mirror of
https://github.com/slackhq/nebula.git
synced 2026-09-30 09:36:37 +02:00
the definitive tun offloads branch (#1704)
This commit is contained in:
Jack Doan
@@ -0,0 +1,187 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math/rand"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The checksum-seeding helpers feed the virtio NEEDS_CSUM contract: the L4
|
||||
// checksum field is pre-loaded with the folded (not inverted) pseudo-header
|
||||
// sum, and the kernel later adds the L4 byte sum and inverts. A wrong seed
|
||||
// produces packets every receiver silently drops, with nothing failing on
|
||||
// our side — so these tests check the helpers against an independent
|
||||
// RFC 1071 reference built from explicit pseudo-header bytes, never against
|
||||
// the production checksum code.
|
||||
|
||||
// refSum accumulates big-endian 16-bit words of b (odd tail zero-padded)
|
||||
// into a wide one's-complement accumulator.
|
||||
func refSum(b []byte) uint64 {
|
||||
var s uint64
|
||||
for i := 0; i+1 < len(b); i += 2 {
|
||||
s += uint64(b[i])<<8 | uint64(b[i+1])
|
||||
}
|
||||
if len(b)%2 == 1 {
|
||||
s += uint64(b[len(b)-1]) << 8
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// refFold folds a wide one's-complement accumulator to 16 bits.
|
||||
func refFold(s uint64) uint16 {
|
||||
for s>>16 != 0 {
|
||||
s = s&0xffff + s>>16
|
||||
}
|
||||
return uint16(s)
|
||||
}
|
||||
|
||||
func TestFoldOnceNoInvertEdgeCases(t *testing.T) {
|
||||
cases := []uint32{
|
||||
0, 1, 0xffff,
|
||||
0x10000, // single carry
|
||||
0x1fffe, // 0xffff + 0xffff: carry produces another 0xffff
|
||||
0xffff0000, // high half only
|
||||
0xfffeffff, // fold yields 0x1fffd: needs a second fold
|
||||
0xffffffff, // worst case
|
||||
0x00010001, // simple two-word
|
||||
}
|
||||
for _, c := range cases {
|
||||
want := refFold(uint64(c))
|
||||
if got := foldOnceNoInvert(c); got != want {
|
||||
t.Errorf("foldOnceNoInvert(%#x) = %#x, want %#x", c, got, want)
|
||||
}
|
||||
// Folding a folded value must be a no-op.
|
||||
if got := foldOnceNoInvert(uint32(foldOnceNoInvert(c))); got != foldOnceNoInvert(c) {
|
||||
t.Errorf("foldOnceNoInvert not idempotent at %#x", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPseudoSumIPv4MatchesReference(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
src, dst [4]byte
|
||||
proto byte
|
||||
l4Len int
|
||||
}{
|
||||
{"simple", [4]byte{10, 0, 0, 1}, [4]byte{10, 0, 0, 2}, 6, 20},
|
||||
{"zero-len", [4]byte{192, 168, 1, 1}, [4]byte{192, 168, 1, 2}, 17, 0},
|
||||
{"max-len", [4]byte{1, 2, 3, 4}, [4]byte{5, 6, 7, 8}, 6, 65535},
|
||||
{"carry-heavy", [4]byte{255, 255, 255, 255}, [4]byte{255, 255, 255, 254}, 17, 65535},
|
||||
{"broadcastish", [4]byte{255, 255, 255, 255}, [4]byte{255, 255, 255, 255}, 255, 65535},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
// RFC 793 pseudo-header: src(4) dst(4) zero(1) proto(1) len(2).
|
||||
ph := make([]byte, 12)
|
||||
copy(ph[0:4], c.src[:])
|
||||
copy(ph[4:8], c.dst[:])
|
||||
ph[9] = c.proto
|
||||
binary.BigEndian.PutUint16(ph[10:12], uint16(c.l4Len))
|
||||
want := refFold(refSum(ph))
|
||||
|
||||
got := foldOnceNoInvert(pseudoSumIPv4(c.src[:], c.dst[:], c.proto, c.l4Len))
|
||||
if got != want {
|
||||
t.Errorf("fold(pseudoSumIPv4) = %#x, want %#x", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPseudoSumIPv6MatchesReference(t *testing.T) {
|
||||
ones := func(b byte) (a [16]byte) {
|
||||
for i := range a {
|
||||
a[i] = b
|
||||
}
|
||||
return
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
src, dst [16]byte
|
||||
proto byte
|
||||
l4Len int
|
||||
}{
|
||||
{"simple", [16]byte{0xfe, 0x80, 15: 1}, [16]byte{0xfe, 0x80, 15: 2}, 6, 20},
|
||||
{"zero-len", [16]byte{0x20, 0x01, 15: 9}, [16]byte{0x20, 0x01, 15: 8}, 17, 0},
|
||||
{"max-u16-len", ones(0xff), ones(0xfe), 6, 65535},
|
||||
{"len-past-u16", ones(0xff), ones(0xff), 17, 0x12345}, // exercises the 32-bit split
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
// RFC 8200 pseudo-header: src(16) dst(16) len(4) zero(3) next(1).
|
||||
ph := make([]byte, 40)
|
||||
copy(ph[0:16], c.src[:])
|
||||
copy(ph[16:32], c.dst[:])
|
||||
binary.BigEndian.PutUint32(ph[32:36], uint32(c.l4Len))
|
||||
ph[39] = c.proto
|
||||
want := refFold(refSum(ph))
|
||||
|
||||
got := foldOnceNoInvert(pseudoSumIPv6(c.src[:], c.dst[:], c.proto, c.l4Len))
|
||||
if got != want {
|
||||
t.Errorf("fold(pseudoSumIPv6) = %#x, want %#x", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPv4HdrChecksumMatchesReference(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(0x1791))
|
||||
for _, hdrLen := range []int{20, 24, 40, 60} {
|
||||
for trial := 0; trial < 200; trial++ {
|
||||
hdr := make([]byte, hdrLen)
|
||||
rng.Read(hdr)
|
||||
hdr[0] = 0x40 | byte(hdrLen/4)
|
||||
hdr[10], hdr[11] = 0, 0 // checksum field zeroed, as the contract requires
|
||||
|
||||
want := ^refFold(refSum(hdr))
|
||||
got := ipv4HdrChecksum(hdr)
|
||||
if got != want {
|
||||
t.Fatalf("ipv4HdrChecksum(len=%d trial=%d) = %#x, want %#x", hdrLen, trial, got, want)
|
||||
}
|
||||
|
||||
// Receiver-side property: with the checksum stored, the full
|
||||
// header must sum to all-ones.
|
||||
binary.BigEndian.PutUint16(hdr[10:12], got)
|
||||
if v := refFold(refSum(hdr)); v != 0xffff {
|
||||
t.Fatalf("stored checksum does not validate: full-header fold = %#x", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestChecksumSeedReceiverAcceptance is the end-to-end property the helpers
|
||||
// exist for: seed the TCP checksum field with fold(pseudoSum), do what the
|
||||
// kernel's NEEDS_CSUM completion does (one's-complement sum over the L4
|
||||
// bytes including the seed, then invert, then store), and verify the result
|
||||
// the way a receiver does (pseudo-header + L4 must sum to all-ones).
|
||||
func TestChecksumSeedReceiverAcceptance(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(0x1826))
|
||||
for trial := 0; trial < 200; trial++ {
|
||||
src := [4]byte{byte(rng.Intn(256)), byte(rng.Intn(256)), byte(rng.Intn(256)), byte(rng.Intn(256))}
|
||||
dst := [4]byte{byte(rng.Intn(256)), byte(rng.Intn(256)), byte(rng.Intn(256)), byte(rng.Intn(256))}
|
||||
payLen := rng.Intn(1500)
|
||||
l4 := make([]byte, 20+payLen)
|
||||
rng.Read(l4)
|
||||
|
||||
// Seed exactly as flushSlot does.
|
||||
seed := foldOnceNoInvert(pseudoSumIPv4(src[:], dst[:], 6, len(l4)))
|
||||
binary.BigEndian.PutUint16(l4[16:18], seed)
|
||||
|
||||
// Kernel NEEDS_CSUM completion: sum the L4 region (seed included,
|
||||
// which is equivalent to summing with the field zeroed and folding
|
||||
// the seed in), invert, store.
|
||||
final := ^refFold(refSum(l4[:16]) + uint64(seed) + refSum(l4[18:]))
|
||||
binary.BigEndian.PutUint16(l4[16:18], final)
|
||||
|
||||
// Receiver validation.
|
||||
ph := make([]byte, 12)
|
||||
copy(ph[0:4], src[:])
|
||||
copy(ph[4:8], dst[:])
|
||||
ph[9] = 6
|
||||
binary.BigEndian.PutUint16(ph[10:12], uint16(len(l4)))
|
||||
if v := refFold(refSum(ph) + refSum(l4)); v != 0xffff {
|
||||
t.Fatalf("trial %d: receiver rejects packet: fold = %#x (seed=%#x final=%#x payLen=%d)",
|
||||
trial, v, seed, final, payLen)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// SortKey identifies a packet's position in its sender's transmission order.
|
||||
type SortKey struct {
|
||||
// Epoch is a receiver-local ordinal for the tunnel (ConnectionState) that decrypted the packet:
|
||||
// a re-handshake replaces the tunnel outright and the replacement's epoch is higher,
|
||||
// so the old tunnel's packets sort first during the cutover overlap.
|
||||
Epoch uint64
|
||||
// Counter is the packet's AEAD message counter within that tunnel.
|
||||
Counter uint64
|
||||
}
|
||||
|
||||
// flowKey identifies a transport flow by {src, dst, sport, dport, family}.
|
||||
// Comparable, so map lookups and linear scans over the slot list stay tight.
|
||||
// Shared by the TCP and UDP coalescers; each coalescer keeps its own
|
||||
// openSlots map, so a TCP and UDP flow on the same 5-tuple-without-proto never alias.
|
||||
type flowKey struct {
|
||||
src, dst [16]byte
|
||||
sport, dport uint16
|
||||
isV6 bool
|
||||
}
|
||||
|
||||
// initialSlots is the starting capacity of the slot pool.
|
||||
// One flow per packet is the worst case, so this matches a typical carrier-side recvmmsg batch on the UDP socket.
|
||||
const initialSlots = 64
|
||||
|
||||
// parseIPAt validates the IP header for lane parsing. newPacket already resolved the L4 protocol
|
||||
// and offset for the firewall, so there is no proto sniff here; the caller's ipHdrLen is
|
||||
// cross-checked instead. A plain header (v4 IHL 20, v6 exactly 40) is the only coalesceable
|
||||
// shape. The v6 check is load-bearing: it rejects extension-header packets whose L4 is not at byte 40.
|
||||
//
|
||||
// The prologues fill fk's addresses and family in place (ports belong to the L4 parser; fk must
|
||||
// be zero on entry so the v4 path leaves src[4:]/dst[4:] clear for map equality) and return pkt
|
||||
// trimmed to the IP-declared length. The receiver-as-out-pointer shape is deliberate: these
|
||||
// functions are too big to inline, and returning structs by value put five 64-byte copies on the
|
||||
// per-packet path.
|
||||
func (fk *flowKey) parseIPAt(pkt []byte, ipHdrLen int) ([]byte, bool) {
|
||||
if len(pkt) < 20 {
|
||||
return nil, false
|
||||
}
|
||||
switch pkt[0] >> 4 {
|
||||
case 4:
|
||||
if ipHdrLen != 20 {
|
||||
return nil, false
|
||||
}
|
||||
return fk.parseIPv4Prologue(pkt)
|
||||
case 6:
|
||||
if ipHdrLen != 40 || len(pkt) < 40 {
|
||||
return nil, false
|
||||
}
|
||||
return fk.parseIPv6Prologue(pkt)
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// parseIPv4Prologue is the shared IPv4 tail of the prologue entries; the caller has verified
|
||||
// len(pkt) >= 20 and the version.
|
||||
func (fk *flowKey) parseIPv4Prologue(pkt []byte) ([]byte, bool) {
|
||||
ihl := int(pkt[0]&0x0f) * 4
|
||||
if ihl != 20 {
|
||||
return nil, false
|
||||
}
|
||||
// Reject any fragmentation (MF or nonzero offset). The dispatcher already gated FragAny; kept
|
||||
// as defense in depth, since a fragment folded into a superpacket would corrupt reassembly.
|
||||
if binary.BigEndian.Uint16(pkt[6:8])&0x3fff != 0 {
|
||||
return nil, false
|
||||
}
|
||||
totalLen := int(binary.BigEndian.Uint16(pkt[2:4]))
|
||||
if totalLen > len(pkt) || totalLen < ihl {
|
||||
return nil, false
|
||||
}
|
||||
fk.isV6 = false
|
||||
copy(fk.src[:4], pkt[12:16])
|
||||
copy(fk.dst[:4], pkt[16:20])
|
||||
return pkt[:totalLen], true
|
||||
}
|
||||
|
||||
// parseIPv6Prologue is the shared IPv6 tail; the caller has verified len(pkt) >= 40, the version,
|
||||
// and that the L4 header sits at byte 40.
|
||||
func (fk *flowKey) parseIPv6Prologue(pkt []byte) ([]byte, bool) {
|
||||
payloadLen := int(binary.BigEndian.Uint16(pkt[4:6]))
|
||||
if 40+payloadLen > len(pkt) {
|
||||
return nil, false
|
||||
}
|
||||
fk.isV6 = true
|
||||
copy(fk.src[:], pkt[8:24])
|
||||
copy(fk.dst[:], pkt[24:40])
|
||||
return pkt[:40+payloadLen], true
|
||||
}
|
||||
|
||||
// ipHeadersMatch compares the IP portion of two packet header prefixes for
|
||||
// byte-for-byte equality on every field that must be identical across coalesced segments.
|
||||
// Size/IPID/IPCsum are masked out.
|
||||
// The full DSCP/ECN byte (IPv4 ToS / IPv6 traffic class) is compared, matching Linux kernel GRO:
|
||||
// segments with differing ECN codepoints must not coalesce,
|
||||
// otherwise ORing e.g. ECT(0) with ECT(1) would fabricate a false CE (congestion) mark or mark a Not-ECT flow as ECN-capable.
|
||||
//
|
||||
// The transport (L4) portion of the header is checked separately by the per-protocol matcher.
|
||||
func ipHeadersMatch(a, b []byte, isV6 bool) bool {
|
||||
if isV6 {
|
||||
// IPv6: [0:4] = version/TC/flow label (TC[1:0] is ECN, so the full TC byte must match),
|
||||
// [6:40] = next_hdr/hop + src + dst. Skip [4:6] payload_len.
|
||||
return bytes.Equal(a[:4], b[:4]) && bytes.Equal(a[6:40], b[6:40])
|
||||
}
|
||||
// IPv4: [0:2] = version/IHL + DSCP|ECN (full ECN byte must match),
|
||||
// [6:10] = flags/fragoff/TTL/proto, [12:20] = src+dst.
|
||||
// Skip [2:4] total len, [4:6] id, [10:12] csum.
|
||||
return bytes.Equal(a[:2], b[:2]) && bytes.Equal(a[6:10], b[6:10]) && bytes.Equal(a[12:20], b[12:20])
|
||||
}
|
||||
|
||||
// ipv4FlagDF is the Don't Fragment bit in the IPv4 flags byte (header byte 6).
|
||||
const ipv4FlagDF = 0x40
|
||||
|
||||
// ipv4CanCoalesceID reports whether an IPv4 packet whose header starts at
|
||||
// nextHdr may join a chain whose seed header is seedHdr as segment index seg
|
||||
// (the seed is segment 0). Kernel GSO re-stamps outgoing segment IDs as
|
||||
// seed_id+n, so coalescing is only transparent when that re-stamp is either
|
||||
// harmless (DF set: RFC 6864 atomic datagrams, the ID carries no meaning) or
|
||||
// reproduces the original IDs exactly (DF clear + IDs already sequential —
|
||||
// the same admission rule kernel GRO applies). Without this, a DF=0 sender
|
||||
// with non-sequential IDs (e.g. OpenBSD's randomized IDs) could have IDs
|
||||
// rewritten into ranges that collide across superpackets, corrupting
|
||||
// reassembly if the packets are fragmented after the TUN write.
|
||||
//
|
||||
// DF itself is guaranteed uniform across a chain by ipHeadersMatch (byte 6
|
||||
// is inside its compared range), so checking the seed's copy suffices.
|
||||
func ipv4CanCoalesceID(seedHdr, nextHdr []byte, seg int) bool {
|
||||
if seedHdr[6]&ipv4FlagDF != 0 {
|
||||
return true
|
||||
}
|
||||
expect := binary.BigEndian.Uint16(seedHdr[4:6]) + uint16(seg)
|
||||
return binary.BigEndian.Uint16(nextHdr[4:6]) == expect
|
||||
}
|
||||
|
||||
// Arena is an injectable byte-slab that hands out non-overlapping borrowed
|
||||
// slices via Reserve and releases them in bulk via Reset.
|
||||
type Arena struct {
|
||||
buf []byte
|
||||
}
|
||||
|
||||
// NewArena returns an Arena with a pre-allocated backing of the given capacity.
|
||||
func NewArena(capacity int) *Arena {
|
||||
return &Arena{buf: make([]byte, 0, capacity)}
|
||||
}
|
||||
|
||||
// Reserve hands out a non-overlapping sz-byte slice from the arena.
|
||||
// If the request doesn't fit the current backing, a fresh, larger backing is allocated.
|
||||
// Already-borrowed slices reference the old backing and remain valid until Reset.
|
||||
func (a *Arena) Reserve(sz int) []byte {
|
||||
if len(a.buf)+sz > cap(a.buf) {
|
||||
newCap := max(cap(a.buf)*2, sz)
|
||||
a.buf = make([]byte, 0, newCap)
|
||||
}
|
||||
start := len(a.buf)
|
||||
a.buf = a.buf[:start+sz]
|
||||
return a.buf[start : start+sz : start+sz]
|
||||
}
|
||||
|
||||
// Reset releases every slice handed out since the last Reset.
|
||||
// Callers must not use any previously-borrowed slice after this returns.
|
||||
// The underlying backing array is retained so subsequent Reserves don't re-allocate.
|
||||
func (a *Arena) Reset() {
|
||||
a.buf = a.buf[:0]
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/slackhq/nebula/test"
|
||||
)
|
||||
|
||||
// stagePackets builds the stagedPacket entries Commit would have produced, so dispatch benchmarks
|
||||
// bypass staging and the sort entirely.
|
||||
func stagePackets(pkts [][]byte) []stagedPacket {
|
||||
staged := make([]stagedPacket, len(pkts))
|
||||
for i, p := range pkts {
|
||||
pp := testPP(p)
|
||||
staged[i] = stagedPacket{
|
||||
pkt: p,
|
||||
key: SortKey{Epoch: 1, Counter: uint64(i + 1)},
|
||||
proto: pp.Protocol,
|
||||
fragAny: pp.FragAny,
|
||||
ipHdrLen: uint16(pp.IPHdrLen),
|
||||
}
|
||||
}
|
||||
return staged
|
||||
}
|
||||
|
||||
func flushLanes(b *testing.B, m *MultiCoalescer) {
|
||||
b.Helper()
|
||||
if m.tcp != nil {
|
||||
if err := m.tcp.Flush(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
if m.udp != nil {
|
||||
if err := m.udp.Flush(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := m.pt.Flush(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// runDispatchBench measures dispatch plus the per-batch lane flush: the post-sort half of the
|
||||
// batcher, which is where the production profile concentrates.
|
||||
func runDispatchBench(b *testing.B, pkts [][]byte, batchSize int) {
|
||||
b.Helper()
|
||||
m := NewMultiCoalescer(nopTunWriter{}, test.NewLogger())
|
||||
staged := stagePackets(pkts)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(pkts[0])))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if err := m.dispatch(staged[i%len(staged)]); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if (i+1)%batchSize == 0 {
|
||||
flushLanes(b, m)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
flushLanes(b, m)
|
||||
}
|
||||
|
||||
// BenchmarkDispatchSingleFlow is the bulk steady state: every packet past the seed appends.
|
||||
func BenchmarkDispatchSingleFlow(b *testing.B) {
|
||||
runDispatchBench(b, buildTCPv4BulkFlow(tcpCoalesceMaxSegs, 1200), tcpCoalesceMaxSegs)
|
||||
}
|
||||
|
||||
// BenchmarkDispatchInterleaved16 stresses the openSlots map: 16 flows round-robined defeats the
|
||||
// lastSlot cache on every packet.
|
||||
func BenchmarkDispatchInterleaved16(b *testing.B) {
|
||||
pkts := buildTCPv4Interleaved(16, tcpCoalesceMaxSegs, 1200)
|
||||
runDispatchBench(b, pkts, len(pkts))
|
||||
}
|
||||
|
||||
// BenchmarkDispatchAckHeavy alternates MSS data with pure ACKs on one flow — the RX shape of a
|
||||
// bidirectional transfer (the peer's data and its ACKs of our data share the tunnel direction).
|
||||
func BenchmarkDispatchAckHeavy(b *testing.B) {
|
||||
pay := make([]byte, 1200)
|
||||
var pkts [][]byte
|
||||
seq := uint32(1000)
|
||||
for range tcpCoalesceMaxSegs / 2 {
|
||||
pkts = append(pkts, buildTCPv4(seq, tcpAck, pay))
|
||||
seq += uint32(len(pay))
|
||||
pkts = append(pkts, buildTCPv4(seq, tcpAck, nil))
|
||||
}
|
||||
runDispatchBench(b, pkts, len(pkts))
|
||||
}
|
||||
|
||||
// BenchmarkDispatchUDPFlow is the QUIC-ish bulk UDP shape.
|
||||
func BenchmarkDispatchUDPFlow(b *testing.B) {
|
||||
pay := make([]byte, 1200)
|
||||
pkts := make([][]byte, udpCoalesceMaxSegs)
|
||||
for i := range pkts {
|
||||
pkts[i] = buildUDPv4(2000, 443, pay)
|
||||
}
|
||||
runDispatchBench(b, pkts, len(pkts))
|
||||
}
|
||||
|
||||
// BenchmarkDispatchSeedHeavy sets PSH on every packet so each one seeds and immediately closes
|
||||
// its own slot — the small-write RPC shape, and the upper bound on what the seed path (including
|
||||
// the parsedTCP-to-slot field transfer) can cost.
|
||||
func BenchmarkDispatchSeedHeavy(b *testing.B) {
|
||||
pay := make([]byte, 1200)
|
||||
pkts := make([][]byte, tcpCoalesceMaxSegs)
|
||||
seq := uint32(1000)
|
||||
for i := range pkts {
|
||||
pkts[i] = buildTCPv4(seq, tcpAckPsh, pay)
|
||||
seq += uint32(len(pay))
|
||||
}
|
||||
runDispatchBench(b, pkts, len(pkts))
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package batch
|
||||
|
||||
//TODO refactor this away
|
||||
// This file holds the lanes' self-parsing Commit entries and the proto-checking parsers behind
|
||||
// them. Production traffic enters the lanes only through MultiCoalescer.dispatch and the At
|
||||
// parsers; these wrappers reproduce that path (including seal-all on unparseable shapes) on top
|
||||
// of a local parse, so tests and benches can drive one lane with nothing but a packet.
|
||||
|
||||
// parseIPPrologue resolves the IP version, requires the L4 protocol to match wantProto (6 TCP,
|
||||
// 17 UDP), and defers to the shared per-version cores. Returns the trimmed packet and the L4
|
||||
// offset; fk must be zero on entry and is filled in place.
|
||||
func (fk *flowKey) parseIPPrologue(pkt []byte, wantProto byte) ([]byte, int, bool) {
|
||||
if len(pkt) < 20 {
|
||||
return nil, 0, false
|
||||
}
|
||||
switch pkt[0] >> 4 {
|
||||
case 4:
|
||||
if pkt[9] != wantProto {
|
||||
return nil, 0, false
|
||||
}
|
||||
trimmed, ok := fk.parseIPv4Prologue(pkt)
|
||||
return trimmed, 20, ok
|
||||
case 6:
|
||||
if len(pkt) < 40 {
|
||||
return nil, 0, false
|
||||
}
|
||||
if pkt[6] != wantProto {
|
||||
return nil, 0, false
|
||||
}
|
||||
trimmed, ok := fk.parseIPv6Prologue(pkt)
|
||||
return trimmed, 40, ok
|
||||
}
|
||||
return nil, 0, false
|
||||
}
|
||||
|
||||
// parseBase extracts the flow key and IP/TCP offsets for any TCP packet, admissible for
|
||||
// coalescing or not. Returns false for non-TCP or malformed input.
|
||||
func (p *parsedTCP) parseBase(pkt []byte) bool {
|
||||
trimmed, ipHdrLen, ok := p.fk.parseIPPrologue(pkt, ipProtoTCP)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return p.parseTail(trimmed, ipHdrLen)
|
||||
}
|
||||
|
||||
// parseBase extracts the flow key and IP/UDP offsets for a UDP packet.
|
||||
func (p *parsedUDP) parseBase(pkt []byte) bool {
|
||||
trimmed, ipHdrLen, ok := p.fk.parseIPPrologue(pkt, ipProtoUDP)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return p.parseTail(trimmed, ipHdrLen)
|
||||
}
|
||||
|
||||
// Commit borrows pkt. The caller must keep pkt valid until the next Flush.
|
||||
func (c *TCPCoalescer) Commit(pkt []byte) error {
|
||||
var info parsedTCP
|
||||
if !info.parseBase(pkt) {
|
||||
// Unparseable: flow key unknown, seal everything so later data cannot emit ahead of it.
|
||||
c.sealAllOpen()
|
||||
c.addVerbatim(pkt)
|
||||
return nil
|
||||
}
|
||||
return c.commitParsed(pkt, &info)
|
||||
}
|
||||
|
||||
// Commit borrows pkt. The caller must keep pkt valid until the next Flush.
|
||||
func (c *UDPCoalescer) Commit(pkt []byte) error {
|
||||
var info parsedUDP
|
||||
if !info.parseBase(pkt) {
|
||||
c.sealAllOpen()
|
||||
c.addVerbatim(pkt)
|
||||
return nil
|
||||
}
|
||||
return c.commitParsed(pkt, &info)
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"slices"
|
||||
|
||||
"github.com/slackhq/nebula/firewall"
|
||||
)
|
||||
|
||||
// MultiCoalescer stages plaintext packets with their (epoch, counter) sort keys and, at Flush,
|
||||
// replays them in sender-transmission order into lane-specific batchers selected by L4 protocol.
|
||||
//
|
||||
// Sorting before dispatch keeps the ordering story simple: each lane consumes packets in
|
||||
// transmission order, builds slots in that order, and emits them in creation order. Wire reorder
|
||||
// inside a flush batch is repaired here, before it can fragment a lane's coalesce chains, so the
|
||||
// lanes carry no reorder-repair machinery.
|
||||
//
|
||||
// The contract is per-tunnel transmission order within each lane, with two exceptions: a pure TCP
|
||||
// ACK may be overtaken by later same-flow data (it does not close the flow's open chain; a late
|
||||
// ACK is just a stale ACK), and an unparseable shape seals every open chain in its lane (its flow
|
||||
// is unknown) and rides the lane as an in-lane verbatim, still in transmission order. Routing
|
||||
// follows the flow: a flow's non-coalesceable shapes ride its protocol lane rather than falling
|
||||
// to the later-flushed pt lane.
|
||||
//
|
||||
// Cross-lane order (TCP vs UDP vs everything else) is not preserved.
|
||||
type MultiCoalescer struct {
|
||||
tcp *TCPCoalescer
|
||||
udp *UDPCoalescer
|
||||
pt *Passthrough
|
||||
|
||||
// staged holds this batch's packets and sort keys until Flush. Borrowed: the caller keeps
|
||||
// each pkt alive until Flush returns.
|
||||
staged []stagedPacket
|
||||
}
|
||||
|
||||
// stagedPacket carries the scalars dispatch needs from the firewall's ParsedPacket, copied by
|
||||
// value: pp is reused by the caller per packet and must not be retained past Commit.
|
||||
type stagedPacket struct {
|
||||
pkt []byte
|
||||
key SortKey
|
||||
proto byte
|
||||
fragAny bool
|
||||
ipHdrLen uint16
|
||||
}
|
||||
|
||||
// NewMultiCoalescer builds a multi-lane batcher over w, based on available protocol support. The
|
||||
// staging sort applies even when no GSO lane is available: passthrough-only platforms still get
|
||||
// transmission-order repair.
|
||||
func NewMultiCoalescer(w io.Writer, l *slog.Logger) *MultiCoalescer {
|
||||
m := &MultiCoalescer{
|
||||
pt: NewPassthrough(w),
|
||||
staged: make([]stagedPacket, 0, initialSlots),
|
||||
}
|
||||
m.tcp = NewTCPCoalescer(w, l)
|
||||
m.udp = NewUDPCoalescer(w)
|
||||
return m
|
||||
}
|
||||
|
||||
// Commit stages pkt for the next Flush; dispatch is deferred so it runs on packets already in
|
||||
// transmission order. key carries the packet's tunnel epoch and message counter. pkt is borrowed:
|
||||
// the caller must keep it valid until the next Flush and not re-use it, and Flush may patch a
|
||||
// coalesced packet's headers in place. pp is the firewall's parse of pkt and is borrowed only
|
||||
// for this call, so the fields dispatch needs are copied here.
|
||||
func (m *MultiCoalescer) Commit(pkt []byte, key SortKey, pp *firewall.ParsedPacket) error {
|
||||
m.staged = append(m.staged, stagedPacket{
|
||||
pkt: pkt,
|
||||
key: key,
|
||||
proto: pp.Protocol,
|
||||
fragAny: pp.FragAny,
|
||||
ipHdrLen: uint16(pp.IPHdrLen),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// compareStaged orders staged packets by (epoch, counter)
|
||||
func compareStaged(a, b stagedPacket) int {
|
||||
if c := cmp.Compare(a.key.Epoch, b.key.Epoch); c != 0 {
|
||||
return c
|
||||
}
|
||||
return cmp.Compare(a.key.Counter, b.key.Counter)
|
||||
}
|
||||
|
||||
// dispatch routes one staged packet to its protocol lane (see commitStaged), or to the verbatim
|
||||
// passthrough when the lane has no GSO support.
|
||||
func (m *MultiCoalescer) dispatch(sp stagedPacket) error {
|
||||
switch sp.proto {
|
||||
case ipProtoTCP:
|
||||
if m.tcp != nil {
|
||||
return m.tcp.commitStaged(sp)
|
||||
}
|
||||
case ipProtoUDP:
|
||||
if m.udp != nil {
|
||||
return m.udp.commitStaged(sp)
|
||||
}
|
||||
}
|
||||
return m.pt.enqueue(sp.pkt)
|
||||
}
|
||||
|
||||
// Flush sorts the staged batch into transmission order, replays it into the lanes, then flushes each lane.
|
||||
// Drains everything and returns the joined errors; one bad packet does not hold up the rest.
|
||||
// After Flush returns, committed payload slices may be recycled.
|
||||
func (m *MultiCoalescer) Flush() error {
|
||||
// Arrival order is already almost sorted (reorder is the exception), which pdqsort detects
|
||||
// and handles in near-linear time.
|
||||
slices.SortFunc(m.staged, compareStaged)
|
||||
|
||||
var errs []error
|
||||
for _, sp := range m.staged {
|
||||
if err := m.dispatch(sp); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
clear(m.staged) // drop borrowed pkt refs
|
||||
m.staged = m.staged[:0]
|
||||
|
||||
if m.tcp != nil {
|
||||
if err := m.tcp.Flush(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if m.udp != nil {
|
||||
if err := m.udp.Flush(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if err := m.pt.Flush(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
@@ -0,0 +1,437 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/slackhq/nebula/firewall"
|
||||
"github.com/slackhq/nebula/test"
|
||||
)
|
||||
|
||||
// keySeq hands out SortKeys with ascending counters in a fixed epoch, for
|
||||
// tests where commit order IS transmission order.
|
||||
type keySeq struct {
|
||||
epoch, counter uint64
|
||||
}
|
||||
|
||||
func (k *keySeq) next() SortKey {
|
||||
k.counter++
|
||||
return SortKey{Epoch: k.epoch, Counter: k.counter}
|
||||
}
|
||||
|
||||
// newTestMultiCoalescer builds a batcher over w.
|
||||
func newTestMultiCoalescer(tb testing.TB, w io.Writer) *MultiCoalescer {
|
||||
tb.Helper()
|
||||
return NewMultiCoalescer(w, test.NewLogger())
|
||||
}
|
||||
|
||||
// TestMultiCoalescerRoutesByProto confirms TCP/UDP/other land in the right
|
||||
// lane: TCP and UDP get coalesced when their lanes are enabled, anything
|
||||
// else (ICMP here) falls through to plain Write.
|
||||
func TestMultiCoalescerRoutesByProto(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
m := newTestMultiCoalescer(t, w)
|
||||
k := &keySeq{epoch: 1}
|
||||
|
||||
tcpPay := make([]byte, 1200)
|
||||
udpPay := make([]byte, 1200)
|
||||
icmp := make([]byte, 28)
|
||||
icmp[0] = 0x45
|
||||
icmp[2] = 0
|
||||
icmp[3] = 28
|
||||
icmp[9] = 1
|
||||
|
||||
if err := m.Commit(buildTCPv4(1000, tcpAck, tcpPay), k.next(), testPP(buildTCPv4(1000, tcpAck, tcpPay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildTCPv4(2200, tcpAck, tcpPay), k.next(), testPP(buildTCPv4(2200, tcpAck, tcpPay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildUDPv4(2000, 53, udpPay), k.next(), testPP(buildUDPv4(2000, 53, udpPay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildUDPv4(2000, 53, udpPay), k.next(), testPP(buildUDPv4(2000, 53, udpPay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(icmp, k.next(), testPP(icmp)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 1 TCP super (2 segments) + 1 UDP super (2 segments) = 2 gso writes.
|
||||
if len(w.gsoWrites) != 2 {
|
||||
t.Fatalf("want 2 gso writes (one TCP + one UDP), got %d", len(w.gsoWrites))
|
||||
}
|
||||
if len(w.writes) != 1 {
|
||||
t.Fatalf("want 1 plain write (ICMP), got %d", len(w.writes))
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiCoalescerRestoresTransmissionOrder is the core staging-sort
|
||||
// property: packets committed out of counter order (wire reorder inside one
|
||||
// flush batch) are replayed into the lanes in transmission order, so the
|
||||
// reorder never fragments the coalesce chain — one superpacket, in seq
|
||||
// order, exactly as if the wire had never reordered. The retransmit shape
|
||||
// falls out of the same key: a retransmit carries a lower seq but a HIGHER
|
||||
// counter (it was encrypted later), so it emits after the data it trails.
|
||||
func TestMultiCoalescerRestoresTransmissionOrder(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
m := newTestMultiCoalescer(t, w)
|
||||
pay := make([]byte, 1200)
|
||||
|
||||
// Transmission order: seq 1000 (c1), 2200 (c2), 3400 (c3).
|
||||
// Arrival order: 3400, 1000, 2200.
|
||||
if err := m.Commit(buildTCPv4(3400, tcpAck, pay), SortKey{Epoch: 1, Counter: 3}, testPP(buildTCPv4(3400, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildTCPv4(1000, tcpAck, pay), SortKey{Epoch: 1, Counter: 1}, testPP(buildTCPv4(1000, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildTCPv4(2200, tcpAck, pay), SortKey{Epoch: 1, Counter: 2}, testPP(buildTCPv4(2200, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.gsoWrites) != 1 || len(w.writes) != 0 {
|
||||
t.Fatalf("want 1 gso write (unfragmented chain), got gso=%d plain=%d", len(w.gsoWrites), len(w.writes))
|
||||
}
|
||||
g := w.gsoWrites[0]
|
||||
if len(g.pays) != 3 {
|
||||
t.Fatalf("segs=%d want 3", len(g.pays))
|
||||
}
|
||||
const ipHdrLen = 20
|
||||
if seedSeq := binary.BigEndian.Uint32(g.hdr[ipHdrLen+4 : ipHdrLen+8]); seedSeq != 1000 {
|
||||
t.Errorf("seed seq=%d want 1000", seedSeq)
|
||||
}
|
||||
|
||||
// Retransmit: seq 1000 again but counter 4 — sorts after seq 4600 (c3).
|
||||
w.writes, w.gsoWrites, w.order = nil, nil, nil
|
||||
if err := m.Commit(buildTCPv4(1000, tcpAck, pay), SortKey{Epoch: 1, Counter: 4}, testPP(buildTCPv4(1000, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildTCPv4(4600, tcpAck, pay), SortKey{Epoch: 1, Counter: 3}, testPP(buildTCPv4(4600, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.writes) != 2 {
|
||||
t.Fatalf("want 2 plain writes, got %d (gso=%d)", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
first := binary.BigEndian.Uint32(w.writes[0][24:28])
|
||||
second := binary.BigEndian.Uint32(w.writes[1][24:28])
|
||||
if first != 4600 || second != 1000 {
|
||||
t.Fatalf("emission (%d, %d), want (4600, 1000): retransmit must not overtake in-flight data", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiCoalescerRestoresOrderAcrossFlows scrambles two interleaved flows;
|
||||
// the staging sort must repair each flow into one superpacket without any
|
||||
// cross-flow contamination.
|
||||
func TestMultiCoalescerRestoresOrderAcrossFlows(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
m := newTestMultiCoalescer(t, w)
|
||||
pay := make([]byte, 1200)
|
||||
|
||||
// Transmission: A.100 (c1), B.500 (c2), A.1300 (c3), B.1700 (c4).
|
||||
// Arrival: A.1300, B.1700, A.100, B.500.
|
||||
if err := m.Commit(buildTCPv4Ports(1000, 2000, 1300, tcpAck, pay), SortKey{Epoch: 1, Counter: 3}, testPP(buildTCPv4Ports(1000, 2000, 1300, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildTCPv4Ports(3000, 2000, 1700, tcpAck, pay), SortKey{Epoch: 1, Counter: 4}, testPP(buildTCPv4Ports(3000, 2000, 1700, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildTCPv4Ports(1000, 2000, 100, tcpAck, pay), SortKey{Epoch: 1, Counter: 1}, testPP(buildTCPv4Ports(1000, 2000, 100, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildTCPv4Ports(3000, 2000, 500, tcpAck, pay), SortKey{Epoch: 1, Counter: 2}, testPP(buildTCPv4Ports(3000, 2000, 500, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.gsoWrites) != 2 {
|
||||
t.Fatalf("want 2 gso writes (one per flow), got %d (plain=%d)", len(w.gsoWrites), len(w.writes))
|
||||
}
|
||||
for i, g := range w.gsoWrites {
|
||||
if len(g.pays) != 2 {
|
||||
t.Errorf("gso[%d] segs=%d want 2", i, len(g.pays))
|
||||
}
|
||||
const ipHdrLen = 20
|
||||
seedSeq := binary.BigEndian.Uint32(g.hdr[ipHdrLen+4 : ipHdrLen+8])
|
||||
sport := binary.BigEndian.Uint16(g.hdr[ipHdrLen : ipHdrLen+2])
|
||||
switch sport {
|
||||
case 1000:
|
||||
if seedSeq != 100 {
|
||||
t.Errorf("flow A seed seq=%d want 100", seedSeq)
|
||||
}
|
||||
case 3000:
|
||||
if seedSeq != 500 {
|
||||
t.Errorf("flow B seed seq=%d want 500", seedSeq)
|
||||
}
|
||||
default:
|
||||
t.Errorf("unexpected sport %d", sport)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiCoalescerEpochOrdersAcrossRehandshake: a re-handshake replaces
|
||||
// the tunnel, and the replacement's counter space starts near zero — raw
|
||||
// counter order would emit the new tunnel's packets first while the old
|
||||
// tunnel's backlog is still arriving. The epoch key must dominate:
|
||||
// everything from the old tunnel emits before anything from the new one.
|
||||
func TestMultiCoalescerEpochOrdersAcrossRehandshake(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
m := newTestMultiCoalescer(t, w)
|
||||
pay := make([]byte, 1200)
|
||||
|
||||
// New session's first data arrives before the old session's last data.
|
||||
if err := m.Commit(buildTCPv4(2200, tcpAck, pay), SortKey{Epoch: 8, Counter: 1}, testPP(buildTCPv4(2200, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildTCPv4(1000, tcpAck, pay), SortKey{Epoch: 7, Counter: 9_000_000}, testPP(buildTCPv4(1000, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Same flow, contiguous seq, identical headers: after the epoch sort the
|
||||
// two segments append into one superpacket seeded by the OLD session's
|
||||
// packet.
|
||||
if len(w.gsoWrites) != 1 {
|
||||
t.Fatalf("want 1 gso write, got %d (plain=%d)", len(w.gsoWrites), len(w.writes))
|
||||
}
|
||||
const ipHdrLen = 20
|
||||
if seedSeq := binary.BigEndian.Uint32(w.gsoWrites[0].hdr[ipHdrLen+4 : ipHdrLen+8]); seedSeq != 1000 {
|
||||
t.Errorf("seed seq=%d want 1000 (old session first)", seedSeq)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiCoalescerNoUSOFallsThrough verifies that on a queue without USO
|
||||
// (older kernel: TSO but no GSO_UDP_L4) the UDP lane never comes up and UDP
|
||||
// packets still reach the kernel via verbatim rather than being lost.
|
||||
func TestMultiCoalescerNoUSOFallsThrough(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true, noUSO: true}
|
||||
m := newTestMultiCoalescer(t, w)
|
||||
k := &keySeq{epoch: 1}
|
||||
if m.udp != nil {
|
||||
t.Fatal("UDP lane must not come up without USO")
|
||||
}
|
||||
|
||||
if err := m.Commit(buildUDPv4(1000, 53, make([]byte, 800)), k.next(), testPP(buildUDPv4(1000, 53, make([]byte, 800)))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildUDPv4(1000, 53, make([]byte, 800)), k.next(), testPP(buildUDPv4(1000, 53, make([]byte, 800)))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.gsoWrites) != 0 {
|
||||
t.Errorf("UDP must NOT be coalesced when USO disabled, got %d gso writes", len(w.gsoWrites))
|
||||
}
|
||||
if len(w.writes) != 2 {
|
||||
t.Errorf("UDP must pass through as 2 plain writes, got %d", len(w.writes))
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiCoalescerNoOffloadsStillSorts covers a queue that can't offload
|
||||
// anything. Both lane constructors refuse, so every packet rides the
|
||||
// verbatim lane — but the staging sort still applies, so emission follows
|
||||
// transmission order even without GSO.
|
||||
func TestMultiCoalescerNoOffloadsStillSorts(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: false}
|
||||
m := newTestMultiCoalescer(t, w)
|
||||
if m.tcp != nil || m.udp != nil {
|
||||
t.Fatal("no lane may come up without offloads")
|
||||
}
|
||||
pkts := [][]byte{
|
||||
buildTCPv4(1000, tcpAck, make([]byte, 1200)),
|
||||
buildUDPv4(1000, 53, make([]byte, 800)),
|
||||
buildTCPv4(2200, tcpAck, make([]byte, 1200)),
|
||||
}
|
||||
// Committed in reverse transmission order; keys carry the truth.
|
||||
for i := len(pkts) - 1; i >= 0; i-- {
|
||||
if err := m.Commit(pkts[i], SortKey{Epoch: 1, Counter: uint64(i + 1)}, testPP(pkts[i])); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := m.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.gsoWrites) != 0 {
|
||||
t.Errorf("no GSO writes possible, got %d", len(w.gsoWrites))
|
||||
}
|
||||
if len(w.writes) != len(pkts) {
|
||||
t.Fatalf("want %d plain writes, got %d", len(pkts), len(w.writes))
|
||||
}
|
||||
// One lane for everything means the sorted order survives end to end.
|
||||
for i, want := range pkts {
|
||||
if !bytes.Equal(w.writes[i], want) {
|
||||
t.Errorf("write %d out of order or corrupt", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// buildUDPv6Fragment builds an IPv6 packet whose extension chain is a
|
||||
// single fragment header (NH=44) naming UDP as the terminal protocol —
|
||||
// a first fragment (offset 0, MF set) carrying the UDP header and a
|
||||
// partial payload.
|
||||
func buildUDPv6Fragment(sport, dport uint16, payload []byte) []byte {
|
||||
const ipHdrLen = 40
|
||||
const fragHdrLen = 8
|
||||
const udpHdrLen = 8
|
||||
total := ipHdrLen + fragHdrLen + udpHdrLen + len(payload)
|
||||
pkt := make([]byte, total)
|
||||
|
||||
pkt[0] = 0x60
|
||||
binary.BigEndian.PutUint16(pkt[4:6], uint16(total-ipHdrLen))
|
||||
pkt[6] = 44 // fragment extension header
|
||||
pkt[7] = 64
|
||||
pkt[8] = 0xfe
|
||||
pkt[9] = 0x80
|
||||
pkt[23] = 1
|
||||
pkt[24] = 0xfe
|
||||
pkt[25] = 0x80
|
||||
pkt[39] = 2
|
||||
|
||||
pkt[40] = ipProtoUDP // fragment's next header
|
||||
binary.BigEndian.PutUint16(pkt[42:44], 0x0001) // offset 0, MF set
|
||||
binary.BigEndian.PutUint32(pkt[44:48], 0x1badf00) // identification
|
||||
|
||||
binary.BigEndian.PutUint16(pkt[48:50], sport)
|
||||
binary.BigEndian.PutUint16(pkt[50:52], dport)
|
||||
binary.BigEndian.PutUint16(pkt[52:54], uint16(udpHdrLen+len(payload)))
|
||||
copy(pkt[56:], payload)
|
||||
return pkt
|
||||
}
|
||||
|
||||
// TestMultiCoalescerIPv6FragmentStaysInLane locks in extension-header
|
||||
// routing: a fragment whose chain terminates in UDP must ride the UDP lane
|
||||
// as an in-lane verbatim — emitted ahead of later same-flow datagrams —
|
||||
// not the verbatim lane, which flushes after every coalescer lane and
|
||||
// would reorder it behind data that arrived after it.
|
||||
func TestMultiCoalescerIPv6FragmentStaysInLane(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
m := newTestMultiCoalescer(t, w)
|
||||
k := &keySeq{epoch: 1}
|
||||
|
||||
if err := m.Commit(buildUDPv6Fragment(2000, 53, make([]byte, 512)), k.next(), testPP(buildUDPv6Fragment(2000, 53, make([]byte, 512)))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildUDPv6(2000, 53, make([]byte, 800)), k.next(), testPP(buildUDPv6(2000, 53, make([]byte, 800)))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildUDPv6(2000, 53, make([]byte, 800)), k.next(), testPP(buildUDPv6(2000, 53, make([]byte, 800)))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.writes) != 1 {
|
||||
t.Fatalf("want the fragment as 1 plain write, got %d", len(w.writes))
|
||||
}
|
||||
if len(w.gsoWrites) != 1 {
|
||||
t.Fatalf("want the two whole datagrams coalesced into 1 gso write, got %d", len(w.gsoWrites))
|
||||
}
|
||||
// Transmission order was fragment-then-data; same-lane routing must keep it.
|
||||
if w.order[0] != "write" {
|
||||
t.Fatalf("fragment must be emitted before later data (in-lane verbatim), order=%v", w.order)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiCoalescerFragmentSealsUDPChains: an unparseable datagram
|
||||
// (fragment) seals every open UDP chain, so datagrams from before and after
|
||||
// it land in separate superpackets and the fragment holds its transmission-
|
||||
// order position between them.
|
||||
func TestMultiCoalescerFragmentSealsUDPChains(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
m := newTestMultiCoalescer(t, w)
|
||||
k := &keySeq{epoch: 1}
|
||||
|
||||
if err := m.Commit(buildUDPv6(2000, 53, make([]byte, 800)), k.next(), testPP(buildUDPv6(2000, 53, make([]byte, 800)))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildUDPv6(2000, 53, make([]byte, 800)), k.next(), testPP(buildUDPv6(2000, 53, make([]byte, 800)))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildUDPv6Fragment(2000, 53, make([]byte, 512)), k.next(), testPP(buildUDPv6Fragment(2000, 53, make([]byte, 512)))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildUDPv6(2000, 53, make([]byte, 800)), k.next(), testPP(buildUDPv6(2000, 53, make([]byte, 800)))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildUDPv6(2000, 53, make([]byte, 800)), k.next(), testPP(buildUDPv6(2000, 53, make([]byte, 800)))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.gsoWrites) != 2 {
|
||||
t.Fatalf("want 2 gso writes (chains sealed around the fragment), got %d", len(w.gsoWrites))
|
||||
}
|
||||
if len(w.writes) != 1 {
|
||||
t.Fatalf("want the fragment as 1 plain write, got %d", len(w.writes))
|
||||
}
|
||||
want := []string{"gso", "write", "gso"}
|
||||
if len(w.order) != 3 || w.order[0] != want[0] || w.order[1] != want[1] || w.order[2] != want[2] {
|
||||
t.Fatalf("emission order = %v, want %v", w.order, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiCoalescerNoTSOFallsThrough mirrors the no-TSO case.
|
||||
func TestMultiCoalescerNoTSOFallsThrough(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true, noTSO: true}
|
||||
m := newTestMultiCoalescer(t, w)
|
||||
k := &keySeq{epoch: 1}
|
||||
if m.tcp != nil {
|
||||
t.Fatal("TCP lane must not come up without TSO")
|
||||
}
|
||||
|
||||
pay := make([]byte, 1200)
|
||||
if err := m.Commit(buildTCPv4(1000, tcpAck, pay), k.next(), testPP(buildTCPv4(1000, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Commit(buildTCPv4(2200, tcpAck, pay), k.next(), testPP(buildTCPv4(2200, tcpAck, pay))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.gsoWrites) != 0 {
|
||||
t.Errorf("TCP must NOT be coalesced when TSO disabled, got %d gso writes", len(w.gsoWrites))
|
||||
}
|
||||
if len(w.writes) != 2 {
|
||||
t.Errorf("TCP must pass through as 2 plain writes, got %d", len(w.writes))
|
||||
}
|
||||
}
|
||||
|
||||
// testPP derives the ParsedPacket newPacket would produce for the packet
|
||||
// shapes the tests build: plain v4/v6, v4 with options or fragment bits set,
|
||||
// and the single-fragment-header v6 shape from buildUDPv6Fragment. Anything
|
||||
// unrecognizable stays zero (proto 0 routes to the passthrough lane).
|
||||
func testPP(pkt []byte) *firewall.ParsedPacket {
|
||||
pp := &firewall.ParsedPacket{}
|
||||
if len(pkt) < 20 {
|
||||
return pp
|
||||
}
|
||||
switch pkt[0] >> 4 {
|
||||
case 4:
|
||||
pp.Protocol = pkt[9]
|
||||
pp.IPHdrLen = int(pkt[0]&0x0f) * 4
|
||||
pp.FragAny = binary.BigEndian.Uint16(pkt[6:8])&0x3fff != 0
|
||||
case 6:
|
||||
pp.Protocol = pkt[6]
|
||||
pp.IPHdrLen = 40
|
||||
if pp.Protocol == 44 { // fragment extension header
|
||||
pp.Protocol = pkt[40]
|
||||
pp.IPHdrLen = 48
|
||||
pp.FragAny = true
|
||||
}
|
||||
}
|
||||
return pp
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// Passthrough is MultiCoalescer's verbatim lane: no batching, packets are written at Flush in the
|
||||
// order enqueued.
|
||||
type Passthrough struct {
|
||||
out io.Writer
|
||||
slots [][]byte
|
||||
}
|
||||
|
||||
func NewPassthrough(w io.Writer) *Passthrough {
|
||||
return &Passthrough{
|
||||
out: w,
|
||||
slots: make([][]byte, 0, 128),
|
||||
}
|
||||
}
|
||||
|
||||
// enqueue accepts one packet, already sorted into transmission order by dispatch.
|
||||
func (p *Passthrough) enqueue(pkt []byte) error {
|
||||
p.slots = append(p.slots, pkt)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Passthrough) Flush() error {
|
||||
var firstErr error
|
||||
for _, s := range p.slots {
|
||||
_, err := p.out.Write(s)
|
||||
if err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
clear(p.slots)
|
||||
p.slots = p.slots[:0]
|
||||
return firstErr
|
||||
}
|
||||
@@ -0,0 +1,472 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"log/slog"
|
||||
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
)
|
||||
|
||||
// ipProtoTCP is the IANA protocol number for TCP. Defined here to help Windows out.
|
||||
const ipProtoTCP = 6
|
||||
|
||||
// tcpCoalesceBufSize caps total bytes per superpacket. Mirrors the kernel's
|
||||
// sk_gso_max_size of ~64KiB; anything beyond this would be rejected anyway.
|
||||
const tcpCoalesceBufSize = 65535
|
||||
|
||||
// tcpCoalesceMaxSegs caps how many segments we'll coalesce into a single
|
||||
// superpacket. Keeping this well below the kernel's TSO ceiling bounds latency.
|
||||
const tcpCoalesceMaxSegs = 64
|
||||
|
||||
// coalesceSlot is one entry in the coalescer's ordered event queue. A verbatim slot holds a single
|
||||
// borrowed packet emitted as-is (pure ACK, non-admissible TCP, unparseable, or oversize seed); a
|
||||
// non-verbatim slot is an in-progress coalesced superpacket. payIovs are borrowed slices of the
|
||||
// caller's plaintext buffers; the caller must keep them alive until Flush.
|
||||
type coalesceSlot struct {
|
||||
verbatim bool
|
||||
// rawPkt is borrowed: the whole packet for verbatim slots, the seed packet for coalesce
|
||||
// slots. A slot that never grows past one segment is emitted from rawPkt so its original
|
||||
// (already valid) L4 checksum ships DATA_VALID instead of making the kernel recompute it.
|
||||
// A multi-segment slot's superpacket header is rawPkt's, patched in place at flush.
|
||||
rawPkt []byte
|
||||
|
||||
fk flowKey
|
||||
hdrLen int
|
||||
ipHdrLen int
|
||||
isV6 bool
|
||||
gsoSize int
|
||||
numSeg int
|
||||
totalPay int
|
||||
nextSeq uint32
|
||||
payIovs [][]byte
|
||||
}
|
||||
|
||||
// TCPCoalescer accumulates adjacent in-flow TCP data segments across multiple concurrent flows and
|
||||
// emits each flow's run as a single TSO superpacket via tio.GSOWriter. Input must be in sender
|
||||
// transmission order (MultiCoalescer sorts by (epoch, counter) before dispatch); slots are emitted
|
||||
// in creation order, so emission reproduces transmission order except for the pure-ACK case in
|
||||
// commitParsed. Owns no locks; one coalescer per TUN write queue.
|
||||
type TCPCoalescer struct {
|
||||
w tio.GSOWriter
|
||||
|
||||
// slots is the ordered event queue. Flush walks it once and emits each
|
||||
// entry as either a WriteGSO (coalesced) or a w.Write (verbatim).
|
||||
slots []*coalesceSlot
|
||||
// openSlots maps a flow key to its open slot so new segments can extend an in-progress
|
||||
// superpacket in O(1). Removal is what closes a chain: on PSH or a short last segment, on a
|
||||
// non-admissible packet for the flow, or in Flush.
|
||||
openSlots map[flowKey]*coalesceSlot
|
||||
// lastSlot caches the most recently touched open slot. Bulk traffic
|
||||
// arrives in same-flow runs (single-flow steady state, or GRO bursts
|
||||
// under multi-flow), so comparing the incoming key against the cached
|
||||
// slot's own fk lets the hot path skip the map lookup (and the aeshash
|
||||
// of a 38-byte key) for the length of each run.
|
||||
// Kept in lockstep with openSlots: nil whenever the slot it pointed
|
||||
// at is removed.
|
||||
lastSlot *coalesceSlot
|
||||
pool []*coalesceSlot // free list for reuse
|
||||
l *slog.Logger
|
||||
}
|
||||
|
||||
// NewTCPCoalescer wraps w, returning nil if w can't accept GSO_TCP writes.
|
||||
func NewTCPCoalescer(w io.Writer, l *slog.Logger) *TCPCoalescer {
|
||||
gw, ok := tio.SupportsGSO(w, tio.GSOProtoTCP)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return &TCPCoalescer{
|
||||
w: gw,
|
||||
slots: make([]*coalesceSlot, 0, initialSlots),
|
||||
openSlots: make(map[flowKey]*coalesceSlot, initialSlots),
|
||||
pool: make([]*coalesceSlot, 0, initialSlots),
|
||||
l: l,
|
||||
}
|
||||
}
|
||||
|
||||
// parsedTCP holds the fields extracted from a single parse so later steps
|
||||
// (admission, slot lookup, canAppend) don't re-walk the header.
|
||||
type parsedTCP struct {
|
||||
fk flowKey
|
||||
ipHdrLen int
|
||||
hdrLen int
|
||||
payLen int
|
||||
seq uint32
|
||||
flags byte
|
||||
}
|
||||
|
||||
// parseAt extracts the flow key and IP/TCP offsets for a packet the dispatcher already knows is
|
||||
// TCP; ipHdrLen is the upstream-resolved L4 offset (see flowKey.parseIPAt). p must be zero on
|
||||
// entry and is filled in place; see flowKey.parseIPAt for why. Returns false for malformed input
|
||||
// or any shape that must not coalesce (IPv4 options/fragmentation, IPv6 extension headers).
|
||||
func (p *parsedTCP) parseAt(pkt []byte, ipHdrLen int) bool {
|
||||
trimmed, ok := p.fk.parseIPAt(pkt, ipHdrLen)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return p.parseTail(trimmed, ipHdrLen)
|
||||
}
|
||||
|
||||
// parseTail layers the TCP-header parse on a validated IP prologue. pkt is the trimmed packet;
|
||||
// fk's addresses are already filled.
|
||||
func (p *parsedTCP) parseTail(pkt []byte, ipHdrLen int) bool {
|
||||
if len(pkt) < ipHdrLen+20 {
|
||||
return false
|
||||
}
|
||||
tcpOff := int(pkt[ipHdrLen+12]>>4) * 4
|
||||
if tcpOff < 20 || tcpOff > 60 {
|
||||
return false
|
||||
}
|
||||
if len(pkt) < ipHdrLen+tcpOff {
|
||||
return false
|
||||
}
|
||||
p.ipHdrLen = ipHdrLen
|
||||
p.hdrLen = ipHdrLen + tcpOff
|
||||
p.payLen = len(pkt) - p.hdrLen
|
||||
p.fk.sport = binary.BigEndian.Uint16(pkt[ipHdrLen : ipHdrLen+2])
|
||||
p.fk.dport = binary.BigEndian.Uint16(pkt[ipHdrLen+2 : ipHdrLen+4])
|
||||
p.seq = binary.BigEndian.Uint32(pkt[ipHdrLen+4 : ipHdrLen+8])
|
||||
p.flags = pkt[ipHdrLen+13]
|
||||
return true
|
||||
}
|
||||
|
||||
// TCP flag bits (byte 13 of the TCP header). Only the bits the coalescer consults are named;
|
||||
// FIN/SYN/RST/URG/CWR are rejected by the negative mask in commitParsed.
|
||||
const (
|
||||
tcpFlagPsh = 0x08
|
||||
tcpFlagAck = 0x10
|
||||
tcpFlagEce = 0x40
|
||||
)
|
||||
|
||||
// sealAllOpen closes every open coalesce chain. Called for unparseable packets: the flow key is
|
||||
// unknown, so any open chain could otherwise absorb later data and emit it ahead of this packet.
|
||||
func (c *TCPCoalescer) sealAllOpen() {
|
||||
clear(c.openSlots)
|
||||
c.lastSlot = nil
|
||||
}
|
||||
|
||||
// sealFlow closes fk's open chain, if any, keeping lastSlot in lockstep. The len guard skips
|
||||
// hashing the 38-byte key when no chains are open (e.g. ack-dominant queues).
|
||||
func (c *TCPCoalescer) sealFlow(fk flowKey) {
|
||||
if len(c.openSlots) == 0 {
|
||||
return
|
||||
}
|
||||
if last := c.lastSlot; last != nil && last.fk == fk {
|
||||
c.lastSlot = nil
|
||||
}
|
||||
delete(c.openSlots, fk)
|
||||
}
|
||||
|
||||
// commitStaged commits one staged packet dispatch routed to this lane. A shape the lane cannot
|
||||
// coalesce (any fragmentation, unparseable header) seals every open chain
|
||||
// and rides the lane as an in-lane verbatim, still in transmission order.
|
||||
func (c *TCPCoalescer) commitStaged(sp stagedPacket) error {
|
||||
if sp.fragAny {
|
||||
c.sealAllOpen()
|
||||
c.addVerbatim(sp.pkt)
|
||||
return nil
|
||||
}
|
||||
var info parsedTCP
|
||||
if !info.parseAt(sp.pkt, int(sp.ipHdrLen)) {
|
||||
c.sealAllOpen()
|
||||
c.addVerbatim(sp.pkt)
|
||||
return nil
|
||||
}
|
||||
return c.commitParsed(sp.pkt, &info)
|
||||
}
|
||||
|
||||
// commitParsed commits one parsed TCP packet. The caller (dispatch, via parseAt) supplies a
|
||||
// valid parse so the header is not re-walked here.
|
||||
func (c *TCPCoalescer) commitParsed(pkt []byte, info *parsedTCP) error {
|
||||
// Admission: only ACK, ACK|PSH, ACK|ECE, ACK|PSH|ECE may ride a coalesce chain. CWR marks a
|
||||
// one-shot congestion transition the receiver must observe at a segment boundary. NB: AccECN
|
||||
// reuses CWR as ACE counter bits; revisit this check if inner hosts adopt AccECN.
|
||||
if info.flags&tcpFlagAck == 0 || info.flags&^(tcpFlagAck|tcpFlagPsh|tcpFlagEce) != 0 {
|
||||
// SYN/FIN/RST/URG/CWR must be observed in sequence. Seal the flow's open slot so later
|
||||
// in-flow packets cannot extend it and emit ahead of this verbatim.
|
||||
c.sealFlow(info.fk)
|
||||
c.addVerbatim(pkt)
|
||||
return nil
|
||||
}
|
||||
if info.payLen == 0 {
|
||||
// Pure ACK: no ordering obligation toward the flow's data. Delivering it after
|
||||
// later-transmitted data only makes it a stale ACK, which receivers ignore. Not sealing
|
||||
// keeps a bidirectional flow's data run coalescing across interleaved peer ACKs, matching
|
||||
// kernel GRO. This is the only place emission deviates from transmission order.
|
||||
c.addVerbatim(pkt)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cached-slot fast path. Arrival isn't per-packet interleaved even with
|
||||
// many flows: wire-side GRO delivers runs of same-flow packets
|
||||
// (deliverSegments splits a superdatagram into up to 64), so the cache
|
||||
// hits for the length of each run and a miss costs one fk compare
|
||||
// before the map lookup carries the weight.
|
||||
var open *coalesceSlot
|
||||
if last := c.lastSlot; last != nil && last.fk == info.fk {
|
||||
open = last
|
||||
} else {
|
||||
open = c.openSlots[info.fk]
|
||||
}
|
||||
if open != nil {
|
||||
if c.canAppend(open, pkt, info) {
|
||||
if c.appendPayload(open, pkt, info) {
|
||||
// Chain closed (PSH or short segment): stop extending it.
|
||||
c.sealFlow(info.fk)
|
||||
} else {
|
||||
c.lastSlot = open
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// Can't extend (seq gap from upstream loss, header change, or a full
|
||||
// chain): evict it from openSlots and fall through to seed a fresh slot.
|
||||
c.sealFlow(info.fk)
|
||||
}
|
||||
c.seed(pkt, info)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *TCPCoalescer) Flush() error {
|
||||
var first error
|
||||
for _, s := range c.slots {
|
||||
var err error
|
||||
if s.verbatim || s.numSeg == 1 {
|
||||
// A slot that never grew is byte-identical to its seed packet; ship the original so
|
||||
// its valid checksum rides the DATA_VALID path instead of a kernel software csum.
|
||||
// rawPkt is only mutated once numSeg >= 2 (PSH propagate, flush patches), so it is
|
||||
// pristine here.
|
||||
_, err = c.w.Write(s.rawPkt)
|
||||
} else {
|
||||
err = c.flushSlot(s)
|
||||
}
|
||||
if err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
c.release(s)
|
||||
}
|
||||
clear(c.slots)
|
||||
c.slots = c.slots[:0]
|
||||
clear(c.openSlots)
|
||||
c.lastSlot = nil
|
||||
|
||||
return first
|
||||
}
|
||||
|
||||
func (c *TCPCoalescer) addVerbatim(pkt []byte) {
|
||||
s := c.take()
|
||||
s.verbatim = true
|
||||
s.rawPkt = pkt
|
||||
c.slots = append(c.slots, s)
|
||||
}
|
||||
|
||||
func (c *TCPCoalescer) seed(pkt []byte, info *parsedTCP) {
|
||||
if info.hdrLen+info.payLen > tcpCoalesceBufSize {
|
||||
// Pathological shape that can't ride a superpacket; emit as-is. No chain for this flow can
|
||||
// be open here (commitParsed evicts before seeding), so sealFlow is defense in depth
|
||||
// against a stale cache entry absorbing later data.
|
||||
c.sealFlow(info.fk)
|
||||
c.addVerbatim(pkt)
|
||||
return
|
||||
}
|
||||
s := c.take()
|
||||
s.verbatim = false
|
||||
// rawPkt serves the numSeg==1 fast path in Flush, is the header source for canAppend, and is
|
||||
// the superpacket header flushSlot patches in place.
|
||||
s.rawPkt = pkt
|
||||
s.hdrLen = info.hdrLen
|
||||
s.ipHdrLen = info.ipHdrLen
|
||||
s.isV6 = info.fk.isV6
|
||||
s.fk = info.fk
|
||||
s.gsoSize = info.payLen
|
||||
s.numSeg = 1
|
||||
s.totalPay = info.payLen
|
||||
s.nextSeq = info.seq + uint32(info.payLen)
|
||||
s.payIovs = append(s.payIovs[:0], pkt[info.hdrLen:info.hdrLen+info.payLen])
|
||||
c.slots = append(c.slots, s)
|
||||
if info.flags&tcpFlagPsh == 0 {
|
||||
c.openSlots[info.fk] = s
|
||||
c.lastSlot = s
|
||||
} else {
|
||||
// PSH on the seed closes the chain immediately; it is never registered as open.
|
||||
// Drop any stale entry for this flow too (defense in depth, unreachable if lastSlot's lockstep invariant holds).
|
||||
c.sealFlow(info.fk)
|
||||
}
|
||||
}
|
||||
|
||||
// canAppend reports whether info's packet extends the slot's seed: same header shape and stable
|
||||
// contents, adjacent seq, not oversized. A closed chain never reaches here; closing removes the
|
||||
// slot from openSlots, the only path in. The header fields read from rawPkt are always pristine:
|
||||
// the only pre-flush mutation is the PSH propagate, which also closes the chain.
|
||||
func (c *TCPCoalescer) canAppend(s *coalesceSlot, pkt []byte, info *parsedTCP) bool {
|
||||
if info.hdrLen != s.hdrLen {
|
||||
return false
|
||||
}
|
||||
if info.seq != s.nextSeq {
|
||||
return false
|
||||
}
|
||||
if s.numSeg >= tcpCoalesceMaxSegs {
|
||||
return false
|
||||
}
|
||||
if info.payLen > s.gsoSize {
|
||||
return false
|
||||
}
|
||||
if s.hdrLen+s.totalPay+info.payLen > tcpCoalesceBufSize {
|
||||
return false
|
||||
}
|
||||
// ECE state must be stable across a burst.
|
||||
// Receivers expect the flag set on every segment of a CE-echoing window or none.
|
||||
seedFlags := s.rawPkt[s.ipHdrLen+13]
|
||||
if (seedFlags^info.flags)&tcpFlagEce != 0 {
|
||||
return false
|
||||
}
|
||||
if !s.isV6 && !ipv4CanCoalesceID(s.rawPkt, pkt, s.numSeg) {
|
||||
return false
|
||||
}
|
||||
if !headersMatch(s.rawPkt[:s.hdrLen], pkt[:info.hdrLen], s.isV6, s.ipHdrLen) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// appendPayload folds info's packet into s and reports whether the chain is now closed: the
|
||||
// segment was sub-gsoSize (kernel TSO allows only the final segment to be short) or carried PSH.
|
||||
// The caller must deregister a closed slot from openSlots.
|
||||
func (c *TCPCoalescer) appendPayload(s *coalesceSlot, pkt []byte, info *parsedTCP) bool {
|
||||
s.payIovs = append(s.payIovs, pkt[info.hdrLen:info.hdrLen+info.payLen])
|
||||
s.numSeg++
|
||||
s.totalPay += info.payLen
|
||||
s.nextSeq = info.seq + uint32(info.payLen)
|
||||
if info.flags&tcpFlagPsh != 0 {
|
||||
// Propagate PSH into the seed header so kernel TSO sets it on the last segment. Mutating
|
||||
// rawPkt is safe: PSH also closes the chain, so no admission check re-reads this header.
|
||||
s.rawPkt[s.ipHdrLen+13] |= tcpFlagPsh
|
||||
}
|
||||
return info.payLen < s.gsoSize || info.flags&tcpFlagPsh != 0
|
||||
}
|
||||
|
||||
func (c *TCPCoalescer) take() *coalesceSlot {
|
||||
if n := len(c.pool); n > 0 {
|
||||
s := c.pool[n-1]
|
||||
c.pool[n-1] = nil
|
||||
c.pool = c.pool[:n-1]
|
||||
return s
|
||||
}
|
||||
return &coalesceSlot{}
|
||||
}
|
||||
|
||||
func (c *TCPCoalescer) release(s *coalesceSlot) {
|
||||
clear(s.payIovs)
|
||||
*s = coalesceSlot{payIovs: s.payIovs[:0]}
|
||||
c.pool = append(c.pool, s)
|
||||
}
|
||||
|
||||
// flushSlot patches the superpacket header in place in rawPkt (total length, IPv4 header
|
||||
// checksum, pseudo-header checksum seed) and calls WriteGSO. The slot is released right after,
|
||||
// so nothing re-reads the patched header. Does not remove the slot from c.slots.
|
||||
func (c *TCPCoalescer) flushSlot(s *coalesceSlot) error {
|
||||
total := s.hdrLen + s.totalPay
|
||||
l4Len := total - s.ipHdrLen
|
||||
hdr := s.rawPkt[:s.hdrLen]
|
||||
|
||||
if s.isV6 {
|
||||
binary.BigEndian.PutUint16(hdr[4:6], uint16(l4Len))
|
||||
} else {
|
||||
binary.BigEndian.PutUint16(hdr[2:4], uint16(total))
|
||||
hdr[10] = 0
|
||||
hdr[11] = 0
|
||||
binary.BigEndian.PutUint16(hdr[10:12], ipv4HdrChecksum(hdr[:s.ipHdrLen]))
|
||||
}
|
||||
|
||||
var psum uint32
|
||||
if s.isV6 {
|
||||
psum = pseudoSumIPv6(hdr[8:24], hdr[24:40], ipProtoTCP, l4Len)
|
||||
} else {
|
||||
psum = pseudoSumIPv4(hdr[12:16], hdr[16:20], ipProtoTCP, l4Len)
|
||||
}
|
||||
tcsum := s.ipHdrLen + 16
|
||||
binary.BigEndian.PutUint16(hdr[tcsum:tcsum+2], foldOnceNoInvert(psum))
|
||||
|
||||
return c.w.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoTCP)
|
||||
}
|
||||
|
||||
// headersMatch compares two IP+TCP header prefixes for byte-for-byte
|
||||
// equality on every field that must be identical across coalesced
|
||||
// segments. Size/IPID/IPCsum/seq/flags/tcpCsum are masked out.
|
||||
func headersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
if !ipHeadersMatch(a, b, isV6) {
|
||||
return false
|
||||
}
|
||||
// TCP: compare [0:4] ports, [8:13] ack+dataoff, [14:16] window,
|
||||
// [18:tcpHdrLen] options (incl. urgent).
|
||||
tcp := ipHdrLen
|
||||
if !bytes.Equal(a[tcp:tcp+4], b[tcp:tcp+4]) {
|
||||
return false
|
||||
}
|
||||
if !bytes.Equal(a[tcp+8:tcp+13], b[tcp+8:tcp+13]) {
|
||||
return false
|
||||
}
|
||||
if !bytes.Equal(a[tcp+14:tcp+16], b[tcp+14:tcp+16]) {
|
||||
return false
|
||||
}
|
||||
if !bytes.Equal(a[tcp+18:], b[tcp+18:]) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ipv4HdrChecksum computes the IPv4 header checksum over hdr (which must
|
||||
// already have its checksum field zeroed) and returns the folded/inverted
|
||||
// 16-bit value to store.
|
||||
func ipv4HdrChecksum(hdr []byte) uint16 {
|
||||
var sum uint32
|
||||
for i := 0; i+1 < len(hdr); i += 2 {
|
||||
sum += uint32(binary.BigEndian.Uint16(hdr[i : i+2]))
|
||||
}
|
||||
if len(hdr)%2 == 1 {
|
||||
sum += uint32(hdr[len(hdr)-1]) << 8
|
||||
}
|
||||
for sum>>16 != 0 {
|
||||
sum = (sum & 0xffff) + (sum >> 16)
|
||||
}
|
||||
return ^uint16(sum)
|
||||
}
|
||||
|
||||
// pseudoSumIPv4 / pseudoSumIPv6 build the L4 pseudo-header partial sum
|
||||
// expected by the virtio NEEDS_CSUM kernel path: the 32-bit accumulator
|
||||
// before folding. proto selects the L4 (TCP or UDP); the UDP coalescer
|
||||
// reuses these helpers.
|
||||
func pseudoSumIPv4(src, dst []byte, proto byte, l4Len int) uint32 {
|
||||
var sum uint32
|
||||
sum += uint32(binary.BigEndian.Uint16(src[0:2]))
|
||||
sum += uint32(binary.BigEndian.Uint16(src[2:4]))
|
||||
sum += uint32(binary.BigEndian.Uint16(dst[0:2]))
|
||||
sum += uint32(binary.BigEndian.Uint16(dst[2:4]))
|
||||
sum += uint32(proto)
|
||||
sum += uint32(l4Len)
|
||||
return sum
|
||||
}
|
||||
|
||||
func pseudoSumIPv6(src, dst []byte, proto byte, l4Len int) uint32 {
|
||||
var sum uint32
|
||||
for i := 0; i < 16; i += 2 {
|
||||
sum += uint32(binary.BigEndian.Uint16(src[i : i+2]))
|
||||
sum += uint32(binary.BigEndian.Uint16(dst[i : i+2]))
|
||||
}
|
||||
sum += uint32(l4Len >> 16)
|
||||
sum += uint32(l4Len & 0xffff)
|
||||
sum += uint32(proto)
|
||||
return sum
|
||||
}
|
||||
|
||||
// foldOnceNoInvert folds the 32-bit accumulator to 16 bits and returns it unchanged (no one's complement).
|
||||
// This is what virtio NEEDS_CSUM wants in the L4 checksum field
|
||||
func foldOnceNoInvert(sum uint32) uint16 {
|
||||
for sum>>16 != 0 {
|
||||
sum = (sum & 0xffff) + (sum >> 16)
|
||||
}
|
||||
return uint16(sum)
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"testing"
|
||||
|
||||
"github.com/slackhq/nebula/firewall"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/test"
|
||||
)
|
||||
|
||||
// nopTunWriter is a zero-alloc tio.GSOWriter for benchmarks. Discards
|
||||
// everything but satisfies the interface the coalescer detects.
|
||||
type nopTunWriter struct{}
|
||||
|
||||
func (nopTunWriter) Write(p []byte) (int, error) { return len(p), nil }
|
||||
func (nopTunWriter) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, _ tio.GSOProto) error {
|
||||
return nil
|
||||
}
|
||||
func (nopTunWriter) Capabilities() tio.Capabilities {
|
||||
return tio.Capabilities{TSO: true, USO: true}
|
||||
}
|
||||
|
||||
// buildTCPv4BulkFlow returns a slice of N adjacent ACK-only TCP segments
|
||||
// on a single 5-tuple, each carrying payloadLen bytes. Seq numbers are
|
||||
// contiguous so every packet is coalesceable onto the previous one.
|
||||
func buildTCPv4BulkFlow(n, payloadLen int) [][]byte {
|
||||
pkts := make([][]byte, n)
|
||||
pay := make([]byte, payloadLen)
|
||||
seq := uint32(1000)
|
||||
for i := range n {
|
||||
pkts[i] = buildTCPv4(seq, tcpAck, pay)
|
||||
seq += uint32(payloadLen)
|
||||
}
|
||||
return pkts
|
||||
}
|
||||
|
||||
// buildTCPv4Interleaved returns nFlows * perFlow packets with per-flow
|
||||
// seq continuity but round-robin across flows — worst case for any
|
||||
// "last-slot" cache.
|
||||
func buildTCPv4Interleaved(nFlows, perFlow, payloadLen int) [][]byte {
|
||||
pay := make([]byte, payloadLen)
|
||||
seqs := make([]uint32, nFlows)
|
||||
for i := range seqs {
|
||||
seqs[i] = uint32(1000 + i*1000000)
|
||||
}
|
||||
pkts := make([][]byte, 0, nFlows*perFlow)
|
||||
for range perFlow {
|
||||
for f := range nFlows {
|
||||
sport := uint16(10000 + f)
|
||||
pkts = append(pkts, buildTCPv4Ports(sport, 2000, seqs[f], tcpAck, pay))
|
||||
seqs[f] += uint32(payloadLen)
|
||||
}
|
||||
}
|
||||
return pkts
|
||||
}
|
||||
|
||||
// buildTCPv4RunInterleaved returns nFlows*perFlow packets delivered in
|
||||
// runs of runLen per flow — the arrival pattern wire-side GRO actually
|
||||
// produces (deliverSegments splits each superdatagram into up to 64
|
||||
// same-flow packets back to back). Contrast with buildTCPv4Interleaved's
|
||||
// per-packet round-robin, the adversarial worst case for a last-slot cache.
|
||||
func buildTCPv4RunInterleaved(nFlows, perFlow, runLen, payloadLen int) [][]byte {
|
||||
pay := make([]byte, payloadLen)
|
||||
seqs := make([]uint32, nFlows)
|
||||
for i := range seqs {
|
||||
seqs[i] = uint32(1000 + i*1000000)
|
||||
}
|
||||
pkts := make([][]byte, 0, nFlows*perFlow)
|
||||
for done := 0; done < perFlow; done += runLen {
|
||||
for f := range nFlows {
|
||||
sport := uint16(10000 + f)
|
||||
for range runLen {
|
||||
pkts = append(pkts, buildTCPv4Ports(sport, 2000, seqs[f], tcpAck, pay))
|
||||
seqs[f] += uint32(payloadLen)
|
||||
}
|
||||
}
|
||||
}
|
||||
return pkts
|
||||
}
|
||||
|
||||
// buildICMPv4 returns a minimal non-TCP packet that takes the verbatim
|
||||
// branch in Commit.
|
||||
func buildICMPv4() []byte {
|
||||
pkt := make([]byte, 28)
|
||||
pkt[0] = 0x45
|
||||
binary.BigEndian.PutUint16(pkt[2:4], 28)
|
||||
pkt[9] = 1 // ICMP
|
||||
copy(pkt[12:16], []byte{10, 0, 0, 1})
|
||||
copy(pkt[16:20], []byte{10, 0, 0, 2})
|
||||
return pkt
|
||||
}
|
||||
|
||||
// runCommitBench drives Commit over pkts batchSize at a time, flushing
|
||||
// between batches, and reports per-packet cost.
|
||||
func runCommitBench(b *testing.B, pkts [][]byte, batchSize int) {
|
||||
b.Helper()
|
||||
c := newTestTCPCoalescer(b, nopTunWriter{})
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(pkts[0])))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
pkt := pkts[i%len(pkts)]
|
||||
if err := c.Commit(pkt); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if (i+1)%batchSize == 0 {
|
||||
if err := c.Flush(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Drain any trailing partial batch so slot state doesn't leak across runs.
|
||||
_ = c.Flush()
|
||||
}
|
||||
|
||||
// BenchmarkCommitSingleFlow is the bulk-TCP steady state: one flow,
|
||||
// contiguous seq, 1200-byte payloads. Every packet past the seed should
|
||||
// append onto the open slot. This is the case we most care about.
|
||||
func BenchmarkCommitSingleFlow(b *testing.B) {
|
||||
pkts := buildTCPv4BulkFlow(tcpCoalesceMaxSegs, 1200)
|
||||
runCommitBench(b, pkts, tcpCoalesceMaxSegs)
|
||||
}
|
||||
|
||||
// BenchmarkCommitInterleaved4 has 4 concurrent bulk flows round-robined.
|
||||
// A single-entry fast-path cache will miss on every packet; an N-way
|
||||
// cache or map lookup carries the weight.
|
||||
func BenchmarkCommitInterleaved4(b *testing.B) {
|
||||
pkts := buildTCPv4Interleaved(4, tcpCoalesceMaxSegs, 1200)
|
||||
runCommitBench(b, pkts, len(pkts))
|
||||
}
|
||||
|
||||
// BenchmarkCommitInterleaved16 stresses the map at higher flow counts.
|
||||
func BenchmarkCommitInterleaved16(b *testing.B) {
|
||||
pkts := buildTCPv4Interleaved(16, tcpCoalesceMaxSegs, 1200)
|
||||
runCommitBench(b, pkts, len(pkts))
|
||||
}
|
||||
|
||||
// BenchmarkCommitRunInterleaved4 is 4 concurrent flows arriving in
|
||||
// GRO-burst runs of 16 — the realistic multi-flow pattern. A last-slot
|
||||
// cache hits for the length of each run; the per-packet round-robin
|
||||
// benches above are its worst case.
|
||||
func BenchmarkCommitRunInterleaved4(b *testing.B) {
|
||||
pkts := buildTCPv4RunInterleaved(4, tcpCoalesceMaxSegs, 16, 1200)
|
||||
runCommitBench(b, pkts, len(pkts))
|
||||
}
|
||||
|
||||
// BenchmarkCommitPassthrough exercises the non-TCP branch: parseBase
|
||||
// bails early and addVerbatim is the only work.
|
||||
func BenchmarkCommitPassthrough(b *testing.B) {
|
||||
pkt := buildICMPv4()
|
||||
pkts := make([][]byte, 64)
|
||||
for i := range pkts {
|
||||
pkts[i] = pkt
|
||||
}
|
||||
runCommitBench(b, pkts, 64)
|
||||
}
|
||||
|
||||
// BenchmarkCommitNonCoalesceableTCP sends SYN|ACK packets on one flow.
|
||||
// Each packet takes the "TCP but not admissible" branch which does a
|
||||
// map delete + verbatim. Measures the seal-without-slot cost.
|
||||
func BenchmarkCommitNonCoalesceableTCP(b *testing.B) {
|
||||
pay := make([]byte, 0)
|
||||
pkts := make([][]byte, 64)
|
||||
for i := range pkts {
|
||||
pkts[i] = buildTCPv4(uint32(1000+i), tcpSyn|tcpAck, pay)
|
||||
}
|
||||
runCommitBench(b, pkts, 64)
|
||||
}
|
||||
|
||||
// runMultiCommitBench drives MultiCoalescer.Commit with in-order keys, so
|
||||
// it includes the staging sort's already-sorted fast path plus the
|
||||
// dispatch-time parse — the full steady-state cost of the batcher. The
|
||||
// ParsedPackets are precomputed: in production they fall out of the
|
||||
// firewall's newPacket, which this bench does not model.
|
||||
func runMultiCommitBench(b *testing.B, pkts [][]byte, batchSize int) {
|
||||
b.Helper()
|
||||
m := NewMultiCoalescer(nopTunWriter{}, test.NewLogger())
|
||||
pps := make([]*firewall.ParsedPacket, len(pkts))
|
||||
for i, p := range pkts {
|
||||
pps[i] = testPP(p)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(pkts[0])))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
j := i % len(pkts)
|
||||
if err := m.Commit(pkts[j], SortKey{Epoch: 1, Counter: uint64(i + 1)}, pps[j]); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if (i+1)%batchSize == 0 {
|
||||
if err := m.Flush(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = m.Flush()
|
||||
}
|
||||
|
||||
// BenchmarkMultiCommitSingleFlow is the multi-lane analogue of
|
||||
// BenchmarkCommitSingleFlow — same workload but routed through the
|
||||
// dispatcher. The delta vs the single-lane bench measures dispatcher
|
||||
// overhead.
|
||||
func BenchmarkMultiCommitSingleFlow(b *testing.B) {
|
||||
pkts := buildTCPv4BulkFlow(tcpCoalesceMaxSegs, 1200)
|
||||
runMultiCommitBench(b, pkts, tcpCoalesceMaxSegs)
|
||||
}
|
||||
|
||||
// BenchmarkMultiCommitInterleaved4 mirrors BenchmarkCommitInterleaved4
|
||||
// through the dispatcher.
|
||||
func BenchmarkMultiCommitInterleaved4(b *testing.B) {
|
||||
pkts := buildTCPv4Interleaved(4, tcpCoalesceMaxSegs, 1200)
|
||||
runMultiCommitBench(b, pkts, len(pkts))
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,59 @@
|
||||
package batch
|
||||
|
||||
import "net/netip"
|
||||
|
||||
const SendBatchCap = 128
|
||||
|
||||
// batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush.
|
||||
type batchWriter interface {
|
||||
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
|
||||
}
|
||||
|
||||
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
|
||||
// One SendBatch is owned by each listenIn goroutine; no locking is needed.
|
||||
// Slots are backed by an Arena (see its docs)
|
||||
type SendBatch struct {
|
||||
out batchWriter
|
||||
bufs [][]byte
|
||||
dsts []netip.AddrPort
|
||||
arena *Arena
|
||||
}
|
||||
|
||||
// NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots
|
||||
func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch {
|
||||
return &SendBatch{
|
||||
out: out,
|
||||
bufs: make([][]byte, 0, batchCap),
|
||||
dsts: make([]netip.AddrPort, 0, batchCap),
|
||||
arena: NewArena(arenaSize),
|
||||
}
|
||||
}
|
||||
|
||||
func (b *SendBatch) Reserve(sz int) []byte {
|
||||
return b.arena.Reserve(sz)
|
||||
}
|
||||
|
||||
// Len reports how many packets are queued for the next Flush. Callers use
|
||||
// it to flush incrementally once a full sendmmsg batch has accumulated,
|
||||
// bounding how long the first packet of a large read batch waits.
|
||||
func (b *SendBatch) Len() int { return len(b.bufs) }
|
||||
|
||||
func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort) {
|
||||
b.bufs = append(b.bufs, pkt)
|
||||
b.dsts = append(b.dsts, dst)
|
||||
}
|
||||
|
||||
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations
|
||||
// were undeliverable; the batch is drained either way.
|
||||
func (b *SendBatch) Flush() (int, error) {
|
||||
var err error
|
||||
written := 0
|
||||
if len(b.bufs) > 0 {
|
||||
written, err = b.out.WriteBatch(b.bufs, b.dsts)
|
||||
}
|
||||
clear(b.bufs)
|
||||
b.bufs = b.bufs[:0]
|
||||
b.dsts = b.dsts[:0]
|
||||
b.arena.Reset()
|
||||
return written, err
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeBatchWriter struct {
|
||||
bufs [][]byte
|
||||
addrs []netip.AddrPort
|
||||
}
|
||||
|
||||
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||
// Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch
|
||||
// returns, so tests must capture data before that happens.
|
||||
w.bufs = make([][]byte, len(bufs))
|
||||
for i, b := range bufs {
|
||||
cp := make([]byte, len(b))
|
||||
copy(cp, b)
|
||||
w.bufs[i] = cp
|
||||
}
|
||||
w.addrs = append(w.addrs[:0], addrs...)
|
||||
return len(bufs), nil
|
||||
}
|
||||
|
||||
func TestSendBatchReserveCommitFlush(t *testing.T) {
|
||||
fw := &fakeBatchWriter{}
|
||||
b := NewSendBatch(fw, 4, 32)
|
||||
|
||||
ap := netip.MustParseAddrPort("10.0.0.1:4242")
|
||||
for i := 0; i < 4; i++ {
|
||||
slot := b.Reserve(32)
|
||||
if cap(slot) != 32 {
|
||||
t.Fatalf("slot %d: cap=%d want 32", i, cap(slot))
|
||||
}
|
||||
pkt := append(slot[:0], byte(i), byte(i+1), byte(i+2))
|
||||
b.Commit(pkt, ap)
|
||||
}
|
||||
if _, err := b.Flush(); err != nil {
|
||||
t.Fatalf("Flush: %v", err)
|
||||
}
|
||||
if len(fw.bufs) != 4 {
|
||||
t.Fatalf("WriteBatch got %d bufs want 4", len(fw.bufs))
|
||||
}
|
||||
for i, buf := range fw.bufs {
|
||||
if len(buf) != 3 || buf[0] != byte(i) {
|
||||
t.Errorf("buf %d: %x", i, buf)
|
||||
}
|
||||
if fw.addrs[i] != ap {
|
||||
t.Errorf("addr %d: got %v want %v", i, fw.addrs[i], ap)
|
||||
}
|
||||
}
|
||||
|
||||
// Flush again with nothing committed — should be a no-op.
|
||||
fw.bufs = nil
|
||||
if _, err := b.Flush(); err != nil {
|
||||
t.Fatalf("empty Flush: %v", err)
|
||||
}
|
||||
if fw.bufs != nil {
|
||||
t.Fatalf("empty Flush triggered WriteBatch")
|
||||
}
|
||||
|
||||
// Reuse after Flush.
|
||||
slot := b.Reserve(32)
|
||||
if cap(slot) != 32 {
|
||||
t.Fatalf("after Flush Reserve wrong cap: %d", cap(slot))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendBatchSlotsDoNotOverlap(t *testing.T) {
|
||||
fw := &fakeBatchWriter{}
|
||||
b := NewSendBatch(fw, 3, 8)
|
||||
ap := netip.MustParseAddrPort("10.0.0.1:80")
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
s := b.Reserve(8)
|
||||
pkt := append(s[:0], byte(0xA0+i), byte(0xB0+i))
|
||||
b.Commit(pkt, ap)
|
||||
}
|
||||
if _, err := b.Flush(); err != nil {
|
||||
t.Fatalf("Flush: %v", err)
|
||||
}
|
||||
|
||||
for i, buf := range fw.bufs {
|
||||
if buf[0] != byte(0xA0+i) || buf[1] != byte(0xB0+i) {
|
||||
t.Errorf("slot %d corrupted: %x", i, buf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendBatchGrowPreservesCommitted(t *testing.T) {
|
||||
fw := &fakeBatchWriter{}
|
||||
// Tiny initial backing forces a grow on the second Reserve.
|
||||
b := NewSendBatch(fw, 1, 4)
|
||||
ap := netip.MustParseAddrPort("10.0.0.1:80")
|
||||
|
||||
s1 := b.Reserve(4)
|
||||
pkt1 := append(s1[:0], 0x11, 0x22, 0x33, 0x44)
|
||||
b.Commit(pkt1, ap)
|
||||
|
||||
s2 := b.Reserve(8) // exceeds remaining cap, triggers grow
|
||||
pkt2 := append(s2[:0], 0xA, 0xB, 0xC, 0xD, 0xE)
|
||||
b.Commit(pkt2, ap)
|
||||
|
||||
// pkt1 must still be intact even though backing reallocated.
|
||||
if pkt1[0] != 0x11 || pkt1[3] != 0x44 {
|
||||
t.Fatalf("first packet corrupted by grow: %x", pkt1)
|
||||
}
|
||||
|
||||
if _, err := b.Flush(); err != nil {
|
||||
t.Fatalf("Flush: %v", err)
|
||||
}
|
||||
if len(fw.bufs) != 2 {
|
||||
t.Fatalf("got %d bufs want 2", len(fw.bufs))
|
||||
}
|
||||
if fw.bufs[0][0] != 0x11 || fw.bufs[0][3] != 0x44 {
|
||||
t.Errorf("first packet on the wire: %x", fw.bufs[0])
|
||||
}
|
||||
if fw.bufs[1][0] != 0xA || fw.bufs[1][4] != 0xE {
|
||||
t.Errorf("second packet on the wire: %x", fw.bufs[1])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,345 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
)
|
||||
|
||||
// ipProtoUDP is the IANA protocol number for UDP.
|
||||
const ipProtoUDP = 17
|
||||
|
||||
// udpCoalesceBufSize caps total bytes per UDP superpacket. Mirrors the
|
||||
// kernel's gso_max_size; payloads beyond this are emitted as-is.
|
||||
const udpCoalesceBufSize = 65535
|
||||
|
||||
// udpCoalesceMaxSegs caps how many segments we'll coalesce. Kernel UDP-GSO
|
||||
// accepts up to 64 segments per skb (UDP_MAX_SEGMENTS); stay under that.
|
||||
const udpCoalesceMaxSegs = 64
|
||||
|
||||
// udpSlot is one entry in the UDPCoalescer's ordered event queue.
|
||||
type udpSlot struct {
|
||||
verbatim bool
|
||||
// rawPkt is borrowed: the whole packet for verbatim slots, the seed
|
||||
// packet for coalesce slots. A coalesce slot that never grows past one
|
||||
// segment is emitted from rawPkt so its original (already valid) L4
|
||||
// checksum ships DATA_VALID instead of making the kernel recompute it.
|
||||
// A multi-segment slot's superpacket header is rawPkt's, patched in place at flush.
|
||||
rawPkt []byte
|
||||
|
||||
fk flowKey
|
||||
hdrLen int
|
||||
ipHdrLen int
|
||||
isV6 bool
|
||||
gsoSize int // per-segment UDP payload length
|
||||
numSeg int
|
||||
totalPay int
|
||||
payIovs [][]byte
|
||||
}
|
||||
|
||||
// UDPCoalescer accumulates adjacent in-flow UDP datagrams across multiple
|
||||
// concurrent flows and emits each flow's run as a single GSO_UDP_L4 superpacket via tio.GSOWriter.
|
||||
// Preserves the in-flow order of packets as they are Commit-ed
|
||||
//
|
||||
// Owns no locks; one coalescer per TUN write queue.
|
||||
type UDPCoalescer struct {
|
||||
w tio.GSOWriter
|
||||
slots []*udpSlot
|
||||
openSlots map[flowKey]*udpSlot
|
||||
// lastSlot caches the most recently touched open slot; see the
|
||||
// TCPCoalescer field of the same name. Single-flow QUIC bulk is the
|
||||
// dominant USO workload, and multi-flow arrival comes in GRO runs, so
|
||||
// the fk compare beats the map's 38-byte key hash on most packets.
|
||||
// Kept in lockstep with openSlots: nil whenever the slot it pointed at
|
||||
// is removed.
|
||||
lastSlot *udpSlot
|
||||
pool []*udpSlot
|
||||
}
|
||||
|
||||
func NewUDPCoalescer(w io.Writer) *UDPCoalescer {
|
||||
gw, ok := tio.SupportsGSO(w, tio.GSOProtoUDP)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return &UDPCoalescer{
|
||||
w: gw,
|
||||
slots: make([]*udpSlot, 0, initialSlots),
|
||||
openSlots: make(map[flowKey]*udpSlot, initialSlots),
|
||||
pool: make([]*udpSlot, 0, initialSlots),
|
||||
}
|
||||
}
|
||||
|
||||
// parsedUDP holds the fields extracted from a single parse so later steps
|
||||
// (admission, slot lookup, canAppend) don't re-walk the header.
|
||||
type parsedUDP struct {
|
||||
fk flowKey
|
||||
ipHdrLen int
|
||||
hdrLen int // ipHdrLen + 8
|
||||
payLen int
|
||||
}
|
||||
|
||||
// parseAt extracts the flow key and IP/UDP offsets for a packet the dispatcher already knows is
|
||||
// UDP; ipHdrLen is the upstream-resolved L4 offset (see flowKey.parseIPAt). p must be zero on
|
||||
// entry and is filled in place. Returns false for malformed input or any shape that must not
|
||||
// coalesce (IPv4 options/fragmentation, IPv6 extension headers).
|
||||
func (p *parsedUDP) parseAt(pkt []byte, ipHdrLen int) bool {
|
||||
trimmed, ok := p.fk.parseIPAt(pkt, ipHdrLen)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return p.parseTail(trimmed, ipHdrLen)
|
||||
}
|
||||
|
||||
// parseTail layers the UDP-header parse on a validated IP prologue. pkt is the trimmed packet;
|
||||
// fk's addresses are already filled.
|
||||
func (p *parsedUDP) parseTail(pkt []byte, ipHdrLen int) bool {
|
||||
if len(pkt) < ipHdrLen+8 {
|
||||
return false
|
||||
}
|
||||
// UDP `length` field: must equal IP-derived length-of-UDP-header-plus-payload.
|
||||
udpLen := int(binary.BigEndian.Uint16(pkt[ipHdrLen+4 : ipHdrLen+6]))
|
||||
if udpLen < 8 || udpLen > len(pkt)-ipHdrLen {
|
||||
return false
|
||||
}
|
||||
p.ipHdrLen = ipHdrLen
|
||||
p.hdrLen = ipHdrLen + 8
|
||||
p.payLen = udpLen - 8
|
||||
p.fk.sport = binary.BigEndian.Uint16(pkt[ipHdrLen : ipHdrLen+2])
|
||||
p.fk.dport = binary.BigEndian.Uint16(pkt[ipHdrLen+2 : ipHdrLen+4])
|
||||
return true
|
||||
}
|
||||
|
||||
// sealFlow closes fk's open chain, if any, keeping lastSlot in lockstep. The len guard skips
|
||||
// hashing the 38-byte key when no chains are open.
|
||||
func (c *UDPCoalescer) sealFlow(fk flowKey) {
|
||||
if len(c.openSlots) == 0 {
|
||||
return
|
||||
}
|
||||
if last := c.lastSlot; last != nil && last.fk == fk {
|
||||
c.lastSlot = nil
|
||||
}
|
||||
delete(c.openSlots, fk)
|
||||
}
|
||||
|
||||
// commitStaged commits one staged packet dispatch routed to this lane. A shape the lane cannot
|
||||
// coalesce (any fragmentation, unparseable header) seals every open chain — its flow is unknown —
|
||||
// and rides the lane as an in-lane verbatim, still in transmission order.
|
||||
func (c *UDPCoalescer) commitStaged(sp stagedPacket) error {
|
||||
if sp.fragAny {
|
||||
c.sealAllOpen()
|
||||
c.addVerbatim(sp.pkt)
|
||||
return nil
|
||||
}
|
||||
var info parsedUDP
|
||||
if !info.parseAt(sp.pkt, int(sp.ipHdrLen)) {
|
||||
c.sealAllOpen()
|
||||
c.addVerbatim(sp.pkt)
|
||||
return nil
|
||||
}
|
||||
return c.commitParsed(sp.pkt, &info)
|
||||
}
|
||||
|
||||
// commitParsed commits one parsed UDP packet. The caller (dispatch, via parseAt) supplies a
|
||||
// valid parse so the header is not re-walked here.
|
||||
func (c *UDPCoalescer) commitParsed(pkt []byte, info *parsedUDP) error {
|
||||
// A zero-length UDP datagram (length == 8) is legal and must reach the TUN, but cannot be
|
||||
// coalesced.
|
||||
if info.payLen == 0 {
|
||||
c.sealFlow(info.fk)
|
||||
c.addVerbatim(pkt)
|
||||
return nil
|
||||
}
|
||||
// Cached-slot fast path; see the TCPCoalescer equivalent.
|
||||
var open *udpSlot
|
||||
if last := c.lastSlot; last != nil && last.fk == info.fk {
|
||||
open = last
|
||||
} else {
|
||||
open = c.openSlots[info.fk]
|
||||
}
|
||||
if open != nil {
|
||||
if c.canAppend(open, pkt, info) {
|
||||
if c.appendPayload(open, pkt, info) {
|
||||
// Chain closed (short segment): stop extending it.
|
||||
c.sealFlow(info.fk)
|
||||
} else {
|
||||
c.lastSlot = open
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// Can't extend: evict it from openSlots and fall through to seed a
|
||||
// fresh slot.
|
||||
c.sealFlow(info.fk)
|
||||
}
|
||||
c.seed(pkt, info)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *UDPCoalescer) Flush() error {
|
||||
var first error
|
||||
for _, s := range c.slots {
|
||||
var err error
|
||||
if s.verbatim || s.numSeg == 1 {
|
||||
// A slot that never grew is byte-identical to the packet it was
|
||||
// seeded from; ship the original so its valid checksum rides the
|
||||
// DATA_VALID path instead of paying a kernel software csum.
|
||||
_, err = c.w.Write(s.rawPkt)
|
||||
} else {
|
||||
err = c.flushSlot(s)
|
||||
}
|
||||
if err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
c.release(s)
|
||||
}
|
||||
clear(c.slots)
|
||||
c.slots = c.slots[:0]
|
||||
clear(c.openSlots)
|
||||
c.lastSlot = nil
|
||||
return first
|
||||
}
|
||||
|
||||
// sealAllOpen closes every open coalesce chain. Called for unparseable packets: the flow key is
|
||||
// unknown, so any open chain could otherwise absorb later data and emit it ahead of this packet.
|
||||
func (c *UDPCoalescer) sealAllOpen() {
|
||||
clear(c.openSlots)
|
||||
c.lastSlot = nil
|
||||
}
|
||||
|
||||
func (c *UDPCoalescer) addVerbatim(pkt []byte) {
|
||||
s := c.take()
|
||||
s.verbatim = true
|
||||
s.rawPkt = pkt
|
||||
c.slots = append(c.slots, s)
|
||||
}
|
||||
|
||||
func (c *UDPCoalescer) seed(pkt []byte, info *parsedUDP) {
|
||||
if info.hdrLen+info.payLen > udpCoalesceBufSize {
|
||||
// Pathological shape that can't ride a superpacket; emit as-is. No chain for this flow can
|
||||
// be open here (commitParsed evicts before seeding), so sealFlow is defense in depth
|
||||
// against a stale cache entry absorbing later data.
|
||||
c.sealFlow(info.fk)
|
||||
c.addVerbatim(pkt)
|
||||
return
|
||||
}
|
||||
s := c.take()
|
||||
s.verbatim = false
|
||||
// rawPkt serves the numSeg==1 fast path in Flush, is the header source for canAppend, and is
|
||||
// the superpacket header flushSlot patches in place.
|
||||
s.rawPkt = pkt
|
||||
s.hdrLen = info.hdrLen
|
||||
s.ipHdrLen = info.ipHdrLen
|
||||
s.isV6 = info.fk.isV6
|
||||
s.fk = info.fk
|
||||
s.gsoSize = info.payLen
|
||||
s.numSeg = 1
|
||||
s.totalPay = info.payLen
|
||||
s.payIovs = append(s.payIovs[:0], pkt[info.hdrLen:info.hdrLen+info.payLen])
|
||||
c.slots = append(c.slots, s)
|
||||
c.openSlots[info.fk] = s
|
||||
c.lastSlot = s
|
||||
}
|
||||
|
||||
// canAppend reports whether info's packet extends the slot's seed.
|
||||
// Kernel UDP-GSO requires every segment except possibly the last to be
|
||||
// exactly gsoSize, and the last may be shorter (≤ gsoSize).
|
||||
func (c *UDPCoalescer) canAppend(s *udpSlot, pkt []byte, info *parsedUDP) bool {
|
||||
if info.hdrLen != s.hdrLen {
|
||||
return false
|
||||
}
|
||||
if s.numSeg >= udpCoalesceMaxSegs {
|
||||
return false
|
||||
}
|
||||
if info.payLen > s.gsoSize {
|
||||
return false
|
||||
}
|
||||
if s.hdrLen+s.totalPay+info.payLen > udpCoalesceBufSize {
|
||||
return false
|
||||
}
|
||||
// Header reads use rawPkt, which is never mutated before flush. A closed chain never reaches
|
||||
// here; closing removes the slot from openSlots, the only path in.
|
||||
if !s.isV6 && !ipv4CanCoalesceID(s.rawPkt, pkt, s.numSeg) {
|
||||
return false
|
||||
}
|
||||
if !udpHeadersMatch(s.rawPkt[:s.hdrLen], pkt[:info.hdrLen], s.isV6, s.ipHdrLen) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// appendPayload folds info's packet into s and reports whether the chain is now closed: kernel
|
||||
// UDP-GSO requires every segment but the last to be exactly gsoSize, so a short segment must be
|
||||
// the final one. The caller must deregister a closed slot from openSlots.
|
||||
func (c *UDPCoalescer) appendPayload(s *udpSlot, pkt []byte, info *parsedUDP) bool {
|
||||
s.payIovs = append(s.payIovs, pkt[info.hdrLen:info.hdrLen+info.payLen])
|
||||
s.numSeg++
|
||||
s.totalPay += info.payLen
|
||||
return info.payLen < s.gsoSize
|
||||
}
|
||||
|
||||
func (c *UDPCoalescer) take() *udpSlot {
|
||||
if n := len(c.pool); n > 0 {
|
||||
s := c.pool[n-1]
|
||||
c.pool[n-1] = nil
|
||||
c.pool = c.pool[:n-1]
|
||||
return s
|
||||
}
|
||||
return &udpSlot{}
|
||||
}
|
||||
|
||||
func (c *UDPCoalescer) release(s *udpSlot) {
|
||||
// Reset every field, identity ones included; see TCPCoalescer.release.
|
||||
clear(s.payIovs)
|
||||
*s = udpSlot{payIovs: s.payIovs[:0]}
|
||||
c.pool = append(c.pool, s)
|
||||
}
|
||||
|
||||
// flushSlot patches the IP header total length / IPv6 payload length and
|
||||
// the UDP length to the *total* across all coalesced segments, then seeds
|
||||
// the UDP checksum field with the pseudo-header partial (single-fold, not
|
||||
// inverted) per virtio NEEDS_CSUM. The patches land in place in rawPkt; the
|
||||
// slot is released right after, so nothing re-reads the patched header.
|
||||
func (c *UDPCoalescer) flushSlot(s *udpSlot) error {
|
||||
hdr := s.rawPkt[:s.hdrLen]
|
||||
total := s.hdrLen + s.totalPay // full IP+UDP+all_payloads bytes
|
||||
l4Len := total - s.ipHdrLen // total UDP (8 + sum of payloads)
|
||||
|
||||
if s.isV6 {
|
||||
binary.BigEndian.PutUint16(hdr[4:6], uint16(l4Len))
|
||||
} else {
|
||||
binary.BigEndian.PutUint16(hdr[2:4], uint16(total))
|
||||
hdr[10] = 0
|
||||
hdr[11] = 0
|
||||
binary.BigEndian.PutUint16(hdr[10:12], ipv4HdrChecksum(hdr[:s.ipHdrLen]))
|
||||
}
|
||||
|
||||
// UDP length field (offset 4 inside the UDP header) = total UDP size.
|
||||
binary.BigEndian.PutUint16(hdr[s.ipHdrLen+4:s.ipHdrLen+6], uint16(l4Len))
|
||||
|
||||
var psum uint32
|
||||
if s.isV6 {
|
||||
psum = pseudoSumIPv6(hdr[8:24], hdr[24:40], ipProtoUDP, l4Len)
|
||||
} else {
|
||||
psum = pseudoSumIPv4(hdr[12:16], hdr[16:20], ipProtoUDP, l4Len)
|
||||
}
|
||||
udpCsumOff := s.ipHdrLen + 6
|
||||
binary.BigEndian.PutUint16(hdr[udpCsumOff:udpCsumOff+2], foldOnceNoInvert(psum))
|
||||
|
||||
return c.w.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoUDP)
|
||||
}
|
||||
|
||||
// udpHeadersMatch compares two IP+UDP header prefixes for byte-equality on
|
||||
// every field that must be identical across coalesced segments
|
||||
func udpHeadersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
if !ipHeadersMatch(a, b, isV6) {
|
||||
return false
|
||||
}
|
||||
// UDP: compare sport+dport ([0:4]). Skip length [4:6] and checksum [6:8]:
|
||||
// length varies (we rewrite at flush) and the checksum will be redone.
|
||||
udp := ipHdrLen
|
||||
return bytes.Equal(a[udp:udp+4], b[udp:udp+4])
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// buildUDPv4BulkFlow returns n equal-size datagrams on one flow — the
|
||||
// steady state for single-flow QUIC bulk, the workload USO exists for.
|
||||
func buildUDPv4BulkFlow(n, payloadLen int) [][]byte {
|
||||
pay := make([]byte, payloadLen)
|
||||
pkts := make([][]byte, n)
|
||||
for i := range pkts {
|
||||
pkts[i] = buildUDPv4(40000, 443, pay)
|
||||
}
|
||||
return pkts
|
||||
}
|
||||
|
||||
// buildUDPv4RunInterleaved mirrors buildTCPv4RunInterleaved: nFlows*perFlow
|
||||
// datagrams arriving in GRO-burst runs of runLen per flow.
|
||||
func buildUDPv4RunInterleaved(nFlows, perFlow, runLen, payloadLen int) [][]byte {
|
||||
pay := make([]byte, payloadLen)
|
||||
pkts := make([][]byte, 0, nFlows*perFlow)
|
||||
for done := 0; done < perFlow; done += runLen {
|
||||
for f := range nFlows {
|
||||
sport := uint16(40000 + f)
|
||||
for range runLen {
|
||||
pkts = append(pkts, buildUDPv4(sport, 443, pay))
|
||||
}
|
||||
}
|
||||
}
|
||||
return pkts
|
||||
}
|
||||
|
||||
// runUDPCommitBench drives UDPCoalescer.Commit over pkts batchSize at a
|
||||
// time, flushing between batches, and reports per-packet cost.
|
||||
func runUDPCommitBench(b *testing.B, pkts [][]byte, batchSize int) {
|
||||
b.Helper()
|
||||
c := newTestUDPCoalescer(b, nopTunWriter{})
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(pkts[0])))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
pkt := pkts[i%len(pkts)]
|
||||
if err := c.Commit(pkt); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if (i+1)%batchSize == 0 {
|
||||
if err := c.Flush(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = c.Flush()
|
||||
}
|
||||
|
||||
// BenchmarkUDPCommitSingleFlow is the single-flow bulk steady state.
|
||||
func BenchmarkUDPCommitSingleFlow(b *testing.B) {
|
||||
pkts := buildUDPv4BulkFlow(udpCoalesceMaxSegs, 1200)
|
||||
runUDPCommitBench(b, pkts, udpCoalesceMaxSegs)
|
||||
}
|
||||
|
||||
// BenchmarkUDPCommitInterleaved4 is the adversarial per-packet round-robin.
|
||||
func BenchmarkUDPCommitInterleaved4(b *testing.B) {
|
||||
pkts := buildUDPv4RunInterleaved(4, udpCoalesceMaxSegs, 1, 1200)
|
||||
runUDPCommitBench(b, pkts, len(pkts))
|
||||
}
|
||||
|
||||
// BenchmarkUDPCommitRunInterleaved4 is 4 flows in GRO-burst runs of 16.
|
||||
func BenchmarkUDPCommitRunInterleaved4(b *testing.B) {
|
||||
pkts := buildUDPv4RunInterleaved(4, udpCoalesceMaxSegs, 16, 1200)
|
||||
runUDPCommitBench(b, pkts, len(pkts))
|
||||
}
|
||||
@@ -0,0 +1,536 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// buildUDPv4 builds a minimal IPv4+UDP packet with the given payload and ports.
|
||||
func buildUDPv4(sport, dport uint16, payload []byte) []byte {
|
||||
const ipHdrLen = 20
|
||||
const udpHdrLen = 8
|
||||
total := ipHdrLen + udpHdrLen + len(payload)
|
||||
pkt := make([]byte, total)
|
||||
|
||||
pkt[0] = 0x45
|
||||
pkt[1] = 0x00
|
||||
binary.BigEndian.PutUint16(pkt[2:4], uint16(total))
|
||||
binary.BigEndian.PutUint16(pkt[4:6], 0)
|
||||
binary.BigEndian.PutUint16(pkt[6:8], 0x4000)
|
||||
pkt[8] = 64
|
||||
pkt[9] = ipProtoUDP
|
||||
copy(pkt[12:16], []byte{10, 0, 0, 1})
|
||||
copy(pkt[16:20], []byte{10, 0, 0, 2})
|
||||
|
||||
binary.BigEndian.PutUint16(pkt[20:22], sport)
|
||||
binary.BigEndian.PutUint16(pkt[22:24], dport)
|
||||
binary.BigEndian.PutUint16(pkt[24:26], uint16(udpHdrLen+len(payload)))
|
||||
binary.BigEndian.PutUint16(pkt[26:28], 0)
|
||||
|
||||
copy(pkt[28:], payload)
|
||||
return pkt
|
||||
}
|
||||
|
||||
// buildUDPv6 builds a minimal IPv6+UDP packet.
|
||||
func buildUDPv6(sport, dport uint16, payload []byte) []byte {
|
||||
const ipHdrLen = 40
|
||||
const udpHdrLen = 8
|
||||
total := ipHdrLen + udpHdrLen + len(payload)
|
||||
pkt := make([]byte, total)
|
||||
|
||||
pkt[0] = 0x60
|
||||
binary.BigEndian.PutUint16(pkt[4:6], uint16(udpHdrLen+len(payload)))
|
||||
pkt[6] = ipProtoUDP
|
||||
pkt[7] = 64
|
||||
pkt[8] = 0xfe
|
||||
pkt[9] = 0x80
|
||||
pkt[23] = 1
|
||||
pkt[24] = 0xfe
|
||||
pkt[25] = 0x80
|
||||
pkt[39] = 2
|
||||
|
||||
binary.BigEndian.PutUint16(pkt[40:42], sport)
|
||||
binary.BigEndian.PutUint16(pkt[42:44], dport)
|
||||
binary.BigEndian.PutUint16(pkt[44:46], uint16(udpHdrLen+len(payload)))
|
||||
binary.BigEndian.PutUint16(pkt[46:48], 0)
|
||||
|
||||
copy(pkt[48:], payload)
|
||||
return pkt
|
||||
}
|
||||
|
||||
// newTestUDPCoalescer builds a coalescer over w and fails the test if w can't
|
||||
// do USO. See newTestTCPCoalescer.
|
||||
func newTestUDPCoalescer(tb testing.TB, w io.Writer) *UDPCoalescer {
|
||||
tb.Helper()
|
||||
c := NewUDPCoalescer(w)
|
||||
if c == nil {
|
||||
tb.Fatal("NewUDPCoalescer: writer does not support USO")
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// TestNewUDPCoalescerRefusesWhenGSOUnavailable mirrors the TCP precondition:
|
||||
// no USO, no coalescer.
|
||||
func TestNewUDPCoalescerRefusesWhenGSOUnavailable(t *testing.T) {
|
||||
if c := NewUDPCoalescer(&fakeTunWriter{gsoEnabled: false}); c != nil {
|
||||
t.Fatalf("want nil for a non-USO writer, got %v", c)
|
||||
}
|
||||
if c := NewUDPCoalescer(&plainOnlyWriter{}); c != nil {
|
||||
t.Fatalf("want nil for a plain writer, got %v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUDPCoalescerNonUDPPassthrough(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
// ICMP packet
|
||||
pkt := make([]byte, 28)
|
||||
pkt[0] = 0x45
|
||||
binary.BigEndian.PutUint16(pkt[2:4], 28)
|
||||
pkt[9] = 1
|
||||
copy(pkt[12:16], []byte{10, 0, 0, 1})
|
||||
copy(pkt[16:20], []byte{10, 0, 0, 2})
|
||||
if err := c.Commit(pkt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("ICMP must pass through unchanged: writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUDPCoalescerSeedThenFlushAlone(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pkt := buildUDPv4(1000, 53, make([]byte, 800))
|
||||
if err := c.Commit(pkt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A slot that never grew past one datagram flushes as a plain Write of
|
||||
// the original packet bytes: the original (already valid) checksum
|
||||
// ships via the DATA_VALID path, so the kernel does no csum work.
|
||||
// WriteGSO is reserved for slots that actually coalesced (>=2 segs).
|
||||
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("single-seg flush: writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
if !bytes.Equal(w.writes[0], pkt) {
|
||||
t.Errorf("plain write not byte-identical to committed packet: got %d bytes want %d", len(w.writes[0]), len(pkt))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUDPCoalescerCoalescesEqualSized(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pay := make([]byte, 1200)
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.gsoWrites) != 1 {
|
||||
t.Fatalf("want 1 gso write, got %d (plain=%d)", len(w.gsoWrites), len(w.writes))
|
||||
}
|
||||
g := w.gsoWrites[0]
|
||||
if g.gsoSize != 1200 {
|
||||
t.Errorf("gsoSize=%d want 1200", g.gsoSize)
|
||||
}
|
||||
if len(g.pays) != 3 {
|
||||
t.Errorf("pay count=%d want 3", len(g.pays))
|
||||
}
|
||||
if g.csumStart != 20 {
|
||||
t.Errorf("csumStart=%d want 20", g.csumStart)
|
||||
}
|
||||
// IP totalLen and UDP length must be the TOTAL across all segments —
|
||||
// the kernel's ip_rcv_core trims skbs to iph->tot_len, so a per-segment
|
||||
// value would silently drop everything but the first segment. Total =
|
||||
// IP(20) + UDP(8) + 3*1200 = 3628.
|
||||
gotTotalLen := binary.BigEndian.Uint16(g.hdr[2:4])
|
||||
if gotTotalLen != 3628 {
|
||||
t.Errorf("ipv4 total_len=%d want 3628 (must be total across segments)", gotTotalLen)
|
||||
}
|
||||
gotUDPLen := binary.BigEndian.Uint16(g.hdr[20+4 : 20+6])
|
||||
if gotUDPLen != 8+3*1200 {
|
||||
t.Errorf("udp len=%d want %d", gotUDPLen, 8+3*1200)
|
||||
}
|
||||
}
|
||||
|
||||
// Last segment may be shorter, sealing the chain.
|
||||
func TestUDPCoalescerShortLastSegmentSeals(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
full := make([]byte, 1200)
|
||||
tail := make([]byte, 600)
|
||||
if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Commit(buildUDPv4(1000, 53, tail)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A 4th packet, even same-sized, must NOT join — chain is sealed.
|
||||
if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The sealed 3-datagram chain is a real superpacket; the re-seed stays
|
||||
// single-segment and flushes as a plain write of the original packet.
|
||||
if len(w.gsoWrites) != 1 || len(w.writes) != 1 {
|
||||
t.Fatalf("want 1 gso (sealed) + 1 plain (new seed), got gso=%d plain=%d", len(w.gsoWrites), len(w.writes))
|
||||
}
|
||||
if len(w.gsoWrites[0].pays) != 3 {
|
||||
t.Errorf("super: want 3 pays, got %d", len(w.gsoWrites[0].pays))
|
||||
}
|
||||
if got, want := len(w.writes[0]), 20+8+1200; got != want {
|
||||
t.Errorf("re-seed plain write len=%d want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A larger-than-gsoSize packet cannot extend the slot — it reseeds.
|
||||
func TestUDPCoalescerLargerThanSeedReseeds(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
if err := c.Commit(buildUDPv4(1000, 53, make([]byte, 800))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Commit(buildUDPv4(1000, 53, make([]byte, 1200))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Both seeds stay single-segment → two plain writes in arrival order.
|
||||
if len(w.writes) != 2 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("want 2 separate plain writes, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
for i, want := range []int{20 + 8 + 800, 20 + 8 + 1200} {
|
||||
if len(w.writes[i]) != want {
|
||||
t.Errorf("write %d len=%d want %d", i, len(w.writes[i]), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Different 5-tuples must not coalesce.
|
||||
func TestUDPCoalescerDifferentFlowsKeepSeparate(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pay := make([]byte, 800)
|
||||
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Commit(buildUDPv4(2000, 53, pay)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Commit(buildUDPv4(2000, 53, pay)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Two flows × 2 datagrams each = 2 superpackets of 2 segments.
|
||||
if len(w.gsoWrites) != 2 {
|
||||
t.Fatalf("want 2 gso writes (one per flow), got %d", len(w.gsoWrites))
|
||||
}
|
||||
for i, g := range w.gsoWrites {
|
||||
if len(g.pays) != 2 {
|
||||
t.Errorf("super %d: want 2 pays, got %d", i, len(g.pays))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Caps at udpCoalesceMaxSegs.
|
||||
func TestUDPCoalescerCapsAtMaxSegs(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pay := make([]byte, 100)
|
||||
for i := 0; i < udpCoalesceMaxSegs+5; i++ {
|
||||
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// First superpacket holds udpCoalesceMaxSegs segments; the spillover
|
||||
// reseeds a new one.
|
||||
if len(w.gsoWrites) != 2 {
|
||||
t.Fatalf("want 2 gso writes (cap then reseed), got %d", len(w.gsoWrites))
|
||||
}
|
||||
if len(w.gsoWrites[0].pays) != udpCoalesceMaxSegs {
|
||||
t.Errorf("first super: pays=%d want %d", len(w.gsoWrites[0].pays), udpCoalesceMaxSegs)
|
||||
}
|
||||
if len(w.gsoWrites[1].pays) != 5 {
|
||||
t.Errorf("second super: pays=%d want 5", len(w.gsoWrites[1].pays))
|
||||
}
|
||||
}
|
||||
|
||||
// Differing IP ECN codepoints must not coalesce: udpHeadersMatch compares
|
||||
// the full ToS byte (matching kernel GRO). A CE-marked datagram mid-run
|
||||
// seals the Not-ECT chain and reseeds; the trailing Not-ECT datagram
|
||||
// reseeds again. All three stay single-segment, so each ships as a plain
|
||||
// write of its original bytes, keeping its own codepoint.
|
||||
func TestUDPCoalescerDifferingECNReseeds(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pay := make([]byte, 800)
|
||||
pkt0 := buildUDPv4(1000, 53, pay) // ECN=00 (Not-ECT)
|
||||
pkt1 := buildUDPv4(1000, 53, pay)
|
||||
pkt1[1] = 0x03 // CE
|
||||
pkt2 := buildUDPv4(1000, 53, pay) // ECN=00 again
|
||||
for _, p := range [][]byte{pkt0, pkt1, pkt2} {
|
||||
if err := c.Commit(p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.writes) != 3 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("want 3 separate plain writes (differing ECN), got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
wantECN := []byte{0x00, 0x03, 0x00}
|
||||
for i, p := range w.writes {
|
||||
if got := p[1] & 0x03; got != wantECN[i] {
|
||||
t.Errorf("write %d ECN=%#x want %#x", i, got, wantECN[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// IPv6 path: same flow, equal-sized → coalesced.
|
||||
func TestUDPCoalescerIPv6Coalesces(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pay := make([]byte, 1200)
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := c.Commit(buildUDPv6(1000, 53, pay)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.gsoWrites) != 1 {
|
||||
t.Fatalf("want 1 gso write, got %d", len(w.gsoWrites))
|
||||
}
|
||||
g := w.gsoWrites[0]
|
||||
if !g.isV6 {
|
||||
t.Errorf("expected v6 write")
|
||||
}
|
||||
if g.csumStart != 40 {
|
||||
t.Errorf("csumStart=%d want 40", g.csumStart)
|
||||
}
|
||||
// IPv6 payload_len and UDP length must be TOTAL — kernel's
|
||||
// ip6_rcv_core trims to payload_len + ipv6 hdr size. Total UDP = 8 +
|
||||
// 3*1200 = 3608.
|
||||
gotPlen := binary.BigEndian.Uint16(g.hdr[4:6])
|
||||
if gotPlen != 8+3*1200 {
|
||||
t.Errorf("ipv6 payload_len=%d want %d (must be total)", gotPlen, 8+3*1200)
|
||||
}
|
||||
gotUDPLen := binary.BigEndian.Uint16(g.hdr[40+4 : 40+6])
|
||||
if gotUDPLen != 8+3*1200 {
|
||||
t.Errorf("udp len=%d want %d", gotUDPLen, 8+3*1200)
|
||||
}
|
||||
}
|
||||
|
||||
// DSCP differences must reseed: udpHeadersMatch compares the full ToS byte.
|
||||
func TestUDPCoalescerDSCPMismatchReseeds(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pay := make([]byte, 800)
|
||||
pkt0 := buildUDPv4(1000, 53, pay)
|
||||
pkt1 := buildUDPv4(1000, 53, pay)
|
||||
pkt1[1] = 0xb8 // EF DSCP, ECN=0
|
||||
if err := c.Commit(pkt0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Commit(pkt1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Both seeds stay single-segment → two plain writes, no gso.
|
||||
if len(w.writes) != 2 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("want 2 separate plain writes (different DSCP), got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
}
|
||||
|
||||
// Fragmented IPv4 must not be coalesced.
|
||||
func TestUDPCoalescerFragmentedIPv4PassesThrough(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pkt := buildUDPv4(1000, 53, make([]byte, 200))
|
||||
binary.BigEndian.PutUint16(pkt[6:8], 0x2000) // MF=1
|
||||
if err := c.Commit(pkt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("frag must pass through plain, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
}
|
||||
|
||||
// A zero-length UDP datagram (UDP length == 8, no payload) is legal and
|
||||
// must be delivered as a plain single datagram — never coalesced. Seeding
|
||||
// it into a GSO slot stores an empty payload iovec that panics WriteGSO
|
||||
// (index-out-of-range on &pay[0]); this is a remote DoS if we ever let it
|
||||
// reach the GSO path. Regression: must not panic and must be written.
|
||||
func TestUDPCoalescerZeroLengthPayloadPassesThrough(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pkt := buildUDPv4(1000, 53, nil) // UDP length 8, zero payload
|
||||
if err := c.Commit(pkt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("zero-length UDP must pass through plain, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
if len(w.writes[0]) != len(pkt) {
|
||||
t.Errorf("delivered %d bytes, want the whole %d-byte datagram", len(w.writes[0]), len(pkt))
|
||||
}
|
||||
}
|
||||
|
||||
// IPv6 zero-length UDP datagram: same verbatim contract as v4.
|
||||
func TestUDPCoalescerZeroLengthPayloadIPv6PassesThrough(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pkt := buildUDPv6(1000, 53, nil) // UDP length 8, zero payload
|
||||
if err := c.Commit(pkt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("zero-length IPv6 UDP must pass through plain, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
if len(w.writes[0]) != len(pkt) {
|
||||
t.Errorf("delivered %d bytes, want the whole %d-byte datagram", len(w.writes[0]), len(pkt))
|
||||
}
|
||||
}
|
||||
|
||||
// A zero-length datagram arriving mid-flow must seal the open chain so the
|
||||
// datagram after it seeds a fresh superpacket *after* the empty one on the
|
||||
// wire — per-flow arrival order (full, empty, full) must be preserved.
|
||||
func TestUDPCoalescerZeroLengthMidFlowSealsAndPreservesOrder(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
full := make([]byte, 800)
|
||||
if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Commit(buildUDPv4(1000, 53, nil)); err != nil { // zero-length
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The empty datagram sealed the first slot, so the trailing full packet
|
||||
// can't join it. All three emit as plain writes (the two full datagrams
|
||||
// stayed single-segment; the empty one is verbatim) in per-flow
|
||||
// arrival order: full, empty, full.
|
||||
if len(w.writes) != 3 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("want 3 plain writes, got gso=%d plain=%d", len(w.gsoWrites), len(w.writes))
|
||||
}
|
||||
for i, want := range []int{20 + 8 + 800, 20 + 8, 20 + 8 + 800} {
|
||||
if len(w.writes[i]) != want {
|
||||
t.Errorf("write %d len=%d want %d (order full, empty, full)", i, len(w.writes[i]), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// IPv4 with options is not admissible (we require IHL=5).
|
||||
func TestUDPCoalescerIPv4WithOptionsPassesThrough(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pkt := buildUDPv4(1000, 53, make([]byte, 200))
|
||||
pkt[0] = 0x46 // IHL = 6 (24-byte IPv4 header — has options)
|
||||
if err := c.Commit(pkt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("ipv4-with-options must pass through plain, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
}
|
||||
|
||||
// TestUDPCoalescerNonAtomicSequentialIDsCoalesce mirrors the TCP rule: DF
|
||||
// clear is fine as long as the IDs already run seed+1 per datagram, so
|
||||
// kernel USO's re-stamp reproduces them.
|
||||
func TestUDPCoalescerNonAtomicSequentialIDsCoalesce(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pay := make([]byte, 1200)
|
||||
|
||||
for i := range 2 {
|
||||
pkt := buildUDPv4(40000, 443, pay)
|
||||
setIPv4ID(pkt, uint16(40+i), false)
|
||||
if err := c.Commit(pkt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.gsoWrites) != 1 || len(w.gsoWrites[0].pays) != 2 {
|
||||
t.Fatalf("sequential-ID DF=0 datagrams must coalesce: gso=%d", len(w.gsoWrites))
|
||||
}
|
||||
}
|
||||
|
||||
// TestUDPCoalescerNonAtomicIDGapReseeds: an ID jump on a DF=0 flow breaks
|
||||
// the chain; each datagram stays a single-segment slot and flushes as a
|
||||
// plain write that keeps its own (meaningful) ID.
|
||||
func TestUDPCoalescerNonAtomicIDGapReseeds(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
pay := make([]byte, 1200)
|
||||
|
||||
p1 := buildUDPv4(40000, 443, pay)
|
||||
setIPv4ID(p1, 40, false)
|
||||
p2 := buildUDPv4(40000, 443, pay)
|
||||
setIPv4ID(p2, 50, false)
|
||||
|
||||
if err := c.Commit(p1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Commit(p2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.writes) != 2 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("ID gap on DF=0 must reseed: writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
for i, want := range []uint16{40, 50} {
|
||||
if id := binary.BigEndian.Uint16(w.writes[i][4:6]); id != want {
|
||||
t.Errorf("write %d: ID=%d want %d (must be preserved)", i, id, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user