unslop, improve the tio interface

This commit is contained in:
JackDoan
2026-07-28 16:20:14 -05:00
parent df8955177e
commit 5c0f6e2b5f
13 changed files with 255 additions and 240 deletions
+1 -10
View File
@@ -301,16 +301,7 @@ func (f *Interface) activate() error {
metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines)) metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
for i := range f.queues { for i := range f.queues {
caps := tio.QueueCapabilities(f.queues[i]) f.batchers[i] = batch.NewMultiCoalescer(f.queues[i], f.l)
if caps.TSO || caps.USO {
// Multi-lane: TCP gets coalesced when TSO is on, UDP when USO
// is on, everything else (and either lane disabled) falls
// through to passthrough so non-IP / non-TCP-UDP traffic still
// reaches the TUN.
f.batchers[i] = batch.NewMultiCoalescer(f.queues[i], f.l, caps.TSO, caps.USO)
} else {
f.batchers[i] = batch.NewPassthrough(f.queues[i])
}
} }
// On error the caller owns the cleanup, Control.Start cancels the service context // On error the caller owns the cleanup, Control.Start cancels the service context
+19 -25
View File
@@ -8,21 +8,20 @@ import (
// flowKey identifies a transport flow by {src, dst, sport, dport, family}. // flowKey identifies a transport flow by {src, dst, sport, dport, family}.
// Comparable, so map lookups and linear scans over the slot list stay tight. // Comparable, so map lookups and linear scans over the slot list stay tight.
// Shared by the TCP and UDP coalescers; each coalescer keeps its own // Shared by the TCP and UDP coalescers; each coalescer keeps its own
// openSlots map, so a TCP and UDP flow on the same 5-tuple-without-proto // openSlots map, so a TCP and UDP flow on the same 5-tuple-without-proto never alias.
// never alias.
type flowKey struct { type flowKey struct {
src, dst [16]byte src, dst [16]byte
sport, dport uint16 sport, dport uint16
isV6 bool isV6 bool
} }
// initialSlots is the starting capacity of the slot pool. One flow per // initialSlots is the starting capacity of the slot pool.
// packet is the worst case so this matches a typical carrier-side // One flow per packet is the worst case,
// recvmmsg batch on the encrypted UDP socket. // so this matches a typical carrier-side recvmmsg batch on the UDP socket.
const initialSlots = 64 const initialSlots = 64
// parsedIP is the IP-level result of parseIPPrologue. The caller layers // parsedIP is the IP-level result of parseIPPrologue.
// L4-specific parsing (TCP / UDP) on top. // The caller layers L4-specific parsing (TCP / UDP) on top.
type parsedIP struct { type parsedIP struct {
fk flowKey fk flowKey
ipHdrLen int ipHdrLen int
@@ -92,15 +91,13 @@ func parseIPPrologue(pkt []byte, wantProto byte) (parsedIP, bool) {
} }
// ipHeadersMatch compares the IP portion of two packet header prefixes for // ipHeadersMatch compares the IP portion of two packet header prefixes for
// byte-for-byte equality on every field that must be identical across // byte-for-byte equality on every field that must be identical across coalesced segments.
// coalesced segments. Size/IPID/IPCsum are masked out. The full DSCP/ECN // Size/IPID/IPCsum are masked out.
// byte (IPv4 ToS / IPv6 traffic class) is compared, matching Linux kernel // The full DSCP/ECN byte (IPv4 ToS / IPv6 traffic class) is compared, matching Linux kernel GRO:
// GRO: segments with differing ECN codepoints must not coalesce, otherwise // segments with differing ECN codepoints must not coalesce,
// ORing e.g. ECT(0) with ECT(1) would fabricate a false CE (congestion) // otherwise ORing e.g. ECT(0) with ECT(1) would fabricate a false CE (congestion) mark or mark a Not-ECT flow as ECN-capable.
// mark or mark a Not-ECT flow as ECN-capable.
// //
// The transport (L4) portion of the header is checked separately by the // The transport (L4) portion of the header is checked separately by the per-protocol matcher.
// per-protocol matcher.
func ipHeadersMatch(a, b []byte, isV6 bool) bool { func ipHeadersMatch(a, b []byte, isV6 bool) bool {
if isV6 { if isV6 {
// IPv6: byte 0 = version/TC[7:4], byte 1 = TC[3:0]/flow[19:16], // IPv6: byte 0 = version/TC[7:4], byte 1 = TC[3:0]/flow[19:16],
@@ -145,17 +142,14 @@ type Arena struct {
buf []byte buf []byte
} }
// NewArena returns an Arena with a pre-allocated backing of the given // NewArena returns an Arena with a pre-allocated backing of the given capacity.
// capacity. Pass 0 if you don't intend to call Reserve (e.g. a test that
// only feeds the coalescer pre-made []byte packets via Commit).
func NewArena(capacity int) *Arena { func NewArena(capacity int) *Arena {
return &Arena{buf: make([]byte, 0, capacity)} return &Arena{buf: make([]byte, 0, capacity)}
} }
// Reserve hands out a non-overlapping sz-byte slice from the arena. If the // Reserve hands out a non-overlapping sz-byte slice from the arena.
// request doesn't fit the current backing, a fresh, larger backing is // If the request doesn't fit the current backing, a fresh, larger backing is allocated.
// allocated; already-borrowed slices reference the old backing and remain // Already-borrowed slices reference the old backing and remain valid until Reset.
// valid until Reset.
func (a *Arena) Reserve(sz int) []byte { func (a *Arena) Reserve(sz int) []byte {
if len(a.buf)+sz > cap(a.buf) { if len(a.buf)+sz > cap(a.buf) {
newCap := max(cap(a.buf)*2, sz) newCap := max(cap(a.buf)*2, sz)
@@ -166,9 +160,9 @@ func (a *Arena) Reserve(sz int) []byte {
return a.buf[start : start+sz : start+sz] return a.buf[start : start+sz : start+sz]
} }
// Reset releases every slice handed out since the last Reset. Callers must // Reset releases every slice handed out since the last Reset.
// not use any previously-borrowed slice after this returns. The underlying // Callers must not use any previously-borrowed slice after this returns.
// backing array is retained so subsequent Reserves don't re-allocate. // The underlying backing array is retained so subsequent Reserves don't re-allocate.
func (a *Arena) Reset() { func (a *Arena) Reset() {
a.buf = a.buf[:0] a.buf = a.buf[:0]
} }
+10 -23
View File
@@ -10,46 +10,33 @@ import (
// on the IP/L4 protocol of the packet. // on the IP/L4 protocol of the packet.
// //
// Lanes are processed independently: the TCP coalescer only sees TCP, the // Lanes are processed independently: the TCP coalescer only sees TCP, the
// UDP coalescer only sees UDP, and the passthrough lane handles everything // UDP coalescer only sees UDP, and the passthrough lane handles everything else.
// else. Per-flow arrival order is preserved because a single 5-tuple only // Per-flow delivery order is preserved because a single 5-tuple only
// ever lands in one lane and each lane preserves its own slot order. // ever lands in one lane and each lane preserves its own slot order.
// //
// Cross-lane order is NOT preserved across the TCP/UDP/passthrough split. // Cross-lane order is intentionally NOT preserved across the TCP/UDP/passthrough split.
// This is acceptable because the carrier-side recvmmsg path already
// stable-sorts by (peer, message counter) before delivering plaintext
// here, so replay-window invariants are unaffected, and apps observe
// correct per-flow ordering; which is all the IP layer guarantees anyway.
// Do not "fix" this by interleaving lane outputs at flush time; that
// negates the entire point of coalescing (each lane needs to see runs of
// adjacent same-flow packets to coalesce them).
type MultiCoalescer struct { type MultiCoalescer struct {
tcp *TCPCoalescer tcp *TCPCoalescer
udp *UDPCoalescer udp *UDPCoalescer
pt *Passthrough pt *Passthrough
} }
// NewMultiCoalescer builds a multi-lane batcher. tcpEnabled lets the caller // NewMultiCoalescer builds a multi-lane batcher over w, based on available protocol support.
// opt out of TCP coalescing (e.g. when the queue can't do TSO); udpEnabled func NewMultiCoalescer(w io.Writer, l *slog.Logger) RxBatcher {
// likewise gates UDP coalescing (only enable when USO was negotiated).
// Either lane disabled redirects its traffic into the passthrough lane.
func NewMultiCoalescer(w io.Writer, l *slog.Logger, tcpEnabled, udpEnabled bool) *MultiCoalescer {
m := &MultiCoalescer{ m := &MultiCoalescer{
pt: NewPassthrough(w), pt: NewPassthrough(w),
} }
if tcpEnabled { m.tcp = NewTCPCoalescer(w, l)
m.tcp = NewTCPCoalescer(w, l) m.udp = NewUDPCoalescer(w)
} if m.tcp == nil && m.udp == nil {
if udpEnabled { return m.pt //no offloads? Use passthrough directly.
m.udp = NewUDPCoalescer(w)
} }
return m return m
} }
// Commit dispatches pkt to the appropriate lane based on IP version + L4 proto. // Commit dispatches pkt to the appropriate lane based on IP version + L4 proto.
//
// On the success path the IP/TCP-or-UDP parse happens here once and the // On the success path the IP/TCP-or-UDP parse happens here once and the
// parsed struct is handed to the lane via commitParsed so the lane doesn't // parsed struct is handed to the lane via commitParsed so the lane doesn't re-walk the header.
// re-walk the header.
func (m *MultiCoalescer) Commit(pkt []byte) error { func (m *MultiCoalescer) Commit(pkt []byte) error {
if len(pkt) < 20 { if len(pkt) < 20 {
return m.pt.Commit(pkt) return m.pt.Commit(pkt)
+73 -11
View File
@@ -1,17 +1,33 @@
package batch package batch
import ( import (
"bytes"
"io"
"testing" "testing"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
) )
// newTestMultiCoalescer builds a batcher over w and asserts it really is
// multi-lane. NewMultiCoalescer collapses to a bare Passthrough when w can
// offload neither protocol, and a test that meant to exercise a lane would
// otherwise pass vacuously.
func newTestMultiCoalescer(tb testing.TB, w io.Writer) *MultiCoalescer {
tb.Helper()
b := NewMultiCoalescer(w, test.NewLogger())
m, ok := b.(*MultiCoalescer)
if !ok {
tb.Fatalf("want a *MultiCoalescer, got %T", b)
}
return m
}
// TestMultiCoalescerRoutesByProto confirms TCP/UDP/other land in the right // TestMultiCoalescerRoutesByProto confirms TCP/UDP/other land in the right
// lane: TCP and UDP get coalesced when their lanes are enabled, anything // lane: TCP and UDP get coalesced when their lanes are enabled, anything
// else (ICMP here) falls through to plain Write. // else (ICMP here) falls through to plain Write.
func TestMultiCoalescerRoutesByProto(t *testing.T) { func TestMultiCoalescerRoutesByProto(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
m := NewMultiCoalescer(w, test.NewLogger(), true, true) m := newTestMultiCoalescer(t, w)
tcpPay := make([]byte, 1200) tcpPay := make([]byte, 1200)
udpPay := make([]byte, 1200) udpPay := make([]byte, 1200)
@@ -48,12 +64,15 @@ func TestMultiCoalescerRoutesByProto(t *testing.T) {
} }
} }
// TestMultiCoalescerDisabledUDPFallsThrough verifies that when the UDP lane // TestMultiCoalescerNoUSOFallsThrough verifies that on a queue without USO
// is disabled (e.g. kernel doesn't support USO), UDP packets still reach // (older kernel: TSO but no GSO_UDP_L4) the UDP lane never comes up and UDP
// the kernel via the passthrough lane rather than being lost. // packets still reach the kernel via passthrough rather than being lost.
func TestMultiCoalescerDisabledUDPFallsThrough(t *testing.T) { func TestMultiCoalescerNoUSOFallsThrough(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true, noUSO: true}
m := NewMultiCoalescer(w, test.NewLogger(), true, false) // TSO on, USO off m := newTestMultiCoalescer(t, w)
if m.udp != nil {
t.Fatal("UDP lane must not come up without USO")
}
if err := m.Commit(buildUDPv4(1000, 53, make([]byte, 800))); err != nil { if err := m.Commit(buildUDPv4(1000, 53, make([]byte, 800))); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -72,10 +91,53 @@ func TestMultiCoalescerDisabledUDPFallsThrough(t *testing.T) {
} }
} }
// TestMultiCoalescerDisabledTCPFallsThrough mirrors the TSO=off case. // TestMultiCoalescerNoOffloadsIsPassthrough covers a queue that can't offload
func TestMultiCoalescerDisabledTCPFallsThrough(t *testing.T) { // anything. Both lane constructors refuse, so there's nothing left to
w := &fakeTunWriter{gsoEnabled: true} // dispatch between and NewMultiCoalescer hands back the passthrough lane
m := NewMultiCoalescer(w, test.NewLogger(), false, true) // TSO off, USO on // itself — no wrapper, no per-packet protocol demux, and every packet reaches
// the kernel in arrival order. This is the case Interface.activate used to
// special-case with a bare Passthrough.
func TestMultiCoalescerNoOffloadsIsPassthrough(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: false}
m := NewMultiCoalescer(w, test.NewLogger())
if _, ok := m.(*Passthrough); !ok {
t.Fatalf("want a bare *Passthrough when neither offload is available, got %T", m)
}
pkts := [][]byte{
buildTCPv4(1000, tcpAck, make([]byte, 1200)),
buildUDPv4(1000, 53, make([]byte, 800)),
buildTCPv4(2200, tcpAck, make([]byte, 1200)),
}
for _, p := range pkts {
if err := m.Commit(p); err != nil {
t.Fatal(err)
}
}
if err := m.Flush(); err != nil {
t.Fatal(err)
}
if len(w.gsoWrites) != 0 {
t.Errorf("no GSO writes possible, got %d", len(w.gsoWrites))
}
if len(w.writes) != len(pkts) {
t.Fatalf("want %d plain writes, got %d", len(pkts), len(w.writes))
}
// One lane for everything means arrival order survives end to end.
for i, want := range pkts {
if !bytes.Equal(w.writes[i], want) {
t.Errorf("write %d out of order or corrupt", i)
}
}
}
// TestMultiCoalescerNoTSOFallsThrough mirrors the no-TSO case.
func TestMultiCoalescerNoTSOFallsThrough(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true, noTSO: true}
m := newTestMultiCoalescer(t, w)
if m.tcp != nil {
t.Fatal("TCP lane must not come up without TSO")
}
pay := make([]byte, 1200) pay := make([]byte, 1200)
if err := m.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil { if err := m.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
+30 -40
View File
@@ -27,13 +27,13 @@ const tcpCoalesceMaxSegs = 64
// into. IPv6 (40) + TCP with full options (60) = 100 bytes. // into. IPv6 (40) + TCP with full options (60) = 100 bytes.
const tcpCoalesceHdrCap = 100 const tcpCoalesceHdrCap = 100
// coalesceSlot is one entry in the coalescer's ordered event queue. When // coalesceSlot is one entry in the coalescer's ordered event queue.
// passthrough is true the slot holds a single borrowed packet that must be // When passthrough is true the slot holds a single borrowed packet that must be
// emitted verbatim (non-TCP, non-admissible TCP, or oversize seed). When // emitted verbatim (non-TCP, non-admissible TCP, or oversize seed).
// passthrough is false the slot is an in-progress coalesced superpacket: // When passthrough is false the slot is an in-progress coalesced superpacket.
// hdrBuf is a mutable copy of the seed's IP+TCP header (we patch total // hdrBuf is a mutable copy of the seed's IP+TCP header
// length and pseudo-header partial at flush), and payIovs are *borrowed* // (we patch total length and pseudo-header partial at flush)
// slices from the caller's plaintext buffers — no payload is ever copied. // payIovs are *borrowed* slices from the caller's plaintext buffers.
// The caller (listenOut) must keep those buffers alive until Flush. // The caller (listenOut) must keep those buffers alive until Flush.
type coalesceSlot struct { type coalesceSlot struct {
passthrough bool passthrough bool
@@ -50,20 +50,18 @@ type coalesceSlot struct {
nextSeq uint32 nextSeq uint32
// sealed marks the chain permanently closed: the last-accepted segment had PSH or was sub-gsoSize, // sealed marks the chain permanently closed: the last-accepted segment had PSH or was sub-gsoSize,
// so no append or flush-time merge may follow. // so no append or flush-time merge may follow.
// Distinct from mere eviction out of openSlots (e.g. on seq mismatch), // Distinct from eviction out of openSlots (e.g. on seq mismatch),
// which leaves sealed=false so reorderForFlush can still merge the slot. // which leaves sealed=false so reorderForFlush can still merge the slot.
sealed bool sealed bool
payIovs [][]byte payIovs [][]byte
} }
// TCPCoalescer accumulates adjacent in-flow TCP data segments across // TCPCoalescer accumulates adjacent in-flow TCP data segments across multiple concurrent flows
// multiple concurrent flows and emits each flow's run as a single TSO // and emits each flow's run as a single TSO superpacket via tio.GSOWriter.
// superpacket via tio.GSOWriter. All output coalesced or not — is // All output, coalesced or not, is deferred until Flush so arrival order is preserved on the wire.
// deferred until Flush so arrival order is preserved on the wire. Owns // Owns no locks; one coalescer per TUN write queue.
// no locks; one coalescer per TUN write queue.
type TCPCoalescer struct { type TCPCoalescer struct {
plainW io.Writer w tio.GSOWriter
gsoW tio.GSOWriter // nil when the queue doesn't support TSO
// slots is the ordered event queue. Flush walks it once and emits each // slots is the ordered event queue. Flush walks it once and emits each
// entry as either a WriteGSO (coalesced) or a plainW.Write (passthrough). // entry as either a WriteGSO (coalesced) or a plainW.Write (passthrough).
@@ -84,18 +82,19 @@ type TCPCoalescer struct {
l *slog.Logger l *slog.Logger
} }
// NewTCPCoalescer wraps w, returning nil if w can't accept GSO_TCP writes.
func NewTCPCoalescer(w io.Writer, l *slog.Logger) *TCPCoalescer { func NewTCPCoalescer(w io.Writer, l *slog.Logger) *TCPCoalescer {
c := &TCPCoalescer{ gw, ok := tio.SupportsGSO(w, tio.GSOProtoTCP)
plainW: w, if !ok {
return nil
}
return &TCPCoalescer{
w: gw,
slots: make([]*coalesceSlot, 0, initialSlots), slots: make([]*coalesceSlot, 0, initialSlots),
openSlots: make(map[flowKey]*coalesceSlot, initialSlots), openSlots: make(map[flowKey]*coalesceSlot, initialSlots),
pool: make([]*coalesceSlot, 0, initialSlots), pool: make([]*coalesceSlot, 0, initialSlots),
l: l, l: l,
} }
if gw, ok := tio.SupportsGSO(w, tio.GSOProtoTCP); ok {
c.gsoW = gw
}
return c
} }
// parsedTCP holds the fields extracted from a single parse so later steps // parsedTCP holds the fields extracted from a single parse so later steps
@@ -111,8 +110,7 @@ type parsedTCP struct {
} }
// parseTCPBase extracts the flow key and IP/TCP offsets for any TCP packet, // parseTCPBase extracts the flow key and IP/TCP offsets for any TCP packet,
// regardless of whether it's admissible for coalescing. Returns ok=false // regardless of whether it's admissible for coalescing. Returns ok=false for non-TCP or malformed input.
// for non-TCP or malformed input.
// Accepts IPv4 (no options or fragmentation) and IPv6 (no extension headers). // Accepts IPv4 (no options or fragmentation) and IPv6 (no extension headers).
func parseTCPBase(pkt []byte) (parsedTCP, bool) { func parseTCPBase(pkt []byte) (parsedTCP, bool) {
var p parsedTCP var p parsedTCP
@@ -169,10 +167,6 @@ func (p parsedTCP) coalesceable() bool {
} }
func (c *TCPCoalescer) Commit(pkt []byte) error { func (c *TCPCoalescer) Commit(pkt []byte) error {
if c.gsoW == nil {
c.addPassthrough(pkt)
return nil
}
info, ok := parseTCPBase(pkt) info, ok := parseTCPBase(pkt)
if !ok { if !ok {
c.addPassthrough(pkt) c.addPassthrough(pkt)
@@ -186,10 +180,6 @@ func (c *TCPCoalescer) Commit(pkt []byte) error {
// Used by MultiCoalescer.Commit to avoid re-walking the IP/TCP header // Used by MultiCoalescer.Commit to avoid re-walking the IP/TCP header
// after the dispatcher has already done so. // after the dispatcher has already done so.
func (c *TCPCoalescer) commitParsed(pkt []byte, info parsedTCP) error { func (c *TCPCoalescer) commitParsed(pkt []byte, info parsedTCP) error {
if c.gsoW == nil {
c.addPassthrough(pkt)
return nil
}
if !info.coalesceable() { if !info.coalesceable() {
// TCP but not admissible (SYN/FIN/RST/URG/CWR or zero-payload). // TCP but not admissible (SYN/FIN/RST/URG/CWR or zero-payload).
// Seal this flow's open slot so later in-flow packets don't extend // Seal this flow's open slot so later in-flow packets don't extend
@@ -240,7 +230,7 @@ func (c *TCPCoalescer) Flush() error {
for _, s := range c.slots { for _, s := range c.slots {
var err error var err error
if s.passthrough { if s.passthrough {
_, err = c.plainW.Write(s.rawPkt) _, err = c.w.Write(s.rawPkt)
} else { } else {
err = c.flushSlot(s) err = c.flushSlot(s)
} }
@@ -266,7 +256,7 @@ func (c *TCPCoalescer) addPassthrough(pkt []byte) {
func (c *TCPCoalescer) seed(pkt []byte, info parsedTCP) { func (c *TCPCoalescer) seed(pkt []byte, info parsedTCP) {
if info.hdrLen > tcpCoalesceHdrCap || info.hdrLen+info.payLen > tcpCoalesceBufSize { if info.hdrLen > tcpCoalesceHdrCap || info.hdrLen+info.payLen > tcpCoalesceBufSize {
// Pathological shape — can't fit our scratch, emit as-is. // Pathological shape. Can't fit our scratch, emit as-is.
c.addPassthrough(pkt) c.addPassthrough(pkt)
return return
} }
@@ -317,8 +307,8 @@ func (c *TCPCoalescer) canAppend(s *coalesceSlot, pkt []byte, info parsedTCP) bo
if s.hdrLen+s.totalPay+info.payLen > tcpCoalesceBufSize { if s.hdrLen+s.totalPay+info.payLen > tcpCoalesceBufSize {
return false return false
} }
// ECE state must be stable across a burst — receivers expect the // ECE state must be stable across a burst.
// flag set on every segment of a CE-echoing window or none. // Receivers expect the flag set on every segment of a CE-echoing window or none.
seedFlags := s.hdrBuf[s.ipHdrLen+13] seedFlags := s.hdrBuf[s.ipHdrLen+13]
if (seedFlags^info.flags)&tcpFlagEce != 0 { if (seedFlags^info.flags)&tcpFlagEce != 0 {
return false return false
@@ -389,7 +379,7 @@ func (c *TCPCoalescer) flushSlot(s *coalesceSlot) error {
tcsum := s.ipHdrLen + 16 tcsum := s.ipHdrLen + 16
binary.BigEndian.PutUint16(hdr[tcsum:tcsum+2], foldOnceNoInvert(psum)) binary.BigEndian.PutUint16(hdr[tcsum:tcsum+2], foldOnceNoInvert(psum))
return c.gsoW.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoTCP) return c.w.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoTCP)
} }
// headersMatch compares two IP+TCP header prefixes for byte-for-byte // headersMatch compares two IP+TCP header prefixes for byte-for-byte
@@ -421,9 +411,9 @@ func headersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
} }
// reorderForFlush neutralizes wire-side reorder that the rxOrder buffer // reorderForFlush neutralizes wire-side reorder that the rxOrder buffer
// couldn't catch (anything crossing a recvmmsg batch boundary). Without // couldn't catch (anything crossing a recvmmsg batch boundary).
// this pass a small wire reorder counter 250 arriving in batch K when // Without this pass a small wire reorder, counter 250 arriving in batch K when
// 200..249 are coming in batch K+1 would seed an out-of-seq slot first // 200..249 are coming in batch K+1, would seed an out-of-seq slot first
// and emit it ahead of the lower-seq slot, manifesting at the inner TCP // and emit it ahead of the lower-seq slot, manifesting at the inner TCP
// receiver as a much larger reorder than the wire actually had. // receiver as a much larger reorder than the wire actually had.
// //
@@ -434,7 +424,7 @@ func headersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
// 2. Sweep once and merge adjacent same-flow slots whose ranges are now // 2. Sweep once and merge adjacent same-flow slots whose ranges are now
// contiguous AND whose tail is gsoSize-aligned. The tail constraint // contiguous AND whose tail is gsoSize-aligned. The tail constraint
// matters because the kernel TSO splitter chops at gsoSize from the // matters because the kernel TSO splitter chops at gsoSize from the
// start of the merged payload — a short segment in the middle would // start of the merged payload. A short segment in the middle would
// desynchronize every later segment. // desynchronize every later segment.
// //
// Passthrough slots act as barriers: the merge check skips them on either // Passthrough slots act as barriers: the merge check skips them on either
+2 -2
View File
@@ -71,7 +71,7 @@ func buildICMPv4() []byte {
// between batches, and reports per-packet cost. // between batches, and reports per-packet cost.
func runCommitBench(b *testing.B, pkts [][]byte, batchSize int) { func runCommitBench(b *testing.B, pkts [][]byte, batchSize int) {
b.Helper() b.Helper()
c := NewTCPCoalescer(nopTunWriter{}, test.NewLogger()) c := newTestTCPCoalescer(b, nopTunWriter{})
b.ReportAllocs() b.ReportAllocs()
b.SetBytes(int64(len(pkts[0]))) b.SetBytes(int64(len(pkts[0])))
b.ResetTimer() b.ResetTimer()
@@ -140,7 +140,7 @@ func BenchmarkCommitNonCoalesceableTCP(b *testing.B) {
// is the bench that shows the savings of skipping the lane's re-parse. // is the bench that shows the savings of skipping the lane's re-parse.
func runMultiCommitBench(b *testing.B, pkts [][]byte, batchSize int) { func runMultiCommitBench(b *testing.B, pkts [][]byte, batchSize int) {
b.Helper() b.Helper()
m := NewMultiCoalescer(nopTunWriter{}, test.NewLogger(), true, true) m := NewMultiCoalescer(nopTunWriter{}, test.NewLogger())
b.ReportAllocs() b.ReportAllocs()
b.SetBytes(int64(len(pkts[0]))) b.SetBytes(int64(len(pkts[0])))
b.ResetTimer() b.ResetTimer()
+64 -43
View File
@@ -2,6 +2,7 @@ package batch
import ( import (
"encoding/binary" "encoding/binary"
"io"
"testing" "testing"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
@@ -11,8 +12,13 @@ import (
// fakeTunWriter records plain Writes and WriteGSO calls without touching a // fakeTunWriter records plain Writes and WriteGSO calls without touching a
// real TUN fd. WriteGSO records the IP header, transport header, and // real TUN fd. WriteGSO records the IP header, transport header, and
// borrowed payload fragments separately so tests can inspect each. // borrowed payload fragments separately so tests can inspect each.
// noTSO / noUSO withhold one offload from an otherwise GSO-capable writer, so
// tests can build the half-capable queues real kernels hand us (USO needs a
// newer kernel than TSO).
type fakeTunWriter struct { type fakeTunWriter struct {
gsoEnabled bool gsoEnabled bool
noTSO bool
noUSO bool
writes [][]byte writes [][]byte
gsoWrites []fakeGSOWrite gsoWrites []fakeGSOWrite
} }
@@ -79,7 +85,7 @@ func (w *fakeTunWriter) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte,
} }
func (w *fakeTunWriter) Capabilities() tio.Capabilities { func (w *fakeTunWriter) Capabilities() tio.Capabilities {
return tio.Capabilities{TSO: w.gsoEnabled, USO: w.gsoEnabled} return tio.Capabilities{TSO: w.gsoEnabled && !w.noTSO, USO: w.gsoEnabled && !w.noUSO}
} }
// buildTCPv4 constructs a minimal IPv4+TCP packet with the given payload, // buildTCPv4 constructs a minimal IPv4+TCP packet with the given payload,
@@ -126,28 +132,43 @@ const (
tcpAckPsh = tcpAck | tcpPsh tcpAckPsh = tcpAck | tcpPsh
) )
func TestCoalescerPassthroughWhenGSOUnavailable(t *testing.T) { // newTestTCPCoalescer builds a coalescer over w and fails the test if w can't
w := &fakeTunWriter{gsoEnabled: false} // do TSO. Every test but TestNewTCPCoalescerRefusesWhenGSOUnavailable wants the
// GSO path, and the constructor now hands back a nil coalescer otherwise.
func newTestTCPCoalescer(tb testing.TB, w io.Writer) *TCPCoalescer {
tb.Helper()
c := NewTCPCoalescer(w, test.NewLogger()) c := NewTCPCoalescer(w, test.NewLogger())
pkt := buildTCPv4(1000, tcpAck, []byte("hello")) if c == nil {
if err := c.Commit(pkt); err != nil { tb.Fatal("NewTCPCoalescer: writer does not support TSO")
t.Fatal(err)
} }
// No sync write — passthrough is deferred to Flush. return c
if len(w.writes) != 0 || len(w.gsoWrites) != 0 { }
t.Fatalf("no Add-time writes: got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
// TestNewTCPCoalescerRefusesWhenGSOUnavailable pins the constructor
// precondition: no TSO, no coalescer. There's no degraded mode — the caller
// (MultiCoalescer) sends TCP down the passthrough lane instead.
func TestNewTCPCoalescerRefusesWhenGSOUnavailable(t *testing.T) {
if c := NewTCPCoalescer(&fakeTunWriter{gsoEnabled: false}, test.NewLogger()); c != nil {
t.Fatalf("want nil for a non-TSO writer, got %v", c)
} }
if err := c.Flush(); err != nil { // A writer that isn't a GSOWriter at all is refused the same way.
t.Fatal(err) if c := NewTCPCoalescer(&plainOnlyWriter{}, test.NewLogger()); c != nil {
} t.Fatalf("want nil for a plain writer, got %v", c)
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
t.Fatalf("want single plain write, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
} }
} }
// plainOnlyWriter is an io.Writer with no GSO support at all — the
// single-packet Queue shape.
type plainOnlyWriter struct{ writes int }
func (w *plainOnlyWriter) Write(p []byte) (int, error) {
w.writes++
return len(p), nil
}
func TestCoalescerNonTCPPassthrough(t *testing.T) { func TestCoalescerNonTCPPassthrough(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pkt := make([]byte, 28) pkt := make([]byte, 28)
pkt[0] = 0x45 pkt[0] = 0x45
binary.BigEndian.PutUint16(pkt[2:4], 28) binary.BigEndian.PutUint16(pkt[2:4], 28)
@@ -167,7 +188,7 @@ func TestCoalescerNonTCPPassthrough(t *testing.T) {
func TestCoalescerSeedThenFlushAlone(t *testing.T) { func TestCoalescerSeedThenFlushAlone(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pkt := buildTCPv4(1000, tcpAck, make([]byte, 1000)) pkt := buildTCPv4(1000, tcpAck, make([]byte, 1000))
if err := c.Commit(pkt); err != nil { if err := c.Commit(pkt); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -194,7 +215,7 @@ func TestCoalescerSeedThenFlushAlone(t *testing.T) {
func TestCoalescerCoalescesAdjacentACKs(t *testing.T) { func TestCoalescerCoalescesAdjacentACKs(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil { if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -234,7 +255,7 @@ func TestCoalescerCoalescesAdjacentACKs(t *testing.T) {
func TestCoalescerRejectsSeqGap(t *testing.T) { func TestCoalescerRejectsSeqGap(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil { if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -253,7 +274,7 @@ func TestCoalescerRejectsSeqGap(t *testing.T) {
func TestCoalescerRejectsFlagMismatch(t *testing.T) { func TestCoalescerRejectsFlagMismatch(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil { if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -274,7 +295,7 @@ func TestCoalescerRejectsFlagMismatch(t *testing.T) {
func TestCoalescerRejectsFIN(t *testing.T) { func TestCoalescerRejectsFIN(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
fin := buildTCPv4(1000, tcpAck|tcpFin, []byte("x")) fin := buildTCPv4(1000, tcpAck|tcpFin, []byte("x"))
if err := c.Commit(fin); err != nil { if err := c.Commit(fin); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -290,7 +311,7 @@ func TestCoalescerRejectsFIN(t *testing.T) {
func TestCoalescerShortLastSegmentClosesChain(t *testing.T) { func TestCoalescerShortLastSegmentClosesChain(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
full := make([]byte, 1200) full := make([]byte, 1200)
half := make([]byte, 500) half := make([]byte, 500)
if err := c.Commit(buildTCPv4(1000, tcpAck, full)); err != nil { if err := c.Commit(buildTCPv4(1000, tcpAck, full)); err != nil {
@@ -325,7 +346,7 @@ func TestCoalescerShortLastSegmentClosesChain(t *testing.T) {
func TestCoalescerPSHFinalizesChain(t *testing.T) { func TestCoalescerPSHFinalizesChain(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil { if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -355,7 +376,7 @@ func TestCoalescerPSHFinalizesChain(t *testing.T) {
// coalescer drops it the sender's push signal never reaches the receiver. // coalescer drops it the sender's push signal never reaches the receiver.
func TestCoalescerPropagatesPSHFromAppended(t *testing.T) { func TestCoalescerPropagatesPSHFromAppended(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
// Seed has no PSH; second segment carries PSH and seals the chain. // Seed has no PSH; second segment carries PSH and seals the chain.
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil { if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
@@ -383,7 +404,7 @@ func TestCoalescerPropagatesPSHFromAppended(t *testing.T) {
func TestCoalescerRejectsDifferentFlow(t *testing.T) { func TestCoalescerRejectsDifferentFlow(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
p1 := buildTCPv4(1000, tcpAck, pay) p1 := buildTCPv4(1000, tcpAck, pay)
p2 := buildTCPv4(2200, tcpAck, pay) p2 := buildTCPv4(2200, tcpAck, pay)
@@ -405,7 +426,7 @@ func TestCoalescerRejectsDifferentFlow(t *testing.T) {
func TestCoalescerRejectsIPOptions(t *testing.T) { func TestCoalescerRejectsIPOptions(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 500) pay := make([]byte, 500)
pkt := buildTCPv4(1000, tcpAck, pay) pkt := buildTCPv4(1000, tcpAck, pay)
// Bump IHL to 6 to simulate 4 bytes of IP options. Don't actually add // Bump IHL to 6 to simulate 4 bytes of IP options. Don't actually add
@@ -425,7 +446,7 @@ func TestCoalescerRejectsIPOptions(t *testing.T) {
func TestCoalescerCapBySegments(t *testing.T) { func TestCoalescerCapBySegments(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 512) pay := make([]byte, 512)
seq := uint32(1000) seq := uint32(1000)
for i := 0; i < tcpCoalesceMaxSegs+5; i++ { for i := 0; i < tcpCoalesceMaxSegs+5; i++ {
@@ -449,7 +470,7 @@ func TestCoalescerCapBySegments(t *testing.T) {
// flows coalesce independently in a single Flush. // flows coalesce independently in a single Flush.
func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) { func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
// Flow A: sport 1000. Flow B: sport 3000. // Flow A: sport 1000. Flow B: sport 3000.
@@ -506,7 +527,7 @@ func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
// writing passthrough packets synchronously. // writing passthrough packets synchronously.
func TestCoalescerPreservesArrivalOrder(t *testing.T) { func TestCoalescerPreservesArrivalOrder(t *testing.T) {
w := &orderedFakeWriter{gsoEnabled: true} w := &orderedFakeWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
// Sequence: coalesceable TCP, ICMP (passthrough), coalesceable TCP on // Sequence: coalesceable TCP, ICMP (passthrough), coalesceable TCP on
// a different flow. Expected emit order: gso(X), plain(ICMP), gso(Y). // a different flow. Expected emit order: gso(X), plain(ICMP), gso(Y).
pay := make([]byte, 1200) pay := make([]byte, 1200)
@@ -574,7 +595,7 @@ func stringSliceEq(a, b []string) bool {
// packet (SYN) mid-flow only flushes its own flow, not others. // packet (SYN) mid-flow only flushes its own flow, not others.
func TestCoalescerInterleavedFlowsPreserveOrdering(t *testing.T) { func TestCoalescerInterleavedFlowsPreserveOrdering(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
// Flow A two segments. // Flow A two segments.
@@ -679,7 +700,7 @@ func buildTCPv6(tcLow byte, seq uint32, flags byte, payload []byte) []byte {
// retains ECE on the wire. // retains ECE on the wire.
func TestCoalescerCoalescesEceFlow(t *testing.T) { func TestCoalescerCoalescesEceFlow(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
flags := byte(tcpAck | tcpEce) flags := byte(tcpAck | tcpEce)
if err := c.Commit(buildTCPv4(1000, flags, pay)); err != nil { if err := c.Commit(buildTCPv4(1000, flags, pay)); err != nil {
@@ -708,7 +729,7 @@ func TestCoalescerCoalescesEceFlow(t *testing.T) {
// in-flow segment seeds a new slot rather than extending the prior burst. // in-flow segment seeds a new slot rather than extending the prior burst.
func TestCoalescerCwrSealsFlow(t *testing.T) { func TestCoalescerCwrSealsFlow(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil { if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -741,7 +762,7 @@ func TestCoalescerCwrSealsFlow(t *testing.T) {
// a CE-echoing window or none. // a CE-echoing window or none.
func TestCoalescerEceMismatchReseeds(t *testing.T) { func TestCoalescerEceMismatchReseeds(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
if err := c.Commit(buildTCPv4(1000, tcpAck|tcpEce, pay)); err != nil { if err := c.Commit(buildTCPv4(1000, tcpAck|tcpEce, pay)); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -771,7 +792,7 @@ func TestCoalescerEceMismatchReseeds(t *testing.T) {
// across the whole burst. // across the whole burst.
func TestCoalescerDifferingECNReseeds(t *testing.T) { func TestCoalescerDifferingECNReseeds(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
if err := c.Commit(buildTCPv4WithToS(ecnECT0, 1000, tcpAck, pay)); err != nil { if err := c.Commit(buildTCPv4WithToS(ecnECT0, 1000, tcpAck, pay)); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -816,7 +837,7 @@ func TestCoalescerDifferingECNReseeds(t *testing.T) {
// codepoint, and neither may end up CE-marked. // codepoint, and neither may end up CE-marked.
func TestCoalescerECT0ThenECT1NoCE(t *testing.T) { func TestCoalescerECT0ThenECT1NoCE(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
if err := c.Commit(buildTCPv4WithToS(ecnECT0, 1000, tcpAck, pay)); err != nil { if err := c.Commit(buildTCPv4WithToS(ecnECT0, 1000, tcpAck, pay)); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -846,7 +867,7 @@ func TestCoalescerECT0ThenECT1NoCE(t *testing.T) {
// six DSCP bits must match too. // six DSCP bits must match too.
func TestCoalescerDscpMismatchReseeds(t *testing.T) { func TestCoalescerDscpMismatchReseeds(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
// Same ECN (Not-ECT), different DSCP (0x10 vs 0x20 in upper 6 bits). // Same ECN (Not-ECT), different DSCP (0x10 vs 0x20 in upper 6 bits).
tosA := byte(0x10<<2) | ecnNotECT tosA := byte(0x10<<2) | ecnNotECT
@@ -869,7 +890,7 @@ func TestCoalescerDscpMismatchReseeds(t *testing.T) {
// TestCoalescerCoalescesEceFlow. // TestCoalescerCoalescesEceFlow.
func TestCoalescerIPv6CoalescesEceFlow(t *testing.T) { func TestCoalescerIPv6CoalescesEceFlow(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
flags := byte(tcpAck | tcpEce) flags := byte(tcpAck | tcpEce)
if err := c.Commit(buildTCPv6(0, 1000, flags, pay)); err != nil { if err := c.Commit(buildTCPv6(0, 1000, flags, pay)); err != nil {
@@ -900,7 +921,7 @@ func TestCoalescerIPv6CoalescesEceFlow(t *testing.T) {
// seen had the wire never reordered. // seen had the wire never reordered.
func TestCoalescerSortsReorderedSeedsAndMerges(t *testing.T) { func TestCoalescerSortsReorderedSeedsAndMerges(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
// Arrival order: seq 1000, 3400, 2200. The 3400 seeds a separate slot // Arrival order: seq 1000, 3400, 2200. The 3400 seeds a separate slot
// because 3400 != nextSeq=2200, then 2200 fails to extend the 3400 slot // because 3400 != nextSeq=2200, then 2200 fails to extend the 3400 slot
@@ -936,7 +957,7 @@ func TestCoalescerSortsReorderedSeedsAndMerges(t *testing.T) {
// without any cross-flow contamination. // without any cross-flow contamination.
func TestCoalescerSortAcrossFlowsMergesEachIndependently(t *testing.T) { func TestCoalescerSortAcrossFlowsMergesEachIndependently(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
// Flow A (sport 1000) seq 100, 1300; flow B (sport 3000) seq 500, 1700. // Flow A (sport 1000) seq 100, 1300; flow B (sport 3000) seq 500, 1700.
// Arrival: A.1300, B.1700, A.100, B.500 — every flow reordered. // Arrival: A.1300, B.1700, A.100, B.500 — every flow reordered.
@@ -987,7 +1008,7 @@ func TestCoalescerSortAcrossFlowsMergesEachIndependently(t *testing.T) {
// boundary by an arbitrary number of segments. // boundary by an arbitrary number of segments.
func TestCoalescerSortKeepsPSHBoundary(t *testing.T) { func TestCoalescerSortKeepsPSHBoundary(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
// Seq 1000 (no PSH) + 2200 (PSH) → seal one slot with PSH set. // Seq 1000 (no PSH) + 2200 (PSH) → seal one slot with PSH set.
// Seq 3400 (no PSH) is contiguous to 3400 from seq 2200+1200; without // Seq 3400 (no PSH) is contiguous to 3400 from seq 2200+1200; without
@@ -1015,7 +1036,7 @@ func TestCoalescerSortKeepsPSHBoundary(t *testing.T) {
// is sorted/merged independently. // is sorted/merged independently.
func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) { func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
// First two segments seed S1 (then a 3400 reorder seeds S2). // First two segments seed S1 (then a 3400 reorder seeds S2).
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil { if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
@@ -1049,7 +1070,7 @@ func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
// 0x30, so ipHeadersMatch (comparing byte 1 fully) still splits them. // 0x30, so ipHeadersMatch (comparing byte 1 fully) still splits them.
func TestCoalescerIPv6DifferingECNReseeds(t *testing.T) { func TestCoalescerIPv6DifferingECNReseeds(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
// tcLow is the low 4 bits of TC; ECN occupies the bottom 2 of those. // tcLow is the low 4 bits of TC; ECN occupies the bottom 2 of those.
if err := c.Commit(buildTCPv6(ecnECT0, 1000, tcpAck, pay)); err != nil { if err := c.Commit(buildTCPv6(ecnECT0, 1000, tcpAck, pay)); err != nil {
@@ -1124,7 +1145,7 @@ func TestSortRunZeroAllocs(t *testing.T) {
// synthesize it from the seal bool. // synthesize it from the seal bool.
func TestCoalescerMergeShortTailDoesNotFabricatePSH(t *testing.T) { func TestCoalescerMergeShortTailDoesNotFabricatePSH(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
short := make([]byte, 600) short := make([]byte, 600)
// Arrival: seq 3400 (full), 4600 (short, seals the slot), then the // Arrival: seq 3400 (full), 4600 (short, seals the slot), then the
@@ -1162,7 +1183,7 @@ func TestCoalescerMergeShortTailDoesNotFabricatePSH(t *testing.T) {
// source slot's tail really carried PSH, the merged header must keep it. // source slot's tail really carried PSH, the merged header must keep it.
func TestCoalescerMergePreservesRealPSH(t *testing.T) { func TestCoalescerMergePreservesRealPSH(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewTCPCoalescer(w, test.NewLogger()) c := newTestTCPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
short := make([]byte, 600) short := make([]byte, 600)
if err := c.Commit(buildTCPv4(3400, tcpAck, pay)); err != nil { if err := c.Commit(buildTCPv4(3400, tcpAck, pay)); err != nil {
+15 -36
View File
@@ -36,44 +36,35 @@ type udpSlot struct {
numSeg int numSeg int
totalPay int totalPay int
// sealed closes the chain: set when a sub-gsoSize segment is appended // sealed closes the chain: set when a sub-gsoSize segment is appended
// (kernel UDP-GSO requires every segment but the last to be exactly // (kernel UDP-GSO requires every segment but the last to be exactly gsoSize)
// gsoSize) or when limits are hit. No further appends after. // or when limits are hit. No further appends after.
sealed bool sealed bool
payIovs [][]byte payIovs [][]byte
} }
// UDPCoalescer accumulates adjacent in-flow UDP datagrams across multiple // UDPCoalescer accumulates adjacent in-flow UDP datagrams across multiple
// concurrent flows and emits each flow's run as a single GSO_UDP_L4 // concurrent flows and emits each flow's run as a single GSO_UDP_L4 superpacket via tio.GSOWriter.
// superpacket via tio.GSOWriter. Falls back to per-packet writes when the
// underlying writer doesn't support USO.
// Preserves the in-flow order of packets as they are Commit-ed // Preserves the in-flow order of packets as they are Commit-ed
// //
// Owns no locks; one coalescer per TUN write queue. // Owns no locks; one coalescer per TUN write queue.
type UDPCoalescer struct { type UDPCoalescer struct {
plainW io.Writer w tio.GSOWriter
gsoW tio.GSOWriter // nil when the queue can't accept GSO_UDP_L4
slots []*udpSlot slots []*udpSlot
openSlots map[flowKey]*udpSlot openSlots map[flowKey]*udpSlot
pool []*udpSlot pool []*udpSlot
} }
// NewUDPCoalescer wraps w. The caller is responsible for only constructing
// this when the underlying Queue's Capabilities advertise USO; otherwise
// the kernel may reject GSO_UDP_L4 writes. If w does not implement
// tio.GSOWriter at all (single-packet Queue), the coalescer degrades to
// plain Writes — same defensive shape as the TCP coalescer.
func NewUDPCoalescer(w io.Writer) *UDPCoalescer { func NewUDPCoalescer(w io.Writer) *UDPCoalescer {
c := &UDPCoalescer{ gw, ok := tio.SupportsGSO(w, tio.GSOProtoUDP)
plainW: w, if !ok {
return nil
}
return &UDPCoalescer{
w: gw,
slots: make([]*udpSlot, 0, initialSlots), slots: make([]*udpSlot, 0, initialSlots),
openSlots: make(map[flowKey]*udpSlot, initialSlots), openSlots: make(map[flowKey]*udpSlot, initialSlots),
pool: make([]*udpSlot, 0, initialSlots), pool: make([]*udpSlot, 0, initialSlots),
} }
if gw, ok := tio.SupportsGSO(w, tio.GSOProtoUDP); ok {
c.gsoW = gw
}
return c
} }
// parsedUDP holds the fields extracted from a single parse so later steps // parsedUDP holds the fields extracted from a single parse so later steps
@@ -115,10 +106,6 @@ func parseUDP(pkt []byte) (parsedUDP, bool) {
// Commit borrows pkt. The caller must keep pkt valid until the next Flush. // Commit borrows pkt. The caller must keep pkt valid until the next Flush.
func (c *UDPCoalescer) Commit(pkt []byte) error { func (c *UDPCoalescer) Commit(pkt []byte) error {
if c.gsoW == nil {
c.addPassthrough(pkt)
return nil
}
info, ok := parseUDP(pkt) info, ok := parseUDP(pkt)
if !ok { if !ok {
c.addPassthrough(pkt) c.addPassthrough(pkt)
@@ -131,16 +118,8 @@ func (c *UDPCoalescer) Commit(pkt []byte) error {
// already verified parseUDP succeeded. Used by MultiCoalescer.Commit to // already verified parseUDP succeeded. Used by MultiCoalescer.Commit to
// avoid re-walking the IP/UDP header. // avoid re-walking the IP/UDP header.
func (c *UDPCoalescer) commitParsed(pkt []byte, info parsedUDP) error { func (c *UDPCoalescer) commitParsed(pkt []byte, info parsedUDP) error {
if c.gsoW == nil {
c.addPassthrough(pkt)
return nil
}
// A zero-length UDP datagram (UDP `length` == 8) is legal and must still // A zero-length UDP datagram (UDP `length` == 8) is legal and must still
// reach the TUN, but it can't be coalesced: a GSO slot would store an // reach the TUN, but it can't be coalesced.
// empty payload iovec and the kernel has nothing to segment. Seal any
// open chain for this flow (so a later, non-empty datagram seeds fresh
// *after* this one and per-flow arrival order is preserved) and deliver
// it as a plain single datagram.
if info.payLen == 0 { if info.payLen == 0 {
delete(c.openSlots, info.fk) delete(c.openSlots, info.fk)
c.addPassthrough(pkt) c.addPassthrough(pkt)
@@ -154,7 +133,7 @@ func (c *UDPCoalescer) commitParsed(pkt []byte, info parsedUDP) error {
} }
return nil return nil
} }
// Can't extend — seal it and fall through to seed a fresh slot. // Can't extend. Seal it and fall through to seed a fresh slot.
delete(c.openSlots, info.fk) delete(c.openSlots, info.fk)
} }
c.seed(pkt, info) c.seed(pkt, info)
@@ -166,7 +145,7 @@ func (c *UDPCoalescer) Flush() error {
for _, s := range c.slots { for _, s := range c.slots {
var err error var err error
if s.passthrough { if s.passthrough {
_, err = c.plainW.Write(s.rawPkt) _, err = c.w.Write(s.rawPkt)
} else { } else {
err = c.flushSlot(s) err = c.flushSlot(s)
} }
@@ -296,7 +275,7 @@ func (c *UDPCoalescer) flushSlot(s *udpSlot) error {
udpCsumOff := s.ipHdrLen + 6 udpCsumOff := s.ipHdrLen + 6
binary.BigEndian.PutUint16(hdr[udpCsumOff:udpCsumOff+2], foldOnceNoInvert(psum)) binary.BigEndian.PutUint16(hdr[udpCsumOff:udpCsumOff+2], foldOnceNoInvert(psum))
return c.gsoW.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoUDP) return c.w.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoUDP)
} }
// udpHeadersMatch compares two IP+UDP header prefixes for byte-equality on // udpHeadersMatch compares two IP+UDP header prefixes for byte-equality on
@@ -308,7 +287,7 @@ func udpHeadersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
if !ipHeadersMatch(a, b, isV6) { if !ipHeadersMatch(a, b, isV6) {
return false return false
} }
// UDP: compare sport+dport ([0:4]). Skip length [4:6] and checksum [6:8] // UDP: compare sport+dport ([0:4]). Skip length [4:6] and checksum [6:8]
// length varies (we rewrite at flush) and the checksum will be redone. // length varies (we rewrite at flush) and the checksum will be redone.
udp := ipHdrLen udp := ipHdrLen
if a[udp] != b[udp] || a[udp+1] != b[udp+1] || a[udp+2] != b[udp+2] || a[udp+3] != b[udp+3] { if a[udp] != b[udp] || a[udp+1] != b[udp+1] || a[udp+2] != b[udp+2] || a[udp+3] != b[udp+3] {
+32 -27
View File
@@ -2,6 +2,7 @@ package batch
import ( import (
"encoding/binary" "encoding/binary"
"io"
"testing" "testing"
) )
@@ -58,27 +59,31 @@ func buildUDPv6(sport, dport uint16, payload []byte) []byte {
return pkt return pkt
} }
func TestUDPCoalescerPassthroughWhenGSOUnavailable(t *testing.T) { // newTestUDPCoalescer builds a coalescer over w and fails the test if w can't
w := &fakeTunWriter{gsoEnabled: false} // do USO. See newTestTCPCoalescer.
func newTestUDPCoalescer(tb testing.TB, w io.Writer) *UDPCoalescer {
tb.Helper()
c := NewUDPCoalescer(w) c := NewUDPCoalescer(w)
pkt := buildUDPv4(1000, 53, make([]byte, 100)) if c == nil {
if err := c.Commit(pkt); err != nil { tb.Fatal("NewUDPCoalescer: writer does not support USO")
t.Fatal(err)
} }
if len(w.writes) != 0 || len(w.gsoWrites) != 0 { return c
t.Fatalf("no Add-time writes: writes=%d gso=%d", len(w.writes), len(w.gsoWrites)) }
// TestNewUDPCoalescerRefusesWhenGSOUnavailable mirrors the TCP precondition:
// no USO, no coalescer.
func TestNewUDPCoalescerRefusesWhenGSOUnavailable(t *testing.T) {
if c := NewUDPCoalescer(&fakeTunWriter{gsoEnabled: false}); c != nil {
t.Fatalf("want nil for a non-USO writer, got %v", c)
} }
if err := c.Flush(); err != nil { if c := NewUDPCoalescer(&plainOnlyWriter{}); c != nil {
t.Fatal(err) t.Fatalf("want nil for a plain writer, got %v", c)
}
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
t.Fatalf("want single plain write, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
} }
} }
func TestUDPCoalescerNonUDPPassthrough(t *testing.T) { func TestUDPCoalescerNonUDPPassthrough(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
// ICMP packet // ICMP packet
pkt := make([]byte, 28) pkt := make([]byte, 28)
pkt[0] = 0x45 pkt[0] = 0x45
@@ -99,7 +104,7 @@ func TestUDPCoalescerNonUDPPassthrough(t *testing.T) {
func TestUDPCoalescerSeedThenFlushAlone(t *testing.T) { func TestUDPCoalescerSeedThenFlushAlone(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
pkt := buildUDPv4(1000, 53, make([]byte, 800)) pkt := buildUDPv4(1000, 53, make([]byte, 800))
if err := c.Commit(pkt); err != nil { if err := c.Commit(pkt); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -116,7 +121,7 @@ func TestUDPCoalescerSeedThenFlushAlone(t *testing.T) {
func TestUDPCoalescerCoalescesEqualSized(t *testing.T) { func TestUDPCoalescerCoalescesEqualSized(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
for i := 0; i < 3; i++ { for i := 0; i < 3; i++ {
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil { if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
@@ -156,7 +161,7 @@ func TestUDPCoalescerCoalescesEqualSized(t *testing.T) {
// Last segment may be shorter, sealing the chain. // Last segment may be shorter, sealing the chain.
func TestUDPCoalescerShortLastSegmentSeals(t *testing.T) { func TestUDPCoalescerShortLastSegmentSeals(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
full := make([]byte, 1200) full := make([]byte, 1200)
tail := make([]byte, 600) tail := make([]byte, 600)
if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil { if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
@@ -189,7 +194,7 @@ func TestUDPCoalescerShortLastSegmentSeals(t *testing.T) {
// A larger-than-gsoSize packet cannot extend the slot — it reseeds. // A larger-than-gsoSize packet cannot extend the slot — it reseeds.
func TestUDPCoalescerLargerThanSeedReseeds(t *testing.T) { func TestUDPCoalescerLargerThanSeedReseeds(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
if err := c.Commit(buildUDPv4(1000, 53, make([]byte, 800))); err != nil { if err := c.Commit(buildUDPv4(1000, 53, make([]byte, 800))); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -207,7 +212,7 @@ func TestUDPCoalescerLargerThanSeedReseeds(t *testing.T) {
// Different 5-tuples must not coalesce. // Different 5-tuples must not coalesce.
func TestUDPCoalescerDifferentFlowsKeepSeparate(t *testing.T) { func TestUDPCoalescerDifferentFlowsKeepSeparate(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
pay := make([]byte, 800) pay := make([]byte, 800)
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil { if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -238,7 +243,7 @@ func TestUDPCoalescerDifferentFlowsKeepSeparate(t *testing.T) {
// Caps at udpCoalesceMaxSegs. // Caps at udpCoalesceMaxSegs.
func TestUDPCoalescerCapsAtMaxSegs(t *testing.T) { func TestUDPCoalescerCapsAtMaxSegs(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
pay := make([]byte, 100) pay := make([]byte, 100)
for i := 0; i < udpCoalesceMaxSegs+5; i++ { for i := 0; i < udpCoalesceMaxSegs+5; i++ {
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil { if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
@@ -267,7 +272,7 @@ func TestUDPCoalescerCapsAtMaxSegs(t *testing.T) {
// trailing Not-ECT datagram seeds another. // trailing Not-ECT datagram seeds another.
func TestUDPCoalescerDifferingECNReseeds(t *testing.T) { func TestUDPCoalescerDifferingECNReseeds(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
pay := make([]byte, 800) pay := make([]byte, 800)
pkt0 := buildUDPv4(1000, 53, pay) // ECN=00 (Not-ECT) pkt0 := buildUDPv4(1000, 53, pay) // ECN=00 (Not-ECT)
pkt1 := buildUDPv4(1000, 53, pay) pkt1 := buildUDPv4(1000, 53, pay)
@@ -298,7 +303,7 @@ func TestUDPCoalescerDifferingECNReseeds(t *testing.T) {
// IPv6 path: same flow, equal-sized → coalesced. // IPv6 path: same flow, equal-sized → coalesced.
func TestUDPCoalescerIPv6Coalesces(t *testing.T) { func TestUDPCoalescerIPv6Coalesces(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
pay := make([]byte, 1200) pay := make([]byte, 1200)
for i := 0; i < 3; i++ { for i := 0; i < 3; i++ {
if err := c.Commit(buildUDPv6(1000, 53, pay)); err != nil { if err := c.Commit(buildUDPv6(1000, 53, pay)); err != nil {
@@ -334,7 +339,7 @@ func TestUDPCoalescerIPv6Coalesces(t *testing.T) {
// DSCP differences must reseed: udpHeadersMatch compares the full ToS byte. // DSCP differences must reseed: udpHeadersMatch compares the full ToS byte.
func TestUDPCoalescerDSCPMismatchReseeds(t *testing.T) { func TestUDPCoalescerDSCPMismatchReseeds(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
pay := make([]byte, 800) pay := make([]byte, 800)
pkt0 := buildUDPv4(1000, 53, pay) pkt0 := buildUDPv4(1000, 53, pay)
pkt1 := buildUDPv4(1000, 53, pay) pkt1 := buildUDPv4(1000, 53, pay)
@@ -356,7 +361,7 @@ func TestUDPCoalescerDSCPMismatchReseeds(t *testing.T) {
// Fragmented IPv4 must not be coalesced. // Fragmented IPv4 must not be coalesced.
func TestUDPCoalescerFragmentedIPv4PassesThrough(t *testing.T) { func TestUDPCoalescerFragmentedIPv4PassesThrough(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
pkt := buildUDPv4(1000, 53, make([]byte, 200)) pkt := buildUDPv4(1000, 53, make([]byte, 200))
binary.BigEndian.PutUint16(pkt[6:8], 0x2000) // MF=1 binary.BigEndian.PutUint16(pkt[6:8], 0x2000) // MF=1
if err := c.Commit(pkt); err != nil { if err := c.Commit(pkt); err != nil {
@@ -377,7 +382,7 @@ func TestUDPCoalescerFragmentedIPv4PassesThrough(t *testing.T) {
// reach the GSO path. Regression: must not panic and must be written. // reach the GSO path. Regression: must not panic and must be written.
func TestUDPCoalescerZeroLengthPayloadPassesThrough(t *testing.T) { func TestUDPCoalescerZeroLengthPayloadPassesThrough(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
pkt := buildUDPv4(1000, 53, nil) // UDP length 8, zero payload pkt := buildUDPv4(1000, 53, nil) // UDP length 8, zero payload
if err := c.Commit(pkt); err != nil { if err := c.Commit(pkt); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -396,7 +401,7 @@ func TestUDPCoalescerZeroLengthPayloadPassesThrough(t *testing.T) {
// IPv6 zero-length UDP datagram: same passthrough contract as v4. // IPv6 zero-length UDP datagram: same passthrough contract as v4.
func TestUDPCoalescerZeroLengthPayloadIPv6PassesThrough(t *testing.T) { func TestUDPCoalescerZeroLengthPayloadIPv6PassesThrough(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
pkt := buildUDPv6(1000, 53, nil) // UDP length 8, zero payload pkt := buildUDPv6(1000, 53, nil) // UDP length 8, zero payload
if err := c.Commit(pkt); err != nil { if err := c.Commit(pkt); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -417,7 +422,7 @@ func TestUDPCoalescerZeroLengthPayloadIPv6PassesThrough(t *testing.T) {
// wire — per-flow arrival order (full, empty, full) must be preserved. // wire — per-flow arrival order (full, empty, full) must be preserved.
func TestUDPCoalescerZeroLengthMidFlowSealsAndPreservesOrder(t *testing.T) { func TestUDPCoalescerZeroLengthMidFlowSealsAndPreservesOrder(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
full := make([]byte, 800) full := make([]byte, 800)
if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil { if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -441,7 +446,7 @@ func TestUDPCoalescerZeroLengthMidFlowSealsAndPreservesOrder(t *testing.T) {
// IPv4 with options is not admissible (we require IHL=5). // IPv4 with options is not admissible (we require IHL=5).
func TestUDPCoalescerIPv4WithOptionsPassesThrough(t *testing.T) { func TestUDPCoalescerIPv4WithOptionsPassesThrough(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true} w := &fakeTunWriter{gsoEnabled: true}
c := NewUDPCoalescer(w) c := newTestUDPCoalescer(t, w)
pkt := buildUDPv4(1000, 53, make([]byte, 200)) pkt := buildUDPv4(1000, 53, make([]byte, 200))
pkt[0] = 0x46 // IHL = 6 (24-byte IPv4 header — has options) pkt[0] = 0x46 // IHL = 6 (24-byte IPv4 header — has options)
if err := c.Commit(pkt); err != nil { if err := c.Commit(pkt); err != nil {
-6
View File
@@ -8,12 +8,6 @@ func protoFromGSOType(_ uint8) (GSOProto, error) {
return 0, fmt.Errorf("GSO unsupported") return 0, fmt.Errorf("GSO unsupported")
} }
// SegmentSuperpacket invokes fn once per segment of pkt. On non-Linux
// builds (and Android/e2e_testing) this package does not provide a Queue
// implementation, so any caller that does construct a Packet here can only
// be operating on non-superpacket bytes and the stub forwards them
// directly. A non-zero GSO field is a programming error from the caller
// and returns an explicit error rather than silently misbehaving.
func SegmentSuperpacket(pkt Packet, fn func(seg []byte) error) error { func SegmentSuperpacket(pkt Packet, fn func(seg []byte) error) error {
if pkt.GSO.IsSuperpacket() { if pkt.GSO.IsSuperpacket() {
return fmt.Errorf("tio: GSO superpacket on platform without segmentation support") return fmt.Errorf("tio: GSO superpacket on platform without segmentation support")
+5 -6
View File
@@ -4,9 +4,8 @@ import "io"
// singleQueue adapts a legacy one-datagram-per-Read source into a Queue. // singleQueue adapts a legacy one-datagram-per-Read source into a Queue.
// Read fills a private scratch buffer and returns exactly one Packet whose // Read fills a private scratch buffer and returns exactly one Packet whose
// Bytes borrow from that buffer, valid only until the next Read, per the // Bytes borrow from that buffer, valid only until the next Read, per the Queue contract.
// Queue contract. Single-reader like every Queue; Write is exactly as safe // Single-reader like every Queue; Write is exactly as safe for concurrent use as the underlying source's Write.
// for concurrent use as the underlying source's Write.
type singleQueue struct { type singleQueue struct {
rw io.ReadWriter rw io.ReadWriter
closer io.Closer // nil: Close is a no-op (the source is shared and owned elsewhere) closer io.Closer // nil: Close is a no-op (the source is shared and owned elsewhere)
@@ -14,9 +13,9 @@ type singleQueue struct {
ret [1]Packet ret [1]Packet
} }
// NewSingleQueue wraps a one-datagram-per-Read ReadWriteCloser (a legacy tun // NewSingleQueue wraps a one-datagram-per-Read ReadWriteCloser (a legacy tun device) into a Queue.
// device) into a Queue. bufSize is the per-queue read scratch size and must // bufSize is the per-queue read scratch size and must be at least the largest datagram the source can return.
// be at least the largest datagram the source can return. Close closes rwc. // Close closes rwc.
func NewSingleQueue(rwc io.ReadWriteCloser, bufSize int) Queue { func NewSingleQueue(rwc io.ReadWriteCloser, bufSize int) Queue {
return &singleQueue{rw: rwc, closer: rwc, buf: make([]byte, bufSize)} return &singleQueue{rw: rwc, closer: rwc, buf: make([]byte, bufSize)}
} }
+3 -10
View File
@@ -93,14 +93,6 @@ type CapsProvider interface {
Capabilities() Capabilities Capabilities() Capabilities
} }
// QueueCapabilities returns q's negotiated offload capabilities, or the zero value when q does not advertise any.
func QueueCapabilities(q io.Writer) Capabilities {
if cp, ok := q.(CapsProvider); ok {
return cp.Capabilities()
}
return Capabilities{}
}
// GSOProto selects the L4 protocol for a GSO superpacket. // GSOProto selects the L4 protocol for a GSO superpacket.
// Determines which VIRTIO_NET_HDR_GSO_* type the writer stamps and which checksum offset // Determines which VIRTIO_NET_HDR_GSO_* type the writer stamps and which checksum offset
// inside the transport header virtio NEEDS_CSUM expects. // inside the transport header virtio NEEDS_CSUM expects.
@@ -126,9 +118,10 @@ const (
// Every segment in pays except possibly the last is exactly the same size. // Every segment in pays except possibly the last is exactly the same size.
// proto picks the L4 protocol so the writer knows which gsoType / CsumOffset to set. // proto picks the L4 protocol so the writer knows which gsoType / CsumOffset to set.
// //
// Callers should also consult CapsProvider (via SupportsGSO or QueueCapabilities) // Callers should also consult CapsProvider (via SupportsGSO) for the per-protocol negotiated capability:
// for the per-protocol negotiated capability: USO may not have been negotiated even when TSO was. // USO may not have been negotiated even when TSO was.
type GSOWriter interface { type GSOWriter interface {
io.Writer
CapsProvider CapsProvider
WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, proto GSOProto) error WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, proto GSOProto) error
} }
+1 -1
View File
@@ -173,7 +173,7 @@ func (r *Offload) Read() ([]Packet, error) {
return nil, err return nil, err
} }
if err := r.decodeRead(n); err != nil { if err := r.decodeRead(n); err != nil {
// Drop and read again — a bad packet should not kill the reader. // Drop and read again. A bad packet should not kill the reader.
continue continue
} }
break break