mirror of
https://github.com/slackhq/nebula.git
synced 2026-08-15 07:17:00 +02:00
improve naming in batch
This commit is contained in:
@@ -12,7 +12,7 @@ import (
|
||||
// on the IP/L4 protocol of the packet.
|
||||
//
|
||||
// Lanes are processed independently: the TCP coalescer only sees TCP, the
|
||||
// UDP coalescer only sees UDP, and the passthrough lane handles everything else.
|
||||
// UDP coalescer only sees UDP, and the verbatim lane handles everything else.
|
||||
// The ordering contract is per-flow DATA order: a flow's payload-bearing
|
||||
// packets are never reordered relative to each other, because a single
|
||||
// 5-tuple only ever lands in one lane and each lane emits its slots in
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
// - pure ACKs, which pass through without sealing the flow's open slot
|
||||
// (a late ACK is just a stale ACK; see TCPCoalescer.commitParsed);
|
||||
// - unparseable in-flow shapes (fragments, IP options), whose lane-level
|
||||
// addPassthrough does not close the flow's open slot either. Closing it
|
||||
// addVerbatim does not close the flow's open slot either. Closing it
|
||||
// would need a full open-slot barrier (the flow key is unknown when the
|
||||
// parse fails) — an accepted tradeoff: mid-flow fragments are rare and
|
||||
// receivers reassemble regardless of arrival order.
|
||||
@@ -31,7 +31,7 @@ import (
|
||||
// terminal proto, so a flow's non-coalesceable shapes ride its lane as
|
||||
// in-lane passthroughs rather than falling to the later-flushed pt lane.
|
||||
//
|
||||
// Cross-lane order is intentionally NOT preserved across the TCP/UDP/passthrough split.
|
||||
// Cross-lane order is intentionally NOT preserved across the TCP/UDP/verbatim split.
|
||||
type MultiCoalescer struct {
|
||||
tcp *TCPCoalescer
|
||||
udp *UDPCoalescer
|
||||
@@ -46,7 +46,7 @@ func NewMultiCoalescer(w io.Writer, l *slog.Logger) RxBatcher {
|
||||
m.tcp = NewTCPCoalescer(w, l)
|
||||
m.udp = NewUDPCoalescer(w)
|
||||
if m.tcp == nil && m.udp == nil {
|
||||
return m.pt //no offloads? Use passthrough directly.
|
||||
return m.pt //no offloads? Use verbatim directly.
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -91,19 +91,8 @@ func (m *MultiCoalescer) Commit(pkt []byte) error {
|
||||
}
|
||||
proto = pkt[6]
|
||||
if isIPv6ExtHeader(proto) {
|
||||
// Walk to the terminal protocol so the packet routes to its
|
||||
// flow's lane. It stays non-coalesceable — the lane's parser
|
||||
// rejects the ext-header shape and emits it as an in-lane
|
||||
// passthrough — but landing in the right lane preserves
|
||||
// per-flow order, exactly like IPv4 fragments (whose header
|
||||
// keeps the L4 proto visible) already do. Fragments are the
|
||||
// case that matters: every fragment names the flow's L4, so a
|
||||
// fragmented datagram travels with its flow's unfragmented
|
||||
// siblings instead of the passthrough lane, which flushes
|
||||
// after every coalescer lane and would emit it behind data
|
||||
// that arrived later. An unresolved walk (truncated or crafted
|
||||
// over-long chain) yields a non-transport number and falls to
|
||||
// the pt lane below.
|
||||
// Walk to the terminal protocol so the packet routes to its flow's protocol lane.
|
||||
// This protects flow ordering.
|
||||
proto, _, _ = iputil.IPv6FindUpperProtocol(pkt)
|
||||
}
|
||||
default:
|
||||
@@ -115,8 +104,8 @@ func (m *MultiCoalescer) Commit(pkt []byte) error {
|
||||
info, ok := parseTCPBase(pkt)
|
||||
if !ok {
|
||||
// Malformed/unsupported TCP shape (IP options, fragments, ...).
|
||||
// Handle this via passthrough support in the TCP coalescer, to attempt to preserve flow order.
|
||||
m.tcp.addPassthrough(pkt)
|
||||
// Handle this via verbatim support in the TCP coalescer, to attempt to preserve flow order.
|
||||
m.tcp.addVerbatim(pkt)
|
||||
return nil
|
||||
}
|
||||
return m.tcp.commitParsed(pkt, info)
|
||||
@@ -125,7 +114,7 @@ func (m *MultiCoalescer) Commit(pkt []byte) error {
|
||||
if m.udp != nil {
|
||||
info, ok := parseUDP(pkt)
|
||||
if !ok {
|
||||
m.udp.addPassthrough(pkt)
|
||||
m.udp.addVerbatim(pkt)
|
||||
return nil
|
||||
}
|
||||
return m.udp.commitParsed(pkt, info)
|
||||
|
||||
@@ -67,7 +67,7 @@ func TestMultiCoalescerRoutesByProto(t *testing.T) {
|
||||
|
||||
// TestMultiCoalescerNoUSOFallsThrough verifies that on a queue without USO
|
||||
// (older kernel: TSO but no GSO_UDP_L4) the UDP lane never comes up and UDP
|
||||
// packets still reach the kernel via passthrough rather than being lost.
|
||||
// packets still reach the kernel via verbatim rather than being lost.
|
||||
func TestMultiCoalescerNoUSOFallsThrough(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true, noUSO: true}
|
||||
m := newTestMultiCoalescer(t, w)
|
||||
@@ -94,7 +94,7 @@ func TestMultiCoalescerNoUSOFallsThrough(t *testing.T) {
|
||||
|
||||
// TestMultiCoalescerNoOffloadsIsPassthrough covers a queue that can't offload
|
||||
// anything. Both lane constructors refuse, so there's nothing left to
|
||||
// dispatch between and NewMultiCoalescer hands back the passthrough lane
|
||||
// dispatch between and NewMultiCoalescer hands back the verbatim lane
|
||||
// itself — no wrapper, no per-packet protocol demux, and every packet reaches
|
||||
// the kernel in arrival order. This is the case Interface.activate used to
|
||||
// special-case with a bare Passthrough.
|
||||
@@ -167,8 +167,8 @@ func buildUDPv6Fragment(sport, dport uint16, payload []byte) []byte {
|
||||
|
||||
// TestMultiCoalescerIPv6FragmentStaysInLane locks in extension-header
|
||||
// routing: a fragment whose chain terminates in UDP must ride the UDP lane
|
||||
// as an in-lane passthrough — emitted ahead of later same-flow datagrams —
|
||||
// not the passthrough lane, which flushes after every coalescer lane and
|
||||
// as an in-lane verbatim — emitted ahead of later same-flow datagrams —
|
||||
// not the verbatim lane, which flushes after every coalescer lane and
|
||||
// would reorder it behind data that arrived after it.
|
||||
func TestMultiCoalescerIPv6FragmentStaysInLane(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
@@ -194,7 +194,7 @@ func TestMultiCoalescerIPv6FragmentStaysInLane(t *testing.T) {
|
||||
}
|
||||
// Arrival order was fragment-then-data; same-lane routing must keep it.
|
||||
if w.order[0] != "write" {
|
||||
t.Fatalf("fragment must be emitted before later data (in-lane passthrough), order=%v", w.order)
|
||||
t.Fatalf("fragment must be emitted before later data (in-lane verbatim), order=%v", w.order)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,25 +28,25 @@ const tcpCoalesceMaxSegs = 64
|
||||
const tcpCoalesceHdrCap = 100
|
||||
|
||||
// coalesceSlot is one entry in the coalescer's ordered event queue.
|
||||
// When passthrough is true the slot holds a single borrowed packet that must be
|
||||
// When verbatim is true the slot holds a single borrowed packet that must be
|
||||
// emitted verbatim (non-TCP, non-admissible TCP, or oversize seed).
|
||||
// When passthrough is false the slot is an in-progress coalesced superpacket.
|
||||
// When verbatim is false the slot is an in-progress coalesced superpacket.
|
||||
// hdrBuf is a mutable copy of the seed's IP+TCP header
|
||||
// (we patch total length and pseudo-header partial at flush)
|
||||
// payIovs are *borrowed* slices from the caller's plaintext buffers.
|
||||
// The caller (listenOut) must keep those buffers alive until Flush.
|
||||
|
||||
const (
|
||||
passthroughFalse = iota
|
||||
// passthroughTrue means a sync-point packet, that may not be re-ordered
|
||||
passthroughTrue
|
||||
// passthroughACK packets are "passed through" without coalescing, but traffic "after" them may be pulled forward to facilitate coalescing.
|
||||
passthroughACK
|
||||
verbatimFalse = iota
|
||||
// verbatimTrue means a sync-point packet, that may not be re-ordered
|
||||
verbatimTrue
|
||||
// verbatimACK packets are "passed through" without coalescing, but traffic "after" them may be pulled forward to facilitate coalescing.
|
||||
verbatimACK
|
||||
)
|
||||
|
||||
type coalesceSlot struct {
|
||||
passthrough uint8
|
||||
// rawPkt is borrowed: the whole packet for passthrough slots, the seed
|
||||
verbatim uint8
|
||||
// rawPkt is borrowed: the whole packet for verbatim slots, the seed
|
||||
// packet for coalesce slots. A coalesce slot that never grows past one
|
||||
// segment is emitted from rawPkt so its original (already valid) L4
|
||||
// checksum ships DATA_VALID instead of making the kernel recompute it.
|
||||
@@ -74,8 +74,8 @@ type coalesceSlot struct {
|
||||
payIovs [][]byte
|
||||
}
|
||||
|
||||
func (c *coalesceSlot) isPassthrough() bool {
|
||||
return c.passthrough != passthroughFalse
|
||||
func (c *coalesceSlot) isVerbatim() bool {
|
||||
return c.verbatim != verbatimFalse
|
||||
}
|
||||
|
||||
// TCPCoalescer accumulates adjacent in-flow TCP data segments across multiple concurrent flows
|
||||
@@ -86,7 +86,7 @@ type TCPCoalescer struct {
|
||||
w tio.GSOWriter
|
||||
|
||||
// slots is the ordered event queue. Flush walks it once and emits each
|
||||
// entry as either a WriteGSO (coalesced) or a w.Write (passthrough).
|
||||
// entry as either a WriteGSO (coalesced) or a w.Write (verbatim).
|
||||
slots []*coalesceSlot
|
||||
// openSlots maps a flow key to its most recent non-sealed slot, so new
|
||||
// segments can extend an in-progress superpacket in O(1). Slots are
|
||||
@@ -208,7 +208,7 @@ func (p parsedTCP) pureAck() bool {
|
||||
func (c *TCPCoalescer) Commit(pkt []byte) error {
|
||||
info, ok := parseTCPBase(pkt)
|
||||
if !ok {
|
||||
c.addPassthrough(pkt)
|
||||
c.addVerbatim(pkt)
|
||||
return nil
|
||||
}
|
||||
return c.commitParsed(pkt, info)
|
||||
@@ -227,13 +227,13 @@ func (c *TCPCoalescer) commitParsed(pkt []byte, info parsedTCP) error {
|
||||
// evict keeps a bidirectional flow's inbound data run coalescing
|
||||
// across the peer ACKs interleaved into it — kernel GRO likewise
|
||||
// doesn't flush held data on a pure ACK.
|
||||
c.addPassthroughACK(pkt, info)
|
||||
c.addVerbatimACK(pkt, info)
|
||||
return nil
|
||||
}
|
||||
// TCP but not admissible (SYN/FIN/RST/URG/CWR or a shape the flow
|
||||
// must observe in sequence). Seal this flow's open slot so later
|
||||
// in-flow packets don't extend it and accidentally reorder past this
|
||||
// passthrough. The len guard skips hashing the 38-byte key on
|
||||
// verbatim. The len guard skips hashing the 38-byte key on
|
||||
// ack-dominant queues, where the map is almost always empty.
|
||||
if len(c.openSlots) != 0 {
|
||||
if last := c.lastSlot; last != nil && last.fk == info.fk {
|
||||
@@ -241,7 +241,7 @@ func (c *TCPCoalescer) commitParsed(pkt []byte, info parsedTCP) error {
|
||||
}
|
||||
delete(c.openSlots, info.fk)
|
||||
}
|
||||
c.addPassthrough(pkt)
|
||||
c.addVerbatim(pkt)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -283,7 +283,7 @@ func (c *TCPCoalescer) Flush() error {
|
||||
var first error
|
||||
for _, s := range c.slots {
|
||||
var err error
|
||||
if s.isPassthrough() || s.numSeg == 1 {
|
||||
if s.isVerbatim() || s.numSeg == 1 {
|
||||
// A slot that never grew (nor absorbed a merge) is byte-identical
|
||||
// to the packet it was seeded from; ship the original so its valid
|
||||
// checksum rides the DATA_VALID path instead of paying a kernel
|
||||
@@ -306,22 +306,22 @@ func (c *TCPCoalescer) Flush() error {
|
||||
return first
|
||||
}
|
||||
|
||||
func (c *TCPCoalescer) addPassthrough(pkt []byte) {
|
||||
func (c *TCPCoalescer) addVerbatim(pkt []byte) {
|
||||
s := c.take()
|
||||
s.passthrough = passthroughTrue
|
||||
s.verbatim = verbatimTrue
|
||||
s.rawPkt = pkt
|
||||
c.slots = append(c.slots, s)
|
||||
}
|
||||
|
||||
// addPassthroughACK commits a pure ACK as a passthrough slot that keeps its
|
||||
// flow identity and sort keys. Unlike addPassthrough slots it does not split
|
||||
// addVerbatimACK commits a pure ACK as a verbatim slot that keeps its
|
||||
// flow identity and sort keys. Unlike addVerbatim slots it does not split
|
||||
// sort runs, so reorderForFlush may sort same-flow data across it (the
|
||||
// contract allows data to overtake a bare ACK). A pure ACK's seq is the
|
||||
// sender's snd_nxt, which orders it after all data the peer sent before it,
|
||||
// and the TSval-first comparator keeps it behind any older-timestamp data.
|
||||
func (c *TCPCoalescer) addPassthroughACK(pkt []byte, info parsedTCP) {
|
||||
func (c *TCPCoalescer) addVerbatimACK(pkt []byte, info parsedTCP) {
|
||||
s := c.take()
|
||||
s.passthrough = passthroughACK
|
||||
s.verbatim = verbatimACK
|
||||
s.rawPkt = pkt
|
||||
s.fk = info.fk
|
||||
s.nextSeq = info.seq // totalPay stays 0, so slotSeedSeq yields info.seq
|
||||
@@ -332,11 +332,11 @@ func (c *TCPCoalescer) addPassthroughACK(pkt []byte, info parsedTCP) {
|
||||
func (c *TCPCoalescer) seed(pkt []byte, info parsedTCP) {
|
||||
if info.hdrLen > tcpCoalesceHdrCap || info.hdrLen+info.payLen > tcpCoalesceBufSize {
|
||||
// Pathological shape. Can't fit our scratch, emit as-is.
|
||||
c.addPassthrough(pkt)
|
||||
c.addVerbatim(pkt)
|
||||
return
|
||||
}
|
||||
s := c.take()
|
||||
s.passthrough = passthroughFalse
|
||||
s.verbatim = verbatimFalse
|
||||
s.rawPkt = pkt // kept for the numSeg==1 fast path in Flush
|
||||
copy(s.hdrBuf[:], pkt[:info.hdrLen])
|
||||
s.hdrLen = info.hdrLen
|
||||
@@ -357,7 +357,7 @@ func (c *TCPCoalescer) seed(pkt []byte, info parsedTCP) {
|
||||
} else if last := c.lastSlot; last != nil && last.fk == info.fk {
|
||||
// PSH-on-seed seals the slot immediately. Any prior cached open
|
||||
// slot for this flow has just been sealed-and-replaced by this
|
||||
// passthrough-shaped seed, so drop the cache too.
|
||||
// verbatim-shaped seed, so drop the cache too.
|
||||
c.lastSlot = nil
|
||||
}
|
||||
}
|
||||
@@ -424,15 +424,15 @@ func (c *TCPCoalescer) take() *coalesceSlot {
|
||||
}
|
||||
|
||||
func (c *TCPCoalescer) release(s *coalesceSlot) {
|
||||
s.passthrough = passthroughFalse
|
||||
s.verbatim = verbatimFalse
|
||||
s.rawPkt = nil
|
||||
clear(s.payIovs)
|
||||
s.payIovs = s.payIovs[:0]
|
||||
s.numSeg = 0
|
||||
s.totalPay = 0
|
||||
s.sealed = false
|
||||
// Zero the identity fields too: addPassthrough doesn't set them, so a
|
||||
// pooled slot reused as a passthrough must not carry a stale flow key
|
||||
// Zero the identity fields too: addVerbatim doesn't set them, so a
|
||||
// pooled slot reused as a verbatim must not carry a stale flow key
|
||||
// that a future refactor could mistake for real.
|
||||
s.fk = flowKey{}
|
||||
s.hdrLen = 0
|
||||
@@ -508,7 +508,7 @@ func headersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
|
||||
// receiver as a much larger reorder than the wire actually had.
|
||||
//
|
||||
// Two phases:
|
||||
// 1. Sort each passthrough-bounded segment of c.slots by (flow, seq).
|
||||
// 1. Sort each verbatim-bounded segment of c.slots by (flow, seq).
|
||||
// Cross-flow ordering inside a segment isn't preserved (it never was
|
||||
// and doesn't matter for any single flow's TCP correctness).
|
||||
// 2. Sweep once and merge adjacent same-flow slots whose ranges are now
|
||||
@@ -517,7 +517,7 @@ func headersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
|
||||
// start of the merged payload. A short segment in the middle would
|
||||
// desynchronize every later segment.
|
||||
//
|
||||
// Passthrough slots act as barriers: the merge check skips them on either
|
||||
// Verbatim slots act as barriers: the merge check skips them on either
|
||||
// side, so a SYN/FIN/RST/CWR is never reordered relative to its flow's
|
||||
// data.
|
||||
func (c *TCPCoalescer) reorderForFlush() {
|
||||
@@ -526,11 +526,11 @@ func (c *TCPCoalescer) reorderForFlush() {
|
||||
}
|
||||
runStart := 0
|
||||
for i := 0; i <= len(c.slots); i++ {
|
||||
// Only hard passthroughs (unparseable, SYN/FIN/RST/CWR, oversized)
|
||||
// split sort runs. Pure-ACK passthroughs stay inside the run so
|
||||
// Only hard verbatims (unparseable, SYN/FIN/RST/CWR, oversized)
|
||||
// split sort runs. Pure-ACK verbatims stay inside the run so
|
||||
// same-flow data separated by an interleaved ACK can still sort
|
||||
// adjacent and merge; their own sort keys keep them ordered.
|
||||
if i < len(c.slots) && c.slots[i].passthrough != passthroughTrue {
|
||||
if i < len(c.slots) && c.slots[i].verbatim != verbatimTrue {
|
||||
continue
|
||||
}
|
||||
c.sortRun(c.slots[runStart:i])
|
||||
@@ -540,7 +540,7 @@ func (c *TCPCoalescer) reorderForFlush() {
|
||||
for _, s := range c.slots {
|
||||
if n := len(out); n > 0 {
|
||||
prev := out[n-1]
|
||||
if !prev.isPassthrough() && !s.isPassthrough() && prev.fk == s.fk {
|
||||
if !prev.isVerbatim() && !s.isVerbatim() && prev.fk == s.fk {
|
||||
// Same-flow neighbors after sort. If they aren't seq-
|
||||
// contiguous it's a real gap: packets the wire reordered
|
||||
// across batches, or actual loss before nebula. Log it so
|
||||
|
||||
@@ -79,7 +79,7 @@ func buildTCPv4RunInterleaved(nFlows, perFlow, runLen, payloadLen int) [][]byte
|
||||
return pkts
|
||||
}
|
||||
|
||||
// buildICMPv4 returns a minimal non-TCP packet that takes the passthrough
|
||||
// buildICMPv4 returns a minimal non-TCP packet that takes the verbatim
|
||||
// branch in Commit.
|
||||
func buildICMPv4() []byte {
|
||||
pkt := make([]byte, 28)
|
||||
@@ -146,7 +146,7 @@ func BenchmarkCommitRunInterleaved4(b *testing.B) {
|
||||
}
|
||||
|
||||
// BenchmarkCommitPassthrough exercises the non-TCP branch: parseTCPBase
|
||||
// bails early and addPassthrough is the only work.
|
||||
// bails early and addVerbatim is the only work.
|
||||
func BenchmarkCommitPassthrough(b *testing.B) {
|
||||
pkt := buildICMPv4()
|
||||
pkts := make([][]byte, 64)
|
||||
@@ -158,7 +158,7 @@ func BenchmarkCommitPassthrough(b *testing.B) {
|
||||
|
||||
// BenchmarkCommitNonCoalesceableTCP sends SYN|ACK packets on one flow.
|
||||
// Each packet takes the "TCP but not admissible" branch which does a
|
||||
// map delete + passthrough. Measures the seal-without-slot cost.
|
||||
// map delete + verbatim. Measures the seal-without-slot cost.
|
||||
func BenchmarkCommitNonCoalesceableTCP(b *testing.B) {
|
||||
pay := make([]byte, 0)
|
||||
pkts := make([][]byte, 64)
|
||||
|
||||
@@ -164,7 +164,7 @@ func newTestTCPCoalescer(tb testing.TB, w io.Writer) *TCPCoalescer {
|
||||
|
||||
// TestNewTCPCoalescerRefusesWhenGSOUnavailable pins the constructor
|
||||
// precondition: no TSO, no coalescer. There's no degraded mode — the caller
|
||||
// (MultiCoalescer) sends TCP down the passthrough lane instead.
|
||||
// (MultiCoalescer) sends TCP down the verbatim lane instead.
|
||||
func TestNewTCPCoalescerRefusesWhenGSOUnavailable(t *testing.T) {
|
||||
if c := NewTCPCoalescer(&fakeTunWriter{gsoEnabled: false}, test.NewLogger()); c != nil {
|
||||
t.Fatalf("want nil for a non-TSO writer, got %v", c)
|
||||
@@ -231,7 +231,7 @@ func TestCoalescerSeedThenFlushAlone(t *testing.T) {
|
||||
|
||||
// TestCoalescerPureAckDoesNotSealRun pins the pure-ACK fast path: a bare
|
||||
// acknowledgment (zero payload, nothing beyond ACK|PSH|ECE) rides its lane
|
||||
// as a passthrough WITHOUT sealing the flow's open slot, so an inbound data
|
||||
// as a verbatim WITHOUT sealing the flow's open slot, so an inbound data
|
||||
// run on a bidirectional connection keeps coalescing across the peer ACKs
|
||||
// interleaved into it. The ACK is emitted after the superpacket (stale ACKs
|
||||
// are ignored by receivers, so the reorder is harmless by design).
|
||||
@@ -388,9 +388,9 @@ func TestCoalescerRejectsFIN(t *testing.T) {
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// FIN isn't admissible — passthrough as plain, no slot, no gso.
|
||||
// FIN isn't admissible — verbatim as plain, no slot, no gso.
|
||||
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("FIN should be passthrough, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
t.Fatalf("FIN should be verbatim, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -529,9 +529,9 @@ func TestCoalescerRejectsIPOptions(t *testing.T) {
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Non-admissible parse → passthrough as plain.
|
||||
// Non-admissible parse → verbatim as plain.
|
||||
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("IP options should passthrough, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
t.Fatalf("IP options should verbatim, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -613,13 +613,13 @@ func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCoalescerPreservesArrivalOrder confirms that with passthrough and
|
||||
// TestCoalescerPreservesArrivalOrder confirms that with verbatim and
|
||||
// coalesced events both queued, Flush emits them in Add order rather than
|
||||
// writing passthrough packets synchronously.
|
||||
// writing verbatim packets synchronously.
|
||||
func TestCoalescerPreservesArrivalOrder(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestTCPCoalescer(t, w)
|
||||
// Sequence: coalesceable TCP, ICMP (passthrough), coalesceable TCP on
|
||||
// Sequence: coalesceable TCP, ICMP (verbatim), coalesceable TCP on
|
||||
// a different flow. Both TCP slots stay single-segment, so all three
|
||||
// emit as plain writes; the packet order (X, ICMP, Y) is asserted by
|
||||
// byte content since the kinds no longer distinguish them.
|
||||
@@ -775,7 +775,7 @@ func buildTCPv6(tcLow byte, seq uint32, flags byte, payload []byte) []byte {
|
||||
|
||||
// TestCoalescerCoalescesEceFlow confirms that ECN-Echo-marked ACKs (an
|
||||
// ECN-aware flow under congestion) keep getting coalesced into a TSO
|
||||
// superpacket instead of falling out to passthrough, and that the seed
|
||||
// superpacket instead of falling out to verbatim, and that the seed
|
||||
// retains ECE on the wire.
|
||||
func TestCoalescerCoalescesEceFlow(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
@@ -804,7 +804,7 @@ func TestCoalescerCoalescesEceFlow(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestCoalescerCwrSealsFlow confirms that a CWR-bearing segment in the
|
||||
// middle of a flow goes to passthrough and seals the open slot, so a later
|
||||
// middle of a flow goes to verbatim and seals the open slot, so a later
|
||||
// in-flow segment seeds a new slot rather than extending the prior burst.
|
||||
func TestCoalescerCwrSealsFlow(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
@@ -824,12 +824,12 @@ func TestCoalescerCwrSealsFlow(t *testing.T) {
|
||||
}
|
||||
// All three emissions are plain writes: the seed before CWR and the
|
||||
// fresh seed after both stay single-segment, and the CWR packet itself
|
||||
// is passthrough. Order: seed, CWR, reseed.
|
||||
// is verbatim. Order: seed, CWR, reseed.
|
||||
if len(w.writes) != 3 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("want 3 plain writes (seed, CWR, reseed), got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
||||
}
|
||||
if flags := w.writes[1][20+13]; flags&tcpCwr == 0 {
|
||||
t.Errorf("middle write flags=0x%02x want CWR (passthrough in arrival order)", flags)
|
||||
t.Errorf("middle write flags=0x%02x want CWR (verbatim in arrival order)", flags)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1115,9 +1115,9 @@ func TestCoalescerSortKeepsPSHBoundary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCoalescerSortKeepsPassthroughBarrier confirms a passthrough slot in
|
||||
// TestCoalescerSortKeepsPassthroughBarrier confirms a verbatim slot in
|
||||
// the middle of the queue prevents the post-sort merge from folding
|
||||
// across it. Reordered same-flow data on either side of the passthrough
|
||||
// across it. Reordered same-flow data on either side of the verbatim
|
||||
// is sorted/merged independently.
|
||||
func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
@@ -1131,7 +1131,7 @@ func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Non-coalesceable packet (SYN+ACK) flushes S1's openSlots entry and
|
||||
// becomes a passthrough barrier in c.slots.
|
||||
// becomes a verbatim barrier in c.slots.
|
||||
if err := c.Commit(buildTCPv4(9999, tcpSyn|tcpAck, pay)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1144,7 +1144,7 @@ func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
|
||||
}
|
||||
// All four packets emit as plain writes: 1000 and 3400 are separate
|
||||
// single-segment slots (not contiguous, so the post-sort merge can't
|
||||
// fold them), the SYN is passthrough, and the post-barrier 2200 stays
|
||||
// fold them), the SYN is verbatim, and the post-barrier 2200 stays
|
||||
// a single-segment slot after the SYN. The pre-barrier sort must land
|
||||
// 1000 before 3400, and 2200 must never move before the SYN.
|
||||
if len(w.writes) != 4 || len(w.gsoWrites) != 0 {
|
||||
|
||||
@@ -24,8 +24,8 @@ const udpCoalesceHdrCap = 64
|
||||
|
||||
// udpSlot is one entry in the UDPCoalescer's ordered event queue.
|
||||
type udpSlot struct {
|
||||
passthrough bool
|
||||
// rawPkt is borrowed: the whole packet for passthrough slots, the seed
|
||||
verbatim bool
|
||||
// rawPkt is borrowed: the whole packet for verbatim slots, the seed
|
||||
// packet for coalesce slots. A coalesce slot that never grows past one
|
||||
// segment is emitted from rawPkt so its original (already valid) L4
|
||||
// checksum ships DATA_VALID instead of making the kernel recompute it.
|
||||
@@ -119,7 +119,7 @@ func parseUDP(pkt []byte) (parsedUDP, bool) {
|
||||
func (c *UDPCoalescer) Commit(pkt []byte) error {
|
||||
info, ok := parseUDP(pkt)
|
||||
if !ok {
|
||||
c.addPassthrough(pkt)
|
||||
c.addVerbatim(pkt)
|
||||
return nil
|
||||
}
|
||||
return c.commitParsed(pkt, info)
|
||||
@@ -139,7 +139,7 @@ func (c *UDPCoalescer) commitParsed(pkt []byte, info parsedUDP) error {
|
||||
}
|
||||
delete(c.openSlots, info.fk)
|
||||
}
|
||||
c.addPassthrough(pkt)
|
||||
c.addVerbatim(pkt)
|
||||
return nil
|
||||
}
|
||||
// Cached-slot fast path; see the TCPCoalescer equivalent.
|
||||
@@ -175,7 +175,7 @@ func (c *UDPCoalescer) Flush() error {
|
||||
var first error
|
||||
for _, s := range c.slots {
|
||||
var err error
|
||||
if s.passthrough || s.numSeg == 1 {
|
||||
if s.verbatim || s.numSeg == 1 {
|
||||
// A slot that never grew is byte-identical to the packet it was
|
||||
// seeded from; ship the original so its valid checksum rides the
|
||||
// DATA_VALID path instead of paying a kernel software csum.
|
||||
@@ -195,20 +195,20 @@ func (c *UDPCoalescer) Flush() error {
|
||||
return first
|
||||
}
|
||||
|
||||
func (c *UDPCoalescer) addPassthrough(pkt []byte) {
|
||||
func (c *UDPCoalescer) addVerbatim(pkt []byte) {
|
||||
s := c.take()
|
||||
s.passthrough = true
|
||||
s.verbatim = true
|
||||
s.rawPkt = pkt
|
||||
c.slots = append(c.slots, s)
|
||||
}
|
||||
|
||||
func (c *UDPCoalescer) seed(pkt []byte, info parsedUDP) {
|
||||
if info.hdrLen > udpCoalesceHdrCap || info.hdrLen+info.payLen > udpCoalesceBufSize {
|
||||
c.addPassthrough(pkt)
|
||||
c.addVerbatim(pkt)
|
||||
return
|
||||
}
|
||||
s := c.take()
|
||||
s.passthrough = false
|
||||
s.verbatim = false
|
||||
s.rawPkt = pkt // kept for the numSeg==1 fast path in Flush
|
||||
copy(s.hdrBuf[:], pkt[:info.hdrLen])
|
||||
s.hdrLen = info.hdrLen
|
||||
@@ -274,7 +274,7 @@ func (c *UDPCoalescer) take() *udpSlot {
|
||||
}
|
||||
|
||||
func (c *UDPCoalescer) release(s *udpSlot) {
|
||||
s.passthrough = false
|
||||
s.verbatim = false
|
||||
s.rawPkt = nil
|
||||
clear(s.payIovs)
|
||||
s.payIovs = s.payIovs[:0]
|
||||
|
||||
@@ -411,7 +411,7 @@ func TestUDPCoalescerZeroLengthPayloadPassesThrough(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// IPv6 zero-length UDP datagram: same passthrough contract as v4.
|
||||
// IPv6 zero-length UDP datagram: same verbatim contract as v4.
|
||||
func TestUDPCoalescerZeroLengthPayloadIPv6PassesThrough(t *testing.T) {
|
||||
w := &fakeTunWriter{gsoEnabled: true}
|
||||
c := newTestUDPCoalescer(t, w)
|
||||
@@ -451,7 +451,7 @@ func TestUDPCoalescerZeroLengthMidFlowSealsAndPreservesOrder(t *testing.T) {
|
||||
}
|
||||
// The empty datagram sealed the first slot, so the trailing full packet
|
||||
// can't join it. All three emit as plain writes (the two full datagrams
|
||||
// stayed single-segment; the empty one is passthrough) in per-flow
|
||||
// stayed single-segment; the empty one is verbatim) in per-flow
|
||||
// arrival order: full, empty, full.
|
||||
if len(w.writes) != 3 || len(w.gsoWrites) != 0 {
|
||||
t.Fatalf("want 3 plain writes, got gso=%d plain=%d", len(w.gsoWrites), len(w.writes))
|
||||
|
||||
Reference in New Issue
Block a user