mirror of
https://github.com/slackhq/nebula.git
synced 2026-08-15 12:27:03 +02:00
unslop, improve the tio interface
This commit is contained in:
+1
-10
@@ -301,16 +301,7 @@ func (f *Interface) activate() error {
|
|||||||
metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
|
metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
|
||||||
|
|
||||||
for i := range f.queues {
|
for i := range f.queues {
|
||||||
caps := tio.QueueCapabilities(f.queues[i])
|
f.batchers[i] = batch.NewMultiCoalescer(f.queues[i], f.l)
|
||||||
if caps.TSO || caps.USO {
|
|
||||||
// Multi-lane: TCP gets coalesced when TSO is on, UDP when USO
|
|
||||||
// is on, everything else (and either lane disabled) falls
|
|
||||||
// through to passthrough so non-IP / non-TCP-UDP traffic still
|
|
||||||
// reaches the TUN.
|
|
||||||
f.batchers[i] = batch.NewMultiCoalescer(f.queues[i], f.l, caps.TSO, caps.USO)
|
|
||||||
} else {
|
|
||||||
f.batchers[i] = batch.NewPassthrough(f.queues[i])
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// On error the caller owns the cleanup, Control.Start cancels the service context
|
// On error the caller owns the cleanup, Control.Start cancels the service context
|
||||||
|
|||||||
@@ -8,21 +8,20 @@ import (
|
|||||||
// flowKey identifies a transport flow by {src, dst, sport, dport, family}.
|
// flowKey identifies a transport flow by {src, dst, sport, dport, family}.
|
||||||
// Comparable, so map lookups and linear scans over the slot list stay tight.
|
// Comparable, so map lookups and linear scans over the slot list stay tight.
|
||||||
// Shared by the TCP and UDP coalescers; each coalescer keeps its own
|
// Shared by the TCP and UDP coalescers; each coalescer keeps its own
|
||||||
// openSlots map, so a TCP and UDP flow on the same 5-tuple-without-proto
|
// openSlots map, so a TCP and UDP flow on the same 5-tuple-without-proto never alias.
|
||||||
// never alias.
|
|
||||||
type flowKey struct {
|
type flowKey struct {
|
||||||
src, dst [16]byte
|
src, dst [16]byte
|
||||||
sport, dport uint16
|
sport, dport uint16
|
||||||
isV6 bool
|
isV6 bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// initialSlots is the starting capacity of the slot pool. One flow per
|
// initialSlots is the starting capacity of the slot pool.
|
||||||
// packet is the worst case so this matches a typical carrier-side
|
// One flow per packet is the worst case,
|
||||||
// recvmmsg batch on the encrypted UDP socket.
|
// so this matches a typical carrier-side recvmmsg batch on the UDP socket.
|
||||||
const initialSlots = 64
|
const initialSlots = 64
|
||||||
|
|
||||||
// parsedIP is the IP-level result of parseIPPrologue. The caller layers
|
// parsedIP is the IP-level result of parseIPPrologue.
|
||||||
// L4-specific parsing (TCP / UDP) on top.
|
// The caller layers L4-specific parsing (TCP / UDP) on top.
|
||||||
type parsedIP struct {
|
type parsedIP struct {
|
||||||
fk flowKey
|
fk flowKey
|
||||||
ipHdrLen int
|
ipHdrLen int
|
||||||
@@ -92,15 +91,13 @@ func parseIPPrologue(pkt []byte, wantProto byte) (parsedIP, bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ipHeadersMatch compares the IP portion of two packet header prefixes for
|
// ipHeadersMatch compares the IP portion of two packet header prefixes for
|
||||||
// byte-for-byte equality on every field that must be identical across
|
// byte-for-byte equality on every field that must be identical across coalesced segments.
|
||||||
// coalesced segments. Size/IPID/IPCsum are masked out. The full DSCP/ECN
|
// Size/IPID/IPCsum are masked out.
|
||||||
// byte (IPv4 ToS / IPv6 traffic class) is compared, matching Linux kernel
|
// The full DSCP/ECN byte (IPv4 ToS / IPv6 traffic class) is compared, matching Linux kernel GRO:
|
||||||
// GRO: segments with differing ECN codepoints must not coalesce, otherwise
|
// segments with differing ECN codepoints must not coalesce,
|
||||||
// ORing e.g. ECT(0) with ECT(1) would fabricate a false CE (congestion)
|
// otherwise ORing e.g. ECT(0) with ECT(1) would fabricate a false CE (congestion) mark or mark a Not-ECT flow as ECN-capable.
|
||||||
// mark or mark a Not-ECT flow as ECN-capable.
|
|
||||||
//
|
//
|
||||||
// The transport (L4) portion of the header is checked separately by the
|
// The transport (L4) portion of the header is checked separately by the per-protocol matcher.
|
||||||
// per-protocol matcher.
|
|
||||||
func ipHeadersMatch(a, b []byte, isV6 bool) bool {
|
func ipHeadersMatch(a, b []byte, isV6 bool) bool {
|
||||||
if isV6 {
|
if isV6 {
|
||||||
// IPv6: byte 0 = version/TC[7:4], byte 1 = TC[3:0]/flow[19:16],
|
// IPv6: byte 0 = version/TC[7:4], byte 1 = TC[3:0]/flow[19:16],
|
||||||
@@ -145,17 +142,14 @@ type Arena struct {
|
|||||||
buf []byte
|
buf []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewArena returns an Arena with a pre-allocated backing of the given
|
// NewArena returns an Arena with a pre-allocated backing of the given capacity.
|
||||||
// capacity. Pass 0 if you don't intend to call Reserve (e.g. a test that
|
|
||||||
// only feeds the coalescer pre-made []byte packets via Commit).
|
|
||||||
func NewArena(capacity int) *Arena {
|
func NewArena(capacity int) *Arena {
|
||||||
return &Arena{buf: make([]byte, 0, capacity)}
|
return &Arena{buf: make([]byte, 0, capacity)}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reserve hands out a non-overlapping sz-byte slice from the arena. If the
|
// Reserve hands out a non-overlapping sz-byte slice from the arena.
|
||||||
// request doesn't fit the current backing, a fresh, larger backing is
|
// If the request doesn't fit the current backing, a fresh, larger backing is allocated.
|
||||||
// allocated; already-borrowed slices reference the old backing and remain
|
// Already-borrowed slices reference the old backing and remain valid until Reset.
|
||||||
// valid until Reset.
|
|
||||||
func (a *Arena) Reserve(sz int) []byte {
|
func (a *Arena) Reserve(sz int) []byte {
|
||||||
if len(a.buf)+sz > cap(a.buf) {
|
if len(a.buf)+sz > cap(a.buf) {
|
||||||
newCap := max(cap(a.buf)*2, sz)
|
newCap := max(cap(a.buf)*2, sz)
|
||||||
@@ -166,9 +160,9 @@ func (a *Arena) Reserve(sz int) []byte {
|
|||||||
return a.buf[start : start+sz : start+sz]
|
return a.buf[start : start+sz : start+sz]
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reset releases every slice handed out since the last Reset. Callers must
|
// Reset releases every slice handed out since the last Reset.
|
||||||
// not use any previously-borrowed slice after this returns. The underlying
|
// Callers must not use any previously-borrowed slice after this returns.
|
||||||
// backing array is retained so subsequent Reserves don't re-allocate.
|
// The underlying backing array is retained so subsequent Reserves don't re-allocate.
|
||||||
func (a *Arena) Reset() {
|
func (a *Arena) Reset() {
|
||||||
a.buf = a.buf[:0]
|
a.buf = a.buf[:0]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,46 +10,33 @@ import (
|
|||||||
// on the IP/L4 protocol of the packet.
|
// on the IP/L4 protocol of the packet.
|
||||||
//
|
//
|
||||||
// Lanes are processed independently: the TCP coalescer only sees TCP, the
|
// Lanes are processed independently: the TCP coalescer only sees TCP, the
|
||||||
// UDP coalescer only sees UDP, and the passthrough lane handles everything
|
// UDP coalescer only sees UDP, and the passthrough lane handles everything else.
|
||||||
// else. Per-flow arrival order is preserved because a single 5-tuple only
|
// Per-flow delivery order is preserved because a single 5-tuple only
|
||||||
// ever lands in one lane and each lane preserves its own slot order.
|
// ever lands in one lane and each lane preserves its own slot order.
|
||||||
//
|
//
|
||||||
// Cross-lane order is NOT preserved across the TCP/UDP/passthrough split.
|
// Cross-lane order is intentionally NOT preserved across the TCP/UDP/passthrough split.
|
||||||
// This is acceptable because the carrier-side recvmmsg path already
|
|
||||||
// stable-sorts by (peer, message counter) before delivering plaintext
|
|
||||||
// here, so replay-window invariants are unaffected, and apps observe
|
|
||||||
// correct per-flow ordering; which is all the IP layer guarantees anyway.
|
|
||||||
// Do not "fix" this by interleaving lane outputs at flush time; that
|
|
||||||
// negates the entire point of coalescing (each lane needs to see runs of
|
|
||||||
// adjacent same-flow packets to coalesce them).
|
|
||||||
type MultiCoalescer struct {
|
type MultiCoalescer struct {
|
||||||
tcp *TCPCoalescer
|
tcp *TCPCoalescer
|
||||||
udp *UDPCoalescer
|
udp *UDPCoalescer
|
||||||
pt *Passthrough
|
pt *Passthrough
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewMultiCoalescer builds a multi-lane batcher. tcpEnabled lets the caller
|
// NewMultiCoalescer builds a multi-lane batcher over w, based on available protocol support.
|
||||||
// opt out of TCP coalescing (e.g. when the queue can't do TSO); udpEnabled
|
func NewMultiCoalescer(w io.Writer, l *slog.Logger) RxBatcher {
|
||||||
// likewise gates UDP coalescing (only enable when USO was negotiated).
|
|
||||||
// Either lane disabled redirects its traffic into the passthrough lane.
|
|
||||||
func NewMultiCoalescer(w io.Writer, l *slog.Logger, tcpEnabled, udpEnabled bool) *MultiCoalescer {
|
|
||||||
m := &MultiCoalescer{
|
m := &MultiCoalescer{
|
||||||
pt: NewPassthrough(w),
|
pt: NewPassthrough(w),
|
||||||
}
|
}
|
||||||
if tcpEnabled {
|
m.tcp = NewTCPCoalescer(w, l)
|
||||||
m.tcp = NewTCPCoalescer(w, l)
|
m.udp = NewUDPCoalescer(w)
|
||||||
}
|
if m.tcp == nil && m.udp == nil {
|
||||||
if udpEnabled {
|
return m.pt //no offloads? Use passthrough directly.
|
||||||
m.udp = NewUDPCoalescer(w)
|
|
||||||
}
|
}
|
||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// Commit dispatches pkt to the appropriate lane based on IP version + L4 proto.
|
// Commit dispatches pkt to the appropriate lane based on IP version + L4 proto.
|
||||||
//
|
|
||||||
// On the success path the IP/TCP-or-UDP parse happens here once and the
|
// On the success path the IP/TCP-or-UDP parse happens here once and the
|
||||||
// parsed struct is handed to the lane via commitParsed so the lane doesn't
|
// parsed struct is handed to the lane via commitParsed so the lane doesn't re-walk the header.
|
||||||
// re-walk the header.
|
|
||||||
func (m *MultiCoalescer) Commit(pkt []byte) error {
|
func (m *MultiCoalescer) Commit(pkt []byte) error {
|
||||||
if len(pkt) < 20 {
|
if len(pkt) < 20 {
|
||||||
return m.pt.Commit(pkt)
|
return m.pt.Commit(pkt)
|
||||||
|
|||||||
@@ -1,17 +1,33 @@
|
|||||||
package batch
|
package batch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/slackhq/nebula/test"
|
"github.com/slackhq/nebula/test"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// newTestMultiCoalescer builds a batcher over w and asserts it really is
|
||||||
|
// multi-lane. NewMultiCoalescer collapses to a bare Passthrough when w can
|
||||||
|
// offload neither protocol, and a test that meant to exercise a lane would
|
||||||
|
// otherwise pass vacuously.
|
||||||
|
func newTestMultiCoalescer(tb testing.TB, w io.Writer) *MultiCoalescer {
|
||||||
|
tb.Helper()
|
||||||
|
b := NewMultiCoalescer(w, test.NewLogger())
|
||||||
|
m, ok := b.(*MultiCoalescer)
|
||||||
|
if !ok {
|
||||||
|
tb.Fatalf("want a *MultiCoalescer, got %T", b)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
// TestMultiCoalescerRoutesByProto confirms TCP/UDP/other land in the right
|
// TestMultiCoalescerRoutesByProto confirms TCP/UDP/other land in the right
|
||||||
// lane: TCP and UDP get coalesced when their lanes are enabled, anything
|
// lane: TCP and UDP get coalesced when their lanes are enabled, anything
|
||||||
// else (ICMP here) falls through to plain Write.
|
// else (ICMP here) falls through to plain Write.
|
||||||
func TestMultiCoalescerRoutesByProto(t *testing.T) {
|
func TestMultiCoalescerRoutesByProto(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
m := NewMultiCoalescer(w, test.NewLogger(), true, true)
|
m := newTestMultiCoalescer(t, w)
|
||||||
|
|
||||||
tcpPay := make([]byte, 1200)
|
tcpPay := make([]byte, 1200)
|
||||||
udpPay := make([]byte, 1200)
|
udpPay := make([]byte, 1200)
|
||||||
@@ -48,12 +64,15 @@ func TestMultiCoalescerRoutesByProto(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMultiCoalescerDisabledUDPFallsThrough verifies that when the UDP lane
|
// TestMultiCoalescerNoUSOFallsThrough verifies that on a queue without USO
|
||||||
// is disabled (e.g. kernel doesn't support USO), UDP packets still reach
|
// (older kernel: TSO but no GSO_UDP_L4) the UDP lane never comes up and UDP
|
||||||
// the kernel via the passthrough lane rather than being lost.
|
// packets still reach the kernel via passthrough rather than being lost.
|
||||||
func TestMultiCoalescerDisabledUDPFallsThrough(t *testing.T) {
|
func TestMultiCoalescerNoUSOFallsThrough(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true, noUSO: true}
|
||||||
m := NewMultiCoalescer(w, test.NewLogger(), true, false) // TSO on, USO off
|
m := newTestMultiCoalescer(t, w)
|
||||||
|
if m.udp != nil {
|
||||||
|
t.Fatal("UDP lane must not come up without USO")
|
||||||
|
}
|
||||||
|
|
||||||
if err := m.Commit(buildUDPv4(1000, 53, make([]byte, 800))); err != nil {
|
if err := m.Commit(buildUDPv4(1000, 53, make([]byte, 800))); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -72,10 +91,53 @@ func TestMultiCoalescerDisabledUDPFallsThrough(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMultiCoalescerDisabledTCPFallsThrough mirrors the TSO=off case.
|
// TestMultiCoalescerNoOffloadsIsPassthrough covers a queue that can't offload
|
||||||
func TestMultiCoalescerDisabledTCPFallsThrough(t *testing.T) {
|
// anything. Both lane constructors refuse, so there's nothing left to
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
// dispatch between and NewMultiCoalescer hands back the passthrough lane
|
||||||
m := NewMultiCoalescer(w, test.NewLogger(), false, true) // TSO off, USO on
|
// itself — no wrapper, no per-packet protocol demux, and every packet reaches
|
||||||
|
// the kernel in arrival order. This is the case Interface.activate used to
|
||||||
|
// special-case with a bare Passthrough.
|
||||||
|
func TestMultiCoalescerNoOffloadsIsPassthrough(t *testing.T) {
|
||||||
|
w := &fakeTunWriter{gsoEnabled: false}
|
||||||
|
m := NewMultiCoalescer(w, test.NewLogger())
|
||||||
|
|
||||||
|
if _, ok := m.(*Passthrough); !ok {
|
||||||
|
t.Fatalf("want a bare *Passthrough when neither offload is available, got %T", m)
|
||||||
|
}
|
||||||
|
pkts := [][]byte{
|
||||||
|
buildTCPv4(1000, tcpAck, make([]byte, 1200)),
|
||||||
|
buildUDPv4(1000, 53, make([]byte, 800)),
|
||||||
|
buildTCPv4(2200, tcpAck, make([]byte, 1200)),
|
||||||
|
}
|
||||||
|
for _, p := range pkts {
|
||||||
|
if err := m.Commit(p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := m.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(w.gsoWrites) != 0 {
|
||||||
|
t.Errorf("no GSO writes possible, got %d", len(w.gsoWrites))
|
||||||
|
}
|
||||||
|
if len(w.writes) != len(pkts) {
|
||||||
|
t.Fatalf("want %d plain writes, got %d", len(pkts), len(w.writes))
|
||||||
|
}
|
||||||
|
// One lane for everything means arrival order survives end to end.
|
||||||
|
for i, want := range pkts {
|
||||||
|
if !bytes.Equal(w.writes[i], want) {
|
||||||
|
t.Errorf("write %d out of order or corrupt", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMultiCoalescerNoTSOFallsThrough mirrors the no-TSO case.
|
||||||
|
func TestMultiCoalescerNoTSOFallsThrough(t *testing.T) {
|
||||||
|
w := &fakeTunWriter{gsoEnabled: true, noTSO: true}
|
||||||
|
m := newTestMultiCoalescer(t, w)
|
||||||
|
if m.tcp != nil {
|
||||||
|
t.Fatal("TCP lane must not come up without TSO")
|
||||||
|
}
|
||||||
|
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
if err := m.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
if err := m.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
||||||
|
|||||||
@@ -27,13 +27,13 @@ const tcpCoalesceMaxSegs = 64
|
|||||||
// into. IPv6 (40) + TCP with full options (60) = 100 bytes.
|
// into. IPv6 (40) + TCP with full options (60) = 100 bytes.
|
||||||
const tcpCoalesceHdrCap = 100
|
const tcpCoalesceHdrCap = 100
|
||||||
|
|
||||||
// coalesceSlot is one entry in the coalescer's ordered event queue. When
|
// coalesceSlot is one entry in the coalescer's ordered event queue.
|
||||||
// passthrough is true the slot holds a single borrowed packet that must be
|
// When passthrough is true the slot holds a single borrowed packet that must be
|
||||||
// emitted verbatim (non-TCP, non-admissible TCP, or oversize seed). When
|
// emitted verbatim (non-TCP, non-admissible TCP, or oversize seed).
|
||||||
// passthrough is false the slot is an in-progress coalesced superpacket:
|
// When passthrough is false the slot is an in-progress coalesced superpacket.
|
||||||
// hdrBuf is a mutable copy of the seed's IP+TCP header (we patch total
|
// hdrBuf is a mutable copy of the seed's IP+TCP header
|
||||||
// length and pseudo-header partial at flush), and payIovs are *borrowed*
|
// (we patch total length and pseudo-header partial at flush)
|
||||||
// slices from the caller's plaintext buffers — no payload is ever copied.
|
// payIovs are *borrowed* slices from the caller's plaintext buffers.
|
||||||
// The caller (listenOut) must keep those buffers alive until Flush.
|
// The caller (listenOut) must keep those buffers alive until Flush.
|
||||||
type coalesceSlot struct {
|
type coalesceSlot struct {
|
||||||
passthrough bool
|
passthrough bool
|
||||||
@@ -50,20 +50,18 @@ type coalesceSlot struct {
|
|||||||
nextSeq uint32
|
nextSeq uint32
|
||||||
// sealed marks the chain permanently closed: the last-accepted segment had PSH or was sub-gsoSize,
|
// sealed marks the chain permanently closed: the last-accepted segment had PSH or was sub-gsoSize,
|
||||||
// so no append or flush-time merge may follow.
|
// so no append or flush-time merge may follow.
|
||||||
// Distinct from mere eviction out of openSlots (e.g. on seq mismatch),
|
// Distinct from eviction out of openSlots (e.g. on seq mismatch),
|
||||||
// which leaves sealed=false so reorderForFlush can still merge the slot.
|
// which leaves sealed=false so reorderForFlush can still merge the slot.
|
||||||
sealed bool
|
sealed bool
|
||||||
payIovs [][]byte
|
payIovs [][]byte
|
||||||
}
|
}
|
||||||
|
|
||||||
// TCPCoalescer accumulates adjacent in-flow TCP data segments across
|
// TCPCoalescer accumulates adjacent in-flow TCP data segments across multiple concurrent flows
|
||||||
// multiple concurrent flows and emits each flow's run as a single TSO
|
// and emits each flow's run as a single TSO superpacket via tio.GSOWriter.
|
||||||
// superpacket via tio.GSOWriter. All output — coalesced or not — is
|
// All output, coalesced or not, is deferred until Flush so arrival order is preserved on the wire.
|
||||||
// deferred until Flush so arrival order is preserved on the wire. Owns
|
// Owns no locks; one coalescer per TUN write queue.
|
||||||
// no locks; one coalescer per TUN write queue.
|
|
||||||
type TCPCoalescer struct {
|
type TCPCoalescer struct {
|
||||||
plainW io.Writer
|
w tio.GSOWriter
|
||||||
gsoW tio.GSOWriter // nil when the queue doesn't support TSO
|
|
||||||
|
|
||||||
// slots is the ordered event queue. Flush walks it once and emits each
|
// slots is the ordered event queue. Flush walks it once and emits each
|
||||||
// entry as either a WriteGSO (coalesced) or a plainW.Write (passthrough).
|
// entry as either a WriteGSO (coalesced) or a plainW.Write (passthrough).
|
||||||
@@ -84,18 +82,19 @@ type TCPCoalescer struct {
|
|||||||
l *slog.Logger
|
l *slog.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewTCPCoalescer wraps w, returning nil if w can't accept GSO_TCP writes.
|
||||||
func NewTCPCoalescer(w io.Writer, l *slog.Logger) *TCPCoalescer {
|
func NewTCPCoalescer(w io.Writer, l *slog.Logger) *TCPCoalescer {
|
||||||
c := &TCPCoalescer{
|
gw, ok := tio.SupportsGSO(w, tio.GSOProtoTCP)
|
||||||
plainW: w,
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &TCPCoalescer{
|
||||||
|
w: gw,
|
||||||
slots: make([]*coalesceSlot, 0, initialSlots),
|
slots: make([]*coalesceSlot, 0, initialSlots),
|
||||||
openSlots: make(map[flowKey]*coalesceSlot, initialSlots),
|
openSlots: make(map[flowKey]*coalesceSlot, initialSlots),
|
||||||
pool: make([]*coalesceSlot, 0, initialSlots),
|
pool: make([]*coalesceSlot, 0, initialSlots),
|
||||||
l: l,
|
l: l,
|
||||||
}
|
}
|
||||||
if gw, ok := tio.SupportsGSO(w, tio.GSOProtoTCP); ok {
|
|
||||||
c.gsoW = gw
|
|
||||||
}
|
|
||||||
return c
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// parsedTCP holds the fields extracted from a single parse so later steps
|
// parsedTCP holds the fields extracted from a single parse so later steps
|
||||||
@@ -111,8 +110,7 @@ type parsedTCP struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parseTCPBase extracts the flow key and IP/TCP offsets for any TCP packet,
|
// parseTCPBase extracts the flow key and IP/TCP offsets for any TCP packet,
|
||||||
// regardless of whether it's admissible for coalescing. Returns ok=false
|
// regardless of whether it's admissible for coalescing. Returns ok=false for non-TCP or malformed input.
|
||||||
// for non-TCP or malformed input.
|
|
||||||
// Accepts IPv4 (no options or fragmentation) and IPv6 (no extension headers).
|
// Accepts IPv4 (no options or fragmentation) and IPv6 (no extension headers).
|
||||||
func parseTCPBase(pkt []byte) (parsedTCP, bool) {
|
func parseTCPBase(pkt []byte) (parsedTCP, bool) {
|
||||||
var p parsedTCP
|
var p parsedTCP
|
||||||
@@ -169,10 +167,6 @@ func (p parsedTCP) coalesceable() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *TCPCoalescer) Commit(pkt []byte) error {
|
func (c *TCPCoalescer) Commit(pkt []byte) error {
|
||||||
if c.gsoW == nil {
|
|
||||||
c.addPassthrough(pkt)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
info, ok := parseTCPBase(pkt)
|
info, ok := parseTCPBase(pkt)
|
||||||
if !ok {
|
if !ok {
|
||||||
c.addPassthrough(pkt)
|
c.addPassthrough(pkt)
|
||||||
@@ -186,10 +180,6 @@ func (c *TCPCoalescer) Commit(pkt []byte) error {
|
|||||||
// Used by MultiCoalescer.Commit to avoid re-walking the IP/TCP header
|
// Used by MultiCoalescer.Commit to avoid re-walking the IP/TCP header
|
||||||
// after the dispatcher has already done so.
|
// after the dispatcher has already done so.
|
||||||
func (c *TCPCoalescer) commitParsed(pkt []byte, info parsedTCP) error {
|
func (c *TCPCoalescer) commitParsed(pkt []byte, info parsedTCP) error {
|
||||||
if c.gsoW == nil {
|
|
||||||
c.addPassthrough(pkt)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !info.coalesceable() {
|
if !info.coalesceable() {
|
||||||
// TCP but not admissible (SYN/FIN/RST/URG/CWR or zero-payload).
|
// TCP but not admissible (SYN/FIN/RST/URG/CWR or zero-payload).
|
||||||
// Seal this flow's open slot so later in-flow packets don't extend
|
// Seal this flow's open slot so later in-flow packets don't extend
|
||||||
@@ -240,7 +230,7 @@ func (c *TCPCoalescer) Flush() error {
|
|||||||
for _, s := range c.slots {
|
for _, s := range c.slots {
|
||||||
var err error
|
var err error
|
||||||
if s.passthrough {
|
if s.passthrough {
|
||||||
_, err = c.plainW.Write(s.rawPkt)
|
_, err = c.w.Write(s.rawPkt)
|
||||||
} else {
|
} else {
|
||||||
err = c.flushSlot(s)
|
err = c.flushSlot(s)
|
||||||
}
|
}
|
||||||
@@ -266,7 +256,7 @@ func (c *TCPCoalescer) addPassthrough(pkt []byte) {
|
|||||||
|
|
||||||
func (c *TCPCoalescer) seed(pkt []byte, info parsedTCP) {
|
func (c *TCPCoalescer) seed(pkt []byte, info parsedTCP) {
|
||||||
if info.hdrLen > tcpCoalesceHdrCap || info.hdrLen+info.payLen > tcpCoalesceBufSize {
|
if info.hdrLen > tcpCoalesceHdrCap || info.hdrLen+info.payLen > tcpCoalesceBufSize {
|
||||||
// Pathological shape — can't fit our scratch, emit as-is.
|
// Pathological shape. Can't fit our scratch, emit as-is.
|
||||||
c.addPassthrough(pkt)
|
c.addPassthrough(pkt)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -317,8 +307,8 @@ func (c *TCPCoalescer) canAppend(s *coalesceSlot, pkt []byte, info parsedTCP) bo
|
|||||||
if s.hdrLen+s.totalPay+info.payLen > tcpCoalesceBufSize {
|
if s.hdrLen+s.totalPay+info.payLen > tcpCoalesceBufSize {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
// ECE state must be stable across a burst — receivers expect the
|
// ECE state must be stable across a burst.
|
||||||
// flag set on every segment of a CE-echoing window or none.
|
// Receivers expect the flag set on every segment of a CE-echoing window or none.
|
||||||
seedFlags := s.hdrBuf[s.ipHdrLen+13]
|
seedFlags := s.hdrBuf[s.ipHdrLen+13]
|
||||||
if (seedFlags^info.flags)&tcpFlagEce != 0 {
|
if (seedFlags^info.flags)&tcpFlagEce != 0 {
|
||||||
return false
|
return false
|
||||||
@@ -389,7 +379,7 @@ func (c *TCPCoalescer) flushSlot(s *coalesceSlot) error {
|
|||||||
tcsum := s.ipHdrLen + 16
|
tcsum := s.ipHdrLen + 16
|
||||||
binary.BigEndian.PutUint16(hdr[tcsum:tcsum+2], foldOnceNoInvert(psum))
|
binary.BigEndian.PutUint16(hdr[tcsum:tcsum+2], foldOnceNoInvert(psum))
|
||||||
|
|
||||||
return c.gsoW.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoTCP)
|
return c.w.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoTCP)
|
||||||
}
|
}
|
||||||
|
|
||||||
// headersMatch compares two IP+TCP header prefixes for byte-for-byte
|
// headersMatch compares two IP+TCP header prefixes for byte-for-byte
|
||||||
@@ -421,9 +411,9 @@ func headersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// reorderForFlush neutralizes wire-side reorder that the rxOrder buffer
|
// reorderForFlush neutralizes wire-side reorder that the rxOrder buffer
|
||||||
// couldn't catch (anything crossing a recvmmsg batch boundary). Without
|
// couldn't catch (anything crossing a recvmmsg batch boundary).
|
||||||
// this pass a small wire reorder — counter 250 arriving in batch K when
|
// Without this pass a small wire reorder, counter 250 arriving in batch K when
|
||||||
// 200..249 are coming in batch K+1 — would seed an out-of-seq slot first
|
// 200..249 are coming in batch K+1, would seed an out-of-seq slot first
|
||||||
// and emit it ahead of the lower-seq slot, manifesting at the inner TCP
|
// and emit it ahead of the lower-seq slot, manifesting at the inner TCP
|
||||||
// receiver as a much larger reorder than the wire actually had.
|
// receiver as a much larger reorder than the wire actually had.
|
||||||
//
|
//
|
||||||
@@ -434,7 +424,7 @@ func headersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
|
|||||||
// 2. Sweep once and merge adjacent same-flow slots whose ranges are now
|
// 2. Sweep once and merge adjacent same-flow slots whose ranges are now
|
||||||
// contiguous AND whose tail is gsoSize-aligned. The tail constraint
|
// contiguous AND whose tail is gsoSize-aligned. The tail constraint
|
||||||
// matters because the kernel TSO splitter chops at gsoSize from the
|
// matters because the kernel TSO splitter chops at gsoSize from the
|
||||||
// start of the merged payload — a short segment in the middle would
|
// start of the merged payload. A short segment in the middle would
|
||||||
// desynchronize every later segment.
|
// desynchronize every later segment.
|
||||||
//
|
//
|
||||||
// Passthrough slots act as barriers: the merge check skips them on either
|
// Passthrough slots act as barriers: the merge check skips them on either
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ func buildICMPv4() []byte {
|
|||||||
// between batches, and reports per-packet cost.
|
// between batches, and reports per-packet cost.
|
||||||
func runCommitBench(b *testing.B, pkts [][]byte, batchSize int) {
|
func runCommitBench(b *testing.B, pkts [][]byte, batchSize int) {
|
||||||
b.Helper()
|
b.Helper()
|
||||||
c := NewTCPCoalescer(nopTunWriter{}, test.NewLogger())
|
c := newTestTCPCoalescer(b, nopTunWriter{})
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.SetBytes(int64(len(pkts[0])))
|
b.SetBytes(int64(len(pkts[0])))
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
@@ -140,7 +140,7 @@ func BenchmarkCommitNonCoalesceableTCP(b *testing.B) {
|
|||||||
// is the bench that shows the savings of skipping the lane's re-parse.
|
// is the bench that shows the savings of skipping the lane's re-parse.
|
||||||
func runMultiCommitBench(b *testing.B, pkts [][]byte, batchSize int) {
|
func runMultiCommitBench(b *testing.B, pkts [][]byte, batchSize int) {
|
||||||
b.Helper()
|
b.Helper()
|
||||||
m := NewMultiCoalescer(nopTunWriter{}, test.NewLogger(), true, true)
|
m := NewMultiCoalescer(nopTunWriter{}, test.NewLogger())
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.SetBytes(int64(len(pkts[0])))
|
b.SetBytes(int64(len(pkts[0])))
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package batch
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
|
"io"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/slackhq/nebula/overlay/tio"
|
"github.com/slackhq/nebula/overlay/tio"
|
||||||
@@ -11,8 +12,13 @@ import (
|
|||||||
// fakeTunWriter records plain Writes and WriteGSO calls without touching a
|
// fakeTunWriter records plain Writes and WriteGSO calls without touching a
|
||||||
// real TUN fd. WriteGSO records the IP header, transport header, and
|
// real TUN fd. WriteGSO records the IP header, transport header, and
|
||||||
// borrowed payload fragments separately so tests can inspect each.
|
// borrowed payload fragments separately so tests can inspect each.
|
||||||
|
// noTSO / noUSO withhold one offload from an otherwise GSO-capable writer, so
|
||||||
|
// tests can build the half-capable queues real kernels hand us (USO needs a
|
||||||
|
// newer kernel than TSO).
|
||||||
type fakeTunWriter struct {
|
type fakeTunWriter struct {
|
||||||
gsoEnabled bool
|
gsoEnabled bool
|
||||||
|
noTSO bool
|
||||||
|
noUSO bool
|
||||||
writes [][]byte
|
writes [][]byte
|
||||||
gsoWrites []fakeGSOWrite
|
gsoWrites []fakeGSOWrite
|
||||||
}
|
}
|
||||||
@@ -79,7 +85,7 @@ func (w *fakeTunWriter) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte,
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (w *fakeTunWriter) Capabilities() tio.Capabilities {
|
func (w *fakeTunWriter) Capabilities() tio.Capabilities {
|
||||||
return tio.Capabilities{TSO: w.gsoEnabled, USO: w.gsoEnabled}
|
return tio.Capabilities{TSO: w.gsoEnabled && !w.noTSO, USO: w.gsoEnabled && !w.noUSO}
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildTCPv4 constructs a minimal IPv4+TCP packet with the given payload,
|
// buildTCPv4 constructs a minimal IPv4+TCP packet with the given payload,
|
||||||
@@ -126,28 +132,43 @@ const (
|
|||||||
tcpAckPsh = tcpAck | tcpPsh
|
tcpAckPsh = tcpAck | tcpPsh
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCoalescerPassthroughWhenGSOUnavailable(t *testing.T) {
|
// newTestTCPCoalescer builds a coalescer over w and fails the test if w can't
|
||||||
w := &fakeTunWriter{gsoEnabled: false}
|
// do TSO. Every test but TestNewTCPCoalescerRefusesWhenGSOUnavailable wants the
|
||||||
|
// GSO path, and the constructor now hands back a nil coalescer otherwise.
|
||||||
|
func newTestTCPCoalescer(tb testing.TB, w io.Writer) *TCPCoalescer {
|
||||||
|
tb.Helper()
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := NewTCPCoalescer(w, test.NewLogger())
|
||||||
pkt := buildTCPv4(1000, tcpAck, []byte("hello"))
|
if c == nil {
|
||||||
if err := c.Commit(pkt); err != nil {
|
tb.Fatal("NewTCPCoalescer: writer does not support TSO")
|
||||||
t.Fatal(err)
|
|
||||||
}
|
}
|
||||||
// No sync write — passthrough is deferred to Flush.
|
return c
|
||||||
if len(w.writes) != 0 || len(w.gsoWrites) != 0 {
|
}
|
||||||
t.Fatalf("no Add-time writes: got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
|
||||||
|
// TestNewTCPCoalescerRefusesWhenGSOUnavailable pins the constructor
|
||||||
|
// precondition: no TSO, no coalescer. There's no degraded mode — the caller
|
||||||
|
// (MultiCoalescer) sends TCP down the passthrough lane instead.
|
||||||
|
func TestNewTCPCoalescerRefusesWhenGSOUnavailable(t *testing.T) {
|
||||||
|
if c := NewTCPCoalescer(&fakeTunWriter{gsoEnabled: false}, test.NewLogger()); c != nil {
|
||||||
|
t.Fatalf("want nil for a non-TSO writer, got %v", c)
|
||||||
}
|
}
|
||||||
if err := c.Flush(); err != nil {
|
// A writer that isn't a GSOWriter at all is refused the same way.
|
||||||
t.Fatal(err)
|
if c := NewTCPCoalescer(&plainOnlyWriter{}, test.NewLogger()); c != nil {
|
||||||
}
|
t.Fatalf("want nil for a plain writer, got %v", c)
|
||||||
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
|
|
||||||
t.Fatalf("want single plain write, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// plainOnlyWriter is an io.Writer with no GSO support at all — the
|
||||||
|
// single-packet Queue shape.
|
||||||
|
type plainOnlyWriter struct{ writes int }
|
||||||
|
|
||||||
|
func (w *plainOnlyWriter) Write(p []byte) (int, error) {
|
||||||
|
w.writes++
|
||||||
|
return len(p), nil
|
||||||
|
}
|
||||||
|
|
||||||
func TestCoalescerNonTCPPassthrough(t *testing.T) {
|
func TestCoalescerNonTCPPassthrough(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pkt := make([]byte, 28)
|
pkt := make([]byte, 28)
|
||||||
pkt[0] = 0x45
|
pkt[0] = 0x45
|
||||||
binary.BigEndian.PutUint16(pkt[2:4], 28)
|
binary.BigEndian.PutUint16(pkt[2:4], 28)
|
||||||
@@ -167,7 +188,7 @@ func TestCoalescerNonTCPPassthrough(t *testing.T) {
|
|||||||
|
|
||||||
func TestCoalescerSeedThenFlushAlone(t *testing.T) {
|
func TestCoalescerSeedThenFlushAlone(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pkt := buildTCPv4(1000, tcpAck, make([]byte, 1000))
|
pkt := buildTCPv4(1000, tcpAck, make([]byte, 1000))
|
||||||
if err := c.Commit(pkt); err != nil {
|
if err := c.Commit(pkt); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -194,7 +215,7 @@ func TestCoalescerSeedThenFlushAlone(t *testing.T) {
|
|||||||
|
|
||||||
func TestCoalescerCoalescesAdjacentACKs(t *testing.T) {
|
func TestCoalescerCoalescesAdjacentACKs(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -234,7 +255,7 @@ func TestCoalescerCoalescesAdjacentACKs(t *testing.T) {
|
|||||||
|
|
||||||
func TestCoalescerRejectsSeqGap(t *testing.T) {
|
func TestCoalescerRejectsSeqGap(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -253,7 +274,7 @@ func TestCoalescerRejectsSeqGap(t *testing.T) {
|
|||||||
|
|
||||||
func TestCoalescerRejectsFlagMismatch(t *testing.T) {
|
func TestCoalescerRejectsFlagMismatch(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -274,7 +295,7 @@ func TestCoalescerRejectsFlagMismatch(t *testing.T) {
|
|||||||
|
|
||||||
func TestCoalescerRejectsFIN(t *testing.T) {
|
func TestCoalescerRejectsFIN(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
fin := buildTCPv4(1000, tcpAck|tcpFin, []byte("x"))
|
fin := buildTCPv4(1000, tcpAck|tcpFin, []byte("x"))
|
||||||
if err := c.Commit(fin); err != nil {
|
if err := c.Commit(fin); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -290,7 +311,7 @@ func TestCoalescerRejectsFIN(t *testing.T) {
|
|||||||
|
|
||||||
func TestCoalescerShortLastSegmentClosesChain(t *testing.T) {
|
func TestCoalescerShortLastSegmentClosesChain(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
full := make([]byte, 1200)
|
full := make([]byte, 1200)
|
||||||
half := make([]byte, 500)
|
half := make([]byte, 500)
|
||||||
if err := c.Commit(buildTCPv4(1000, tcpAck, full)); err != nil {
|
if err := c.Commit(buildTCPv4(1000, tcpAck, full)); err != nil {
|
||||||
@@ -325,7 +346,7 @@ func TestCoalescerShortLastSegmentClosesChain(t *testing.T) {
|
|||||||
|
|
||||||
func TestCoalescerPSHFinalizesChain(t *testing.T) {
|
func TestCoalescerPSHFinalizesChain(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -355,7 +376,7 @@ func TestCoalescerPSHFinalizesChain(t *testing.T) {
|
|||||||
// coalescer drops it the sender's push signal never reaches the receiver.
|
// coalescer drops it the sender's push signal never reaches the receiver.
|
||||||
func TestCoalescerPropagatesPSHFromAppended(t *testing.T) {
|
func TestCoalescerPropagatesPSHFromAppended(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
// Seed has no PSH; second segment carries PSH and seals the chain.
|
// Seed has no PSH; second segment carries PSH and seals the chain.
|
||||||
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
||||||
@@ -383,7 +404,7 @@ func TestCoalescerPropagatesPSHFromAppended(t *testing.T) {
|
|||||||
|
|
||||||
func TestCoalescerRejectsDifferentFlow(t *testing.T) {
|
func TestCoalescerRejectsDifferentFlow(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
p1 := buildTCPv4(1000, tcpAck, pay)
|
p1 := buildTCPv4(1000, tcpAck, pay)
|
||||||
p2 := buildTCPv4(2200, tcpAck, pay)
|
p2 := buildTCPv4(2200, tcpAck, pay)
|
||||||
@@ -405,7 +426,7 @@ func TestCoalescerRejectsDifferentFlow(t *testing.T) {
|
|||||||
|
|
||||||
func TestCoalescerRejectsIPOptions(t *testing.T) {
|
func TestCoalescerRejectsIPOptions(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 500)
|
pay := make([]byte, 500)
|
||||||
pkt := buildTCPv4(1000, tcpAck, pay)
|
pkt := buildTCPv4(1000, tcpAck, pay)
|
||||||
// Bump IHL to 6 to simulate 4 bytes of IP options. Don't actually add
|
// Bump IHL to 6 to simulate 4 bytes of IP options. Don't actually add
|
||||||
@@ -425,7 +446,7 @@ func TestCoalescerRejectsIPOptions(t *testing.T) {
|
|||||||
|
|
||||||
func TestCoalescerCapBySegments(t *testing.T) {
|
func TestCoalescerCapBySegments(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 512)
|
pay := make([]byte, 512)
|
||||||
seq := uint32(1000)
|
seq := uint32(1000)
|
||||||
for i := 0; i < tcpCoalesceMaxSegs+5; i++ {
|
for i := 0; i < tcpCoalesceMaxSegs+5; i++ {
|
||||||
@@ -449,7 +470,7 @@ func TestCoalescerCapBySegments(t *testing.T) {
|
|||||||
// flows coalesce independently in a single Flush.
|
// flows coalesce independently in a single Flush.
|
||||||
func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
|
func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
|
|
||||||
// Flow A: sport 1000. Flow B: sport 3000.
|
// Flow A: sport 1000. Flow B: sport 3000.
|
||||||
@@ -506,7 +527,7 @@ func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
|
|||||||
// writing passthrough packets synchronously.
|
// writing passthrough packets synchronously.
|
||||||
func TestCoalescerPreservesArrivalOrder(t *testing.T) {
|
func TestCoalescerPreservesArrivalOrder(t *testing.T) {
|
||||||
w := &orderedFakeWriter{gsoEnabled: true}
|
w := &orderedFakeWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
// Sequence: coalesceable TCP, ICMP (passthrough), coalesceable TCP on
|
// Sequence: coalesceable TCP, ICMP (passthrough), coalesceable TCP on
|
||||||
// a different flow. Expected emit order: gso(X), plain(ICMP), gso(Y).
|
// a different flow. Expected emit order: gso(X), plain(ICMP), gso(Y).
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
@@ -574,7 +595,7 @@ func stringSliceEq(a, b []string) bool {
|
|||||||
// packet (SYN) mid-flow only flushes its own flow, not others.
|
// packet (SYN) mid-flow only flushes its own flow, not others.
|
||||||
func TestCoalescerInterleavedFlowsPreserveOrdering(t *testing.T) {
|
func TestCoalescerInterleavedFlowsPreserveOrdering(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
|
|
||||||
// Flow A two segments.
|
// Flow A two segments.
|
||||||
@@ -679,7 +700,7 @@ func buildTCPv6(tcLow byte, seq uint32, flags byte, payload []byte) []byte {
|
|||||||
// retains ECE on the wire.
|
// retains ECE on the wire.
|
||||||
func TestCoalescerCoalescesEceFlow(t *testing.T) {
|
func TestCoalescerCoalescesEceFlow(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
flags := byte(tcpAck | tcpEce)
|
flags := byte(tcpAck | tcpEce)
|
||||||
if err := c.Commit(buildTCPv4(1000, flags, pay)); err != nil {
|
if err := c.Commit(buildTCPv4(1000, flags, pay)); err != nil {
|
||||||
@@ -708,7 +729,7 @@ func TestCoalescerCoalescesEceFlow(t *testing.T) {
|
|||||||
// in-flow segment seeds a new slot rather than extending the prior burst.
|
// in-flow segment seeds a new slot rather than extending the prior burst.
|
||||||
func TestCoalescerCwrSealsFlow(t *testing.T) {
|
func TestCoalescerCwrSealsFlow(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -741,7 +762,7 @@ func TestCoalescerCwrSealsFlow(t *testing.T) {
|
|||||||
// a CE-echoing window or none.
|
// a CE-echoing window or none.
|
||||||
func TestCoalescerEceMismatchReseeds(t *testing.T) {
|
func TestCoalescerEceMismatchReseeds(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
if err := c.Commit(buildTCPv4(1000, tcpAck|tcpEce, pay)); err != nil {
|
if err := c.Commit(buildTCPv4(1000, tcpAck|tcpEce, pay)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -771,7 +792,7 @@ func TestCoalescerEceMismatchReseeds(t *testing.T) {
|
|||||||
// across the whole burst.
|
// across the whole burst.
|
||||||
func TestCoalescerDifferingECNReseeds(t *testing.T) {
|
func TestCoalescerDifferingECNReseeds(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
if err := c.Commit(buildTCPv4WithToS(ecnECT0, 1000, tcpAck, pay)); err != nil {
|
if err := c.Commit(buildTCPv4WithToS(ecnECT0, 1000, tcpAck, pay)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -816,7 +837,7 @@ func TestCoalescerDifferingECNReseeds(t *testing.T) {
|
|||||||
// codepoint, and neither may end up CE-marked.
|
// codepoint, and neither may end up CE-marked.
|
||||||
func TestCoalescerECT0ThenECT1NoCE(t *testing.T) {
|
func TestCoalescerECT0ThenECT1NoCE(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
if err := c.Commit(buildTCPv4WithToS(ecnECT0, 1000, tcpAck, pay)); err != nil {
|
if err := c.Commit(buildTCPv4WithToS(ecnECT0, 1000, tcpAck, pay)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -846,7 +867,7 @@ func TestCoalescerECT0ThenECT1NoCE(t *testing.T) {
|
|||||||
// six DSCP bits must match too.
|
// six DSCP bits must match too.
|
||||||
func TestCoalescerDscpMismatchReseeds(t *testing.T) {
|
func TestCoalescerDscpMismatchReseeds(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
// Same ECN (Not-ECT), different DSCP (0x10 vs 0x20 in upper 6 bits).
|
// Same ECN (Not-ECT), different DSCP (0x10 vs 0x20 in upper 6 bits).
|
||||||
tosA := byte(0x10<<2) | ecnNotECT
|
tosA := byte(0x10<<2) | ecnNotECT
|
||||||
@@ -869,7 +890,7 @@ func TestCoalescerDscpMismatchReseeds(t *testing.T) {
|
|||||||
// TestCoalescerCoalescesEceFlow.
|
// TestCoalescerCoalescesEceFlow.
|
||||||
func TestCoalescerIPv6CoalescesEceFlow(t *testing.T) {
|
func TestCoalescerIPv6CoalescesEceFlow(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
flags := byte(tcpAck | tcpEce)
|
flags := byte(tcpAck | tcpEce)
|
||||||
if err := c.Commit(buildTCPv6(0, 1000, flags, pay)); err != nil {
|
if err := c.Commit(buildTCPv6(0, 1000, flags, pay)); err != nil {
|
||||||
@@ -900,7 +921,7 @@ func TestCoalescerIPv6CoalescesEceFlow(t *testing.T) {
|
|||||||
// seen had the wire never reordered.
|
// seen had the wire never reordered.
|
||||||
func TestCoalescerSortsReorderedSeedsAndMerges(t *testing.T) {
|
func TestCoalescerSortsReorderedSeedsAndMerges(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
// Arrival order: seq 1000, 3400, 2200. The 3400 seeds a separate slot
|
// Arrival order: seq 1000, 3400, 2200. The 3400 seeds a separate slot
|
||||||
// because 3400 != nextSeq=2200, then 2200 fails to extend the 3400 slot
|
// because 3400 != nextSeq=2200, then 2200 fails to extend the 3400 slot
|
||||||
@@ -936,7 +957,7 @@ func TestCoalescerSortsReorderedSeedsAndMerges(t *testing.T) {
|
|||||||
// without any cross-flow contamination.
|
// without any cross-flow contamination.
|
||||||
func TestCoalescerSortAcrossFlowsMergesEachIndependently(t *testing.T) {
|
func TestCoalescerSortAcrossFlowsMergesEachIndependently(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
// Flow A (sport 1000) seq 100, 1300; flow B (sport 3000) seq 500, 1700.
|
// Flow A (sport 1000) seq 100, 1300; flow B (sport 3000) seq 500, 1700.
|
||||||
// Arrival: A.1300, B.1700, A.100, B.500 — every flow reordered.
|
// Arrival: A.1300, B.1700, A.100, B.500 — every flow reordered.
|
||||||
@@ -987,7 +1008,7 @@ func TestCoalescerSortAcrossFlowsMergesEachIndependently(t *testing.T) {
|
|||||||
// boundary by an arbitrary number of segments.
|
// boundary by an arbitrary number of segments.
|
||||||
func TestCoalescerSortKeepsPSHBoundary(t *testing.T) {
|
func TestCoalescerSortKeepsPSHBoundary(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
// Seq 1000 (no PSH) + 2200 (PSH) → seal one slot with PSH set.
|
// Seq 1000 (no PSH) + 2200 (PSH) → seal one slot with PSH set.
|
||||||
// Seq 3400 (no PSH) is contiguous to 3400 from seq 2200+1200; without
|
// Seq 3400 (no PSH) is contiguous to 3400 from seq 2200+1200; without
|
||||||
@@ -1015,7 +1036,7 @@ func TestCoalescerSortKeepsPSHBoundary(t *testing.T) {
|
|||||||
// is sorted/merged independently.
|
// is sorted/merged independently.
|
||||||
func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
|
func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
// First two segments seed S1 (then a 3400 reorder seeds S2).
|
// First two segments seed S1 (then a 3400 reorder seeds S2).
|
||||||
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
|
||||||
@@ -1049,7 +1070,7 @@ func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
|
|||||||
// 0x30, so ipHeadersMatch (comparing byte 1 fully) still splits them.
|
// 0x30, so ipHeadersMatch (comparing byte 1 fully) still splits them.
|
||||||
func TestCoalescerIPv6DifferingECNReseeds(t *testing.T) {
|
func TestCoalescerIPv6DifferingECNReseeds(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
// tcLow is the low 4 bits of TC; ECN occupies the bottom 2 of those.
|
// tcLow is the low 4 bits of TC; ECN occupies the bottom 2 of those.
|
||||||
if err := c.Commit(buildTCPv6(ecnECT0, 1000, tcpAck, pay)); err != nil {
|
if err := c.Commit(buildTCPv6(ecnECT0, 1000, tcpAck, pay)); err != nil {
|
||||||
@@ -1124,7 +1145,7 @@ func TestSortRunZeroAllocs(t *testing.T) {
|
|||||||
// synthesize it from the seal bool.
|
// synthesize it from the seal bool.
|
||||||
func TestCoalescerMergeShortTailDoesNotFabricatePSH(t *testing.T) {
|
func TestCoalescerMergeShortTailDoesNotFabricatePSH(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
short := make([]byte, 600)
|
short := make([]byte, 600)
|
||||||
// Arrival: seq 3400 (full), 4600 (short, seals the slot), then the
|
// Arrival: seq 3400 (full), 4600 (short, seals the slot), then the
|
||||||
@@ -1162,7 +1183,7 @@ func TestCoalescerMergeShortTailDoesNotFabricatePSH(t *testing.T) {
|
|||||||
// source slot's tail really carried PSH, the merged header must keep it.
|
// source slot's tail really carried PSH, the merged header must keep it.
|
||||||
func TestCoalescerMergePreservesRealPSH(t *testing.T) {
|
func TestCoalescerMergePreservesRealPSH(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewTCPCoalescer(w, test.NewLogger())
|
c := newTestTCPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
short := make([]byte, 600)
|
short := make([]byte, 600)
|
||||||
if err := c.Commit(buildTCPv4(3400, tcpAck, pay)); err != nil {
|
if err := c.Commit(buildTCPv4(3400, tcpAck, pay)); err != nil {
|
||||||
|
|||||||
@@ -36,44 +36,35 @@ type udpSlot struct {
|
|||||||
numSeg int
|
numSeg int
|
||||||
totalPay int
|
totalPay int
|
||||||
// sealed closes the chain: set when a sub-gsoSize segment is appended
|
// sealed closes the chain: set when a sub-gsoSize segment is appended
|
||||||
// (kernel UDP-GSO requires every segment but the last to be exactly
|
// (kernel UDP-GSO requires every segment but the last to be exactly gsoSize)
|
||||||
// gsoSize) or when limits are hit. No further appends after.
|
// or when limits are hit. No further appends after.
|
||||||
sealed bool
|
sealed bool
|
||||||
payIovs [][]byte
|
payIovs [][]byte
|
||||||
}
|
}
|
||||||
|
|
||||||
// UDPCoalescer accumulates adjacent in-flow UDP datagrams across multiple
|
// UDPCoalescer accumulates adjacent in-flow UDP datagrams across multiple
|
||||||
// concurrent flows and emits each flow's run as a single GSO_UDP_L4
|
// concurrent flows and emits each flow's run as a single GSO_UDP_L4 superpacket via tio.GSOWriter.
|
||||||
// superpacket via tio.GSOWriter. Falls back to per-packet writes when the
|
|
||||||
// underlying writer doesn't support USO.
|
|
||||||
// Preserves the in-flow order of packets as they are Commit-ed
|
// Preserves the in-flow order of packets as they are Commit-ed
|
||||||
//
|
//
|
||||||
// Owns no locks; one coalescer per TUN write queue.
|
// Owns no locks; one coalescer per TUN write queue.
|
||||||
type UDPCoalescer struct {
|
type UDPCoalescer struct {
|
||||||
plainW io.Writer
|
w tio.GSOWriter
|
||||||
gsoW tio.GSOWriter // nil when the queue can't accept GSO_UDP_L4
|
|
||||||
|
|
||||||
slots []*udpSlot
|
slots []*udpSlot
|
||||||
openSlots map[flowKey]*udpSlot
|
openSlots map[flowKey]*udpSlot
|
||||||
pool []*udpSlot
|
pool []*udpSlot
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewUDPCoalescer wraps w. The caller is responsible for only constructing
|
|
||||||
// this when the underlying Queue's Capabilities advertise USO; otherwise
|
|
||||||
// the kernel may reject GSO_UDP_L4 writes. If w does not implement
|
|
||||||
// tio.GSOWriter at all (single-packet Queue), the coalescer degrades to
|
|
||||||
// plain Writes — same defensive shape as the TCP coalescer.
|
|
||||||
func NewUDPCoalescer(w io.Writer) *UDPCoalescer {
|
func NewUDPCoalescer(w io.Writer) *UDPCoalescer {
|
||||||
c := &UDPCoalescer{
|
gw, ok := tio.SupportsGSO(w, tio.GSOProtoUDP)
|
||||||
plainW: w,
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &UDPCoalescer{
|
||||||
|
w: gw,
|
||||||
slots: make([]*udpSlot, 0, initialSlots),
|
slots: make([]*udpSlot, 0, initialSlots),
|
||||||
openSlots: make(map[flowKey]*udpSlot, initialSlots),
|
openSlots: make(map[flowKey]*udpSlot, initialSlots),
|
||||||
pool: make([]*udpSlot, 0, initialSlots),
|
pool: make([]*udpSlot, 0, initialSlots),
|
||||||
}
|
}
|
||||||
if gw, ok := tio.SupportsGSO(w, tio.GSOProtoUDP); ok {
|
|
||||||
c.gsoW = gw
|
|
||||||
}
|
|
||||||
return c
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// parsedUDP holds the fields extracted from a single parse so later steps
|
// parsedUDP holds the fields extracted from a single parse so later steps
|
||||||
@@ -115,10 +106,6 @@ func parseUDP(pkt []byte) (parsedUDP, bool) {
|
|||||||
|
|
||||||
// Commit borrows pkt. The caller must keep pkt valid until the next Flush.
|
// Commit borrows pkt. The caller must keep pkt valid until the next Flush.
|
||||||
func (c *UDPCoalescer) Commit(pkt []byte) error {
|
func (c *UDPCoalescer) Commit(pkt []byte) error {
|
||||||
if c.gsoW == nil {
|
|
||||||
c.addPassthrough(pkt)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
info, ok := parseUDP(pkt)
|
info, ok := parseUDP(pkt)
|
||||||
if !ok {
|
if !ok {
|
||||||
c.addPassthrough(pkt)
|
c.addPassthrough(pkt)
|
||||||
@@ -131,16 +118,8 @@ func (c *UDPCoalescer) Commit(pkt []byte) error {
|
|||||||
// already verified parseUDP succeeded. Used by MultiCoalescer.Commit to
|
// already verified parseUDP succeeded. Used by MultiCoalescer.Commit to
|
||||||
// avoid re-walking the IP/UDP header.
|
// avoid re-walking the IP/UDP header.
|
||||||
func (c *UDPCoalescer) commitParsed(pkt []byte, info parsedUDP) error {
|
func (c *UDPCoalescer) commitParsed(pkt []byte, info parsedUDP) error {
|
||||||
if c.gsoW == nil {
|
|
||||||
c.addPassthrough(pkt)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
// A zero-length UDP datagram (UDP `length` == 8) is legal and must still
|
// A zero-length UDP datagram (UDP `length` == 8) is legal and must still
|
||||||
// reach the TUN, but it can't be coalesced: a GSO slot would store an
|
// reach the TUN, but it can't be coalesced.
|
||||||
// empty payload iovec and the kernel has nothing to segment. Seal any
|
|
||||||
// open chain for this flow (so a later, non-empty datagram seeds fresh
|
|
||||||
// *after* this one and per-flow arrival order is preserved) and deliver
|
|
||||||
// it as a plain single datagram.
|
|
||||||
if info.payLen == 0 {
|
if info.payLen == 0 {
|
||||||
delete(c.openSlots, info.fk)
|
delete(c.openSlots, info.fk)
|
||||||
c.addPassthrough(pkt)
|
c.addPassthrough(pkt)
|
||||||
@@ -154,7 +133,7 @@ func (c *UDPCoalescer) commitParsed(pkt []byte, info parsedUDP) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// Can't extend — seal it and fall through to seed a fresh slot.
|
// Can't extend. Seal it and fall through to seed a fresh slot.
|
||||||
delete(c.openSlots, info.fk)
|
delete(c.openSlots, info.fk)
|
||||||
}
|
}
|
||||||
c.seed(pkt, info)
|
c.seed(pkt, info)
|
||||||
@@ -166,7 +145,7 @@ func (c *UDPCoalescer) Flush() error {
|
|||||||
for _, s := range c.slots {
|
for _, s := range c.slots {
|
||||||
var err error
|
var err error
|
||||||
if s.passthrough {
|
if s.passthrough {
|
||||||
_, err = c.plainW.Write(s.rawPkt)
|
_, err = c.w.Write(s.rawPkt)
|
||||||
} else {
|
} else {
|
||||||
err = c.flushSlot(s)
|
err = c.flushSlot(s)
|
||||||
}
|
}
|
||||||
@@ -296,7 +275,7 @@ func (c *UDPCoalescer) flushSlot(s *udpSlot) error {
|
|||||||
udpCsumOff := s.ipHdrLen + 6
|
udpCsumOff := s.ipHdrLen + 6
|
||||||
binary.BigEndian.PutUint16(hdr[udpCsumOff:udpCsumOff+2], foldOnceNoInvert(psum))
|
binary.BigEndian.PutUint16(hdr[udpCsumOff:udpCsumOff+2], foldOnceNoInvert(psum))
|
||||||
|
|
||||||
return c.gsoW.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoUDP)
|
return c.w.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoUDP)
|
||||||
}
|
}
|
||||||
|
|
||||||
// udpHeadersMatch compares two IP+UDP header prefixes for byte-equality on
|
// udpHeadersMatch compares two IP+UDP header prefixes for byte-equality on
|
||||||
@@ -308,7 +287,7 @@ func udpHeadersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
|
|||||||
if !ipHeadersMatch(a, b, isV6) {
|
if !ipHeadersMatch(a, b, isV6) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
// UDP: compare sport+dport ([0:4]). Skip length [4:6] and checksum [6:8] —
|
// UDP: compare sport+dport ([0:4]). Skip length [4:6] and checksum [6:8]
|
||||||
// length varies (we rewrite at flush) and the checksum will be redone.
|
// length varies (we rewrite at flush) and the checksum will be redone.
|
||||||
udp := ipHdrLen
|
udp := ipHdrLen
|
||||||
if a[udp] != b[udp] || a[udp+1] != b[udp+1] || a[udp+2] != b[udp+2] || a[udp+3] != b[udp+3] {
|
if a[udp] != b[udp] || a[udp+1] != b[udp+1] || a[udp+2] != b[udp+2] || a[udp+3] != b[udp+3] {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package batch
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
|
"io"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -58,27 +59,31 @@ func buildUDPv6(sport, dport uint16, payload []byte) []byte {
|
|||||||
return pkt
|
return pkt
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUDPCoalescerPassthroughWhenGSOUnavailable(t *testing.T) {
|
// newTestUDPCoalescer builds a coalescer over w and fails the test if w can't
|
||||||
w := &fakeTunWriter{gsoEnabled: false}
|
// do USO. See newTestTCPCoalescer.
|
||||||
|
func newTestUDPCoalescer(tb testing.TB, w io.Writer) *UDPCoalescer {
|
||||||
|
tb.Helper()
|
||||||
c := NewUDPCoalescer(w)
|
c := NewUDPCoalescer(w)
|
||||||
pkt := buildUDPv4(1000, 53, make([]byte, 100))
|
if c == nil {
|
||||||
if err := c.Commit(pkt); err != nil {
|
tb.Fatal("NewUDPCoalescer: writer does not support USO")
|
||||||
t.Fatal(err)
|
|
||||||
}
|
}
|
||||||
if len(w.writes) != 0 || len(w.gsoWrites) != 0 {
|
return c
|
||||||
t.Fatalf("no Add-time writes: writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
}
|
||||||
|
|
||||||
|
// TestNewUDPCoalescerRefusesWhenGSOUnavailable mirrors the TCP precondition:
|
||||||
|
// no USO, no coalescer.
|
||||||
|
func TestNewUDPCoalescerRefusesWhenGSOUnavailable(t *testing.T) {
|
||||||
|
if c := NewUDPCoalescer(&fakeTunWriter{gsoEnabled: false}); c != nil {
|
||||||
|
t.Fatalf("want nil for a non-USO writer, got %v", c)
|
||||||
}
|
}
|
||||||
if err := c.Flush(); err != nil {
|
if c := NewUDPCoalescer(&plainOnlyWriter{}); c != nil {
|
||||||
t.Fatal(err)
|
t.Fatalf("want nil for a plain writer, got %v", c)
|
||||||
}
|
|
||||||
if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
|
|
||||||
t.Fatalf("want single plain write, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUDPCoalescerNonUDPPassthrough(t *testing.T) {
|
func TestUDPCoalescerNonUDPPassthrough(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
// ICMP packet
|
// ICMP packet
|
||||||
pkt := make([]byte, 28)
|
pkt := make([]byte, 28)
|
||||||
pkt[0] = 0x45
|
pkt[0] = 0x45
|
||||||
@@ -99,7 +104,7 @@ func TestUDPCoalescerNonUDPPassthrough(t *testing.T) {
|
|||||||
|
|
||||||
func TestUDPCoalescerSeedThenFlushAlone(t *testing.T) {
|
func TestUDPCoalescerSeedThenFlushAlone(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
pkt := buildUDPv4(1000, 53, make([]byte, 800))
|
pkt := buildUDPv4(1000, 53, make([]byte, 800))
|
||||||
if err := c.Commit(pkt); err != nil {
|
if err := c.Commit(pkt); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -116,7 +121,7 @@ func TestUDPCoalescerSeedThenFlushAlone(t *testing.T) {
|
|||||||
|
|
||||||
func TestUDPCoalescerCoalescesEqualSized(t *testing.T) {
|
func TestUDPCoalescerCoalescesEqualSized(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
for i := 0; i < 3; i++ {
|
for i := 0; i < 3; i++ {
|
||||||
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
|
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
|
||||||
@@ -156,7 +161,7 @@ func TestUDPCoalescerCoalescesEqualSized(t *testing.T) {
|
|||||||
// Last segment may be shorter, sealing the chain.
|
// Last segment may be shorter, sealing the chain.
|
||||||
func TestUDPCoalescerShortLastSegmentSeals(t *testing.T) {
|
func TestUDPCoalescerShortLastSegmentSeals(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
full := make([]byte, 1200)
|
full := make([]byte, 1200)
|
||||||
tail := make([]byte, 600)
|
tail := make([]byte, 600)
|
||||||
if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
|
if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
|
||||||
@@ -189,7 +194,7 @@ func TestUDPCoalescerShortLastSegmentSeals(t *testing.T) {
|
|||||||
// A larger-than-gsoSize packet cannot extend the slot — it reseeds.
|
// A larger-than-gsoSize packet cannot extend the slot — it reseeds.
|
||||||
func TestUDPCoalescerLargerThanSeedReseeds(t *testing.T) {
|
func TestUDPCoalescerLargerThanSeedReseeds(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
if err := c.Commit(buildUDPv4(1000, 53, make([]byte, 800))); err != nil {
|
if err := c.Commit(buildUDPv4(1000, 53, make([]byte, 800))); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -207,7 +212,7 @@ func TestUDPCoalescerLargerThanSeedReseeds(t *testing.T) {
|
|||||||
// Different 5-tuples must not coalesce.
|
// Different 5-tuples must not coalesce.
|
||||||
func TestUDPCoalescerDifferentFlowsKeepSeparate(t *testing.T) {
|
func TestUDPCoalescerDifferentFlowsKeepSeparate(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
pay := make([]byte, 800)
|
pay := make([]byte, 800)
|
||||||
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
|
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -238,7 +243,7 @@ func TestUDPCoalescerDifferentFlowsKeepSeparate(t *testing.T) {
|
|||||||
// Caps at udpCoalesceMaxSegs.
|
// Caps at udpCoalesceMaxSegs.
|
||||||
func TestUDPCoalescerCapsAtMaxSegs(t *testing.T) {
|
func TestUDPCoalescerCapsAtMaxSegs(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
pay := make([]byte, 100)
|
pay := make([]byte, 100)
|
||||||
for i := 0; i < udpCoalesceMaxSegs+5; i++ {
|
for i := 0; i < udpCoalesceMaxSegs+5; i++ {
|
||||||
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
|
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
|
||||||
@@ -267,7 +272,7 @@ func TestUDPCoalescerCapsAtMaxSegs(t *testing.T) {
|
|||||||
// trailing Not-ECT datagram seeds another.
|
// trailing Not-ECT datagram seeds another.
|
||||||
func TestUDPCoalescerDifferingECNReseeds(t *testing.T) {
|
func TestUDPCoalescerDifferingECNReseeds(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
pay := make([]byte, 800)
|
pay := make([]byte, 800)
|
||||||
pkt0 := buildUDPv4(1000, 53, pay) // ECN=00 (Not-ECT)
|
pkt0 := buildUDPv4(1000, 53, pay) // ECN=00 (Not-ECT)
|
||||||
pkt1 := buildUDPv4(1000, 53, pay)
|
pkt1 := buildUDPv4(1000, 53, pay)
|
||||||
@@ -298,7 +303,7 @@ func TestUDPCoalescerDifferingECNReseeds(t *testing.T) {
|
|||||||
// IPv6 path: same flow, equal-sized → coalesced.
|
// IPv6 path: same flow, equal-sized → coalesced.
|
||||||
func TestUDPCoalescerIPv6Coalesces(t *testing.T) {
|
func TestUDPCoalescerIPv6Coalesces(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
pay := make([]byte, 1200)
|
pay := make([]byte, 1200)
|
||||||
for i := 0; i < 3; i++ {
|
for i := 0; i < 3; i++ {
|
||||||
if err := c.Commit(buildUDPv6(1000, 53, pay)); err != nil {
|
if err := c.Commit(buildUDPv6(1000, 53, pay)); err != nil {
|
||||||
@@ -334,7 +339,7 @@ func TestUDPCoalescerIPv6Coalesces(t *testing.T) {
|
|||||||
// DSCP differences must reseed: udpHeadersMatch compares the full ToS byte.
|
// DSCP differences must reseed: udpHeadersMatch compares the full ToS byte.
|
||||||
func TestUDPCoalescerDSCPMismatchReseeds(t *testing.T) {
|
func TestUDPCoalescerDSCPMismatchReseeds(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
pay := make([]byte, 800)
|
pay := make([]byte, 800)
|
||||||
pkt0 := buildUDPv4(1000, 53, pay)
|
pkt0 := buildUDPv4(1000, 53, pay)
|
||||||
pkt1 := buildUDPv4(1000, 53, pay)
|
pkt1 := buildUDPv4(1000, 53, pay)
|
||||||
@@ -356,7 +361,7 @@ func TestUDPCoalescerDSCPMismatchReseeds(t *testing.T) {
|
|||||||
// Fragmented IPv4 must not be coalesced.
|
// Fragmented IPv4 must not be coalesced.
|
||||||
func TestUDPCoalescerFragmentedIPv4PassesThrough(t *testing.T) {
|
func TestUDPCoalescerFragmentedIPv4PassesThrough(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
pkt := buildUDPv4(1000, 53, make([]byte, 200))
|
pkt := buildUDPv4(1000, 53, make([]byte, 200))
|
||||||
binary.BigEndian.PutUint16(pkt[6:8], 0x2000) // MF=1
|
binary.BigEndian.PutUint16(pkt[6:8], 0x2000) // MF=1
|
||||||
if err := c.Commit(pkt); err != nil {
|
if err := c.Commit(pkt); err != nil {
|
||||||
@@ -377,7 +382,7 @@ func TestUDPCoalescerFragmentedIPv4PassesThrough(t *testing.T) {
|
|||||||
// reach the GSO path. Regression: must not panic and must be written.
|
// reach the GSO path. Regression: must not panic and must be written.
|
||||||
func TestUDPCoalescerZeroLengthPayloadPassesThrough(t *testing.T) {
|
func TestUDPCoalescerZeroLengthPayloadPassesThrough(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
pkt := buildUDPv4(1000, 53, nil) // UDP length 8, zero payload
|
pkt := buildUDPv4(1000, 53, nil) // UDP length 8, zero payload
|
||||||
if err := c.Commit(pkt); err != nil {
|
if err := c.Commit(pkt); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -396,7 +401,7 @@ func TestUDPCoalescerZeroLengthPayloadPassesThrough(t *testing.T) {
|
|||||||
// IPv6 zero-length UDP datagram: same passthrough contract as v4.
|
// IPv6 zero-length UDP datagram: same passthrough contract as v4.
|
||||||
func TestUDPCoalescerZeroLengthPayloadIPv6PassesThrough(t *testing.T) {
|
func TestUDPCoalescerZeroLengthPayloadIPv6PassesThrough(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
pkt := buildUDPv6(1000, 53, nil) // UDP length 8, zero payload
|
pkt := buildUDPv6(1000, 53, nil) // UDP length 8, zero payload
|
||||||
if err := c.Commit(pkt); err != nil {
|
if err := c.Commit(pkt); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -417,7 +422,7 @@ func TestUDPCoalescerZeroLengthPayloadIPv6PassesThrough(t *testing.T) {
|
|||||||
// wire — per-flow arrival order (full, empty, full) must be preserved.
|
// wire — per-flow arrival order (full, empty, full) must be preserved.
|
||||||
func TestUDPCoalescerZeroLengthMidFlowSealsAndPreservesOrder(t *testing.T) {
|
func TestUDPCoalescerZeroLengthMidFlowSealsAndPreservesOrder(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
full := make([]byte, 800)
|
full := make([]byte, 800)
|
||||||
if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
|
if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -441,7 +446,7 @@ func TestUDPCoalescerZeroLengthMidFlowSealsAndPreservesOrder(t *testing.T) {
|
|||||||
// IPv4 with options is not admissible (we require IHL=5).
|
// IPv4 with options is not admissible (we require IHL=5).
|
||||||
func TestUDPCoalescerIPv4WithOptionsPassesThrough(t *testing.T) {
|
func TestUDPCoalescerIPv4WithOptionsPassesThrough(t *testing.T) {
|
||||||
w := &fakeTunWriter{gsoEnabled: true}
|
w := &fakeTunWriter{gsoEnabled: true}
|
||||||
c := NewUDPCoalescer(w)
|
c := newTestUDPCoalescer(t, w)
|
||||||
pkt := buildUDPv4(1000, 53, make([]byte, 200))
|
pkt := buildUDPv4(1000, 53, make([]byte, 200))
|
||||||
pkt[0] = 0x46 // IHL = 6 (24-byte IPv4 header — has options)
|
pkt[0] = 0x46 // IHL = 6 (24-byte IPv4 header — has options)
|
||||||
if err := c.Commit(pkt); err != nil {
|
if err := c.Commit(pkt); err != nil {
|
||||||
|
|||||||
@@ -8,12 +8,6 @@ func protoFromGSOType(_ uint8) (GSOProto, error) {
|
|||||||
return 0, fmt.Errorf("GSO unsupported")
|
return 0, fmt.Errorf("GSO unsupported")
|
||||||
}
|
}
|
||||||
|
|
||||||
// SegmentSuperpacket invokes fn once per segment of pkt. On non-Linux
|
|
||||||
// builds (and Android/e2e_testing) this package does not provide a Queue
|
|
||||||
// implementation, so any caller that does construct a Packet here can only
|
|
||||||
// be operating on non-superpacket bytes and the stub forwards them
|
|
||||||
// directly. A non-zero GSO field is a programming error from the caller
|
|
||||||
// and returns an explicit error rather than silently misbehaving.
|
|
||||||
func SegmentSuperpacket(pkt Packet, fn func(seg []byte) error) error {
|
func SegmentSuperpacket(pkt Packet, fn func(seg []byte) error) error {
|
||||||
if pkt.GSO.IsSuperpacket() {
|
if pkt.GSO.IsSuperpacket() {
|
||||||
return fmt.Errorf("tio: GSO superpacket on platform without segmentation support")
|
return fmt.Errorf("tio: GSO superpacket on platform without segmentation support")
|
||||||
|
|||||||
@@ -4,9 +4,8 @@ import "io"
|
|||||||
|
|
||||||
// singleQueue adapts a legacy one-datagram-per-Read source into a Queue.
|
// singleQueue adapts a legacy one-datagram-per-Read source into a Queue.
|
||||||
// Read fills a private scratch buffer and returns exactly one Packet whose
|
// Read fills a private scratch buffer and returns exactly one Packet whose
|
||||||
// Bytes borrow from that buffer, valid only until the next Read, per the
|
// Bytes borrow from that buffer, valid only until the next Read, per the Queue contract.
|
||||||
// Queue contract. Single-reader like every Queue; Write is exactly as safe
|
// Single-reader like every Queue; Write is exactly as safe for concurrent use as the underlying source's Write.
|
||||||
// for concurrent use as the underlying source's Write.
|
|
||||||
type singleQueue struct {
|
type singleQueue struct {
|
||||||
rw io.ReadWriter
|
rw io.ReadWriter
|
||||||
closer io.Closer // nil: Close is a no-op (the source is shared and owned elsewhere)
|
closer io.Closer // nil: Close is a no-op (the source is shared and owned elsewhere)
|
||||||
@@ -14,9 +13,9 @@ type singleQueue struct {
|
|||||||
ret [1]Packet
|
ret [1]Packet
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewSingleQueue wraps a one-datagram-per-Read ReadWriteCloser (a legacy tun
|
// NewSingleQueue wraps a one-datagram-per-Read ReadWriteCloser (a legacy tun device) into a Queue.
|
||||||
// device) into a Queue. bufSize is the per-queue read scratch size and must
|
// bufSize is the per-queue read scratch size and must be at least the largest datagram the source can return.
|
||||||
// be at least the largest datagram the source can return. Close closes rwc.
|
// Close closes rwc.
|
||||||
func NewSingleQueue(rwc io.ReadWriteCloser, bufSize int) Queue {
|
func NewSingleQueue(rwc io.ReadWriteCloser, bufSize int) Queue {
|
||||||
return &singleQueue{rw: rwc, closer: rwc, buf: make([]byte, bufSize)}
|
return &singleQueue{rw: rwc, closer: rwc, buf: make([]byte, bufSize)}
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-10
@@ -93,14 +93,6 @@ type CapsProvider interface {
|
|||||||
Capabilities() Capabilities
|
Capabilities() Capabilities
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueueCapabilities returns q's negotiated offload capabilities, or the zero value when q does not advertise any.
|
|
||||||
func QueueCapabilities(q io.Writer) Capabilities {
|
|
||||||
if cp, ok := q.(CapsProvider); ok {
|
|
||||||
return cp.Capabilities()
|
|
||||||
}
|
|
||||||
return Capabilities{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// GSOProto selects the L4 protocol for a GSO superpacket.
|
// GSOProto selects the L4 protocol for a GSO superpacket.
|
||||||
// Determines which VIRTIO_NET_HDR_GSO_* type the writer stamps and which checksum offset
|
// Determines which VIRTIO_NET_HDR_GSO_* type the writer stamps and which checksum offset
|
||||||
// inside the transport header virtio NEEDS_CSUM expects.
|
// inside the transport header virtio NEEDS_CSUM expects.
|
||||||
@@ -126,9 +118,10 @@ const (
|
|||||||
// Every segment in pays except possibly the last is exactly the same size.
|
// Every segment in pays except possibly the last is exactly the same size.
|
||||||
// proto picks the L4 protocol so the writer knows which gsoType / CsumOffset to set.
|
// proto picks the L4 protocol so the writer knows which gsoType / CsumOffset to set.
|
||||||
//
|
//
|
||||||
// Callers should also consult CapsProvider (via SupportsGSO or QueueCapabilities)
|
// Callers should also consult CapsProvider (via SupportsGSO) for the per-protocol negotiated capability:
|
||||||
// for the per-protocol negotiated capability: USO may not have been negotiated even when TSO was.
|
// USO may not have been negotiated even when TSO was.
|
||||||
type GSOWriter interface {
|
type GSOWriter interface {
|
||||||
|
io.Writer
|
||||||
CapsProvider
|
CapsProvider
|
||||||
WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, proto GSOProto) error
|
WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, proto GSOProto) error
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -173,7 +173,7 @@ func (r *Offload) Read() ([]Packet, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := r.decodeRead(n); err != nil {
|
if err := r.decodeRead(n); err != nil {
|
||||||
// Drop and read again — a bad packet should not kill the reader.
|
// Drop and read again. A bad packet should not kill the reader.
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
|
|||||||
Reference in New Issue
Block a user