unslop, improve the tio interface

overlay/batch/tcp_coalesce_test.go

@@ -2,6 +2,7 @@ package batch
 
 import (
 	"encoding/binary"
+	"io"
 	"testing"
 
 	"github.com/slackhq/nebula/overlay/tio"
@@ -11,8 +12,13 @@ import (
 // fakeTunWriter records plain Writes and WriteGSO calls without touching a
 // real TUN fd. WriteGSO records the IP header, transport header, and
 // borrowed payload fragments separately so tests can inspect each.
+// noTSO / noUSO withhold one offload from an otherwise GSO-capable writer, so
+// tests can build the half-capable queues real kernels hand us (USO needs a
+// newer kernel than TSO).
 type fakeTunWriter struct {
 	gsoEnabled bool
+	noTSO      bool
+	noUSO      bool
 	writes     [][]byte
 	gsoWrites  []fakeGSOWrite
 }
@@ -79,7 +85,7 @@ func (w *fakeTunWriter) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte,
 }
 
 func (w *fakeTunWriter) Capabilities() tio.Capabilities {
-	return tio.Capabilities{TSO: w.gsoEnabled, USO: w.gsoEnabled}
+	return tio.Capabilities{TSO: w.gsoEnabled && !w.noTSO, USO: w.gsoEnabled && !w.noUSO}
 }
 
 // buildTCPv4 constructs a minimal IPv4+TCP packet with the given payload,
@@ -126,28 +132,43 @@ const (
 	tcpAckPsh = tcpAck | tcpPsh
 )
 
-func TestCoalescerPassthroughWhenGSOUnavailable(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: false}
+// newTestTCPCoalescer builds a coalescer over w and fails the test if w can't
+// do TSO. Every test but TestNewTCPCoalescerRefusesWhenGSOUnavailable wants the
+// GSO path, and the constructor now hands back a nil coalescer otherwise.
+func newTestTCPCoalescer(tb testing.TB, w io.Writer) *TCPCoalescer {
+	tb.Helper()
 	c := NewTCPCoalescer(w, test.NewLogger())
-	pkt := buildTCPv4(1000, tcpAck, []byte("hello"))
-	if err := c.Commit(pkt); err != nil {
-		t.Fatal(err)
+	if c == nil {
+		tb.Fatal("NewTCPCoalescer: writer does not support TSO")
 	}
-	// No sync write — passthrough is deferred to Flush.
-	if len(w.writes) != 0 || len(w.gsoWrites) != 0 {
-		t.Fatalf("no Add-time writes: got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
+	return c
+}
+
+// TestNewTCPCoalescerRefusesWhenGSOUnavailable pins the constructor
+// precondition: no TSO, no coalescer. There's no degraded mode — the caller
+// (MultiCoalescer) sends TCP down the passthrough lane instead.
+func TestNewTCPCoalescerRefusesWhenGSOUnavailable(t *testing.T) {
+	if c := NewTCPCoalescer(&fakeTunWriter{gsoEnabled: false}, test.NewLogger()); c != nil {
+		t.Fatalf("want nil for a non-TSO writer, got %v", c)
 	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
-		t.Fatalf("want single plain write, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
+	// A writer that isn't a GSOWriter at all is refused the same way.
+	if c := NewTCPCoalescer(&plainOnlyWriter{}, test.NewLogger()); c != nil {
+		t.Fatalf("want nil for a plain writer, got %v", c)
 	}
 }
 
+// plainOnlyWriter is an io.Writer with no GSO support at all — the
+// single-packet Queue shape.
+type plainOnlyWriter struct{ writes int }
+
+func (w *plainOnlyWriter) Write(p []byte) (int, error) {
+	w.writes++
+	return len(p), nil
+}
+
 func TestCoalescerNonTCPPassthrough(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pkt := make([]byte, 28)
 	pkt[0] = 0x45
 	binary.BigEndian.PutUint16(pkt[2:4], 28)
@@ -167,7 +188,7 @@ func TestCoalescerNonTCPPassthrough(t *testing.T) {
 
 func TestCoalescerSeedThenFlushAlone(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pkt := buildTCPv4(1000, tcpAck, make([]byte, 1000))
 	if err := c.Commit(pkt); err != nil {
 		t.Fatal(err)
@@ -194,7 +215,7 @@ func TestCoalescerSeedThenFlushAlone(t *testing.T) {
 
 func TestCoalescerCoalescesAdjacentACKs(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -234,7 +255,7 @@ func TestCoalescerCoalescesAdjacentACKs(t *testing.T) {
 
 func TestCoalescerRejectsSeqGap(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -253,7 +274,7 @@ func TestCoalescerRejectsSeqGap(t *testing.T) {
 
 func TestCoalescerRejectsFlagMismatch(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -274,7 +295,7 @@ func TestCoalescerRejectsFlagMismatch(t *testing.T) {
 
 func TestCoalescerRejectsFIN(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	fin := buildTCPv4(1000, tcpAck|tcpFin, []byte("x"))
 	if err := c.Commit(fin); err != nil {
 		t.Fatal(err)
@@ -290,7 +311,7 @@ func TestCoalescerRejectsFIN(t *testing.T) {
 
 func TestCoalescerShortLastSegmentClosesChain(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	full := make([]byte, 1200)
 	half := make([]byte, 500)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, full)); err != nil {
@@ -325,7 +346,7 @@ func TestCoalescerShortLastSegmentClosesChain(t *testing.T) {
 
 func TestCoalescerPSHFinalizesChain(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -355,7 +376,7 @@ func TestCoalescerPSHFinalizesChain(t *testing.T) {
 // coalescer drops it the sender's push signal never reaches the receiver.
 func TestCoalescerPropagatesPSHFromAppended(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	// Seed has no PSH; second segment carries PSH and seals the chain.
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
@@ -383,7 +404,7 @@ func TestCoalescerPropagatesPSHFromAppended(t *testing.T) {
 
 func TestCoalescerRejectsDifferentFlow(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	p1 := buildTCPv4(1000, tcpAck, pay)
 	p2 := buildTCPv4(2200, tcpAck, pay)
@@ -405,7 +426,7 @@ func TestCoalescerRejectsDifferentFlow(t *testing.T) {
 
 func TestCoalescerRejectsIPOptions(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 500)
 	pkt := buildTCPv4(1000, tcpAck, pay)
 	// Bump IHL to 6 to simulate 4 bytes of IP options. Don't actually add
@@ -425,7 +446,7 @@ func TestCoalescerRejectsIPOptions(t *testing.T) {
 
 func TestCoalescerCapBySegments(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 512)
 	seq := uint32(1000)
 	for i := 0; i < tcpCoalesceMaxSegs+5; i++ {
@@ -449,7 +470,7 @@ func TestCoalescerCapBySegments(t *testing.T) {
 // flows coalesce independently in a single Flush.
 func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 
 	// Flow A: sport 1000. Flow B: sport 3000.
@@ -506,7 +527,7 @@ func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
 // writing passthrough packets synchronously.
 func TestCoalescerPreservesArrivalOrder(t *testing.T) {
 	w := &orderedFakeWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	// Sequence: coalesceable TCP, ICMP (passthrough), coalesceable TCP on
 	// a different flow. Expected emit order: gso(X), plain(ICMP), gso(Y).
 	pay := make([]byte, 1200)
@@ -574,7 +595,7 @@ func stringSliceEq(a, b []string) bool {
 // packet (SYN) mid-flow only flushes its own flow, not others.
 func TestCoalescerInterleavedFlowsPreserveOrdering(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 
 	// Flow A two segments.
@@ -679,7 +700,7 @@ func buildTCPv6(tcLow byte, seq uint32, flags byte, payload []byte) []byte {
 // retains ECE on the wire.
 func TestCoalescerCoalescesEceFlow(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	flags := byte(tcpAck | tcpEce)
 	if err := c.Commit(buildTCPv4(1000, flags, pay)); err != nil {
@@ -708,7 +729,7 @@ func TestCoalescerCoalescesEceFlow(t *testing.T) {
 // in-flow segment seeds a new slot rather than extending the prior burst.
 func TestCoalescerCwrSealsFlow(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -741,7 +762,7 @@ func TestCoalescerCwrSealsFlow(t *testing.T) {
 // a CE-echoing window or none.
 func TestCoalescerEceMismatchReseeds(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck|tcpEce, pay)); err != nil {
 		t.Fatal(err)
@@ -771,7 +792,7 @@ func TestCoalescerEceMismatchReseeds(t *testing.T) {
 // across the whole burst.
 func TestCoalescerDifferingECNReseeds(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4WithToS(ecnECT0, 1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -816,7 +837,7 @@ func TestCoalescerDifferingECNReseeds(t *testing.T) {
 // codepoint, and neither may end up CE-marked.
 func TestCoalescerECT0ThenECT1NoCE(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4WithToS(ecnECT0, 1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -846,7 +867,7 @@ func TestCoalescerECT0ThenECT1NoCE(t *testing.T) {
 // six DSCP bits must match too.
 func TestCoalescerDscpMismatchReseeds(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	// Same ECN (Not-ECT), different DSCP (0x10 vs 0x20 in upper 6 bits).
 	tosA := byte(0x10<<2) | ecnNotECT
@@ -869,7 +890,7 @@ func TestCoalescerDscpMismatchReseeds(t *testing.T) {
 // TestCoalescerCoalescesEceFlow.
 func TestCoalescerIPv6CoalescesEceFlow(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	flags := byte(tcpAck | tcpEce)
 	if err := c.Commit(buildTCPv6(0, 1000, flags, pay)); err != nil {
@@ -900,7 +921,7 @@ func TestCoalescerIPv6CoalescesEceFlow(t *testing.T) {
 // seen had the wire never reordered.
 func TestCoalescerSortsReorderedSeedsAndMerges(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	// Arrival order: seq 1000, 3400, 2200. The 3400 seeds a separate slot
 	// because 3400 != nextSeq=2200, then 2200 fails to extend the 3400 slot
@@ -936,7 +957,7 @@ func TestCoalescerSortsReorderedSeedsAndMerges(t *testing.T) {
 // without any cross-flow contamination.
 func TestCoalescerSortAcrossFlowsMergesEachIndependently(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	// Flow A (sport 1000) seq 100, 1300; flow B (sport 3000) seq 500, 1700.
 	// Arrival: A.1300, B.1700, A.100, B.500 — every flow reordered.
@@ -987,7 +1008,7 @@ func TestCoalescerSortAcrossFlowsMergesEachIndependently(t *testing.T) {
 // boundary by an arbitrary number of segments.
 func TestCoalescerSortKeepsPSHBoundary(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	// Seq 1000 (no PSH) + 2200 (PSH) → seal one slot with PSH set.
 	// Seq 3400 (no PSH) is contiguous to 3400 from seq 2200+1200; without
@@ -1015,7 +1036,7 @@ func TestCoalescerSortKeepsPSHBoundary(t *testing.T) {
 // is sorted/merged independently.
 func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	// First two segments seed S1 (then a 3400 reorder seeds S2).
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
@@ -1049,7 +1070,7 @@ func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
 // 0x30, so ipHeadersMatch (comparing byte 1 fully) still splits them.
 func TestCoalescerIPv6DifferingECNReseeds(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	// tcLow is the low 4 bits of TC; ECN occupies the bottom 2 of those.
 	if err := c.Commit(buildTCPv6(ecnECT0, 1000, tcpAck, pay)); err != nil {
@@ -1124,7 +1145,7 @@ func TestSortRunZeroAllocs(t *testing.T) {
 // synthesize it from the seal bool.
 func TestCoalescerMergeShortTailDoesNotFabricatePSH(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	short := make([]byte, 600)
 	// Arrival: seq 3400 (full), 4600 (short, seals the slot), then the
@@ -1162,7 +1183,7 @@ func TestCoalescerMergeShortTailDoesNotFabricatePSH(t *testing.T) {
 // source slot's tail really carried PSH, the merged header must keep it.
 func TestCoalescerMergePreservesRealPSH(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger())
+	c := newTestTCPCoalescer(t, w)
 	pay := make([]byte, 1200)
 	short := make([]byte, 600)
 	if err := c.Commit(buildTCPv4(3400, tcpAck, pay)); err != nil {