Probe for another path when the data path finds a tunnel's remote unreachable

overlay/batch/tx_batch.go

@@ -6,7 +6,14 @@ const SendBatchCap = 128
 
 // batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush.
 type batchWriter interface {
-	WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
+	WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
+}
+
+// Failure is a packet the underlay would not send, with the tag it was committed under
+type Failure struct {
+	Tag uint32
+	Dst netip.AddrPort
+	Err error
 }
 
 // SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
@@ -16,17 +23,29 @@ type SendBatch struct {
 	out   batchWriter
 	bufs  [][]byte
 	dsts  []netip.AddrPort
+	tags  []uint32
 	arena *Arena
+
+	failures []Failure
+	// recordFailure is bound once so handing it to WriteBatch does not allocate on every Flush
+	recordFailure func(i int, err error)
 }
 
 // NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots
 func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch {
-	return &SendBatch{
+	b := &SendBatch{
 		out:   out,
 		bufs:  make([][]byte, 0, batchCap),
 		dsts:  make([]netip.AddrPort, 0, batchCap),
+		tags:  make([]uint32, 0, batchCap),
 		arena: NewArena(arenaSize),
 	}
+	b.recordFailure = b.failed
+	return b
+}
+
+func (b *SendBatch) failed(i int, err error) {
+	b.failures = append(b.failures, Failure{Tag: b.tags[i], Dst: b.dsts[i], Err: err})
 }
 
 func (b *SendBatch) Reserve(sz int) []byte {
@@ -38,22 +57,35 @@ func (b *SendBatch) Reserve(sz int) []byte {
 // bounding how long the first packet of a large read batch waits.
 func (b *SendBatch) Len() int { return len(b.bufs) }
 
-func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort) {
+// Commit queues pkt for dst. tag comes back in Failures if the underlay refuses the packet, so the caller can tell
+// whose packet it was.
+func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort, tag uint32) {
 	b.bufs = append(b.bufs, pkt)
 	b.dsts = append(b.dsts, dst)
+	b.tags = append(b.tags, tag)
 }
 
 // Flush writes every queued packet and reports how many actually went out. A short count means some destinations
-// were undeliverable; the batch is drained either way.
+// were undeliverable, Failures says which; the batch is drained either way.
 func (b *SendBatch) Flush() (int, error) {
+	clear(b.failures)
+	b.failures = b.failures[:0]
+
 	var err error
 	written := 0
 	if len(b.bufs) > 0 {
-		written, err = b.out.WriteBatch(b.bufs, b.dsts)
+		written, err = b.out.WriteBatch(b.bufs, b.dsts, b.recordFailure)
 	}
 	clear(b.bufs)
 	b.bufs = b.bufs[:0]
 	b.dsts = b.dsts[:0]
+	b.tags = b.tags[:0]
 	b.arena.Reset()
 	return written, err
 }
+
+// Failures returns every packet the last Flush could not send, in the order they were committed. Valid until the
+// next Flush.
+func (b *SendBatch) Failures() []Failure {
+	return b.failures
+}

overlay/batch/tx_batch_test.go

@@ -1,7 +1,9 @@
 package batch
 
 import (
+	"errors"
 	"net/netip"
+	"slices"
 	"testing"
 )
 
@@ -10,7 +12,7 @@ type fakeBatchWriter struct {
 	addrs []netip.AddrPort
 }
 
-func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
+func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, _ func(int, error)) (int, error) {
 	// Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch
 	// returns, so tests must capture data before that happens.
 	w.bufs = make([][]byte, len(bufs))
@@ -34,7 +36,7 @@ func TestSendBatchReserveCommitFlush(t *testing.T) {
 			t.Fatalf("slot %d: cap=%d want 32", i, cap(slot))
 		}
 		pkt := append(slot[:0], byte(i), byte(i+1), byte(i+2))
-		b.Commit(pkt, ap)
+		b.Commit(pkt, ap, 0)
 	}
 	if _, err := b.Flush(); err != nil {
 		t.Fatalf("Flush: %v", err)
@@ -75,7 +77,7 @@ func TestSendBatchSlotsDoNotOverlap(t *testing.T) {
 	for i := range 3 {
 		s := b.Reserve(8)
 		pkt := append(s[:0], byte(0xA0+i), byte(0xB0+i))
-		b.Commit(pkt, ap)
+		b.Commit(pkt, ap, 0)
 	}
 	if _, err := b.Flush(); err != nil {
 		t.Fatalf("Flush: %v", err)
@@ -96,11 +98,11 @@ func TestSendBatchGrowPreservesCommitted(t *testing.T) {
 
 	s1 := b.Reserve(4)
 	pkt1 := append(s1[:0], 0x11, 0x22, 0x33, 0x44)
-	b.Commit(pkt1, ap)
+	b.Commit(pkt1, ap, 0)
 
 	s2 := b.Reserve(8) // exceeds remaining cap, triggers grow
 	pkt2 := append(s2[:0], 0xA, 0xB, 0xC, 0xD, 0xE)
-	b.Commit(pkt2, ap)
+	b.Commit(pkt2, ap, 0)
 
 	// pkt1 must still be intact even though backing reallocated.
 	if pkt1[0] != 0x11 || pkt1[3] != 0x44 {
@@ -120,3 +122,68 @@ func TestSendBatchGrowPreservesCommitted(t *testing.T) {
 		t.Errorf("second packet on the wire: %x", fw.bufs[1])
 	}
 }
+
+// failingBatchWriter refuses every packet whose first byte is in refuse
+type failingBatchWriter struct {
+	refuse map[byte]error
+}
+
+func (w *failingBatchWriter) WriteBatch(bufs [][]byte, _ []netip.AddrPort, failed func(int, error)) (int, error) {
+	written := 0
+	for i, b := range bufs {
+		if err, ok := w.refuse[b[0]]; ok {
+			failed(i, err)
+			continue
+		}
+		written++
+	}
+	return written, nil
+}
+
+func TestSendBatchReportsFailedTags(t *testing.T) {
+	errNoRoute := errors.New("no route")
+	fw := &failingBatchWriter{refuse: map[byte]error{2: errNoRoute, 3: errNoRoute}}
+
+	b := NewSendBatch(fw, 4, 64)
+
+	dsts := []netip.AddrPort{
+		netip.MustParseAddrPort("10.0.0.1:4242"),
+		netip.MustParseAddrPort("[fd00::2]:4242"),
+		netip.MustParseAddrPort("[fd00::3]:4242"),
+		netip.MustParseAddrPort("10.0.0.4:4242"),
+	}
+	for i, dst := range dsts {
+		pkt := append(b.Reserve(8)[:0], byte(i+1))
+		b.Commit(pkt, dst, uint32(100+i))
+	}
+
+	written, err := b.Flush()
+	if err != nil {
+		t.Fatalf("Flush: %v", err)
+	}
+	if written != 2 {
+		t.Errorf("written = %d, want 2", written)
+	}
+	want := []Failure{{101, dsts[1], errNoRoute}, {102, dsts[2], errNoRoute}}
+	if got := b.Failures(); !slices.Equal(got, want) {
+		t.Errorf("failures = %v, want %v", got, want)
+	}
+
+	// Tags do not leak into the next flush
+	b.Commit(append(b.Reserve(8)[:0], 2), dsts[0], 7)
+	if _, err := b.Flush(); err != nil {
+		t.Fatalf("Flush: %v", err)
+	}
+	if want := []Failure{{7, dsts[0], errNoRoute}}; !slices.Equal(b.Failures(), want) {
+		t.Errorf("second flush failures = %v, want %v", b.Failures(), want)
+	}
+
+	// A clean flush forgets the last one's failures
+	b.Commit(append(b.Reserve(8)[:0], 1), dsts[0], 8)
+	if _, err := b.Flush(); err != nil {
+		t.Fatalf("Flush: %v", err)
+	}
+	if got := b.Failures(); len(got) != 0 {
+		t.Errorf("clean flush failures = %v, want none", got)
+	}
+}