Probe for another path when the data path finds a tunnel's remote unreachable

udp/conn.go

@@ -41,8 +41,9 @@ type Conn interface {
 	//
 	// Returns the number of packets successfully written. A destination the kernel rejects costs only
 	// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
+	// failed, when not nil, is called with the index of every packet a destination cost us and why.
 	// Not safe for concurrent use on the same Conn.
-	WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
+	WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
 	// ReloadConfig applies the listen settings that can change, r is nil at startup
 	ReloadConfig(c *config.C, r *config.Reload)
 	SupportsMultipleReaders() bool
@@ -66,7 +67,7 @@ func (NoopConn) SupportsMultipleReaders() bool {
 func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
 	return nil
 }
-func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
+func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
 	return len(bufs), nil
 }
 func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) {

udp/udp_darwin.go

@@ -139,7 +139,7 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
 	}
 }
 
-func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
+func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
 	// An un-sendable destination costs its own packet, never the ones behind it in the batch.
 	// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
 	// silently drops the rest of a burst (linux blocks instead). Poll for
@@ -150,6 +150,9 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
 			written++
 		} else {
 			u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
+			if failed != nil {
+				failed(i, err)
+			}
 		}
 	}
 	return written, nil

udp/udp_generic.go

@@ -44,7 +44,7 @@ func (u *GenericConn) WriteTo(b []byte, addr netip.AddrPort) error {
 	return err
 }
 
-func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
+func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
 	// An un-sendable destination costs its own packet, never the ones behind it in the batch.
 	written := 0
 	for i, b := range bufs {
@@ -52,6 +52,9 @@ func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
 			written++
 		} else {
 			u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
+			if failed != nil {
+				failed(i, err)
+			}
 		}
 	}
 	return written, nil

udp/udp_linux.go

@@ -358,8 +358,8 @@ func sendto(fd int, b []byte, addr netip.AddrPort, isV4 bool) error {
 
 // WriteBatch sends bufs via sendmmsg(2), coalescing same-destination runs into UDP-GSO superpackets when supported.
 // See batchWriter in udp_linux_writebatch.go for the mechanics.
-func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
-	return u.bw.WriteBatch(bufs, addrs)
+func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
+	return u.bw.WriteBatch(bufs, addrs, failed)
 }
 
 // writeSockaddr encodes addr into buf (which must be at least SizeofSockaddrInet6 bytes).

udp/udp_linux_fixes_test.go

@@ -3,6 +3,7 @@
 package udp
 
 import (
+	"errors"
 	"fmt"
 	"log/slog"
 	"net"
@@ -96,13 +97,22 @@ func TestWriteBatchBadFamilyDeliversOthers(t *testing.T) {
 	bufs := [][]byte{[]byte("AAA"), []byte("BBB"), []byte("CCC")}
 	addrs := []netip.AddrPort{good, bad, good}
 
-	n, err := sender.WriteBatch(bufs, addrs)
+	var failed []int
+	n, err := sender.WriteBatch(bufs, addrs, func(i int, err error) {
+		if !IsUnreachable(err) {
+			t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
+		}
+		failed = append(failed, i)
+	})
 	if err != nil {
 		t.Fatalf("WriteBatch returned error, want nil (bad dest should be isolated): %v", err)
 	}
 	if n != 2 {
 		t.Errorf("WriteBatch wrote %d packets, want 2 of 3 (the bad-family dest is the only casualty)", n)
 	}
+	if want := []int{1}; !slices.Equal(failed, want) {
+		t.Errorf("failed reported %v, want %v", failed, want)
+	}
 
 	got := map[string]bool{}
 	rx.SetReadDeadline(time.Now().Add(2 * time.Second))
@@ -154,7 +164,7 @@ func TestWriteBatchUnreachableDestDeliversOthers(t *testing.T) {
 	addrs := []netip.AddrPort{good, good, bad, good, good}
 
 	// The bad destination is reported, but only after every other packet has been attempted.
-	if _, err := sender.WriteBatch(bufs, addrs); err == nil {
+	if _, err := sender.WriteBatch(bufs, addrs, nil); err == nil {
 		t.Log("WriteBatch returned nil; kernel accepted the reserved address, delivery assertions still apply")
 	}
 
@@ -376,7 +386,7 @@ func TestWriteBatchPartialSendRewind(t *testing.T) {
 				return accept, nil
 			}
 
-			written, err := w.WriteBatch(bufs, addrs)
+			written, err := w.WriteBatch(bufs, addrs, nil)
 			if err != nil {
 				t.Fatalf("WriteBatch: %v", err)
 			}
@@ -429,13 +439,22 @@ func TestWriteBatchSkipUnroutableRunAccounting(t *testing.T) {
 				return accept, nil
 			}
 
-			written, err := w.WriteBatch(bufs, addrs)
+			var failed []int
+			written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
+				if !IsUnreachable(err) {
+					t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
+				}
+				failed = append(failed, i)
+			})
 			if err != nil {
 				t.Fatalf("WriteBatch: %v", err)
 			}
 			if written != 4 {
 				t.Errorf("written = %d, want 4 (the unroutable run is the only casualty)", written)
 			}
+			if want := []int{2}; !slices.Equal(failed, want) {
+				t.Errorf("failed reported %v, want %v", failed, want)
+			}
 			wantTags := []byte{1, 2, 4, 5}
 			if len(wire) != len(wantTags) {
 				t.Fatalf("wire got %d packets, want %d (dup or loss around the skip)", len(wire), len(wantTags))
@@ -486,13 +505,23 @@ func TestWriteBatchMidChunkRejectResumes(t *testing.T) {
 		}
 	}
 
-	written, err := w.WriteBatch(bufs, addrs)
+	var failed []int
+	written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
+		if !errors.Is(err, unix.EPERM) {
+			t.Errorf("packet %d failed with %v, want EPERM", i, err)
+		}
+		failed = append(failed, i)
+	})
 	if err != nil {
 		t.Fatalf("WriteBatch: %v", err)
 	}
 	if written != 3 {
 		t.Errorf("written = %d, want 3 (B's rejected run is the only casualty)", written)
 	}
+	// Every packet in B's run is reported, not just the entry
+	if want := []int{2, 3}; !slices.Equal(failed, want) {
+		t.Errorf("failed reported %v, want %v", failed, want)
+	}
 	wantTags := []byte{1, 2, 5}
 	if len(wire) != len(wantTags) {
 		t.Fatalf("wire got %d packets, want %d (dup or loss around the mid-chunk reject)", len(wire), len(wantTags))
@@ -543,7 +572,7 @@ func TestWriteBatchMidChunkEIODisablesGSOWithoutDup(t *testing.T) {
 		}
 	}
 
-	written, err := w.WriteBatch(bufs, addrs)
+	written, err := w.WriteBatch(bufs, addrs, nil)
 	if err != nil {
 		t.Fatalf("WriteBatch: %v", err)
 	}
@@ -571,7 +600,7 @@ func TestWriteBatchZeroProgress(t *testing.T) {
 	w.sendFn = func(start, n int) (int, error) { return 0, nil }
 	bufs := [][]byte{make([]byte, 100)}
 	addrs := []netip.AddrPort{netip.MustParseAddrPort("127.0.0.1:4242")}
-	if _, err := w.WriteBatch(bufs, addrs); err == nil {
+	if _, err := w.WriteBatch(bufs, addrs, nil); err == nil {
 		t.Fatal("WriteBatch = nil error on zero progress, want error")
 	}
 }
@@ -597,7 +626,7 @@ func TestWriteBatchEIODisablesGSOAndReplays(t *testing.T) {
 		return n, nil
 	}
 
-	written, err := w.WriteBatch(bufs, addrs)
+	written, err := w.WriteBatch(bufs, addrs, nil)
 	if err != nil {
 		t.Fatalf("WriteBatch: %v", err)
 	}
@@ -674,7 +703,7 @@ func TestGSOEngagesOnLoopback(t *testing.T) {
 		addrs[i] = dst
 	}
 
-	written, err := sc.WriteBatch(bufs, addrs)
+	written, err := sc.WriteBatch(bufs, addrs, nil)
 	if err != nil {
 		t.Fatalf("WriteBatch: %v", err)
 	}

udp/udp_linux_offloads_test.go

@@ -86,7 +86,7 @@ func TestWriteBatchOffloadsDisabledScripted(t *testing.T) {
 		addrs[i] = dst
 	}
 
-	written, err := w.WriteBatch(bufs, addrs)
+	written, err := w.WriteBatch(bufs, addrs, nil)
 	if err != nil {
 		t.Fatalf("WriteBatch: %v", err)
 	}
@@ -163,7 +163,7 @@ func TestOffloadsDisabledOnLoopback(t *testing.T) {
 		addrs[i] = dst
 	}
 
-	written, err := sc.WriteBatch(bufs, addrs)
+	written, err := sc.WriteBatch(bufs, addrs, nil)
 	if err != nil {
 		t.Fatalf("WriteBatch: %v", err)
 	}

udp/udp_linux_writebatch.go

@@ -190,8 +190,8 @@ func parseRelease(r string) (major, minor int) {
 // its packets are dropped and the rest of the chunk resumes in place.
 //
 // Returns the number of packets sent. An error means the call itself failed.
-// A short count means some destinations were undeliverable.
-func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
+// A short count means some destinations were undeliverable, failed hears about each of their packets.
+func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
 	if len(bufs) != len(addrs) {
 		return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
 	}
@@ -233,6 +233,7 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
 				if w.l.Enabled(context.Background(), slog.LevelDebug) {
 					w.l.Debug("skipping unroutable batch destination", "udpAddr", addrs[i], "packets", runLen, "error", err)
 				}
+				reportFailed(failed, i, i+runLen, err)
 				i += runLen
 				continue
 			}
@@ -300,12 +301,23 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
 					"gso", w.gsoSupported,
 				)
 			}
+			reportFailed(failed, w.entryEnd[done]-w.entryPkts[done], w.entryEnd[done], serr)
 			done++
 		}
 	}
 	return written, nil
 }
 
+// reportFailed tells failed, if anyone is listening, that every packet in bufs[start:end] was lost to err
+func reportFailed(failed func(i int, err error), start, end int, err error) {
+	if failed == nil {
+		return
+	}
+	for i := start; i < end; i++ {
+		failed(i, err)
+	}
+}
+
 // planRun returns the length of the run starting at start and its segment
 // size (len(bufs[start])). A run of length 1 carries no UDP_SEGMENT cmsg
 // and is sent as a plain datagram; without GSO support planRun always returns 1.

udp/udp_linux_writebatch_alloc_test.go

@@ -80,11 +80,11 @@ func TestWriteBatchNoAllocs(t *testing.T) {
 
 			var werr error
 			// Warm-up outside the measured runs.
-			if _, err := tx.WriteBatch(bufs, addrs); err != nil {
+			if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
 				t.Fatalf("WriteBatch warm-up: %v", err)
 			}
 			allocs := testing.AllocsPerRun(100, func() {
-				if _, err := tx.WriteBatch(bufs, addrs); err != nil {
+				if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
 					werr = err
 				}
 			})

udp/udp_rio_windows.go

@@ -319,7 +319,7 @@ func (u *RIOConn) WriteTo(buf []byte, ip netip.AddrPort) error {
 	return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
 }
 
-func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
+func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
 	// An un-sendable destination costs its own packet, never the ones behind it in the batch.
 	written := 0
 	for i, b := range bufs {
@@ -327,6 +327,9 @@ func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
 			written++
 		} else {
 			u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
+			if failed != nil {
+				failed(i, err)
+			}
 		}
 	}
 	return written, nil

udp/udp_tester.go

@@ -217,13 +217,16 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
 		return nil
 	}
 }
-func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
+func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
 	written := 0
 	for i, b := range bufs {
 		if err := u.WriteTo(b, addrs[i]); err == nil {
 			written++
 		} else {
 			u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
+			if failed != nil {
+				failed(i, err)
+			}
 		}
 	}
 	return written, nil