Probe a tunnel's other remotes when its current one is unreachable

control_tester.go

@@ -117,6 +117,13 @@ func (c *Control) SetUDPAddr(addr netip.AddrPort) {
 	c.f.outside.(*udp.TesterConn).SetAddr(addr)
 }
 
+// SetDualStackUDPAddrs gives this node one underlay address per family, sending from whichever matches the
+// destination. An invalid AddrPort leaves the node without that family, sends to it fail with ENETUNREACH. Register
+// any address the node did not start on with the router as well or nothing will route back.
+func (c *Control) SetDualStackUDPAddrs(v4, v6 netip.AddrPort) {
+	c.f.outside.(*udp.TesterConn).SetDualStackAddrs(v4, v6)
+}
+
 // SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
 // whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
 func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {

e2e/dual_stack_roam_test.go

@@ -0,0 +1,240 @@
+//go:build e2e_testing
+
+package e2e
+
+import (
+	"net/netip"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/udp"
+)
+
+// TestDualStackRoamToV4Only measures how long a laptop's tunnels take to come back after it moves from a dual stack
+// network, where its tunnels formed over IPv6, to one with no IPv6 service at all: the office to hotel wifi. Every
+// peer still has an IPv4 address the laptop can reach, so nothing about the move should strand a tunnel.
+//
+// It is a measurement, not a pass/fail assertion, and it runs on the production connection manager timers.
+//
+//	go test -tags e2e_testing -v -run TestDualStackRoamToV4Only ./e2e/
+func TestDualStackRoamToV4Only(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		meSends   bool
+		themSends bool
+		rebind    bool
+		lhOverV4  bool
+	}{
+		{name: "both send", meSends: true, themSends: true},
+		{name: "both send, rebind", meSends: true, themSends: true, rebind: true},
+		{name: "laptop sends", meSends: true},
+		{name: "peer sends", themSends: true},
+		{name: "peer sends, rebind", themSends: true, rebind: true},
+		{name: "peer sends, lh over v4", themSends: true, lhOverV4: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fwd, rev := measureDualStackRoam(t, tc.meSends, tc.themSends, tc.rebind, tc.lhOverV4)
+			t.Logf("RESULT %-24s laptop->peer %-9v peer->laptop %-9v", tc.name, fmtRecovery(fwd), fmtRecovery(rev))
+		})
+	}
+}
+
+func fmtRecovery(d time.Duration) string {
+	if d == 0 {
+		return "never"
+	}
+	return d.Round(time.Millisecond).String()
+}
+
+type dualStackNode struct {
+	c       *nebula.Control
+	vpnAddr netip.Addr
+	v4, v6  netip.AddrPort
+	adv     atomic.Pointer[[]netip.Addr]
+}
+
+// move puts the node on a new network, advertising only what it can be reached on there
+func (n *dualStackNode) move(v4, v6 netip.AddrPort) {
+	n.v4, n.v6 = v4, v6
+	n.c.SetDualStackUDPAddrs(v4, v6)
+	var adv []netip.Addr
+	for _, a := range []netip.AddrPort{v4, v6} {
+		if a.IsValid() {
+			adv = append(adv, a.Addr())
+		}
+	}
+	n.adv.Store(&adv)
+}
+
+func newDualStackNode(ca cert.Certificate, caKey []byte, name, vpnNet string, v4, v6 netip.AddrPort, overrides m) *dualStackNode {
+	c, vpnNets, _, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, name, vpnNet, v4, overrides)
+	n := &dualStackNode{c: c, vpnAddr: vpnNets[0].Addr()}
+	n.move(v4, v6)
+	c.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
+		return *n.adv.Load()
+	})
+	return n
+}
+
+// measureDualStackRoam returns how long after the move the first packet made it laptop->peer and peer->laptop, zero
+// for a direction that never recovered.
+func measureDualStackRoam(t *testing.T, meSends, themSends, rebind, lhOverV4 bool) (time.Duration, time.Duration) {
+	t.Helper()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	ap := netip.MustParseAddrPort
+	lhV4, lhV6 := ap("10.0.0.1:4242"), ap("[fd00::1]:4242")
+	lh := newDualStackNode(ca, caKey, "lh", "10.128.0.1/24", lhV4, lhV6, m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	// Production timers rather than the 2s the harness defaults to, this is about how long a real laptop waits
+	peerCfg := m{
+		"lighthouse": m{
+			"hosts":    []any{lh.vpnAddr.String()},
+			"interval": 10,
+		},
+		"static_host_map": m{
+			lh.vpnAddr.String(): []any{lhV4.String(), lhV6.String()},
+		},
+		"timers": m{
+			"connection_alive_interval": 5,
+			"pending_deletion_interval": 10,
+		},
+	}
+
+	me := newDualStackNode(ca, caKey, "me", "10.128.0.2/24", ap("10.0.0.2:4242"), ap("[fd00::2]:4242"), peerCfg)
+	them := newDualStackNode(ca, caKey, "them", "10.128.0.3/24", ap("10.0.0.3:4242"), ap("[fd00::3]:4242"), peerCfg)
+
+	r := router.NewR(t, lh.c, me.c, them.c)
+	r.CancelFlowLogs()
+	for _, n := range []*dualStackNode{lh, me, them} {
+		r.AddRoute(n.v6.Addr(), n.v6.Port(), n.c)
+	}
+	defer func() {
+		lh.c.Stop()
+		me.c.Stop()
+		them.c.Stop()
+	}()
+
+	lh.c.Start()
+	me.c.Start()
+	them.c.Start()
+
+	// At the office IPv6 wins every race, so everything forms over it. The lighthouse tunnel optionally does not,
+	// for a static_host_map that only lists the lighthouse's IPv4 address.
+	officeV6 := func(p *udp.Packet, _ *nebula.Control) router.ExitType {
+		lhTraffic := p.To == lhV4 || p.To == lhV6 || p.From == lhV4 || p.From == lhV6
+		if p.To.Addr().Is4() == (lhOverV4 && lhTraffic) {
+			return router.KeepRouting
+		}
+		return router.Drop
+	}
+	r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
+
+	me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("establish")))
+	them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("establish")))
+	r.RouteForAllExitFuncOrTimeout(time.Second*2, officeV6)
+	drainTun(me.c)
+	drainTun(them.c)
+
+	for _, want := range []struct {
+		name    string
+		c       *nebula.Control
+		vpnAddr netip.Addr
+		v6      bool
+	}{
+		{"laptop->peer", me.c, them.vpnAddr, true},
+		{"peer->laptop", them.c, me.vpnAddr, true},
+		{"laptop->lh", me.c, lh.vpnAddr, !lhOverV4},
+	} {
+		hi := want.c.GetHostInfoByVpnAddr(want.vpnAddr, false)
+		if hi == nil {
+			t.Fatalf("%s: no tunnel at the office", want.name)
+		}
+		if hi.CurrentRemote.Addr().Is6() != want.v6 {
+			t.Fatalf("%s: tunnel formed over %v, the scenario wants v6=%v", want.name, hi.CurrentRemote, want.v6)
+		}
+	}
+
+	// Hotel wifi: a new NAT'd IPv4 address and no IPv6 at all. The office addresses are gone, nothing reaches them.
+	officeV4, officeV6Addr := me.v4, me.v6
+	hotel := ap("10.0.0.99:4242")
+	me.move(hotel, netip.AddrPort{})
+	r.AddRoute(hotel.Addr(), hotel.Port(), me.c)
+
+	start := time.Now()
+	if rebind {
+		me.c.RebindUDPServer()
+	}
+
+	var fwd, rev atomic.Int64
+	stop := make(chan struct{})
+	defer close(stop)
+	watchTun(them.c, stop, start, &fwd)
+	watchTun(me.c, stop, start, &rev)
+
+	go func() {
+		tick := time.NewTicker(time.Millisecond * 200)
+		defer tick.Stop()
+		for {
+			select {
+			case <-stop:
+				return
+			case <-tick.C:
+				if meSends {
+					me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("retry")))
+				}
+				if themSends {
+					them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("retry")))
+				}
+			}
+		}
+	}()
+
+	// When only one side has traffic, the other direction recovering is a bonus we still report but do not wait for
+	done := func() bool {
+		return (!meSends || fwd.Load() != 0) && (!themSends || rev.Load() != 0)
+	}
+	r.RouteForAllExitFuncOrTimeout(time.Second*90, func(p *udp.Packet, _ *nebula.Control) router.ExitType {
+		if p.To == officeV4 || p.To == officeV6Addr {
+			return router.Drop
+		}
+		if done() {
+			return router.ExitNow
+		}
+		return router.KeepRouting
+	})
+
+	return time.Duration(fwd.Load()), time.Duration(rev.Load())
+}
+
+func drainTun(c *nebula.Control) {
+	for {
+		select {
+		case <-c.GetTunTxChan():
+		default:
+			return
+		}
+	}
+}
+
+// watchTun records how long after start the first packet came out of c's tun, and keeps the tun drained after
+func watchTun(c *nebula.Control, stop chan struct{}, start time.Time, first *atomic.Int64) {
+	go func() {
+		for {
+			select {
+			case <-stop:
+				return
+			case <-c.GetTunTxChan():
+				first.CompareAndSwap(0, int64(time.Since(start)))
+			}
+		}
+	}()
+}

e2e/router/router.go

@@ -497,11 +497,10 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []
 		// Nope, lets push the sender along
 		case p := <-udpTx:
 			r.Lock()
-			a := sender.GetUDPAddr()
+			c := r.getControl(p)
-			c := r.getControl(a, p.To, p)
 			if c == nil {
 				r.Unlock()
-				panic("No control for udp tx " + a.String())
+				panic("No control for udp tx " + p.To.String())
 			}
 			fp := r.unlockedInjectFlow(sender, c, p, false)
 			c.InjectUDPPacket(p) // copies internally; original is ours to release
@@ -622,8 +621,7 @@ func (r *R) stopFanInWorkers() {
 func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
 	r.Lock()
 	defer r.Unlock()
-	a := from.GetUDPAddr()
+	c := r.getControl(p)
-	c := r.getControl(a, p.To, p)
 	if c == nil {
 		panic(fmt.Sprintf("No control for udp tx %s", p.To))
 	}
@@ -671,7 +669,7 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
 			panic(err)
 		}
 
-		receiver := r.getControl(sender.GetUDPAddr(), p.To, p)
+		receiver := r.getControl(p)
 		if receiver == nil {
 			r.Unlock()
 			panic("Can't RouteExitFunc for host: " + p.To.String())
@@ -767,7 +765,7 @@ func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc)
 
 		r.Lock()
 		p := rx.Interface().(*udp.Packet)
-		receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
+		receiver := r.getControl(p)
 		if receiver == nil {
 			r.Unlock()
 			panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
@@ -876,7 +874,7 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
 		r.Lock()
 
 		p := rx.Interface().(*udp.Packet)
-		receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
+		receiver := r.getControl(p)
 		if receiver == nil {
 			r.Unlock()
 			panic("Can't RouteForAllExitFunc for host: " + p.To.String())
@@ -939,7 +937,7 @@ func (r *R) FlushAll() {
 	})
 
 	for {
-		x, rx, ok := reflect.Select(sc)
+		_, rx, ok := reflect.Select(sc)
 		if !ok {
 			return
 		}
@@ -947,7 +945,7 @@ func (r *R) FlushAll() {
 
 		p := rx.Interface().(*udp.Packet)
 
-		receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
+		receiver := r.getControl(p)
 		if receiver == nil {
 			r.Unlock()
 			panic("Can't FlushAll for host: " + p.To.String())
@@ -958,9 +956,11 @@ func (r *R) FlushAll() {
 	}
 }
 
-// getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change
+// getControl performs or seeds NAT translation and returns the control for p.To, p from fields may change
 // This is an internal router function, the caller must hold the lock
-func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control {
+func (r *R) getControl(p *udp.Packet) *nebula.Control {
+	// Key on the address the packet actually left from, a dual stack sender has one per family
+	fromAddr, toAddr := p.From, p.To
 	if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
 		p.From = newAddr
 	}

hostmap.go

@@ -264,6 +264,9 @@ type HostInfo struct {
 	// This is used to limit lighthouse re-queries in chatty clients
 	nextLHQuery atomic.Int64
 
+	// nextUnreachableProbe is the earliest we will probe our other remotes again after failing to reach the current one
+	nextUnreachableProbe atomic.Int64
+
 	// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
 	// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
 	// This is used to avoid an attack where a handshake packet is replayed after some time
@@ -766,6 +769,33 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
 	}
 }
 
+// unreachableProbeInterval bounds how often a tunnel whose remote we cannot reach probes its other remotes
+const unreachableProbeInterval = time.Second
+
+// remoteUnreachable is called when a send to our current remote failed because we have no route to it, typically
+// after moving to a network without that address family. Sends a test packet to every other remote we know for this
+// host: the first one that arrives roams the peer over to wherever we are now, and its reply roams us onto that path.
+func (i *HostInfo) remoteUnreachable(ifce *Interface, dead netip.AddrPort) {
+	if i.remotes == nil || i.ConnectionState == nil {
+		return
+	}
+
+	now := time.Now().UnixNano()
+	next := i.nextUnreachableProbe.Load()
+	if now < next || !i.nextUnreachableProbe.CompareAndSwap(next, now+int64(unreachableProbeInterval)) {
+		return
+	}
+
+	nb := make([]byte, 12)
+	out := make([]byte, mtu)
+	i.remotes.ForEach(ifce.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) {
+		if addr == dead {
+			return
+		}
+		ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), nb, out)
+	})
+}
+
 // Bits within HostInfo.state, everything above stateEpochShift is the epoch
 const (
 	stateIn uint32 = 1 << iota

inside.go

@@ -15,6 +15,7 @@ import (
 	"github.com/slackhq/nebula/overlay/batch"
 	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
+	"github.com/slackhq/nebula/udp"
 )
 
 func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
@@ -611,6 +612,9 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 					"udpAddr", hr,
 				)
 			}
+			if udp.IsUnreachable(err) {
+				hostinfo.remoteUnreachable(f, hr)
+			}
 		}
 	} else {
 		// Try to send via a relay

udp/udp_tester.go

@@ -6,10 +6,12 @@ import (
 	"context"
 	"io"
 	"log/slog"
+	"net"
 	"net/netip"
 	"os"
 	"sync"
 	"sync/atomic"
+	"syscall"
 
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
@@ -63,10 +65,19 @@ func acquirePacket() *Packet {
 	}
 }
 
+// testerAddrs is where a TesterConn sends from. A single address host sends everything from addr, whatever the
+// destination. A dual stack host sends from the address matching the destination's family and has no route at all
+// for a family it holds no address in.
+type testerAddrs struct {
+	addr   netip.AddrPort
+	v4, v6 netip.AddrPort
+	dual   bool
+}
+
 type TesterConn struct {
-	// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
+	// addrs is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
 	// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
-	addr atomic.Pointer[netip.AddrPort]
+	addrs atomic.Pointer[testerAddrs]
 
 	RxPackets chan *Packet // Packets to receive into nebula
 	TxPackets chan *Packet // Packets transmitted outside by nebula
@@ -94,14 +105,45 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
 	return c, nil
 }
 
-// GetAddr returns the underlay address this conn currently sends from.
+// GetAddr returns the underlay address this conn currently sends from. For a dual stack conn that is the IPv4
+// address, or the IPv6 one when there is no IPv4 service.
 func (u *TesterConn) GetAddr() netip.AddrPort {
-	return *u.addr.Load()
+	return u.addrs.Load().addr
 }
 
 // SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
 func (u *TesterConn) SetAddr(addr netip.AddrPort) {
-	u.addr.Store(&addr)
+	u.addrs.Store(&testerAddrs{addr: addr})
+}
+
+// SetDualStackAddrs gives this conn one underlay address per family, standing in for a host whose routing table picks
+// the source by destination. Pass an invalid AddrPort for a family the network does not offer, sends to that family
+// then fail with ENETUNREACH like they would on a host with no route for it.
+func (u *TesterConn) SetDualStackAddrs(v4, v6 netip.AddrPort) {
+	a := &testerAddrs{v4: v4, v6: v6, dual: true}
+	if v4.IsValid() {
+		a.addr = v4
+	} else {
+		a.addr = v6
+	}
+	u.addrs.Store(a)
+}
+
+// sourceFor returns the address a send to dst leaves from
+func (u *TesterConn) sourceFor(dst netip.AddrPort) (netip.AddrPort, error) {
+	a := u.addrs.Load()
+	if !a.dual {
+		return a.addr, nil
+	}
+
+	src := a.v6
+	if dst.Addr().Unmap().Is4() {
+		src = a.v4
+	}
+	if !src.IsValid() {
+		return netip.AddrPort{}, &net.OpError{Op: "write", Net: "udp", Addr: net.UDPAddrFromAddrPort(dst), Err: syscall.ENETUNREACH}
+	}
+	return src, nil
 }
 
 // Send will place a UdpPacket onto the receive queue for nebula to consume
@@ -153,6 +195,11 @@ func (u *TesterConn) Get(block bool) *Packet {
 //********************************************************************************************************************//
 
 func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
+	from, err := u.sourceFor(addr)
+	if err != nil {
+		return err
+	}
+
 	p := acquirePacket()
 	if cap(p.Data) < len(b) {
 		p.Data = make([]byte, len(b))
@@ -160,7 +207,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
 		p.Data = p.Data[:len(b)]
 	}
 	copy(p.Data, b)
-	p.From = u.GetAddr()
+	p.From = from
 	p.To = addr
 	select {
 	case <-u.done:

udp/unreachable.go

@@ -0,0 +1,19 @@
+//go:build !windows
+
+package udp
+
+import (
+	"errors"
+	"syscall"
+)
+
+// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
+// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
+// to an IPv6 remote into one of these.
+func IsUnreachable(err error) bool {
+	return errors.Is(err, syscall.ENETUNREACH) ||
+		errors.Is(err, syscall.EHOSTUNREACH) ||
+		errors.Is(err, syscall.EADDRNOTAVAIL) ||
+		errors.Is(err, syscall.EAFNOSUPPORT) ||
+		errors.Is(err, ErrInvalidIPv6RemoteForSocket)
+}

udp/unreachable_windows.go

@@ -0,0 +1,18 @@
+package udp
+
+import (
+	"errors"
+
+	"golang.org/x/sys/windows"
+)
+
+// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
+// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
+// to an IPv6 remote into one of these.
+func IsUnreachable(err error) bool {
+	return errors.Is(err, windows.WSAENETUNREACH) ||
+		errors.Is(err, windows.WSAEHOSTUNREACH) ||
+		errors.Is(err, windows.WSAEADDRNOTAVAIL) ||
+		errors.Is(err, windows.WSAEAFNOSUPPORT) ||
+		errors.Is(err, ErrInvalidIPv6RemoteForSocket)
+}