From 0ff077fea3c60cddc06e329dca1091ef1e15bd9c Mon Sep 17 00:00:00 2001 From: JackDoan Date: Tue, 20 Oct 2026 13:10:31 -0500 Subject: [PATCH] Probe a tunnel's other remotes when its current one is unreachable --- control_tester.go | 7 ++ e2e/dual_stack_roam_test.go | 240 ++++++++++++++++++++++++++++++++++++ e2e/router/router.go | 24 ++-- hostmap.go | 30 +++++ inside.go | 4 + udp/udp_tester.go | 59 ++++++++- udp/unreachable.go | 19 +++ udp/unreachable_windows.go | 18 +++ 8 files changed, 383 insertions(+), 18 deletions(-) create mode 100644 e2e/dual_stack_roam_test.go create mode 100644 udp/unreachable.go create mode 100644 udp/unreachable_windows.go diff --git a/control_tester.go b/control_tester.go index c4f96f20..59c3447b 100644 --- a/control_tester.go +++ b/control_tester.go @@ -117,6 +117,13 @@ func (c *Control) SetUDPAddr(addr netip.AddrPort) { c.f.outside.(*udp.TesterConn).SetAddr(addr) } +// SetDualStackUDPAddrs gives this node one underlay address per family, sending from whichever matches the +// destination. An invalid AddrPort leaves the node without that family, sends to it fail with ENETUNREACH. Register +// any address the node did not start on with the router as well or nothing will route back. +func (c *Control) SetDualStackUDPAddrs(v4, v6 netip.AddrPort) { + c.f.outside.(*udp.TesterConn).SetDualStackAddrs(v4, v6) +} + // SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of // whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker. func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) { diff --git a/e2e/dual_stack_roam_test.go b/e2e/dual_stack_roam_test.go new file mode 100644 index 00000000..9511d589 --- /dev/null +++ b/e2e/dual_stack_roam_test.go @@ -0,0 +1,240 @@ +//go:build e2e_testing + +package e2e + +import ( + "net/netip" + "sync/atomic" + "testing" + "time" + + "github.com/slackhq/nebula" + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/e2e/router" + "github.com/slackhq/nebula/udp" +) + +// TestDualStackRoamToV4Only measures how long a laptop's tunnels take to come back after it moves from a dual stack +// network, where its tunnels formed over IPv6, to one with no IPv6 service at all: the office to hotel wifi. Every +// peer still has an IPv4 address the laptop can reach, so nothing about the move should strand a tunnel. +// +// It is a measurement, not a pass/fail assertion, and it runs on the production connection manager timers. +// +// go test -tags e2e_testing -v -run TestDualStackRoamToV4Only ./e2e/ +func TestDualStackRoamToV4Only(t *testing.T) { + for _, tc := range []struct { + name string + meSends bool + themSends bool + rebind bool + lhOverV4 bool + }{ + {name: "both send", meSends: true, themSends: true}, + {name: "both send, rebind", meSends: true, themSends: true, rebind: true}, + {name: "laptop sends", meSends: true}, + {name: "peer sends", themSends: true}, + {name: "peer sends, rebind", themSends: true, rebind: true}, + {name: "peer sends, lh over v4", themSends: true, lhOverV4: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + fwd, rev := measureDualStackRoam(t, tc.meSends, tc.themSends, tc.rebind, tc.lhOverV4) + t.Logf("RESULT %-24s laptop->peer %-9v peer->laptop %-9v", tc.name, fmtRecovery(fwd), fmtRecovery(rev)) + }) + } +} + +func fmtRecovery(d time.Duration) string { + if d == 0 { + return "never" + } + return d.Round(time.Millisecond).String() +} + +type dualStackNode struct { + c *nebula.Control + vpnAddr netip.Addr + v4, v6 netip.AddrPort + adv atomic.Pointer[[]netip.Addr] +} + +// move puts the node on a new network, advertising only what it can be reached on there +func (n *dualStackNode) move(v4, v6 netip.AddrPort) { + n.v4, n.v6 = v4, v6 + n.c.SetDualStackUDPAddrs(v4, v6) + var adv []netip.Addr + for _, a := range []netip.AddrPort{v4, v6} { + if a.IsValid() { + adv = append(adv, a.Addr()) + } + } + n.adv.Store(&adv) +} + +func newDualStackNode(ca cert.Certificate, caKey []byte, name, vpnNet string, v4, v6 netip.AddrPort, overrides m) *dualStackNode { + c, vpnNets, _, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, name, vpnNet, v4, overrides) + n := &dualStackNode{c: c, vpnAddr: vpnNets[0].Addr()} + n.move(v4, v6) + c.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr { + return *n.adv.Load() + }) + return n +} + +// measureDualStackRoam returns how long after the move the first packet made it laptop->peer and peer->laptop, zero +// for a direction that never recovered. +func measureDualStackRoam(t *testing.T, meSends, themSends, rebind, lhOverV4 bool) (time.Duration, time.Duration) { + t.Helper() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + + ap := netip.MustParseAddrPort + lhV4, lhV6 := ap("10.0.0.1:4242"), ap("[fd00::1]:4242") + lh := newDualStackNode(ca, caKey, "lh", "10.128.0.1/24", lhV4, lhV6, m{ + "lighthouse": m{"am_lighthouse": true}, + }) + + // Production timers rather than the 2s the harness defaults to, this is about how long a real laptop waits + peerCfg := m{ + "lighthouse": m{ + "hosts": []any{lh.vpnAddr.String()}, + "interval": 10, + }, + "static_host_map": m{ + lh.vpnAddr.String(): []any{lhV4.String(), lhV6.String()}, + }, + "timers": m{ + "connection_alive_interval": 5, + "pending_deletion_interval": 10, + }, + } + + me := newDualStackNode(ca, caKey, "me", "10.128.0.2/24", ap("10.0.0.2:4242"), ap("[fd00::2]:4242"), peerCfg) + them := newDualStackNode(ca, caKey, "them", "10.128.0.3/24", ap("10.0.0.3:4242"), ap("[fd00::3]:4242"), peerCfg) + + r := router.NewR(t, lh.c, me.c, them.c) + r.CancelFlowLogs() + for _, n := range []*dualStackNode{lh, me, them} { + r.AddRoute(n.v6.Addr(), n.v6.Port(), n.c) + } + defer func() { + lh.c.Stop() + me.c.Stop() + them.c.Stop() + }() + + lh.c.Start() + me.c.Start() + them.c.Start() + + // At the office IPv6 wins every race, so everything forms over it. The lighthouse tunnel optionally does not, + // for a static_host_map that only lists the lighthouse's IPv4 address. + officeV6 := func(p *udp.Packet, _ *nebula.Control) router.ExitType { + lhTraffic := p.To == lhV4 || p.To == lhV6 || p.From == lhV4 || p.From == lhV6 + if p.To.Addr().Is4() == (lhOverV4 && lhTraffic) { + return router.KeepRouting + } + return router.Drop + } + r.RouteForAllExitFuncOrTimeout(time.Second, officeV6) + + me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("establish"))) + them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("establish"))) + r.RouteForAllExitFuncOrTimeout(time.Second*2, officeV6) + drainTun(me.c) + drainTun(them.c) + + for _, want := range []struct { + name string + c *nebula.Control + vpnAddr netip.Addr + v6 bool + }{ + {"laptop->peer", me.c, them.vpnAddr, true}, + {"peer->laptop", them.c, me.vpnAddr, true}, + {"laptop->lh", me.c, lh.vpnAddr, !lhOverV4}, + } { + hi := want.c.GetHostInfoByVpnAddr(want.vpnAddr, false) + if hi == nil { + t.Fatalf("%s: no tunnel at the office", want.name) + } + if hi.CurrentRemote.Addr().Is6() != want.v6 { + t.Fatalf("%s: tunnel formed over %v, the scenario wants v6=%v", want.name, hi.CurrentRemote, want.v6) + } + } + + // Hotel wifi: a new NAT'd IPv4 address and no IPv6 at all. The office addresses are gone, nothing reaches them. + officeV4, officeV6Addr := me.v4, me.v6 + hotel := ap("10.0.0.99:4242") + me.move(hotel, netip.AddrPort{}) + r.AddRoute(hotel.Addr(), hotel.Port(), me.c) + + start := time.Now() + if rebind { + me.c.RebindUDPServer() + } + + var fwd, rev atomic.Int64 + stop := make(chan struct{}) + defer close(stop) + watchTun(them.c, stop, start, &fwd) + watchTun(me.c, stop, start, &rev) + + go func() { + tick := time.NewTicker(time.Millisecond * 200) + defer tick.Stop() + for { + select { + case <-stop: + return + case <-tick.C: + if meSends { + me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("retry"))) + } + if themSends { + them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("retry"))) + } + } + } + }() + + // When only one side has traffic, the other direction recovering is a bonus we still report but do not wait for + done := func() bool { + return (!meSends || fwd.Load() != 0) && (!themSends || rev.Load() != 0) + } + r.RouteForAllExitFuncOrTimeout(time.Second*90, func(p *udp.Packet, _ *nebula.Control) router.ExitType { + if p.To == officeV4 || p.To == officeV6Addr { + return router.Drop + } + if done() { + return router.ExitNow + } + return router.KeepRouting + }) + + return time.Duration(fwd.Load()), time.Duration(rev.Load()) +} + +func drainTun(c *nebula.Control) { + for { + select { + case <-c.GetTunTxChan(): + default: + return + } + } +} + +// watchTun records how long after start the first packet came out of c's tun, and keeps the tun drained after +func watchTun(c *nebula.Control, stop chan struct{}, start time.Time, first *atomic.Int64) { + go func() { + for { + select { + case <-stop: + return + case <-c.GetTunTxChan(): + first.CompareAndSwap(0, int64(time.Since(start))) + } + } + }() +} diff --git a/e2e/router/router.go b/e2e/router/router.go index 0a29f042..30055555 100644 --- a/e2e/router/router.go +++ b/e2e/router/router.go @@ -497,11 +497,10 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) [] // Nope, lets push the sender along case p := <-udpTx: r.Lock() - a := sender.GetUDPAddr() - c := r.getControl(a, p.To, p) + c := r.getControl(p) if c == nil { r.Unlock() - panic("No control for udp tx " + a.String()) + panic("No control for udp tx " + p.To.String()) } fp := r.unlockedInjectFlow(sender, c, p, false) c.InjectUDPPacket(p) // copies internally; original is ours to release @@ -622,8 +621,7 @@ func (r *R) stopFanInWorkers() { func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) { r.Lock() defer r.Unlock() - a := from.GetUDPAddr() - c := r.getControl(a, p.To, p) + c := r.getControl(p) if c == nil { panic(fmt.Sprintf("No control for udp tx %s", p.To)) } @@ -671,7 +669,7 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) { panic(err) } - receiver := r.getControl(sender.GetUDPAddr(), p.To, p) + receiver := r.getControl(p) if receiver == nil { r.Unlock() panic("Can't RouteExitFunc for host: " + p.To.String()) @@ -767,7 +765,7 @@ func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc) r.Lock() p := rx.Interface().(*udp.Packet) - receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p) + receiver := r.getControl(p) if receiver == nil { r.Unlock() panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String()) @@ -876,7 +874,7 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) { r.Lock() p := rx.Interface().(*udp.Packet) - receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p) + receiver := r.getControl(p) if receiver == nil { r.Unlock() panic("Can't RouteForAllExitFunc for host: " + p.To.String()) @@ -939,7 +937,7 @@ func (r *R) FlushAll() { }) for { - x, rx, ok := reflect.Select(sc) + _, rx, ok := reflect.Select(sc) if !ok { return } @@ -947,7 +945,7 @@ func (r *R) FlushAll() { p := rx.Interface().(*udp.Packet) - receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p) + receiver := r.getControl(p) if receiver == nil { r.Unlock() panic("Can't FlushAll for host: " + p.To.String()) @@ -958,9 +956,11 @@ func (r *R) FlushAll() { } } -// getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change +// getControl performs or seeds NAT translation and returns the control for p.To, p from fields may change // This is an internal router function, the caller must hold the lock -func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control { +func (r *R) getControl(p *udp.Packet) *nebula.Control { + // Key on the address the packet actually left from, a dual stack sender has one per family + fromAddr, toAddr := p.From, p.To if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok { p.From = newAddr } diff --git a/hostmap.go b/hostmap.go index 24dc6411..60505cc2 100644 --- a/hostmap.go +++ b/hostmap.go @@ -264,6 +264,9 @@ type HostInfo struct { // This is used to limit lighthouse re-queries in chatty clients nextLHQuery atomic.Int64 + // nextUnreachableProbe is the earliest we will probe our other remotes again after failing to reach the current one + nextUnreachableProbe atomic.Int64 + // lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally // Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator // This is used to avoid an attack where a handshake packet is replayed after some time @@ -766,6 +769,33 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac } } +// unreachableProbeInterval bounds how often a tunnel whose remote we cannot reach probes its other remotes +const unreachableProbeInterval = time.Second + +// remoteUnreachable is called when a send to our current remote failed because we have no route to it, typically +// after moving to a network without that address family. Sends a test packet to every other remote we know for this +// host: the first one that arrives roams the peer over to wherever we are now, and its reply roams us onto that path. +func (i *HostInfo) remoteUnreachable(ifce *Interface, dead netip.AddrPort) { + if i.remotes == nil || i.ConnectionState == nil { + return + } + + now := time.Now().UnixNano() + next := i.nextUnreachableProbe.Load() + if now < next || !i.nextUnreachableProbe.CompareAndSwap(next, now+int64(unreachableProbeInterval)) { + return + } + + nb := make([]byte, 12) + out := make([]byte, mtu) + i.remotes.ForEach(ifce.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) { + if addr == dead { + return + } + ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), nb, out) + }) +} + // Bits within HostInfo.state, everything above stateEpochShift is the epoch const ( stateIn uint32 = 1 << iota diff --git a/inside.go b/inside.go index 696b9d97..f79702e6 100644 --- a/inside.go +++ b/inside.go @@ -15,6 +15,7 @@ import ( "github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/routing" + "github.com/slackhq/nebula/udp" ) func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) { @@ -611,6 +612,9 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType "udpAddr", hr, ) } + if udp.IsUnreachable(err) { + hostinfo.remoteUnreachable(f, hr) + } } } else { // Try to send via a relay diff --git a/udp/udp_tester.go b/udp/udp_tester.go index 292111ab..e556ce9e 100644 --- a/udp/udp_tester.go +++ b/udp/udp_tester.go @@ -6,10 +6,12 @@ import ( "context" "io" "log/slog" + "net" "net/netip" "os" "sync" "sync/atomic" + "syscall" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" @@ -63,10 +65,19 @@ func acquirePacket() *Packet { } } +// testerAddrs is where a TesterConn sends from. A single address host sends everything from addr, whatever the +// destination. A dual stack host sends from the address matching the destination's family and has no route at all +// for a family it holds no address in. +type testerAddrs struct { + addr netip.AddrPort + v4, v6 netip.AddrPort + dual bool +} + type TesterConn struct { - // addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can + // addrs is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can // move it mid-run to simulate roaming, so it is atomic rather than a plain field. - addr atomic.Pointer[netip.AddrPort] + addrs atomic.Pointer[testerAddrs] RxPackets chan *Packet // Packets to receive into nebula TxPackets chan *Packet // Packets transmitted outside by nebula @@ -94,14 +105,45 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) { return c, nil } -// GetAddr returns the underlay address this conn currently sends from. +// GetAddr returns the underlay address this conn currently sends from. For a dual stack conn that is the IPv4 +// address, or the IPv6 one when there is no IPv4 service. func (u *TesterConn) GetAddr() netip.AddrPort { - return *u.addr.Load() + return u.addrs.Load().addr } // SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network. func (u *TesterConn) SetAddr(addr netip.AddrPort) { - u.addr.Store(&addr) + u.addrs.Store(&testerAddrs{addr: addr}) +} + +// SetDualStackAddrs gives this conn one underlay address per family, standing in for a host whose routing table picks +// the source by destination. Pass an invalid AddrPort for a family the network does not offer, sends to that family +// then fail with ENETUNREACH like they would on a host with no route for it. +func (u *TesterConn) SetDualStackAddrs(v4, v6 netip.AddrPort) { + a := &testerAddrs{v4: v4, v6: v6, dual: true} + if v4.IsValid() { + a.addr = v4 + } else { + a.addr = v6 + } + u.addrs.Store(a) +} + +// sourceFor returns the address a send to dst leaves from +func (u *TesterConn) sourceFor(dst netip.AddrPort) (netip.AddrPort, error) { + a := u.addrs.Load() + if !a.dual { + return a.addr, nil + } + + src := a.v6 + if dst.Addr().Unmap().Is4() { + src = a.v4 + } + if !src.IsValid() { + return netip.AddrPort{}, &net.OpError{Op: "write", Net: "udp", Addr: net.UDPAddrFromAddrPort(dst), Err: syscall.ENETUNREACH} + } + return src, nil } // Send will place a UdpPacket onto the receive queue for nebula to consume @@ -153,6 +195,11 @@ func (u *TesterConn) Get(block bool) *Packet { //********************************************************************************************************************// func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error { + from, err := u.sourceFor(addr) + if err != nil { + return err + } + p := acquirePacket() if cap(p.Data) < len(b) { p.Data = make([]byte, len(b)) @@ -160,7 +207,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error { p.Data = p.Data[:len(b)] } copy(p.Data, b) - p.From = u.GetAddr() + p.From = from p.To = addr select { case <-u.done: diff --git a/udp/unreachable.go b/udp/unreachable.go new file mode 100644 index 00000000..ccec0fc4 --- /dev/null +++ b/udp/unreachable.go @@ -0,0 +1,19 @@ +//go:build !windows + +package udp + +import ( + "errors" + "syscall" +) + +// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it +// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send +// to an IPv6 remote into one of these. +func IsUnreachable(err error) bool { + return errors.Is(err, syscall.ENETUNREACH) || + errors.Is(err, syscall.EHOSTUNREACH) || + errors.Is(err, syscall.EADDRNOTAVAIL) || + errors.Is(err, syscall.EAFNOSUPPORT) || + errors.Is(err, ErrInvalidIPv6RemoteForSocket) +} diff --git a/udp/unreachable_windows.go b/udp/unreachable_windows.go new file mode 100644 index 00000000..fe8178a7 --- /dev/null +++ b/udp/unreachable_windows.go @@ -0,0 +1,18 @@ +package udp + +import ( + "errors" + + "golang.org/x/sys/windows" +) + +// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it +// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send +// to an IPv6 remote into one of these. +func IsUnreachable(err error) bool { + return errors.Is(err, windows.WSAENETUNREACH) || + errors.Is(err, windows.WSAEHOSTUNREACH) || + errors.Is(err, windows.WSAEADDRNOTAVAIL) || + errors.Is(err, windows.WSAEAFNOSUPPORT) || + errors.Is(err, ErrInvalidIPv6RemoteForSocket) +}