diff --git a/connection_manager.go b/connection_manager.go index 88f31321..0802d6a3 100644 --- a/connection_manager.go +++ b/connection_manager.go @@ -44,6 +44,11 @@ type connectionManager struct { inactivityTimeout atomic.Int64 dropInactive atomic.Bool + // Wake-from-sleep handling, sampled once per tick in Start + wakeDetector *wakeDetector + clearOnWake atomic.Bool + wakeClearThreshold atomic.Int64 + l *slog.Logger } @@ -54,6 +59,7 @@ func newConnectionManagerFromConfig(l *slog.Logger, c *config.C, hm *HostMap, p punchy: p, relayUsed: make(map[uint32]struct{}), relayUsedLock: &sync.RWMutex{}, + wakeDetector: newWakeDetector(), } cm.reload(c, true) @@ -98,12 +104,38 @@ func (cm *connectionManager) reload(c *config.C, initial bool) { ) } } + + if initial || c.HasChanged("tunnels.clear_on_wake") { + old := cm.clearOnWake.Load() + cm.clearOnWake.Store(c.GetBool("tunnels.clear_on_wake", true)) + if !initial { + cm.l.Info("Clear on wake setting has changed", + "oldBool", old, + "newBool", cm.clearOnWake.Load(), + ) + } + } + + if initial || c.HasChanged("tunnels.wake_clear_threshold") { + old := cm.getWakeClearThreshold() + cm.wakeClearThreshold.Store((int64)(c.GetDuration("tunnels.wake_clear_threshold", 30*time.Second))) + if !initial { + cm.l.Info("Wake clear threshold has changed", + "oldDuration", old, + "newDuration", cm.getWakeClearThreshold(), + ) + } + } } func (cm *connectionManager) getInactivityTimeout() time.Duration { return (time.Duration)(cm.inactivityTimeout.Load()) } +func (cm *connectionManager) getWakeClearThreshold() time.Duration { + return (time.Duration)(cm.wakeClearThreshold.Load()) +} + func (cm *connectionManager) In(h *HostInfo) { h.in.Store(true) } @@ -136,6 +168,73 @@ func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time) return in, out } +// checkWake runs once per tick and clears every tunnel when the machine has just returned from system sleep. +// Tunnels rarely survive a suspend: our NAT mappings have expired and our address has usually changed, so every +// established hostinfo is a corpse that will eat 15-20s of traffic checks before the wheel declares it dead. +// Clearing now means the first packet after wake starts a fresh handshake immediately. +// +// The suspend itself costs nothing here: the ticker driving us is frozen with the rest of the process and this +// fires within one tick of resume. +func (cm *connectionManager) checkWake() { + slept, ok := cm.wakeDetector.Sample() + if !ok || slept == 0 { + return + } + + // The clock pair is read non-atomically, so scheduling jitter shows up as tiny sub-millisecond "sleeps". + // Keep the floor well above that so a zero/nonsense threshold can't clear tunnels on every tick. + threshold := max(cm.getWakeClearThreshold(), time.Second) + + if slept < threshold { + // Short suspends (lid closed and quickly reopened) often come back before NAT state expires; those + // tunnels may well be alive, leave them to the normal traffic checks. + if slept >= time.Second { + cm.l.Debug("Woke from sleep below the clear threshold, leaving tunnels alone", + "sleptFor", slept, + "threshold", threshold, + ) + } + return + } + + if !cm.clearOnWake.Load() { + cm.l.Info("Woke from sleep, tunnels.clear_on_wake is disabled so tunnels are left to the normal traffic checks", "sleptFor", slept) + return + } + + closed := cm.clearAllTunnels() + cm.l.Info("Woke from sleep, cleared tunnels", "sleptFor", slept, "tunnelsCleared", closed) + + // Our public address almost certainly changed; get it to the lighthouses as soon as possible so peers can + // find us again. The update rides over a fresh lighthouse handshake. If the network isn't back up yet these + // sends fail harmlessly and the periodic update worker retries within lighthouse.interval. + cm.intf.lightHouse.TriggerUpdate() +} + +// clearAllTunnels closes every tunnel in the hostmap locally, without notifying the remotes. It is the wake-from- +// sleep counterpart to Control.CloseAllTunnels: after a suspend the remotes stopped hearing from us long ago, and +// close packets fired into a network that may not even be up yet are wasted, so we only tear down our own state +// and let the next packet to each host start a fresh handshake. +func (cm *connectionManager) clearAllTunnels() int { + cm.hostMap.RLock() + hostinfos := make([]*HostInfo, 0, len(cm.hostMap.Indexes)) + for _, h := range cm.hostMap.Indexes { + hostinfos = append(hostinfos, h) + } + cm.hostMap.RUnlock() + + for _, h := range hostinfos { + cm.intf.closeTunnel(h) + } + + // With every tunnel gone no relay can be in use, drop the usage tracking wholesale. + cm.relayUsedLock.Lock() + clear(cm.relayUsed) + cm.relayUsedLock.Unlock() + + return len(hostinfos) +} + func (cm *connectionManager) Start(ctx context.Context) { clockSource := time.NewTicker(cm.trafficTimer.t.tickDuration) defer clockSource.Stop() @@ -150,6 +249,7 @@ func (cm *connectionManager) Start(ctx context.Context) { return case now := <-clockSource.C: + cm.checkWake() cm.trafficTimer.Advance(now) for { localIndex, has := cm.trafficTimer.Purge() diff --git a/connection_manager_test.go b/connection_manager_test.go index e167e5f2..426633fc 100644 --- a/connection_manager_test.go +++ b/connection_manager_test.go @@ -501,3 +501,85 @@ func (d *dummyCert) MarshalJSON() ([]byte, error) { func (d *dummyCert) Copy() cert.Certificate { return d } + +func TestConnectionManager_WakeClear(t *testing.T) { + l := test.NewLogger() + localrange := netip.MustParsePrefix("10.1.1.1/24") + vpnIp := netip.MustParseAddr("172.1.1.2") + preferredRanges := []netip.Prefix{localrange} + + // Very incomplete mock objects + hostMap := newHostMap(l) + hostMap.preferredRanges.Store(&preferredRanges) + + cs := &CertState{ + initiatingVersion: cert.Version1, + privateKey: []byte{}, + v1Cert: &dummyCert{version: cert.Version1}, + v1Credential: nil, + } + + lh := newTestLighthouse() + ifce := &Interface{ + hostMap: hostMap, + inside: &overlaytest.NoopTun{}, + outside: &udp.NoopConn{}, + firewall: &Firewall{}, + lightHouse: lh, + pki: &PKI{}, + handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig), + l: l, + } + ifce.pki.cs.Store(cs) + + // Create manager + conf := config.NewC(test.NewLogger()) + punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil) + nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy) + nc.intf = ifce + + // Drive the wake detector from a fake clock pair + suspended := time.Duration(0) + nc.wakeDetector = &wakeDetector{read: func() (time.Duration, bool) { return suspended, true }} + nc.checkWake() // primes the baseline + + addTunnel := func(localIndex uint32) *HostInfo { + hostinfo := &HostInfo{ + vpnAddrs: []netip.Addr{vpnIp}, + localIndexId: localIndex, + remoteIndexId: 9901, + } + hostinfo.ConnectionState = &ConnectionState{ + myCert: &dummyCert{version: cert.Version1}, + } + nc.hostMap.unlockedAddHostInfo(hostinfo, ifce) + return hostinfo + } + + addTunnel(1099) + nc.RelayUsed(5000) + + // No suspend, nothing happens + nc.checkWake() + assert.Contains(t, nc.hostMap.Indexes, uint32(1099)) + + // A suspend below the threshold leaves tunnels alone + suspended += 5 * time.Second + nc.checkWake() + assert.Contains(t, nc.hostMap.Indexes, uint32(1099)) + + // A suspend past the threshold clears everything, including relay usage tracking + suspended += time.Hour + nc.checkWake() + assert.Empty(t, nc.hostMap.Indexes) + assert.Empty(t, nc.hostMap.Hosts) + assert.Empty(t, nc.relayUsed) + + // With clear_on_wake disabled the tunnels survive a long suspend + addTunnel(1100) + nc.clearOnWake.Store(false) + suspended += time.Hour + nc.checkWake() + assert.Contains(t, nc.hostMap.Indexes, uint32(1100)) + assert.Contains(t, nc.hostMap.Hosts, vpnIp) +} diff --git a/examples/config.yml b/examples/config.yml index 4f7fd1e7..a3291d90 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -390,6 +390,20 @@ logging: # This setting is reloadable #inactivity_timeout: 10m + # clear_on_wake controls whether all tunnels are immediately torn down (locally, without notifying the remotes) + # when the machine detects it has just woken from system sleep. Tunnels rarely survive a suspend: NAT mappings + # expire and the machine's address usually changes, so waiting for the normal liveness checks costs 15-20 seconds + # of black-holed traffic per tunnel after wake. Clearing them means the first packet after wake starts a fresh + # handshake right away. + # This setting is reloadable + #clear_on_wake: true + + # wake_clear_threshold is the minimum time the machine must have been suspended for clear_on_wake to act. + # Suspends shorter than this often come back before NAT state expires, so those tunnels may still be alive and + # are left to the normal liveness checks. Values below 1s are treated as 1s. + # This setting is reloadable + #wake_clear_threshold: 30s + # Nebula security group configuration firewall: # Action to take when a packet is not allowed by the firewall rules. diff --git a/wake_clock_darwin.go b/wake_clock_darwin.go new file mode 100644 index 00000000..7dc12e0b --- /dev/null +++ b/wake_clock_darwin.go @@ -0,0 +1,29 @@ +//go:build darwin + +package nebula + +import ( + "time" + + "golang.org/x/sys/unix" +) + +// suspendClockDelta returns CLOCK_MONOTONIC - CLOCK_UPTIME_RAW. On macOS CLOCK_MONOTONIC keeps counting across +// system sleep while CLOCK_UPTIME_RAW (mach_absolute_time) pauses, so the difference grows by time spent asleep. +// +// The pausing clock is read first so scheduling jitter between the two reads biases the delta positive; the +// wakeDetector clamps out the noise. +// +// Caveat: on Apple Silicon the hardware timebase keeps ticking through sleep, which can make both clocks advance +// and the spread stay flat, leaving this detector blind. That fails safe (no clears, behavior as before); IOKit +// power notifications are the follow-up for full coverage on those machines. +func suspendClockDelta() (time.Duration, bool) { + var uptime, mono unix.Timespec + if err := unix.ClockGettime(unix.CLOCK_UPTIME_RAW, &uptime); err != nil { + return 0, false + } + if err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &mono); err != nil { + return 0, false + } + return time.Duration(mono.Nano() - uptime.Nano()), true +} diff --git a/wake_clock_generic.go b/wake_clock_generic.go new file mode 100644 index 00000000..103ce068 --- /dev/null +++ b/wake_clock_generic.go @@ -0,0 +1,11 @@ +//go:build !linux && !darwin && !windows + +package nebula + +import "time" + +// suspendClockDelta reports that this platform has no usable clock pair for detecting system sleep; the wake +// detector stays dormant and dead tunnels are left to the normal traffic checks. +func suspendClockDelta() (time.Duration, bool) { + return 0, false +} diff --git a/wake_clock_linux.go b/wake_clock_linux.go new file mode 100644 index 00000000..697087fb --- /dev/null +++ b/wake_clock_linux.go @@ -0,0 +1,26 @@ +//go:build linux + +package nebula + +import ( + "time" + + "golang.org/x/sys/unix" +) + +// suspendClockDelta returns CLOCK_BOOTTIME - CLOCK_MONOTONIC. CLOCK_MONOTONIC pauses while the system is suspended +// and CLOCK_BOOTTIME does not, so the difference only ever grows, and only by time spent suspended. Both reads are +// vDSO calls, cheap enough for a hot ticker. +// +// The pausing clock is read first so scheduling jitter between the two reads biases the delta positive; the +// wakeDetector clamps out the noise. +func suspendClockDelta() (time.Duration, bool) { + var mono, boot unix.Timespec + if err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &mono); err != nil { + return 0, false + } + if err := unix.ClockGettime(unix.CLOCK_BOOTTIME, &boot); err != nil { + return 0, false + } + return time.Duration(boot.Nano() - mono.Nano()), true +} diff --git a/wake_clock_windows.go b/wake_clock_windows.go new file mode 100644 index 00000000..66d69971 --- /dev/null +++ b/wake_clock_windows.go @@ -0,0 +1,41 @@ +//go:build windows + +package nebula + +import ( + "sync" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + procQueryInterruptTime = windows.NewLazySystemDLL("kernelbase.dll").NewProc("QueryInterruptTime") + procQueryUnbiasedInterruptTime = windows.NewLazySystemDLL("kernel32.dll").NewProc("QueryUnbiasedInterruptTime") + + // QueryInterruptTime needs Windows 10; probe once and stay dormant on anything older. + wakeClockAvailable = sync.OnceValue(func() bool { + return procQueryInterruptTime.Find() == nil && procQueryUnbiasedInterruptTime.Find() == nil + }) +) + +// suspendClockDelta returns interrupt time minus unbiased interrupt time, both in 100ns units. The unbiased count +// excludes time the system spends suspended while the biased one includes it, so the difference grows by exactly +// the time spent asleep. +// +// The pausing (unbiased) clock is read first so scheduling jitter between the two reads biases the delta positive; +// the wakeDetector clamps out the noise. +func suspendClockDelta() (time.Duration, bool) { + if !wakeClockAvailable() { + return 0, false + } + + var unbiased, biased uint64 + if r1, _, _ := procQueryUnbiasedInterruptTime.Call(uintptr(unsafe.Pointer(&unbiased))); r1 == 0 { + return 0, false + } + // Returns void, cannot fail once resolved. + _, _, _ = procQueryInterruptTime.Call(uintptr(unsafe.Pointer(&biased))) + return time.Duration(int64(biased-unbiased)) * 100, true +} diff --git a/wake_detector.go b/wake_detector.go new file mode 100644 index 00000000..f978fba6 --- /dev/null +++ b/wake_detector.go @@ -0,0 +1,47 @@ +package nebula + +import "time" + +// wakeDetector notices when the machine has returned from system sleep and measures how long it was suspended. +// +// It samples the spread between two kernel clocks: one that pauses across a suspend and one that keeps counting +// (suspendClockDelta, per platform). While the machine is awake the spread is constant no matter how starved, +// stopped, or stepped this process is — SIGSTOP, debugger pauses, scheduler starvation, and NTP adjustments move +// both clocks together or neither, so none of them can fake a wake. A true suspend is the only thing that grows +// the spread, and it grows by exactly the time spent suspended. +// +// Sample is intended to piggyback on a ticker the caller already runs; it costs two clock reads. It is not safe +// for concurrent use. +type wakeDetector struct { + // read returns the current spread between the two clocks, false if this platform can't provide one. + read func() (time.Duration, bool) + last time.Duration + primed bool +} + +func newWakeDetector() *wakeDetector { + return &wakeDetector{read: suspendClockDelta} +} + +// Sample returns how long the machine was suspended since the previous call, 0 if it wasn't, and false if the +// platform has no way to tell. The first call primes the baseline and always reports 0. +func (w *wakeDetector) Sample() (time.Duration, bool) { + delta, ok := w.read() + if !ok { + return 0, false + } + + if !w.primed { + w.primed = true + w.last = delta + return 0, true + } + + slept := delta - w.last + w.last = delta + if slept < 0 { + // The clock pair is read non-atomically so tiny negative jitter is possible; it is never a wake. + slept = 0 + } + return slept, true +} diff --git a/wake_detector_test.go b/wake_detector_test.go new file mode 100644 index 00000000..5a799e49 --- /dev/null +++ b/wake_detector_test.go @@ -0,0 +1,67 @@ +package nebula + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestWakeDetector(t *testing.T) { + delta := time.Duration(0) + ok := true + w := &wakeDetector{read: func() (time.Duration, bool) { return delta, ok }} + + // The first sample primes the baseline and never reports a wake, even with a pre-existing spread + delta = 3 * time.Hour + slept, sok := w.Sample() + assert.True(t, sok) + assert.Equal(t, time.Duration(0), slept) + + // A stable spread means the machine never slept + slept, sok = w.Sample() + assert.True(t, sok) + assert.Equal(t, time.Duration(0), slept) + + // The spread grows by exactly the time spent suspended + delta += 42 * time.Second + slept, sok = w.Sample() + assert.True(t, sok) + assert.Equal(t, 42*time.Second, slept) + + // A wake is reported once, then the baseline moves with it + slept, sok = w.Sample() + assert.True(t, sok) + assert.Equal(t, time.Duration(0), slept) + + // Negative jitter from the non-atomic clock pair reads clamps to zero + delta -= time.Microsecond + slept, sok = w.Sample() + assert.True(t, sok) + assert.Equal(t, time.Duration(0), slept) + + // Consecutive suspends both report; the clamped jitter moved the baseline so it is not double-counted + delta += time.Minute + slept, _ = w.Sample() + assert.Equal(t, time.Minute, slept) + delta += time.Hour + slept, _ = w.Sample() + assert.Equal(t, time.Hour, slept) + + // An unsupported platform read reports not-ok + ok = false + _, sok = w.Sample() + assert.False(t, sok) +} + +// TestWakeDetectorPlatformClock smoke tests the real clock pair: two samples close together must not report a +// wake on a machine that isn't suspending mid-test. +func TestWakeDetectorPlatformClock(t *testing.T) { + w := newWakeDetector() + if _, ok := w.Sample(); !ok { + t.Skip("platform has no suspend clock pair") + } + slept, ok := w.Sample() + assert.True(t, ok) + assert.Less(t, slept, time.Second) +}