make boringcrypto: add checklinkname flag for go1.23

Starting with go1.23, we need to set -checklinkname=0 when building for
boringcrypto because we need to use go:linkname to access `newGCMTLS`.

Note that this does break builds when using a go version less than
go1.23.0. We can probably assume that someone using this Makefile and
manually building is using the latest release of Go though.

See:

- https://go.dev/doc/go1.23#linker

CHANGELOG.md

@@ -7,60 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-## [1.9.7] - 2025-10-8
-
-### Security
-
-- Fix an opportunity for emitting a packet that was sent with a spoofed source address within the configured vpn network. (#1494)
-
-### Changed
-
-- Disable sending `recv_error` messages when a packet is received outside the allowable counter window. (#1459)
-- Improve error messages and remove some unnecessary fatal conditions in the Windows and generic udp listener. (#1543)
-
-## [1.9.6] - 2025-7-15
-
-### Added
-
-- Support dropping inactive tunnels. This is disabled by default in this release but can be enabled with `tunnels.drop_inactive`. See example config for more details. (#1413)
-
-### Fixed
-
-- Fix Darwin freeze due to presence of some Network Extensions (#1426)
-- Ensure the same relay tunnel is always used when multiple relay tunnels are present (#1422)
-- Fix Windows freeze due to ICMP error handling (#1412)
-- Fix relay migration panic (#1403)
-
-## [1.9.5] - 2024-12-05
-
-### Added
-
-- Gracefully ignore v2 certificates. (#1282)
-
-### Fixed
-
-- Fix relays that refuse to re-establish after one of the remote tunnel pairs breaks. (#1277)
-
-## [1.9.4] - 2024-09-09
-
-### Added
-
-- Support UDP dialing with gVisor. (#1181)
-
-### Changed
-
-- Make some Nebula state programmatically available via control object. (#1188)
-- Switch internal representation of IPs to netip, to prepare for IPv6 support
-  in the overlay. (#1173)
-- Minor build and cleanup changes. (#1171, #1164, #1162)
-- Various dependency updates. (#1195, #1190, #1174, #1168, #1167, #1161, #1147, #1146)
-
-### Fixed
-
-- Fix a bug on big endian hosts, like mips. (#1194)
-- Fix a rare panic if a local index collision happens. (#1191)
-- Fix integer wraparound in the calculation of handshake timeouts on 32-bit targets. (#1185)
-
 ## [1.9.3] - 2024-06-06
 
 ### Fixed
@@ -698,11 +644,7 @@ created.)
 
 - Initial public release.
 
-[Unreleased]: https://github.com/slackhq/nebula/compare/v1.9.7...HEAD
+[Unreleased]: https://github.com/slackhq/nebula/compare/v1.9.3...HEAD
-[1.9.7]: https://github.com/slackhq/nebula/releases/tag/v1.9.7
-[1.9.6]: https://github.com/slackhq/nebula/releases/tag/v1.9.6
-[1.9.5]: https://github.com/slackhq/nebula/releases/tag/v1.9.5
-[1.9.4]: https://github.com/slackhq/nebula/releases/tag/v1.9.4
 [1.9.3]: https://github.com/slackhq/nebula/releases/tag/v1.9.3
 [1.9.2]: https://github.com/slackhq/nebula/releases/tag/v1.9.2
 [1.9.1]: https://github.com/slackhq/nebula/releases/tag/v1.9.1

Makefile

@@ -133,6 +133,8 @@ build/linux-mips-softfloat/%: LDFLAGS += -s -w
 # boringcrypto
 build/linux-amd64-boringcrypto/%: GOENV += GOEXPERIMENT=boringcrypto CGO_ENABLED=1
 build/linux-arm64-boringcrypto/%: GOENV += GOEXPERIMENT=boringcrypto CGO_ENABLED=1
+build/linux-amd64-boringcrypto/%: LDFLAGS += -checklinkname=0
+build/linux-arm64-boringcrypto/%: LDFLAGS += -checklinkname=0
 
 build/%/nebula: .FORCE
 	GOOS=$(firstword $(subst -, , $*)) \

cert/ca.go

@@ -24,39 +24,31 @@ func NewCAPool() *NebulaCAPool {
 
 // NewCAPoolFromBytes will create a new CA pool from the provided
 // input bytes, which must be a PEM-encoded set of nebula certificates.
-// If the pool contains unsupported certificates, they will generate warnings
-// in the []error return arg.
 // If the pool contains any expired certificates, an ErrExpired will be
 // returned along with the pool. The caller must handle any such errors.
-func NewCAPoolFromBytes(caPEMs []byte) (*NebulaCAPool, []error, error) {
+func NewCAPoolFromBytes(caPEMs []byte) (*NebulaCAPool, error) {
 	pool := NewCAPool()
 	var err error
-	var warnings []error
+	var expired bool
-	good := 0
-
 	for {
 		caPEMs, err = pool.AddCACertificate(caPEMs)
 		if errors.Is(err, ErrExpired) {
-			warnings = append(warnings, err)
+			expired = true
-		} else if errors.Is(err, ErrInvalidPEMCertificateUnsupported) {
+			err = nil
-			warnings = append(warnings, err)
+		}
-		} else if err != nil {
+		if err != nil {
-			return nil, warnings, err
+			return nil, err
-		} else {
-			// Only consider a good certificate if there were no errors present
-			good++
 		}
-
 		if len(caPEMs) == 0 || strings.TrimSpace(string(caPEMs)) == "" {
 			break
 		}
 	}
 
-	if good == 0 {
+	if expired {
-		return nil, warnings, errors.New("no valid CA certificates present")
+		return pool, ErrExpired
 	}
 
-	return pool, warnings, nil
+	return pool, nil
 }
 
 // AddCACertificate verifies a Nebula CA certificate and adds it to the pool

cert/cert.go

@@ -28,7 +28,6 @@ const publicKeyLen = 32
 
 const (
 	CertBanner                       = "NEBULA CERTIFICATE"
-	CertificateV2Banner              = "NEBULA CERTIFICATE V2"
 	X25519PrivateKeyBanner           = "NEBULA X25519 PRIVATE KEY"
 	X25519PublicKeyBanner            = "NEBULA X25519 PUBLIC KEY"
 	EncryptedEd25519PrivateKeyBanner = "NEBULA ED25519 ENCRYPTED PRIVATE KEY"
@@ -164,9 +163,6 @@ func UnmarshalNebulaCertificateFromPEM(b []byte) (*NebulaCertificate, []byte, er
 	if p == nil {
 		return nil, r, fmt.Errorf("input did not contain a valid PEM encoded block")
 	}
-	if p.Type == CertificateV2Banner {
-		return nil, r, fmt.Errorf("%w: %s", ErrInvalidPEMCertificateUnsupported, p.Type)
-	}
 	if p.Type != CertBanner {
 		return nil, r, fmt.Errorf("bytes did not contain a proper nebula certificate banner")
 	}

cert/cert_test.go

@@ -5,7 +5,6 @@ import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
-	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -573,13 +572,6 @@ CmYKEG5lYnVsYSBQMjU2IHRlc3Qo4s+7mgYw4tXrsAc6QQRkaW2jFmllYvN4+/k2
 76gvQAGgBgESRzBFAiEAib0/te6eMiZOKD8gdDeloMTS0wGuX2t0C7TFdUhAQzgC
 IBNWYMep3ysx9zCgknfG5dKtwGTaqF++BWKDYdyl34KX
 -----END NEBULA CERTIFICATE-----
-`
-
-	v2 := `
-# valid PEM with the V2 header
------BEGIN NEBULA CERTIFICATE V2-----
-CmYKEG5lYnVsYSBQMjU2IHRlc3Qo4s+7mgYw4tXrsAc6QQRkaW2jFmllYvN4+/k2
------END NEBULA CERTIFICATE V2-----
 `
 
 	rootCA := NebulaCertificate{
@@ -600,46 +592,33 @@ CmYKEG5lYnVsYSBQMjU2IHRlc3Qo4s+7mgYw4tXrsAc6QQRkaW2jFmllYvN4+/k2
 		},
 	}
 
-	p, warn, err := NewCAPoolFromBytes([]byte(noNewLines))
+	p, err := NewCAPoolFromBytes([]byte(noNewLines))
 	assert.Nil(t, err)
-	assert.Nil(t, warn)
 	assert.Equal(t, p.CAs[string("c9bfaf7ce8e84b2eeda2e27b469f4b9617bde192efd214b68891ecda6ed49522")].Details.Name, rootCA.Details.Name)
 	assert.Equal(t, p.CAs[string("5c9c3f23e7ee7fe97637cbd3a0a5b854154d1d9aaaf7b566a51f4a88f76b64cd")].Details.Name, rootCA01.Details.Name)
 
-	pp, warn, err := NewCAPoolFromBytes([]byte(withNewLines))
+	pp, err := NewCAPoolFromBytes([]byte(withNewLines))
 	assert.Nil(t, err)
-	assert.Nil(t, warn)
 	assert.Equal(t, pp.CAs[string("c9bfaf7ce8e84b2eeda2e27b469f4b9617bde192efd214b68891ecda6ed49522")].Details.Name, rootCA.Details.Name)
 	assert.Equal(t, pp.CAs[string("5c9c3f23e7ee7fe97637cbd3a0a5b854154d1d9aaaf7b566a51f4a88f76b64cd")].Details.Name, rootCA01.Details.Name)
 
 	// expired cert, no valid certs
-	ppp, warn, err := NewCAPoolFromBytes([]byte(expired))
+	ppp, err := NewCAPoolFromBytes([]byte(expired))
-	assert.Error(t, err, "no valid CA certificates present")
+	assert.Equal(t, ErrExpired, err)
-	assert.Len(t, warn, 1)
+	assert.Equal(t, ppp.CAs[string("152070be6bb19bc9e3bde4c2f0e7d8f4ff5448b4c9856b8eccb314fade0229b0")].Details.Name, "expired")
-	assert.Error(t, warn[0], ErrExpired)
-	assert.Nil(t, ppp)
 
 	// expired cert, with valid certs
-	pppp, warn, err := NewCAPoolFromBytes(append([]byte(expired), noNewLines...))
+	pppp, err := NewCAPoolFromBytes(append([]byte(expired), noNewLines...))
-	assert.Len(t, warn, 1)
+	assert.Equal(t, ErrExpired, err)
-	assert.Nil(t, err)
-	assert.Error(t, warn[0], ErrExpired)
 	assert.Equal(t, pppp.CAs[string("c9bfaf7ce8e84b2eeda2e27b469f4b9617bde192efd214b68891ecda6ed49522")].Details.Name, rootCA.Details.Name)
 	assert.Equal(t, pppp.CAs[string("5c9c3f23e7ee7fe97637cbd3a0a5b854154d1d9aaaf7b566a51f4a88f76b64cd")].Details.Name, rootCA01.Details.Name)
 	assert.Equal(t, pppp.CAs[string("152070be6bb19bc9e3bde4c2f0e7d8f4ff5448b4c9856b8eccb314fade0229b0")].Details.Name, "expired")
 	assert.Equal(t, len(pppp.CAs), 3)
 
-	ppppp, warn, err := NewCAPoolFromBytes([]byte(p256))
+	ppppp, err := NewCAPoolFromBytes([]byte(p256))
 	assert.Nil(t, err)
-	assert.Nil(t, warn)
 	assert.Equal(t, ppppp.CAs[string("a7938893ec8c4ef769b06d7f425e5e46f7a7f5ffa49c3bcf4a86b608caba9159")].Details.Name, rootCAP256.Details.Name)
 	assert.Equal(t, len(ppppp.CAs), 1)
-
-	pppppp, warn, err := NewCAPoolFromBytes(append([]byte(p256), []byte(v2)...))
-	assert.Nil(t, err)
-	assert.True(t, errors.Is(warn[0], ErrInvalidPEMCertificateUnsupported))
-	assert.Equal(t, pppppp.CAs[string("a7938893ec8c4ef769b06d7f425e5e46f7a7f5ffa49c3bcf4a86b608caba9159")].Details.Name, rootCAP256.Details.Name)
-	assert.Equal(t, len(pppppp.CAs), 1)
 }
 
 func appendByteSlices(b ...[]byte) []byte {

cert/errors.go

@@ -11,5 +11,4 @@ var (
 	ErrNotSelfSigned     = errors.New("certificate is not self-signed")
 	ErrBlockListed       = errors.New("certificate is in the block list")
 	ErrSignatureMismatch = errors.New("certificate signature did not match")
-	ErrInvalidPEMCertificateUnsupported = errors.New("bytes contain an unsupported certificate format")
 )

connection_manager.go

@@ -4,16 +4,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
-	"fmt"
 	"net/netip"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
 )
 
@@ -30,6 +27,12 @@ const (
 )
 
 type connectionManager struct {
+	in     map[uint32]struct{}
+	inLock *sync.RWMutex
+
+	out     map[uint32]struct{}
+	outLock *sync.RWMutex
+
 	// relayUsed holds which relay localIndexs are in use
 	relayUsed     map[uint32]struct{}
 	relayUsedLock *sync.RWMutex
@@ -37,117 +40,117 @@ type connectionManager struct {
 	hostMap                 *HostMap
 	trafficTimer            *LockingTimerWheel[uint32]
 	intf                    *Interface
+	pendingDeletion         map[uint32]struct{}
 	punchy                  *Punchy
-
-	// Configuration settings
 	checkInterval           time.Duration
 	pendingDeletionInterval time.Duration
-	inactivityTimeout       atomic.Int64
-	dropInactive            atomic.Bool
-
 	metricsTxPunchy         metrics.Counter
 
 	l *logrus.Logger
 }
 
-func newConnectionManagerFromConfig(l *logrus.Logger, c *config.C, hm *HostMap, p *Punchy) *connectionManager {
+func newConnectionManager(ctx context.Context, l *logrus.Logger, intf *Interface, checkInterval, pendingDeletionInterval time.Duration, punchy *Punchy) *connectionManager {
-	cm := &connectionManager{
+	var max time.Duration
-		hostMap:         hm,
+	if checkInterval < pendingDeletionInterval {
-		l:               l,
+		max = pendingDeletionInterval
-		punchy:          p,
+	} else {
+		max = checkInterval
+	}
+
+	nc := &connectionManager{
+		hostMap:                 intf.hostMap,
+		in:                      make(map[uint32]struct{}),
+		inLock:                  &sync.RWMutex{},
+		out:                     make(map[uint32]struct{}),
+		outLock:                 &sync.RWMutex{},
 		relayUsed:               make(map[uint32]struct{}),
 		relayUsedLock:           &sync.RWMutex{},
+		trafficTimer:            NewLockingTimerWheel[uint32](time.Millisecond*500, max),
+		intf:                    intf,
+		pendingDeletion:         make(map[uint32]struct{}),
+		checkInterval:           checkInterval,
+		pendingDeletionInterval: pendingDeletionInterval,
+		punchy:                  punchy,
 		metricsTxPunchy:         metrics.GetOrRegisterCounter("messages.tx.punchy", nil),
+		l:                       l,
 	}
 
-	cm.reload(c, true)
+	nc.Start(ctx)
-	c.RegisterReloadCallback(func(c *config.C) {
+	return nc
-		cm.reload(c, false)
-	})
-
-	return cm
 }
 
-func (cm *connectionManager) reload(c *config.C, initial bool) {
+func (n *connectionManager) In(localIndex uint32) {
-	if initial {
+	n.inLock.RLock()
-		cm.checkInterval = time.Duration(c.GetInt("timers.connection_alive_interval", 5)) * time.Second
-		cm.pendingDeletionInterval = time.Duration(c.GetInt("timers.pending_deletion_interval", 10)) * time.Second
-
-		// We want at least a minimum resolution of 500ms per tick so that we can hit these intervals
-		// pretty close to their configured duration.
-		// The inactivity duration is checked each time a hostinfo ticks through so we don't need the wheel to contain it.
-		minDuration := min(time.Millisecond*500, cm.checkInterval, cm.pendingDeletionInterval)
-		maxDuration := max(cm.checkInterval, cm.pendingDeletionInterval)
-		cm.trafficTimer = NewLockingTimerWheel[uint32](minDuration, maxDuration)
-	}
-
-	if initial || c.HasChanged("tunnels.inactivity_timeout") {
-		old := cm.getInactivityTimeout()
-		cm.inactivityTimeout.Store((int64)(c.GetDuration("tunnels.inactivity_timeout", 10*time.Minute)))
-		if !initial {
-			cm.l.WithField("oldDuration", old).
-				WithField("newDuration", cm.getInactivityTimeout()).
-				Info("Inactivity timeout has changed")
-		}
-	}
-
-	if initial || c.HasChanged("tunnels.drop_inactive") {
-		old := cm.dropInactive.Load()
-		cm.dropInactive.Store(c.GetBool("tunnels.drop_inactive", false))
-		if !initial {
-			cm.l.WithField("oldBool", old).
-				WithField("newBool", cm.dropInactive.Load()).
-				Info("Drop inactive setting has changed")
-		}
-	}
-}
-
-func (cm *connectionManager) getInactivityTimeout() time.Duration {
-	return (time.Duration)(cm.inactivityTimeout.Load())
-}
-
-func (cm *connectionManager) In(h *HostInfo) {
-	h.in.Store(true)
-}
-
-func (cm *connectionManager) Out(h *HostInfo) {
-	h.out.Store(true)
-}
-
-func (cm *connectionManager) RelayUsed(localIndex uint32) {
-	cm.relayUsedLock.RLock()
 	// If this already exists, return
-	if _, ok := cm.relayUsed[localIndex]; ok {
+	if _, ok := n.in[localIndex]; ok {
-		cm.relayUsedLock.RUnlock()
+		n.inLock.RUnlock()
 		return
 	}
-	cm.relayUsedLock.RUnlock()
+	n.inLock.RUnlock()
-	cm.relayUsedLock.Lock()
+	n.inLock.Lock()
-	cm.relayUsed[localIndex] = struct{}{}
+	n.in[localIndex] = struct{}{}
-	cm.relayUsedLock.Unlock()
+	n.inLock.Unlock()
+}
+
+func (n *connectionManager) Out(localIndex uint32) {
+	n.outLock.RLock()
+	// If this already exists, return
+	if _, ok := n.out[localIndex]; ok {
+		n.outLock.RUnlock()
+		return
+	}
+	n.outLock.RUnlock()
+	n.outLock.Lock()
+	n.out[localIndex] = struct{}{}
+	n.outLock.Unlock()
+}
+
+func (n *connectionManager) RelayUsed(localIndex uint32) {
+	n.relayUsedLock.RLock()
+	// If this already exists, return
+	if _, ok := n.relayUsed[localIndex]; ok {
+		n.relayUsedLock.RUnlock()
+		return
+	}
+	n.relayUsedLock.RUnlock()
+	n.relayUsedLock.Lock()
+	n.relayUsed[localIndex] = struct{}{}
+	n.relayUsedLock.Unlock()
 }
 
 // getAndResetTrafficCheck returns if there was any inbound or outbound traffic within the last tick and
 // resets the state for this local index
-func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time) (bool, bool) {
+func (n *connectionManager) getAndResetTrafficCheck(localIndex uint32) (bool, bool) {
-	in := h.in.Swap(false)
+	n.inLock.Lock()
-	out := h.out.Swap(false)
+	n.outLock.Lock()
-	if in || out {
+	_, in := n.in[localIndex]
-		h.lastUsed = now
+	_, out := n.out[localIndex]
-	}
+	delete(n.in, localIndex)
+	delete(n.out, localIndex)
+	n.inLock.Unlock()
+	n.outLock.Unlock()
 	return in, out
 }
 
-// AddTrafficWatch must be called for every new HostInfo.
+func (n *connectionManager) AddTrafficWatch(localIndex uint32) {
-// We will continue to monitor the HostInfo until the tunnel is dropped.
+	// Use a write lock directly because it should be incredibly rare that we are ever already tracking this index
-func (cm *connectionManager) AddTrafficWatch(h *HostInfo) {
+	n.outLock.Lock()
-	if h.out.Swap(true) == false {
+	if _, ok := n.out[localIndex]; ok {
-		cm.trafficTimer.Add(h.localIndexId, cm.checkInterval)
+		n.outLock.Unlock()
+		return
 	}
+	n.out[localIndex] = struct{}{}
+	n.trafficTimer.Add(localIndex, n.checkInterval)
+	n.outLock.Unlock()
 }
 
-func (cm *connectionManager) Start(ctx context.Context) {
+func (n *connectionManager) Start(ctx context.Context) {
-	clockSource := time.NewTicker(cm.trafficTimer.t.tickDuration)
+	go n.Run(ctx)
+}
+
+func (n *connectionManager) Run(ctx context.Context) {
+	//TODO: this tick should be based on the min wheel tick? Check firewall
+	clockSource := time.NewTicker(500 * time.Millisecond)
 	defer clockSource.Stop()
 
 	p := []byte("")
@@ -160,61 +163,61 @@ func (cm *connectionManager) Start(ctx context.Context) {
 			return
 
 		case now := <-clockSource.C:
-			cm.trafficTimer.Advance(now)
+			n.trafficTimer.Advance(now)
 			for {
-				localIndex, has := cm.trafficTimer.Purge()
+				localIndex, has := n.trafficTimer.Purge()
 				if !has {
 					break
 				}
 
-				cm.doTrafficCheck(localIndex, p, nb, out, now)
+				n.doTrafficCheck(localIndex, p, nb, out, now)
 			}
 		}
 	}
 }
 
-func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
+func (n *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
-	decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now)
+	decision, hostinfo, primary := n.makeTrafficDecision(localIndex, now)
 
 	switch decision {
 	case deleteTunnel:
-		if cm.hostMap.DeleteHostInfo(hostinfo) {
+		if n.hostMap.DeleteHostInfo(hostinfo) {
 			// Only clearing the lighthouse cache if this is the last hostinfo for this vpn ip in the hostmap
-			cm.intf.lightHouse.DeleteVpnIp(hostinfo.vpnIp)
+			n.intf.lightHouse.DeleteVpnIp(hostinfo.vpnIp)
 		}
 
 	case closeTunnel:
-		cm.intf.sendCloseTunnel(hostinfo)
+		n.intf.sendCloseTunnel(hostinfo)
-		cm.intf.closeTunnel(hostinfo)
+		n.intf.closeTunnel(hostinfo)
 
 	case swapPrimary:
-		cm.swapPrimary(hostinfo, primary)
+		n.swapPrimary(hostinfo, primary)
 
 	case migrateRelays:
-		cm.migrateRelayUsed(hostinfo, primary)
+		n.migrateRelayUsed(hostinfo, primary)
 
 	case tryRehandshake:
-		cm.tryRehandshake(hostinfo)
+		n.tryRehandshake(hostinfo)
 
 	case sendTestPacket:
-		cm.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, nb, out)
+		n.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, nb, out)
 	}
 
-	cm.resetRelayTrafficCheck(hostinfo)
+	n.resetRelayTrafficCheck(hostinfo)
 }
 
-func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
+func (n *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
 	if hostinfo != nil {
-		cm.relayUsedLock.Lock()
+		n.relayUsedLock.Lock()
-		defer cm.relayUsedLock.Unlock()
+		defer n.relayUsedLock.Unlock()
 		// No need to migrate any relays, delete usage info now.
 		for _, idx := range hostinfo.relayState.CopyRelayForIdxs() {
-			delete(cm.relayUsed, idx)
+			delete(n.relayUsed, idx)
 		}
 	}
 }
 
-func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo) {
+func (n *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo) {
 	relayFor := oldhostinfo.relayState.CopyAllRelayFor()
 
 	for _, r := range relayFor {
@@ -224,51 +227,46 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
 		var relayFrom netip.Addr
 		var relayTo netip.Addr
 		switch {
-		case ok:
+		case ok && existing.State == Established:
-			switch existing.State {
-			case Established, PeerRequested, Disestablished:
 			// This relay already exists in newhostinfo, then do nothing.
 			continue
-			case Requested:
+		case ok && existing.State == Requested:
-				// The relayed connection exists in a Requested state; re-send the request
+			// The relay exists in a Requested state; re-send the request
 			index = existing.LocalIndex
 			switch r.Type {
 			case TerminalType:
-					relayFrom = cm.intf.myVpnNet.Addr()
+				relayFrom = n.intf.myVpnNet.Addr()
 				relayTo = existing.PeerIp
 			case ForwardingType:
 				relayFrom = existing.PeerIp
 				relayTo = newhostinfo.vpnIp
 			default:
 				// should never happen
-					panic(fmt.Sprintf("Migrating unknown relay type: %v", r.Type))
-				}
 			}
 		case !ok:
-			cm.relayUsedLock.RLock()
+			n.relayUsedLock.RLock()
-			if _, relayUsed := cm.relayUsed[r.LocalIndex]; !relayUsed {
+			if _, relayUsed := n.relayUsed[r.LocalIndex]; !relayUsed {
 				// The relay hasn't been used; don't migrate it.
-				cm.relayUsedLock.RUnlock()
+				n.relayUsedLock.RUnlock()
 				continue
 			}
-			cm.relayUsedLock.RUnlock()
+			n.relayUsedLock.RUnlock()
 			// The relay doesn't exist at all; create some relay state and send the request.
 			var err error
-			index, err = AddRelay(cm.l, newhostinfo, cm.hostMap, r.PeerIp, nil, r.Type, Requested)
+			index, err = AddRelay(n.l, newhostinfo, n.hostMap, r.PeerIp, nil, r.Type, Requested)
 			if err != nil {
-				cm.l.WithError(err).Error("failed to migrate relay to new hostinfo")
+				n.l.WithError(err).Error("failed to migrate relay to new hostinfo")
 				continue
 			}
 			switch r.Type {
 			case TerminalType:
-				relayFrom = cm.intf.myVpnNet.Addr()
+				relayFrom = n.intf.myVpnNet.Addr()
 				relayTo = r.PeerIp
 			case ForwardingType:
 				relayFrom = r.PeerIp
 				relayTo = newhostinfo.vpnIp
 			default:
 				// should never happen
-				panic(fmt.Sprintf("Migrating unknown relay type: %v", r.Type))
 			}
 		}
 
@@ -285,10 +283,10 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
 		}
 		msg, err := req.Marshal()
 		if err != nil {
-			cm.l.WithError(err).Error("failed to marshal Control message to migrate relay")
+			n.l.WithError(err).Error("failed to marshal Control message to migrate relay")
 		} else {
-			cm.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, make([]byte, 12), make([]byte, mtu))
+			n.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, make([]byte, 12), make([]byte, mtu))
-			cm.l.WithFields(logrus.Fields{
+			n.l.WithFields(logrus.Fields{
 				"relayFrom":           req.RelayFromIp,
 				"relayTo":             req.RelayToIp,
 				"initiatorRelayIndex": req.InitiatorRelayIndex,
@@ -299,45 +297,46 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
 	}
 }
 
-func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) {
+func (n *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) {
-	// Read lock the main hostmap to order decisions based on tunnels being the primary tunnel
+	n.hostMap.RLock()
-	cm.hostMap.RLock()
+	defer n.hostMap.RUnlock()
-	defer cm.hostMap.RUnlock()
 
-	hostinfo := cm.hostMap.Indexes[localIndex]
+	hostinfo := n.hostMap.Indexes[localIndex]
 	if hostinfo == nil {
-		cm.l.WithField("localIndex", localIndex).Debugln("Not found in hostmap")
+		n.l.WithField("localIndex", localIndex).Debugf("Not found in hostmap")
+		delete(n.pendingDeletion, localIndex)
 		return doNothing, nil, nil
 	}
 
-	if cm.isInvalidCertificate(now, hostinfo) {
+	if n.isInvalidCertificate(now, hostinfo) {
+		delete(n.pendingDeletion, hostinfo.localIndexId)
 		return closeTunnel, hostinfo, nil
 	}
 
-	primary := cm.hostMap.Hosts[hostinfo.vpnIp]
+	primary := n.hostMap.Hosts[hostinfo.vpnIp]
 	mainHostInfo := true
 	if primary != nil && primary != hostinfo {
 		mainHostInfo = false
 	}
 
 	// Check for traffic on this hostinfo
-	inTraffic, outTraffic := cm.getAndResetTrafficCheck(hostinfo, now)
+	inTraffic, outTraffic := n.getAndResetTrafficCheck(localIndex)
 
 	// A hostinfo is determined alive if there is incoming traffic
 	if inTraffic {
 		decision := doNothing
-		if cm.l.Level >= logrus.DebugLevel {
+		if n.l.Level >= logrus.DebugLevel {
-			hostinfo.logger(cm.l).
+			hostinfo.logger(n.l).
 				WithField("tunnelCheck", m{"state": "alive", "method": "passive"}).
 				Debug("Tunnel status")
 		}
-		hostinfo.pendingDeletion.Store(false)
+		delete(n.pendingDeletion, hostinfo.localIndexId)
 
 		if mainHostInfo {
 			decision = tryRehandshake
 
 		} else {
-			if cm.shouldSwapPrimary(hostinfo, primary) {
+			if n.shouldSwapPrimary(hostinfo, primary) {
 				decision = swapPrimary
 			} else {
 				// migrate the relays to the primary, if in use.
@@ -345,55 +344,46 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 			}
 		}
 
-		cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
+		n.trafficTimer.Add(hostinfo.localIndexId, n.checkInterval)
 
 		if !outTraffic {
 			// Send a punch packet to keep the NAT state alive
-			cm.sendPunch(hostinfo)
+			n.sendPunch(hostinfo)
 		}
 
 		return decision, hostinfo, primary
 	}
 
-	if hostinfo.pendingDeletion.Load() {
+	if _, ok := n.pendingDeletion[hostinfo.localIndexId]; ok {
 		// We have already sent a test packet and nothing was returned, this hostinfo is dead
-		hostinfo.logger(cm.l).
+		hostinfo.logger(n.l).
 			WithField("tunnelCheck", m{"state": "dead", "method": "active"}).
 			Info("Tunnel status")
 
+		delete(n.pendingDeletion, hostinfo.localIndexId)
 		return deleteTunnel, hostinfo, nil
 	}
 
 	decision := doNothing
 	if hostinfo != nil && hostinfo.ConnectionState != nil && mainHostInfo {
 		if !outTraffic {
-			inactiveFor, isInactive := cm.isInactive(hostinfo, now)
-			if isInactive {
-				// Tunnel is inactive, tear it down
-				hostinfo.logger(cm.l).
-					WithField("inactiveDuration", inactiveFor).
-					WithField("primary", mainHostInfo).
-					Info("Dropping tunnel due to inactivity")
-
-				return closeTunnel, hostinfo, primary
-			}
-
 			// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
 			// Just maintain NAT state if configured to do so.
-			cm.sendPunch(hostinfo)
+			n.sendPunch(hostinfo)
-			cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
+			n.trafficTimer.Add(hostinfo.localIndexId, n.checkInterval)
 			return doNothing, nil, nil
+
 		}
 
-		if cm.punchy.GetTargetEverything() {
+		if n.punchy.GetTargetEverything() {
 			// This is similar to the old punchy behavior with a slight optimization.
 			// We aren't receiving traffic but we are sending it, punch on all known
 			// ips in case we need to re-prime NAT state
-			cm.sendPunch(hostinfo)
+			n.sendPunch(hostinfo)
 		}
 
-		if cm.l.Level >= logrus.DebugLevel {
+		if n.l.Level >= logrus.DebugLevel {
-			hostinfo.logger(cm.l).
+			hostinfo.logger(n.l).
 				WithField("tunnelCheck", m{"state": "testing", "method": "active"}).
 				Debug("Tunnel status")
 		}
@@ -402,118 +392,95 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 		decision = sendTestPacket
 
 	} else {
-		if cm.l.Level >= logrus.DebugLevel {
+		if n.l.Level >= logrus.DebugLevel {
-			hostinfo.logger(cm.l).Debugf("Hostinfo sadness")
+			hostinfo.logger(n.l).Debugf("Hostinfo sadness")
 		}
 	}
 
-	hostinfo.pendingDeletion.Store(true)
+	n.pendingDeletion[hostinfo.localIndexId] = struct{}{}
-	cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
+	n.trafficTimer.Add(hostinfo.localIndexId, n.pendingDeletionInterval)
 	return decision, hostinfo, nil
 }
 
-func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) {
+func (n *connectionManager) shouldSwapPrimary(current, primary *HostInfo) bool {
-	if cm.dropInactive.Load() == false {
-		// We aren't configured to drop inactive tunnels
-		return 0, false
-	}
-
-	inactiveDuration := now.Sub(hostinfo.lastUsed)
-	if inactiveDuration < cm.getInactivityTimeout() {
-		// It's not considered inactive
-		return inactiveDuration, false
-	}
-
-	// The tunnel is inactive
-	return inactiveDuration, true
-}
-
-func (cm *connectionManager) shouldSwapPrimary(current, primary *HostInfo) bool {
 	// The primary tunnel is the most recent handshake to complete locally and should work entirely fine.
 	// If we are here then we have multiple tunnels for a host pair and neither side believes the same tunnel is primary.
 	// Let's sort this out.
 
-	if current.vpnIp.Compare(cm.intf.myVpnNet.Addr()) < 0 {
+	if current.vpnIp.Compare(n.intf.myVpnNet.Addr()) < 0 {
 		// Only one side should flip primary because if both flip then we may never resolve to a single tunnel.
 		// vpn ip is static across all tunnels for this host pair so lets use that to determine who is flipping.
 		// The remotes vpn ip is lower than mine. I will not flip.
 		return false
 	}
 
-	certState := cm.intf.pki.GetCertState()
+	certState := n.intf.pki.GetCertState()
 	return bytes.Equal(current.ConnectionState.myCert.Signature, certState.Certificate.Signature)
 }
 
-func (cm *connectionManager) swapPrimary(current, primary *HostInfo) {
+func (n *connectionManager) swapPrimary(current, primary *HostInfo) {
-	cm.hostMap.Lock()
+	n.hostMap.Lock()
 	// Make sure the primary is still the same after the write lock. This avoids a race with a rehandshake.
-	if cm.hostMap.Hosts[current.vpnIp] == primary {
+	if n.hostMap.Hosts[current.vpnIp] == primary {
-		cm.hostMap.unlockedMakePrimary(current)
+		n.hostMap.unlockedMakePrimary(current)
 	}
-	cm.hostMap.Unlock()
+	n.hostMap.Unlock()
 }
 
 // isInvalidCertificate will check if we should destroy a tunnel if pki.disconnect_invalid is true and
 // the certificate is no longer valid. Block listed certificates will skip the pki.disconnect_invalid
 // check and return true.
-func (cm *connectionManager) isInvalidCertificate(now time.Time, hostinfo *HostInfo) bool {
+func (n *connectionManager) isInvalidCertificate(now time.Time, hostinfo *HostInfo) bool {
 	remoteCert := hostinfo.GetCert()
 	if remoteCert == nil {
 		return false
 	}
 
-	valid, err := remoteCert.VerifyWithCache(now, cm.intf.pki.GetCAPool())
+	valid, err := remoteCert.VerifyWithCache(now, n.intf.pki.GetCAPool())
 	if valid {
 		return false
 	}
 
-	if !cm.intf.disconnectInvalid.Load() && err != cert.ErrBlockListed {
+	if !n.intf.disconnectInvalid.Load() && err != cert.ErrBlockListed {
 		// Block listed certificates should always be disconnected
 		return false
 	}
 
 	fingerprint, _ := remoteCert.Sha256Sum()
-	hostinfo.logger(cm.l).WithError(err).
+	hostinfo.logger(n.l).WithError(err).
 		WithField("fingerprint", fingerprint).
 		Info("Remote certificate is no longer valid, tearing down the tunnel")
 
 	return true
 }
 
-func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
+func (n *connectionManager) sendPunch(hostinfo *HostInfo) {
-	if !cm.punchy.GetPunch() {
+	if !n.punchy.GetPunch() {
 		// Punching is disabled
 		return
 	}
 
-	if cm.intf.lightHouse.IsLighthouseIP(hostinfo.vpnIp) {
+	if n.punchy.GetTargetEverything() {
-		// Do not punch to lighthouses, we assume our lighthouse update interval is good enough.
+		hostinfo.remotes.ForEach(n.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
-		// In the event the update interval is not sufficient to maintain NAT state then a publicly available lighthouse
+			n.metricsTxPunchy.Inc(1)
-		// would lose the ability to notify us and punchy.respond would become unreliable.
+			n.intf.outside.WriteTo([]byte{1}, addr)
-		return
-	}
-
-	if cm.punchy.GetTargetEverything() {
-		hostinfo.remotes.ForEach(cm.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
-			cm.metricsTxPunchy.Inc(1)
-			cm.intf.outside.WriteTo([]byte{1}, addr)
 		})
 
 	} else if hostinfo.remote.IsValid() {
-		cm.metricsTxPunchy.Inc(1)
+		n.metricsTxPunchy.Inc(1)
-		cm.intf.outside.WriteTo([]byte{1}, hostinfo.remote)
+		n.intf.outside.WriteTo([]byte{1}, hostinfo.remote)
 	}
 }
 
-func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
+func (n *connectionManager) tryRehandshake(hostinfo *HostInfo) {
-	certState := cm.intf.pki.GetCertState()
+	certState := n.intf.pki.GetCertState()
 	if bytes.Equal(hostinfo.ConnectionState.myCert.Signature, certState.Certificate.Signature) {
 		return
 	}
 
-	cm.l.WithField("vpnIp", hostinfo.vpnIp).
+	n.l.WithField("vpnIp", hostinfo.vpnIp).
 		WithField("reason", "local certificate is not current").
 		Info("Re-handshaking with remote")
 
-	cm.intf.handshakeManager.StartHandshake(hostinfo.vpnIp, nil)
+	n.intf.handshakeManager.StartHandshake(hostinfo.vpnIp, nil)
 }

connection_manager_test.go

@@ -1,6 +1,7 @@
 package nebula
 
 import (
+	"context"
 	"crypto/ed25519"
 	"crypto/rand"
 	"net"
@@ -64,10 +65,10 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	ifce.pki.cs.Store(cs)
 
 	// Create manager
-	conf := config.NewC(l)
+	ctx, cancel := context.WithCancel(context.Background())
-	punchy := NewPunchyFromConfig(l, conf)
+	defer cancel()
-	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
+	punchy := NewPunchyFromConfig(l, config.NewC(l))
-	nc.intf = ifce
+	nc := newConnectionManager(ctx, l, ifce, 5, 10, punchy)
 	p := []byte("")
 	nb := make([]byte, 12, 12)
 	out := make([]byte, mtu)
@@ -85,32 +86,31 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
 
 	// We saw traffic out to vpnIp
-	nc.Out(hostinfo)
+	nc.Out(hostinfo.localIndexId)
-	nc.In(hostinfo)
+	nc.In(hostinfo.localIndexId)
-	assert.False(t, hostinfo.pendingDeletion.Load())
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp)
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
-	assert.True(t, hostinfo.out.Load())
+	assert.Contains(t, nc.out, hostinfo.localIndexId)
-	assert.True(t, hostinfo.in.Load())
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.False(t, hostinfo.pendingDeletion.Load())
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId)
-	assert.False(t, hostinfo.out.Load())
+	assert.NotContains(t, nc.out, hostinfo.localIndexId)
-	assert.False(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.in, hostinfo.localIndexId)
 
 	// Do another traffic check tick, this host should be pending deletion now
-	nc.Out(hostinfo)
+	nc.Out(hostinfo.localIndexId)
-	assert.True(t, hostinfo.out.Load())
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.True(t, hostinfo.pendingDeletion.Load())
+	assert.Contains(t, nc.pendingDeletion, hostinfo.localIndexId)
-	assert.False(t, hostinfo.out.Load())
+	assert.NotContains(t, nc.out, hostinfo.localIndexId)
-	assert.False(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.in, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp)
 
 	// Do a final traffic check tick, the host should now be removed
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId)
 	assert.NotContains(t, nc.hostMap.Hosts, hostinfo.vpnIp)
 	assert.NotContains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 }
@@ -148,10 +148,10 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	ifce.pki.cs.Store(cs)
 
 	// Create manager
-	conf := config.NewC(l)
+	ctx, cancel := context.WithCancel(context.Background())
-	punchy := NewPunchyFromConfig(l, conf)
+	defer cancel()
-	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
+	punchy := NewPunchyFromConfig(l, config.NewC(l))
-	nc.intf = ifce
+	nc := newConnectionManager(ctx, l, ifce, 5, 10, punchy)
 	p := []byte("")
 	nb := make([]byte, 12, 12)
 	out := make([]byte, mtu)
@@ -169,130 +169,33 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
 
 	// We saw traffic out to vpnIp
-	nc.Out(hostinfo)
+	nc.Out(hostinfo.localIndexId)
-	nc.In(hostinfo)
+	nc.In(hostinfo.localIndexId)
-	assert.True(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.vpnIp)
-	assert.True(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.pendingDeletion.Load())
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp)
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.False(t, hostinfo.pendingDeletion.Load())
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId)
-	assert.False(t, hostinfo.out.Load())
+	assert.NotContains(t, nc.out, hostinfo.localIndexId)
-	assert.False(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.in, hostinfo.localIndexId)
 
 	// Do another traffic check tick, this host should be pending deletion now
-	nc.Out(hostinfo)
+	nc.Out(hostinfo.localIndexId)
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.True(t, hostinfo.pendingDeletion.Load())
+	assert.Contains(t, nc.pendingDeletion, hostinfo.localIndexId)
-	assert.False(t, hostinfo.out.Load())
+	assert.NotContains(t, nc.out, hostinfo.localIndexId)
-	assert.False(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.in, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp)
 
 	// We saw traffic, should no longer be pending deletion
-	nc.In(hostinfo)
+	nc.In(hostinfo.localIndexId)
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.False(t, hostinfo.pendingDeletion.Load())
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId)
-	assert.False(t, hostinfo.out.Load())
+	assert.NotContains(t, nc.out, hostinfo.localIndexId)
-	assert.False(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.in, hostinfo.localIndexId)
-	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
-	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp)
-}
-
-func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
-	l := test.NewLogger()
-	vpncidr := netip.MustParsePrefix("172.1.1.1/24")
-	localrange := netip.MustParsePrefix("10.1.1.1/24")
-	vpnIp := netip.MustParseAddr("172.1.1.2")
-	preferredRanges := []netip.Prefix{localrange}
-
-	// Very incomplete mock objects
-	hostMap := newHostMap(l, vpncidr)
-	hostMap.preferredRanges.Store(&preferredRanges)
-
-	cs := &CertState{
-		RawCertificate:      []byte{},
-		PrivateKey:          []byte{},
-		Certificate:         &cert.NebulaCertificate{},
-		RawCertificateNoKey: []byte{},
-	}
-
-	lh := newTestLighthouse()
-	ifce := &Interface{
-		hostMap:          hostMap,
-		inside:           &test.NoopTun{},
-		outside:          &udp.NoopConn{},
-		firewall:         &Firewall{},
-		lightHouse:       lh,
-		pki:              &PKI{},
-		handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
-		l:                l,
-	}
-	ifce.pki.cs.Store(cs)
-
-	// Create manager
-	conf := config.NewC(l)
-	conf.Settings["tunnels"] = map[interface{}]interface{}{
-		"drop_inactive": true,
-	}
-	punchy := NewPunchyFromConfig(l, conf)
-	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
-	assert.True(t, nc.dropInactive.Load())
-	nc.intf = ifce
-
-	// Add an ip we have established a connection w/ to hostmap
-	hostinfo := &HostInfo{
-		vpnIp:         vpnIp,
-		localIndexId:  1099,
-		remoteIndexId: 9901,
-	}
-	hostinfo.ConnectionState = &ConnectionState{
-		myCert: &cert.NebulaCertificate{},
-		H:      &noise.HandshakeState{},
-	}
-	nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
-
-	// Do a traffic check tick, in and out should be cleared but should not be pending deletion
-	nc.Out(hostinfo)
-	nc.In(hostinfo)
-	assert.True(t, hostinfo.out.Load())
-	assert.True(t, hostinfo.in.Load())
-
-	now := time.Now()
-	decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
-	assert.Equal(t, tryRehandshake, decision)
-	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
-
-	decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
-	assert.Equal(t, doNothing, decision)
-	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
-
-	// Do another traffic check tick, should still not be pending deletion
-	decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
-	assert.Equal(t, doNothing, decision)
-	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
-	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
-	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp)
-
-	// Finally advance beyond the inactivity timeout
-	decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
-	assert.Equal(t, closeTunnel, decision)
-	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp)
 }
@@ -370,10 +273,10 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 	ifce.disconnectInvalid.Store(true)
 
 	// Create manager
-	conf := config.NewC(l)
+	ctx, cancel := context.WithCancel(context.Background())
-	punchy := NewPunchyFromConfig(l, conf)
+	defer cancel()
-	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
+	punchy := NewPunchyFromConfig(l, config.NewC(l))
-	nc.intf = ifce
+	nc := newConnectionManager(ctx, l, ifce, 5, 10, punchy)
 	ifce.connectionManager = nc
 
 	hostinfo := &HostInfo{

control.go

@@ -34,7 +34,6 @@ type Control struct {
 	statsStart      func()
 	dnsStart        func()
 	lighthouseStart func()
-	connectionManagerStart func(context.Context)
 }
 
 type ControlHostInfo struct {
@@ -64,9 +63,6 @@ func (c *Control) Start() {
 	if c.dnsStart != nil {
 		go c.dnsStart()
 	}
-	if c.connectionManagerStart != nil {
-		go c.connectionManagerStart(c.ctx)
-	}
 	if c.lighthouseStart != nil {
 		c.lighthouseStart()
 	}

control_test.go

@@ -66,7 +66,7 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 		localIndexId:  201,
 		vpnIp:         vpnIp,
 		relayState: RelayState{
-			relays:        nil,
+			relays:        map[netip.Addr]struct{}{},
 			relayForByIp:  map[netip.Addr]*Relay{},
 			relayForByIdx: map[uint32]*Relay{},
 		},
@@ -85,7 +85,7 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 		localIndexId:  201,
 		vpnIp:         vpnIp2,
 		relayState: RelayState{
-			relays:        nil,
+			relays:        map[netip.Addr]struct{}{},
 			relayForByIp:  map[netip.Addr]*Relay{},
 			relayForByIdx: map[uint32]*Relay{},
 		},

e2e/handshakes_test.go

@@ -4,13 +4,11 @@
 package e2e
 
 import (
+	"fmt"
 	"net/netip"
-	"slices"
 	"testing"
 	"time"
 
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/e2e/router"
@@ -371,137 +369,6 @@ func TestRelays(t *testing.T) {
 	//TODO: assert we actually used the relay even though it should be impossible for a tunnel to have occurred without it
 }
 
-func TestReestablishRelays(t *testing.T) {
-	ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(ca, caKey, "relay  ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them   ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnIpNet.Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnIpNet.Addr(), []netip.Addr{relayVpnIpNet.Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnIpNet.Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunUDPPacket(theirVpnIpNet.Addr(), 80, 80, []byte("Hi from me"))
-
-	p := r.RouteForAllUntilTxTun(theirControl)
-	r.Log("Assert the tunnel works")
-	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet.Addr(), theirVpnIpNet.Addr(), 80, 80)
-
-	t.Log("Ensure packet traversal from them to me via the relay")
-	theirControl.InjectTunUDPPacket(myVpnIpNet.Addr(), 80, 80, []byte("Hi from them"))
-
-	p = r.RouteForAllUntilTxTun(myControl)
-	r.Log("Assert the tunnel works")
-	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnIpNet.Addr(), myVpnIpNet.Addr(), 80, 80)
-
-	// If we break the relay's connection to 'them', 'me' needs to detect and recover the connection
-	r.Log("Close the tunnel")
-	relayControl.CloseTunnel(theirVpnIpNet.Addr(), true)
-
-	start := len(myControl.GetHostmap().Indexes)
-	curIndexes := len(myControl.GetHostmap().Indexes)
-	for curIndexes >= start {
-		curIndexes = len(myControl.GetHostmap().Indexes)
-		r.Logf("Wait for the dead index to go away:start=%v indexes, current=%v indexes", start, curIndexes)
-		myControl.InjectTunUDPPacket(theirVpnIpNet.Addr(), 80, 80, []byte("Hi from me should fail"))
-
-		r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
-			return router.RouteAndExit
-		})
-		time.Sleep(2 * time.Second)
-	}
-	r.Log("Dead index went away. Woot!")
-	r.RenderHostmaps("Me removed hostinfo", myControl, relayControl, theirControl)
-	// Next packet should re-establish a relayed connection and work just great.
-
-	t.Logf("Assert the tunnel...")
-	for {
-		t.Log("RouteForAllUntilTxTun")
-		myControl.InjectLightHouseAddr(relayVpnIpNet.Addr(), relayUdpAddr)
-		myControl.InjectRelays(theirVpnIpNet.Addr(), []netip.Addr{relayVpnIpNet.Addr()})
-		relayControl.InjectLightHouseAddr(theirVpnIpNet.Addr(), theirUdpAddr)
-		myControl.InjectTunUDPPacket(theirVpnIpNet.Addr(), 80, 80, []byte("Hi from me"))
-
-		p = r.RouteForAllUntilTxTun(theirControl)
-		r.Log("Assert the tunnel works")
-		packet := gopacket.NewPacket(p, layers.LayerTypeIPv4, gopacket.Lazy)
-		v4 := packet.Layer(layers.LayerTypeIPv4).(*layers.IPv4)
-		if slices.Compare(v4.SrcIP, myVpnIpNet.Addr().AsSlice()) != 0 {
-			t.Logf("SrcIP is unexpected...this is not the packet I'm looking for. Keep looking")
-			continue
-		}
-		if slices.Compare(v4.DstIP, theirVpnIpNet.Addr().AsSlice()) != 0 {
-			t.Logf("DstIP is unexpected...this is not the packet I'm looking for. Keep looking")
-			continue
-		}
-
-		udp := packet.Layer(layers.LayerTypeUDP).(*layers.UDP)
-		if udp == nil {
-			t.Log("Not a UDP packet. This is not the packet I'm looking for. Keep looking")
-			continue
-		}
-		data := packet.ApplicationLayer()
-		if data == nil {
-			t.Log("No data found in packet. This is not the packet I'm looking for. Keep looking.")
-			continue
-		}
-		if string(data.Payload()) != "Hi from me" {
-			t.Logf("Unexpected payload: '%v', keep looking", string(data.Payload()))
-			continue
-		}
-		t.Log("I found my lost packet. I am so happy.")
-		break
-	}
-	t.Log("Assert the tunnel works the other way, too")
-	for {
-		t.Log("RouteForAllUntilTxTun")
-		theirControl.InjectTunUDPPacket(myVpnIpNet.Addr(), 80, 80, []byte("Hi from them"))
-
-		p = r.RouteForAllUntilTxTun(myControl)
-		r.Log("Assert the tunnel works")
-		packet := gopacket.NewPacket(p, layers.LayerTypeIPv4, gopacket.Lazy)
-		v4 := packet.Layer(layers.LayerTypeIPv4).(*layers.IPv4)
-		if slices.Compare(v4.DstIP, myVpnIpNet.Addr().AsSlice()) != 0 {
-			t.Logf("Dst is unexpected...this is not the packet I'm looking for. Keep looking")
-			continue
-		}
-		if slices.Compare(v4.SrcIP, theirVpnIpNet.Addr().AsSlice()) != 0 {
-			t.Logf("SrcIP is unexpected...this is not the packet I'm looking for. Keep looking")
-			continue
-		}
-
-		udp := packet.Layer(layers.LayerTypeUDP).(*layers.UDP)
-		if udp == nil {
-			t.Log("Not a UDP packet. This is not the packet I'm looking for. Keep looking")
-			continue
-		}
-		data := packet.ApplicationLayer()
-		if data == nil {
-			t.Log("No data found in packet. This is not the packet I'm looking for. Keep looking.")
-			continue
-		}
-		if string(data.Payload()) != "Hi from them" {
-			t.Logf("Unexpected payload: '%v', keep looking", string(data.Payload()))
-			continue
-		}
-		t.Log("I found my lost packet. I am so happy.")
-		break
-	}
-	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
-
-}
-
 func TestStage1RaceRelays(t *testing.T) {
 	//NOTE: this is a race between me and relay resulting in a full tunnel from me to them via relay
 	ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
@@ -963,8 +830,9 @@ func TestRehandshakingLoser(t *testing.T) {
 	t.Log("Stand up a tunnel between me and them")
 	assertTunnel(t, myVpnIpNet.Addr(), theirVpnIpNet.Addr(), myControl, theirControl, r)
 
-	myControl.GetHostInfoByVpnIp(theirVpnIpNet.Addr(), false)
+	tt1 := myControl.GetHostInfoByVpnIp(theirVpnIpNet.Addr(), false)
-	theirControl.GetHostInfoByVpnIp(myVpnIpNet.Addr(), false)
+	tt2 := theirControl.GetHostInfoByVpnIp(myVpnIpNet.Addr(), false)
+	fmt.Println(tt1.LocalIndex, tt2.LocalIndex)
 
 	r.RenderHostmaps("Starting hostmaps", myControl, theirControl)
 

e2e/router/router.go

@@ -690,7 +690,6 @@ func (r *R) FlushAll() {
 			r.Unlock()
 			panic("Can't FlushAll for host: " + p.To.String())
 		}
-		receiver.InjectUDPPacket(p)
 		r.Unlock()
 	}
 }

e2e/tunnels_test.go

@@ -1,55 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula/e2e/router"
-)
-
-func TestDropInactiveTunnels(t *testing.T) {
-	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
-	// under ideal conditions
-	ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(ca, caKey, "me", "10.128.0.1/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "5s"}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them", "10.128.0.2/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "10m"}})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet.Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet.Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-
-	r.Log("Assert the tunnel between me and them works")
-	assertTunnel(t, myVpnIpNet.Addr(), theirVpnIpNet.Addr(), myControl, theirControl, r)
-
-	r.Log("Go inactive and wait for the tunnels to get dropped")
-	waitStart := time.Now()
-	for {
-		myIndexes := len(myControl.GetHostmap().Indexes)
-		theirIndexes := len(theirControl.GetHostmap().Indexes)
-		if myIndexes == 0 && theirIndexes == 0 {
-			break
-		}
-
-		since := time.Since(waitStart)
-		r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since)
-		if since > time.Second*30 {
-			t.Fatal("Tunnel should have been declared inactive after 5 seconds and before 30 seconds")
-		}
-
-		time.Sleep(1 * time.Second)
-		r.FlushAll()
-	}
-
-	r.Logf("Inactive tunnels were dropped within %v", time.Since(waitStart))
-	myControl.Stop()
-	theirControl.Stop()
-}

examples/config.yml

@@ -303,18 +303,6 @@ logging:
   # after receiving the response for lighthouse queries
   #trigger_buffer: 64
 
-# Tunnel manager settings
-#tunnels:
-  # drop_inactive controls whether inactive tunnels are maintained or dropped after the inactive_timeout period has
-  # elapsed.
-  # In general, it is a good idea to enable this setting. It will be enabled by default in a future release.
-  # This setting is reloadable
-  #drop_inactive: false
-
-  # inactivity_timeout controls how long a tunnel MUST NOT see any inbound or outbound traffic before being considered
-  # inactive and eligible to be dropped.
-  # This setting is reloadable
-  #inactivity_timeout: 10m
 
 # Nebula security group configuration
 firewall:

examples/go_service/main.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"fmt"
 	"log"
-	"net"
 
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/service"
@@ -55,16 +54,16 @@ pki:
   cert: /home/rice/Developer/nebula-config/app.crt
   key: /home/rice/Developer/nebula-config/app.key
 `
-	var cfg config.C
+	var config config.C
-	if err := cfg.LoadString(configStr); err != nil {
+	if err := config.LoadString(configStr); err != nil {
 		return err
 	}
-	svc, err := service.New(&cfg)
+	service, err := service.New(&config)
 	if err != nil {
 		return err
 	}
 
-	ln, err := svc.Listen("tcp", ":1234")
+	ln, err := service.Listen("tcp", ":1234")
 	if err != nil {
 		return err
 	}
@@ -74,24 +73,16 @@ pki:
 			log.Printf("accept error: %s", err)
 			break
 		}
-		defer func(conn net.Conn) {
+		defer conn.Close()
-			_ = conn.Close()
-		}(conn)
 
 		log.Printf("got connection")
 
-		_, err = conn.Write([]byte("hello world\n"))
+		conn.Write([]byte("hello world\n"))
-		if err != nil {
-			log.Printf("write error: %s", err)
-		}
 
 		scanner := bufio.NewScanner(conn)
 		for scanner.Scan() {
 			message := scanner.Text()
-			_, err = fmt.Fprintf(conn, "echo: %q\n", message)
+			fmt.Fprintf(conn, "echo: %q\n", message)
-			if err != nil {
-				log.Printf("write error: %s", err)
-			}
 			log.Printf("got message %q", message)
 		}
 
@@ -101,8 +92,8 @@ pki:
 		}
 	}
 
-	_ = svc.Close()
+	service.Close()
-	if err := svc.Wait(); err != nil {
+	if err := service.Wait(); err != nil {
 		return err
 	}
 	return nil

go.mod

@@ -23,12 +23,12 @@ require (
 	github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
 	github.com/stretchr/testify v1.9.0
 	github.com/vishvananda/netlink v1.2.1-beta.2
-	golang.org/x/crypto v0.26.0
+	golang.org/x/crypto v0.25.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.28.0
+	golang.org/x/net v0.27.0
 	golang.org/x/sync v0.8.0
-	golang.org/x/sys v0.24.0
+	golang.org/x/sys v0.23.0
-	golang.org/x/term v0.23.0
+	golang.org/x/term v0.22.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
 	golang.zx2c4.com/wireguard/windows v0.5.3

go.sum

@@ -151,8 +151,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
@@ -171,8 +171,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -199,11 +199,11 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
-golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
+golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
-golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
+golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

handshake_ix.go

@@ -151,7 +151,7 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 		HandshakePacket:   make(map[uint8][]byte, 0),
 		lastHandshakeTime: hs.Details.Time,
 		relayState: RelayState{
-			relays:        nil,
+			relays:        map[netip.Addr]struct{}{},
 			relayForByIp:  map[netip.Addr]*Relay{},
 			relayForByIdx: map[uint32]*Relay{},
 		},
@@ -322,9 +322,6 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 			return
 		}
 		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnIp)
-		// I successfully received a handshake. Just in case I marked this tunnel as 'Disestablished', ensure
-		// it's correctly marked as working.
-		via.relayHI.relayState.UpdateRelayForByIdxState(via.remoteIdx, Established)
 		f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
 		f.l.WithField("vpnIp", vpnIp).WithField("relay", via.relayHI.vpnIp).
 			WithField("certName", certName).
@@ -335,7 +332,7 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 			Info("Handshake message sent")
 	}
 
-	f.connectionManager.AddTrafficWatch(hostinfo)
+	f.connectionManager.AddTrafficWatch(hostinfo.localIndexId)
 
 	hostinfo.remotes.ResetBlockedRemotes()
 
@@ -493,7 +490,7 @@ func ixHandshakeStage2(f *Interface, addr netip.AddrPort, via *ViaSender, hh *Ha
 
 	// Complete our handshake and update metrics, this will replace any existing tunnels for this vpnIp
 	f.handshakeManager.Complete(hostinfo, f)
-	f.connectionManager.AddTrafficWatch(hostinfo)
+	f.connectionManager.AddTrafficWatch(hostinfo.localIndexId)
 
 	if f.l.Level >= logrus.DebugLevel {
 		hostinfo.logger(f.l).Debugf("Sending %d stored packets", len(hh.packetStore))

handshake_manager.go

@@ -278,8 +278,48 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 				continue
 			}
 			// Check the relay HostInfo to see if we already established a relay through it
-			existingRelay, ok := relayHostInfo.relayState.QueryRelayForByIp(vpnIp)
+			if existingRelay, ok := relayHostInfo.relayState.QueryRelayForByIp(vpnIp); ok {
-			if !ok {
+				switch existingRelay.State {
+				case Established:
+					hostinfo.logger(hm.l).WithField("relay", relay.String()).Info("Send handshake via relay")
+					hm.f.SendVia(relayHostInfo, existingRelay, hostinfo.HandshakePacket[0], make([]byte, 12), make([]byte, mtu), false)
+				case Requested:
+					hostinfo.logger(hm.l).WithField("relay", relay.String()).Info("Re-send CreateRelay request")
+
+					//TODO: IPV6-WORK
+					myVpnIpB := hm.f.myVpnNet.Addr().As4()
+					theirVpnIpB := vpnIp.As4()
+
+					// Re-send the CreateRelay request, in case the previous one was lost.
+					m := NebulaControl{
+						Type:                NebulaControl_CreateRelayRequest,
+						InitiatorRelayIndex: existingRelay.LocalIndex,
+						RelayFromIp:         binary.BigEndian.Uint32(myVpnIpB[:]),
+						RelayToIp:           binary.BigEndian.Uint32(theirVpnIpB[:]),
+					}
+					msg, err := m.Marshal()
+					if err != nil {
+						hostinfo.logger(hm.l).
+							WithError(err).
+							Error("Failed to marshal Control message to create relay")
+					} else {
+						// This must send over the hostinfo, not over hm.Hosts[ip]
+						hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
+						hm.l.WithFields(logrus.Fields{
+							"relayFrom":           hm.f.myVpnNet.Addr(),
+							"relayTo":             vpnIp,
+							"initiatorRelayIndex": existingRelay.LocalIndex,
+							"relay":               relay}).
+							Info("send CreateRelayRequest")
+					}
+				default:
+					hostinfo.logger(hm.l).
+						WithField("vpnIp", vpnIp).
+						WithField("state", existingRelay.State).
+						WithField("relay", relayHostInfo.vpnIp).
+						Errorf("Relay unexpected state")
+				}
+			} else {
 				// No relays exist or requested yet.
 				if relayHostInfo.remote.IsValid() {
 					idx, err := AddRelay(hm.l, relayHostInfo, hm.mainHostMap, vpnIp, nil, TerminalType, Requested)
@@ -312,52 +352,6 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 							Info("send CreateRelayRequest")
 					}
 				}
-				continue
-			}
-			switch existingRelay.State {
-			case Established:
-				hostinfo.logger(hm.l).WithField("relay", relay.String()).Info("Send handshake via relay")
-				hm.f.SendVia(relayHostInfo, existingRelay, hostinfo.HandshakePacket[0], make([]byte, 12), make([]byte, mtu), false)
-			case Disestablished:
-				// Mark this relay as 'requested'
-				relayHostInfo.relayState.UpdateRelayForByIpState(vpnIp, Requested)
-				fallthrough
-			case Requested:
-				hostinfo.logger(hm.l).WithField("relay", relay.String()).Info("Re-send CreateRelay request")
-				// Re-send the CreateRelay request, in case the previous one was lost.
-				relayFrom := hm.f.myVpnNet.Addr().As4()
-				relayTo := vpnIp.As4()
-				m := NebulaControl{
-					Type:                NebulaControl_CreateRelayRequest,
-					InitiatorRelayIndex: existingRelay.LocalIndex,
-					RelayFromIp:         binary.BigEndian.Uint32(relayFrom[:]),
-					RelayToIp:           binary.BigEndian.Uint32(relayTo[:]),
-				}
-
-				msg, err := m.Marshal()
-				if err != nil {
-					hostinfo.logger(hm.l).
-						WithError(err).
-						Error("Failed to marshal Control message to create relay")
-				} else {
-					// This must send over the hostinfo, not over hm.Hosts[ip]
-					hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
-					hm.l.WithFields(logrus.Fields{
-						"relayFrom":           hm.f.myVpnNet,
-						"relayTo":             vpnIp,
-						"initiatorRelayIndex": existingRelay.LocalIndex,
-						"relay":               relay}).
-						Info("send CreateRelayRequest")
-				}
-			case PeerRequested:
-				// PeerRequested only occurs in Forwarding relays, not Terminal relays, and this is a Terminal relay case.
-				fallthrough
-			default:
-				hostinfo.logger(hm.l).
-					WithField("vpnIp", vpnIp).
-					WithField("state", existingRelay.State).
-					WithField("relay", relay).
-					Errorf("Relay unexpected state")
 			}
 		}
 	}
@@ -403,7 +397,7 @@ func (hm *HandshakeManager) StartHandshake(vpnIp netip.Addr, cacheCb func(*Hands
 		vpnIp:           vpnIp,
 		HandshakePacket: make(map[uint8][]byte, 0),
 		relayState: RelayState{
-			relays:        nil,
+			relays:        map[netip.Addr]struct{}{},
 			relayForByIp:  map[netip.Addr]*Relay{},
 			relayForByIdx: map[uint32]*Relay{},
 		},

hostmap.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"net"
 	"net/netip"
-	"slices"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -22,6 +21,7 @@ const defaultPromoteEvery = 1000       // Count of packets sent before we try mo
 const defaultReQueryEvery = 5000       // Count of packets sent before re-querying a hostinfo to the lighthouse
 const defaultReQueryWait = time.Minute // Minimum amount of seconds to wait before re-querying a hostinfo the lighthouse. Evaluated every ReQueryEvery
 const MaxRemotes = 10
+const maxRecvError = 4
 
 // MaxHostInfosPerVpnIp is the max number of hostinfos we will track for a given vpn ip
 // 5 allows for an initial handshake and each host pair re-handshaking twice
@@ -35,7 +35,6 @@ const (
 	Requested = iota
 	PeerRequested
 	Established
-	Disestablished
 )
 
 const (
@@ -69,7 +68,7 @@ type HostMap struct {
 type RelayState struct {
 	sync.RWMutex
 
-	relays        []netip.Addr          // Ordered set of VpnIp's of Hosts to use as relays to access this peer
+	relays        map[netip.Addr]struct{} // Set of VpnIp's of Hosts to use as relays to access this peer
 	relayForByIp  map[netip.Addr]*Relay   // Maps VpnIps of peers for which this HostInfo is a relay to some Relay info
 	relayForByIdx map[uint32]*Relay       // Maps a local index to some Relay info
 }
@@ -77,34 +76,7 @@ type RelayState struct {
 func (rs *RelayState) DeleteRelay(ip netip.Addr) {
 	rs.Lock()
 	defer rs.Unlock()
-	for idx, val := range rs.relays {
+	delete(rs.relays, ip)
-		if val == ip {
-			rs.relays = append(rs.relays[:idx], rs.relays[idx+1:]...)
-			return
-		}
-	}
-}
-
-func (rs *RelayState) UpdateRelayForByIpState(vpnIp netip.Addr, state int) {
-	rs.Lock()
-	defer rs.Unlock()
-	if r, ok := rs.relayForByIp[vpnIp]; ok {
-		newRelay := *r
-		newRelay.State = state
-		rs.relayForByIp[newRelay.PeerIp] = &newRelay
-		rs.relayForByIdx[newRelay.LocalIndex] = &newRelay
-	}
-}
-
-func (rs *RelayState) UpdateRelayForByIdxState(idx uint32, state int) {
-	rs.Lock()
-	defer rs.Unlock()
-	if r, ok := rs.relayForByIdx[idx]; ok {
-		newRelay := *r
-		newRelay.State = state
-		rs.relayForByIp[newRelay.PeerIp] = &newRelay
-		rs.relayForByIdx[newRelay.LocalIndex] = &newRelay
-	}
 }
 
 func (rs *RelayState) CopyAllRelayFor() []*Relay {
@@ -127,16 +99,16 @@ func (rs *RelayState) GetRelayForByIp(ip netip.Addr) (*Relay, bool) {
 func (rs *RelayState) InsertRelayTo(ip netip.Addr) {
 	rs.Lock()
 	defer rs.Unlock()
-	if !slices.Contains(rs.relays, ip) {
+	rs.relays[ip] = struct{}{}
-		rs.relays = append(rs.relays, ip)
-	}
 }
 
 func (rs *RelayState) CopyRelayIps() []netip.Addr {
-	ret := make([]netip.Addr, len(rs.relays))
 	rs.RLock()
 	defer rs.RUnlock()
-	copy(ret, rs.relays)
+	ret := make([]netip.Addr, 0, len(rs.relays))
+	for ip := range rs.relays {
+		ret = append(ret, ip)
+	}
 	return ret
 }
 
@@ -219,6 +191,7 @@ type HostInfo struct {
 	remoteIndexId   uint32
 	localIndexId    uint32
 	vpnIp           netip.Addr
+	recvError       atomic.Uint32
 	remoteCidr      *bart.Table[struct{}]
 	relayState      RelayState
 
@@ -246,14 +219,6 @@ type HostInfo struct {
 	// Used to track other hostinfos for this vpn ip since only 1 can be primary
 	// Synchronised via hostmap lock and not the hostinfo lock.
 	next, prev *HostInfo
-
-	//TODO: in, out, and others might benefit from being an atomic.Int32. We could collapse connectionManager pendingDeletion, relayUsed, and in/out into this 1 thing
-	in, out, pendingDeletion atomic.Bool
-
-	// lastUsed tracks the last time ConnectionManager checked the tunnel and it was in use.
-	// This value will be behind against actual tunnel utilization in the hot path.
-	// This should only be used by the ConnectionManagers ticker routine.
-	lastUsed time.Time
 }
 
 type ViaSender struct {
@@ -396,7 +361,6 @@ func (hm *HostMap) unlockedMakePrimary(hostinfo *HostInfo) {
 
 func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) {
 	primary, ok := hm.Hosts[hostinfo.vpnIp]
-	isLastHostinfo := hostinfo.next == nil && hostinfo.prev == nil
 	if ok && primary == hostinfo {
 		// The vpnIp pointer points to the same hostinfo as the local index id, we can remove it
 		delete(hm.Hosts, hostinfo.vpnIp)
@@ -446,12 +410,6 @@ func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) {
 			Debug("Hostmap hostInfo deleted")
 	}
 
-	if isLastHostinfo {
-		// I have lost connectivity to my peers. My relay tunnel is likely broken. Mark the next
-		// hops as 'Disestablished' so that new relay tunnels are created in the future.
-		hm.unlockedDisestablishVpnAddrRelayFor(hostinfo)
-	}
-	// Clean up any local relay indexes for which I am acting as a relay hop
 	for _, localRelayIdx := range hostinfo.relayState.CopyRelayForIdxs() {
 		delete(hm.Relays, localRelayIdx)
 	}
@@ -512,27 +470,6 @@ func (hm *HostMap) QueryVpnIpRelayFor(targetIp, relayHostIp netip.Addr) (*HostIn
 	return nil, nil, errors.New("unable to find host with relay")
 }
 
-func (hm *HostMap) unlockedDisestablishVpnAddrRelayFor(hi *HostInfo) {
-	for _, relayHostIp := range hi.relayState.CopyRelayIps() {
-		if h, ok := hm.Hosts[relayHostIp]; ok {
-			for h != nil {
-				h.relayState.UpdateRelayForByIpState(hi.vpnIp, Disestablished)
-				h = h.next
-			}
-		}
-	}
-	for _, rs := range hi.relayState.CopyAllRelayFor() {
-		if rs.Type == ForwardingType {
-			if h, ok := hm.Hosts[rs.PeerIp]; ok {
-				for h != nil {
-					h.relayState.UpdateRelayForByIpState(hi.vpnIp, Disestablished)
-					h = h.next
-				}
-			}
-		}
-	}
-}
-
 func (hm *HostMap) queryVpnIp(vpnIp netip.Addr, promoteIfce *Interface) *HostInfo {
 	hm.RLock()
 	if h, ok := hm.Hosts[vpnIp]; ok {
@@ -703,6 +640,13 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, newRemote netip.AddrPort) b
 	return false
 }
 
+func (i *HostInfo) RecvErrorExceeded() bool {
+	if i.recvError.Add(1) >= maxRecvError {
+		return true
+	}
+	return true
+}
+
 func (i *HostInfo) CreateRemoteCIDR(c *cert.NebulaCertificate) {
 	if len(c.Details.Ips) == 1 && len(c.Details.Subnets) == 0 {
 		// Simple case, no CIDRTree needed
@@ -714,7 +658,8 @@ func (i *HostInfo) CreateRemoteCIDR(c *cert.NebulaCertificate) {
 		//TODO: IPV6-WORK what to do when ip is invalid?
 		nip, _ := netip.AddrFromSlice(ip.IP)
 		nip = nip.Unmap()
-		remoteCidr.Insert(netip.PrefixFrom(nip, nip.BitLen()), struct{}{})
+		bits, _ := ip.Mask.Size()
+		remoteCidr.Insert(netip.PrefixFrom(nip, bits), struct{}{})
 	}
 
 	for _, n := range c.Details.Subnets {

hostmap_test.go

@@ -1,15 +1,12 @@
 package nebula
 
 import (
-	"net"
 	"net/netip"
 	"testing"
 
-	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestHostMap_MakePrimary(t *testing.T) {
@@ -89,40 +86,6 @@ func TestHostMap_MakePrimary(t *testing.T) {
 	assert.Nil(t, h2.next)
 }
 
-func TestHostInfo_CreateRemoteCIDR(t *testing.T) {
-	h := HostInfo{}
-	c := &cert.NebulaCertificate{
-		Details: cert.NebulaCertificateDetails{
-			Ips: []*net.IPNet{
-				{
-					IP:   net.IPv4(1, 2, 3, 4),
-					Mask: net.IPv4Mask(255, 255, 255, 0),
-				},
-			},
-		},
-	}
-
-	// remoteCidr should be empty with only 1 ip address present in the certificate
-	h.CreateRemoteCIDR(c)
-	assert.Empty(t, h.remoteCidr)
-
-	// remoteCidr should be populated if there is also a subnet in the certificate
-	c.Details.Subnets = []*net.IPNet{
-		{
-			IP:   net.IPv4(9, 2, 3, 4),
-			Mask: net.IPv4Mask(255, 255, 255, 0),
-		},
-	}
-	h.CreateRemoteCIDR(c)
-	assert.NotEmpty(t, h.remoteCidr)
-	_, ok := h.remoteCidr.Lookup(netip.MustParseAddr("1.2.3.0"))
-	assert.False(t, ok, "An ip address within the certificates network should not be found")
-	_, ok = h.remoteCidr.Lookup(netip.MustParseAddr("1.2.3.4"))
-	assert.True(t, ok, "An exact ip address match should be found")
-	_, ok = h.remoteCidr.Lookup(netip.MustParseAddr("9.2.3.4"))
-	assert.True(t, ok, "An ip address within the subnets should be found")
-}
-
 func TestHostMap_DeleteHostInfo(t *testing.T) {
 	l := test.NewLogger()
 	hm := newHostMap(
@@ -262,31 +225,3 @@ func TestHostMap_reload(t *testing.T) {
 	c.ReloadConfigString("preferred_ranges: [1.1.1.1/32]")
 	assert.EqualValues(t, []string{"1.1.1.1/32"}, toS(hm.GetPreferredRanges()))
 }
-
-func TestHostMap_RelayState(t *testing.T) {
-	h1 := &HostInfo{vpnIp: netip.MustParseAddr("0.0.0.1"), localIndexId: 1}
-	a1 := netip.MustParseAddr("::1")
-	a2 := netip.MustParseAddr("2001::1")
-
-	h1.relayState.InsertRelayTo(a1)
-	assert.Equal(t, h1.relayState.relays, []netip.Addr{a1})
-	h1.relayState.InsertRelayTo(a2)
-	assert.Equal(t, h1.relayState.relays, []netip.Addr{a1, a2})
-	// Ensure that the first relay added is the first one returned in the copy
-	currentRelays := h1.relayState.CopyRelayIps()
-	require.Len(t, currentRelays, 2)
-	assert.Equal(t, currentRelays[0], a1)
-
-	// Deleting the last one in the list works ok
-	h1.relayState.DeleteRelay(a2)
-	assert.Equal(t, h1.relayState.relays, []netip.Addr{a1})
-
-	// Deleting an element not in the list works ok
-	h1.relayState.DeleteRelay(a2)
-	assert.Equal(t, h1.relayState.relays, []netip.Addr{a1})
-
-	// Deleting the only element in the list works ok
-	h1.relayState.DeleteRelay(a1)
-	assert.Equal(t, h1.relayState.relays, []netip.Addr{})
-
-}

inside.go

@@ -213,7 +213,7 @@ func (f *Interface) SendVia(via *HostInfo,
 	c := via.ConnectionState.messageCounter.Add(1)
 
 	out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
-	f.connectionManager.Out(via)
+	f.connectionManager.Out(via.localIndexId)
 
 	// Authenticate the header and payload, but do not encrypt for this message type.
 	// The payload consists of the inner, unencrypted Nebula header, as well as the end-to-end encrypted payload.
@@ -282,7 +282,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 
 	//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
 	out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
-	f.connectionManager.Out(hostinfo)
+	f.connectionManager.Out(hostinfo.localIndexId)
 
 	// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against
 	// all our IPs and enable a faster roaming.

interface.go

@@ -33,7 +33,8 @@ type InterfaceConfig struct {
 	ServeDns                bool
 	HandshakeManager        *HandshakeManager
 	lightHouse              *LightHouse
-	connectionManager  *connectionManager
+	checkInterval           time.Duration
+	pendingDeletionInterval time.Duration
 	DropLocalBroadcast      bool
 	DropMulticast           bool
 	routines                int
@@ -153,9 +154,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 	if c.Firewall == nil {
 		return nil, errors.New("no firewall rules")
 	}
-	if c.connectionManager == nil {
-		return nil, errors.New("no connection manager")
-	}
 
 	certificate := c.pki.GetCertState().Certificate
 
@@ -198,7 +196,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		readers:            make([]io.ReadWriteCloser, c.routines),
 		myVpnNet:           myVpnNet,
 		relayManager:       c.relayManager,
-		connectionManager:  c.connectionManager,
 
 		conntrackCacheTimeout: c.ConntrackCacheTimeout,
 
@@ -222,7 +219,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 	ifce.reQueryEvery.Store(c.reQueryEvery)
 	ifce.reQueryWait.Store(int64(c.reQueryWait))
 
-	ifce.connectionManager.intf = ifce
+	ifce.connectionManager = newConnectionManager(ctx, c.l, ifce, c.checkInterval, c.pendingDeletionInterval, c.punchy)
 
 	return ifce, nil
 }

main.go

@@ -199,7 +199,6 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 
 	hostMap := NewHostMapFromConfig(l, tunCidr, c)
 	punchy := NewPunchyFromConfig(l, c)
-	connManager := newConnectionManagerFromConfig(l, c, hostMap, punchy)
 	lightHouse, err := NewLightHouseFromConfig(ctx, l, c, tunCidr, udpConns[0], punchy)
 	if err != nil {
 		return nil, util.ContextualizeIfNeeded("Failed to initialize lighthouse handler", err)
@@ -235,6 +234,9 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		}
 	}
 
+	checkInterval := c.GetInt("timers.connection_alive_interval", 5)
+	pendingDeletionInterval := c.GetInt("timers.pending_deletion_interval", 10)
+
 	ifConfig := &InterfaceConfig{
 		HostMap:                 hostMap,
 		Inside:                  tun,
@@ -244,8 +246,9 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		Firewall:                fw,
 		ServeDns:                serveDns,
 		HandshakeManager:        handshakeManager,
-		connectionManager:  connManager,
 		lightHouse:              lightHouse,
+		checkInterval:           time.Second * time.Duration(checkInterval),
+		pendingDeletionInterval: time.Second * time.Duration(pendingDeletionInterval),
 		tryPromoteEvery:         c.GetUint32("counters.try_promote", defaultPromoteEvery),
 		reQueryEvery:            c.GetUint32("counters.requery_every_packets", defaultReQueryEvery),
 		reQueryWait:             c.GetDuration("timers.requery_wait_duration", defaultReQueryWait),
@@ -322,6 +325,5 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		statsStart,
 		dnsStart,
 		lightHouse.StartUpdateWorker,
-		connManager.Start,
 	}, nil
 }

outside.go

@@ -102,7 +102,7 @@ func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []
 			// Pull the Roaming parts up here, and return in all call paths.
 			f.handleHostRoaming(hostinfo, ip)
 			// Track usage of both the HostInfo and the Relay for the received & authenticated packet
-			f.connectionManager.In(hostinfo)
+			f.connectionManager.In(hostinfo.localIndexId)
 			f.connectionManager.RelayUsed(h.RemoteIndex)
 
 			relay, ok := hostinfo.relayState.QueryRelayForByIdx(h.RemoteIndex)
@@ -246,7 +246,7 @@ func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []
 
 	f.handleHostRoaming(hostinfo, ip)
 
-	f.connectionManager.In(hostinfo)
+	f.connectionManager.In(hostinfo.localIndexId)
 }
 
 // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
@@ -286,18 +286,16 @@ func (f *Interface) handleHostRoaming(hostinfo *HostInfo, ip netip.AddrPort) {
 
 }
 
-// handleEncrypted returns true if a packet should be processed, false otherwise
 func (f *Interface) handleEncrypted(ci *ConnectionState, addr netip.AddrPort, h *header.H) bool {
-	// If connectionstate does not exist, send a recv error, if possible, to encourage a fast reconnect
+	// If connectionstate exists and the replay protector allows, process packet
-	if ci == nil {
+	// Else, send recv errors for 300 seconds after a restart to allow fast reconnection.
+	if ci == nil || !ci.window.Check(f.l, h.MessageCounter) {
 		if addr.IsValid() {
 			f.maybeSendRecvError(addr, h.RemoteIndex)
-		}
+			return false
+		} else {
 			return false
 		}
-	// If the window check fails, refuse to process the packet, but don't send a recv error
-	if !ci.window.Check(f.l, h.MessageCounter) {
-		return false
 	}
 
 	return true
@@ -420,7 +418,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 		return false
 	}
 
-	f.connectionManager.In(hostinfo)
+	f.connectionManager.In(hostinfo.localIndexId)
 	_, err = f.readers[q].Write(out)
 	if err != nil {
 		f.l.WithError(err).Error("Failed to write to tun")
@@ -460,6 +458,10 @@ func (f *Interface) handleRecvError(addr netip.AddrPort, h *header.H) {
 		return
 	}
 
+	if !hostinfo.RecvErrorExceeded() {
+		return
+	}
+
 	if hostinfo.remote.IsValid() && hostinfo.remote != addr {
 		f.l.Infoln("Someone spoofing recv_errors? ", addr, hostinfo.remote)
 		return

pki.go

@@ -223,13 +223,22 @@ func loadCAPoolFromConfig(l *logrus.Logger, c *config.C) (*cert.NebulaCAPool, er
 		}
 	}
 
-	caPool, warnings, err := cert.NewCAPoolFromBytes(rawCA)
+	caPool, err := cert.NewCAPoolFromBytes(rawCA)
-	for _, w := range warnings {
+	if errors.Is(err, cert.ErrExpired) {
-		l.WithError(w).Warn("parsing a CA certificate failed")
+		var expired int
+		for _, crt := range caPool.CAs {
+			if crt.Expired(time.Now()) {
+				expired++
+				l.WithField("cert", crt).Warn("expired certificate present in CA pool")
+			}
 		}
 
-	if err != nil {
+		if expired >= len(caPool.CAs) {
-		return nil, fmt.Errorf("could not create CA certificate pool: %s", err)
+			return nil, errors.New("no valid CA certificates present")
+		}
+
+	} else if err != nil {
+		return nil, fmt.Errorf("error while adding CA certificate to CA trust store: %s", err)
 	}
 
 	for _, fp := range c.GetStringSlice("pki.blocklist", []string{}) {

relay_manager.go

@@ -146,14 +146,10 @@ func (rm *relayManager) handleCreateRelayResponse(h *HostInfo, f *Interface, m *
 		rm.l.WithField("relayTo", peerHostInfo.vpnIp).Error("peerRelay does not have Relay state for relayTo")
 		return
 	}
-	switch peerRelay.State {
+	if peerRelay.State == PeerRequested {
-	case Requested:
-		// I initiated the request to this peer, but haven't heard back from the peer yet. I must wait for this peer
-		// to respond to complete the connection.
-	case PeerRequested, Disestablished, Established:
-		peerHostInfo.relayState.UpdateRelayForByIpState(targetAddr, Established)
 		//TODO: IPV6-WORK
 		b = peerHostInfo.vpnIp.As4()
+		peerRelay.State = Established
 		resp := NebulaControl{
 			Type:                NebulaControl_CreateRelayResponse,
 			ResponderRelayIndex: peerRelay.LocalIndex,
@@ -219,21 +215,6 @@ func (rm *relayManager) handleCreateRelayRequest(h *HostInfo, f *Interface, m *N
 						"existingRemoteIndex": existingRelay.RemoteIndex}).Error("Existing relay mismatch with CreateRelayRequest")
 					return
 				}
-			case Disestablished:
-				if existingRelay.RemoteIndex != m.InitiatorRelayIndex {
-					// We got a brand new Relay request, because its index is different than what we saw before.
-					// This should never happen. The peer should never change an index, once created.
-					logMsg.WithFields(logrus.Fields{
-						"existingRemoteIndex": existingRelay.RemoteIndex}).Error("Existing relay mismatch with CreateRelayRequest")
-					return
-				}
-				// Mark the relay as 'Established' because it's safe to use again
-				h.relayState.UpdateRelayForByIpState(from, Established)
-			case PeerRequested:
-				// I should never be in this state, because I am terminal, not forwarding.
-				logMsg.WithFields(logrus.Fields{
-					"existingRemoteIndex": existingRelay.RemoteIndex,
-					"state":               existingRelay.State}).Error("Unexpected Relay State found")
 			}
 		} else {
 			_, err := AddRelay(rm.l, h, f.hostMap, from, &m.InitiatorRelayIndex, TerminalType, Established)
@@ -245,7 +226,7 @@ func (rm *relayManager) handleCreateRelayRequest(h *HostInfo, f *Interface, m *N
 
 		relay, ok := h.relayState.QueryRelayForByIp(from)
 		if !ok {
-			logMsg.WithField("from", from).Error("Relay State not found")
+			logMsg.Error("Relay State not found")
 			return
 		}
 
@@ -292,22 +273,29 @@ func (rm *relayManager) handleCreateRelayRequest(h *HostInfo, f *Interface, m *N
 			// Only create relays to peers for whom I have a direct connection
 			return
 		}
+		sendCreateRequest := false
 		var index uint32
 		var err error
 		targetRelay, ok := peer.relayState.QueryRelayForByIp(from)
 		if ok {
 			index = targetRelay.LocalIndex
+			if targetRelay.State == Requested {
+				sendCreateRequest = true
+			}
 		} else {
 			// Allocate an index in the hostMap for this relay peer
 			index, err = AddRelay(rm.l, peer, f.hostMap, from, nil, ForwardingType, Requested)
 			if err != nil {
 				return
 			}
+			sendCreateRequest = true
 		}
-		peer.relayState.UpdateRelayForByIpState(from, Requested)
+		if sendCreateRequest {
-		// Send a CreateRelayRequest to the peer.
+			//TODO: IPV6-WORK
-		fromB := from.As4()
+			fromB := h.vpnIp.As4()
 			targetB := target.As4()
+
+			// Send a CreateRelayRequest to the peer.
 			req := NebulaControl{
 				Type:                NebulaControl_CreateRelayRequest,
 				InitiatorRelayIndex: index,
@@ -326,18 +314,62 @@ func (rm *relayManager) handleCreateRelayRequest(h *HostInfo, f *Interface, m *N
 					"relayTo":             target,
 					"initiatorRelayIndex": req.InitiatorRelayIndex,
 					"responderRelayIndex": req.ResponderRelayIndex,
-				"vpnAddr":             target}).
+					"vpnIp":               target}).
 					Info("send CreateRelayRequest")
+			}
+		}
 		// Also track the half-created Relay state just received
-			_, ok := h.relayState.QueryRelayForByIp(target)
+		relay, ok := h.relayState.QueryRelayForByIp(target)
 		if !ok {
 			// Add the relay
-				_, err := AddRelay(rm.l, h, f.hostMap, target, &m.InitiatorRelayIndex, ForwardingType, PeerRequested)
+			state := PeerRequested
+			if targetRelay != nil && targetRelay.State == Established {
+				state = Established
+			}
+			_, err := AddRelay(rm.l, h, f.hostMap, target, &m.InitiatorRelayIndex, ForwardingType, state)
 			if err != nil {
 				logMsg.
 					WithError(err).Error("relayManager Failed to allocate a local index for relay")
 				return
 			}
+		} else {
+			switch relay.State {
+			case Established:
+				if relay.RemoteIndex != m.InitiatorRelayIndex {
+					// We got a brand new Relay request, because its index is different than what we saw before.
+					// This should never happen. The peer should never change an index, once created.
+					logMsg.WithFields(logrus.Fields{
+						"existingRemoteIndex": relay.RemoteIndex}).Error("Existing relay mismatch with CreateRelayRequest")
+					return
+				}
+				//TODO: IPV6-WORK
+				fromB := h.vpnIp.As4()
+				targetB := target.As4()
+				resp := NebulaControl{
+					Type:                NebulaControl_CreateRelayResponse,
+					ResponderRelayIndex: relay.LocalIndex,
+					InitiatorRelayIndex: relay.RemoteIndex,
+					RelayFromIp:         binary.BigEndian.Uint32(fromB[:]),
+					RelayToIp:           binary.BigEndian.Uint32(targetB[:]),
+				}
+				msg, err := resp.Marshal()
+				if err != nil {
+					rm.l.
+						WithError(err).Error("relayManager Failed to marshal Control CreateRelayResponse message to create relay")
+				} else {
+					f.SendMessageToHostInfo(header.Control, 0, h, msg, make([]byte, 12), make([]byte, mtu))
+					rm.l.WithFields(logrus.Fields{
+						//TODO: IPV6-WORK more lazy, used to use resp object
+						"relayFrom":           h.vpnIp,
+						"relayTo":             target,
+						"initiatorRelayIndex": resp.InitiatorRelayIndex,
+						"responderRelayIndex": resp.ResponderRelayIndex,
+						"vpnIp":               h.vpnIp}).
+						Info("send CreateRelayResponse")
+				}
+
+			case Requested:
+				// Keep waiting for the other relay to complete
 			}
 		}
 	}

service/service.go

@@ -8,7 +8,6 @@ import (
 	"log"
 	"math"
 	"net"
-	"net/netip"
 	"os"
 	"strings"
 	"sync"
@@ -154,48 +153,24 @@ func New(config *config.C) (*Service, error) {
 	return &s, nil
 }
 
-func getProtocolNumber(addr netip.Addr) tcpip.NetworkProtocolNumber {
+// DialContext dials the provided address. Currently only TCP is supported.
-	if addr.Is6() {
+func (s *Service) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
-		return ipv6.ProtocolNumber
+	if network != "tcp" && network != "tcp4" {
-	}
+		return nil, errors.New("only tcp is supported")
-	return ipv4.ProtocolNumber
 	}
 
-// DialContext dials the provided address.
-func (s *Service) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
-	switch network {
-	case "udp", "udp4", "udp6":
-		addr, err := net.ResolveUDPAddr(network, address)
-		if err != nil {
-			return nil, err
-		}
-		fullAddr := tcpip.FullAddress{
-			NIC:  nicID,
-			Addr: tcpip.AddrFromSlice(addr.IP),
-			Port: uint16(addr.Port),
-		}
-		num := getProtocolNumber(addr.AddrPort().Addr())
-		return gonet.DialUDP(s.ipstack, nil, &fullAddr, num)
-	case "tcp", "tcp4", "tcp6":
 	addr, err := net.ResolveTCPAddr(network, address)
 	if err != nil {
 		return nil, err
 	}
+
 	fullAddr := tcpip.FullAddress{
 		NIC:  nicID,
 		Addr: tcpip.AddrFromSlice(addr.IP),
 		Port: uint16(addr.Port),
 	}
-		num := getProtocolNumber(addr.AddrPort().Addr())
-		return gonet.DialContextTCP(ctx, s.ipstack, fullAddr, num)
-	default:
-		return nil, fmt.Errorf("unknown network type: %s", network)
-	}
-}
 
-// Dial dials the provided address
+	return gonet.DialContextTCP(ctx, s.ipstack, fullAddr, ipv4.ProtocolNumber)
-func (s *Service) Dial(network, address string) (net.Conn, error) {
-	return s.DialContext(context.Background(), network, address)
 }
 
 // Listen listens on the provided address. Currently only TCP with wildcard

udp/errors.go

@@ -1,5 +0,0 @@
-package udp
-
-import "errors"
-
-var ErrInvalidIPv6RemoteForSocket = errors.New("listener is IPv4, but writing to IPv6 remote")

udp/udp_darwin.go

@@ -6,63 +6,17 @@ package udp
 // Darwin support is primarily implemented in udp_generic, besides NewListenConfig
 
 import (
-	"context"
-	"encoding/binary"
-	"errors"
 	"fmt"
 	"net"
 	"net/netip"
 	"syscall"
-	"unsafe"
 
 	"github.com/sirupsen/logrus"
-	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/firewall"
-	"github.com/slackhq/nebula/header"
 	"golang.org/x/sys/unix"
 )
 
-type StdConn struct {
-	*net.UDPConn
-	isV4  bool
-	sysFd uintptr
-	l     *logrus.Logger
-}
-
-var _ Conn = &StdConn{}
-
 func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
-	lc := NewListenConfig(multi)
+	return NewGenericListener(l, ip, port, multi, batch)
-	pc, err := lc.ListenPacket(context.TODO(), "udp", net.JoinHostPort(ip.String(), fmt.Sprintf("%v", port)))
-	if err != nil {
-		return nil, err
-	}
-
-	if uc, ok := pc.(*net.UDPConn); ok {
-		c := &StdConn{UDPConn: uc, l: l}
-
-		rc, err := uc.SyscallConn()
-		if err != nil {
-			return nil, fmt.Errorf("failed to open udp socket: %w", err)
-		}
-
-		err = rc.Control(func(fd uintptr) {
-			c.sysFd = fd
-		})
-		if err != nil {
-			return nil, fmt.Errorf("failed to get udp fd: %w", err)
-		}
-
-		la, err := c.LocalAddr()
-		if err != nil {
-			return nil, err
-		}
-		c.isV4 = la.Addr().Is4()
-
-		return c, nil
-	}
-
-	return nil, fmt.Errorf("unexpected PacketConn: %T %#v", pc, pc)
 }
 
 func NewListenConfig(multi bool) net.ListenConfig {
@@ -89,130 +43,16 @@ func NewListenConfig(multi bool) net.ListenConfig {
 	}
 }
 
-//go:linkname sendto golang.org/x/sys/unix.sendto
+func (u *GenericConn) Rebind() error {
-//go:noescape
+	rc, err := u.UDPConn.SyscallConn()
-func sendto(s int, buf []byte, flags int, to unsafe.Pointer, addrlen int32) (err error)
-
-func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
-	var sa unsafe.Pointer
-	var addrLen int32
-
-	if u.isV4 {
-		if ap.Addr().Is6() {
-			return ErrInvalidIPv6RemoteForSocket
-		}
-
-		var rsa unix.RawSockaddrInet6
-		rsa.Family = unix.AF_INET6
-		rsa.Addr = ap.Addr().As16()
-		binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ap.Port())
-		sa = unsafe.Pointer(&rsa)
-		addrLen = syscall.SizeofSockaddrInet4
-	} else {
-		var rsa unix.RawSockaddrInet6
-		rsa.Family = unix.AF_INET6
-		rsa.Addr = ap.Addr().As16()
-		binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ap.Port())
-		sa = unsafe.Pointer(&rsa)
-		addrLen = syscall.SizeofSockaddrInet6
-	}
-
-	// Golang stdlib doesn't handle EAGAIN correctly in some situations so we do writes ourselves
-	// See https://github.com/golang/go/issues/73919
-	for {
-		//_, _, err := unix.Syscall6(unix.SYS_SENDTO, u.sysFd, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), 0, sa, addrLen)
-		err := sendto(int(u.sysFd), b, 0, sa, addrLen)
-		if err == nil {
-			// Written, get out before the error handling
-			return nil
-		}
-
-		if errors.Is(err, syscall.EINTR) {
-			// Write was interrupted, retry
-			continue
-		}
-
-		if errors.Is(err, syscall.EAGAIN) {
-			return &net.OpError{Op: "sendto", Err: unix.EWOULDBLOCK}
-		}
-
-		if errors.Is(err, syscall.EBADF) {
-			return net.ErrClosed
-		}
-
-		return &net.OpError{Op: "sendto", Err: err}
-	}
-}
-
-func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
-	a := u.UDPConn.LocalAddr()
-
-	switch v := a.(type) {
-	case *net.UDPAddr:
-		addr, ok := netip.AddrFromSlice(v.IP)
-		if !ok {
-			return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP)
-		}
-		return netip.AddrPortFrom(addr, uint16(v.Port)), nil
-
-	default:
-		return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a)
-	}
-}
-
-func (u *StdConn) ReloadConfig(c *config.C) {
-	// TODO
-}
-
-func NewUDPStatsEmitter(udpConns []Conn) func() {
-	// No UDP stats for non-linux
-	return func() {}
-}
-
-func (u *StdConn) ListenOut(r EncReader, lhf LightHouseHandlerFunc, cache *firewall.ConntrackCacheTicker, q int) {
-	plaintext := make([]byte, MTU)
-	buffer := make([]byte, MTU)
-	h := &header.H{}
-	fwPacket := &firewall.Packet{}
-	nb := make([]byte, 12, 12)
-
-	for {
-		// Just read one packet at a time
-		n, rua, err := u.ReadFromUDPAddrPort(buffer)
 	if err != nil {
-			if errors.Is(err, net.ErrClosed) {
+		return err
-				u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
-				return
-			}
-
-			u.l.WithError(err).Error("unexpected udp socket receive error")
-		}
-
-		r(
-			netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()),
-			plaintext[:0],
-			buffer[:n],
-			h,
-			fwPacket,
-			lhf,
-			nb,
-			q,
-			cache.Get(u.l),
-		)
-	}
-}
-
-func (u *StdConn) Rebind() error {
-	var err error
-	if u.isV4 {
-		err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IP, syscall.IP_BOUND_IF, 0)
-	} else {
-		err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0)
 	}
 
+	return rc.Control(func(fd uintptr) {
+		err := syscall.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_BOUND_IF, 0)
 		if err != nil {
 			u.l.WithError(err).Error("Failed to rebind udp socket")
 		}
-
+	})
-	return nil
 }

udp/udp_generic.go

@@ -1,7 +1,6 @@
-//go:build (!linux || android) && !e2e_testing && !darwin
+//go:build (!linux || android) && !e2e_testing
 // +build !linux android
 // +build !e2e_testing
-// +build !darwin
 
 // udp_generic implements the nebula UDP interface in pure Go stdlib. This
 // means it can be used on platforms like Darwin and Windows.
@@ -10,11 +9,9 @@ package udp
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net"
 	"net/netip"
-	"time"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
@@ -82,23 +79,13 @@ func (u *GenericConn) ListenOut(r EncReader, lhf LightHouseHandlerFunc, cache *f
 	fwPacket := &firewall.Packet{}
 	nb := make([]byte, 12, 12)
 
-	var lastRecvErr time.Time
-
 	for {
 		// Just read one packet at a time
 		n, rua, err := u.ReadFromUDPAddrPort(buffer)
 		if err != nil {
-			if errors.Is(err, net.ErrClosed) {
 			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
 			return
 		}
-			// Dampen unexpected message warns to once per minute
-			if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute {
-				lastRecvErr = time.Now()
-				u.l.WithError(err).Warn("unexpected udp socket receive error")
-			}
-			continue
-		}
 
 		r(
 			netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()),

udp/udp_linux.go

@@ -218,7 +218,9 @@ func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
 	var rsa unix.RawSockaddrInet6
 	rsa.Family = unix.AF_INET6
 	rsa.Addr = ip.Addr().As16()
-	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
+	port := ip.Port()
+	// Little Endian -> Network Endian
+	rsa.Port = (port >> 8) | ((port & 0xff) << 8)
 
 	for {
 		_, _, err := unix.Syscall6(
@@ -243,13 +245,15 @@ func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
 
 func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
 	if !ip.Addr().Is4() {
-		return ErrInvalidIPv6RemoteForSocket
+		return fmt.Errorf("Listener is IPv4, but writing to IPv6 remote")
 	}
 
 	var rsa unix.RawSockaddrInet4
 	rsa.Family = unix.AF_INET
 	rsa.Addr = ip.Addr().As4()
-	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
+	port := ip.Port()
+	// Little Endian -> Network Endian
+	rsa.Port = (port >> 8) | ((port & 0xff) << 8)
 
 	for {
 		_, _, err := unix.Syscall6(

udp/udp_rio_windows.go

@@ -14,7 +14,6 @@ import (
 	"sync"
 	"sync/atomic"
 	"syscall"
-	"time"
 	"unsafe"
 
 	"github.com/sirupsen/logrus"
@@ -70,7 +69,7 @@ func NewRIOListener(l *logrus.Logger, addr netip.Addr, port int) (*RIOConn, erro
 
 	u := &RIOConn{l: l}
 
-	err := u.bind(l, &windows.SockaddrInet6{Addr: addr.As16(), Port: port})
+	err := u.bind(&windows.SockaddrInet6{Addr: addr.As16(), Port: port})
 	if err != nil {
 		return nil, fmt.Errorf("bind: %w", err)
 	}
@@ -86,58 +85,34 @@ func NewRIOListener(l *logrus.Logger, addr netip.Addr, port int) (*RIOConn, erro
 	return u, nil
 }
 
-func (u *RIOConn) bind(l *logrus.Logger, sa windows.Sockaddr) error {
+func (u *RIOConn) bind(sa windows.Sockaddr) error {
 	var err error
 	u.sock, err = winrio.Socket(windows.AF_INET6, windows.SOCK_DGRAM, windows.IPPROTO_UDP)
 	if err != nil {
-		return fmt.Errorf("winrio.Socket error: %w", err)
+		return err
 	}
 
 	// Enable v4 for this socket
 	syscall.SetsockoptInt(syscall.Handle(u.sock), syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 0)
 
-	// Disable reporting of PORT_UNREACHABLE and NET_UNREACHABLE errors from the UDP socket receive call.
-	// These errors are returned on Windows during UDP receives based on the receipt of ICMP packets. Disable
-	// the UDP receive error returns with these ioctl calls.
-	ret := uint32(0)
-	flag := uint32(0)
-	size := uint32(unsafe.Sizeof(flag))
-	err = syscall.WSAIoctl(syscall.Handle(u.sock), syscall.SIO_UDP_CONNRESET, (*byte)(unsafe.Pointer(&flag)), size, nil, 0, &ret, nil, 0)
-	if err != nil {
-		// This is a best-effort to prevent errors from being returned by the udp recv operation.
-		// Quietly log a failure and continue.
-		l.WithError(err).Debug("failed to set UDP_CONNRESET ioctl")
-	}
-
-	ret = 0
-	flag = 0
-	size = uint32(unsafe.Sizeof(flag))
-	SIO_UDP_NETRESET := uint32(syscall.IOC_IN | syscall.IOC_VENDOR | 15)
-	err = syscall.WSAIoctl(syscall.Handle(u.sock), SIO_UDP_NETRESET, (*byte)(unsafe.Pointer(&flag)), size, nil, 0, &ret, nil, 0)
-	if err != nil {
-		// This is a best-effort to prevent errors from being returned by the udp recv operation.
-		// Quietly log a failure and continue.
-		l.WithError(err).Debug("failed to set UDP_NETRESET ioctl")
-	}
-
 	err = u.rx.Open()
 	if err != nil {
-		return fmt.Errorf("error rx.Open(): %w", err)
+		return err
 	}
 
 	err = u.tx.Open()
 	if err != nil {
-		return fmt.Errorf("error tx.Open(): %w", err)
+		return err
 	}
 
 	u.rq, err = winrio.CreateRequestQueue(u.sock, packetsPerRing, 1, packetsPerRing, 1, u.rx.cq, u.tx.cq, 0)
 	if err != nil {
-		return fmt.Errorf("error CreateRequestQueue: %w", err)
+		return err
 	}
 
 	err = windows.Bind(u.sock, sa)
 	if err != nil {
-		return fmt.Errorf("error windows.Bind(): %w", err)
+		return err
 	}
 
 	return nil
@@ -150,24 +125,13 @@ func (u *RIOConn) ListenOut(r EncReader, lhf LightHouseHandlerFunc, cache *firew
 	fwPacket := &firewall.Packet{}
 	nb := make([]byte, 12, 12)
 
-	var lastRecvErr time.Time
-
 	for {
 		// Just read one packet at a time
 		n, rua, err := u.receive(buffer)
-
 		if err != nil {
-			if errors.Is(err, net.ErrClosed) {
 			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
 			return
 		}
-			// Dampen unexpected message warns to once per minute
-			if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute {
-				lastRecvErr = time.Now()
-				u.l.WithError(err).Warn("unexpected udp socket receive error")
-			}
-			continue
-		}
 
 		r(
 			netip.AddrPortFrom(netip.AddrFrom16(rua.Addr).Unmap(), (rua.Port>>8)|((rua.Port&0xff)<<8)),