Group the HostInfo fields the packet paths touch

.github/workflows/smoke.yml

@@ -60,12 +60,4 @@ jobs:
       working-directory: ./.github/workflows/smoke
       run: NAME="smoke-p256" ./smoke.sh
 
-    - name: setup docker image for multiport
-      working-directory: ./.github/workflows/smoke
-      run: NAME="smoke-multiport" MULTIPORT_TX=true MULTIPORT_RX=true MULTIPORT_HANDSHAKE=true ./build.sh
-
-    - name: run smoke
-      working-directory: ./.github/workflows/smoke
-      run: NAME="smoke-multiport" ./smoke.sh
-
     timeout-minutes: 10

.github/workflows/smoke/genconfig.sh

@@ -48,10 +48,6 @@ listen:
 
 tun:
   dev: ${TUN_DEV:-tun0}
-  multiport:
-    tx_enabled: ${MULTIPORT_TX:-false}
-    rx_enabled: ${MULTIPORT_RX:-false}
-    tx_handshake: ${MULTIPORT_HANDSHAKE:-false}
 
 firewall:
   inbound_action: reject

Makefile

@@ -268,10 +268,6 @@ smoke-relay-docker: bin-docker
 	cd .github/workflows/smoke/ && ./build-relay.sh
 	cd .github/workflows/smoke/ && ./smoke-relay.sh
 
-smoke-multiport-docker: bin-docker
-	cd .github/workflows/smoke/ && NAME="smoke-multiport" MULTIPORT_TX=true MULTIPORT_RX=true MULTIPORT_HANDSHAKE=true ./build.sh
-	cd .github/workflows/smoke/ && NAME="smoke-multiport" ./smoke.sh
-
 smoke-docker-ipv6: export SMOKE_OVERLAY_IPV6 = 1
 smoke-docker-ipv6: smoke-docker
 

config/multiport.go

@@ -1,10 +0,0 @@
-package config
-
-type MultiPortConfig struct {
-	Tx               bool
-	Rx               bool
-	TxBasePort       uint16
-	TxPorts          int
-	TxHandshake      bool
-	TxHandshakeDelay int64
-}

connection_manager.go

@@ -105,11 +105,17 @@ func (cm *connectionManager) getInactivityTimeout() time.Duration {
 }
 
 func (cm *connectionManager) In(h *HostInfo) {
-	h.in.Store(true)
+	h.markIn()
 }
 
-func (cm *connectionManager) Out(h *HostInfo) {
-	h.out.Store(true)
+// OutRelay records relayed traffic, leaving the rebind epoch for the direct path to this host to consume
+func (cm *connectionManager) OutRelay(h *HostInfo) {
+	h.markOutOnly()
+}
+
+// Out records outbound traffic and reports whether we rebound since this tunnel last sent
+func (cm *connectionManager) Out(h *HostInfo) bool {
+	return h.markOut(cm.intf.rebindEpoch.Load())
 }
 
 func (cm *connectionManager) RelayUsed(localIndex uint32) {
@@ -128,8 +134,7 @@ func (cm *connectionManager) RelayUsed(localIndex uint32) {
 // getAndResetTrafficCheck returns if there was any inbound or outbound traffic within the last tick and
 // resets the state for this local index
 func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time) (bool, bool) {
-	in := h.in.Swap(false)
-	out := h.out.Swap(false)
+	in, out := h.takeTraffic()
 	if in || out {
 		h.lastUsed = now
 	}
@@ -340,7 +345,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 				"tunnelCheck", m{"state": "alive", "method": "passive"},
 			)
 		}
-		hostinfo.pendingDeletion.Store(false)
+		hostinfo.setPendingDeletion(false)
 
 		if mainHostInfo {
 			decision = tryRehandshake
@@ -363,7 +368,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 		return decision, hostinfo, primary
 	}
 
-	if hostinfo.pendingDeletion.Load() {
+	if hostinfo.isPendingDeletion() {
 		// We have already sent a test packet and nothing was returned, this hostinfo is dead
 		hostinfo.logger(cm.l).Info("Tunnel status",
 			"tunnelCheck", m{"state": "dead", "method": "active"},
@@ -414,7 +419,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 		}
 	}
 
-	hostinfo.pendingDeletion.Store(true)
+	hostinfo.setPendingDeletion(true)
 	cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
 	return decision, hostinfo, nil
 }

connection_manager_test.go

@@ -86,25 +86,25 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	// We saw traffic out to vpnIp
 	nc.Out(hostinfo)
 	nc.In(hostinfo)
-	assert.False(t, hostinfo.pendingDeletion.Load())
+	assert.False(t, hostinfo.isPendingDeletion())
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
-	assert.True(t, hostinfo.out.Load())
-	assert.True(t, hostinfo.in.Load())
+	assert.True(t, hostinfo.sentSinceCheck())
+	assert.True(t, (hostinfo.state.Load()&stateIn != 0))
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
+	assert.False(t, hostinfo.isPendingDeletion())
+	assert.False(t, hostinfo.sentSinceCheck())
+	assert.False(t, (hostinfo.state.Load()&stateIn != 0))
 
 	// Do another traffic check tick, this host should be pending deletion now
 	nc.Out(hostinfo)
-	assert.True(t, hostinfo.out.Load())
+	assert.True(t, hostinfo.sentSinceCheck())
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.True(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
+	assert.True(t, hostinfo.isPendingDeletion())
+	assert.False(t, hostinfo.sentSinceCheck())
+	assert.False(t, (hostinfo.state.Load()&stateIn != 0))
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 
@@ -168,33 +168,33 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	// We saw traffic out to vpnIp
 	nc.Out(hostinfo)
 	nc.In(hostinfo)
-	assert.True(t, hostinfo.in.Load())
-	assert.True(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.pendingDeletion.Load())
+	assert.True(t, (hostinfo.state.Load()&stateIn != 0))
+	assert.True(t, hostinfo.sentSinceCheck())
+	assert.False(t, hostinfo.isPendingDeletion())
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
+	assert.False(t, hostinfo.isPendingDeletion())
+	assert.False(t, hostinfo.sentSinceCheck())
+	assert.False(t, (hostinfo.state.Load()&stateIn != 0))
 
 	// Do another traffic check tick, this host should be pending deletion now
 	nc.Out(hostinfo)
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.True(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
+	assert.True(t, hostinfo.isPendingDeletion())
+	assert.False(t, hostinfo.sentSinceCheck())
+	assert.False(t, (hostinfo.state.Load()&stateIn != 0))
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 
 	// We saw traffic, should no longer be pending deletion
 	nc.In(hostinfo)
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
+	assert.False(t, hostinfo.isPendingDeletion())
+	assert.False(t, hostinfo.sentSinceCheck())
+	assert.False(t, (hostinfo.state.Load()&stateIn != 0))
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 }
@@ -253,31 +253,31 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
 	// Do a traffic check tick, in and out should be cleared but should not be pending deletion
 	nc.Out(hostinfo)
 	nc.In(hostinfo)
-	assert.True(t, hostinfo.out.Load())
-	assert.True(t, hostinfo.in.Load())
+	assert.True(t, hostinfo.sentSinceCheck())
+	assert.True(t, (hostinfo.state.Load()&stateIn != 0))
 
 	now := time.Now()
 	decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
 	assert.Equal(t, tryRehandshake, decision)
 	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
+	assert.False(t, hostinfo.isPendingDeletion())
+	assert.False(t, hostinfo.sentSinceCheck())
+	assert.False(t, (hostinfo.state.Load()&stateIn != 0))
 
 	decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
 	assert.Equal(t, doNothing, decision)
 	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
+	assert.False(t, hostinfo.isPendingDeletion())
+	assert.False(t, hostinfo.sentSinceCheck())
+	assert.False(t, (hostinfo.state.Load()&stateIn != 0))
 
 	// Do another traffic check tick, should still not be pending deletion
 	decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
 	assert.Equal(t, doNothing, decision)
 	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
+	assert.False(t, hostinfo.isPendingDeletion())
+	assert.False(t, hostinfo.sentSinceCheck())
+	assert.False(t, (hostinfo.state.Load()&stateIn != 0))
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 
@@ -285,9 +285,9 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
 	decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
 	assert.Equal(t, closeTunnel, decision)
 	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
+	assert.False(t, hostinfo.isPendingDeletion())
+	assert.False(t, hostinfo.sentSinceCheck())
+	assert.False(t, (hostinfo.state.Load()&stateIn != 0))
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 }

connection_state_test.go

@@ -8,7 +8,6 @@ import (
 	"github.com/flynn/noise"
 	"github.com/slackhq/nebula/cert"
 	ct "github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/handshake"
 	"github.com/slackhq/nebula/header"
 	"github.com/stretchr/testify/assert"
@@ -56,7 +55,6 @@ func runTestHandshake(t *testing.T) (initR, respR *handshake.Result) {
 		cert.Version2, initCreds, verifier,
 		func() (uint32, error) { return 1000, nil },
 		true, header.HandshakeIXPSK0,
-		config.MultiPortConfig{},
 	)
 	require.NoError(t, err)
 
@@ -64,7 +62,6 @@ func runTestHandshake(t *testing.T) (initR, respR *handshake.Result) {
 		cert.Version2, respCreds, verifier,
 		func() (uint32, error) { return 2000, nil },
 		false, header.HandshakeIXPSK0,
-		config.MultiPortConfig{},
 	)
 	require.NoError(t, err)
 

control.go

@@ -212,7 +212,7 @@ func (c *Control) RebindUDPServer() {
 	c.f.lightHouse.SendUpdate()
 
 	// Let the main interface know that we rebound so that underlying tunnels know to trigger punches from their remotes
-	c.f.rebindCount++
+	c.f.rebindEpoch.Add(1)
 }
 
 // ListHostmapHosts returns details about the actual or pending (handshaking) hostmap by vpn ip

control_tester.go

@@ -123,6 +123,16 @@ func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
 	c.f.lightHouse.localAddrsFn = fn
 }
 
+// GetRebindEpochFor returns the rebind epoch a tunnel last sent under, so a test can tell whether a send
+// consumed the epoch edge without having to infer it from lighthouse traffic.
+func (c *Control) GetRebindEpochFor(vpnAddr netip.Addr) (uint32, bool) {
+	h := c.f.hostMap.QueryVpnAddr(vpnAddr)
+	if h == nil {
+		return 0, false
+	}
+	return h.state.Load() >> stateEpochShift, true
+}
+
 func (c *Control) KillPendingTunnel(vpnIp netip.Addr) bool {
 	hostinfo := c.f.handshakeManager.QueryVpnAddr(vpnIp)
 	if hostinfo == nil {

e2e/rebind_test.go

@@ -223,3 +223,57 @@ func TestRebindAdvertisesNewAddressAfterMove(t *testing.T) {
 	lhControl.Stop()
 	myControl.Stop()
 }
+
+// A relayed send records traffic but must not consume the rebind epoch. If it does, the next direct send to the
+// relay host sees the epoch already current and never requeries, so the far side is never told to punch at our
+// new address. This pins the SendVia call site, which the unit tests cannot reach.
+func TestRebindRequeriesAfterRelayedSend(t *testing.T) {
+	t.Parallel()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	// No lighthouse on purpose: it would hand out a direct address for them and nothing would relay.
+	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
+	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
+
+	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
+	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
+	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+
+	r := router.NewR(t, myControl, relayControl, theirControl)
+	defer r.RenderFlow()
+
+	myControl.Start()
+	relayControl.Start()
+	theirControl.Start()
+
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("establish")))
+	r.RouteForAllUntilTxTun(theirControl)
+	r.RouteFor(time.Millisecond * 500)
+
+	hi := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
+	require.NotNil(t, hi, "expected a tunnel to them")
+	require.NotEmpty(t, hi.CurrentRelaysToMe, "them must be reachable only via the relay for this test to mean anything")
+	// sendNoMetrics only reaches SendVia when there is no direct remote, so pin that too. Without this the test
+	// keeps passing while quietly sending direct and never exercising the relay path.
+	require.False(t, hi.CurrentRemote.IsValid(), "them must have no direct remote, otherwise SendVia is never called")
+
+	before, ok := myControl.GetRebindEpochFor(relayVpnIpNet[0].Addr())
+	require.True(t, ok, "expected a tunnel to the relay")
+
+	myControl.RebindUDPServer()
+
+	// Traffic to them goes through SendVia on the relay tunnel. That must record traffic without consuming the
+	// relay tunnel's own epoch edge, which belongs to the direct path.
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("relayed")))
+	r.RouteForAllUntilTxTun(theirControl)
+
+	after, ok := myControl.GetRebindEpochFor(relayVpnIpNet[0].Addr())
+	require.True(t, ok)
+	assert.Equal(t, before, after,
+		"a relayed send consumed the relay tunnel's rebind epoch, so the next direct send will not requery")
+
+	myControl.Stop()
+	relayControl.Stop()
+	theirControl.Stop()
+}

e2e/recovery_timing_test.go

@@ -0,0 +1,136 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package e2e
+
+import (
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/udp"
+)
+
+// TestRecoveryTiming measures how long a tunnel takes to come back after the peer stops accepting our traffic,
+// which is what a laptop waking on a new network looks like from the peer's side: its NAT has no state for where
+// we are now, so everything we send disappears.
+//
+// It is a measurement, not a pass/fail assertion. Recovery is timed to the moment the peer punches back at us,
+// since that is when its NAT opens and the tunnel is usable again.
+//
+//	go test -tags e2e_testing -v -run TestRecoveryTiming ./e2e/
+func TestRecoveryTiming(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		rebind bool
+	}{
+		{"no trigger", false},
+		{"rebind counter", true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			d, lost := measureRecovery(t, tc.rebind)
+			t.Logf("RESULT %-16s recovered in %-9v (%d packets lost)", tc.name, d.Round(time.Millisecond), lost)
+		})
+	}
+}
+
+// measureRecovery returns how long until the peer punched back, and how many of our packets died meanwhile. When
+// rebind is true we call RebindUDPServer once the tunnel goes dark, which is what the darwin network change
+// monitor does and what iOS has always done. When false, nothing tells nebula anything is wrong.
+func measureRecovery(t *testing.T, rebind bool) (time.Duration, int) {
+	t.Helper()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	peerCfg := m{
+		"lighthouse": m{
+			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
+			"interval": 600,
+			"local_allow_list": m{
+				"10.0.0.0/24": true,
+				"::/0":        false,
+			},
+		},
+		"static_host_map": m{
+			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
+		},
+	}
+
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", peerCfg)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", peerCfg)
+
+	r := router.NewR(t, lhControl, myControl, theirControl)
+	defer r.RenderFlow()
+	defer func() {
+		lhControl.Stop()
+		myControl.Stop()
+		theirControl.Stop()
+	}()
+
+	lhControl.Start()
+	myControl.Start()
+	theirControl.Start()
+	r.RouteFor(time.Millisecond * 500)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("establish")))
+	r.RouteFor(time.Second)
+	if myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) == nil {
+		t.Fatal("failed to establish the tunnel we are measuring")
+	}
+	r.RouteFor(time.Millisecond * 500)
+
+	// From here the peer's NAT has no state for us, everything we send it disappears
+	start := time.Now()
+	blackholed := 0
+	var recovered time.Duration
+
+	if rebind {
+		myControl.RebindUDPServer()
+	}
+
+	// Keep the tun busy the way someone retrying a stalled connection would
+	stop := make(chan struct{})
+	defer close(stop)
+	go func() {
+		tick := time.NewTicker(time.Millisecond * 200)
+		defer tick.Stop()
+		for {
+			select {
+			case <-stop:
+				return
+			case <-tick.C:
+				myControl.InjectTunPacket(BuildTunUDPPacket(
+					theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("retry")))
+			}
+		}
+	}()
+
+	r.RouteForAllExitFuncOrTimeout(time.Second*30, func(p *udp.Packet, c *nebula.Control) router.ExitType {
+		if c == theirControl && p.From == myControl.GetUDPAddr() {
+			blackholed++
+			return router.Drop
+		}
+
+		// The peer reaching us directly is the moment its NAT opened, whether that is a punch or a handshake
+		if c == myControl && p.From == theirUdpAddr {
+			recovered = time.Since(start)
+			return router.RouteAndExit
+		}
+
+		return router.KeepRouting
+	})
+
+	if recovered == 0 {
+		t.Fatalf("no recovery within 30s (%d packets blackholed)", blackholed)
+	}
+	return recovered, blackholed
+}

e2e/router/router.go

@@ -153,6 +153,9 @@ const (
 	ExitNow ExitType = 1
 	// RouteAndExit routes this packet and exits immediately afterwards
 	RouteAndExit ExitType = 2
+	// Drop discards this packet without delivering it and keeps routing. Use it to simulate a blackhole, such as
+	// a restrictive NAT refusing traffic from an address it has not seen.
+	Drop ExitType = 3
 )
 
 type ExitFunc func(packet *udp.Packet, receiver *nebula.Control) ExitType
@@ -163,7 +166,9 @@ type ExitFunc func(packet *udp.Packet, receiver *nebula.Control) ExitType
 func NewR(t testing.TB, controls ...*nebula.Control) *R {
 	ctx, cancel := context.WithCancel(context.Background())
 
-	if err := os.MkdirAll("mermaid", 0755); err != nil {
+	// t.Name() contains a slash for subtests, so the flow log can land in a nested directory
+	fn := filepath.Join("mermaid", fmt.Sprintf("%s.md", t.Name()))
+	if err := os.MkdirAll(filepath.Dir(fn), 0755); err != nil {
 		panic(err)
 	}
 
@@ -174,7 +179,7 @@ func NewR(t testing.TB, controls ...*nebula.Control) *R {
 		outNat:       make(map[outNatKey]netip.AddrPort),
 		flow:         []flowEntry{},
 		ignoreFlows:  []ignoreFlow{},
-		fn:           filepath.Join("mermaid", fmt.Sprintf("%s.md", t.Name())),
+		fn:           fn,
 		t:            t,
 		cancelRender: cancel,
 	}
@@ -687,6 +692,10 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
 			p.Release()
 			return
 
+		case Drop:
+			// Record it so the flow log shows the attempt, but never hand it to the receiver
+			r.unlockedInjectFlow(sender, receiver, p, false)
+
 		case KeepRouting:
 			fp := r.unlockedInjectFlow(sender, receiver, p, false)
 			receiver.InjectUDPPacket(p)
@@ -779,6 +788,10 @@ func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc)
 			p.Release()
 			return true
 
+		case Drop:
+			// Record it so the flow log shows the attempt, but never hand it to the receiver
+			r.unlockedInjectFlow(cm[x], receiver, p, false)
+
 		case KeepRouting:
 			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
 			receiver.InjectUDPPacket(p)
@@ -884,6 +897,10 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
 			p.Release()
 			return
 
+		case Drop:
+			// Record it so the flow log shows the attempt, but never hand it to the receiver
+			r.unlockedInjectFlow(cm[x], receiver, p, false)
+
 		case KeepRouting:
 			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
 			receiver.InjectUDPPacket(p)

examples/config.yml

@@ -328,47 +328,6 @@ tun:
   # SO_RCVBUFFORCE is used to avoid having to raise the system wide max
   #use_system_route_table_buffer_size: 0
 
-  # EXPERIMENTAL: This option may change or disappear in the future.
-  # Multiport spreads outgoing UDP packets across multiple UDP send ports,
-  # which allows nebula to work around any issues on the underlay network.
-  # Some example issues this could work around:
-  #  - UDP rate limits on a per flow basis.
-  #  - Partial underlay network failure in which some flows work and some don't
-  # Agreement is done during the handshake to decide if multiport mode will
-  # be used for a given tunnel (one side must have tx_enabled set, the other
-  # side must have rx_enabled set)
-  #
-  # NOTE: you cannot use multiport on a host if you are relying on UDP hole
-  # punching to get through a NAT or firewall.
-  #
-  # NOTE: Linux only (uses raw sockets to send). Also currently only works
-  # with IPv4 underlay network remotes.
-  #
-  # The default values are listed below:
-  #multiport:
-    # This host support sending via multiple UDP ports.
-    #tx_enabled: false
-    #
-    # This host supports receiving packets sent from multiple UDP ports.
-    #rx_enabled: false
-    #
-    # How many UDP ports to use when sending. The lowest source port will be
-    # listen.port and go up to (but not including) listen.port + tx_ports.
-    #tx_ports: 100
-    #
-    # NOTE: All of your hosts must be running a version of Nebula that supports
-    # multiport if you want to enable this feature. Older versions of Nebula
-    # will be confused by these multiport handshakes.
-    #
-    # If handshakes are not getting a response, attempt to transmit handshakes
-    # using random UDP source ports (to get around partial underlay network
-    # failures).
-    #tx_handshake: false
-    #
-    # How many unresponded handshakes we should send before we attempt to
-    # send multiport handshakes.
-    #tx_handshake_delay: 2
-
 # Configure logging level
 logging:
   # trace, debug, info, warn, or error. Default is info and is reloadable.

firewall/packet.go

@@ -3,7 +3,6 @@ package firewall
 import (
 	"encoding/json"
 	"fmt"
-	mathrand "math/rand"
 	"net/netip"
 )
 
@@ -66,30 +65,3 @@ func (fp Packet) MarshalJSON() ([]byte, error) {
 		"Fragment":   fp.Fragment,
 	})
 }
-
-// UDPSendPort calculates the UDP port to send from when using multiport mode.
-// The result will be from [0, numBuckets)
-func (fp Packet) UDPSendPort(numBuckets int) uint16 {
-	if numBuckets <= 1 {
-		return 0
-	}
-
-	// If there is no port (like an ICMP packet), pick a random UDP send port
-	if fp.LocalPort == 0 {
-		return uint16(mathrand.Intn(numBuckets))
-	}
-
-	// A decent enough 32bit hash function
-	// Prospecting for Hash Functions
-	// - https://nullprogram.com/blog/2018/07/31/
-	// - https://github.com/skeeto/hash-prospector
-	//   [16 21f0aaad 15 d35a2d97 15] = 0.10760229515479501
-	x := (uint32(fp.LocalPort) << 16) | uint32(fp.RemotePort)
-	x ^= x >> 16
-	x *= 0x21f0aaad
-	x ^= x >> 15
-	x *= 0xd35a2d97
-	x ^= x >> 15
-
-	return uint16(x) % uint16(numBuckets)
-}

handshake/handshake.proto

@@ -24,14 +24,6 @@ message NebulaHandshakeDetails {
   uint64 Cookie = 4 [deprecated = true];
   uint64 Time = 5;
   uint32 CertVersion = 8;
-
-  MultiPortDetails InitiatorMultiPort = 6;
-  MultiPortDetails ResponderMultiPort = 7;
-}
-
-message MultiPortDetails {
-  bool RxSupported = 1;
-  bool TxSupported = 2;
-  uint32 BasePort = 3;
-  uint32 TotalPorts = 4;
+  // reserved for WIP multiport
+  reserved 6, 7;
 }

handshake/helpers_test.go

@@ -8,7 +8,6 @@ import (
 	"github.com/flynn/noise"
 	"github.com/slackhq/nebula/cert"
 	ct "github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
 	"github.com/stretchr/testify/require"
 )
@@ -72,7 +71,6 @@ func newTestMachine(
 		cs.version, cs.getCredential,
 		verifier, func() (uint32, error) { return localIndex, nil },
 		initiator, header.HandshakeIXPSK0,
-		config.MultiPortConfig{},
 	)
 	require.NoError(t, err)
 	return m

handshake/machine.go

@@ -3,13 +3,11 @@ package handshake
 import (
 	"bytes"
 	"fmt"
-	"math"
 	"slices"
 	"time"
 
 	"github.com/flynn/noise"
 	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
 )
 
@@ -41,10 +39,6 @@ type Result struct {
 	HandshakeTime uint64
 	MessageIndex  uint64 // number of messages exchanged during the handshake
 	Initiator     bool
-
-	MultiportRx       bool
-	MultiportTx       bool
-	MultiportBasePort uint16
 }
 
 // Machine drives a Noise handshake through N messages. It handles Noise
@@ -73,8 +67,6 @@ type Machine struct {
 	remoteCertSet  bool
 	payloadSet     bool
 	failed         bool
-
-	multiport config.MultiPortConfig
 }
 
 // NewMachine creates a handshake state machine. The subtype determines both
@@ -88,7 +80,6 @@ func NewMachine(
 	allocIndex IndexAllocator,
 	initiator bool,
 	subtype header.MessageSubType,
-	multiport config.MultiPortConfig,
 ) (*Machine, error) {
 	info, err := subtypeInfoFor(subtype)
 	if err != nil {
@@ -117,8 +108,6 @@ func NewMachine(
 			Initiator: initiator,
 			Cipher:    cred.cipherSuite,
 		},
-
-		multiport: multiport,
 	}, nil
 }
 
@@ -309,7 +298,7 @@ func (m *Machine) processPayload(msg []byte, flags msgFlags) error {
 	}
 
 	// Assert the payload contains exactly what we expect
-	hasPayloadData := payload.InitiatorIndex != 0 || payload.ResponderIndex != 0 || payload.Time != 0 || payload.InitiatorMultiPort != nil || payload.ResponderMultiPort != nil
+	hasPayloadData := payload.InitiatorIndex != 0 || payload.ResponderIndex != 0 || payload.Time != 0
 	if hasPayloadData != flags.expectsPayload {
 		m.failed = true
 		return ErrUnexpectedContent
@@ -326,22 +315,8 @@ func (m *Machine) processPayload(msg []byte, flags msgFlags) error {
 		var remoteIndex uint32
 		if m.result.Initiator {
 			remoteIndex = payload.ResponderIndex
-			if payload.ResponderMultiPort != nil {
-				m.result.MultiportRx = payload.ResponderMultiPort.RxSupported
-				m.result.MultiportTx = payload.ResponderMultiPort.TxSupported
-				if payload.ResponderMultiPort.BasePort <= math.MaxUint16 {
-					m.result.MultiportBasePort = uint16(payload.ResponderMultiPort.BasePort)
-				}
-			}
 		} else {
 			remoteIndex = payload.InitiatorIndex
-			if payload.InitiatorMultiPort != nil {
-				m.result.MultiportRx = payload.InitiatorMultiPort.RxSupported
-				m.result.MultiportTx = payload.InitiatorMultiPort.TxSupported
-				if payload.InitiatorMultiPort.BasePort <= math.MaxUint16 {
-					m.result.MultiportBasePort = uint16(payload.InitiatorMultiPort.BasePort)
-				}
-			}
 		}
 		// The payload presence check above can be satisfied by Time alone, so a payload
 		// could still carry a zero index here. We need to reject it.
@@ -422,28 +397,11 @@ func (m *Machine) marshalOutgoing(flags msgFlags) ([]byte, error) {
 
 		if m.result.Initiator {
 			p.InitiatorIndex = m.result.LocalIndex
-			if m.multiport.Rx || m.multiport.Tx {
-				p.InitiatorMultiPort = &PayloadMultiPortDetails{
-					RxSupported: m.multiport.Rx,
-					TxSupported: m.multiport.Tx,
-					BasePort:    uint32(m.multiport.TxBasePort),
-					TotalPorts:  uint32(m.multiport.TxPorts),
-				}
-			}
 		} else {
 			p.ResponderIndex = m.result.LocalIndex
 			p.InitiatorIndex = m.result.RemoteIndex
-			if m.multiport.Rx || m.multiport.Tx {
-				p.ResponderMultiPort = &PayloadMultiPortDetails{
-					RxSupported: m.multiport.Rx,
-					TxSupported: m.multiport.Tx,
-					BasePort:    uint32(m.multiport.TxBasePort),
-					TotalPorts:  uint32(m.multiport.TxPorts),
-				}
-			}
 		}
 		p.Time = uint64(time.Now().UnixNano())
-
 	}
 	if flags.expectsCert {
 		cred := m.getCred(m.myVersion)

handshake/machine_test.go

@@ -8,7 +8,6 @@ import (
 	"github.com/flynn/noise"
 	"github.com/slackhq/nebula/cert"
 	ct "github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/noiseutil"
 	"github.com/stretchr/testify/assert"
@@ -445,7 +444,6 @@ func TestMachineThreeMessagePattern(t *testing.T) {
 		initCS.getCredential, v,
 		func() (uint32, error) { return 1000, nil },
 		true, header.HandshakeXXPSK0,
-		config.MultiPortConfig{},
 	)
 	require.NoError(t, err)
 
@@ -454,7 +452,6 @@ func TestMachineThreeMessagePattern(t *testing.T) {
 		respCS.getCredential, v,
 		func() (uint32, error) { return 2000, nil },
 		false, header.HandshakeXXPSK0,
-		config.MultiPortConfig{},
 	)
 	require.NoError(t, err)
 

handshake/payload.go

@@ -20,16 +20,6 @@ type Payload struct {
 	ResponderIndex uint32
 	Time           uint64
 	CertVersion    uint32
-
-	InitiatorMultiPort *PayloadMultiPortDetails
-	ResponderMultiPort *PayloadMultiPortDetails
-}
-
-type PayloadMultiPortDetails struct {
-	RxSupported bool
-	TxSupported bool
-	BasePort    uint32
-	TotalPorts  uint32
 }
 
 // Proto field numbers for NebulaHandshakeDetails
@@ -39,17 +29,6 @@ const (
 	fieldResponderIndex = 3 // uint32
 	fieldTime           = 5 // uint64
 	fieldCertVersion    = 8 // uint32
-
-	fieldInitiatorMultiPort = 6 // MultiPortDetails
-	fieldResponderMultiPort = 7 // MultiPortDetails
-)
-
-// Proto field numbers for MultiPortDetails
-const (
-	fieldMultiportRxSupported = 1 // bool
-	fieldMultiportTxSupported = 2 // bool
-	fieldMultiportBasePort    = 3 // uint32
-	fieldMultiportTotalPorts  = 4 // uint32
 )
 
 // MarshalPayload encodes a handshake payload in protobuf wire format compatible
@@ -78,16 +57,6 @@ func MarshalPayload(out []byte, p Payload) []byte {
 		details = protowire.AppendTag(details, fieldCertVersion, protowire.VarintType)
 		details = protowire.AppendVarint(details, uint64(p.CertVersion))
 	}
-	if p.InitiatorMultiPort != nil {
-		details = protowire.AppendTag(details, fieldInitiatorMultiPort, protowire.BytesType)
-		details = protowire.AppendVarint(details, uint64(p.InitiatorMultiPort.size()))
-		details = p.InitiatorMultiPort.marshal(details)
-	}
-	if p.ResponderMultiPort != nil {
-		details = protowire.AppendTag(details, fieldResponderMultiPort, protowire.BytesType)
-		details = protowire.AppendVarint(details, uint64(p.ResponderMultiPort.size()))
-		details = p.ResponderMultiPort.marshal(details)
-	}
 
 	out = protowire.AppendTag(out, 1, protowire.BytesType)
 	out = protowire.AppendBytes(out, details)
@@ -95,23 +64,6 @@ func MarshalPayload(out []byte, p Payload) []byte {
 	return out
 }
 
-func (p PayloadMultiPortDetails) marshal(details []byte) []byte {
-	details = protowire.AppendTag(details, fieldMultiportRxSupported, protowire.VarintType)
-	details = protowire.AppendVarint(details, protowire.EncodeBool(p.RxSupported))
-	details = protowire.AppendTag(details, fieldMultiportTxSupported, protowire.VarintType)
-	details = protowire.AppendVarint(details, protowire.EncodeBool(p.TxSupported))
-	details = protowire.AppendTag(details, fieldMultiportBasePort, protowire.VarintType)
-	details = protowire.AppendVarint(details, uint64(p.BasePort))
-	details = protowire.AppendTag(details, fieldMultiportTotalPorts, protowire.VarintType)
-	details = protowire.AppendVarint(details, uint64(p.TotalPorts))
-
-	return details
-}
-
-func (p PayloadMultiPortDetails) size() int {
-	return 4 + 2 + protowire.SizeVarint(uint64(p.BasePort)) + protowire.SizeVarint(uint64(p.TotalPorts))
-}
-
 // UnmarshalPayload decodes a protobuf-encoded NebulaHandshake message.
 func UnmarshalPayload(b []byte) (Payload, error) {
 	var p Payload
@@ -209,97 +161,6 @@ func unmarshalPayloadDetails(p *Payload, b []byte) error {
 			}
 			p.CertVersion = uint32(v)
 			b = b[n:]
-		case fieldInitiatorMultiPort:
-			if typ != protowire.BytesType {
-				return errInvalidHandshakeDetails
-			}
-			d, n := protowire.ConsumeBytes(b)
-			if n < 0 {
-				return errInvalidHandshakeMessage
-			}
-			b = b[n:]
-			p.InitiatorMultiPort = new(PayloadMultiPortDetails)
-			if err := unmarshalPayloadMultiPortDetails(p.InitiatorMultiPort, d); err != nil {
-				return err
-			}
-		case fieldResponderMultiPort:
-			if typ != protowire.BytesType {
-				return errInvalidHandshakeDetails
-			}
-			d, n := protowire.ConsumeBytes(b)
-			if n < 0 {
-				return errInvalidHandshakeMessage
-			}
-			b = b[n:]
-			p.ResponderMultiPort = new(PayloadMultiPortDetails)
-			if err := unmarshalPayloadMultiPortDetails(p.ResponderMultiPort, d); err != nil {
-				return err
-			}
-		default:
-			n := protowire.ConsumeFieldValue(num, typ, b)
-			if n < 0 {
-				return errInvalidHandshakeDetails
-			}
-			b = b[n:]
-		}
-	}
-	return nil
-}
-
-func unmarshalPayloadMultiPortDetails(p *PayloadMultiPortDetails, b []byte) error {
-	for len(b) > 0 {
-		num, typ, n := protowire.ConsumeTag(b)
-		if n < 0 {
-			return errInvalidHandshakeDetails
-		}
-		b = b[n:]
-
-		// For known field numbers, reject any non-matching wire type as a
-		// hard error rather than silently skipping. The caller will catch
-		// missing-field cases downstream, but a wire-type mismatch on a tag
-		// we know is a peer protocol violation worth flagging here.
-		// Repeated occurrences of a singular field follow proto3 last-wins.
-		switch num {
-		case fieldMultiportRxSupported:
-			if typ != protowire.VarintType {
-				return errInvalidHandshakeDetails
-			}
-			v, n := protowire.ConsumeVarint(b)
-			if n < 0 || v > math.MaxUint32 {
-				return errInvalidHandshakeDetails
-			}
-			p.RxSupported = protowire.DecodeBool(v)
-			b = b[n:]
-		case fieldMultiportTxSupported:
-			if typ != protowire.VarintType {
-				return errInvalidHandshakeDetails
-			}
-			v, n := protowire.ConsumeVarint(b)
-			if n < 0 || v > math.MaxUint32 {
-				return errInvalidHandshakeDetails
-			}
-			p.TxSupported = protowire.DecodeBool(v)
-			b = b[n:]
-		case fieldMultiportBasePort:
-			if typ != protowire.VarintType {
-				return errInvalidHandshakeDetails
-			}
-			v, n := protowire.ConsumeVarint(b)
-			if n < 0 || v > math.MaxUint32 {
-				return errInvalidHandshakeDetails
-			}
-			p.BasePort = uint32(v)
-			b = b[n:]
-		case fieldMultiportTotalPorts:
-			if typ != protowire.VarintType {
-				return errInvalidHandshakeDetails
-			}
-			v, n := protowire.ConsumeVarint(b)
-			if n < 0 || v > math.MaxUint32 {
-				return errInvalidHandshakeDetails
-			}
-			p.TotalPorts = uint32(v)
-			b = b[n:]
 		default:
 			n := protowire.ConsumeFieldValue(num, typ, b)
 			if n < 0 {

handshake/payload_test.go

@@ -117,24 +117,24 @@ func TestPayloadUnknownFields(t *testing.T) {
 		assert.Equal(t, uint32(88), got.ResponderIndex)
 	})
 
-	// t.Run("reserved fields 6 and 7 are skipped", func(t *testing.T) {
-	// 	// Fields 6 and 7 are reserved in the proto definition
-	// 	var details []byte
-	// 	details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType)
-	// 	details = protowire.AppendVarint(details, 100)
-	// 	details = protowire.AppendTag(details, 6, protowire.VarintType)
-	// 	details = protowire.AppendVarint(details, 1)
-	// 	details = protowire.AppendTag(details, 7, protowire.VarintType)
-	// 	details = protowire.AppendVarint(details, 2)
+	t.Run("reserved fields 6 and 7 are skipped", func(t *testing.T) {
+		// Fields 6 and 7 are reserved in the proto definition
+		var details []byte
+		details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType)
+		details = protowire.AppendVarint(details, 100)
+		details = protowire.AppendTag(details, 6, protowire.VarintType)
+		details = protowire.AppendVarint(details, 1)
+		details = protowire.AppendTag(details, 7, protowire.VarintType)
+		details = protowire.AppendVarint(details, 2)
 
-	// 	var data []byte
-	// 	data = protowire.AppendTag(data, 1, protowire.BytesType)
-	// 	data = protowire.AppendBytes(data, details)
+		var data []byte
+		data = protowire.AppendTag(data, 1, protowire.BytesType)
+		data = protowire.AppendBytes(data, details)
 
-	// 	got, err := UnmarshalPayload(data)
-	// 	require.NoError(t, err)
-	// 	assert.Equal(t, uint32(100), got.InitiatorIndex)
-	// })
+		got, err := UnmarshalPayload(data)
+		require.NoError(t, err)
+		assert.Equal(t, uint32(100), got.InitiatorIndex)
+	})
 }
 
 func TestPayloadBytesConsumed(t *testing.T) {

handshake_manager.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/rcrowley/go-metrics"
 	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/handshake"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/udp"
@@ -72,9 +71,6 @@ type HandshakeManager struct {
 	f                      *Interface
 	l                      *slog.Logger
 
-	multiPort config.MultiPortConfig
-	udpRaw    *udp.RawConn
-
 	// can be used to trigger outbound handshake for the given vpnIp
 	trigger chan netip.Addr
 }
@@ -295,7 +291,6 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 
 	// Send the handshake to all known ips, stage 2 takes care of assigning the hostinfo.remote based on the first to reply
 	var sentTo []netip.AddrPort
-	var sentMultiport bool
 	hostinfo.remotes.ForEach(hm.mainHostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) {
 		hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1)
 		err := hm.outside.WriteTo(stage0, addr)
@@ -316,29 +311,6 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 		} else {
 			sentTo = append(sentTo, addr)
 		}
-
-		// Attempt a multiport handshake if we are past the TxHandshakeDelay attempts
-		if hm.multiPort.TxHandshake && hm.udpRaw != nil && hh.counter >= hm.multiPort.TxHandshakeDelay {
-			sentMultiport = true
-			// We need to re-allocate with 8 bytes at the start of SOCK_RAW
-			raw := hostinfo.HandshakePacket[0x80]
-			if raw == nil {
-				raw = make([]byte, len(hostinfo.HandshakePacket[0])+udp.RawOverhead)
-				copy(raw[udp.RawOverhead:], hostinfo.HandshakePacket[0])
-				hostinfo.HandshakePacket[0x80] = raw
-			}
-
-			hm.messageMetrics.Tx(header.Handshake, header.MessageSubType(hostinfo.HandshakePacket[0][1]), 1)
-			err = hm.udpRaw.WriteTo(raw, udp.RandomSendPort.UDPSendPort(hm.multiPort.TxPorts), addr)
-			if err != nil {
-				hostinfo.logger(hm.l).Error("Failed to send handshake message",
-					"error", err,
-					"udpAddr", addr,
-					"initiatorIndex", hostinfo.localIndexId,
-					"handshake", hsFields,
-				)
-			}
-		}
 	})
 
 	// Don't be too noisy or confusing if we fail to send a handshake - if we don't get through we'll eventually log a timeout,
@@ -348,7 +320,6 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 			"udpAddrs", sentTo,
 			"initiatorIndex", hostinfo.localIndexId,
 			"handshake", hsFields,
-			"multiportHandshake", sentMultiport,
 		)
 	} else if hm.l.Enabled(context.Background(), slog.LevelDebug) {
 		hostinfo.logger(hm.l).Debug("Handshake message sent",
@@ -701,7 +672,6 @@ func (hm *HandshakeManager) buildStage0Packet(hh *HandshakeHostInfo) bool {
 		v, cs.GetCredential,
 		hm.certVerifier(), func() (uint32, error) { return hm.allocateIndex(hh) },
 		true, header.HandshakeIXPSK0,
-		hm.multiPort,
 	)
 	if err != nil {
 		hm.f.l.Error("Failed to create handshake machine",
@@ -743,7 +713,6 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
 		v, cs.GetCredential,
 		hm.certVerifier(), func() (uint32, error) { return generateIndex(f.l) },
 		false, header.HandshakeIXPSK0,
-		hm.multiPort,
 	)
 	if err != nil {
 		f.l.Error("Failed to create handshake machine", "from", via, "error", err)
@@ -768,12 +737,6 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
 		return
 	}
 
-	if !via.IsRelayed && result.MultiportTx && result.MultiportBasePort != via.UdpAddr.Port() {
-		// The other side sent us a handshake from a different port, make sure
-		// we send responses back to the BasePort
-		via.UdpAddr = netip.AddrPortFrom(via.UdpAddr.Addr(), result.MultiportBasePort)
-	}
-
 	remoteCert := result.RemoteCert
 	if remoteCert == nil {
 		f.l.Error("Handshake did not produce a peer certificate", "from", via)
@@ -798,8 +761,6 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
 			relayForByAddr: map[netip.Addr]*Relay{},
 			relayForByIdx:  map[uint32]*Relay{},
 		},
-		multiportTx: hm.multiPort.Tx && result.MultiportRx,
-		multiportRx: hm.multiPort.Rx && result.MultiportTx,
 	}
 
 	msg := "Handshake message received"
@@ -816,8 +777,6 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
 		"initiatorIndex", result.RemoteIndex,
 		"responderIndex", result.LocalIndex,
 		"handshake", m{"stage": uint64(machine.MessageIndex()), "style": header.SubTypeName(header.Handshake, machine.Subtype())},
-		"multiportTx", hostinfo.multiportTx,
-		"multiportRx", hostinfo.multiportRx,
 	)
 
 	// packet aliases the listener's incoming buffer, so this copy must stay.
@@ -908,14 +867,6 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
 		return
 	}
 
-	if !via.IsRelayed && result.MultiportTx && result.MultiportBasePort != via.UdpAddr.Port() {
-		// The other side sent us a handshake from a different port, make sure
-		// we send responses back to the BasePort
-		via.UdpAddr = netip.AddrPortFrom(via.UdpAddr.Addr(), result.MultiportBasePort)
-	}
-	hostinfo.multiportTx = hm.multiPort.Tx && result.MultiportRx
-	hostinfo.multiportRx = hm.multiPort.Rx && result.MultiportTx
-
 	// Handshake complete; build the ConnectionState now that we have keys and a verified peer cert.
 	hostinfo.ConnectionState = newConnectionStateFromResult(result)
 
@@ -1010,8 +961,6 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
 		"handshake", m{"stage": uint64(machine.MessageIndex()), "style": header.SubTypeName(header.Handshake, machine.Subtype())},
 		"durationNs", duration,
 		"sentCachedPackets", len(hh.packetStore),
-		"multiportTx", hostinfo.multiportTx,
-		"multiportRx", hostinfo.multiportRx,
 	)
 
 	hostinfo.vpnAddrs = vpnAddrs
@@ -1153,10 +1102,6 @@ func (hm *HandshakeManager) handleCheckAndCompleteError(err error, existing, hos
 
 	switch err {
 	case ErrAlreadySeen:
-		if hostinfo.multiportRx {
-			// The other host is sending to us with multiport, so only grab the IP
-			via.UdpAddr = netip.AddrPortFrom(via.UdpAddr.Addr(), hostinfo.GetRemote().Port())
-		}
 		if existing.SetRemoteIfPreferred(f.hostMap, via) {
 			f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
 		}

hostmap.go

@@ -238,28 +238,30 @@ const (
 )
 
 type HostInfo struct {
+	// The first cache line is everything the packet paths touch.
+
 	remote          atomic.Pointer[netip.AddrPort]
-	remotes         *RemoteList
-	promoteCounter  atomic.Uint32
 	ConnectionState *ConnectionState
-	remoteIndexId   uint32
-	localIndexId    uint32
+
+	// Traffic bits, pendingDeletion, and the rebind epoch we last sent under
+	state atomic.Uint32
+
+	promoteCounter atomic.Uint32
+	remoteIndexId  uint32
+	localIndexId   uint32
+	remotes        *RemoteList
 
 	// vpnAddrs is a list of vpn addresses assigned to this host that are within our own vpn networks
 	// The host may have other vpn addresses that are outside our
 	// vpn networks but were removed because they are not usable
 	vpnAddrs []netip.Addr
 
+	// Everything below is off the packet path: handshakes, relays, roaming and the connection manager.
+
 	// networks is a combination of specific vpn addresses (not prefixes!) and full unsafe networks assigned to this host.
 	networks   *bart.Table[NetworkType]
 	relayState RelayState
 
-	// If true, we should send to this remote using multiport
-	multiportTx bool
-
-	// If true, we will receive from this remote using multiport
-	multiportRx bool
-
 	// HandshakePacket records the packets used to create this hostinfo
 	// We need these to avoid replayed handshake packets creating new hostinfos which causes churn
 	HandshakePacket map[uint8][]byte
@@ -268,11 +270,6 @@ type HostInfo struct {
 	// This is used to limit lighthouse re-queries in chatty clients
 	nextLHQuery atomic.Int64
 
-	// lastRebindCount is the other side of Interface.rebindCount, if these values don't match then we need to ask LH
-	// for a punch from the remote end of this tunnel. The goal being to prime their conntrack for our traffic just like
-	// with a handshake
-	lastRebindCount int8
-
 	// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
 	// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
 	// This is used to avoid an attack where a handshake packet is replayed after some time
@@ -281,9 +278,6 @@ type HostInfo struct {
 	lastRoam       time.Time
 	lastRoamRemote netip.AddrPort
 
-	//TODO: in, out, and others might benefit from being an atomic.Int32. We could collapse connectionManager pendingDeletion, relayUsed, and in/out into this 1 thing
-	in, out, pendingDeletion atomic.Bool
-
 	// lastUsed tracks the last time ConnectionManager checked the tunnel and it was in use.
 	// This value will be behind against actual tunnel utilization in the hot path.
 	// This should only be used by the ConnectionManagers ticker routine.
@@ -664,7 +658,7 @@ func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
 	hm.Indexes[hostinfo.localIndexId] = hostinfo
 	hm.RemoteIndexes[hostinfo.remoteIndexId] = hostinfo
 
-	hostinfo.out.Store(true)
+	hostinfo.markOut(f.rebindEpoch.Load())
 	if f.connectionManager != nil { // f.connectionManager is only nil in some unit tests
 		f.connectionManager.trafficTimer.Add(hostinfo.localIndexId, f.connectionManager.checkInterval)
 	}
@@ -765,6 +759,68 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
 	}
 }
 
+// Bits within HostInfo.state, everything above stateEpochShift is the epoch
+const (
+	stateIn uint32 = 1 << iota
+	stateOut
+	statePendingDeletion
+
+	stateFlags      = stateIn | stateOut | statePendingDeletion
+	stateEpochShift = 3
+)
+
+// markIn records inbound traffic
+func (i *HostInfo) markIn() {
+	if i.state.Load()&stateIn == 0 {
+		i.state.Or(stateIn)
+	}
+}
+
+// markOut records a send and reports whether the epoch moved, meaning we want a punch from the far side
+func (i *HostInfo) markOut(epoch uint32) bool {
+	e := epoch << stateEpochShift
+	for {
+		old := i.state.Load()
+		if old&stateOut != 0 && old&^stateFlags == e {
+			return false
+		}
+
+		if i.state.CompareAndSwap(old, old&stateFlags|stateOut|e) {
+			return old&^stateFlags != e
+		}
+	}
+}
+
+// markOutOnly records a send without consuming the rebind epoch, for paths that cannot act on a requery
+func (i *HostInfo) markOutOnly() {
+	if i.state.Load()&stateOut == 0 {
+		i.state.Or(stateOut)
+	}
+}
+
+// sentSinceCheck reports whether anything has been sent since the connection manager last looked
+func (i *HostInfo) sentSinceCheck() bool {
+	return i.state.Load()&stateOut != 0
+}
+
+// takeTraffic clears both traffic bits, leaving the epoch alone, and reports what they were
+func (i *HostInfo) takeTraffic() (in bool, out bool) {
+	old := i.state.And(^(stateIn | stateOut))
+	return old&stateIn != 0, old&stateOut != 0
+}
+
+func (i *HostInfo) setPendingDeletion(v bool) {
+	if v {
+		i.state.Or(statePendingDeletion)
+	} else {
+		i.state.And(^statePendingDeletion)
+	}
+}
+
+func (i *HostInfo) isPendingDeletion() bool {
+	return i.state.Load()&statePendingDeletion != 0
+}
+
 func (i *HostInfo) GetCert() *cert.CachedCertificate {
 	if i.ConnectionState != nil {
 		return i.ConnectionState.peerCert

hostmap_test.go

@@ -401,3 +401,43 @@ func TestHostMap_RelayState(t *testing.T) {
 	assert.Equal(t, []netip.Addr{}, h1.relayState.relays)
 
 }
+
+func TestHostInfo_markOut(t *testing.T) {
+	h := &HostInfo{}
+	h.markOut(5) // stamped when the tunnel was added
+
+	// A tunnel already on the current epoch has nothing to report, which is what keeps a fresh tunnel from
+	// requerying on its first packet
+	assert.False(t, h.markOut(5), "an unchanged epoch should not report a move")
+	assert.True(t, h.sentSinceCheck(), "the send is still recorded as traffic")
+
+	// A rebind is observed exactly once, so we requery once per rebind
+	assert.True(t, h.markOut(6), "a bumped epoch should report a move")
+	assert.False(t, h.markOut(6), "the epoch move should only be reported once")
+
+	// Traffic and pendingDeletion live in the same word and must survive an epoch change
+	h.setPendingDeletion(true)
+	h.markIn()
+	assert.True(t, h.markOut(7))
+	assert.True(t, h.isPendingDeletion(), "pendingDeletion must survive an epoch change")
+	in, out := h.takeTraffic()
+	assert.True(t, in, "inbound traffic must survive an epoch change")
+	assert.True(t, out)
+
+	// Clearing the traffic bits leaves the epoch alone, otherwise an idle tunnel would requery forever
+	assert.False(t, h.markOut(7), "takeTraffic must not disturb the epoch")
+}
+
+// A relayed send records traffic but must leave the rebind epoch for the direct path to consume, otherwise
+// relaying to a host swallows the requery that gets the far side punching at our new address.
+func TestHostInfo_markOutOnly(t *testing.T) {
+	h := &HostInfo{}
+	h.markOut(5)
+
+	h.markOutOnly()
+	assert.True(t, h.sentSinceCheck(), "a relayed send is still outbound traffic")
+	assert.False(t, h.markOut(5), "a relayed send must not disturb the epoch")
+
+	assert.True(t, h.markOut(6), "a relayed send must not consume the epoch edge")
+	assert.False(t, h.markOut(6))
+}

inside.go

@@ -10,7 +10,6 @@ import (
 	"github.com/slackhq/nebula/iputil"
 	"github.com/slackhq/nebula/noiseutil"
 	"github.com/slackhq/nebula/routing"
-	"github.com/slackhq/nebula/udp"
 )
 
 func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
@@ -74,7 +73,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 
 	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
-		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q, fwPacket)
+		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
 
 	} else {
 		f.rejectInside(packet, out, q)
@@ -123,7 +122,7 @@ func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *
 		return
 	}
 
-	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q, nil)
+	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q)
 }
 
 // Handshake will attempt to initiate a tunnel with the provided vpn address. This is a no-op if the tunnel is already established or being established
@@ -236,7 +235,7 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
 		return
 	}
 
-	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, 0, nil)
+	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, 0)
 }
 
 // SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr.
@@ -268,12 +267,12 @@ func (f *Interface) SendMessageToHostInfo(t header.MessageType, st header.Messag
 
 func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p, nb, out []byte) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, nb, out, 0, nil)
+	f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, nb, out, 0)
 }
 
 func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0, nil)
+	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0)
 }
 
 // SendVia sends a payload through a Relay tunnel. No authentication or encryption is done
@@ -298,7 +297,7 @@ func (f *Interface) SendVia(via *HostInfo,
 	c := via.ConnectionState.messageCounter.Add(1)
 
 	out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
-	f.connectionManager.Out(via)
+	f.connectionManager.OutRelay(via)
 
 	// Authenticate the header and payload, but do not encrypt for this message type.
 	// The payload consists of the inner, unencrypted Nebula header, as well as the end-to-end encrypted payload.
@@ -341,27 +340,10 @@ func (f *Interface) SendVia(via *HostInfo,
 	f.connectionManager.RelayUsed(relay.LocalIndex)
 }
 
-func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, q int, udpPortGetter udp.SendPortGetter) {
+func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, q int) {
 	if ci.eKey == nil {
 		return
 	}
-
-	multiport := f.multiPort.Tx && hostinfo.multiportTx
-	rawOut := out
-	if multiport {
-		if len(out) < udp.RawOverhead {
-			// NOTE: This is because some spots in the code send us `out[:0]`, so
-			// we need to expand the slice back out to get our 8 bytes back.
-			out = out[:udp.RawOverhead]
-		}
-		// Preserve bytes needed for the raw socket
-		out = out[udp.RawOverhead:]
-
-		if udpPortGetter == nil {
-			udpPortGetter = udp.RandomSendPort
-		}
-	}
-
 	useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid()
 	fullOut := out
 
@@ -383,17 +365,11 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 
 	//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
 	out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
-	f.connectionManager.Out(hostinfo)
-
-	// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against
-	// all our addrs and enable a faster roaming.
-	if t != header.CloseTunnel && hostinfo.lastRebindCount != f.rebindCount {
-		//NOTE: there is an update hole if a tunnel isn't used and exactly 256 rebinds occur before the tunnel is
-		// finally used again. This tunnel would eventually be torn down and recreated if this action didn't help.
+	// We rebound since this tunnel last sent, ask the lighthouse to get the far side punching at us again
+	if f.connectionManager.Out(hostinfo) && t != header.CloseTunnel {
 		f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
-		hostinfo.lastRebindCount = f.rebindCount
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			f.l.Debug("Lighthouse update triggered for punch due to rebind counter",
+			f.l.Debug("Lighthouse update triggered for punch due to rebind epoch",
 				"vpnAddrs", hostinfo.vpnAddrs,
 			)
 		}
@@ -414,13 +390,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 	}
 
 	if remote.IsValid() {
-		if multiport {
-			rawOut = rawOut[:len(out)+udp.RawOverhead]
-			port := udpPortGetter.UDPSendPort(f.multiPort.TxPorts)
-			err = f.udpRaw.WriteTo(rawOut, port, remote)
-		} else {
-			err = f.writers[q].WriteTo(out, remote)
-		}
+		err = f.writers[q].WriteTo(out, remote)
 		if err != nil {
 			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
 				"error", err,
@@ -428,13 +398,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 			)
 		}
 	} else if hr := hostinfo.GetRemote(); hr.IsValid() {
-		if multiport {
-			rawOut = rawOut[:len(out)+udp.RawOverhead]
-			port := udpPortGetter.UDPSendPort(f.multiPort.TxPorts)
-			err = f.udpRaw.WriteTo(rawOut, port, hr)
-		} else {
-			err = f.writers[q].WriteTo(out, hr)
-		}
+		err = f.writers[q].WriteTo(out, hr)
 		if err != nil {
 			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
 				"error", err,

interface.go

@@ -82,8 +82,8 @@ type Interface struct {
 	sendRecvErrorConfig   recvErrorConfig
 	acceptRecvErrorConfig recvErrorConfig
 
-	// rebindCount is used to decide if an active tunnel should trigger a punch notification through a lighthouse
-	rebindCount int8
+	// Bumped on every udp rebind, tunnels compare it to decide they need a punch from the far side
+	rebindEpoch atomic.Uint32
 	version     string
 
 	conntrackCacheTimeout time.Duration
@@ -99,9 +99,6 @@ type Interface struct {
 	// triggerShutdown is a function that will be run exactly once, when onFatal swaps something non-nil into fatalErr
 	triggerShutdown func()
 
-	udpRaw    *udp.RawConn
-	multiPort config.MultiPortConfig
-
 	metricHandshakes    metrics.Histogram
 	messageMetrics      *MessageMetrics
 	cachedPacketMetrics *cachedPacketMetrics
@@ -109,15 +106,6 @@ type Interface struct {
 	l *slog.Logger
 }
 
-type MultiPortConfig struct {
-	Tx               bool
-	Rx               bool
-	TxBasePort       uint16
-	TxPorts          int
-	TxHandshake      bool
-	TxHandshakeDelay int64
-}
-
 type EncWriter interface {
 	SendVia(via *HostInfo,
 		relay *Relay,
@@ -261,8 +249,6 @@ func (f *Interface) activate() error {
 
 	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
 
-	metrics.GetOrRegisterGauge("multiport.tx_ports", nil).Update(int64(f.multiPort.TxPorts))
-
 	// Prepare n tun queues
 	var reader io.ReadWriteCloser = f.inside
 	for i := 0; i < f.routines; i++ {
@@ -519,8 +505,6 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
 
 	udpStats := udp.NewUDPStatsEmitter(f.writers)
 
-	var rawStats func()
-
 	certExpirationGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil)
 	certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil)
 	certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil)
@@ -535,13 +519,6 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
 		certExpirationGauge.Update(int64(defaultCrt.NotAfter().Sub(time.Now()) / time.Second))
 		certInitiatingVersion.Update(int64(defaultCrt.Version()))
 
-		if f.udpRaw != nil {
-			if rawStats == nil {
-				rawStats = udp.NewRawStatsEmitter(f.udpRaw)
-			}
-			rawStats()
-		}
-
 		// Report the max certificate version we are capable of using
 		if certState.v2Cert != nil {
 			certMaxVersion.Update(int64(certState.v2Cert.Version()))

main.go

@@ -244,39 +244,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		ifce.writers = udpConns
 		lightHouse.ifce = ifce
 
-		loadMultiPortConfig := func(c *config.C) {
-			ifce.multiPort.Rx = c.GetBool("tun.multiport.rx_enabled", false)
-
-			tx := c.GetBool("tun.multiport.tx_enabled", false)
-
-			if tx && ifce.udpRaw == nil {
-				ifce.udpRaw, err = udp.NewRawConn(l, c.GetString("listen.host", "0.0.0.0"), port, uint16(port))
-				if err != nil {
-					l.Error("Failed to get raw socket for tun.multiport.tx_enabled", "error", err)
-					ifce.udpRaw = nil
-					tx = false
-				}
-			}
-
-			if tx {
-				ifce.multiPort.TxBasePort = uint16(port)
-				ifce.multiPort.TxPorts = c.GetInt("tun.multiport.tx_ports", 100)
-				ifce.multiPort.TxHandshake = c.GetBool("tun.multiport.tx_handshake", false)
-				ifce.multiPort.TxHandshakeDelay = int64(c.GetInt("tun.multiport.tx_handshake_delay", 2))
-				ifce.udpRaw.ReloadConfig(c)
-			}
-			ifce.multiPort.Tx = tx
-
-			// TODO: if we upstream this, make this cleaner
-			handshakeManager.udpRaw = ifce.udpRaw
-			handshakeManager.multiPort = ifce.multiPort
-
-			l.Info("Multiport configured", "multiPort", ifce.multiPort)
-		}
-
-		loadMultiPortConfig(c)
-		c.RegisterReloadCallback(loadMultiPortConfig)
-
 		ifce.RegisterConfigChangeCallbacks(c)
 		ifce.reloadDisconnectInvalid(c)
 		ifce.reloadSendRecvError(c)

outside.go

@@ -264,15 +264,6 @@ func (f *Interface) sendCloseTunnel(h *HostInfo) {
 func (f *Interface) handleHostRoaming(hostinfo *HostInfo, via ViaSender) {
 	curRemote := hostinfo.GetRemote()
 	if !via.IsRelayed && curRemote != via.UdpAddr {
-		if hostinfo.multiportRx {
-			// If the remote is sending with multiport, we aren't roaming unless
-			// the IP has changed
-			if curRemote.Addr().Compare(via.UdpAddr.Addr()) == 0 {
-				return
-			}
-			// Keep the port from the original hostinfo, because the remote is transmitting from multiport ports
-			via.UdpAddr = netip.AddrPortFrom(via.UdpAddr.Addr(), curRemote.Port())
-		}
 		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
 			if f.l.Enabled(context.Background(), slog.LevelDebug) {
 				hostinfo.logger(f.l).Debug("lighthouse.remote_allow_list denied roaming", "newAddr", via.UdpAddr)

udp/udp_raw.go

@@ -1,16 +0,0 @@
-package udp
-
-import mathrand "math/rand"
-
-type SendPortGetter interface {
-	// UDPSendPort returns the port to use
-	UDPSendPort(maxPort int) uint16
-}
-
-type randomSendPort struct{}
-
-func (randomSendPort) UDPSendPort(maxPort int) uint16 {
-	return uint16(mathrand.Intn(maxPort))
-}
-
-var RandomSendPort = randomSendPort{}

udp/udp_raw_linux.go

@@ -1,191 +0,0 @@
-//go:build !android && !e2e_testing
-// +build !android,!e2e_testing
-
-package udp
-
-import (
-	"encoding/binary"
-	"fmt"
-	"log/slog"
-	"net"
-	"net/netip"
-	"syscall"
-	"unsafe"
-
-	"github.com/rcrowley/go-metrics"
-	"github.com/slackhq/nebula/config"
-	"golang.org/x/net/ipv4"
-	"golang.org/x/sys/unix"
-)
-
-// RawOverhead is the number of bytes that need to be reserved at the start of
-// the raw bytes passed to (*RawConn).WriteTo. This is used by WriteTo to prefix
-// the IP and UDP headers.
-const RawOverhead = 28
-
-type RawConn struct {
-	sysFd    int
-	basePort uint16
-	l        *slog.Logger
-}
-
-func NewRawConn(l *slog.Logger, ip string, port int, basePort uint16) (*RawConn, error) {
-	syscall.ForkLock.RLock()
-	// With IPPROTO_UDP, the linux kernel tries to deliver every UDP packet
-	// received in the system to our socket. This constantly overflows our
-	// buffer and marks our socket as having dropped packets. This makes the
-	// stats on the socket useless.
-	//
-	// In contrast, IPPROTO_RAW is not delivered any packets and thus our read
-	// buffer will not fill up and mark as having dropped packets. The only
-	// difference is that we have to assemble the IP header as well, but this
-	// is fairly easy since Linux does the checksum for us.
-	//
-	// TODO: How to get this working with Inet6 correctly? I was having issues
-	// with the source address when testing before, probably need to `bind(2)`?
-	fd, err := unix.Socket(unix.AF_INET, unix.SOCK_RAW, unix.IPPROTO_RAW)
-	if err == nil {
-		unix.CloseOnExec(fd)
-	}
-	syscall.ForkLock.RUnlock()
-	if err != nil {
-		return nil, err
-	}
-
-	// We only want to send, not recv. This will hopefully help the kernel avoid
-	// wasting time on us
-	if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, 0); err != nil {
-		return nil, fmt.Errorf("unable to set SO_RCVBUF: %s", err)
-	}
-
-	var lip [16]byte
-	copy(lip[:], net.ParseIP(ip))
-
-	// TODO do we need to `bind(2)` so that we send from the correct address/interface?
-	if err = unix.Bind(fd, &unix.SockaddrInet6{Addr: lip, Port: port}); err != nil {
-		return nil, fmt.Errorf("unable to bind to socket: %s", err)
-	}
-
-	return &RawConn{
-		sysFd:    fd,
-		basePort: basePort,
-		l:        l,
-	}, nil
-}
-
-// WriteTo must be called with raw leaving the first `udp.RawOverhead` bytes empty,
-// for the IP/UDP headers.
-func (u *RawConn) WriteTo(raw []byte, fromPort uint16, ip netip.AddrPort) error {
-	var rsa unix.RawSockaddrInet4
-	rsa.Family = unix.AF_INET
-	rsa.Addr = ip.Addr().As4()
-
-	totalLen := len(raw)
-	udpLen := totalLen - ipv4.HeaderLen
-
-	// IP header
-	raw[0] = byte(ipv4.Version<<4 | (ipv4.HeaderLen >> 2 & 0x0f))
-	raw[1] = 0 // tos
-	binary.BigEndian.PutUint16(raw[2:4], uint16(totalLen))
-	binary.BigEndian.PutUint16(raw[4:6], 0)   // id (linux does it for us)
-	binary.BigEndian.PutUint16(raw[6:8], 0)   // frag options
-	raw[8] = byte(64)                         // ttl
-	raw[9] = byte(17)                         // protocol
-	binary.BigEndian.PutUint16(raw[10:12], 0) // checksum (linux does it for us)
-	binary.BigEndian.PutUint32(raw[12:16], 0) // src (linux does it for us)
-	copy(raw[16:20], rsa.Addr[:])             // dst
-
-	// UDP header
-	fromPort = u.basePort + fromPort
-	binary.BigEndian.PutUint16(raw[20:22], uint16(fromPort))  // src port
-	binary.BigEndian.PutUint16(raw[22:24], uint16(ip.Port())) // dst port
-	binary.BigEndian.PutUint16(raw[24:26], uint16(udpLen))    // UDP length
-	binary.BigEndian.PutUint16(raw[26:28], 0)                 // checksum (optional)
-
-	for {
-		_, _, err := unix.Syscall6(
-			unix.SYS_SENDTO,
-			uintptr(u.sysFd),
-			uintptr(unsafe.Pointer(&raw[0])),
-			uintptr(len(raw)),
-			uintptr(0),
-			uintptr(unsafe.Pointer(&rsa)),
-			uintptr(unix.SizeofSockaddrInet4),
-		)
-
-		if err != 0 {
-			return &net.OpError{Op: "sendto", Err: err}
-		}
-
-		//TODO: handle incomplete writes
-
-		return nil
-	}
-}
-
-func (u *RawConn) ReloadConfig(c *config.C) {
-	b := c.GetInt("listen.write_buffer", 0)
-	if b <= 0 {
-		return
-	}
-
-	if err := u.SetSendBuffer(b); err != nil {
-		u.l.Error("Failed to set listen.write_buffer", "error", err)
-		return
-	}
-
-	s, err := u.GetSendBuffer()
-	if err != nil {
-		u.l.Warn("Failed to get listen.write_buffer", "error", err)
-		return
-	}
-
-	u.l.Info("listen.write_buffer was set", "size", s)
-}
-
-func (u *RawConn) SetSendBuffer(n int) error {
-	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
-}
-
-func (u *RawConn) GetSendBuffer() (int, error) {
-	return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF)
-}
-
-func (u *RawConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
-	var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
-	_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
-	if err != 0 {
-		return err
-	}
-	return nil
-}
-
-func NewRawStatsEmitter(rawConn *RawConn) func() {
-	// Check if our kernel supports SO_MEMINFO before registering the gauges
-	var gauges [unix.SK_MEMINFO_VARS]metrics.Gauge
-	var meminfo [unix.SK_MEMINFO_VARS]uint32
-	if err := rawConn.getMemInfo(&meminfo); err == nil {
-		gauges = [unix.SK_MEMINFO_VARS]metrics.Gauge{
-			metrics.GetOrRegisterGauge("raw.rmem_alloc", nil),
-			metrics.GetOrRegisterGauge("raw.rcvbuf", nil),
-			metrics.GetOrRegisterGauge("raw.wmem_alloc", nil),
-			metrics.GetOrRegisterGauge("raw.sndbuf", nil),
-			metrics.GetOrRegisterGauge("raw.fwd_alloc", nil),
-			metrics.GetOrRegisterGauge("raw.wmem_queued", nil),
-			metrics.GetOrRegisterGauge("raw.optmem", nil),
-			metrics.GetOrRegisterGauge("raw.backlog", nil),
-			metrics.GetOrRegisterGauge("raw.drops", nil),
-		}
-	} else {
-		// return no-op because we don't support SO_MEMINFO
-		return func() {}
-	}
-
-	return func() {
-		if err := rawConn.getMemInfo(&meminfo); err == nil {
-			for j := 0; j < unix.SK_MEMINFO_VARS; j++ {
-				gauges[j].Update(int64(meminfo[j]))
-			}
-		}
-	}
-}

udp/udp_raw_unsupported.go

@@ -1,29 +0,0 @@
-//go:build !linux || android || e2e_testing
-// +build !linux android e2e_testing
-
-package udp
-
-import (
-	"fmt"
-	"log/slog"
-	"net/netip"
-	"runtime"
-
-	"github.com/slackhq/nebula/config"
-)
-
-const RawOverhead = 0
-
-type RawConn struct{}
-
-func NewRawConn(l *slog.Logger, ip string, port int, basePort uint16) (*RawConn, error) {
-	return nil, fmt.Errorf("multiport tx is not supported on %s", runtime.GOOS)
-}
-
-func (u *RawConn) WriteTo(raw []byte, fromPort uint16, addr netip.AddrPort) error {
-	return fmt.Errorf("multiport tx is not supported on %s", runtime.GOOS)
-}
-
-func (u *RawConn) ReloadConfig(c *config.C) {}
-
-func NewRawStatsEmitter(rawConn *RawConn) func() { return func() {} }