diff --git a/examples/config.yml b/examples/config.yml index 6b1bb3d0..a6aeda74 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -175,7 +175,9 @@ listen: # listen.port+i instead of sharing one port via SO_REUSEPORT, and one extra # tunnel ("lane") per routine is negotiated with capable peers: lane i # handshakes from local port listen.port+i to the peer's advertised -# base+(i mod peer_ports). Each lane is a full Noise session with its own +# base+((i + pair_offset) mod peer_ports), where pair_offset is a per-pair +# hash that spreads many small peers across a big peer's whole port range. +# Each lane is a full Noise session with its own # keys, nonce counter and replay window, so flows taking different paths # never fight over shared replay state. # diff --git a/handshake_manager.go b/handshake_manager.go index aefc866a..2b2b0685 100644 --- a/handshake_manager.go +++ b/handshake_manager.go @@ -740,7 +740,11 @@ func (hm *HandshakeManager) maybeAllocLaneState(hostinfo *HostInfo, result *hand // A certified-but-hostile peer doesn't get to size our state. peerPorts = 256 } - hostinfo.lanes = newLaneState(hm.f.routines, uint16(peerPorts), uint16(result.PeerBasePort)) + var offset uint16 + if len(hm.f.myVpnAddrs) > 0 && len(hostinfo.vpnAddrs) > 0 { + offset = lanePortOffset(hm.f.myVpnAddrs[0], hostinfo.vpnAddrs[0], uint16(peerPorts)) + } + hostinfo.lanes = newLaneState(hm.f.routines, uint16(peerPorts), uint16(result.PeerBasePort), offset) } // EnsureLanes starts lane handshakes for every empty, non-pending, retry-due @@ -785,7 +789,7 @@ func (hm *HandshakeManager) startLaneHandshake(base *HostInfo, i int) { ls.noteLaneFailure(i) return } - target := netip.AddrPortFrom(remote.Addr(), ls.peerBasePort+uint16(i%int(ls.peerPortCount))) + target := netip.AddrPortFrom(remote.Addr(), ls.laneTargetPort(i)) hostinfo := &HostInfo{ vpnAddrs: slices.Clone(base.vpnAddrs), diff --git a/hostmap.go b/hostmap.go index 9ac6e38b..7a97d87c 100644 --- a/hostmap.go +++ b/hostmap.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "hash/fnv" "log/slog" "net" "net/netip" @@ -316,9 +317,11 @@ type laneState struct { sync.Mutex // peerPortCount/peerBasePort are the peer's advert from the base - // handshake; lane i targets peerBasePort + (i % peerPortCount). + // handshake; portOffset is the per-pair rotation from lanePortOffset. + // Lane i targets peerBasePort + ((i + portOffset) % peerPortCount). peerPortCount uint16 peerBasePort uint16 + portOffset uint16 // txLanes[i] is our established, initiator-owned lane for routine i, or // nil. Index 0 is always nil — the base tunnel is lane 0. A pointer is @@ -337,10 +340,11 @@ type laneState struct { peerLanes []*HostInfo } -func newLaneState(laneCount int, peerPortCount, peerBasePort uint16) *laneState { +func newLaneState(laneCount int, peerPortCount, peerBasePort, portOffset uint16) *laneState { return &laneState{ peerPortCount: peerPortCount, peerBasePort: peerBasePort, + portOffset: portOffset, txLanes: make([]atomic.Pointer[HostInfo], laneCount), txPending: make([]bool, laneCount), txFails: make([]uint8, laneCount), @@ -348,6 +352,45 @@ func newLaneState(laneCount int, peerPortCount, peerBasePort uint16) *laneState } } +// laneTargetPort returns the peer port that owned lane i handshakes to and +// egresses toward. The caller must ensure peerPortCount != 0. +func (ls *laneState) laneTargetPort(i int) uint16 { + return ls.peerBasePort + uint16((i+int(ls.portOffset))%int(ls.peerPortCount)) +} + +// lanePortOffset returns the rotation applied to this pair's lane target +// ports, in [0, peerPortCount). Without it every low-routine peer would aim +// its few lanes at a big peer's first few ports, concentrating the big +// peer's receive work on a couple of sockets; the hash spreads pairs across +// the whole advertised range. +// +// Both sides hash the same sorted vpn-address pair and the higher address +// negates the result, so when port counts match the two sides' rotations +// cancel: our lane i's 4-tuple is still the reverse of a peer-owned lane's, +// and each outbound lane handshake opens the conntrack entry its partner +// arrives through. (The one lane a nonzero rotation lands on the peer's base +// port has no partner lane; behind a port-restricted NAT it may not form and +// its routine rides the base tunnel — the standard lane fallback.) +func lanePortOffset(myAddr, peerAddr netip.Addr, peerPortCount uint16) uint16 { + if peerPortCount == 0 { + return 0 + } + lo, hi := myAddr, peerAddr + if hi.Less(lo) { + lo, hi = hi, lo + } + h := fnv.New32a() + b := lo.As16() + h.Write(b[:]) + b = hi.As16() + h.Write(b[:]) + o := uint16(h.Sum32() % uint32(peerPortCount)) + if myAddr == hi { + o = (peerPortCount - o) % peerPortCount + } + return o +} + const ( laneRetryBase = 5 * time.Second laneRetryMax = 60 * time.Second diff --git a/lanes_test.go b/lanes_test.go index 9a0f7de0..128888f9 100644 --- a/lanes_test.go +++ b/lanes_test.go @@ -27,7 +27,7 @@ func newTestBaseHostInfo(vpnIp netip.Addr, localIdx, remoteIdx uint32, laneCount HandshakePacket: map[uint8][]byte{}, } base.SetRemote(netip.MustParseAddrPort("192.0.2.1:4242")) - base.lanes = newLaneState(laneCount, uint16(laneCount), 4242) + base.lanes = newLaneState(laneCount, uint16(laneCount), 4242, 0) return base } @@ -47,6 +47,62 @@ func newTestLaneHostInfo(base *HostInfo, laneIndex uint16, localIdx, remoteIdx u return lane } +func TestLanePortOffset(t *testing.T) { + a := netip.MustParseAddr("10.0.0.1") + b := netip.MustParseAddr("10.0.0.2") + + // Deterministic and in range. + for _, count := range []uint16{1, 2, 3, 4, 16, 256} { + o := lanePortOffset(a, b, count) + assert.Equal(t, o, lanePortOffset(a, b, count), "count %d not deterministic", count) + assert.Less(t, o, count, "count %d out of range", count) + } + assert.Equal(t, uint16(0), lanePortOffset(a, b, 0), "zero port count") + + // The two sides' rotations cancel when port counts match, preserving the + // lane-i-reverses-lane-j conntrack pairing. + for _, count := range []uint16{2, 3, 4, 7, 16} { + for i := range 32 { + peer := netip.AddrFrom4([4]byte{192, 0, 2, byte(i)}) + oA := lanePortOffset(a, peer, count) + oB := lanePortOffset(peer, a, count) + assert.Equal(t, uint16(0), (oA+oB)%count, + "offsets don't cancel for peer %s count %d", peer, count) + } + } + + // Distinct small peers land on distinct rotations of a big peer's range, + // not all on the same first ports. + const bigPeerPorts = 16 + distinct := map[uint16]struct{}{} + for i := range 64 { + client := netip.AddrFrom4([4]byte{192, 0, 2, byte(i)}) + distinct[lanePortOffset(client, a, bigPeerPorts)] = struct{}{} + } + assert.GreaterOrEqual(t, len(distinct), 8, "64 clients only produced %d distinct offsets", len(distinct)) +} + +func TestLaneTargetPort(t *testing.T) { + // No rotation: lane i targets base+i, wrapping past the peer's range. + ls := newLaneState(4, 4, 4242, 0) + for i, want := range map[int]uint16{1: 4243, 2: 4244, 3: 4245, 5: 4243} { + assert.Equal(t, want, ls.laneTargetPort(i), "lane %d", i) + } + + // Rotation shifts the whole mapping; the wrapped lane lands on the base + // port itself, which is a valid distinct 4-tuple (our source port differs). + ls = newLaneState(4, 4, 4242, 3) + for i, want := range map[int]uint16{1: 4242, 2: 4243, 3: 4244} { + assert.Equal(t, want, ls.laneTargetPort(i), "rotated lane %d", i) + } + + // Fewer peer ports than local lanes: rotation still spreads across all of + // the peer's ports. + ls = newLaneState(16, 2, 4242, 1) + assert.Equal(t, uint16(4242), ls.laneTargetPort(1)) + assert.Equal(t, uint16(4243), ls.laneTargetPort(2)) +} + func TestLaneHostmapLifecycle(t *testing.T) { l := test.NewLogger() hostMap := newHostMap(l)