crazy multiport stuff

(cherry picked from commit 0488793a62)

main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"math"
 	"net"
 	"net/netip"
 	"os"
@@ -146,6 +147,22 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 	udpConns := make([]udp.Conn, routines)
 	port := c.GetInt("listen.port", 0)
 
+	// Multiport lanes: bind `routines` consecutive UDP ports (listen.port+i)
+	// instead of SO_REUSEPORT sharing one, and negotiate one extra tunnel per
+	// routine with capable peers so each routine's traffic rides its own
+	// underlay 5-tuple. Defaults on, degrading gracefully when preconditions
+	// aren't met — managed deployments (dnclient) can't be hard-errored on
+	// config they don't control.
+	multiport := c.GetBool("multiport.enabled", true)
+	if multiport && routines < 2 {
+		l.Info("multiport disabled: requires routines > 1")
+		multiport = false
+	}
+	if multiport && port != 0 && port+routines-1 > math.MaxUint16 {
+		l.Warn("multiport disabled: would bind ports beyond 65535", "listen.port", port, "routines", routines)
+		multiport = false
+	}
+
 	// Callers get no handle to these until the Control is returned, release them on any error.
 	defer func() {
 		if reterr != nil {
@@ -175,38 +192,100 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 			listenHost = ips[0].Unmap()
 		}
 
-		for i := 0; i < routines; i++ {
-			listen := netip.AddrPortFrom(listenHost, uint16(port))
-			l.Info("listening", "addr", listen)
-			batchSize := c.GetInt("listen.batch", 64)
-			if batchSize < 1 {
-				oldBatch := batchSize
-				batchSize = 1
-				l.Warn("listen.batch size is invalid", "provided", oldBatch, "overridden to", batchSize)
-			}
-			udpSettings := udp.Settings{
-				Listen:   listen,
-				Multi:    routines > 1,
-				Batch:    batchSize,
-				Offloads: c.GetBool("listen.udp_offloads", false),
-			}
-			udpServer, err := udp.NewListener(l, udpSettings)
-			if err != nil {
-				return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err)
-			}
-			udpServer.ReloadConfig(c)
-			udpConns[i] = udpServer
+		batchSize := c.GetInt("listen.batch", 64)
+		if batchSize < 1 {
+			oldBatch := batchSize
+			batchSize = 1
+			l.Warn("listen.batch size is invalid", "provided", oldBatch, "overridden to", batchSize)
+		}
+		offloads := c.GetBool("listen.udp_offloads", false)
 
-			// If port is dynamic, discover it before the next pass through the for loop
-			// This way all routines will use the same port correctly
-			if port == 0 {
-				uPort, err := udpServer.LocalAddr()
-				if err != nil {
-					return nil, util.NewContextualError("Failed to get listening port", nil, err)
+		// Every lane socket needs its own reader; a platform that can't run
+		// multiple readers would silently strand sockets 1..n-1 as blackholes.
+		// Probe capability before committing to per-port binds.
+		if multiport {
+			probe, err := udp.NewListener(l, udp.Settings{
+				Listen: netip.AddrPortFrom(listenHost, 0),
+				Batch:  1,
+			})
+			if err != nil {
+				// We could not confirm support, so don't gamble a bound port range on it. The real bind below reports
+				// the underlying error if it is not transient.
+				l.Warn("multiport disabled: could not probe udp reader support", "error", err)
+				multiport = false
+			} else {
+				if !probe.SupportsMultipleReaders() {
+					l.Warn("multiport disabled: this platform does not support multiple udp readers")
+					multiport = false
 				}
-				port = int(uPort.Port())
+				_ = probe.Close()
 			}
 		}
+
+		// With a dynamic listen.port, multiport binds socket 0 dynamically and
+		// then claims the next routines-1 ports above it; if that range turns
+		// out to be partially occupied, re-roll with a fresh dynamic port.
+		dynamic := port == 0
+		var bindErr error
+		for attempt := 0; attempt < 6; attempt++ {
+			bindErr = nil
+			for i := 0; i < routines; i++ {
+				lPort := port
+				if multiport {
+					lPort = port + i
+				}
+				udpServer, err := udp.NewListener(l, udp.Settings{
+					Listen: netip.AddrPortFrom(listenHost, uint16(lPort)),
+					// Multiport gives every routine its own port, so SO_REUSEPORT sharing is neither needed nor wanted:
+					// the destination port alone must decide which socket, and therefore which routine, sees a flow.
+					Multi:    routines > 1 && !multiport,
+					Batch:    batchSize,
+					Offloads: offloads,
+				})
+				if err != nil {
+					bindErr = util.NewContextualError("Failed to open udp listener", m{"queue": i}, err)
+					break
+				}
+				udpServer.ReloadConfig(c)
+				udpConns[i] = udpServer
+
+				// If port is dynamic, discover it before the next pass through the for loop
+				// This way all routines will use the same port correctly
+				if port == 0 {
+					uPort, err := udpServer.LocalAddr()
+					if err != nil {
+						return nil, util.NewContextualError("Failed to get listening port", nil, err)
+					}
+					port = int(uPort.Port())
+					if multiport && port+routines-1 > math.MaxUint16 {
+						bindErr = util.NewContextualError("multiport dynamic port too close to 65535", m{"port": port}, nil)
+						break
+					}
+				}
+				bound := port
+				if multiport {
+					bound = port + i
+				}
+				l.Info("listening", "addr", netip.AddrPortFrom(listenHost, uint16(bound)), "socket", i)
+			}
+			if bindErr == nil {
+				break
+			}
+			if !(multiport && dynamic) {
+				return nil, bindErr
+			}
+			for i := range udpConns {
+				if udpConns[i] != nil {
+					_ = udpConns[i].Close()
+					udpConns[i] = nil
+				}
+			}
+			port = 0
+			l.Debug("multiport dynamic port range collided, retrying", "attempt", attempt+1)
+		}
+		if bindErr != nil {
+			return nil, bindErr
+		}
 	}
 
 	hostMap := NewHostMapFromConfig(l, c)
@@ -231,6 +310,16 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		messageMetrics: messageMetrics,
 	}
 
+	if multiport {
+		lanes := c.GetInt("multiport.lanes", 0)
+		if lanes <= 0 || lanes > routines {
+			lanes = routines
+		}
+		handshakeConfig.laneCount = lanes
+		handshakeConfig.lanePortCount = uint16(routines)
+		handshakeConfig.laneBasePort = uint16(port)
+	}
+
 	handshakeManager := NewHandshakeManager(l, hostMap, lightHouse, udpConns[0], handshakeConfig)
 	lightHouse.handshakeTrigger = handshakeManager.trigger
 
@@ -286,6 +375,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		DropLocalBroadcast:    c.GetBool("tun.drop_local_broadcast", false),
 		DropMulticast:         c.GetBool("tun.drop_multicast", false),
 		routines:              routines,
+		Multiport:             multiport,
 		MessageMetrics:        messageMetrics,
 		version:               buildVersion,
 		relayManager:          NewRelayManager(ctx, l, hostMap, c),