PMTUD exploration, start small then grow

overlay/device.go

@@ -15,4 +15,13 @@ type Device interface {
 	RoutesFor(netip.Addr) routing.Gateways
 	SupportsMultiqueue() bool
 	NewMultiQueueReader() (io.ReadWriteCloser, error)
+	// SupportsPerPeerMTU reports whether SetPeerMTU is implemented for real on
+	// this platform. PMTUD requires this; the manager will refuse to enable when
+	// false even if the operator set tun.max_mtu, because a discovered MTU we
+	// can't actually install does the operator no good.
+	SupportsPerPeerMTU() bool
+	// SetPeerMTU installs a per-peer MTU on the routing table so the kernel will
+	// surface PTB / EMSGSIZE for inside packets to that peer that would exceed mtu.
+	// Pass mtu=0 to remove the override and let the device default apply.
+	SetPeerMTU(addr netip.Addr, mtu int) error
 }

overlay/overlaytest/noop.go

@@ -39,6 +39,14 @@ func (NoopTun) Write([]byte) (int, error) {
 	return 0, nil
 }
 
+func (NoopTun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (NoopTun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (NoopTun) SupportsMultiqueue() bool {
 	return false
 }

overlay/tun_android.go

@@ -95,6 +95,14 @@ func (t *tun) Name() string {
 	return "android"
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }

overlay/tun_darwin.go

@@ -548,6 +548,14 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }

overlay/tun_disabled.go

@@ -106,6 +106,14 @@ func (t *disabledTun) Write(b []byte) (int, error) {
 	return len(b), nil
 }
 
+func (t *disabledTun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *disabledTun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *disabledTun) SupportsMultiqueue() bool {
 	return true
 }

overlay/tun_freebsd.go

@@ -561,6 +561,14 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }

overlay/tun_ios.go

@@ -151,6 +151,14 @@ func (t *tun) Name() string {
 	return "iOS"
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }

overlay/tun_linux.go

@@ -368,6 +368,13 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		}
 	}
 
+	// tun.max_mtu raises the device MTU above tun.mtu so PMTUD has headroom to
+	// install per-peer routes between tun.mtu (floor) and tun.max_mtu (ceiling).
+	// When unset (default 0) the device MTU is unchanged from existing behavior.
+	if pmtudCeiling := c.GetInt("tun.max_mtu", 0); pmtudCeiling > newMaxMTU {
+		newMaxMTU = pmtudCeiling
+	}
+
 	t.MaxMTU = newMaxMTU
 	t.DefaultMTU = newDefaultMTU
 
@@ -596,7 +603,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
 		LinkIndex: t.deviceIndex,
 		Dst:       dr,
 		MTU:       t.DefaultMTU,
-		AdvMSS:    t.advMSS(Route{}),
+		AdvMSS:    t.advMSS(Route{Cidr: cidr}),
 		Scope:     unix.RT_SCOPE_LINK,
 		Src:       net.IP(cidr.Addr().AsSlice()),
 		Protocol:  unix.RTPROT_KERNEL,
@@ -705,17 +712,68 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return true
+}
+
+// SetPeerMTU installs a host route (/32 for an IPv4 vpn address, /128 for an IPv6
+// vpn address) to addr through this tun device with the given MTU. This causes
+// the kernel to reject (or surface PTB to apps for) inside packets to addr that
+// would exceed mtu. Pass mtu=0 to remove the override and let the per-vpn-network
+// route apply again. PoC: assumes addr is reachable directly via this device.
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	bits := addr.BitLen()
+	prefix := netip.PrefixFrom(addr, bits)
+
+	dr := &net.IPNet{
+		IP:   addr.AsSlice(),
+		Mask: net.CIDRMask(bits, bits),
+	}
+
+	if mtu == 0 {
+		nr := netlink.Route{
+			LinkIndex: t.deviceIndex,
+			Dst:       dr,
+			Scope:     unix.RT_SCOPE_LINK,
+		}
+		if err := netlink.RouteDel(&nr); err != nil {
+			return fmt.Errorf("failed to remove per-peer mtu route %v: %w", prefix, err)
+		}
+		return nil
+	}
+
+	nr := netlink.Route{
+		LinkIndex: t.deviceIndex,
+		Dst:       dr,
+		MTU:       mtu,
+		AdvMSS:    t.advMSS(Route{Cidr: prefix, MTU: mtu}),
+		Scope:     unix.RT_SCOPE_LINK,
+	}
+	if err := netlink.RouteReplace(&nr); err != nil {
+		return fmt.Errorf("failed to set per-peer mtu route %v mtu=%d: %w", prefix, mtu, err)
+	}
+	return nil
+}
+
 func (t *tun) advMSS(r Route) int {
 	mtu := r.MTU
 	if r.MTU == 0 {
 		mtu = t.DefaultMTU
 	}
 
-	// We only need to set advmss if the route MTU does not match the device MTU
-	if mtu != t.MaxMTU {
-		return mtu - 40
+	// We only need to set advmss if the route MTU does not match the device MTU.
+	if mtu == t.MaxMTU {
+		return 0
 	}
-	return 0
+
+	// MSS = MTU - (IP header + TCP header). TCP is always 20 bytes; IP is 20 for
+	// v4 and 40 for v6. r.Cidr is the route destination so it tells us which
+	// family this route is in. If Cidr is unset (empty Route) we default to v4.
+	addr := r.Cidr.Addr()
+	if addr.Is6() && !addr.Is4In6() {
+		return mtu - 60
+	}
+	return mtu - 40
 }
 
 func (t *tun) watchRoutes() {

overlay/tun_netbsd.go

@@ -390,6 +390,14 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }

overlay/tun_openbsd.go

@@ -310,6 +310,14 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }

overlay/tun_tester.go

@@ -105,6 +105,14 @@ func (t *TestTun) Name() string {
 	return t.Device
 }
 
+func (t *TestTun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *TestTun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *TestTun) Write(b []byte) (n int, err error) {
 	if t.closed.Load() {
 		return 0, io.ErrClosedPipe

overlay/tun_windows.go

@@ -229,6 +229,14 @@ func (t *winTun) Name() string {
 	return t.Device
 }
 
+func (t *winTun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *winTun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *winTun) Read(b []byte) (int, error) {
 	return t.tun.Read(b, 0)
 }

overlay/user.go

@@ -46,6 +46,14 @@ func (d *UserDevice) RoutesFor(ip netip.Addr) routing.Gateways {
 	return routing.Gateways{routing.NewGateway(ip, 1)}
 }
 
+func (d *UserDevice) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (d *UserDevice) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (d *UserDevice) SupportsMultiqueue() bool {
 	return true
 }