smoke: Prove windows honors NlMtu with a 9000 byte ping

Follow-up to #1871. The Get-NetIPInterface check proves the value was written, not that windows fragments to it.

A 9000 byte payload only crosses the tunnel as fragments cut at NlMtu on the way out of the adapter. Left at the adapter default it reaches nebula whole, overflows the udp.MTU (9001) write buffer and is dropped, so the ping never answers. Anything smaller would ride out as one oversized datagram and survive on IP fragmentation of the underlay, which is why a payload that is merely larger than tun.mtu would not catch a regression.

Both families are pinged from the windows side so the v4 NlMtu, which was always right, gets the same guard.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J34ExJUUPKWUvr2g5GMREi

.github/workflows/smoke/smoke-windows.ps1

@@ -51,15 +51,19 @@ wsl -d $Distro -- bash -c "rm -rf $WslDir && mkdir -p $WslDir" | Out-Null
 $DevName = 'nebula-smoke'
 $Ip1 = '192.168.241.1'
 $Ip2 = '192.168.241.2'
+# Dual stack on purpose: a v4-only overlay never exercises the v6 side of tun.mtu.
+$Ip6_1 = 'fd42:4242:241::1'
+$Ip6_2 = 'fd42:4242:241::2'
+$Mtu = 1300
 $Port = 4242
 
 & $NebulaCert ca -name 'smoke-ca' -out-crt "$WorkDir\ca.crt" -out-key "$WorkDir\ca.key"
 if ($LASTEXITCODE -ne 0) { throw "nebula-cert ca failed (exit $LASTEXITCODE)" }
 
-& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key"
+& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24,$Ip6_1/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key"
 if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign lighthouse failed (exit $LASTEXITCODE)" }
 
-& $NebulaCert sign -name 'peer' -networks "$Ip2/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key"
+& $NebulaCert sign -name 'peer' -networks "$Ip2/24,$Ip6_2/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key"
 if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign peer failed (exit $LASTEXITCODE)" }
 
 # Windows lighthouse config.
@@ -82,7 +86,7 @@ tun:
   drop_local_broadcast: false
   drop_multicast: false
   tx_queue: 500
-  mtu: 1300
+  mtu: $Mtu
   network_category: private
 logging:
   level: info
@@ -126,7 +130,7 @@ tun:
   drop_local_broadcast: false
   drop_multicast: false
   tx_queue: 500
-  mtu: 1300
+  mtu: $Mtu
 logging:
   level: info
   format: text
@@ -169,7 +173,7 @@ Write-Host '=== WSL diagnostic ==='
 wsl --version 2>&1 | Out-Host
 wsl --list --verbose 2>&1 | Out-Host
 wsl -d $Distro -u root -- uname -a | Out-Host
-wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; ls -l /dev/net/tun"
+wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; { echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6; echo 0 > /proc/sys/net/ipv6/conf/default/disable_ipv6; } 2>/dev/null || true; ls -l /dev/net/tun"
 if ($LASTEXITCODE -ne 0) { throw "failed to prepare /dev/net/tun in WSL (TUN support missing?)" }
 
 # Deliberately no New-NetFirewallRule calls here -- nebula's windows_bypass_wdf
@@ -214,6 +218,16 @@ try {
     }
     Write-Host "OK: $DevName NetworkCategory=Private"
 
+    # v6 silently kept the adapter default of 65535 while v4 was correct.
+    foreach ($family in @('IPv4', 'IPv6')) {
+        Wait-Until -TimeoutSec 30 -What "$DevName $family NlMtu=$Mtu" -Predicate {
+            if ($lhProc.HasExited) { throw "lighthouse exited (code $($lhProc.ExitCode)) before $family mtu was set" }
+            $rows = @(Get-NetIPInterface -InterfaceAlias $DevName -AddressFamily $family -ErrorAction SilentlyContinue)
+            $rows.Count -gt 0 -and -not ($rows | Where-Object { $_.NlMtu -ne $Mtu })
+        }
+        Write-Host "OK: $DevName $family NlMtu=$Mtu"
+    }
+
     Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip2" -Predicate {
         if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before tun was ready" }
         $r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet $Ip2' && echo yes"
@@ -221,6 +235,13 @@ try {
     }
     Write-Host "OK: WSL nebula1 has $Ip2"
 
+    Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip6_2" -Predicate {
+        if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 address was up" }
+        $r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet6 $Ip6_2' && echo yes"
+        ("$r").Trim() -eq 'yes'
+    }
+    Write-Host "OK: WSL nebula1 has $Ip6_2"
+
     Wait-Until -TimeoutSec 30 -What "ping from WSL peer to windows lighthouse ($Ip1)" -Predicate {
         if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before ping succeeded" }
         $r = wsl -d $Distro -u root -- bash -c "ping -c1 -W1 $Ip1 >/dev/null 2>&1 && echo OK"
@@ -234,6 +255,28 @@ try {
     }
     Write-Host "OK: windows lighthouse -> WSL peer"
 
+    # Otherwise the v6 networks only prove the interface exists, not that it forwards.
+    Wait-Until -TimeoutSec 30 -What "v6 ping from WSL peer to windows lighthouse ($Ip6_1)" -Predicate {
+        if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 ping succeeded" }
+        $r = wsl -d $Distro -u root -- bash -c "ping -6 -c1 -W1 $Ip6_1 >/dev/null 2>&1 && echo OK"
+        ("$r").Trim() -eq 'OK'
+    }
+    Write-Host "OK: WSL peer -> windows lighthouse over v6"
+
+    # The NlMtu check above proves the value was written, not that windows honors it. A payload this size only
+    # crosses the tunnel as fragments cut at NlMtu on the way out of the adapter. Left at the adapter default it
+    # reaches nebula whole, overflows the udp.MTU (9001) write buffer and is dropped, so the ping never answers.
+    # Anything smaller would ride out as one oversized datagram and survive on IP fragmentation of the underlay.
+    $BigPayload = 9000
+    foreach ($target in @(@{ Family = 'v4'; Ip = $Ip2 }, @{ Family = 'v6'; Ip = $Ip6_2 })) {
+        Wait-Until -TimeoutSec 30 -What "$($target.Family) ping with a $BigPayload byte payload from windows lighthouse to WSL peer ($($target.Ip))" -Predicate {
+            if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the large $($target.Family) ping succeeded" }
+            $null = & ping.exe -n 1 -w 1000 -l $BigPayload $target.Ip
+            $LASTEXITCODE -eq 0
+        }
+        Write-Host "OK: windows lighthouse -> WSL peer, $BigPayload byte $($target.Family) payload"
+    }
+
     Write-Host ''
     Write-Host 'All smoke checks passed.'
 }

overlay/tun_windows.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"sync/atomic"
 	"syscall"
 	"unsafe"
@@ -182,6 +183,7 @@ func (t *winTun) addRoutes(logErrors bool) error {
 	luid := winipcfg.LUID(t.tun.LUID())
 	routes := *t.Routes.Load()
 	foundDefault4 := false
+	carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() })
 
 	for _, r := range routes {
 		if len(r.Via) == 0 || !r.Install {
@@ -189,6 +191,9 @@ func (t *winTun) addRoutes(logErrors bool) error {
 			continue
 		}
 
+		// A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter.
+		carriesV6 = carriesV6 || r.Cidr.Addr().Is6()
+
 		// Add our unsafe route as an on-link route to the nebula tun device.
 		err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric))
 		if err != nil {
@@ -210,6 +215,11 @@ func (t *winTun) addRoutes(logErrors bool) error {
 		}
 	}
 
+	return t.setMTU(luid, foundDefault4, carriesV6)
+}
+
+// setMTU applies tun.mtu per address family. The default route metric rides along on the v4 handle.
+func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error {
 	ipif, err := luid.IPInterface(windows.AF_INET)
 	if err != nil {
 		return fmt.Errorf("failed to get ip interface: %w", err)
@@ -224,6 +234,25 @@ func (t *winTun) addRoutes(logErrors bool) error {
 	if err := ipif.Set(); err != nil {
 		return fmt.Errorf("failed to set ip interface: %w", err)
 	}
+
+	// Windows tracks NLMTU per family and wintun sets neither, so v6 keeps the adapter default of 65535.
+	// Gated so a v4-only overlay under 1280 boots; a v6 one deliberately does not, as linux also refuses.
+	if !carriesV6 {
+		return nil
+	}
+
+	ipif6, err := luid.IPInterface(windows.AF_INET6)
+	if err != nil {
+		// No v6 on the adapter means there is no NLMTU to get wrong. A failed Set below is not the same thing.
+		t.l.Info("Skipping ipv6 MTU, no ipv6 interface on this adapter", "error", err)
+		return nil
+	}
+
+	ipif6.NLMTU = uint32(t.MTU)
+	if err := ipif6.Set(); err != nil {
+		return fmt.Errorf("failed to set ipv6 interface: %w", err)
+	}
+
 	return nil
 }
 

udp/udp_rio_windows.go

@@ -31,7 +31,8 @@ func procyield(cycles uint32)
 
 const (
 	packetsPerRing = 1024
-	bytesPerPacket = 2048 - 32
+	// Caps tun.mtu at MTU-32 direct, MTU-64 relayed, unenforced anywhere else. 17.6MB page locked per socket.
+	bytesPerPacket = MTU
 	receiveSpins   = 15
 )
 
@@ -69,12 +70,14 @@ func NewRIOListener(l *slog.Logger, addr netip.Addr, port int) (*RIOConn, error)
 
 	err := u.bind(l, &windows.SockaddrInet6{Addr: addr.As16(), Port: port})
 	if err != nil {
+		u.close()
 		return nil, fmt.Errorf("bind: %w", err)
 	}
 
 	for i := 0; i < packetsPerRing; i++ {
 		err = u.insertReceiveRequest()
 		if err != nil {
+			u.close()
 			return nil, fmt.Errorf("init rx ring: %w", err)
 		}
 	}
@@ -356,15 +359,25 @@ func (u *RIOConn) Close() error {
 		return nil
 	}
 
+	u.close()
+	return nil
+}
+
+// Also unwinds a partial build from NewRIOListener, where isOpen is false and Close would no-op.
+// Socket first, unlike wireguard-go: receive() re-arms every slot, so freeing the rings under a live socket
+// hands the kernel freed pages for all packetsPerRing outstanding receives.
+func (u *RIOConn) close() {
+	// WSASocket reports failure as InvalidHandle, not zero.
+	if u.sock != 0 && u.sock != windows.InvalidHandle {
+		windows.CloseHandle(u.sock)
+	}
+	u.sock = 0
+
 	windows.PostQueuedCompletionStatus(u.rx.iocp, 0, 0, nil)
 	windows.PostQueuedCompletionStatus(u.tx.iocp, 0, 0, nil)
 
 	u.rx.CloseAndZero()
 	u.tx.CloseAndZero()
-	if u.sock != 0 {
-		windows.CloseHandle(u.sock)
-	}
-	return nil
 }
 
 func (ring *ringBuffer) Push() *ringPacket {