diff --git a/.github/workflows/smoke/smoke-windows.ps1 b/.github/workflows/smoke/smoke-windows.ps1 index 0436598d..e94768a9 100644 --- a/.github/workflows/smoke/smoke-windows.ps1 +++ b/.github/workflows/smoke/smoke-windows.ps1 @@ -51,15 +51,19 @@ wsl -d $Distro -- bash -c "rm -rf $WslDir && mkdir -p $WslDir" | Out-Null $DevName = 'nebula-smoke' $Ip1 = '192.168.241.1' $Ip2 = '192.168.241.2' +# Dual stack on purpose: a v4-only overlay never exercises the v6 side of tun.mtu. +$Ip6_1 = 'fd42:4242:241::1' +$Ip6_2 = 'fd42:4242:241::2' +$Mtu = 1300 $Port = 4242 & $NebulaCert ca -name 'smoke-ca' -out-crt "$WorkDir\ca.crt" -out-key "$WorkDir\ca.key" if ($LASTEXITCODE -ne 0) { throw "nebula-cert ca failed (exit $LASTEXITCODE)" } -& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key" +& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24,$Ip6_1/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key" if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign lighthouse failed (exit $LASTEXITCODE)" } -& $NebulaCert sign -name 'peer' -networks "$Ip2/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key" +& $NebulaCert sign -name 'peer' -networks "$Ip2/24,$Ip6_2/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key" if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign peer failed (exit $LASTEXITCODE)" } # Windows lighthouse config. @@ -82,7 +86,7 @@ tun: drop_local_broadcast: false drop_multicast: false tx_queue: 500 - mtu: 1300 + mtu: $Mtu network_category: private logging: level: info @@ -126,7 +130,7 @@ tun: drop_local_broadcast: false drop_multicast: false tx_queue: 500 - mtu: 1300 + mtu: $Mtu logging: level: info format: text @@ -169,7 +173,7 @@ Write-Host '=== WSL diagnostic ===' wsl --version 2>&1 | Out-Host wsl --list --verbose 2>&1 | Out-Host wsl -d $Distro -u root -- uname -a | Out-Host -wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; ls -l /dev/net/tun" +wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; { echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6; echo 0 > /proc/sys/net/ipv6/conf/default/disable_ipv6; } 2>/dev/null || true; ls -l /dev/net/tun" if ($LASTEXITCODE -ne 0) { throw "failed to prepare /dev/net/tun in WSL (TUN support missing?)" } # Deliberately no New-NetFirewallRule calls here -- nebula's windows_bypass_wdf @@ -214,6 +218,16 @@ try { } Write-Host "OK: $DevName NetworkCategory=Private" + # v6 silently kept the adapter default of 65535 while v4 was correct. + foreach ($family in @('IPv4', 'IPv6')) { + Wait-Until -TimeoutSec 30 -What "$DevName $family NlMtu=$Mtu" -Predicate { + if ($lhProc.HasExited) { throw "lighthouse exited (code $($lhProc.ExitCode)) before $family mtu was set" } + $rows = @(Get-NetIPInterface -InterfaceAlias $DevName -AddressFamily $family -ErrorAction SilentlyContinue) + $rows.Count -gt 0 -and -not ($rows | Where-Object { $_.NlMtu -ne $Mtu }) + } + Write-Host "OK: $DevName $family NlMtu=$Mtu" + } + Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip2" -Predicate { if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before tun was ready" } $r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet $Ip2' && echo yes" @@ -221,6 +235,13 @@ try { } Write-Host "OK: WSL nebula1 has $Ip2" + Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip6_2" -Predicate { + if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 address was up" } + $r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet6 $Ip6_2' && echo yes" + ("$r").Trim() -eq 'yes' + } + Write-Host "OK: WSL nebula1 has $Ip6_2" + Wait-Until -TimeoutSec 30 -What "ping from WSL peer to windows lighthouse ($Ip1)" -Predicate { if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before ping succeeded" } $r = wsl -d $Distro -u root -- bash -c "ping -c1 -W1 $Ip1 >/dev/null 2>&1 && echo OK" @@ -234,6 +255,14 @@ try { } Write-Host "OK: windows lighthouse -> WSL peer" + # Otherwise the v6 networks only prove the interface exists, not that it forwards. + Wait-Until -TimeoutSec 30 -What "v6 ping from WSL peer to windows lighthouse ($Ip6_1)" -Predicate { + if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 ping succeeded" } + $r = wsl -d $Distro -u root -- bash -c "ping -6 -c1 -W1 $Ip6_1 >/dev/null 2>&1 && echo OK" + ("$r").Trim() -eq 'OK' + } + Write-Host "OK: WSL peer -> windows lighthouse over v6" + Write-Host '' Write-Host 'All smoke checks passed.' } diff --git a/overlay/tun_windows.go b/overlay/tun_windows.go index 6be85ffc..56c18131 100644 --- a/overlay/tun_windows.go +++ b/overlay/tun_windows.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "sync/atomic" "syscall" "unsafe" @@ -182,6 +183,7 @@ func (t *winTun) addRoutes(logErrors bool) error { luid := winipcfg.LUID(t.tun.LUID()) routes := *t.Routes.Load() foundDefault4 := false + carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() }) for _, r := range routes { if len(r.Via) == 0 || !r.Install { @@ -189,6 +191,9 @@ func (t *winTun) addRoutes(logErrors bool) error { continue } + // A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter. + carriesV6 = carriesV6 || r.Cidr.Addr().Is6() + // Add our unsafe route as an on-link route to the nebula tun device. err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric)) if err != nil { @@ -210,6 +215,11 @@ func (t *winTun) addRoutes(logErrors bool) error { } } + return t.setMTU(luid, foundDefault4, carriesV6) +} + +// setMTU applies tun.mtu per address family. The default route metric rides along on the v4 handle. +func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error { ipif, err := luid.IPInterface(windows.AF_INET) if err != nil { return fmt.Errorf("failed to get ip interface: %w", err) @@ -224,6 +234,25 @@ func (t *winTun) addRoutes(logErrors bool) error { if err := ipif.Set(); err != nil { return fmt.Errorf("failed to set ip interface: %w", err) } + + // Windows tracks NLMTU per family and wintun sets neither, so v6 keeps the adapter default of 65535. + // Gated so a v4-only overlay under 1280 boots; a v6 one deliberately does not, as linux also refuses. + if !carriesV6 { + return nil + } + + ipif6, err := luid.IPInterface(windows.AF_INET6) + if err != nil { + // No v6 on the adapter means there is no NLMTU to get wrong. A failed Set below is not the same thing. + t.l.Info("Skipping ipv6 MTU, no ipv6 interface on this adapter", "error", err) + return nil + } + + ipif6.NLMTU = uint32(t.MTU) + if err := ipif6.Set(); err != nil { + return fmt.Errorf("failed to set ipv6 interface: %w", err) + } + return nil } diff --git a/udp/udp_rio_windows.go b/udp/udp_rio_windows.go index b04770af..5df42ff7 100644 --- a/udp/udp_rio_windows.go +++ b/udp/udp_rio_windows.go @@ -31,7 +31,8 @@ func procyield(cycles uint32) const ( packetsPerRing = 1024 - bytesPerPacket = 2048 - 32 + // Caps tun.mtu at MTU-32 direct, MTU-64 relayed, unenforced anywhere else. 17.6MB page locked per socket. + bytesPerPacket = MTU receiveSpins = 15 ) @@ -69,12 +70,14 @@ func NewRIOListener(l *slog.Logger, addr netip.Addr, port int) (*RIOConn, error) err := u.bind(l, &windows.SockaddrInet6{Addr: addr.As16(), Port: port}) if err != nil { + u.close() return nil, fmt.Errorf("bind: %w", err) } for i := 0; i < packetsPerRing; i++ { err = u.insertReceiveRequest() if err != nil { + u.close() return nil, fmt.Errorf("init rx ring: %w", err) } } @@ -356,15 +359,25 @@ func (u *RIOConn) Close() error { return nil } + u.close() + return nil +} + +// Also unwinds a partial build from NewRIOListener, where isOpen is false and Close would no-op. +// Socket first, unlike wireguard-go: receive() re-arms every slot, so freeing the rings under a live socket +// hands the kernel freed pages for all packetsPerRing outstanding receives. +func (u *RIOConn) close() { + // WSASocket reports failure as InvalidHandle, not zero. + if u.sock != 0 && u.sock != windows.InvalidHandle { + windows.CloseHandle(u.sock) + } + u.sock = 0 + windows.PostQueuedCompletionStatus(u.rx.iocp, 0, 0, nil) windows.PostQueuedCompletionStatus(u.tx.iocp, 0, 0, nil) u.rx.CloseAndZero() u.tx.CloseAndZero() - if u.sock != 0 { - windows.CloseHandle(u.sock) - } - return nil } func (ring *ringBuffer) Push() *ringPacket {