Disable DNS registration on the Windows tunnel adapter (#1914)

On a domain-joined Windows host the DNS Client registers every adapter's
addresses under the host name in AD DNS when "Register this connection's
addresses in DNS" is on, which is the default for a new adapter. When the
Nebula adapter registers, LAN clients that are not on the overlay resolve
the host name to its overlay IP and RDP, SMB and PowerShell Remoting to it
fail intermittently, with nothing pointing at the tunnel as the cause.
Windows registers on address assignment, periodically, and on every
ipconfig /registerdns.

Right after the wintun adapter is created and before Activate assigns any
address, write RegistrationEnabled=0, DisableDynamicUpdate=1 and
MaxNumberOfAddressesToRegister=0 under both the Tcpip and Tcpip6
Parameters\Interfaces\{GUID} keys. The adapter checkbox alone has proven
insufficient in practice, as has disabling dynamic updates alone, so all
three are set, matching what Tailscale and NetBird do. A failure to write
the keys is logged as a warning and does not fail adapter creation.

overlay/dns_registration_windows.go

@@ -0,0 +1,57 @@
+//go:build !e2e_testing
+
+package overlay
+
+import (
+	"fmt"
+	"log/slog"
+
+	"golang.org/x/sys/windows"
+	"golang.org/x/sys/windows/registry"
+)
+
+var tcpipInterfaceKeys = []string{
+	`SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces`,
+	`SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces`,
+}
+
+// disableDNSRegistration prevents publishing overlay addresses in the host's
+// DNS records, where clients outside the overlay cannot reach them.
+// Failures are non-fatal so registry access cannot prevent tunnel creation.
+func disableDNSRegistration(l *slog.Logger, guid windows.GUID) {
+	id := guid.String()
+	for _, base := range tcpipInterfaceKeys {
+		path := base + `\` + id
+		if err := setDNSRegistrationValues(path); err != nil {
+			l.Warn("Failed to disable DNS registration on the tun adapter", "error", err, "key", path)
+			continue
+		}
+		l.Debug("Disabled DNS registration on the tun adapter", "key", path)
+	}
+}
+
+func setDNSRegistrationValues(path string) error {
+	// The stack may not have created the adapter's interface key yet.
+	k, _, err := registry.CreateKey(registry.LOCAL_MACHINE, path, registry.SET_VALUE)
+	if err != nil {
+		return fmt.Errorf("open HKLM\\%s: %w", path, err)
+	}
+	defer k.Close()
+
+	// Neither RegistrationEnabled nor DisableDynamicUpdate alone reliably
+	// prevents registration, so all three settings are required.
+	values := []struct {
+		name string
+		data uint32
+	}{
+		{"RegistrationEnabled", 0},
+		{"DisableDynamicUpdate", 1},
+		{"MaxNumberOfAddressesToRegister", 0},
+	}
+	for _, v := range values {
+		if err := k.SetDWordValue(v.name, v.data); err != nil {
+			return fmt.Errorf("set %s: %w", v.name, err)
+		}
+	}
+	return nil
+}

overlay/tun_windows.go

@@ -104,6 +104,9 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
 	}
 	t.tun = tunDevice.(*wintun.NativeTun)
 
+	// Disable registration before Activate assigns addresses and triggers it.
+	disableDNSRegistration(l, *guid)
+
 	c.RegisterReloadCallback(func(c *config.C) {
 		err := t.reload(c, false)
 		if err != nil {