Report how a config reload went: failures, restarts required, and the keys that need one (#1935)

overlay/tun.go

@@ -41,9 +41,16 @@ func (e *NameError) Error() string {
 type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error)
 
 func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
+	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+		r.RestartRequiredIfChanged("tun.disabled")
+	})
+
 	switch {
 	case c.GetBool("tun.disabled", false):
 		tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 5000), c.GetBool("stats.message_metrics", false), l)
+		c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+			r.RestartRequiredIfChanged("tun.tx_queue", "stats.message_metrics")
+		})
 		return tun, nil
 
 	default:

overlay/tun_android.go

@@ -38,16 +38,17 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 		l:               l,
 	}
 
-	err := t.reload(c, true)
+	err := t.reload(c, true, nil)
 	if err != nil {
 		_ = file.Close()
 		return nil, err
 	}
 
-	c.RegisterReloadCallback(func(c *config.C) {
-		err := t.reload(c, false)
+	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+		err := t.reload(c, false, r)
 		if err != nil {
 			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+			r.Failed()
 		}
 	})
 
@@ -67,7 +68,7 @@ func (t *tun) Activate() error {
 	return nil
 }
 
-func (t *tun) reload(c *config.C, initial bool) error {
+func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
 	change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
 	if err != nil {
 		return err

overlay/tun_bypass_windows.go

@@ -10,6 +10,9 @@ import (
 	"github.com/slackhq/nebula/wfp"
 )
 
+// interfaceBypassKeys are only read when the interface bypass is installed
+var interfaceBypassKeys = []string{"tun.windows_bypass_wdf"}
+
 // installInterfaceBypass installs a WFP PERMIT filter scoped to the wintun interface LUID so inbound traffic on the
 // nebula adapter bypasses Windows Defender Firewall.
 func installInterfaceBypass(l *slog.Logger, luid uint64) closer {

overlay/tun_bypass_windows_386.go

@@ -4,6 +4,9 @@ package overlay
 
 import "log/slog"
 
+// interfaceBypassKeys is empty, windows-386 has no bypass
+var interfaceBypassKeys []string
+
 // installInterfaceBypass is a no-op on windows-386 because we don't currently build for it.
 func installInterfaceBypass(_ *slog.Logger, _ uint64) closer {
 	return nil

overlay/tun_darwin.go

@@ -132,15 +132,17 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 		l:           l,
 	}
 
-	err = t.reload(c, true)
+	err = t.reload(c, true, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	c.RegisterReloadCallback(func(c *config.C) {
-		err := t.reload(c, false)
+	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+		r.RestartRequiredIfChanged("tun.dev", "tun.mtu")
+		err := t.reload(c, false, r)
 		if err != nil {
 			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+			r.Failed()
 		}
 	})
 
@@ -174,14 +176,15 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 		l:           l,
 	}
 
-	if err := t.reload(c, true); err != nil {
+	if err := t.reload(c, true, nil); err != nil {
 		_ = file.Close()
 		return nil, err
 	}
 
-	c.RegisterReloadCallback(func(c *config.C) {
-		if err := t.reload(c, false); err != nil {
+	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+		if err := t.reload(c, false, r); err != nil {
 			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+			r.Failed()
 		}
 	})
 
@@ -268,7 +271,7 @@ func (t *tun) Activate() error {
 	}
 
 	// Unsafe path routes
-	return t.addRoutes(false)
+	return t.addRoutes(false, nil)
 }
 
 // activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
@@ -366,7 +369,7 @@ func (t *tun) activate6(network netip.Prefix) error {
 	return nil
 }
 
-func (t *tun) reload(c *config.C, initial bool) error {
+func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
 	change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
 	if err != nil {
 		return err
@@ -387,16 +390,18 @@ func (t *tun) reload(c *config.C, initial bool) error {
 
 	if !initial {
 		// Remove first, if the system removes a wanted route hopefully it will be re-added next
-		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r)
 		if err != nil {
 			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
+			r.Failed()
 		}
 
 		// Ensure any routes we actually want are installed
-		err = t.addRoutes(true)
+		err = t.addRoutes(true, r)
 		if err != nil {
 			// Catch any stray logs
 			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
+			r.Failed()
 		}
 	}
 
@@ -489,7 +494,7 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
 	return nil, nil
 }
 
-func (t *tun) addRoutes(logErrors bool) error {
+func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
 	// The route tree is still ours, the system routing table is not
 	if t.hostOwned {
 		return nil
@@ -524,6 +529,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 				retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
 				if logErrors {
 					retErr.Log(t.l)
+					rl.Failed()
 				} else {
 					return retErr
 				}
@@ -538,7 +544,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 	return nil
 }
 
-func (t *tun) removeRoutes(routes []Route) error {
+func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
 	linkAddr := t.linkAddr.Load()
 	if t.hostOwned || linkAddr == nil {
 		return nil
@@ -551,8 +557,12 @@ func (t *tun) removeRoutes(routes []Route) error {
 		}
 
 		err := delRoute(r.Cidr, linkAddr)
-		if err != nil {
+		if errors.Is(err, unix.ESRCH) {
+			// Already gone, which is what we wanted
+			t.l.Info("Route was already removed", "route", r)
+		} else if err != nil {
 			t.l.Error("Failed to remove route", "error", err, "route", r)
+			rl.Failed()
 		} else {
 			t.l.Info("Removed route", "route", r)
 		}

overlay/tun_darwin_route_test.go

@@ -72,8 +72,8 @@ func TestTunRoutesBeforeActivate(t *testing.T) {
 	}}
 	tn.Routes.Store(&routes)
 
-	assert.NoError(t, tn.addRoutes(true))
-	assert.NoError(t, tn.removeRoutes(routes))
+	assert.NoError(t, tn.addRoutes(true, nil))
+	assert.NoError(t, tn.removeRoutes(routes, nil))
 }
 
 // A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that
@@ -96,7 +96,9 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
 	t.Cleanup(func() { _ = tn.Close() })
 	require.NoError(t, tn.Activate())
 
-	require.NoError(t, c.ReloadConfigString("tun: {}\n"))
+	res := c.ReloadConfigString("tun: {}\n")
+	require.NoError(t, res.Err)
+	assert.False(t, res.Failed, "removing a route we never installed failed the reload")
 	msgs, err := fetchRoutes()
 	require.NoError(t, err)
 	assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))

overlay/tun_freebsd.go

@@ -385,16 +385,18 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 		},
 	}
 
-	err = t.reload(c, true)
+	err = t.reload(c, true, nil)
 	if err != nil {
 		return nil, err
 	}
 	closeOnErr = false
 
-	c.RegisterReloadCallback(func(c *config.C) {
-		err := t.reload(c, false)
+	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+		r.RestartRequiredIfChanged("tun.dev", "tun.mtu")
+		err := t.reload(c, false, r)
 		if err != nil {
 			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+			r.Failed()
 		}
 	})
 
@@ -493,7 +495,7 @@ func (t *tun) Activate() error {
 		}
 	}
 
-	return t.addRoutes(false)
+	return t.addRoutes(false, nil)
 }
 
 func (t *tun) setMTU() error {
@@ -509,7 +511,7 @@ func (t *tun) setMTU() error {
 	return err
 }
 
-func (t *tun) reload(c *config.C, initial bool) error {
+func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
 	change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
 	if err != nil {
 		return err
@@ -530,16 +532,18 @@ func (t *tun) reload(c *config.C, initial bool) error {
 
 	if !initial {
 		// Remove first, if the system removes a wanted route hopefully it will be re-added next
-		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r)
 		if err != nil {
 			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
+			r.Failed()
 		}
 
 		// Ensure any routes we actually want are installed
-		err = t.addRoutes(true)
+		err = t.addRoutes(true, r)
 		if err != nil {
 			// Catch any stray logs
 			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
+			r.Failed()
 		}
 	}
 
@@ -563,7 +567,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
 	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
 }
 
-func (t *tun) addRoutes(logErrors bool) error {
+func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
 	routes := *t.Routes.Load()
 	for _, r := range routes {
 		if len(r.Via) == 0 || !r.Install {
@@ -576,6 +580,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
+				rl.Failed()
 			} else {
 				return retErr
 			}
@@ -587,7 +592,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 	return nil
 }
 
-func (t *tun) removeRoutes(routes []Route) error {
+func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
 	for _, r := range routes {
 		if len(r.Via) == 0 || !r.Install {
 			// addRoutes never installs these
@@ -595,8 +600,12 @@ func (t *tun) removeRoutes(routes []Route) error {
 		}
 
 		err := delRoute(r.Cidr, t.linkAddr)
-		if err != nil {
+		if errors.Is(err, unix.ESRCH) {
+			// Already gone, which is what we wanted
+			t.l.Info("Route was already removed", "route", r)
+		} else if err != nil {
 			t.l.Error("Failed to remove route", "error", err, "route", r)
+			rl.Failed()
 		} else {
 			t.l.Info("Removed route", "route", r)
 		}

overlay/tun_ios.go

@@ -47,16 +47,17 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 		l:               l,
 	}
 
-	err := t.reload(c, true)
+	err := t.reload(c, true, nil)
 	if err != nil {
 		_ = file.Close()
 		return nil, err
 	}
 
-	c.RegisterReloadCallback(func(c *config.C) {
-		err := t.reload(c, false)
+	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+		err := t.reload(c, false, r)
 		if err != nil {
 			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+			r.Failed()
 		}
 	})
 
@@ -67,7 +68,7 @@ func (t *tun) Activate() error {
 	return nil
 }
 
-func (t *tun) reload(c *config.C, initial bool) error {
+func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
 	change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
 	if err != nil {
 		return err

overlay/tun_linux.go

@@ -185,6 +185,10 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
 	}
 
 	t.Device = name
+	// Only read when we create the device, an fd handed to us already has its name and offloads
+	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+		r.RestartRequiredIfChanged("tun.dev", "tun.use_offloads")
+	})
 
 	return t, nil
 }
@@ -228,22 +232,24 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
 		l:                         l,
 	}
 
-	if err = t.reload(c, true); err != nil {
+	if err = t.reload(c, true, nil); err != nil {
 		_ = t.Close()
 		return nil, err
 	}
 
-	c.RegisterReloadCallback(func(c *config.C) {
-		err := t.reload(c, false)
+	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+		r.RestartRequiredIfChanged("tun.tx_queue", "tun.use_system_route_table", "tun.use_system_route_table_buffer_size")
+		err := t.reload(c, false, r)
 		if err != nil {
 			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+			r.Failed()
 		}
 	})
 
 	return t, nil
 }
 
-func (t *tun) reload(c *config.C, initial bool) error {
+func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
 	routeChange, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
 	if err != nil {
 		return err
@@ -297,7 +303,7 @@ func (t *tun) reload(c *config.C, initial bool) error {
 
 	if !initial {
 		if oldMaxMTU != newMaxMTU {
-			t.setMTU()
+			t.setMTU(r)
 			t.l.Info("Set max MTU", "mtu", t.MaxMTU, "oldMTU", oldMaxMTU)
 		}
 
@@ -305,7 +311,8 @@ func (t *tun) reload(c *config.C, initial bool) error {
 			for i := range t.vpnNetworks {
 				err := t.setDefaultRoute(t.vpnNetworks[i])
 				if err != nil {
-					t.l.Warn(err.Error())
+					t.l.Error(err.Error())
+					r.Failed()
 				} else {
 					t.l.Info("Set default MTU", "mtu", t.DefaultMTU, "oldMTU", oldDefaultMTU)
 				}
@@ -313,13 +320,14 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		}
 
 		// Remove first, if the system removes a wanted route hopefully it will be re-added next
-		t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r)
 
 		// Ensure any routes we actually want are installed
-		err = t.addRoutes(true)
+		err = t.addRoutes(true, r)
 		if err != nil {
 			// This should never be called since addRoutes should log its own errors in a reload condition
 			util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l)
+			r.Failed()
 		}
 	}
 
@@ -474,7 +482,7 @@ func (t *tun) Activate() error {
 	t.deviceIndex = link.Attrs().Index
 
 	// Setup our default MTU
-	t.setMTU()
+	t.setMTU(nil)
 
 	// Set the transmit queue length
 	ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)}
@@ -506,7 +514,7 @@ func (t *tun) Activate() error {
 	}
 
 	// Set the routes
-	if err = t.addRoutes(false); err != nil {
+	if err = t.addRoutes(false, nil); err != nil {
 		return err
 	}
 
@@ -519,12 +527,13 @@ func (t *tun) Activate() error {
 	return nil
 }
 
-func (t *tun) setMTU() {
+func (t *tun) setMTU(r *config.Reload) {
 	// Set the MTU on the device
 	ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)}
 	if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
 		// This is currently a non fatal condition because the route table must have the MTU set appropriately as well
 		t.l.Error("Failed to set tun mtu", "error", err)
+		r.Failed()
 	}
 }
 
@@ -577,7 +586,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
 	return nil
 }
 
-func (t *tun) addRoutes(logErrors bool) error {
+func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
 	// Path routes
 	routes := *t.Routes.Load()
 	for _, r := range routes {
@@ -607,6 +616,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
+				rl.Failed()
 			} else {
 				return retErr
 			}
@@ -618,7 +628,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 	return nil
 }
 
-func (t *tun) removeRoutes(routes []Route) {
+func (t *tun) removeRoutes(routes []Route, rl *config.Reload) {
 	for _, r := range routes {
 		if !r.Install {
 			continue
@@ -642,8 +652,12 @@ func (t *tun) removeRoutes(routes []Route) {
 		}
 
 		err := netlink.RouteDel(&nr)
-		if err != nil {
+		if errors.Is(err, unix.ESRCH) {
+			// Already gone, which is what we wanted
+			t.l.Info("Route was already removed", "route", r)
+		} else if err != nil {
 			t.l.Error("Failed to remove route", "error", err, "route", r)
+			rl.Failed()
 		} else {
 			t.l.Info("Removed route", "route", r)
 		}

overlay/tun_netbsd.go

@@ -103,15 +103,17 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 		l:           l,
 	}
 
-	err = t.reload(c, true)
+	err = t.reload(c, true, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	c.RegisterReloadCallback(func(c *config.C) {
-		err := t.reload(c, false)
+	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+		r.RestartRequiredIfChanged("tun.dev", "tun.mtu")
+		err := t.reload(c, false, r)
 		if err != nil {
 			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+			r.Failed()
 		}
 	})
 
@@ -324,7 +326,7 @@ func (t *tun) Activate() error {
 		}
 	}
 
-	return t.addRoutes(false)
+	return t.addRoutes(false, nil)
 }
 
 func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
@@ -339,7 +341,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
 	return err
 }
 
-func (t *tun) reload(c *config.C, initial bool) error {
+func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
 	change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
 	if err != nil {
 		return err
@@ -360,16 +362,18 @@ func (t *tun) reload(c *config.C, initial bool) error {
 
 	if !initial {
 		// Remove first, if the system removes a wanted route hopefully it will be re-added next
-		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r)
 		if err != nil {
 			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
+			r.Failed()
 		}
 
 		// Ensure any routes we actually want are installed
-		err = t.addRoutes(true)
+		err = t.addRoutes(true, r)
 		if err != nil {
 			// Catch any stray logs
 			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
+			r.Failed()
 		}
 	}
 
@@ -393,7 +397,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
 	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
 }
 
-func (t *tun) addRoutes(logErrors bool) error {
+func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
 	routes := *t.Routes.Load()
 
 	for _, r := range routes {
@@ -407,6 +411,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
+				rl.Failed()
 			} else {
 				return retErr
 			}
@@ -418,7 +423,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 	return nil
 }
 
-func (t *tun) removeRoutes(routes []Route) error {
+func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
 	for _, r := range routes {
 		if len(r.Via) == 0 || !r.Install {
 			// addRoutes never installs these
@@ -426,8 +431,12 @@ func (t *tun) removeRoutes(routes []Route) error {
 		}
 
 		err := delRoute(r.Cidr, t.vpnNetworks)
-		if err != nil {
+		if errors.Is(err, unix.ESRCH) {
+			// Already gone, which is what we wanted
+			t.l.Info("Route was already removed", "route", r)
+		} else if err != nil {
 			t.l.Error("Failed to remove route", "error", err, "route", r)
+			rl.Failed()
 		} else {
 			t.l.Info("Removed route", "route", r)
 		}

overlay/tun_openbsd.go

@@ -94,15 +94,17 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 		l:           l,
 	}
 
-	err = t.reload(c, true)
+	err = t.reload(c, true, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	c.RegisterReloadCallback(func(c *config.C) {
-		err := t.reload(c, false)
+	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+		r.RestartRequiredIfChanged("tun.dev", "tun.mtu")
+		err := t.reload(c, false, r)
 		if err != nil {
 			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+			r.Failed()
 		}
 	})
 
@@ -303,7 +305,7 @@ func (t *tun) Activate() error {
 		}
 	}
 
-	return t.addRoutes(false)
+	return t.addRoutes(false, nil)
 }
 
 func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
@@ -318,7 +320,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
 	return err
 }
 
-func (t *tun) reload(c *config.C, initial bool) error {
+func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
 	change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
 	if err != nil {
 		return err
@@ -339,16 +341,18 @@ func (t *tun) reload(c *config.C, initial bool) error {
 
 	if !initial {
 		// Remove first, if the system removes a wanted route hopefully it will be re-added next
-		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r)
 		if err != nil {
 			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
+			r.Failed()
 		}
 
 		// Ensure any routes we actually want are installed
-		err = t.addRoutes(true)
+		err = t.addRoutes(true, r)
 		if err != nil {
 			// Catch any stray logs
 			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
+			r.Failed()
 		}
 	}
 
@@ -372,7 +376,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
 	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
 }
 
-func (t *tun) addRoutes(logErrors bool) error {
+func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
 	routes := *t.Routes.Load()
 
 	for _, r := range routes {
@@ -386,6 +390,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
+				rl.Failed()
 			} else {
 				return retErr
 			}
@@ -397,7 +402,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 	return nil
 }
 
-func (t *tun) removeRoutes(routes []Route) error {
+func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
 	for _, r := range routes {
 		if len(r.Via) == 0 || !r.Install {
 			// addRoutes never installs these
@@ -405,8 +410,12 @@ func (t *tun) removeRoutes(routes []Route) error {
 		}
 
 		err := delRoute(r.Cidr, t.vpnNetworks)
-		if err != nil {
+		if errors.Is(err, unix.ESRCH) {
+			// Already gone, which is what we wanted
+			t.l.Info("Route was already removed", "route", r)
+		} else if err != nil {
 			t.l.Error("Failed to remove route", "error", err, "route", r)
+			rl.Failed()
 		} else {
 			t.l.Info("Removed route", "route", r)
 		}

overlay/tun_reload_linux_test.go

@@ -4,6 +4,7 @@ package overlay
 
 import (
 	"net/netip"
+	"os"
 	"testing"
 
 	"github.com/slackhq/nebula/config"
@@ -25,10 +26,10 @@ func TestReloadMTUOnlyKeepsRoutes(t *testing.T) {
 		ioctlFd:          ^uintptr(0),
 		routesFromSystem: map[netip.Prefix]routing.Gateways{},
 	}
-	require.NoError(t, tn.reload(c, true))
+	require.NoError(t, tn.reload(c, true, nil))
 
-	require.NoError(t, c.ReloadConfigString("tun:\n  mtu: 1400\n"+routes))
-	_ = tn.reload(c, false)
+	require.NoError(t, c.ReloadConfigString("tun:\n  mtu: 1400\n"+routes).Err)
+	_ = tn.reload(c, false, nil)
 
 	got := *tn.Routes.Load()
 	require.Len(t, got, 1)
@@ -50,7 +51,57 @@ func TestReloadMaxMTUIsTheLargestRoute(t *testing.T) {
 		ioctlFd:          ^uintptr(0),
 		routesFromSystem: map[netip.Prefix]routing.Gateways{},
 	}
-	require.NoError(t, tn.reload(c, true))
+	require.NoError(t, tn.reload(c, true, nil))
 	assert.Equal(t, 1500, tn.MaxMTU)
 	assert.Equal(t, 1300, tn.DefaultMTU)
 }
+
+// There is no device behind this tun, so adding a route always fails. Removing one is refused without root, as root the
+// kernel answers that the route isn't there, which is what we wanted
+func TestReloadRouteFailures(t *testing.T) {
+	route := "tun:\n  mtu: 1300\n  unsafe_routes:\n    - route: 192.168.100.0/24\n      via: 10.1.0.1\n"
+	cases := []struct {
+		name      string
+		from, to  string
+		wantFails bool
+	}{
+		{"added", "tun:\n  mtu: 1300\n", route, true},
+		{"removed", route, "tun:\n  mtu: 1300\n", os.Geteuid() != 0},
+		{"unchanged", route, route + "punchy:\n  punch: true\n", false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			c := config.NewC(test.NewLogger())
+			require.NoError(t, c.LoadString(tc.from))
+			tn := &tun{
+				l:                test.NewLogger(),
+				vpnNetworks:      []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")},
+				ioctlFd:          ^uintptr(0),
+				routesFromSystem: map[netip.Prefix]routing.Gateways{},
+			}
+			require.NoError(t, tn.reload(c, true, nil))
+			c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+				require.NoError(t, tn.reload(c, false, r))
+			})
+
+			res := c.ReloadConfigString(tc.to)
+			require.NoError(t, res.Err)
+			assert.Equal(t, tc.wantFails, res.Failed)
+		})
+	}
+}
+
+// Keys only read when the device is created need a restart to apply
+func TestTunReloadRestartKeys(t *testing.T) {
+	c := config.NewC(test.NewLogger())
+	require.NoError(t, c.LoadString("tun:\n  dev: nebrk0\n"))
+	tn, err := NewDeviceFromConfig(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, 1)
+	if err != nil {
+		t.Skipf("can't create a tun device here: %v", err)
+	}
+	t.Cleanup(func() { _ = tn.Close() })
+
+	res := c.ReloadConfigString("tun:\n  disabled: true\n  dev: nebrk1\n  tx_queue: 100\n")
+	require.NoError(t, res.Err)
+	assert.ElementsMatch(t, []string{"tun.disabled", "tun.dev", "tun.tx_queue"}, res.RestartKeys)
+}

overlay/tun_test.go

@@ -0,0 +1,24 @@
+package overlay
+
+import (
+	"net/netip"
+	"testing"
+
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// tun.disabled and the disabled tun's settings are only read when the device is made
+func TestDisabledTunRestartKeys(t *testing.T) {
+	c := config.NewC(test.NewLogger())
+	require.NoError(t, c.LoadString("tun:\n  disabled: true\n"))
+	d, err := NewDeviceFromConfig(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, 1)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = d.Close() })
+
+	res := c.ReloadConfigString("tun:\n  disabled: false\n  tx_queue: 100\n")
+	require.NoError(t, res.Err)
+	assert.ElementsMatch(t, []string{"tun.disabled", "tun.tx_queue"}, res.RestartKeys)
+}

overlay/tun_windows.go

@@ -84,7 +84,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
 		l:               l,
 	}
 
-	err = t.reload(c, true)
+	err = t.reload(c, true, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -108,17 +108,20 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
 	// Disable registration before Activate assigns addresses and triggers it.
 	disableDNSRegistration(l, *guid)
 
-	c.RegisterReloadCallback(func(c *config.C) {
-		err := t.reload(c, false)
+	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
+		r.RestartRequiredIfChanged("tun.dev", "tun.mtu", "tun.network_category")
+		r.RestartRequiredIfChanged(interfaceBypassKeys...)
+		err := t.reload(c, false, r)
 		if err != nil {
 			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+			r.Failed()
 		}
 	})
 
 	return t, nil
 }
 
-func (t *winTun) reload(c *config.C, initial bool) error {
+func (t *winTun) reload(c *config.C, initial bool, r *config.Reload) error {
 	change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
 	if err != nil {
 		return err
@@ -139,16 +142,18 @@ func (t *winTun) reload(c *config.C, initial bool) error {
 
 	if !initial {
 		// Remove first, if the system removes a wanted route hopefully it will be re-added next
-		err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes))
+		err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes), r)
 		if err != nil {
 			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
+			r.Failed()
 		}
 
 		// Ensure any routes we actually want are installed
-		err = t.addRoutes(true)
+		err = t.addRoutes(true, r)
 		if err != nil {
 			// Catch any stray logs
 			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
+			r.Failed()
 		}
 	}
 
@@ -163,7 +168,7 @@ func (t *winTun) Activate() error {
 		return fmt.Errorf("failed to set address: %w", err)
 	}
 
-	err = t.addRoutes(false)
+	err = t.addRoutes(false, nil)
 	if err != nil {
 		return err
 	}
@@ -182,7 +187,7 @@ func (t *winTun) Activate() error {
 	return nil
 }
 
-func (t *winTun) addRoutes(logErrors bool) error {
+func (t *winTun) addRoutes(logErrors bool, rl *config.Reload) error {
 	luid := winipcfg.LUID(t.tun.LUID())
 	routes := *t.Routes.Load()
 	foundDefault4 := false
@@ -213,6 +218,7 @@ func (t *winTun) addRoutes(logErrors bool) error {
 			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
+				rl.Failed()
 				continue
 			} else {
 				return retErr
@@ -274,7 +280,7 @@ func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error
 	return nil
 }
 
-func (t *winTun) removeRoutes(routes []Route) error {
+func (t *winTun) removeRoutes(routes []Route, rl *config.Reload) error {
 	luid := winipcfg.LUID(t.tun.LUID())
 
 	for _, r := range routes {
@@ -284,8 +290,12 @@ func (t *winTun) removeRoutes(routes []Route) error {
 
 		// See comment on luid.AddRoute
 		err := luid.DeleteRoute(r.Cidr, unspecifiedNextHop(r.Cidr))
-		if err != nil {
+		if errors.Is(err, windows.ERROR_NOT_FOUND) {
+			// Already gone, which is what we wanted
+			t.l.Info("Route was already removed", "route", r)
+		} else if err != nil {
 			t.l.Error("Failed to remove route", "error", err, "route", r)
+			rl.Failed()
 		} else {
 			t.l.Info("Removed route", "route", r)
 		}

overlay/tun_windows_route_test.go

@@ -53,6 +53,14 @@ func (rt *routeTest) metric(t *testing.T, cidr string) uint32 {
 	return row.Metric
 }
 
+// reload applies cfg and expects nothing in it to fail
+func (rt *routeTest) reload(t *testing.T, cfg string) {
+	t.Helper()
+	res := rt.c.ReloadConfigString(cfg)
+	require.NoError(t, res.Err)
+	assert.False(t, res.Failed, "the reload reported a failure")
+}
+
 func unsafeRoutes(routes ...string) string {
 	var b strings.Builder
 	b.WriteString("tun:\n  dev: nebroutetest\n  unsafe_routes:\n")
@@ -77,7 +85,7 @@ func TestWinTun_ReloadKeepsInstalledRoutes(t *testing.T) {
 		rt := newRouteTest(t, unsafeRoutes(a))
 		require.NoError(t, rt.tun.Activate())
 
-		require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a, b)))
+		rt.reload(t, unsafeRoutes(a, b))
 		assert.NotContains(t, rt.log.String(), "Failed to add route")
 		rt.metric(t, a)
 		rt.metric(t, b)
@@ -92,7 +100,7 @@ func TestWinTun_ReloadMetricChangeApplies(t *testing.T) {
 		require.NoError(t, rt.tun.Activate())
 		require.Equal(t, uint32(100), rt.metric(t, a))
 
-		require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200")))
+		rt.reload(t, unsafeRoutes(a+"@200"))
 		assert.NotContains(t, rt.log.String(), "Failed to add route")
 		assert.NotContains(t, rt.log.String(), "Removed route")
 		assert.Equal(t, uint32(200), rt.metric(t, a))
@@ -120,7 +128,7 @@ func TestWinTun_DuplicateCidrInstallsTheLastEntry(t *testing.T) {
 		require.NoError(t, rt.tun.Activate())
 		assert.Equal(t, uint32(200), rt.metric(t, a))
 
-		require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@100", a+"@200", b)))
+		rt.reload(t, unsafeRoutes(a+"@100", a+"@200", b))
 		assert.Equal(t, uint32(200), rt.metric(t, a))
 		assert.NotContains(t, rt.log.String(), "Updated route metric")
 		assert.NotContains(t, rt.log.String(), "Failed to add route")
@@ -134,7 +142,7 @@ func TestWinTun_DroppingAnUninstalledDuplicateKeepsTheRoute(t *testing.T) {
 		rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200"))
 		require.NoError(t, rt.tun.Activate())
 
-		require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200")))
+		rt.reload(t, unsafeRoutes(a+"@200"))
 		assert.NotContains(t, rt.log.String(), "Removed route")
 		assert.Equal(t, uint32(200), rt.metric(t, a))
 	})
@@ -155,12 +163,24 @@ func TestWinTun_ReloadMatchesAFreshStart(t *testing.T) {
 
 		rt := newRouteTest(t, unsafeRoutes(a+"@100"))
 		require.NoError(t, rt.tun.Activate())
-		require.NoError(t, rt.c.ReloadConfigString(routes))
+		rt.reload(t, routes)
 		_, err = rt.luid.Route(cidr, unspecifiedNextHop(cidr))
 		assert.Error(t, err, "the reload left it installed")
 	})
 }
 
+// Removing a route that is already gone is what we wanted, not a failure
+func TestWinTun_ReloadRouteAlreadyGone(t *testing.T) {
+	forFamilies(t, func(t *testing.T, a, b string) {
+		rt := newRouteTest(t, unsafeRoutes(a, b))
+		require.NoError(t, rt.tun.Activate())
+		p := netip.MustParsePrefix(b)
+		require.NoError(t, rt.luid.DeleteRoute(p, unspecifiedNextHop(p)))
+
+		rt.reload(t, unsafeRoutes(a))
+	})
+}
+
 // forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4
 func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) {
 	t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") })