mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 19:56:52 +01:00
darwin: Add an unsafe_route once the route holding its prefix is removed (#1942)
This commit is contained in:
1 parent
8a4bd8b10f
commit
cacfa8fac4
4 files changed
+874
-84
No files matched your search
@@ -331,6 +331,9 @@ tun:
|
|||||||
# `mtu`: will default to tun mtu if this option is not specified
|
# `mtu`: will default to tun mtu if this option is not specified
|
||||||
# `metric`: will default to 0 if this option is not specified
|
# `metric`: will default to 0 if this option is not specified
|
||||||
# `install`: will default to true, controls whether this route is installed in the systems routing table.
|
# `install`: will default to true, controls whether this route is installed in the systems routing table.
|
||||||
|
# On macOS, a route whose destination another interface already routes, like the local network, is installed once
|
||||||
|
# that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another
|
||||||
|
# interface routes its destination and lets it go.
|
||||||
# This setting is reloadable.
|
# This setting is reloadable.
|
||||||
unsafe_routes:
|
unsafe_routes:
|
||||||
#- route: 172.16.1.0/24
|
#- route: 172.16.1.0/24
|
||||||
|
|||||||
+54
-84
@@ -3,7 +3,6 @@
|
|||||||
package overlay
|
package overlay
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"math/bits"
|
"math/bits"
|
||||||
@@ -12,6 +11,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
"time"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
"github.com/gaissmai/bart"
|
"github.com/gaissmai/bart"
|
||||||
@@ -35,6 +35,26 @@ type tun struct {
|
|||||||
// and routes for it. NEPacketTunnelProvider on darwin does this.
|
// and routes for it. NEPacketTunnelProvider on darwin does this.
|
||||||
hostOwned bool
|
hostOwned bool
|
||||||
l *slog.Logger
|
l *slog.Logger
|
||||||
|
|
||||||
|
// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
|
||||||
|
// fields below it
|
||||||
|
routesMu sync.Mutex
|
||||||
|
// routeStates is who held each installable unsafe_route prefix when we last looked
|
||||||
|
routeStates map[netip.Prefix]routeState
|
||||||
|
// routeWatch is the routing socket watchRoutes reads, Close closes it
|
||||||
|
routeWatch *os.File
|
||||||
|
// routeSweep reconciles the unsafe_routes every routeSweepEvery
|
||||||
|
routeSweep *time.Timer
|
||||||
|
routeSweepEvery time.Duration
|
||||||
|
closed bool
|
||||||
|
|
||||||
|
// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
|
||||||
|
// lock so the route watcher never waits on a reconcile
|
||||||
|
routeMoversMu sync.Mutex
|
||||||
|
routeMovers map[netip.Prefix]routeMover
|
||||||
|
|
||||||
|
// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
|
||||||
|
lastRouteEvent atomic.Int64
|
||||||
}
|
}
|
||||||
|
|
||||||
type ifReq struct {
|
type ifReq struct {
|
||||||
@@ -139,11 +159,12 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
|
|||||||
}
|
}
|
||||||
|
|
||||||
t := &tun{
|
t := &tun{
|
||||||
f: os.NewFile(uintptr(fd), ""),
|
f: os.NewFile(uintptr(fd), ""),
|
||||||
Device: name,
|
Device: name,
|
||||||
vpnNetworks: vpnNetworks,
|
vpnNetworks: vpnNetworks,
|
||||||
DefaultMTU: getMTU(l, c),
|
DefaultMTU: getMTU(l, c),
|
||||||
l: l,
|
routeSweepEvery: routeSweepInterval,
|
||||||
|
l: l,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = t.reload(c, true, nil)
|
err = t.reload(c, true, nil)
|
||||||
@@ -219,6 +240,16 @@ func utunNameFromFd(fd int) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) Close() error {
|
func (t *tun) Close() error {
|
||||||
|
t.routesMu.Lock()
|
||||||
|
t.closed = true
|
||||||
|
if t.routeWatch != nil {
|
||||||
|
_ = t.routeWatch.Close()
|
||||||
|
}
|
||||||
|
if t.routeSweep != nil {
|
||||||
|
t.routeSweep.Stop()
|
||||||
|
}
|
||||||
|
t.routesMu.Unlock()
|
||||||
|
|
||||||
if t.f != nil {
|
if t.f != nil {
|
||||||
return t.f.Close()
|
return t.f.Close()
|
||||||
}
|
}
|
||||||
@@ -287,8 +318,22 @@ func (t *tun) Activate() error {
|
|||||||
return fmt.Errorf("failed to run tun device: %s", err)
|
return fmt.Errorf("failed to run tun device: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Watch before adding so a route removed in between is still noticed
|
||||||
|
watch, err := openRouteSocket()
|
||||||
|
if err != nil {
|
||||||
|
t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Unsafe path routes
|
// Unsafe path routes
|
||||||
return t.addRoutes(false, nil)
|
if err = t.addRoutes(false, nil); err != nil {
|
||||||
|
if watch != nil {
|
||||||
|
_ = watch.Close()
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
t.startRouteWatch(watch)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
|
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
|
||||||
@@ -393,7 +438,8 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !initial && !change {
|
if !initial && !change {
|
||||||
return nil
|
// Any reload puts back the unsafe_routes someone deleted
|
||||||
|
return t.addRoutes(true, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
routeTree, err := makeRouteTree(t.l, routes, false)
|
routeTree, err := makeRouteTree(t.l, routes, false)
|
||||||
@@ -511,82 +557,6 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
|
|||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
|
|
||||||
// The route tree is still ours, the system routing table is not
|
|
||||||
if t.hostOwned {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Activate hasn't run yet, it will install the routes
|
|
||||||
linkAddr := t.linkAddr.Load()
|
|
||||||
if linkAddr == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
routes := *t.Routes.Load()
|
|
||||||
systemRoutes := sync.OnceValues(fetchRoutes)
|
|
||||||
|
|
||||||
for _, r := range routes {
|
|
||||||
if len(r.Via) == 0 || !r.Install {
|
|
||||||
// We don't allow route MTUs so only install routes with a via
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err := addRoute(r.Cidr, linkAddr)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, unix.EEXIST) {
|
|
||||||
// Every route we added already exists on a reload, only warn about someone else's
|
|
||||||
msgs, err := systemRoutes()
|
|
||||||
if err != nil {
|
|
||||||
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
|
|
||||||
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
|
|
||||||
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
|
||||||
if logErrors {
|
|
||||||
retErr.Log(t.l)
|
|
||||||
rl.Failed()
|
|
||||||
} else {
|
|
||||||
return retErr
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
t.l.Info("Added route", "route", r)
|
|
||||||
// A cidr listed twice would otherwise find our own route missing from an older fetch
|
|
||||||
systemRoutes = sync.OnceValues(fetchRoutes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
|
|
||||||
linkAddr := t.linkAddr.Load()
|
|
||||||
if t.hostOwned || linkAddr == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, r := range routes {
|
|
||||||
if len(r.Via) == 0 || !r.Install {
|
|
||||||
// addRoutes never installs these
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err := delRoute(r.Cidr, linkAddr)
|
|
||||||
if errors.Is(err, unix.ESRCH) {
|
|
||||||
// Already gone, which is what we wanted
|
|
||||||
t.l.Info("Route was already removed", "route", r)
|
|
||||||
} else if err != nil {
|
|
||||||
t.l.Error("Failed to remove route", "error", err, "route", r)
|
|
||||||
rl.Failed()
|
|
||||||
} else {
|
|
||||||
t.l.Info("Removed route", "route", r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
|
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
|
||||||
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -3,12 +3,17 @@
|
|||||||
package overlay
|
package overlay
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
"github.com/slackhq/nebula/routing"
|
"github.com/slackhq/nebula/routing"
|
||||||
@@ -104,6 +109,367 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
|
|||||||
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
|
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
|
||||||
|
const (
|
||||||
|
// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
|
||||||
|
kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
|
||||||
|
addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
|
||||||
|
// With no interface address, which x/net/route skips
|
||||||
|
interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRoutesMayHaveMoved(t *testing.T) {
|
||||||
|
cidr := netip.MustParsePrefix("192.168.1.0/24")
|
||||||
|
via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
|
||||||
|
routes := []Route{{Cidr: cidr, Via: via, Install: true}}
|
||||||
|
// A request like route(8) sends, the kernel echoes it to every routing socket
|
||||||
|
request := func(typ, flags int, dst, mask [4]byte) []byte {
|
||||||
|
m := &netroute.RouteMessage{
|
||||||
|
Version: unix.RTM_VERSION,
|
||||||
|
Type: typ,
|
||||||
|
Flags: flags,
|
||||||
|
ID: 1234,
|
||||||
|
Addrs: []netroute.Addr{
|
||||||
|
unix.RTAX_DST: &netroute.Inet4Addr{IP: dst},
|
||||||
|
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
|
||||||
|
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
b, err := m.Marshal()
|
||||||
|
require.NoError(t, err)
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
raw := func(s string) []byte {
|
||||||
|
b, err := hex.DecodeString(s)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
|
||||||
|
routesMayHaveMoved := func(b []byte, routes []Route) bool {
|
||||||
|
moved, _, _ := routesMayHaveMoved(b, routes)
|
||||||
|
return moved
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
|
||||||
|
assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
|
||||||
|
assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
|
||||||
|
|
||||||
|
failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
|
||||||
|
binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
|
||||||
|
assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
|
||||||
|
|
||||||
|
_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
|
||||||
|
require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
|
||||||
|
assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
|
||||||
|
neighbor := raw(kernelSubnetDelete)
|
||||||
|
binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
|
||||||
|
assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
|
||||||
|
|
||||||
|
assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
|
||||||
|
assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A request names its prefix and who made it, the kernel's own and interface changes name neither
|
||||||
|
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
|
||||||
|
cidr := netip.MustParsePrefix("192.168.1.0/24")
|
||||||
|
routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
|
||||||
|
m := &netroute.RouteMessage{
|
||||||
|
Version: unix.RTM_VERSION,
|
||||||
|
Type: unix.RTM_CHANGE,
|
||||||
|
Flags: unix.RTF_UP,
|
||||||
|
ID: 1234,
|
||||||
|
Addrs: []netroute.Addr{
|
||||||
|
unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
|
||||||
|
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
|
||||||
|
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
b, err := m.Marshal()
|
||||||
|
require.NoError(t, err)
|
||||||
|
moved, p, pid := routesMayHaveMoved(b, routes)
|
||||||
|
assert.True(t, moved)
|
||||||
|
assert.Equal(t, cidr, p)
|
||||||
|
assert.Equal(t, int32(1234), pid)
|
||||||
|
|
||||||
|
for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
|
||||||
|
b, err := hex.DecodeString(s)
|
||||||
|
require.NoError(t, err)
|
||||||
|
moved, p, pid := routesMayHaveMoved(b, routes)
|
||||||
|
assert.True(t, moved, name)
|
||||||
|
assert.False(t, p.IsValid(), name)
|
||||||
|
assert.Zero(t, pid, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessName(t *testing.T) {
|
||||||
|
assert.Equal(t, "launchd", processName(1))
|
||||||
|
self := filepath.Base(os.Args[0])
|
||||||
|
assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
|
||||||
|
assert.Empty(t, processName(-1))
|
||||||
|
}
|
||||||
|
|
||||||
|
// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
|
||||||
|
// the minute between sweeps when it isn't zero.
|
||||||
|
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
|
||||||
|
requireRoot(t)
|
||||||
|
|
||||||
|
cfg := "tun:\n unsafe_routes:\n"
|
||||||
|
for _, cidr := range cidrs {
|
||||||
|
via := "10.250.0.2"
|
||||||
|
if netip.MustParsePrefix(cidr).Addr().Is6() {
|
||||||
|
via = "fd00:250::2"
|
||||||
|
}
|
||||||
|
cfg += " - route: " + cidr + "\n via: " + via + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
l := test.NewLogger()
|
||||||
|
c := config.NewC(l)
|
||||||
|
require.NoError(t, c.LoadString(cfg))
|
||||||
|
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
if sweep != 0 {
|
||||||
|
tn.routeSweepEvery = sweep
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = tn.Close() })
|
||||||
|
require.NoError(t, tn.Activate())
|
||||||
|
return tn, c, cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireRoot(t *testing.T) {
|
||||||
|
if os.Geteuid() != 0 {
|
||||||
|
t.Skip("creating interfaces and adding routes needs root")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// routeVia is the index of the interface that routes cidr, 0 if none does
|
||||||
|
func routeVia(t *testing.T, cidr string) int {
|
||||||
|
msgs, err := fetchRoutes()
|
||||||
|
require.NoError(t, err)
|
||||||
|
return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
|
||||||
|
}
|
||||||
|
|
||||||
|
// routeStateOf is what tn made of cidr when it last looked
|
||||||
|
func routeStateOf(tn *tun, cidr string) routeState {
|
||||||
|
tn.routesMu.Lock()
|
||||||
|
defer tn.routesMu.Unlock()
|
||||||
|
return tn.routeStates[netip.MustParsePrefix(cidr)]
|
||||||
|
}
|
||||||
|
|
||||||
|
func routeArgs(action, cidr string, extra ...string) []string {
|
||||||
|
args := []string{"-q", "-n", action}
|
||||||
|
if netip.MustParsePrefix(cidr).Addr().Is6() {
|
||||||
|
args = append(args, "-inet6")
|
||||||
|
}
|
||||||
|
return append(append(args, "-net", cidr), extra...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func routeCmd(t *testing.T, action, cidr string, extra ...string) {
|
||||||
|
requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
|
||||||
|
}
|
||||||
|
|
||||||
|
// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
|
||||||
|
// says so
|
||||||
|
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
require.NoError(t, err, string(out))
|
||||||
|
require.NotContains(t, string(out), "writing to routing socket")
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanupRoute deletes cidr's route when the test ends, whoever holds it
|
||||||
|
func cleanupRoute(t *testing.T, cidr string) {
|
||||||
|
t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
|
||||||
|
}
|
||||||
|
|
||||||
|
// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
|
||||||
|
func routeThroughLo0(t *testing.T, cidr string) int {
|
||||||
|
requireRoot(t)
|
||||||
|
routeCmd(t, "add", cidr, "-interface", "lo0")
|
||||||
|
cleanupRoute(t, cidr)
|
||||||
|
lo0, err := net.InterfaceByName("lo0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
return lo0.Index
|
||||||
|
}
|
||||||
|
|
||||||
|
// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
|
||||||
|
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
|
||||||
|
for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
|
||||||
|
t.Run(cidr, func(t *testing.T) {
|
||||||
|
lo0 := routeThroughLo0(t, cidr)
|
||||||
|
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||||
|
require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
|
||||||
|
|
||||||
|
routeCmd(t, "delete", cidr)
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
|
||||||
|
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
|
||||||
|
cidr := "10.251.1.0/24"
|
||||||
|
lo0 := routeThroughLo0(t, cidr)
|
||||||
|
_, c, _ := newRouteTestTun(t, 0, cidr)
|
||||||
|
|
||||||
|
res := c.ReloadConfigString("tun: {}\n")
|
||||||
|
require.NoError(t, res.Err)
|
||||||
|
assert.False(t, res.Failed)
|
||||||
|
assert.Equal(t, lo0, routeVia(t, cidr))
|
||||||
|
}
|
||||||
|
|
||||||
|
// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
|
||||||
|
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
|
||||||
|
cidr := "10.251.3.0/24"
|
||||||
|
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||||
|
ours := tn.linkAddr.Load().Index
|
||||||
|
require.Equal(t, ours, routeVia(t, cidr))
|
||||||
|
|
||||||
|
cleanupRoute(t, cidr)
|
||||||
|
requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
|
||||||
|
lo0, err := net.InterfaceByName("lo0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
time.Sleep(2 * routeRetryDelay)
|
||||||
|
require.Equal(t, lo0.Index, routeVia(t, cidr))
|
||||||
|
|
||||||
|
routeCmd(t, "delete", cidr)
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
|
||||||
|
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
|
||||||
|
cidr := "10.251.4.0/24"
|
||||||
|
tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
|
||||||
|
ours := tn.linkAddr.Load().Index
|
||||||
|
require.Equal(t, ours, routeVia(t, cidr))
|
||||||
|
|
||||||
|
routeCmd(t, "delete", cidr)
|
||||||
|
assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
|
||||||
|
|
||||||
|
// A reload with nothing changed is how to ask for it back
|
||||||
|
res := c.ReloadConfigString(cfg)
|
||||||
|
require.NoError(t, res.Err)
|
||||||
|
assert.False(t, res.Failed)
|
||||||
|
assert.Equal(t, ours, routeVia(t, cidr))
|
||||||
|
}
|
||||||
|
|
||||||
|
// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
|
||||||
|
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
|
||||||
|
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
|
||||||
|
for name, tc := range map[string]struct {
|
||||||
|
cidr string
|
||||||
|
sweep time.Duration
|
||||||
|
misfile func(t *testing.T, tn *tun, cidr string)
|
||||||
|
}{
|
||||||
|
// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
|
||||||
|
"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
|
||||||
|
require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
}},
|
||||||
|
// The sweep that caps a table that never settles
|
||||||
|
"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
|
||||||
|
tn.reconcileRoutes()
|
||||||
|
require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
|
||||||
|
ours := tn.linkAddr.Load().Index
|
||||||
|
require.Equal(t, ours, routeVia(t, tc.cidr))
|
||||||
|
|
||||||
|
routeCmd(t, "delete", tc.cidr)
|
||||||
|
tc.misfile(t, tn, tc.cidr)
|
||||||
|
|
||||||
|
lo0 := routeThroughLo0(t, tc.cidr)
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
s := routeStateOf(tn, tc.cidr)
|
||||||
|
return s.kind == routeYielded && s.holder == lo0
|
||||||
|
}, 5*time.Second, 50*time.Millisecond)
|
||||||
|
|
||||||
|
routeCmd(t, "delete", tc.cidr)
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
|
||||||
|
// taken for a hand delete
|
||||||
|
func TestTunYieldsToARouteChangedAway(t *testing.T) {
|
||||||
|
for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
|
||||||
|
t.Run(cidr, func(t *testing.T) {
|
||||||
|
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||||
|
ours := tn.linkAddr.Load().Index
|
||||||
|
require.Equal(t, ours, routeVia(t, cidr))
|
||||||
|
|
||||||
|
cleanupRoute(t, cidr)
|
||||||
|
routeCmd(t, "change", cidr, "-interface", "lo0")
|
||||||
|
lo0, err := net.InterfaceByName("lo0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, lo0.Index, routeVia(t, cidr))
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
s := routeStateOf(tn, cidr)
|
||||||
|
return s.kind == routeYielded && s.holder == lo0.Index
|
||||||
|
}, 5*time.Second, 50*time.Millisecond)
|
||||||
|
|
||||||
|
routeCmd(t, "delete", cidr)
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
|
||||||
|
// ethernet configd routed the lan through
|
||||||
|
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
|
||||||
|
requireRoot(t)
|
||||||
|
var feth string
|
||||||
|
for i := 100; i < 200 && feth == ""; i++ {
|
||||||
|
if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
|
||||||
|
feth = fmt.Sprintf("feth%d", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
require.NotEmpty(t, feth)
|
||||||
|
require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
|
||||||
|
t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
|
||||||
|
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
|
||||||
|
|
||||||
|
// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
|
||||||
|
// refuses one named by the feth itself
|
||||||
|
cidr := "10.251.2.0/24"
|
||||||
|
routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
|
||||||
|
cleanupRoute(t, cidr)
|
||||||
|
ifc, err := net.InterfaceByName(feth)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||||
|
require.Equal(t, ifc.Index, routeVia(t, cidr))
|
||||||
|
|
||||||
|
// Losing the address first leaves the route, the reconcile finds it still held
|
||||||
|
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
|
||||||
|
time.Sleep(2 * routeRetryDelay)
|
||||||
|
require.Equal(t, ifc.Index, routeVia(t, cidr))
|
||||||
|
|
||||||
|
// Only RTM_IFINFO says the interface went, its route goes with no message of its own
|
||||||
|
require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
|
||||||
|
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
|
||||||
|
cidr := "10.251.5.0/24"
|
||||||
|
routeThroughLo0(t, cidr)
|
||||||
|
tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
|
||||||
|
|
||||||
|
tn.routesMu.Lock()
|
||||||
|
require.NoError(t, tn.routeWatch.Close())
|
||||||
|
tn.routesMu.Unlock()
|
||||||
|
|
||||||
|
routeCmd(t, "delete", cidr)
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
|
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
|
||||||
func TestTunDevNeedsRestart(t *testing.T) {
|
func TestTunDevNeedsRestart(t *testing.T) {
|
||||||
for name, want := range map[string]bool{
|
for name, want := range map[string]bool{
|
||||||
|
|||||||
@@ -0,0 +1,451 @@
|
|||||||
|
//go:build !ios && !e2e_testing
|
||||||
|
|
||||||
|
package overlay
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/netip"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
"unsafe"
|
||||||
|
|
||||||
|
"github.com/slackhq/nebula/config"
|
||||||
|
"github.com/slackhq/nebula/util"
|
||||||
|
netroute "golang.org/x/net/route"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
|
||||||
|
// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
|
||||||
|
// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
|
||||||
|
// or until another interface takes the prefix, after which ours follows theirs like any other.
|
||||||
|
|
||||||
|
const (
|
||||||
|
// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
|
||||||
|
// replaces a route by deleting and adding, looking in between would find the prefix free
|
||||||
|
routeRetryDelay = time.Second
|
||||||
|
|
||||||
|
// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
|
||||||
|
// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
|
||||||
|
// a message for each
|
||||||
|
routeSweepInterval = time.Minute
|
||||||
|
|
||||||
|
// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
|
||||||
|
routeSocketRcvBuf = 1 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
type routeStateKind int
|
||||||
|
|
||||||
|
const (
|
||||||
|
routeUnknown routeStateKind = iota
|
||||||
|
// routeInstalled is our route holding the prefix
|
||||||
|
routeInstalled
|
||||||
|
// routeYielded is another interface holding the prefix, ours is added once it lets go
|
||||||
|
routeYielded
|
||||||
|
// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
|
||||||
|
// another interface takes the prefix
|
||||||
|
routeRemoved
|
||||||
|
// routeFailed is adding our route failing for some other reason, it is retried
|
||||||
|
routeFailed
|
||||||
|
)
|
||||||
|
|
||||||
|
type routeState struct {
|
||||||
|
kind routeStateKind
|
||||||
|
// holder is the index of the interface holding the prefix when kind is routeYielded
|
||||||
|
holder int
|
||||||
|
}
|
||||||
|
|
||||||
|
// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
|
||||||
|
// is looked up as the message arrives, route(8) has exited by the time the table settles
|
||||||
|
type routeMover struct {
|
||||||
|
pid int32
|
||||||
|
name string
|
||||||
|
}
|
||||||
|
|
||||||
|
// logArgs appends m to a log line's args, when we know who it was
|
||||||
|
func (m routeMover) logArgs(args ...any) []any {
|
||||||
|
if m.pid == 0 {
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
args = append(args, "pid", m.pid)
|
||||||
|
if m.name != "" {
|
||||||
|
args = append(args, "process", m.name)
|
||||||
|
}
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
|
||||||
|
// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
|
||||||
|
func processName(pid int32) string {
|
||||||
|
kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
|
||||||
|
if err != nil || kp.Proc.P_pid != pid {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return unix.ByteSliceToString(kp.Proc.P_comm[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// installRoutes returns the routes we put in the system routing table, one per prefix
|
||||||
|
func installRoutes(routes []Route) []Route {
|
||||||
|
seen := make(map[netip.Prefix]struct{}, len(routes))
|
||||||
|
var out []Route
|
||||||
|
for _, r := range routes {
|
||||||
|
if len(r.Via) == 0 || !r.Install {
|
||||||
|
// We don't allow route MTUs so only install routes with a via
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
p := r.Cidr.Masked()
|
||||||
|
if _, ok := seen[p]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[p] = struct{}{}
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
|
||||||
|
// interface holds its prefix. It runs on start and on every reload.
|
||||||
|
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
|
||||||
|
// The route tree is still ours, the system routing table is not
|
||||||
|
if t.hostOwned {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routesMu.Lock()
|
||||||
|
defer t.routesMu.Unlock()
|
||||||
|
|
||||||
|
// Activate hasn't run yet, it will install the routes
|
||||||
|
linkAddr := t.linkAddr.Load()
|
||||||
|
if linkAddr == nil || t.closed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var msgs []netroute.Message
|
||||||
|
var msgsErr error
|
||||||
|
fetched := false
|
||||||
|
states := make(map[netip.Prefix]routeState)
|
||||||
|
for _, r := range installRoutes(*t.Routes.Load()) {
|
||||||
|
p := r.Cidr.Masked()
|
||||||
|
prev := t.routeStates[p]
|
||||||
|
|
||||||
|
err := addRoute(p, linkAddr)
|
||||||
|
if err == nil {
|
||||||
|
t.l.Info("Added route", "route", r)
|
||||||
|
states[p] = routeState{kind: routeInstalled}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !errors.Is(err, unix.EEXIST) {
|
||||||
|
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
||||||
|
if !logErrors {
|
||||||
|
return retErr
|
||||||
|
}
|
||||||
|
retErr.Log(t.l)
|
||||||
|
rl.Failed()
|
||||||
|
states[p] = routeState{kind: routeFailed}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every route we added already exists on a reload, only warn about someone else's
|
||||||
|
if !fetched {
|
||||||
|
msgs, msgsErr = fetchRoutes()
|
||||||
|
fetched = true
|
||||||
|
}
|
||||||
|
if msgsErr != nil {
|
||||||
|
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
|
||||||
|
states[p] = prev
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch holder := routeInterfaceIn(msgs, p); holder {
|
||||||
|
case linkAddr.Index:
|
||||||
|
states[p] = routeState{kind: routeInstalled}
|
||||||
|
case 0:
|
||||||
|
// Removed since the add, the watcher will see to it
|
||||||
|
states[p] = prev
|
||||||
|
default:
|
||||||
|
if prev.kind != routeYielded || prev.holder != holder {
|
||||||
|
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
|
||||||
|
}
|
||||||
|
states[p] = routeState{kind: routeYielded, holder: holder}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routeStates = states
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
|
||||||
|
if t.hostOwned {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routesMu.Lock()
|
||||||
|
defer t.routesMu.Unlock()
|
||||||
|
|
||||||
|
linkAddr := t.linkAddr.Load()
|
||||||
|
if linkAddr == nil || t.closed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
|
||||||
|
msgs, err := fetchRoutes()
|
||||||
|
if err != nil {
|
||||||
|
var left []netip.Prefix
|
||||||
|
for _, r := range installRoutes(routes) {
|
||||||
|
left = append(left, r.Cidr)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, r := range installRoutes(routes) {
|
||||||
|
p := r.Cidr.Masked()
|
||||||
|
// Whatever happens next, the prefix is no longer one we know about
|
||||||
|
delete(t.routeStates, p)
|
||||||
|
if routeInterfaceIn(msgs, p) != linkAddr.Index {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
err = delRoute(p, linkAddr)
|
||||||
|
if errors.Is(err, unix.ESRCH) {
|
||||||
|
// Already gone, which is what we wanted
|
||||||
|
t.l.Info("Route was already removed", "route", r)
|
||||||
|
} else if err != nil {
|
||||||
|
t.l.Error("Failed to remove route", "error", err, "route", r)
|
||||||
|
rl.Failed()
|
||||||
|
} else {
|
||||||
|
t.l.Info("Removed route", "route", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
|
||||||
|
// changed since it last looked, it runs every routeSweepEvery.
|
||||||
|
func (t *tun) reconcileRoutes() {
|
||||||
|
t.routesMu.Lock()
|
||||||
|
defer t.routesMu.Unlock()
|
||||||
|
|
||||||
|
linkAddr := t.linkAddr.Load()
|
||||||
|
if linkAddr == nil || t.closed {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routeMoversMu.Lock()
|
||||||
|
movers := t.routeMovers
|
||||||
|
t.routeMovers = nil
|
||||||
|
t.routeMoversMu.Unlock()
|
||||||
|
|
||||||
|
msgs, err := fetchRoutes()
|
||||||
|
if err != nil {
|
||||||
|
t.l.Error("Failed to read the system routing table", "error", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
states := make(map[netip.Prefix]routeState)
|
||||||
|
for _, r := range installRoutes(*t.Routes.Load()) {
|
||||||
|
p := r.Cidr.Masked()
|
||||||
|
prev := t.routeStates[p]
|
||||||
|
by := movers[p]
|
||||||
|
|
||||||
|
holder := routeInterfaceIn(msgs, p)
|
||||||
|
switch {
|
||||||
|
case holder == linkAddr.Index:
|
||||||
|
states[p] = routeState{kind: routeInstalled}
|
||||||
|
|
||||||
|
case holder != 0:
|
||||||
|
// Another interface taking a prefix we left alone puts ours back in line behind theirs
|
||||||
|
if prev.kind != routeYielded || prev.holder != holder {
|
||||||
|
t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
|
||||||
|
}
|
||||||
|
states[p] = routeState{kind: routeYielded, holder: holder}
|
||||||
|
|
||||||
|
case prev.kind == routeInstalled:
|
||||||
|
t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
|
||||||
|
states[p] = routeState{kind: routeRemoved}
|
||||||
|
|
||||||
|
case prev.kind == routeRemoved:
|
||||||
|
states[p] = prev
|
||||||
|
|
||||||
|
default:
|
||||||
|
err := addRoute(p, linkAddr)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
if prev.kind == routeYielded {
|
||||||
|
t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
|
||||||
|
} else {
|
||||||
|
t.l.Info("Added route", "route", r)
|
||||||
|
}
|
||||||
|
states[p] = routeState{kind: routeInstalled}
|
||||||
|
case errors.Is(err, unix.EEXIST):
|
||||||
|
// Taken since the fetch, the next look will see by whom
|
||||||
|
states[p] = prev
|
||||||
|
default:
|
||||||
|
if prev.kind != routeFailed {
|
||||||
|
util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
|
||||||
|
}
|
||||||
|
states[p] = routeState{kind: routeFailed}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.routeStates = states
|
||||||
|
}
|
||||||
|
|
||||||
|
// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
|
||||||
|
// routing table settles after anything that may have moved a prefix
|
||||||
|
func (t *tun) startRouteWatch(watch *os.File) {
|
||||||
|
t.routesMu.Lock()
|
||||||
|
defer t.routesMu.Unlock()
|
||||||
|
if t.closed {
|
||||||
|
if watch != nil {
|
||||||
|
_ = watch.Close()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
|
||||||
|
if watch != nil {
|
||||||
|
t.routeWatch = watch
|
||||||
|
go t.watchRoutes(watch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tun) sweepRoutes() {
|
||||||
|
// A table still changing gets reconciled once it settles
|
||||||
|
if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
|
||||||
|
t.reconcileRoutes()
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routesMu.Lock()
|
||||||
|
defer t.routesMu.Unlock()
|
||||||
|
if !t.closed {
|
||||||
|
t.routeSweep.Reset(t.routeSweepEvery)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func openRouteSocket() (*os.File, error) {
|
||||||
|
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
|
||||||
|
}
|
||||||
|
// Fewer drops, the sweep catches the rest
|
||||||
|
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
|
||||||
|
// Non-blocking puts it on the poller, so Close interrupts a Read
|
||||||
|
if err = unix.SetNonblock(fd, true); err != nil {
|
||||||
|
_ = unix.Close(fd)
|
||||||
|
return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
|
||||||
|
}
|
||||||
|
return os.NewFile(uintptr(fd), "route"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
|
||||||
|
// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
|
||||||
|
func (t *tun) watchRoutes(f *os.File) {
|
||||||
|
var reconciled atomic.Int64
|
||||||
|
settle := time.AfterFunc(time.Hour, func() {
|
||||||
|
reconciled.Store(time.Now().UnixNano())
|
||||||
|
t.reconcileRoutes()
|
||||||
|
})
|
||||||
|
settle.Stop()
|
||||||
|
defer settle.Stop()
|
||||||
|
|
||||||
|
var burst time.Time
|
||||||
|
self := int32(os.Getpid())
|
||||||
|
b := make([]byte, os.Getpagesize())
|
||||||
|
for {
|
||||||
|
n, err := f.Read(b)
|
||||||
|
if err != nil {
|
||||||
|
if !errors.Is(err, os.ErrClosed) {
|
||||||
|
t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
|
||||||
|
if !moved {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Our own are already in routeStates
|
||||||
|
if p.IsValid() && pid != self {
|
||||||
|
t.routeMoversMu.Lock()
|
||||||
|
if t.routeMovers == nil {
|
||||||
|
t.routeMovers = make(map[netip.Prefix]routeMover)
|
||||||
|
}
|
||||||
|
t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
|
||||||
|
t.routeMoversMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
t.lastRouteEvent.Store(now.UnixNano())
|
||||||
|
if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
|
||||||
|
burst = now
|
||||||
|
}
|
||||||
|
settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
|
||||||
|
// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
|
||||||
|
// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
|
||||||
|
// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
|
||||||
|
// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
|
||||||
|
// and the pid of the process that made it.
|
||||||
|
func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
|
||||||
|
if len(b) < 4 {
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
switch int(b[3]) {
|
||||||
|
case unix.RTM_IFINFO, unix.RTM_DELADDR:
|
||||||
|
return true, p, 0
|
||||||
|
case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
|
||||||
|
default:
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(b) < unix.SizeofRtMsghdr {
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
|
||||||
|
// A failed request is reported too, and a scoped route never held the prefix
|
||||||
|
if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
if hdr.Pid == 0 {
|
||||||
|
// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
|
||||||
|
// could be holding a prefix
|
||||||
|
return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Replies to a request carry the requester's padded sockaddrs
|
||||||
|
msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
|
||||||
|
if err != nil || len(msgs) != 1 {
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
m, ok := msgs[0].(*netroute.RouteMessage)
|
||||||
|
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
|
||||||
|
if !ok {
|
||||||
|
return false, netip.Prefix{}, 0
|
||||||
|
}
|
||||||
|
p = p.Masked()
|
||||||
|
for _, r := range installRoutes(routes) {
|
||||||
|
if r.Cidr.Masked() == p {
|
||||||
|
return true, p, hdr.Pid
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, netip.Prefix{}, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// interfaceName is the name of the interface with index i, for logs
|
||||||
|
func interfaceName(i int) string {
|
||||||
|
if ifc, err := net.InterfaceByIndex(i); err == nil {
|
||||||
|
return ifc.Name
|
||||||
|
}
|
||||||
|
return strconv.Itoa(i)
|
||||||
|
}
|
||||||
Reference in new issue
Block a user