claude first stab and first cleanup - ugh

overlay/tun_linux.go

@@ -1,5 +1,5 @@
-//go:build !android && !e2e_testing
-// +build !android,!e2e_testing
+//go:build linux && !android && !e2e_testing
+// +build linux,!android,!e2e_testing
 
 package overlay
 
@@ -9,133 +9,102 @@ import (
 	"net"
 	"net/netip"
 	"os"
-	"strings"
-	"sync/atomic"
 	"time"
 	"unsafe"
 
-	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
+	wgtun "golang.zx2c4.com/wireguard/tun"
 )
 
 type tun struct {
-	io.ReadWriteCloser
-	fd          int
-	Device      string
-	vpnNetworks []netip.Prefix
-	MaxMTU      int
-	DefaultMTU  int
-	TXQueueLen  int
-	deviceIndex int
-	ioctlFd     uintptr
-
-	Routes                    atomic.Pointer[[]Route]
-	routeTree                 atomic.Pointer[bart.Table[routing.Gateways]]
-	routeChan                 chan struct{}
+	deviceIndex               int
+	ioctlFd                   uintptr
 	useSystemRoutes           bool
 	useSystemRoutesBufferSize int
-
-	l *logrus.Logger
 }
 
-func (t *tun) Networks() []netip.Prefix {
-	return t.vpnNetworks
-}
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*wgTun, error) {
+	deviceName := c.GetString("tun.dev", "")
+	mtu := c.GetInt("tun.mtu", DefaultMTU)
 
-type ifReq struct {
-	Name  [16]byte
-	Flags uint16
-	pad   [8]byte
-}
-
-type ifreqMTU struct {
-	Name [16]byte
-	MTU  int32
-	pad  [8]byte
-}
-
-type ifreqQLEN struct {
-	Name  [16]byte
-	Value int32
-	pad   [8]byte
-}
-
-func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
-	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
-
-	t, err := newTunGeneric(c, l, file, vpnNetworks)
+	// Create WireGuard TUN device
+	tunDevice, err := wgtun.CreateTUN(deviceName, mtu)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to create TUN device: %w", err)
 	}
 
-	t.Device = "tun0"
-
-	return t, nil
-}
-
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
-	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
+	// Get the actual device name
+	actualName, err := tunDevice.Name()
 	if err != nil {
-		// If /dev/net/tun doesn't exist, try to create it (will happen in docker)
-		if os.IsNotExist(err) {
-			err = os.MkdirAll("/dev/net", 0755)
-			if err != nil {
-				return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
-			}
-			err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200)))
-			if err != nil {
-				return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err)
-			}
-
-			fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0)
-			if err != nil {
-				return nil, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
-			}
-		} else {
-			return nil, err
-		}
+		tunDevice.Close()
+		return nil, fmt.Errorf("failed to get TUN device name: %w", err)
 	}
 
-	var req ifReq
-	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
-	if multiqueue {
-		req.Flags |= unix.IFF_MULTI_QUEUE
-	}
-	copy(req.Name[:], c.GetString("tun.dev", ""))
-	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
-		return nil, err
-	}
-	name := strings.Trim(string(req.Name[:]), "\x00")
-
-	file := os.NewFile(uintptr(fd), "/dev/net/tun")
-	t, err := newTunGeneric(c, l, file, vpnNetworks)
-	if err != nil {
-		return nil, err
+	t := &wgTun{
+		tunDevice:   tunDevice,
+		vpnNetworks: vpnNetworks,
+		MaxMTU:      mtu,
+		DefaultMTU:  mtu,
+		l:           l,
 	}
 
-	t.Device = name
-
-	return t, nil
-}
-
-func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []netip.Prefix) (*tun, error) {
-	t := &tun{
-		ReadWriteCloser:           file,
-		fd:                        int(file.Fd()),
-		vpnNetworks:               vpnNetworks,
-		TXQueueLen:                c.GetInt("tun.tx_queue", 500),
+	// Create Linux-specific route manager
+	routeManager := &tun{
 		useSystemRoutes:           c.GetBool("tun.use_system_route_table", false),
 		useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
-		l:                         l,
+	}
+	t.routeManager = routeManager
+
+	err = t.reload(c, true)
+	if err != nil {
+		tunDevice.Close()
+		return nil, err
 	}
 
-	err := t.reload(c, true)
+	c.RegisterReloadCallback(func(c *config.C) {
+		err := t.reload(c, false)
+		if err != nil {
+			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+		}
+	})
+
+	l.WithField("name", actualName).Info("Created WireGuard TUN device")
+
+	return t, nil
+}
+
+func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*wgTun, error) {
+	// Create TUN device from file descriptor
+	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
+	mtu := c.GetInt("tun.mtu", DefaultMTU)
+	tunDevice, err := wgtun.CreateTUNFromFile(file, mtu)
 	if err != nil {
+		return nil, fmt.Errorf("failed to create TUN device from fd: %w", err)
+	}
+
+	t := &wgTun{
+		tunDevice:   tunDevice,
+		vpnNetworks: vpnNetworks,
+		MaxMTU:      mtu,
+		DefaultMTU:  mtu,
+		l:           l,
+	}
+
+	// Create Linux-specific route manager
+	routeManager := &tun{
+		useSystemRoutes:           c.GetBool("tun.use_system_route_table", false),
+		useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
+	}
+	t.routeManager = routeManager
+
+	err = t.reload(c, true)
+	if err != nil {
+		tunDevice.Close()
 		return nil, err
 	}
 
@@ -149,273 +118,106 @@ func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []n
 	return t, nil
 }
 
-func (t *tun) reload(c *config.C, initial bool) error {
-	routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
+func (rm *tun) Activate(t *wgTun) error {
+	name, err := t.tunDevice.Name()
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to get device name: %w", err)
 	}
 
-	if !initial && !routeChange && !c.HasChanged("tun.mtu") {
-		return nil
-	}
-
-	routeTree, err := makeRouteTree(t.l, routes, true)
-	if err != nil {
-		return err
-	}
-
-	oldDefaultMTU := t.DefaultMTU
-	oldMaxMTU := t.MaxMTU
-	newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU)
-	newMaxMTU := newDefaultMTU
-	for i, r := range routes {
-		if r.MTU == 0 {
-			routes[i].MTU = newDefaultMTU
-		}
-
-		if r.MTU > t.MaxMTU {
-			newMaxMTU = r.MTU
-		}
-	}
-
-	t.MaxMTU = newMaxMTU
-	t.DefaultMTU = newDefaultMTU
-
-	// Teach nebula how to handle the routes before establishing them in the system table
-	oldRoutes := t.Routes.Swap(&routes)
-	t.routeTree.Store(routeTree)
-
-	if !initial {
-		if oldMaxMTU != newMaxMTU {
-			t.setMTU()
-			t.l.Infof("Set max MTU to %v was %v", t.MaxMTU, oldMaxMTU)
-		}
-
-		if oldDefaultMTU != newDefaultMTU {
-			for i := range t.vpnNetworks {
-				err := t.setDefaultRoute(t.vpnNetworks[i])
-				if err != nil {
-					t.l.Warn(err)
-				} else {
-					t.l.Infof("Set default MTU to %v was %v", t.DefaultMTU, oldDefaultMTU)
-				}
-			}
-		}
-
-		// Remove first, if the system removes a wanted route hopefully it will be re-added next
-		t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
-
-		// Ensure any routes we actually want are installed
-		err = t.addRoutes(true)
-		if err != nil {
-			// This should never be called since addRoutes should log its own errors in a reload condition
-			util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l)
-		}
-	}
-
-	return nil
-}
-
-func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
-	if err != nil {
-		return nil, err
-	}
-
-	var req ifReq
-	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
-	copy(req.Name[:], t.Device)
-	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
-		return nil, err
-	}
-
-	file := os.NewFile(uintptr(fd), "/dev/net/tun")
-
-	return file, nil
-}
-
-func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
-	r, _ := t.routeTree.Load().Lookup(ip)
-	return r
-}
-
-func (t *tun) Write(b []byte) (int, error) {
-	var nn int
-	maximum := len(b)
-
-	for {
-		n, err := unix.Write(t.fd, b[nn:maximum])
-		if n > 0 {
-			nn += n
-		}
-		if nn == len(b) {
-			return nn, err
-		}
-
-		if err != nil {
-			return nn, err
-		}
-
-		if n == 0 {
-			return nn, io.ErrUnexpectedEOF
-		}
-	}
-}
-
-func (t *tun) deviceBytes() (o [16]byte) {
-	for i, c := range t.Device {
-		o[i] = byte(c)
-	}
-	return
-}
-
-func hasNetlinkAddr(al []*netlink.Addr, x netlink.Addr) bool {
-	for i := range al {
-		if al[i].Equal(x) {
-			return true
-		}
-	}
-	return false
-}
-
-// addIPs uses netlink to add all addresses that don't exist, then it removes ones that should not be there
-func (t *tun) addIPs(link netlink.Link) error {
-	newAddrs := make([]*netlink.Addr, len(t.vpnNetworks))
-	for i := range t.vpnNetworks {
-		newAddrs[i] = &netlink.Addr{
-			IPNet: &net.IPNet{
-				IP:   t.vpnNetworks[i].Addr().AsSlice(),
-				Mask: net.CIDRMask(t.vpnNetworks[i].Bits(), t.vpnNetworks[i].Addr().BitLen()),
-			},
-			Label: t.vpnNetworks[i].Addr().Zone(),
-		}
-	}
-
-	//add all new addresses
-	for i := range newAddrs {
-		//AddrReplace still adds new IPs, but if their properties change it will change them as well
-		if err := netlink.AddrReplace(link, newAddrs[i]); err != nil {
-			return err
-		}
-	}
-
-	//iterate over remainder, remove whoever shouldn't be there
-	al, err := netlink.AddrList(link, netlink.FAMILY_ALL)
-	if err != nil {
-		return fmt.Errorf("failed to get tun address list: %s", err)
-	}
-
-	for i := range al {
-		if hasNetlinkAddr(newAddrs, al[i]) {
-			continue
-		}
-		err = netlink.AddrDel(link, &al[i])
-		if err != nil {
-			t.l.WithError(err).Error("failed to remove address from tun address list")
-		} else {
-			t.l.WithField("removed", al[i].String()).Info("removed address not listed in cert(s)")
-		}
-	}
-
-	return nil
-}
-
-func (t *tun) Activate() error {
-	devName := t.deviceBytes()
-
-	if t.useSystemRoutes {
+	if t.routeManager.useSystemRoutes {
 		t.watchRoutes()
 	}
 
+	// Get the netlink device
+	link, err := netlink.LinkByName(name)
+	if err != nil {
+		return fmt.Errorf("failed to get tun device link: %s", err)
+	}
+
+	rm.deviceIndex = link.Attrs().Index
+
+	// Open socket for ioctl operations
 	s, err := unix.Socket(
-		unix.AF_INET, //because everything we use t.ioctlFd for is address family independent, this is fine
+		unix.AF_INET,
 		unix.SOCK_DGRAM,
 		unix.IPPROTO_IP,
 	)
 	if err != nil {
 		return err
 	}
-	t.ioctlFd = uintptr(s)
+	rm.ioctlFd = uintptr(s)
 
-	// Set the device name
-	ifrf := ifReq{Name: devName}
-	if err = ioctl(t.ioctlFd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
-		return fmt.Errorf("failed to set tun device name: %s", err)
-	}
-
-	link, err := netlink.LinkByName(t.Device)
-	if err != nil {
-		return fmt.Errorf("failed to get tun device link: %s", err)
-	}
-
-	t.deviceIndex = link.Attrs().Index
-
-	// Setup our default MTU
-	t.setMTU()
+	// Set the MTU
+	rm.SetMTU(t, t.MaxMTU)
 
 	// Set the transmit queue length
-	ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)}
-	if err = ioctl(t.ioctlFd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil {
-		// If we can't set the queue length nebula will still work but it may lead to packet loss
+	txQueueLen := 500 // default
+	devName := deviceBytes(name)
+	ifrq := ifreqQLEN{Name: devName, Value: int32(txQueueLen)}
+	if err = ioctl(t.routeManager.ioctlFd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil {
 		t.l.WithError(err).Error("Failed to set tun tx queue length")
 	}
 
+	// Disable IPv6 link-local address generation
 	const modeNone = 1
 	if err = netlink.LinkSetIP6AddrGenMode(link, modeNone); err != nil {
 		t.l.WithError(err).Warn("Failed to disable link local address generation")
 	}
 
-	if err = t.addIPs(link); err != nil {
+	// Add IP addresses
+	if err = t.routeManager.addIPs(t, link); err != nil {
 		return err
 	}
 
 	// Bring up the interface
-	ifrf.Flags = ifrf.Flags | unix.IFF_UP
-	if err = ioctl(t.ioctlFd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
+	if err = netlink.LinkSetUp(link); err != nil {
 		return fmt.Errorf("failed to bring the tun device up: %s", err)
 	}
 
-	//set route MTU
+	// Set route MTU
 	for i := range t.vpnNetworks {
-		if err = t.setDefaultRoute(t.vpnNetworks[i]); err != nil {
+		if err = t.routeManager.SetDefaultRoute(t, t.vpnNetworks[i]); err != nil {
 			return fmt.Errorf("failed to set default route MTU: %w", err)
 		}
 	}
 
 	// Set the routes
-	if err = t.addRoutes(false); err != nil {
+	if err = t.routeManager.AddRoutes(t, false); err != nil {
 		return err
 	}
 
-	// Run the interface
-	ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
-	if err = ioctl(t.ioctlFd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
-		return fmt.Errorf("failed to run tun device: %s", err)
-	}
-
 	return nil
 }
 
-func (t *tun) setMTU() {
-	// Set the MTU on the device
-	ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)}
-	if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
-		// This is currently a non fatal condition because the route table must have the MTU set appropriately as well
+func (rm *tun) SetMTU(t *wgTun, mtu int) {
+	name, err := t.tunDevice.Name()
+	if err != nil {
+		t.l.WithError(err).Error("Failed to get device name for MTU set")
+		return
+	}
+
+	link, err := netlink.LinkByName(name)
+	if err != nil {
+		t.l.WithError(err).Error("Failed to get link for MTU set")
+		return
+	}
+
+	if err := netlink.LinkSetMTU(link, mtu); err != nil {
 		t.l.WithError(err).Error("Failed to set tun mtu")
 	}
 }
 
-func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
+func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error {
 	dr := &net.IPNet{
 		IP:   cidr.Masked().Addr().AsSlice(),
 		Mask: net.CIDRMask(cidr.Bits(), cidr.Addr().BitLen()),
 	}
 
 	nr := netlink.Route{
-		LinkIndex: t.deviceIndex,
+		LinkIndex: t.routeManager.deviceIndex,
 		Dst:       dr,
 		MTU:       t.DefaultMTU,
-		AdvMSS:    t.advMSS(Route{}),
+		AdvMSS:    advMSS(Route{}, t.DefaultMTU, t.MaxMTU),
 		Scope:     unix.RT_SCOPE_LINK,
 		Src:       net.IP(cidr.Addr().AsSlice()),
 		Protocol:  unix.RTPROT_KERNEL,
@@ -425,7 +227,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
 	err := netlink.RouteReplace(&nr)
 	if err != nil {
 		t.l.WithError(err).WithField("cidr", cidr).Warn("Failed to set default route MTU, retrying")
-		//retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument`
+		// Retry twice more
 		for i := 0; i < 2; i++ {
 			time.Sleep(100 * time.Millisecond)
 			err = netlink.RouteReplace(&nr)
@@ -443,8 +245,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
 	return nil
 }
 
-func (t *tun) addRoutes(logErrors bool) error {
-	// Path routes
+func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error {
 	routes := *t.Routes.Load()
 	for _, r := range routes {
 		if !r.Install {
@@ -457,10 +258,10 @@ func (t *tun) addRoutes(logErrors bool) error {
 		}
 
 		nr := netlink.Route{
-			LinkIndex: t.deviceIndex,
+			LinkIndex: t.routeManager.deviceIndex,
 			Dst:       dr,
 			MTU:       r.MTU,
-			AdvMSS:    t.advMSS(r),
+			AdvMSS:    advMSS(r, t.DefaultMTU, t.MaxMTU),
 			Scope:     unix.RT_SCOPE_LINK,
 		}
 
@@ -484,7 +285,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 	return nil
 }
 
-func (t *tun) removeRoutes(routes []Route) {
+func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) {
 	for _, r := range routes {
 		if !r.Install {
 			continue
@@ -496,10 +297,10 @@ func (t *tun) removeRoutes(routes []Route) {
 		}
 
 		nr := netlink.Route{
-			LinkIndex: t.deviceIndex,
+			LinkIndex: t.routeManager.deviceIndex,
 			Dst:       dr,
 			MTU:       r.MTU,
-			AdvMSS:    t.advMSS(r),
+			AdvMSS:    advMSS(r, t.DefaultMTU, t.MaxMTU),
 			Scope:     unix.RT_SCOPE_LINK,
 		}
 
@@ -516,30 +317,108 @@ func (t *tun) removeRoutes(routes []Route) {
 	}
 }
 
-func (t *tun) Name() string {
-	return t.Device
+func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) {
+	// For Linux with WireGuard TUN, we can reuse the same device
+	// The vectorized I/O will handle batching
+	return &wgTunReader{
+		parent:    t,
+		tunDevice: t.tunDevice,
+		batchSize: 64, // Default batch size
+		offset:    0,
+		l:         t.l,
+	}, nil
 }
 
-func (t *tun) advMSS(r Route) int {
+// Helper functions
+
+func deviceBytes(name string) [16]byte {
+	var o [16]byte
+	for i, c := range name {
+		if i >= 16 {
+			break
+		}
+		o[i] = byte(c)
+	}
+	return o
+}
+
+func advMSS(r Route, defaultMTU, maxMTU int) int {
 	mtu := r.MTU
 	if r.MTU == 0 {
-		mtu = t.DefaultMTU
+		mtu = defaultMTU
 	}
 
 	// We only need to set advmss if the route MTU does not match the device MTU
-	if mtu != t.MaxMTU {
+	if mtu != maxMTU {
 		return mtu - 40
 	}
 	return 0
 }
 
-func (t *tun) watchRoutes() {
+type ifreqQLEN struct {
+	Name  [16]byte
+	Value int32
+	pad   [8]byte
+}
+
+func hasNetlinkAddr(al []*netlink.Addr, x netlink.Addr) bool {
+	for i := range al {
+		if al[i].Equal(x) {
+			return true
+		}
+	}
+	return false
+}
+
+func (rm *tun) addIPs(t *wgTun, link netlink.Link) error {
+	newAddrs := make([]*netlink.Addr, len(t.vpnNetworks))
+	for i := range t.vpnNetworks {
+		newAddrs[i] = &netlink.Addr{
+			IPNet: &net.IPNet{
+				IP:   t.vpnNetworks[i].Addr().AsSlice(),
+				Mask: net.CIDRMask(t.vpnNetworks[i].Bits(), t.vpnNetworks[i].Addr().BitLen()),
+			},
+			Label: t.vpnNetworks[i].Addr().Zone(),
+		}
+	}
+
+	// Add all new addresses
+	for i := range newAddrs {
+		if err := netlink.AddrReplace(link, newAddrs[i]); err != nil {
+			return err
+		}
+	}
+
+	// Iterate over remainder, remove whoever shouldn't be there
+	al, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+	if err != nil {
+		return fmt.Errorf("failed to get tun address list: %s", err)
+	}
+
+	for i := range al {
+		if hasNetlinkAddr(newAddrs, al[i]) {
+			continue
+		}
+		err = netlink.AddrDel(link, &al[i])
+		if err != nil {
+			t.l.WithError(err).Error("failed to remove address from tun address list")
+		} else {
+			t.l.WithField("removed", al[i].String()).Info("removed address not listed in cert(s)")
+		}
+	}
+
+	return nil
+}
+
+// watchRoutes monitors system route changes
+func (t *wgTun) watchRoutes() {
+
 	rch := make(chan netlink.RouteUpdate)
 	doneChan := make(chan struct{})
 
 	netlinkOptions := netlink.RouteSubscribeOptions{
-		ReceiveBufferSize:      t.useSystemRoutesBufferSize,
-		ReceiveBufferForceSize: t.useSystemRoutesBufferSize != 0,
+		ReceiveBufferSize:      t.routeManager.useSystemRoutesBufferSize,
+		ReceiveBufferForceSize: t.routeManager.useSystemRoutesBufferSize != 0,
 		ErrorCallback:          func(e error) { t.l.WithError(e).Errorf("netlink error") },
 	}
 
@@ -557,87 +436,19 @@ func (t *tun) watchRoutes() {
 				if ok {
 					t.updateRoutes(r)
 				} else {
-					// may be should do something here as
-					// netlink stops sending updates
 					return
 				}
 			case <-doneChan:
-				// netlink.RouteSubscriber will close the rch for us
 				return
 			}
 		}
 	}()
 }
 
-func (t *tun) isGatewayInVpnNetworks(gwAddr netip.Addr) bool {
-	withinNetworks := false
-	for i := range t.vpnNetworks {
-		if t.vpnNetworks[i].Contains(gwAddr) {
-			withinNetworks = true
-			break
-		}
-	}
-
-	return withinNetworks
-}
-
-func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways {
-
-	var gateways routing.Gateways
-
-	link, err := netlink.LinkByName(t.Device)
-	if err != nil {
-		t.l.WithField("Devicename", t.Device).Error("Ignoring route update: failed to get link by name")
-		return gateways
-	}
-
-	// If this route is relevant to our interface and there is a gateway then add it
-	if r.LinkIndex == link.Attrs().Index && len(r.Gw) > 0 {
-		gwAddr, ok := netip.AddrFromSlice(r.Gw)
-		if !ok {
-			t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway address")
-		} else {
-			gwAddr = gwAddr.Unmap()
-
-			if !t.isGatewayInVpnNetworks(gwAddr) {
-				// Gateway isn't in our overlay network, ignore
-				t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
-			} else {
-				gateways = append(gateways, routing.NewGateway(gwAddr, 1))
-			}
-		}
-	}
-
-	for _, p := range r.MultiPath {
-		// If this route is relevant to our interface and there is a gateway then add it
-		if p.LinkIndex == link.Attrs().Index && len(p.Gw) > 0 {
-			gwAddr, ok := netip.AddrFromSlice(p.Gw)
-			if !ok {
-				t.l.WithField("route", r).Debug("Ignoring multipath route update, invalid gateway address")
-			} else {
-				gwAddr = gwAddr.Unmap()
-
-				if !t.isGatewayInVpnNetworks(gwAddr) {
-					// Gateway isn't in our overlay network, ignore
-					t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
-				} else {
-					// p.Hops+1 = weight of the route
-					gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1))
-				}
-			}
-		}
-	}
-
-	routing.CalculateBucketsForGateways(gateways)
-	return gateways
-}
-
-func (t *tun) updateRoutes(r netlink.RouteUpdate) {
-
-	gateways := t.getGatewaysFromRoute(&r.Route)
+func (t *wgTun) updateRoutes(r netlink.RouteUpdate) {
+	gateways := t.getGatewaysFromRoute(&r.Route, t.routeManager.deviceIndex)
 
 	if len(gateways) == 0 {
-		// No gateways relevant to our network, no routing changes required.
 		t.l.WithField("route", r).Debug("Ignoring route update, no gateways")
 		return
 	}
@@ -661,7 +472,6 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
 	if r.Type == unix.RTM_NEWROUTE {
 		t.l.WithField("destination", dst).WithField("via", gateways).Info("Adding route")
 		newTree.Insert(dst, gateways)
-
 	} else {
 		t.l.WithField("destination", dst).WithField("via", gateways).Info("Removing route")
 		newTree.Delete(dst)
@@ -669,18 +479,71 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
 	t.routeTree.Store(newTree)
 }
 
-func (t *tun) Close() error {
-	if t.routeChan != nil {
-		close(t.routeChan)
+func (t *wgTun) getGatewaysFromRoute(r *netlink.Route, deviceIndex int) routing.Gateways {
+	var gateways routing.Gateways
+
+	name, err := t.tunDevice.Name()
+	if err != nil {
+		t.l.Error("Ignoring route update: failed to get device name")
+		return gateways
 	}
 
-	if t.ReadWriteCloser != nil {
-		_ = t.ReadWriteCloser.Close()
+	link, err := netlink.LinkByName(name)
+	if err != nil {
+		t.l.WithField("DeviceName", name).Error("Ignoring route update: failed to get link by name")
+		return gateways
 	}
 
-	if t.ioctlFd > 0 {
-		_ = os.NewFile(t.ioctlFd, "ioctlFd").Close()
+	// If this route is relevant to our interface and there is a gateway then add it
+	if r.LinkIndex == link.Attrs().Index && len(r.Gw) > 0 {
+		gwAddr, ok := netip.AddrFromSlice(r.Gw)
+		if !ok {
+			t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway address")
+		} else {
+			gwAddr = gwAddr.Unmap()
+
+			if !t.isGatewayInVpnNetworks(gwAddr) {
+				t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
+			} else {
+				gateways = append(gateways, routing.NewGateway(gwAddr, 1))
+			}
+		}
 	}
 
+	for _, p := range r.MultiPath {
+		if p.LinkIndex == link.Attrs().Index && len(p.Gw) > 0 {
+			gwAddr, ok := netip.AddrFromSlice(p.Gw)
+			if !ok {
+				t.l.WithField("route", r).Debug("Ignoring multipath route update, invalid gateway address")
+			} else {
+				gwAddr = gwAddr.Unmap()
+
+				if !t.isGatewayInVpnNetworks(gwAddr) {
+					t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
+				} else {
+					gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1))
+				}
+			}
+		}
+	}
+
+	routing.CalculateBucketsForGateways(gateways)
+	return gateways
+}
+
+func (t *wgTun) isGatewayInVpnNetworks(gwAddr netip.Addr) bool {
+	for i := range t.vpnNetworks {
+		if t.vpnNetworks[i].Contains(gwAddr) {
+			return true
+		}
+	}
+	return false
+}
+
+func ioctl(a1, a2, a3 uintptr) error {
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, a1, a2, a3)
+	if errno != 0 {
+		return errno
+	}
 	return nil
 }