diff --git a/go.mod b/go.mod index b060abc..dc5c9fa 100644 --- a/go.mod +++ b/go.mod @@ -29,11 +29,11 @@ require ( golang.org/x/sys v0.37.0 golang.org/x/term v0.36.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 - golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b + golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb golang.zx2c4.com/wireguard/windows v0.5.3 google.golang.org/protobuf v1.36.8 gopkg.in/yaml.v3 v3.0.1 - gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe + gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c ) require ( @@ -49,6 +49,6 @@ require ( github.com/vishvananda/netns v0.0.5 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/mod v0.24.0 // indirect - golang.org/x/time v0.5.0 // indirect + golang.org/x/time v0.7.0 // indirect golang.org/x/tools v0.33.0 // indirect ) diff --git a/go.sum b/go.sum index 3aee30a..83c3e57 100644 --- a/go.sum +++ b/go.sum @@ -215,8 +215,8 @@ golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -230,8 +230,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= -golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo= -golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4= +golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb h1:whnFRlWMcXI9d+ZbWg+4sHnLp52d5yiIPUxMBSt4X9A= +golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb/go.mod h1:rpwXGsirqLqN2L0JDJQlwOboGHmptD5ZD6T2VmcqhTw= golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE= golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -257,5 +257,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g= -gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU= +gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c h1:m/r7OM+Y2Ty1sgBQ7Qb27VgIMBW8ZZhT4gLnUyDIhzI= +gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c/go.mod h1:3r5CMtNQMKIvBlrmM9xWUNamjKBYPOWyXOjmg5Kts3g= diff --git a/inside.go b/inside.go index d24ed31..e8aa420 100644 --- a/inside.go +++ b/inside.go @@ -33,8 +33,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet // routes packets from the Nebula addr to the Nebula addr through the Nebula // TUN device. if immediatelyForwardToSelf { - _, err := f.readers[q].Write(packet) - if err != nil { + if err := f.writeTun(q, packet); err != nil { f.l.WithError(err).Error("Failed to forward to tun") } } @@ -91,8 +90,7 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) { return } - _, err := f.readers[q].Write(out) - if err != nil { + if err := f.writeTun(q, out); err != nil { f.l.WithError(err).Error("Failed to write to tun") } } diff --git a/interface.go b/interface.go index 082906d..82634df 100644 --- a/interface.go +++ b/interface.go @@ -110,6 +110,16 @@ type EncWriter interface { GetCertState() *CertState } +// BatchReader is an interface for readers that support vectorized packet reading +type BatchReader interface { + BatchRead() ([][]byte, []int, error) +} + +// BatchWriter is an interface for writers that support vectorized packet writing +type BatchWriter interface { + BatchWrite([][]byte) (int, error) +} + type sendRecvErrorConfig uint8 const ( @@ -279,6 +289,16 @@ func (f *Interface) listenOut(i int) { func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) { runtime.LockOSThread() + // Check if reader supports batch operations + if batchReader, ok := reader.(BatchReader); ok { + err := f.listenInBatch(batchReader, i) + if err != nil { + f.l.WithError(err).Error("Fatal error in batch packet reader, exiting goroutine") + } + return + } + + // Fall back to single-packet mode packet := make([]byte, mtu) out := make([]byte, mtu) fwPacket := &firewall.Packet{} @@ -293,15 +313,78 @@ func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) { return } - f.l.WithError(err).Error("Error while reading outbound packet") - // This only seems to happen when something fatal happens to the fd, so exit. - os.Exit(2) + f.l.WithError(err).Error("Fatal error while reading outbound packet, exiting goroutine") + return } f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get(f.l)) } } +// listenInBatch handles vectorized packet reading for improved performance +func (f *Interface) listenInBatch(reader BatchReader, i int) error { + // Allocate per-packet state + fwPackets := make([]*firewall.Packet, 64) // Match batch size + outBuffers := make([][]byte, 64) + nbBuffers := make([][]byte, 64) + + for j := 0; j < 64; j++ { + fwPackets[j] = &firewall.Packet{} + outBuffers[j] = make([]byte, mtu) + nbBuffers[j] = make([]byte, 12, 12) + } + + conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout) + + for { + packets, sizes, err := reader.BatchRead() + if err != nil { + if errors.Is(err, os.ErrClosed) && f.closed.Load() { + return nil + } + + return fmt.Errorf("error while batch reading outbound packets: %w", err) + } + + // Process each packet in the batch + cache := conntrackCache.Get(f.l) + for idx := 0; idx < len(packets); idx++ { + if idx < len(sizes) && sizes[idx] > 0 { + // Use modulo to reuse fw packet state if batch is larger than our pre-allocated state + stateIdx := idx % len(fwPackets) + f.consumeInsidePacket(packets[idx][:sizes[idx]], fwPackets[stateIdx], nbBuffers[stateIdx], outBuffers[stateIdx], i, cache) + } + } + } +} + +// writeTunBatch attempts to write multiple packets to the TUN device using batch operations if supported +func (f *Interface) writeTunBatch(q int, packets [][]byte) error { + if len(packets) == 0 { + return nil + } + + // Check if the reader/writer supports batch operations + if batchWriter, ok := f.readers[q].(BatchWriter); ok { + _, err := batchWriter.BatchWrite(packets) + return err + } + + // Fall back to writing packets individually + for _, packet := range packets { + if _, err := f.readers[q].Write(packet); err != nil { + return err + } + } + return nil +} + +// writeTun writes a single packet to the TUN device +func (f *Interface) writeTun(q int, packet []byte) error { + _, err := f.readers[q].Write(packet) + return err +} + func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) { c.RegisterReloadCallback(f.reloadFirewall) c.RegisterReloadCallback(f.reloadSendRecvError) diff --git a/overlay/route.go b/overlay/route.go index 6198958..641cf74 100644 --- a/overlay/route.go +++ b/overlay/route.go @@ -3,7 +3,6 @@ package overlay import ( "fmt" "math" - "net" "net/netip" "runtime" "strconv" @@ -305,29 +304,3 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) { return routes, nil } - -func ipWithin(o *net.IPNet, i *net.IPNet) bool { - // Make sure o contains the lowest form of i - if !o.Contains(i.IP.Mask(i.Mask)) { - return false - } - - // Find the max ip in i - ip4 := i.IP.To4() - if ip4 == nil { - return false - } - - last := make(net.IP, len(ip4)) - copy(last, ip4) - for x := range ip4 { - last[x] |= ^i.Mask[x] - } - - // Make sure o contains the max - if !o.Contains(last) { - return false - } - - return true -} diff --git a/overlay/route_test.go b/overlay/route_test.go index 9a959a5..996095c 100644 --- a/overlay/route_test.go +++ b/overlay/route_test.go @@ -225,6 +225,7 @@ func Test_parseUnsafeRoutes(t *testing.T) { // no mtu c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": "127.0.0.1", "route": "1.0.0.0/8"}}} routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) + require.NoError(t, err) assert.Len(t, routes, 1) assert.Equal(t, 0, routes[0].MTU) @@ -318,7 +319,7 @@ func Test_makeRouteTree(t *testing.T) { ip, err = netip.ParseAddr("1.1.0.1") require.NoError(t, err) - r, ok = routeTree.Lookup(ip) + _, ok = routeTree.Lookup(ip) assert.False(t, ok) } diff --git a/overlay/tun.go b/overlay/tun.go index 3a61d18..4a6377d 100644 --- a/overlay/tun.go +++ b/overlay/tun.go @@ -1,8 +1,6 @@ package overlay import ( - "fmt" - "net" "net/netip" "github.com/sirupsen/logrus" @@ -72,51 +70,3 @@ func findRemovedRoutes(newRoutes, oldRoutes []Route) []Route { return removed } - -func prefixToMask(prefix netip.Prefix) netip.Addr { - pLen := 128 - if prefix.Addr().Is4() { - pLen = 32 - } - - addr, _ := netip.AddrFromSlice(net.CIDRMask(prefix.Bits(), pLen)) - return addr -} - -func flipBytes(b []byte) []byte { - for i := 0; i < len(b); i++ { - b[i] ^= 0xFF - } - return b -} -func orBytes(a []byte, b []byte) []byte { - ret := make([]byte, len(a)) - for i := 0; i < len(a); i++ { - ret[i] = a[i] | b[i] - } - return ret -} - -func getBroadcast(cidr netip.Prefix) netip.Addr { - broadcast, _ := netip.AddrFromSlice( - orBytes( - cidr.Addr().AsSlice(), - flipBytes(prefixToMask(cidr).AsSlice()), - ), - ) - return broadcast -} - -func selectGateway(dest netip.Prefix, gateways []netip.Prefix) (netip.Prefix, error) { - for _, gateway := range gateways { - if dest.Addr().Is4() && gateway.Addr().Is4() { - return gateway, nil - } - - if dest.Addr().Is6() && gateway.Addr().Is6() { - return gateway, nil - } - } - - return netip.Prefix{}, fmt.Errorf("no gateway found for %v in the list of vpn networks", dest) -} diff --git a/overlay/tun_darwin.go b/overlay/tun_darwin.go index 34c2a71..47ad7b4 100644 --- a/overlay/tun_darwin.go +++ b/overlay/tun_darwin.go @@ -1,5 +1,5 @@ -//go:build !ios && !e2e_testing -// +build !ios,!e2e_testing +//go:build darwin && !ios && !e2e_testing +// +build darwin,!ios,!e2e_testing package overlay @@ -8,48 +8,27 @@ import ( "fmt" "io" "net/netip" - "os" - "sync/atomic" - "syscall" "unsafe" - "github.com/gaissmai/bart" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" netroute "golang.org/x/net/route" "golang.org/x/sys/unix" + wgtun "golang.zx2c4.com/wireguard/tun" ) type tun struct { - io.ReadWriteCloser - Device string - vpnNetworks []netip.Prefix - DefaultMTU int - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[bart.Table[routing.Gateways]] - linkAddr *netroute.LinkAddr - l *logrus.Logger - - // cache out buffer since we need to prepend 4 bytes for tun metadata - out []byte + linkAddr *netroute.LinkAddr } +// ioctl structures for Darwin network configuration type ifReq struct { Name [unix.IFNAMSIZ]byte Flags uint16 pad [8]byte } -const ( - _SIOCAIFADDR_IN6 = 2155899162 - _UTUN_OPT_IFNAME = 2 - _IN6_IFF_NODAD = 0x0020 - _IN6_IFF_SECURED = 0x0400 - utunControlName = "com.apple.net.utun_control" -) - type ifreqMTU struct { Name [16]byte MTU int32 @@ -79,60 +58,46 @@ type ifreqAlias6 struct { Lifetime addrLifetime } -func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { - name := c.GetString("tun.dev", "") - ifIndex := -1 - if name != "" && name != "utun" { - _, err := fmt.Sscanf(name, "utun%d", &ifIndex) - if err != nil || ifIndex < 0 { - // NOTE: we don't make this error so we don't break existing - // configs that set a name before it was used. - l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring") - ifIndex = -1 - } - } +const ( + _SIOCAIFADDR_IN6 = 2155899162 + _IN6_IFF_NODAD = 0x0020 +) - fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL) +func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*wgTun, error) { + return nil, fmt.Errorf("newTunFromFd not supported on Darwin") +} + +func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*wgTun, error) { + deviceName := c.GetString("tun.dev", "utun") + mtu := c.GetInt("tun.mtu", DefaultMTU) + + // Create WireGuard TUN device + tunDevice, err := wgtun.CreateTUN(deviceName, mtu) if err != nil { - return nil, fmt.Errorf("system socket: %v", err) + return nil, fmt.Errorf("failed to create TUN device: %w", err) } - var ctlInfo = &unix.CtlInfo{} - copy(ctlInfo.Name[:], utunControlName) - - err = unix.IoctlCtlInfo(fd, ctlInfo) + // Get the actual device name + actualName, err := tunDevice.Name() if err != nil { - return nil, fmt.Errorf("CTLIOCGINFO: %v", err) + tunDevice.Close() + return nil, fmt.Errorf("failed to get TUN device name: %w", err) } - err = unix.Connect(fd, &unix.SockaddrCtl{ - ID: ctlInfo.Id, - Unit: uint32(ifIndex) + 1, - }) - if err != nil { - return nil, fmt.Errorf("SYS_CONNECT: %v", err) + t := &wgTun{ + tunDevice: tunDevice, + vpnNetworks: vpnNetworks, + MaxMTU: mtu, + DefaultMTU: mtu, + l: l, } - name, err = unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME) - if err != nil { - return nil, fmt.Errorf("failed to retrieve tun name: %w", err) - } - - err = unix.SetNonblock(fd, true) - if err != nil { - return nil, fmt.Errorf("SetNonblock: %v", err) - } - - t := &tun{ - ReadWriteCloser: os.NewFile(uintptr(fd), ""), - Device: name, - vpnNetworks: vpnNetworks, - DefaultMTU: c.GetInt("tun.mtu", DefaultMTU), - l: l, - } + // Create Darwin-specific route manager + t.routeManager = &tun{} err = t.reload(c, true) if err != nil { + tunDevice.Close() return nil, err } @@ -143,215 +108,252 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) ( } }) + l.WithField("name", actualName).Info("Created WireGuard TUN device") + return t, nil } -func (t *tun) deviceBytes() (o [16]byte) { - for i, c := range t.Device { - o[i] = byte(c) - } - return -} - -func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) { - return nil, fmt.Errorf("newTunFromFd not supported in Darwin") -} - -func (t *tun) Close() error { - if t.ReadWriteCloser != nil { - return t.ReadWriteCloser.Close() - } - return nil -} - -func (t *tun) Activate() error { - devName := t.deviceBytes() - - s, err := unix.Socket( - unix.AF_INET, - unix.SOCK_DGRAM, - unix.IPPROTO_IP, - ) +func (rm *tun) Activate(t *wgTun) error { + name, err := t.tunDevice.Name() if err != nil { - return err - } - defer unix.Close(s) - - fd := uintptr(s) - - // Set the MTU on the device - ifm := ifreqMTU{Name: devName, MTU: int32(t.DefaultMTU)} - if err = ioctl(fd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil { - return fmt.Errorf("failed to set tun mtu: %v", err) + return fmt.Errorf("failed to get device name: %w", err) } - // Get the device flags - ifrf := ifReq{Name: devName} - if err = ioctl(fd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { - return fmt.Errorf("failed to get tun flags: %s", err) + // Set the MTU + rm.SetMTU(t, t.MaxMTU) + + // Add IP addresses + for _, network := range t.vpnNetworks { + if err := rm.addIP(t, name, network); err != nil { + return err + } } - linkAddr, err := getLinkAddr(t.Device) + // Bring up the interface using ioctl + if err := rm.bringUpInterface(name); err != nil { + return fmt.Errorf("failed to bring up interface: %w", err) + } + + // Get the link address for routing + linkAddr, err := getLinkAddr(name) if err != nil { - return err + return fmt.Errorf("failed to get link address: %w", err) } if linkAddr == nil { return fmt.Errorf("unable to discover link_addr for tun interface") } - t.linkAddr = linkAddr + rm.linkAddr = linkAddr - for _, network := range t.vpnNetworks { - if network.Addr().Is4() { - err = t.activate4(network) - if err != nil { - return err + // Set the routes + if err := rm.AddRoutes(t, false); err != nil { + return err + } + + return nil +} + +func (rm *tun) bringUpInterface(name string) error { + // Open a socket for ioctl + fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0) + if err != nil { + return fmt.Errorf("failed to create socket: %w", err) + } + defer unix.Close(fd) + + // Get current flags + var ifrf ifReq + copy(ifrf.Name[:], name) + + if err := ioctl(uintptr(fd), unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { + return fmt.Errorf("failed to get interface flags: %w", err) + } + + // Set IFF_UP and IFF_RUNNING flags + ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING + + if err := ioctl(uintptr(fd), unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { + return fmt.Errorf("failed to set interface flags: %w", err) + } + + return nil +} + +func (rm *tun) SetMTU(t *wgTun, mtu int) { + name, err := t.tunDevice.Name() + if err != nil { + t.l.WithError(err).Error("Failed to get device name for MTU set") + return + } + + // Open a socket for ioctl + fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0) + if err != nil { + t.l.WithError(err).Error("Failed to create socket for MTU set") + return + } + defer unix.Close(fd) + + // Prepare the ioctl request + var ifr ifreqMTU + copy(ifr.Name[:], name) + ifr.MTU = int32(mtu) + + // Set the MTU using ioctl + if err := ioctl(uintptr(fd), unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifr))); err != nil { + t.l.WithError(err).Error("Failed to set tun mtu via ioctl") + } +} + +func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error { + // On Darwin, routes are set via ifconfig and route commands + return nil +} + +func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error { + routes := *t.Routes.Load() + for _, r := range routes { + if !r.Install { + continue + } + + err := rm.addRoute(r.Cidr) + if err != nil { + if errors.Is(err, unix.EEXIST) { + t.l.WithField("route", r.Cidr). + Warnf("unable to add unsafe_route, identical route already exists") + } else { + retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) + if logErrors { + retErr.Log(t.l) + } else { + return retErr + } } } else { - err = t.activate6(network) - if err != nil { - return err - } + t.l.WithField("route", r).Info("Added route") } } - // Run the interface - ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING - if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { - return fmt.Errorf("failed to run tun device: %s", err) - } - - // Unsafe path routes - return t.addRoutes(false) + return nil } -func (t *tun) activate4(network netip.Prefix) error { - s, err := unix.Socket( - unix.AF_INET, - unix.SOCK_DGRAM, - unix.IPPROTO_IP, - ) +func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) { + for _, r := range routes { + if !r.Install { + continue + } + + err := rm.delRoute(r.Cidr) + if err != nil { + t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + } else { + t.l.WithField("route", r).Info("Removed route") + } + } +} + +func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) { + // Darwin doesn't support multi-queue TUN devices in the same way as Linux + // Return a reader that wraps the same device + return &wgTunReader{ + parent: t, + tunDevice: t.tunDevice, + batchSize: 64, + offset: 0, + l: t.l, + }, nil +} + +func (rm *tun) addIP(t *wgTun, name string, network netip.Prefix) error { + addr := network.Addr() + + if addr.Is4() { + return rm.addIPv4(name, network) + } else { + return rm.addIPv6(name, network) + } +} + +func (rm *tun) addIPv4(name string, network netip.Prefix) error { + // Open an IPv4 socket for ioctl + s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) if err != nil { - return err + return fmt.Errorf("failed to create IPv4 socket: %w", err) } defer unix.Close(s) - ifr := ifreqAlias4{ - Name: t.deviceBytes(), - Addr: unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: network.Addr().As4(), - }, - DstAddr: unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: network.Addr().As4(), - }, - MaskAddr: unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: prefixToMask(network).As4(), - }, + var ifr ifreqAlias4 + copy(ifr.Name[:], name) + + // Set the address + ifr.Addr = unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: network.Addr().As4(), + } + + // Set the destination address (same as address for point-to-point) + ifr.DstAddr = unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: network.Addr().As4(), + } + + // Set the netmask + ifr.MaskAddr = unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: prefixToMask(network).As4(), } if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil { - return fmt.Errorf("failed to set tun v4 address: %s", err) - } - - err = addRoute(network, t.linkAddr) - if err != nil { - return err + return fmt.Errorf("failed to set IPv4 address via ioctl: %w", err) } return nil } -func (t *tun) activate6(network netip.Prefix) error { - s, err := unix.Socket( - unix.AF_INET6, - unix.SOCK_DGRAM, - unix.IPPROTO_IP, - ) +func (rm *tun) addIPv6(name string, network netip.Prefix) error { + // Open an IPv6 socket for ioctl + s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP) if err != nil { - return err + return fmt.Errorf("failed to create IPv6 socket: %w", err) } defer unix.Close(s) - ifr := ifreqAlias6{ - Name: t.deviceBytes(), - Addr: unix.RawSockaddrInet6{ - Len: unix.SizeofSockaddrInet6, - Family: unix.AF_INET6, - Addr: network.Addr().As16(), - }, - PrefixMask: unix.RawSockaddrInet6{ - Len: unix.SizeofSockaddrInet6, - Family: unix.AF_INET6, - Addr: prefixToMask(network).As16(), - }, - Lifetime: addrLifetime{ - // never expires - Vltime: 0xffffffff, - Pltime: 0xffffffff, - }, - Flags: _IN6_IFF_NODAD, + var ifr ifreqAlias6 + copy(ifr.Name[:], name) + + // Set the address + ifr.Addr = unix.RawSockaddrInet6{ + Len: unix.SizeofSockaddrInet6, + Family: unix.AF_INET6, + Addr: network.Addr().As16(), } + // Set the prefix mask + ifr.PrefixMask = unix.RawSockaddrInet6{ + Len: unix.SizeofSockaddrInet6, + Family: unix.AF_INET6, + Addr: prefixToMask(network).As16(), + } + + // Set lifetime (never expires) + ifr.Lifetime = addrLifetime{ + Vltime: 0xffffffff, + Pltime: 0xffffffff, + } + + // Set flags (no DAD - Duplicate Address Detection) + ifr.Flags = _IN6_IFF_NODAD + if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil { - return fmt.Errorf("failed to set tun address: %s", err) + return fmt.Errorf("failed to set IPv6 address via ioctl: %w", err) } return nil } -func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) - if err != nil { - return err - } - - if !initial && !change { - return nil - } - - routeTree, err := makeRouteTree(t.l, routes, false) - if err != nil { - return err - } - - // Teach nebula how to handle the routes before establishing them in the system table - oldRoutes := t.Routes.Swap(&routes) - t.routeTree.Store(routeTree) - - if !initial { - // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) - if err != nil { - util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) - } - - // Ensure any routes we actually want are installed - err = t.addRoutes(true) - if err != nil { - // Catch any stray logs - util.LogWithContextIfNeeded("Failed to add routes", err, t.l) - } - } - - return nil -} - -func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { - r, ok := t.routeTree.Load().Lookup(ip) - if ok { - return r - } - return routing.Gateways{} -} - -// Get the LinkAddr for the interface of the given name -// Is there an easier way to fetch this when we create the interface? -// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers. func getLinkAddr(name string) (*netroute.LinkAddr, error) { rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0) if err != nil { @@ -377,53 +379,7 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) { return nil, nil } -func (t *tun) addRoutes(logErrors bool) error { - routes := *t.Routes.Load() - - for _, r := range routes { - if len(r.Via) == 0 || !r.Install { - // We don't allow route MTUs so only install routes with a via - continue - } - - err := addRoute(r.Cidr, t.linkAddr) - if err != nil { - if errors.Is(err, unix.EEXIST) { - t.l.WithField("route", r.Cidr). - Warnf("unable to add unsafe_route, identical route already exists") - } else { - retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) - if logErrors { - retErr.Log(t.l) - } else { - return retErr - } - } - } else { - t.l.WithField("route", r).Info("Added route") - } - } - - return nil -} - -func (t *tun) removeRoutes(routes []Route) error { - for _, r := range routes { - if !r.Install { - continue - } - - err := delRoute(r.Cidr, t.linkAddr) - if err != nil { - t.l.WithError(err).WithField("route", r).Error("Failed to remove route") - } else { - t.l.WithField("route", r).Info("Removed route") - } - } - return nil -} - -func addRoute(prefix netip.Prefix, gateway netroute.Addr) error { +func (rm *tun) addRoute(prefix netip.Prefix) error { sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) if err != nil { return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) @@ -441,13 +397,13 @@ func addRoute(prefix netip.Prefix, gateway netroute.Addr) error { route.Addrs = []netroute.Addr{ unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, - unix.RTAX_GATEWAY: gateway, + unix.RTAX_GATEWAY: rm.linkAddr, } } else { route.Addrs = []netroute.Addr{ unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, - unix.RTAX_GATEWAY: gateway, + unix.RTAX_GATEWAY: rm.linkAddr, } } @@ -464,7 +420,7 @@ func addRoute(prefix netip.Prefix, gateway netroute.Addr) error { return nil } -func delRoute(prefix netip.Prefix, gateway netroute.Addr) error { +func (rm *tun) delRoute(prefix netip.Prefix) error { sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) if err != nil { return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) @@ -481,13 +437,13 @@ func delRoute(prefix netip.Prefix, gateway netroute.Addr) error { route.Addrs = []netroute.Addr{ unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, - unix.RTAX_GATEWAY: gateway, + unix.RTAX_GATEWAY: rm.linkAddr, } } else { route.Addrs = []netroute.Addr{ unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, - unix.RTAX_GATEWAY: gateway, + unix.RTAX_GATEWAY: rm.linkAddr, } } @@ -495,6 +451,7 @@ func delRoute(prefix netip.Prefix, gateway netroute.Addr) error { if err != nil { return fmt.Errorf("failed to create route.RouteMessage: %w", err) } + _, err = unix.Write(sock, data[:]) if err != nil { return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) @@ -503,52 +460,34 @@ func delRoute(prefix netip.Prefix, gateway netroute.Addr) error { return nil } -func (t *tun) Read(to []byte) (int, error) { - buf := make([]byte, len(to)+4) - - n, err := t.ReadWriteCloser.Read(buf) - - copy(to, buf[4:]) - return n - 4, err +func ioctl(a1, a2, a3 uintptr) error { + _, _, errno := unix.Syscall(unix.SYS_IOCTL, a1, a2, a3) + if errno != 0 { + return errno + } + return nil } -// Write is only valid for single threaded use -func (t *tun) Write(from []byte) (int, error) { - buf := t.out - if cap(buf) < len(from)+4 { - buf = make([]byte, len(from)+4) - t.out = buf - } - buf = buf[:len(from)+4] - - if len(from) == 0 { - return 0, syscall.EIO - } - - // Determine the IP Family for the NULL L2 Header - ipVer := from[0] >> 4 - if ipVer == 4 { - buf[3] = syscall.AF_INET - } else if ipVer == 6 { - buf[3] = syscall.AF_INET6 +func prefixToMask(prefix netip.Prefix) netip.Addr { + bits := prefix.Bits() + if prefix.Addr().Is4() { + // Create IPv4 netmask from prefix length + mask := ^uint32(0) << (32 - bits) + return netip.AddrFrom4([4]byte{ + byte(mask >> 24), + byte(mask >> 16), + byte(mask >> 8), + byte(mask), + }) } else { - return 0, fmt.Errorf("unable to determine IP version from packet") + // Create IPv6 netmask from prefix length + var mask [16]byte + for i := 0; i < bits/8; i++ { + mask[i] = 0xff + } + if bits%8 != 0 { + mask[bits/8] = ^byte(0) << (8 - bits%8) + } + return netip.AddrFrom16(mask) } - - copy(buf[4:], from) - - n, err := t.ReadWriteCloser.Write(buf) - return n - 4, err -} - -func (t *tun) Networks() []netip.Prefix { - return t.vpnNetworks -} - -func (t *tun) Name() string { - return t.Device -} - -func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { - return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin") } diff --git a/overlay/tun_freebsd.go b/overlay/tun_freebsd.go index 0dd8228..913c6f8 100644 --- a/overlay/tun_freebsd.go +++ b/overlay/tun_freebsd.go @@ -1,284 +1,77 @@ -//go:build !e2e_testing -// +build !e2e_testing +//go:build freebsd && !e2e_testing +// +build freebsd,!e2e_testing package overlay import ( - "bytes" - "errors" "fmt" "io" - "io/fs" "net/netip" - "sync/atomic" + "os/exec" + "strconv" + "strings" "syscall" - "time" "unsafe" - "github.com/gaissmai/bart" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" - netroute "golang.org/x/net/route" "golang.org/x/sys/unix" + wgtun "golang.zx2c4.com/wireguard/tun" ) -const ( - // FIODGNAME is defined in sys/sys/filio.h on FreeBSD - // For 32-bit systems, use FIODGNAME_32 (not defined in this file: 0x80086678) - FIODGNAME = 0x80106678 - TUNSIFMODE = 0x8004745e - TUNSIFHEAD = 0x80047460 - OSIOCAIFADDR_IN6 = 0x8088691b - IN6_IFF_NODAD = 0x0020 -) - -type fiodgnameArg struct { - length int32 - pad [4]byte - buf unsafe.Pointer -} +type tun struct{} +// ifreqRename is used for renaming network interfaces on FreeBSD type ifreqRename struct { Name [unix.IFNAMSIZ]byte Data uintptr } -type ifreqDestroy struct { - Name [unix.IFNAMSIZ]byte - pad [16]byte +func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*wgTun, error) { + return nil, fmt.Errorf("newTunFromFd not supported on FreeBSD") } -type ifReq struct { - Name [unix.IFNAMSIZ]byte - Flags uint16 -} +func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*wgTun, error) { + deviceName := c.GetString("tun.dev", "tun") + mtu := c.GetInt("tun.mtu", DefaultMTU) -type ifreqMTU struct { - Name [unix.IFNAMSIZ]byte - MTU int32 -} - -type addrLifetime struct { - Expire uint64 - Preferred uint64 - Vltime uint32 - Pltime uint32 -} - -type ifreqAlias4 struct { - Name [unix.IFNAMSIZ]byte - Addr unix.RawSockaddrInet4 - DstAddr unix.RawSockaddrInet4 - MaskAddr unix.RawSockaddrInet4 - VHid uint32 -} - -type ifreqAlias6 struct { - Name [unix.IFNAMSIZ]byte - Addr unix.RawSockaddrInet6 - DstAddr unix.RawSockaddrInet6 - PrefixMask unix.RawSockaddrInet6 - Flags uint32 - Lifetime addrLifetime - VHid uint32 -} - -type tun struct { - Device string - vpnNetworks []netip.Prefix - MTU int - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[bart.Table[routing.Gateways]] - linkAddr *netroute.LinkAddr - l *logrus.Logger - devFd int -} - -func (t *tun) Read(to []byte) (int, error) { - // use readv() to read from the tunnel device, to eliminate the need for copying the buffer - if t.devFd < 0 { - return -1, syscall.EINVAL - } - - // first 4 bytes is protocol family, in network byte order - head := make([]byte, 4) - - iovecs := []syscall.Iovec{ - {&head[0], 4}, - {&to[0], uint64(len(to))}, - } - - n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2)) - - var err error - if errno != 0 { - err = syscall.Errno(errno) - } else { - err = nil - } - // fix bytes read number to exclude header - bytesRead := int(n) - if bytesRead < 0 { - return bytesRead, err - } else if bytesRead < 4 { - return 0, err - } else { - return bytesRead - 4, err - } -} - -// Write is only valid for single threaded use -func (t *tun) Write(from []byte) (int, error) { - // use writev() to write to the tunnel device, to eliminate the need for copying the buffer - if t.devFd < 0 { - return -1, syscall.EINVAL - } - - if len(from) <= 1 { - return 0, syscall.EIO - } - ipVer := from[0] >> 4 - var head []byte - // first 4 bytes is protocol family, in network byte order - if ipVer == 4 { - head = []byte{0, 0, 0, syscall.AF_INET} - } else if ipVer == 6 { - head = []byte{0, 0, 0, syscall.AF_INET6} - } else { - return 0, fmt.Errorf("unable to determine IP version from packet") - } - iovecs := []syscall.Iovec{ - {&head[0], 4}, - {&from[0], uint64(len(from))}, - } - - n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2)) - - var err error - if errno != 0 { - err = syscall.Errno(errno) - } else { - err = nil - } - - return int(n) - 4, err -} - -func (t *tun) Close() error { - if t.devFd >= 0 { - err := syscall.Close(t.devFd) - if err != nil { - t.l.WithError(err).Error("Error closing device") - } - t.devFd = -1 - - c := make(chan struct{}) - go func() { - // destroying the interface can block if a read() is still pending. Do this asynchronously. - defer close(c) - s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) - if err == nil { - defer syscall.Close(s) - ifreq := ifreqDestroy{Name: t.deviceBytes()} - err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq))) - } - if err != nil { - t.l.WithError(err).Error("Error destroying tunnel") - } - }() - - // wait up to 1 second so we start blocking at the ioctl - select { - case <-c: - case <-time.After(1 * time.Second): - } - } - - return nil -} - -func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) { - return nil, fmt.Errorf("newTunFromFd not supported in FreeBSD") -} - -func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { - // Try to open existing tun device - var fd int - var err error - deviceName := c.GetString("tun.dev", "") - if deviceName != "" { - fd, err = syscall.Open("/dev/"+deviceName, syscall.O_RDWR, 0) - } - if errors.Is(err, fs.ErrNotExist) || deviceName == "" { - // If the device doesn't already exist, request a new one and rename it - fd, err = syscall.Open("/dev/tun", syscall.O_RDWR, 0) - } + // Create WireGuard TUN device + tunDevice, err := wgtun.CreateTUN(deviceName, mtu) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create TUN device: %w", err) } - // Read the name of the interface - var name [16]byte - arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)} - ctrlErr := ioctl(uintptr(fd), FIODGNAME, uintptr(unsafe.Pointer(&arg))) - - if ctrlErr == nil { - // set broadcast mode and multicast - ifmode := uint32(unix.IFF_BROADCAST | unix.IFF_MULTICAST) - ctrlErr = ioctl(uintptr(fd), TUNSIFMODE, uintptr(unsafe.Pointer(&ifmode))) - } - - if ctrlErr == nil { - // turn on link-layer mode, to support ipv6 - ifhead := uint32(1) - ctrlErr = ioctl(uintptr(fd), TUNSIFHEAD, uintptr(unsafe.Pointer(&ifhead))) - } - - if ctrlErr != nil { - return nil, err - } - - ifName := string(bytes.TrimRight(name[:], "\x00")) - if deviceName == "" { - deviceName = ifName + // Get the actual device name + actualName, err := tunDevice.Name() + if err != nil { + tunDevice.Close() + return nil, fmt.Errorf("failed to get TUN device name: %w", err) } // If the name doesn't match the desired interface name, rename it now - if ifName != deviceName { - s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) - if err != nil { - return nil, err + if actualName != deviceName && deviceName != "" && deviceName != "tun" { + if err := renameInterface(actualName, deviceName); err != nil { + tunDevice.Close() + return nil, fmt.Errorf("failed to rename interface from %s to %s: %w", actualName, deviceName, err) } - defer syscall.Close(s) - - fd := uintptr(s) - - var fromName [16]byte - var toName [16]byte - copy(fromName[:], ifName) - copy(toName[:], deviceName) - - ifrr := ifreqRename{ - Name: fromName, - Data: uintptr(unsafe.Pointer(&toName)), - } - - // Set the device name - ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr))) + actualName = deviceName } - t := &tun{ - Device: deviceName, + t := &wgTun{ + tunDevice: tunDevice, vpnNetworks: vpnNetworks, - MTU: c.GetInt("tun.mtu", DefaultMTU), + MaxMTU: mtu, + DefaultMTU: mtu, l: l, - devFd: fd, } + // Create FreeBSD-specific route manager + t.routeManager = &tun{} + err = t.reload(c, true) if err != nil { + tunDevice.Close() return nil, err } @@ -289,180 +82,86 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) ( } }) + l.WithField("name", actualName).Info("Created WireGuard TUN device") + return t, nil } -func (t *tun) addIp(cidr netip.Prefix) error { - if cidr.Addr().Is4() { - ifr := ifreqAlias4{ - Name: t.deviceBytes(), - Addr: unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: cidr.Addr().As4(), - }, - DstAddr: unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: getBroadcast(cidr).As4(), - }, - MaskAddr: unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: prefixToMask(cidr).As4(), - }, - VHid: 0, - } - s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) - if err != nil { - return err - } - defer syscall.Close(s) - // Note: unix.SIOCAIFADDR corresponds to FreeBSD's OSIOCAIFADDR - if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil { - return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err) - } - return nil - } - - if cidr.Addr().Is6() { - ifr := ifreqAlias6{ - Name: t.deviceBytes(), - Addr: unix.RawSockaddrInet6{ - Len: unix.SizeofSockaddrInet6, - Family: unix.AF_INET6, - Addr: cidr.Addr().As16(), - }, - PrefixMask: unix.RawSockaddrInet6{ - Len: unix.SizeofSockaddrInet6, - Family: unix.AF_INET6, - Addr: prefixToMask(cidr).As16(), - }, - Lifetime: addrLifetime{ - Expire: 0, - Preferred: 0, - Vltime: 0xffffffff, - Pltime: 0xffffffff, - }, - Flags: IN6_IFF_NODAD, - } - s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) - if err != nil { - return err - } - defer syscall.Close(s) - - if err := ioctl(uintptr(s), OSIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil { - return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err) - } - return nil - } - - return fmt.Errorf("unknown address type %v", cidr) -} - -func (t *tun) Activate() error { - // Setup our default MTU - err := t.setMTU() +func (rm *tun) Activate(t *wgTun) error { + name, err := t.tunDevice.Name() if err != nil { - return err + return fmt.Errorf("failed to get device name: %w", err) } - linkAddr, err := getLinkAddr(t.Device) - if err != nil { - return err - } - if linkAddr == nil { - return fmt.Errorf("unable to discover link_addr for tun interface") - } - t.linkAddr = linkAddr + // Set the MTU + rm.SetMTU(t, t.MaxMTU) - for i := range t.vpnNetworks { - err := t.addIp(t.vpnNetworks[i]) - if err != nil { + // Add IP addresses + for _, network := range t.vpnNetworks { + if err := rm.addIP(t, name, network); err != nil { return err } } - return t.addRoutes(false) -} + // Bring up the interface + if err := runCommandBSD("ifconfig", name, "up"); err != nil { + return fmt.Errorf("failed to bring up interface: %w", err) + } -func (t *tun) setMTU() error { - // Set the MTU on the device - s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) - if err != nil { + // Set the routes + if err := rm.AddRoutes(t, false); err != nil { return err } - defer syscall.Close(s) - - ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MTU)} - err = ioctl(uintptr(s), unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))) - return err -} - -func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) - if err != nil { - return err - } - - if !initial && !change { - return nil - } - - routeTree, err := makeRouteTree(t.l, routes, false) - if err != nil { - return err - } - - // Teach nebula how to handle the routes before establishing them in the system table - oldRoutes := t.Routes.Swap(&routes) - t.routeTree.Store(routeTree) - - if !initial { - // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) - if err != nil { - util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) - } - - // Ensure any routes we actually want are installed - err = t.addRoutes(true) - if err != nil { - // Catch any stray logs - util.LogWithContextIfNeeded("Failed to add routes", err, t.l) - } - } return nil } -func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { - r, _ := t.routeTree.Load().Lookup(ip) - return r +func (rm *tun) SetMTU(t *wgTun, mtu int) { + name, err := t.tunDevice.Name() + if err != nil { + t.l.WithError(err).Error("Failed to get device name for MTU set") + return + } + + if err := runCommandBSD("ifconfig", name, "mtu", strconv.Itoa(mtu)); err != nil { + t.l.WithError(err).Error("Failed to set tun mtu") + } } -func (t *tun) Networks() []netip.Prefix { - return t.vpnNetworks +func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error { + // On FreeBSD, routes are set via ifconfig and route commands + return nil } -func (t *tun) Name() string { - return t.Device -} +func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error { + name, err := t.tunDevice.Name() + if err != nil { + return fmt.Errorf("failed to get device name: %w", err) + } -func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { - return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd") -} - -func (t *tun) addRoutes(logErrors bool) error { routes := *t.Routes.Load() for _, r := range routes { - if len(r.Via) == 0 || !r.Install { - // We don't allow route MTUs so only install routes with a via + if !r.Install { continue } - err := addRoute(r.Cidr, t.linkAddr) + // Add route using route command + args := []string{"add"} + + if r.Cidr.Addr().Is6() { + args = append(args, "-inet6") + } else { + args = append(args, "-inet") + } + + args = append(args, r.Cidr.String(), "-interface", name) + + if r.Metric > 0 { + // FreeBSD doesn't support route metrics directly like Linux + t.l.WithField("route", r).Warn("Route metrics are not fully supported on FreeBSD") + } + + err := runCommandBSD("route", args...) if err != nil { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { @@ -478,142 +177,100 @@ func (t *tun) addRoutes(logErrors bool) error { return nil } -func (t *tun) removeRoutes(routes []Route) error { +func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) { + name, err := t.tunDevice.Name() + if err != nil { + t.l.WithError(err).Error("Failed to get device name for route removal") + return + } + for _, r := range routes { if !r.Install { continue } - err := delRoute(r.Cidr, t.linkAddr) + args := []string{"delete"} + + if r.Cidr.Addr().Is6() { + args = append(args, "-inet6") + } else { + args = append(args, "-inet") + } + + args = append(args, r.Cidr.String(), "-interface", name) + + err := runCommandBSD("route", args...) if err != nil { t.l.WithError(err).WithField("route", r).Error("Failed to remove route") } else { t.l.WithField("route", r).Info("Removed route") } } - return nil } -func (t *tun) deviceBytes() (o [16]byte) { - for i, c := range t.Device { - o[i] = byte(c) - } - return +func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) { + // FreeBSD doesn't support multi-queue TUN devices in the same way as Linux + // Return a reader that wraps the same device + return &wgTunReader{ + parent: t, + tunDevice: t.tunDevice, + batchSize: 64, + offset: 0, + l: t.l, + }, nil } -func addRoute(prefix netip.Prefix, gateway netroute.Addr) error { - sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) - if err != nil { - return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) - } - defer unix.Close(sock) +func (rm *tun) addIP(t *wgTun, name string, network netip.Prefix) error { + addr := network.Addr() - route := &netroute.RouteMessage{ - Version: unix.RTM_VERSION, - Type: unix.RTM_ADD, - Flags: unix.RTF_UP, - Seq: 1, - } - - if prefix.Addr().Is4() { - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, - unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, - unix.RTAX_GATEWAY: gateway, + if addr.Is4() { + // For IPv4: ifconfig tun0 10.0.0.1/24 + if err := runCommandBSD("ifconfig", name, network.String()); err != nil { + return fmt.Errorf("failed to add IPv4 address: %w", err) } } else { - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, - unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, - unix.RTAX_GATEWAY: gateway, + // For IPv6: ifconfig tun0 inet6 add 2001:db8::1/64 + if err := runCommandBSD("ifconfig", name, "inet6", "add", network.String()); err != nil { + return fmt.Errorf("failed to add IPv6 address: %w", err) } } - data, err := route.Marshal() - if err != nil { - return fmt.Errorf("failed to create route.RouteMessage: %w", err) - } - - _, err = unix.Write(sock, data[:]) - if err != nil { - if errors.Is(err, unix.EEXIST) { - // Try to do a change - route.Type = unix.RTM_CHANGE - data, err = route.Marshal() - if err != nil { - return fmt.Errorf("failed to create route.RouteMessage for change: %w", err) - } - _, err = unix.Write(sock, data[:]) - fmt.Println("DOING CHANGE") - return err - } - return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) - } - return nil } -func delRoute(prefix netip.Prefix, gateway netroute.Addr) error { - sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) +func runCommandBSD(name string, args ...string) error { + cmd := exec.Command(name, args...) + output, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + return fmt.Errorf("%s %s failed: %w\nOutput: %s", name, strings.Join(args, " "), err, string(output)) } - defer unix.Close(sock) + return nil +} - route := netroute.RouteMessage{ - Version: unix.RTM_VERSION, - Type: unix.RTM_DELETE, - Seq: 1, +func renameInterface(fromName, toName string) error { + s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) + if err != nil { + return fmt.Errorf("failed to create socket: %w", err) + } + defer syscall.Close(s) + + fd := uintptr(s) + + var fromNameBytes [unix.IFNAMSIZ]byte + var toNameBytes [unix.IFNAMSIZ]byte + copy(fromNameBytes[:], fromName) + copy(toNameBytes[:], toName) + + ifrr := ifreqRename{ + Name: fromNameBytes, + Data: uintptr(unsafe.Pointer(&toNameBytes)), } - if prefix.Addr().Is4() { - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, - unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, - unix.RTAX_GATEWAY: gateway, - } - } else { - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, - unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, - unix.RTAX_GATEWAY: gateway, - } - } - - data, err := route.Marshal() - if err != nil { - return fmt.Errorf("failed to create route.RouteMessage: %w", err) - } - _, err = unix.Write(sock, data[:]) - if err != nil { - return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) + // Set the device name using SIOCSIFNAME ioctl + _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr))) + if errno != 0 { + return fmt.Errorf("SIOCSIFNAME ioctl failed: %w", errno) } return nil } - -// getLinkAddr Gets the link address for the interface of the given name -func getLinkAddr(name string) (*netroute.LinkAddr, error) { - rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0) - if err != nil { - return nil, err - } - msgs, err := netroute.ParseRIB(unix.NET_RT_IFLIST, rib) - if err != nil { - return nil, err - } - - for _, m := range msgs { - switch m := m.(type) { - case *netroute.InterfaceMessage: - if m.Name == name { - sa, ok := m.Addrs[unix.RTAX_IFP].(*netroute.LinkAddr) - if ok { - return sa, nil - } - } - } - } - - return nil, nil -} diff --git a/overlay/tun_linux.go b/overlay/tun_linux.go index 44d8746..c19a74b 100644 --- a/overlay/tun_linux.go +++ b/overlay/tun_linux.go @@ -1,5 +1,5 @@ -//go:build !android && !e2e_testing -// +build !android,!e2e_testing +//go:build linux && !android && !e2e_testing +// +build linux,!android,!e2e_testing package overlay @@ -9,133 +9,102 @@ import ( "net" "net/netip" "os" - "strings" - "sync/atomic" "time" "unsafe" - "github.com/gaissmai/bart" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + wgtun "golang.zx2c4.com/wireguard/tun" ) type tun struct { - io.ReadWriteCloser - fd int - Device string - vpnNetworks []netip.Prefix - MaxMTU int - DefaultMTU int - TXQueueLen int - deviceIndex int - ioctlFd uintptr - - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[bart.Table[routing.Gateways]] - routeChan chan struct{} + deviceIndex int + ioctlFd uintptr useSystemRoutes bool useSystemRoutesBufferSize int - - l *logrus.Logger } -func (t *tun) Networks() []netip.Prefix { - return t.vpnNetworks -} +func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*wgTun, error) { + deviceName := c.GetString("tun.dev", "") + mtu := c.GetInt("tun.mtu", DefaultMTU) -type ifReq struct { - Name [16]byte - Flags uint16 - pad [8]byte -} - -type ifreqMTU struct { - Name [16]byte - MTU int32 - pad [8]byte -} - -type ifreqQLEN struct { - Name [16]byte - Value int32 - pad [8]byte -} - -func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) { - file := os.NewFile(uintptr(deviceFd), "/dev/net/tun") - - t, err := newTunGeneric(c, l, file, vpnNetworks) + // Create WireGuard TUN device + tunDevice, err := wgtun.CreateTUN(deviceName, mtu) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create TUN device: %w", err) } - t.Device = "tun0" - - return t, nil -} - -func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) { - fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0) + // Get the actual device name + actualName, err := tunDevice.Name() if err != nil { - // If /dev/net/tun doesn't exist, try to create it (will happen in docker) - if os.IsNotExist(err) { - err = os.MkdirAll("/dev/net", 0755) - if err != nil { - return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err) - } - err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200))) - if err != nil { - return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err) - } - - fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0) - if err != nil { - return nil, fmt.Errorf("created /dev/net/tun, but still failed: %w", err) - } - } else { - return nil, err - } + tunDevice.Close() + return nil, fmt.Errorf("failed to get TUN device name: %w", err) } - var req ifReq - req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI) - if multiqueue { - req.Flags |= unix.IFF_MULTI_QUEUE - } - copy(req.Name[:], c.GetString("tun.dev", "")) - if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil { - return nil, err - } - name := strings.Trim(string(req.Name[:]), "\x00") - - file := os.NewFile(uintptr(fd), "/dev/net/tun") - t, err := newTunGeneric(c, l, file, vpnNetworks) - if err != nil { - return nil, err + t := &wgTun{ + tunDevice: tunDevice, + vpnNetworks: vpnNetworks, + MaxMTU: mtu, + DefaultMTU: mtu, + l: l, } - t.Device = name - - return t, nil -} - -func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []netip.Prefix) (*tun, error) { - t := &tun{ - ReadWriteCloser: file, - fd: int(file.Fd()), - vpnNetworks: vpnNetworks, - TXQueueLen: c.GetInt("tun.tx_queue", 500), + // Create Linux-specific route manager + routeManager := &tun{ useSystemRoutes: c.GetBool("tun.use_system_route_table", false), useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0), - l: l, + } + t.routeManager = routeManager + + err = t.reload(c, true) + if err != nil { + tunDevice.Close() + return nil, err } - err := t.reload(c, true) + c.RegisterReloadCallback(func(c *config.C) { + err := t.reload(c, false) + if err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) + + l.WithField("name", actualName).Info("Created WireGuard TUN device") + + return t, nil +} + +func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*wgTun, error) { + // Create TUN device from file descriptor + file := os.NewFile(uintptr(deviceFd), "/dev/net/tun") + mtu := c.GetInt("tun.mtu", DefaultMTU) + tunDevice, err := wgtun.CreateTUNFromFile(file, mtu) if err != nil { + return nil, fmt.Errorf("failed to create TUN device from fd: %w", err) + } + + t := &wgTun{ + tunDevice: tunDevice, + vpnNetworks: vpnNetworks, + MaxMTU: mtu, + DefaultMTU: mtu, + l: l, + } + + // Create Linux-specific route manager + routeManager := &tun{ + useSystemRoutes: c.GetBool("tun.use_system_route_table", false), + useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0), + } + t.routeManager = routeManager + + err = t.reload(c, true) + if err != nil { + tunDevice.Close() return nil, err } @@ -149,273 +118,106 @@ func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []n return t, nil } -func (t *tun) reload(c *config.C, initial bool) error { - routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) +func (rm *tun) Activate(t *wgTun) error { + name, err := t.tunDevice.Name() if err != nil { - return err + return fmt.Errorf("failed to get device name: %w", err) } - if !initial && !routeChange && !c.HasChanged("tun.mtu") { - return nil - } - - routeTree, err := makeRouteTree(t.l, routes, true) - if err != nil { - return err - } - - oldDefaultMTU := t.DefaultMTU - oldMaxMTU := t.MaxMTU - newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU) - newMaxMTU := newDefaultMTU - for i, r := range routes { - if r.MTU == 0 { - routes[i].MTU = newDefaultMTU - } - - if r.MTU > t.MaxMTU { - newMaxMTU = r.MTU - } - } - - t.MaxMTU = newMaxMTU - t.DefaultMTU = newDefaultMTU - - // Teach nebula how to handle the routes before establishing them in the system table - oldRoutes := t.Routes.Swap(&routes) - t.routeTree.Store(routeTree) - - if !initial { - if oldMaxMTU != newMaxMTU { - t.setMTU() - t.l.Infof("Set max MTU to %v was %v", t.MaxMTU, oldMaxMTU) - } - - if oldDefaultMTU != newDefaultMTU { - for i := range t.vpnNetworks { - err := t.setDefaultRoute(t.vpnNetworks[i]) - if err != nil { - t.l.Warn(err) - } else { - t.l.Infof("Set default MTU to %v was %v", t.DefaultMTU, oldDefaultMTU) - } - } - } - - // Remove first, if the system removes a wanted route hopefully it will be re-added next - t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) - - // Ensure any routes we actually want are installed - err = t.addRoutes(true) - if err != nil { - // This should never be called since addRoutes should log its own errors in a reload condition - util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l) - } - } - - return nil -} - -func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { - fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0) - if err != nil { - return nil, err - } - - var req ifReq - req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE) - copy(req.Name[:], t.Device) - if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil { - return nil, err - } - - file := os.NewFile(uintptr(fd), "/dev/net/tun") - - return file, nil -} - -func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { - r, _ := t.routeTree.Load().Lookup(ip) - return r -} - -func (t *tun) Write(b []byte) (int, error) { - var nn int - maximum := len(b) - - for { - n, err := unix.Write(t.fd, b[nn:maximum]) - if n > 0 { - nn += n - } - if nn == len(b) { - return nn, err - } - - if err != nil { - return nn, err - } - - if n == 0 { - return nn, io.ErrUnexpectedEOF - } - } -} - -func (t *tun) deviceBytes() (o [16]byte) { - for i, c := range t.Device { - o[i] = byte(c) - } - return -} - -func hasNetlinkAddr(al []*netlink.Addr, x netlink.Addr) bool { - for i := range al { - if al[i].Equal(x) { - return true - } - } - return false -} - -// addIPs uses netlink to add all addresses that don't exist, then it removes ones that should not be there -func (t *tun) addIPs(link netlink.Link) error { - newAddrs := make([]*netlink.Addr, len(t.vpnNetworks)) - for i := range t.vpnNetworks { - newAddrs[i] = &netlink.Addr{ - IPNet: &net.IPNet{ - IP: t.vpnNetworks[i].Addr().AsSlice(), - Mask: net.CIDRMask(t.vpnNetworks[i].Bits(), t.vpnNetworks[i].Addr().BitLen()), - }, - Label: t.vpnNetworks[i].Addr().Zone(), - } - } - - //add all new addresses - for i := range newAddrs { - //AddrReplace still adds new IPs, but if their properties change it will change them as well - if err := netlink.AddrReplace(link, newAddrs[i]); err != nil { - return err - } - } - - //iterate over remainder, remove whoever shouldn't be there - al, err := netlink.AddrList(link, netlink.FAMILY_ALL) - if err != nil { - return fmt.Errorf("failed to get tun address list: %s", err) - } - - for i := range al { - if hasNetlinkAddr(newAddrs, al[i]) { - continue - } - err = netlink.AddrDel(link, &al[i]) - if err != nil { - t.l.WithError(err).Error("failed to remove address from tun address list") - } else { - t.l.WithField("removed", al[i].String()).Info("removed address not listed in cert(s)") - } - } - - return nil -} - -func (t *tun) Activate() error { - devName := t.deviceBytes() - - if t.useSystemRoutes { + if t.routeManager.useSystemRoutes { t.watchRoutes() } + // Get the netlink device + link, err := netlink.LinkByName(name) + if err != nil { + return fmt.Errorf("failed to get tun device link: %s", err) + } + + rm.deviceIndex = link.Attrs().Index + + // Open socket for ioctl operations s, err := unix.Socket( - unix.AF_INET, //because everything we use t.ioctlFd for is address family independent, this is fine + unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP, ) if err != nil { return err } - t.ioctlFd = uintptr(s) + rm.ioctlFd = uintptr(s) - // Set the device name - ifrf := ifReq{Name: devName} - if err = ioctl(t.ioctlFd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { - return fmt.Errorf("failed to set tun device name: %s", err) - } - - link, err := netlink.LinkByName(t.Device) - if err != nil { - return fmt.Errorf("failed to get tun device link: %s", err) - } - - t.deviceIndex = link.Attrs().Index - - // Setup our default MTU - t.setMTU() + // Set the MTU + rm.SetMTU(t, t.MaxMTU) // Set the transmit queue length - ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)} - if err = ioctl(t.ioctlFd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil { - // If we can't set the queue length nebula will still work but it may lead to packet loss + txQueueLen := 500 // default + devName := deviceBytes(name) + ifrq := ifreqQLEN{Name: devName, Value: int32(txQueueLen)} + if err = ioctl(t.routeManager.ioctlFd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil { t.l.WithError(err).Error("Failed to set tun tx queue length") } + // Disable IPv6 link-local address generation const modeNone = 1 if err = netlink.LinkSetIP6AddrGenMode(link, modeNone); err != nil { t.l.WithError(err).Warn("Failed to disable link local address generation") } - if err = t.addIPs(link); err != nil { + // Add IP addresses + if err = t.routeManager.addIPs(t, link); err != nil { return err } // Bring up the interface - ifrf.Flags = ifrf.Flags | unix.IFF_UP - if err = ioctl(t.ioctlFd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { + if err = netlink.LinkSetUp(link); err != nil { return fmt.Errorf("failed to bring the tun device up: %s", err) } - //set route MTU + // Set route MTU for i := range t.vpnNetworks { - if err = t.setDefaultRoute(t.vpnNetworks[i]); err != nil { + if err = t.routeManager.SetDefaultRoute(t, t.vpnNetworks[i]); err != nil { return fmt.Errorf("failed to set default route MTU: %w", err) } } // Set the routes - if err = t.addRoutes(false); err != nil { + if err = t.routeManager.AddRoutes(t, false); err != nil { return err } - // Run the interface - ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING - if err = ioctl(t.ioctlFd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { - return fmt.Errorf("failed to run tun device: %s", err) - } - return nil } -func (t *tun) setMTU() { - // Set the MTU on the device - ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)} - if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil { - // This is currently a non fatal condition because the route table must have the MTU set appropriately as well +func (rm *tun) SetMTU(t *wgTun, mtu int) { + name, err := t.tunDevice.Name() + if err != nil { + t.l.WithError(err).Error("Failed to get device name for MTU set") + return + } + + link, err := netlink.LinkByName(name) + if err != nil { + t.l.WithError(err).Error("Failed to get link for MTU set") + return + } + + if err := netlink.LinkSetMTU(link, mtu); err != nil { t.l.WithError(err).Error("Failed to set tun mtu") } } -func (t *tun) setDefaultRoute(cidr netip.Prefix) error { +func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error { dr := &net.IPNet{ IP: cidr.Masked().Addr().AsSlice(), Mask: net.CIDRMask(cidr.Bits(), cidr.Addr().BitLen()), } nr := netlink.Route{ - LinkIndex: t.deviceIndex, + LinkIndex: t.routeManager.deviceIndex, Dst: dr, MTU: t.DefaultMTU, - AdvMSS: t.advMSS(Route{}), + AdvMSS: advMSS(Route{}, t.DefaultMTU, t.MaxMTU), Scope: unix.RT_SCOPE_LINK, Src: net.IP(cidr.Addr().AsSlice()), Protocol: unix.RTPROT_KERNEL, @@ -425,7 +227,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error { err := netlink.RouteReplace(&nr) if err != nil { t.l.WithError(err).WithField("cidr", cidr).Warn("Failed to set default route MTU, retrying") - //retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument` + // Retry twice more for i := 0; i < 2; i++ { time.Sleep(100 * time.Millisecond) err = netlink.RouteReplace(&nr) @@ -443,8 +245,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error { return nil } -func (t *tun) addRoutes(logErrors bool) error { - // Path routes +func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error { routes := *t.Routes.Load() for _, r := range routes { if !r.Install { @@ -457,10 +258,10 @@ func (t *tun) addRoutes(logErrors bool) error { } nr := netlink.Route{ - LinkIndex: t.deviceIndex, + LinkIndex: t.routeManager.deviceIndex, Dst: dr, MTU: r.MTU, - AdvMSS: t.advMSS(r), + AdvMSS: advMSS(r, t.DefaultMTU, t.MaxMTU), Scope: unix.RT_SCOPE_LINK, } @@ -484,7 +285,7 @@ func (t *tun) addRoutes(logErrors bool) error { return nil } -func (t *tun) removeRoutes(routes []Route) { +func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) { for _, r := range routes { if !r.Install { continue @@ -496,10 +297,10 @@ func (t *tun) removeRoutes(routes []Route) { } nr := netlink.Route{ - LinkIndex: t.deviceIndex, + LinkIndex: t.routeManager.deviceIndex, Dst: dr, MTU: r.MTU, - AdvMSS: t.advMSS(r), + AdvMSS: advMSS(r, t.DefaultMTU, t.MaxMTU), Scope: unix.RT_SCOPE_LINK, } @@ -516,30 +317,108 @@ func (t *tun) removeRoutes(routes []Route) { } } -func (t *tun) Name() string { - return t.Device +func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) { + // For Linux with WireGuard TUN, we can reuse the same device + // The vectorized I/O will handle batching + return &wgTunReader{ + parent: t, + tunDevice: t.tunDevice, + batchSize: 64, // Default batch size + offset: 0, + l: t.l, + }, nil } -func (t *tun) advMSS(r Route) int { +// Helper functions + +func deviceBytes(name string) [16]byte { + var o [16]byte + for i, c := range name { + if i >= 16 { + break + } + o[i] = byte(c) + } + return o +} + +func advMSS(r Route, defaultMTU, maxMTU int) int { mtu := r.MTU if r.MTU == 0 { - mtu = t.DefaultMTU + mtu = defaultMTU } // We only need to set advmss if the route MTU does not match the device MTU - if mtu != t.MaxMTU { + if mtu != maxMTU { return mtu - 40 } return 0 } -func (t *tun) watchRoutes() { +type ifreqQLEN struct { + Name [16]byte + Value int32 + pad [8]byte +} + +func hasNetlinkAddr(al []*netlink.Addr, x netlink.Addr) bool { + for i := range al { + if al[i].Equal(x) { + return true + } + } + return false +} + +func (rm *tun) addIPs(t *wgTun, link netlink.Link) error { + newAddrs := make([]*netlink.Addr, len(t.vpnNetworks)) + for i := range t.vpnNetworks { + newAddrs[i] = &netlink.Addr{ + IPNet: &net.IPNet{ + IP: t.vpnNetworks[i].Addr().AsSlice(), + Mask: net.CIDRMask(t.vpnNetworks[i].Bits(), t.vpnNetworks[i].Addr().BitLen()), + }, + Label: t.vpnNetworks[i].Addr().Zone(), + } + } + + // Add all new addresses + for i := range newAddrs { + if err := netlink.AddrReplace(link, newAddrs[i]); err != nil { + return err + } + } + + // Iterate over remainder, remove whoever shouldn't be there + al, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return fmt.Errorf("failed to get tun address list: %s", err) + } + + for i := range al { + if hasNetlinkAddr(newAddrs, al[i]) { + continue + } + err = netlink.AddrDel(link, &al[i]) + if err != nil { + t.l.WithError(err).Error("failed to remove address from tun address list") + } else { + t.l.WithField("removed", al[i].String()).Info("removed address not listed in cert(s)") + } + } + + return nil +} + +// watchRoutes monitors system route changes +func (t *wgTun) watchRoutes() { + rch := make(chan netlink.RouteUpdate) doneChan := make(chan struct{}) netlinkOptions := netlink.RouteSubscribeOptions{ - ReceiveBufferSize: t.useSystemRoutesBufferSize, - ReceiveBufferForceSize: t.useSystemRoutesBufferSize != 0, + ReceiveBufferSize: t.routeManager.useSystemRoutesBufferSize, + ReceiveBufferForceSize: t.routeManager.useSystemRoutesBufferSize != 0, ErrorCallback: func(e error) { t.l.WithError(e).Errorf("netlink error") }, } @@ -557,87 +436,19 @@ func (t *tun) watchRoutes() { if ok { t.updateRoutes(r) } else { - // may be should do something here as - // netlink stops sending updates return } case <-doneChan: - // netlink.RouteSubscriber will close the rch for us return } } }() } -func (t *tun) isGatewayInVpnNetworks(gwAddr netip.Addr) bool { - withinNetworks := false - for i := range t.vpnNetworks { - if t.vpnNetworks[i].Contains(gwAddr) { - withinNetworks = true - break - } - } - - return withinNetworks -} - -func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways { - - var gateways routing.Gateways - - link, err := netlink.LinkByName(t.Device) - if err != nil { - t.l.WithField("Devicename", t.Device).Error("Ignoring route update: failed to get link by name") - return gateways - } - - // If this route is relevant to our interface and there is a gateway then add it - if r.LinkIndex == link.Attrs().Index && len(r.Gw) > 0 { - gwAddr, ok := netip.AddrFromSlice(r.Gw) - if !ok { - t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway address") - } else { - gwAddr = gwAddr.Unmap() - - if !t.isGatewayInVpnNetworks(gwAddr) { - // Gateway isn't in our overlay network, ignore - t.l.WithField("route", r).Debug("Ignoring route update, not in our network") - } else { - gateways = append(gateways, routing.NewGateway(gwAddr, 1)) - } - } - } - - for _, p := range r.MultiPath { - // If this route is relevant to our interface and there is a gateway then add it - if p.LinkIndex == link.Attrs().Index && len(p.Gw) > 0 { - gwAddr, ok := netip.AddrFromSlice(p.Gw) - if !ok { - t.l.WithField("route", r).Debug("Ignoring multipath route update, invalid gateway address") - } else { - gwAddr = gwAddr.Unmap() - - if !t.isGatewayInVpnNetworks(gwAddr) { - // Gateway isn't in our overlay network, ignore - t.l.WithField("route", r).Debug("Ignoring route update, not in our network") - } else { - // p.Hops+1 = weight of the route - gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1)) - } - } - } - } - - routing.CalculateBucketsForGateways(gateways) - return gateways -} - -func (t *tun) updateRoutes(r netlink.RouteUpdate) { - - gateways := t.getGatewaysFromRoute(&r.Route) +func (t *wgTun) updateRoutes(r netlink.RouteUpdate) { + gateways := t.getGatewaysFromRoute(&r.Route, t.routeManager.deviceIndex) if len(gateways) == 0 { - // No gateways relevant to our network, no routing changes required. t.l.WithField("route", r).Debug("Ignoring route update, no gateways") return } @@ -661,7 +472,6 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) { if r.Type == unix.RTM_NEWROUTE { t.l.WithField("destination", dst).WithField("via", gateways).Info("Adding route") newTree.Insert(dst, gateways) - } else { t.l.WithField("destination", dst).WithField("via", gateways).Info("Removing route") newTree.Delete(dst) @@ -669,18 +479,71 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) { t.routeTree.Store(newTree) } -func (t *tun) Close() error { - if t.routeChan != nil { - close(t.routeChan) +func (t *wgTun) getGatewaysFromRoute(r *netlink.Route, deviceIndex int) routing.Gateways { + var gateways routing.Gateways + + name, err := t.tunDevice.Name() + if err != nil { + t.l.Error("Ignoring route update: failed to get device name") + return gateways } - if t.ReadWriteCloser != nil { - _ = t.ReadWriteCloser.Close() + link, err := netlink.LinkByName(name) + if err != nil { + t.l.WithField("DeviceName", name).Error("Ignoring route update: failed to get link by name") + return gateways } - if t.ioctlFd > 0 { - _ = os.NewFile(t.ioctlFd, "ioctlFd").Close() + // If this route is relevant to our interface and there is a gateway then add it + if r.LinkIndex == link.Attrs().Index && len(r.Gw) > 0 { + gwAddr, ok := netip.AddrFromSlice(r.Gw) + if !ok { + t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway address") + } else { + gwAddr = gwAddr.Unmap() + + if !t.isGatewayInVpnNetworks(gwAddr) { + t.l.WithField("route", r).Debug("Ignoring route update, not in our network") + } else { + gateways = append(gateways, routing.NewGateway(gwAddr, 1)) + } + } } + for _, p := range r.MultiPath { + if p.LinkIndex == link.Attrs().Index && len(p.Gw) > 0 { + gwAddr, ok := netip.AddrFromSlice(p.Gw) + if !ok { + t.l.WithField("route", r).Debug("Ignoring multipath route update, invalid gateway address") + } else { + gwAddr = gwAddr.Unmap() + + if !t.isGatewayInVpnNetworks(gwAddr) { + t.l.WithField("route", r).Debug("Ignoring route update, not in our network") + } else { + gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1)) + } + } + } + } + + routing.CalculateBucketsForGateways(gateways) + return gateways +} + +func (t *wgTun) isGatewayInVpnNetworks(gwAddr netip.Addr) bool { + for i := range t.vpnNetworks { + if t.vpnNetworks[i].Contains(gwAddr) { + return true + } + } + return false +} + +func ioctl(a1, a2, a3 uintptr) error { + _, _, errno := unix.Syscall(unix.SYS_IOCTL, a1, a2, a3) + if errno != 0 { + return errno + } return nil } diff --git a/overlay/tun_netbsd.go b/overlay/tun_netbsd.go index 49ac19f..2fecb3d 100644 --- a/overlay/tun_netbsd.go +++ b/overlay/tun_netbsd.go @@ -1,111 +1,59 @@ -//go:build !e2e_testing -// +build !e2e_testing +//go:build netbsd && !e2e_testing +// +build netbsd,!e2e_testing package overlay import ( - "errors" "fmt" "io" "net/netip" - "os" - "regexp" - "sync/atomic" - "syscall" - "unsafe" + "os/exec" + "strconv" + "strings" - "github.com/gaissmai/bart" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" - netroute "golang.org/x/net/route" - "golang.org/x/sys/unix" + wgtun "golang.zx2c4.com/wireguard/tun" ) -const ( - SIOCAIFADDR_IN6 = 0x8080696b - TUNSIFHEAD = 0x80047442 - TUNSIFMODE = 0x80047458 -) +type tun struct{} -type ifreqAlias4 struct { - Name [unix.IFNAMSIZ]byte - Addr unix.RawSockaddrInet4 - DstAddr unix.RawSockaddrInet4 - MaskAddr unix.RawSockaddrInet4 +func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*wgTun, error) { + return nil, fmt.Errorf("newTunFromFd not supported on NetBSD") } -type ifreqAlias6 struct { - Name [unix.IFNAMSIZ]byte - Addr unix.RawSockaddrInet6 - DstAddr unix.RawSockaddrInet6 - PrefixMask unix.RawSockaddrInet6 - Flags uint32 - Lifetime addrLifetime -} +func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*wgTun, error) { + deviceName := c.GetString("tun.dev", "tun") + mtu := c.GetInt("tun.mtu", DefaultMTU) -type ifreq struct { - Name [unix.IFNAMSIZ]byte - data int -} - -type addrLifetime struct { - Expire uint64 - Preferred uint64 - Vltime uint32 - Pltime uint32 -} - -type tun struct { - Device string - vpnNetworks []netip.Prefix - MTU int - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[bart.Table[routing.Gateways]] - l *logrus.Logger - f *os.File - fd int -} - -var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`) - -func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) { - return nil, fmt.Errorf("newTunFromFd not supported in NetBSD") -} - -func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { - // Try to open tun device - var err error - deviceName := c.GetString("tun.dev", "") - if deviceName == "" { - return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") - } - if !deviceNameRE.MatchString(deviceName) { - return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") - } - - fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0) + // Create WireGuard TUN device + tunDevice, err := wgtun.CreateTUN(deviceName, mtu) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create TUN device: %w", err) } - err = unix.SetNonblock(fd, true) + // Get the actual device name + actualName, err := tunDevice.Name() if err != nil { - l.WithError(err).Warn("Failed to set the tun device as nonblocking") + tunDevice.Close() + return nil, fmt.Errorf("failed to get TUN device name: %w", err) } - t := &tun{ - f: os.NewFile(uintptr(fd), ""), - fd: fd, - Device: deviceName, + t := &wgTun{ + tunDevice: tunDevice, vpnNetworks: vpnNetworks, - MTU: c.GetInt("tun.mtu", DefaultMTU), + MaxMTU: mtu, + DefaultMTU: mtu, l: l, } + // Create NetBSD-specific route manager + t.routeManager = &tun{} + err = t.reload(c, true) if err != nil { + tunDevice.Close() return nil, err } @@ -116,294 +64,86 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) ( } }) + l.WithField("name", actualName).Info("Created WireGuard TUN device") + return t, nil } -func (t *tun) Close() error { - if t.f != nil { - if err := t.f.Close(); err != nil { - return fmt.Errorf("error closing tun file: %w", err) - } - - // t.f.Close should have handled it for us but let's be extra sure - _ = unix.Close(t.fd) - - s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) - if err != nil { - return err - } - defer syscall.Close(s) - - ifr := ifreq{Name: t.deviceBytes()} - err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifr))) - return err - } - return nil -} - -func (t *tun) Read(to []byte) (int, error) { - rc, err := t.f.SyscallConn() +func (rm *tun) Activate(t *wgTun) error { + name, err := t.tunDevice.Name() if err != nil { - return 0, fmt.Errorf("failed to get syscall conn for tun: %w", err) + return fmt.Errorf("failed to get device name: %w", err) } - var errno syscall.Errno - var n uintptr - err = rc.Read(func(fd uintptr) bool { - // first 4 bytes is protocol family, in network byte order - head := [4]byte{} - iovecs := []syscall.Iovec{ - {&head[0], 4}, - {&to[0], uint64(len(to))}, - } + // Set the MTU + rm.SetMTU(t, t.MaxMTU) - n, _, errno = syscall.Syscall(syscall.SYS_READV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2)) - if errno.Temporary() { - // We got an EAGAIN, EINTR, or EWOULDBLOCK, go again - return false - } - return true - }) - if err != nil { - if err == syscall.EBADF || err.Error() == "use of closed file" { - // Go doesn't export poll.ErrFileClosing but happily reports it to us so here we are - // https://github.com/golang/go/blob/master/src/internal/poll/fd_poll_runtime.go#L121 - return 0, os.ErrClosed - } - return 0, fmt.Errorf("failed to make read call for tun: %w", err) - } - - if errno != 0 { - return 0, fmt.Errorf("failed to make inner read call for tun: %w", errno) - } - - // fix bytes read number to exclude header - bytesRead := int(n) - if bytesRead < 0 { - return bytesRead, nil - } else if bytesRead < 4 { - return 0, nil - } else { - return bytesRead - 4, nil - } -} - -// Write is only valid for single threaded use -func (t *tun) Write(from []byte) (int, error) { - if len(from) <= 1 { - return 0, syscall.EIO - } - - ipVer := from[0] >> 4 - var head [4]byte - // first 4 bytes is protocol family, in network byte order - if ipVer == 4 { - head[3] = syscall.AF_INET - } else if ipVer == 6 { - head[3] = syscall.AF_INET6 - } else { - return 0, fmt.Errorf("unable to determine IP version from packet") - } - - rc, err := t.f.SyscallConn() - if err != nil { - return 0, err - } - - var errno syscall.Errno - var n uintptr - err = rc.Write(func(fd uintptr) bool { - iovecs := []syscall.Iovec{ - {&head[0], 4}, - {&from[0], uint64(len(from))}, - } - - n, _, errno = syscall.Syscall(syscall.SYS_WRITEV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2)) - // According to NetBSD documentation for TUN, writes will only return errors in which - // this packet will never be delivered so just go on living life. - return true - }) - if err != nil { - return 0, err - } - - if errno != 0 { - return 0, errno - } - - return int(n) - 4, err -} - -func (t *tun) addIp(cidr netip.Prefix) error { - if cidr.Addr().Is4() { - var req ifreqAlias4 - req.Name = t.deviceBytes() - req.Addr = unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: cidr.Addr().As4(), - } - req.DstAddr = unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: cidr.Addr().As4(), - } - req.MaskAddr = unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: prefixToMask(cidr).As4(), - } - - s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) - if err != nil { - return err - } - defer syscall.Close(s) - - if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&req))); err != nil { - return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr(), err) - } - - return nil - } - - if cidr.Addr().Is6() { - var req ifreqAlias6 - req.Name = t.deviceBytes() - req.Addr = unix.RawSockaddrInet6{ - Len: unix.SizeofSockaddrInet6, - Family: unix.AF_INET6, - Addr: cidr.Addr().As16(), - } - req.PrefixMask = unix.RawSockaddrInet6{ - Len: unix.SizeofSockaddrInet6, - Family: unix.AF_INET6, - Addr: prefixToMask(cidr).As16(), - } - req.Lifetime = addrLifetime{ - Vltime: 0xffffffff, - Pltime: 0xffffffff, - } - - s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP) - if err != nil { - return err - } - defer syscall.Close(s) - - if err := ioctl(uintptr(s), SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&req))); err != nil { - return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err) - } - return nil - } - - return fmt.Errorf("unknown address type %v", cidr) -} - -func (t *tun) Activate() error { - mode := int32(unix.IFF_BROADCAST) - err := ioctl(uintptr(t.fd), TUNSIFMODE, uintptr(unsafe.Pointer(&mode))) - if err != nil { - return fmt.Errorf("failed to set tun device mode: %w", err) - } - - v := 1 - err = ioctl(uintptr(t.fd), TUNSIFHEAD, uintptr(unsafe.Pointer(&v))) - if err != nil { - return fmt.Errorf("failed to set tun device head: %w", err) - } - - err = t.doIoctlByName(unix.SIOCSIFMTU, uint32(t.MTU)) - if err != nil { - return fmt.Errorf("failed to set tun mtu: %w", err) - } - - for i := range t.vpnNetworks { - err = t.addIp(t.vpnNetworks[i]) - if err != nil { + // Add IP addresses + for _, network := range t.vpnNetworks { + if err := rm.addIP(t, name, network); err != nil { return err } } - return t.addRoutes(false) -} + // Bring up the interface + if err := runCommandBSD("ifconfig", name, "up"); err != nil { + return fmt.Errorf("failed to bring up interface: %w", err) + } -func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { - s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) - if err != nil { + // Set the routes + if err := rm.AddRoutes(t, false); err != nil { return err } - defer syscall.Close(s) - - ir := ifreq{Name: t.deviceBytes(), data: int(value)} - err = ioctl(uintptr(s), ctl, uintptr(unsafe.Pointer(&ir))) - return err -} - -func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) - if err != nil { - return err - } - - if !initial && !change { - return nil - } - - routeTree, err := makeRouteTree(t.l, routes, false) - if err != nil { - return err - } - - // Teach nebula how to handle the routes before establishing them in the system table - oldRoutes := t.Routes.Swap(&routes) - t.routeTree.Store(routeTree) - - if !initial { - // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) - if err != nil { - util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) - } - - // Ensure any routes we actually want are installed - err = t.addRoutes(true) - if err != nil { - // Catch any stray logs - util.LogWithContextIfNeeded("Failed to add routes", err, t.l) - } - } return nil } -func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { - r, _ := t.routeTree.Load().Lookup(ip) - return r +func (rm *tun) SetMTU(t *wgTun, mtu int) { + name, err := t.tunDevice.Name() + if err != nil { + t.l.WithError(err).Error("Failed to get device name for MTU set") + return + } + + if err := runCommandBSD("ifconfig", name, "mtu", strconv.Itoa(mtu)); err != nil { + t.l.WithError(err).Error("Failed to set tun mtu") + } } -func (t *tun) Networks() []netip.Prefix { - return t.vpnNetworks +func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error { + // On NetBSD, routes are set via ifconfig and route commands + return nil } -func (t *tun) Name() string { - return t.Device -} +func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error { + name, err := t.tunDevice.Name() + if err != nil { + return fmt.Errorf("failed to get device name: %w", err) + } -func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { - return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd") -} - -func (t *tun) addRoutes(logErrors bool) error { routes := *t.Routes.Load() - for _, r := range routes { - if len(r.Via) == 0 || !r.Install { - // We don't allow route MTUs so only install routes with a via + if !r.Install { continue } - err := addRoute(r.Cidr, t.vpnNetworks) + // Add route using route command + args := []string{"add"} + + if r.Cidr.Addr().Is6() { + args = append(args, "-inet6") + } else { + args = append(args, "-inet") + } + + args = append(args, r.Cidr.String(), "-interface", name) + + if r.Metric > 0 { + // NetBSD doesn't support route metrics directly like Linux + t.l.WithField("route", r).Warn("Route metrics are not fully supported on NetBSD") + } + + err := runCommandBSD("route", args...) if err != nil { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { @@ -419,131 +159,72 @@ func (t *tun) addRoutes(logErrors bool) error { return nil } -func (t *tun) removeRoutes(routes []Route) error { +func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) { + name, err := t.tunDevice.Name() + if err != nil { + t.l.WithError(err).Error("Failed to get device name for route removal") + return + } + for _, r := range routes { if !r.Install { continue } - err := delRoute(r.Cidr, t.vpnNetworks) + args := []string{"delete"} + + if r.Cidr.Addr().Is6() { + args = append(args, "-inet6") + } else { + args = append(args, "-inet") + } + + args = append(args, r.Cidr.String(), "-interface", name) + + err := runCommandBSD("route", args...) if err != nil { t.l.WithError(err).WithField("route", r).Error("Failed to remove route") } else { t.l.WithField("route", r).Info("Removed route") } } - return nil } -func (t *tun) deviceBytes() (o [16]byte) { - for i, c := range t.Device { - o[i] = byte(c) - } - return +func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) { + // NetBSD doesn't support multi-queue TUN devices in the same way as Linux + // Return a reader that wraps the same device + return &wgTunReader{ + parent: t, + tunDevice: t.tunDevice, + batchSize: 64, + offset: 0, + l: t.l, + }, nil } -func addRoute(prefix netip.Prefix, gateways []netip.Prefix) error { - sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) - if err != nil { - return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) - } - defer unix.Close(sock) +func (rm *tun) addIP(t *wgTun, name string, network netip.Prefix) error { + addr := network.Addr() - route := &netroute.RouteMessage{ - Version: unix.RTM_VERSION, - Type: unix.RTM_ADD, - Flags: unix.RTF_UP | unix.RTF_GATEWAY, - Seq: 1, - } - - if prefix.Addr().Is4() { - gw, err := selectGateway(prefix, gateways) - if err != nil { - return err - } - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, - unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, - unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()}, + if addr.Is4() { + // For IPv4: ifconfig tun0 10.0.0.1/24 + if err := runCommandBSD("ifconfig", name, network.String()); err != nil { + return fmt.Errorf("failed to add IPv4 address: %w", err) } } else { - gw, err := selectGateway(prefix, gateways) - if err != nil { - return err + // For IPv6: ifconfig tun0 inet6 add 2001:db8::1/64 + if err := runCommandBSD("ifconfig", name, "inet6", "add", network.String()); err != nil { + return fmt.Errorf("failed to add IPv6 address: %w", err) } - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, - unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, - unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()}, - } - } - - data, err := route.Marshal() - if err != nil { - return fmt.Errorf("failed to create route.RouteMessage: %w", err) - } - - _, err = unix.Write(sock, data[:]) - if err != nil { - if errors.Is(err, unix.EEXIST) { - // Try to do a change - route.Type = unix.RTM_CHANGE - data, err = route.Marshal() - if err != nil { - return fmt.Errorf("failed to create route.RouteMessage for change: %w", err) - } - _, err = unix.Write(sock, data[:]) - return err - } - return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) } return nil } -func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error { - sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) +func runCommandBSD(name string, args ...string) error { + cmd := exec.Command(name, args...) + output, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + return fmt.Errorf("%s %s failed: %w\nOutput: %s", name, strings.Join(args, " "), err, string(output)) } - defer unix.Close(sock) - - route := netroute.RouteMessage{ - Version: unix.RTM_VERSION, - Type: unix.RTM_DELETE, - Seq: 1, - } - - if prefix.Addr().Is4() { - gw, err := selectGateway(prefix, gateways) - if err != nil { - return err - } - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, - unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, - unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()}, - } - } else { - gw, err := selectGateway(prefix, gateways) - if err != nil { - return err - } - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, - unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, - unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()}, - } - } - - data, err := route.Marshal() - if err != nil { - return fmt.Errorf("failed to create route.RouteMessage: %w", err) - } - _, err = unix.Write(sock, data[:]) - if err != nil { - return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) - } - return nil } diff --git a/overlay/tun_notwin.go b/overlay/tun_notwin.go deleted file mode 100644 index 2fab927..0000000 --- a/overlay/tun_notwin.go +++ /dev/null @@ -1,14 +0,0 @@ -//go:build !windows -// +build !windows - -package overlay - -import "syscall" - -func ioctl(a1, a2, a3 uintptr) error { - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, a1, a2, a3) - if errno != 0 { - return errno - } - return nil -} diff --git a/overlay/tun_openbsd.go b/overlay/tun_openbsd.go index 52d5297..9227b8a 100644 --- a/overlay/tun_openbsd.go +++ b/overlay/tun_openbsd.go @@ -1,104 +1,59 @@ -//go:build !e2e_testing -// +build !e2e_testing +//go:build openbsd && !e2e_testing +// +build openbsd,!e2e_testing package overlay import ( - "errors" "fmt" "io" "net/netip" - "os" - "regexp" - "sync/atomic" - "syscall" - "unsafe" + "os/exec" + "strconv" + "strings" - "github.com/gaissmai/bart" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" - netroute "golang.org/x/net/route" - "golang.org/x/sys/unix" + wgtun "golang.zx2c4.com/wireguard/tun" ) -const ( - SIOCAIFADDR_IN6 = 0x8080691a -) +type tun struct{} -type ifreqAlias4 struct { - Name [unix.IFNAMSIZ]byte - Addr unix.RawSockaddrInet4 - DstAddr unix.RawSockaddrInet4 - MaskAddr unix.RawSockaddrInet4 +func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*wgTun, error) { + return nil, fmt.Errorf("newTunFromFd not supported on OpenBSD") } -type ifreqAlias6 struct { - Name [unix.IFNAMSIZ]byte - Addr unix.RawSockaddrInet6 - DstAddr unix.RawSockaddrInet6 - PrefixMask unix.RawSockaddrInet6 - Flags uint32 - Lifetime [2]uint32 -} +func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*wgTun, error) { + deviceName := c.GetString("tun.dev", "tun") + mtu := c.GetInt("tun.mtu", DefaultMTU) -type ifreq struct { - Name [unix.IFNAMSIZ]byte - data int -} - -type tun struct { - Device string - vpnNetworks []netip.Prefix - MTU int - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[bart.Table[routing.Gateways]] - l *logrus.Logger - f *os.File - fd int - // cache out buffer since we need to prepend 4 bytes for tun metadata - out []byte -} - -var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`) - -func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) { - return nil, fmt.Errorf("newTunFromFd not supported in openbsd") -} - -func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { - // Try to open tun device - var err error - deviceName := c.GetString("tun.dev", "") - if deviceName == "" { - return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") - } - if !deviceNameRE.MatchString(deviceName) { - return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") - } - - fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0) + // Create WireGuard TUN device + tunDevice, err := wgtun.CreateTUN(deviceName, mtu) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create TUN device: %w", err) } - err = unix.SetNonblock(fd, true) + // Get the actual device name + actualName, err := tunDevice.Name() if err != nil { - l.WithError(err).Warn("Failed to set the tun device as nonblocking") + tunDevice.Close() + return nil, fmt.Errorf("failed to get TUN device name: %w", err) } - t := &tun{ - f: os.NewFile(uintptr(fd), ""), - fd: fd, - Device: deviceName, + t := &wgTun{ + tunDevice: tunDevice, vpnNetworks: vpnNetworks, - MTU: c.GetInt("tun.mtu", DefaultMTU), + MaxMTU: mtu, + DefaultMTU: mtu, l: l, } + // Create OpenBSD-specific route manager + t.routeManager = &tun{} + err = t.reload(c, true) if err != nil { + tunDevice.Close() return nil, err } @@ -109,221 +64,86 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) ( } }) + l.WithField("name", actualName).Info("Created WireGuard TUN device") + return t, nil } -func (t *tun) Close() error { - if t.f != nil { - if err := t.f.Close(); err != nil { - return fmt.Errorf("error closing tun file: %w", err) - } - - // t.f.Close should have handled it for us but let's be extra sure - _ = unix.Close(t.fd) - } - return nil -} - -func (t *tun) Read(to []byte) (int, error) { - buf := make([]byte, len(to)+4) - - n, err := t.f.Read(buf) - - copy(to, buf[4:]) - return n - 4, err -} - -// Write is only valid for single threaded use -func (t *tun) Write(from []byte) (int, error) { - buf := t.out - if cap(buf) < len(from)+4 { - buf = make([]byte, len(from)+4) - t.out = buf - } - buf = buf[:len(from)+4] - - if len(from) == 0 { - return 0, syscall.EIO - } - - // Determine the IP Family for the NULL L2 Header - ipVer := from[0] >> 4 - if ipVer == 4 { - buf[3] = syscall.AF_INET - } else if ipVer == 6 { - buf[3] = syscall.AF_INET6 - } else { - return 0, fmt.Errorf("unable to determine IP version from packet") - } - - copy(buf[4:], from) - - n, err := t.f.Write(buf) - return n - 4, err -} - -func (t *tun) addIp(cidr netip.Prefix) error { - if cidr.Addr().Is4() { - var req ifreqAlias4 - req.Name = t.deviceBytes() - req.Addr = unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: cidr.Addr().As4(), - } - req.DstAddr = unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: cidr.Addr().As4(), - } - req.MaskAddr = unix.RawSockaddrInet4{ - Len: unix.SizeofSockaddrInet4, - Family: unix.AF_INET, - Addr: prefixToMask(cidr).As4(), - } - - s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) - if err != nil { - return err - } - defer syscall.Close(s) - - if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&req))); err != nil { - return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr(), err) - } - - err = addRoute(cidr, t.vpnNetworks) - if err != nil { - return fmt.Errorf("failed to set route for vpn network %v: %w", cidr, err) - } - - return nil - } - - if cidr.Addr().Is6() { - var req ifreqAlias6 - req.Name = t.deviceBytes() - req.Addr = unix.RawSockaddrInet6{ - Len: unix.SizeofSockaddrInet6, - Family: unix.AF_INET6, - Addr: cidr.Addr().As16(), - } - req.PrefixMask = unix.RawSockaddrInet6{ - Len: unix.SizeofSockaddrInet6, - Family: unix.AF_INET6, - Addr: prefixToMask(cidr).As16(), - } - req.Lifetime[0] = 0xffffffff - req.Lifetime[1] = 0xffffffff - - s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP) - if err != nil { - return err - } - defer syscall.Close(s) - - if err := ioctl(uintptr(s), SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&req))); err != nil { - return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err) - } - - return nil - } - - return fmt.Errorf("unknown address type %v", cidr) -} - -func (t *tun) Activate() error { - err := t.doIoctlByName(unix.SIOCSIFMTU, uint32(t.MTU)) +func (rm *tun) Activate(t *wgTun) error { + name, err := t.tunDevice.Name() if err != nil { - return fmt.Errorf("failed to set tun mtu: %w", err) + return fmt.Errorf("failed to get device name: %w", err) } - for i := range t.vpnNetworks { - err = t.addIp(t.vpnNetworks[i]) - if err != nil { + // Set the MTU + rm.SetMTU(t, t.MaxMTU) + + // Add IP addresses + for _, network := range t.vpnNetworks { + if err := rm.addIP(t, name, network); err != nil { return err } } - return t.addRoutes(false) -} + // Bring up the interface + if err := runCommandBSD("ifconfig", name, "up"); err != nil { + return fmt.Errorf("failed to bring up interface: %w", err) + } -func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { - s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) - if err != nil { + // Set the routes + if err := rm.AddRoutes(t, false); err != nil { return err } - defer syscall.Close(s) - - ir := ifreq{Name: t.deviceBytes(), data: int(value)} - err = ioctl(uintptr(s), ctl, uintptr(unsafe.Pointer(&ir))) - return err -} - -func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) - if err != nil { - return err - } - - if !initial && !change { - return nil - } - - routeTree, err := makeRouteTree(t.l, routes, false) - if err != nil { - return err - } - - // Teach nebula how to handle the routes before establishing them in the system table - oldRoutes := t.Routes.Swap(&routes) - t.routeTree.Store(routeTree) - - if !initial { - // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) - if err != nil { - util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) - } - - // Ensure any routes we actually want are installed - err = t.addRoutes(true) - if err != nil { - // Catch any stray logs - util.LogWithContextIfNeeded("Failed to add routes", err, t.l) - } - } return nil } -func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { - r, _ := t.routeTree.Load().Lookup(ip) - return r +func (rm *tun) SetMTU(t *wgTun, mtu int) { + name, err := t.tunDevice.Name() + if err != nil { + t.l.WithError(err).Error("Failed to get device name for MTU set") + return + } + + if err := runCommandBSD("ifconfig", name, "mtu", strconv.Itoa(mtu)); err != nil { + t.l.WithError(err).Error("Failed to set tun mtu") + } } -func (t *tun) Networks() []netip.Prefix { - return t.vpnNetworks +func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error { + // On OpenBSD, routes are set via ifconfig and route commands + return nil } -func (t *tun) Name() string { - return t.Device -} +func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error { + name, err := t.tunDevice.Name() + if err != nil { + return fmt.Errorf("failed to get device name: %w", err) + } -func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { - return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd") -} - -func (t *tun) addRoutes(logErrors bool) error { routes := *t.Routes.Load() - for _, r := range routes { - if len(r.Via) == 0 || !r.Install { - // We don't allow route MTUs so only install routes with a via + if !r.Install { continue } - err := addRoute(r.Cidr, t.vpnNetworks) + // Add route using route command + args := []string{"add"} + + if r.Cidr.Addr().Is6() { + args = append(args, "-inet6") + } else { + args = append(args, "-inet") + } + + args = append(args, r.Cidr.String(), "-interface", name) + + if r.Metric > 0 { + // OpenBSD doesn't support route metrics directly like Linux + t.l.WithField("route", r).Warn("Route metrics are not fully supported on OpenBSD") + } + + err := runCommandBSD("route", args...) if err != nil { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { @@ -339,131 +159,72 @@ func (t *tun) addRoutes(logErrors bool) error { return nil } -func (t *tun) removeRoutes(routes []Route) error { +func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) { + name, err := t.tunDevice.Name() + if err != nil { + t.l.WithError(err).Error("Failed to get device name for route removal") + return + } + for _, r := range routes { if !r.Install { continue } - err := delRoute(r.Cidr, t.vpnNetworks) + args := []string{"delete"} + + if r.Cidr.Addr().Is6() { + args = append(args, "-inet6") + } else { + args = append(args, "-inet") + } + + args = append(args, r.Cidr.String(), "-interface", name) + + err := runCommandBSD("route", args...) if err != nil { t.l.WithError(err).WithField("route", r).Error("Failed to remove route") } else { t.l.WithField("route", r).Info("Removed route") } } - return nil } -func (t *tun) deviceBytes() (o [16]byte) { - for i, c := range t.Device { - o[i] = byte(c) - } - return +func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) { + // OpenBSD doesn't support multi-queue TUN devices in the same way as Linux + // Return a reader that wraps the same device + return &wgTunReader{ + parent: t, + tunDevice: t.tunDevice, + batchSize: 64, + offset: 0, + l: t.l, + }, nil } -func addRoute(prefix netip.Prefix, gateways []netip.Prefix) error { - sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) - if err != nil { - return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) - } - defer unix.Close(sock) +func (rm *tun) addIP(t *wgTun, name string, network netip.Prefix) error { + addr := network.Addr() - route := &netroute.RouteMessage{ - Version: unix.RTM_VERSION, - Type: unix.RTM_ADD, - Flags: unix.RTF_UP | unix.RTF_GATEWAY, - Seq: 1, - } - - if prefix.Addr().Is4() { - gw, err := selectGateway(prefix, gateways) - if err != nil { - return err - } - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, - unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, - unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()}, + if addr.Is4() { + // For IPv4: ifconfig tun0 10.0.0.1/24 + if err := runCommandBSD("ifconfig", name, network.String()); err != nil { + return fmt.Errorf("failed to add IPv4 address: %w", err) } } else { - gw, err := selectGateway(prefix, gateways) - if err != nil { - return err + // For IPv6: ifconfig tun0 inet6 add 2001:db8::1/64 + if err := runCommandBSD("ifconfig", name, "inet6", "add", network.String()); err != nil { + return fmt.Errorf("failed to add IPv6 address: %w", err) } - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, - unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, - unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()}, - } - } - - data, err := route.Marshal() - if err != nil { - return fmt.Errorf("failed to create route.RouteMessage: %w", err) - } - - _, err = unix.Write(sock, data[:]) - if err != nil { - if errors.Is(err, unix.EEXIST) { - // Try to do a change - route.Type = unix.RTM_CHANGE - data, err = route.Marshal() - if err != nil { - return fmt.Errorf("failed to create route.RouteMessage for change: %w", err) - } - _, err = unix.Write(sock, data[:]) - return err - } - return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) } return nil } -func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error { - sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) +func runCommandBSD(name string, args ...string) error { + cmd := exec.Command(name, args...) + output, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + return fmt.Errorf("%s %s failed: %w\nOutput: %s", name, strings.Join(args, " "), err, string(output)) } - defer unix.Close(sock) - - route := netroute.RouteMessage{ - Version: unix.RTM_VERSION, - Type: unix.RTM_DELETE, - Seq: 1, - } - - if prefix.Addr().Is4() { - gw, err := selectGateway(prefix, gateways) - if err != nil { - return err - } - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, - unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, - unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()}, - } - } else { - gw, err := selectGateway(prefix, gateways) - if err != nil { - return err - } - route.Addrs = []netroute.Addr{ - unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, - unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, - unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()}, - } - } - - data, err := route.Marshal() - if err != nil { - return fmt.Errorf("failed to create route.RouteMessage: %w", err) - } - _, err = unix.Write(sock, data[:]) - if err != nil { - return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) - } - return nil } diff --git a/overlay/tun_wg.go b/overlay/tun_wg.go new file mode 100644 index 0000000..2186ad0 --- /dev/null +++ b/overlay/tun_wg.go @@ -0,0 +1,279 @@ +//go:build !android && !e2e_testing +// +build !android,!e2e_testing + +package overlay + +import ( + "fmt" + "io" + "net/netip" + "sync" + "sync/atomic" + + "github.com/gaissmai/bart" + "github.com/sirupsen/logrus" + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/routing" + "github.com/slackhq/nebula/util" + wgtun "golang.zx2c4.com/wireguard/tun" +) + +// wgTun wraps a WireGuard TUN device and implements the overlay.Device interface +type wgTun struct { + tunDevice wgtun.Device + vpnNetworks []netip.Prefix + MaxMTU int + DefaultMTU int + + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[bart.Table[routing.Gateways]] + routeChan chan struct{} + + // Platform-specific route management + routeManager *tun + + l *logrus.Logger +} + +// BatchReader interface for readers that support vectorized I/O +type BatchReader interface { + BatchRead() ([][]byte, []int, error) +} + +// BatchWriter interface for writers that support vectorized I/O +type BatchWriter interface { + BatchWrite(packets [][]byte) (int, error) +} + +// wgTunReader wraps a single TUN queue for multi-queue support +type wgTunReader struct { + parent *wgTun + tunDevice wgtun.Device + buffers [][]byte + sizes []int + offset int + batchSize int + l *logrus.Logger +} + +var ( + bufferPool = sync.Pool{ + New: func() interface{} { + buf := make([]byte, 9001) // MTU size + return &buf + }, + } +) + +func (t *wgTun) Networks() []netip.Prefix { + return t.vpnNetworks +} + +func (t *wgTun) Name() string { + name, err := t.tunDevice.Name() + if err != nil { + t.l.WithError(err).Error("Failed to get TUN device name") + return "unknown" + } + return name +} + +func (t *wgTun) RoutesFor(ip netip.Addr) routing.Gateways { + r, _ := t.routeTree.Load().Lookup(ip) + return r +} + +func (t *wgTun) Activate() error { + if t.routeManager == nil { + return fmt.Errorf("route manager not initialized") + } + return t.routeManager.Activate(t) +} + +// Read implements single-packet read for backward compatibility +func (t *wgTun) Read(b []byte) (int, error) { + bufs := [][]byte{b} + sizes := []int{0} + n, err := t.tunDevice.Read(bufs, sizes, 0) + if err != nil { + return 0, err + } + if n == 0 { + return 0, io.ErrNoProgress + } + return sizes[0], nil +} + +// Write implements single-packet write for backward compatibility +func (t *wgTun) Write(b []byte) (int, error) { + bufs := [][]byte{b} + offset := 0 + + // WireGuard TUN expects the packet data to start at offset 0 + n, err := t.tunDevice.Write(bufs, offset) + if err != nil { + return 0, err + } + if n == 0 { + return 0, io.ErrShortWrite + } + return len(b), nil +} + +func (t *wgTun) Close() error { + if t.routeChan != nil { + close(t.routeChan) + } + + if t.tunDevice != nil { + return t.tunDevice.Close() + } + + return nil +} + +func (t *wgTun) NewMultiQueueReader() (io.ReadWriteCloser, error) { + // For WireGuard TUN, we need to create separate TUN device instances for multi-queue + // The platform-specific implementation will handle this + if t.routeManager == nil { + return nil, fmt.Errorf("route manager not initialized for multi-queue reader") + } + + return t.routeManager.NewMultiQueueReader(t) +} + +func (t *wgTun) reload(c *config.C, initial bool) error { + routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) + if err != nil { + return err + } + + if !initial && !routeChange && !c.HasChanged("tun.mtu") { + return nil + } + + routeTree, err := makeRouteTree(t.l, routes, true) + if err != nil { + return err + } + + oldDefaultMTU := t.DefaultMTU + oldMaxMTU := t.MaxMTU + newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU) + newMaxMTU := newDefaultMTU + for i, r := range routes { + if r.MTU == 0 { + routes[i].MTU = newDefaultMTU + } + + if r.MTU > t.MaxMTU { + newMaxMTU = r.MTU + } + } + + t.MaxMTU = newMaxMTU + t.DefaultMTU = newDefaultMTU + + // Teach nebula how to handle the routes before establishing them in the system table + oldRoutes := t.Routes.Swap(&routes) + t.routeTree.Store(routeTree) + + if !initial && t.routeManager != nil { + if oldMaxMTU != newMaxMTU { + t.routeManager.SetMTU(t, t.MaxMTU) + t.l.Infof("Set max MTU to %v was %v", t.MaxMTU, oldMaxMTU) + } + + if oldDefaultMTU != newDefaultMTU { + for i := range t.vpnNetworks { + err := t.routeManager.SetDefaultRoute(t, t.vpnNetworks[i]) + if err != nil { + t.l.Warn(err) + } else { + t.l.Infof("Set default MTU to %v was %v", t.DefaultMTU, oldDefaultMTU) + } + } + } + + // Remove first, if the system removes a wanted route hopefully it will be re-added next + t.routeManager.RemoveRoutes(t, findRemovedRoutes(routes, *oldRoutes)) + + // Ensure any routes we actually want are installed + err = t.routeManager.AddRoutes(t, true) + if err != nil { + // This should never be called since AddRoutes should log its own errors in a reload condition + util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l) + } + } + + return nil +} + +// BatchRead reads multiple packets from the TUN device using vectorized I/O +func (r *wgTunReader) BatchRead() ([][]byte, []int, error) { + // Reuse buffers from pool + if len(r.buffers) == 0 { + r.buffers = make([][]byte, r.batchSize) + r.sizes = make([]int, r.batchSize) + for i := 0; i < r.batchSize; i++ { + buf := bufferPool.Get().(*[]byte) + r.buffers[i] = (*buf)[:cap(*buf)] + } + } + + n, err := r.tunDevice.Read(r.buffers, r.sizes, r.offset) + if err != nil { + return nil, nil, err + } + + return r.buffers[:n], r.sizes[:n], nil +} + +// Read implements io.Reader for wgTunReader (single packet for compatibility) +func (r *wgTunReader) Read(b []byte) (int, error) { + bufs := [][]byte{b} + sizes := []int{0} + n, err := r.tunDevice.Read(bufs, sizes, r.offset) + if err != nil { + return 0, err + } + if n == 0 { + return 0, io.ErrNoProgress + } + return sizes[0], nil +} + +// Write implements io.Writer for wgTunReader +func (r *wgTunReader) Write(b []byte) (int, error) { + bufs := [][]byte{b} + n, err := r.tunDevice.Write(bufs, r.offset) + if err != nil { + return 0, err + } + if n == 0 { + return 0, io.ErrShortWrite + } + return len(b), nil +} + +// BatchWrite writes multiple packets to the TUN device using vectorized I/O +func (r *wgTunReader) BatchWrite(packets [][]byte) (int, error) { + return r.tunDevice.Write(packets, r.offset) +} + +func (r *wgTunReader) Close() error { + // Return buffers to pool + for i := range r.buffers { + if r.buffers[i] != nil { + bufferPool.Put(&r.buffers[i]) + r.buffers[i] = nil + } + } + r.buffers = nil + r.sizes = nil + + if r.tunDevice != nil { + return r.tunDevice.Close() + } + return nil +} diff --git a/overlay/tun_windows.go b/overlay/tun_windows.go index 7aac128..130c856 100644 --- a/overlay/tun_windows.go +++ b/overlay/tun_windows.go @@ -1,84 +1,76 @@ -//go:build !e2e_testing -// +build !e2e_testing +//go:build windows && !e2e_testing +// +build windows,!e2e_testing package overlay import ( "crypto" + "encoding/binary" "fmt" "io" "net/netip" - "os" - "path/filepath" - "runtime" - "sync/atomic" - "syscall" - "unsafe" - "github.com/gaissmai/bart" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" - "github.com/slackhq/nebula/wintun" "golang.org/x/sys/windows" + wgtun "golang.zx2c4.com/wireguard/tun" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" ) const tunGUIDLabel = "Fixed Nebula Windows GUID v1" -type winTun struct { - Device string - vpnNetworks []netip.Prefix - MTU int - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[bart.Table[routing.Gateways]] - l *logrus.Logger - - tun *wintun.NativeTun +type tun struct { + luid windows.LUID } -func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (Device, error) { +func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*wgTun, error) { return nil, fmt.Errorf("newTunFromFd not supported in Windows") } -func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*winTun, error) { - err := checkWinTunExists() +func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*wgTun, error) { + deviceName := c.GetString("tun.dev", "Nebula") + mtu := c.GetInt("tun.mtu", DefaultMTU) + + // Create WireGuard TUN device + tunDevice, err := wgtun.CreateTUN(deviceName, mtu) if err != nil { - return nil, fmt.Errorf("can not load the wintun driver: %w", err) + return nil, fmt.Errorf("failed to create TUN device: %w", err) } - deviceName := c.GetString("tun.dev", "") - guid, err := generateGUIDByDeviceName(deviceName) + // Get the actual device name + actualName, err := tunDevice.Name() if err != nil { - return nil, fmt.Errorf("generate GUID failed: %w", err) + tunDevice.Close() + return nil, fmt.Errorf("failed to get TUN device name: %w", err) } - t := &winTun{ - Device: deviceName, + t := &wgTun{ + tunDevice: tunDevice, vpnNetworks: vpnNetworks, - MTU: c.GetInt("tun.mtu", DefaultMTU), + MaxMTU: mtu, + DefaultMTU: mtu, l: l, } + // Create Windows-specific route manager + rm := &tun{} + + // Get LUID from the device name + luid, err := winipcfg.LUIDFromAlias(actualName) + if err != nil { + tunDevice.Close() + return nil, fmt.Errorf("failed to get LUID: %w", err) + } + rm.luid = luid + t.routeManager = rm + err = t.reload(c, true) if err != nil { + tunDevice.Close() return nil, err } - var tunDevice wintun.Device - tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU) - if err != nil { - // Windows 10 has an issue with unclean shutdowns not fully cleaning up the wintun device. - // Trying a second time resolves the issue. - l.WithError(err).Debug("Failed to create wintun device, retrying") - tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU) - if err != nil { - return nil, fmt.Errorf("create TUN device failed: %w", err) - } - } - t.tun = tunDevice.(*wintun.NativeTun) - c.RegisterReloadCallback(func(c *config.C) { err := t.reload(c, false) if err != nil { @@ -86,206 +78,132 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) ( } }) + l.WithField("name", actualName).Info("Created WireGuard TUN device") + return t, nil } -func (t *winTun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) +func (rm *tun) Activate(t *wgTun) error { + // Set MTU + err := rm.setMTU(t, t.MaxMTU) if err != nil { - return err + return fmt.Errorf("failed to set MTU: %w", err) } - if !initial && !change { - return nil - } - - routeTree, err := makeRouteTree(t.l, routes, false) - if err != nil { - return err - } - - // Teach nebula how to handle the routes before establishing them in the system table - oldRoutes := t.Routes.Swap(&routes) - t.routeTree.Store(routeTree) - - if !initial { - // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) - if err != nil { - util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) - } - - // Ensure any routes we actually want are installed - err = t.addRoutes(true) - if err != nil { - // Catch any stray logs - util.LogWithContextIfNeeded("Failed to add routes", err, t.l) + // Add IP addresses + for _, network := range t.vpnNetworks { + if err := rm.addIP(t, network); err != nil { + return err } } - return nil -} - -func (t *winTun) Activate() error { - luid := winipcfg.LUID(t.tun.LUID()) - - err := luid.SetIPAddresses(t.vpnNetworks) - if err != nil { - return fmt.Errorf("failed to set address: %w", err) - } - - err = t.addRoutes(false) - if err != nil { + // Add routes + if err := rm.AddRoutes(t, false); err != nil { return err } return nil } -func (t *winTun) addRoutes(logErrors bool) error { - luid := winipcfg.LUID(t.tun.LUID()) +func (rm *tun) SetMTU(t *wgTun, mtu int) { + if err := rm.setMTU(t, mtu); err != nil { + t.l.WithError(err).Error("Failed to set MTU") + } +} + +func (rm *tun) setMTU(t *wgTun, mtu int) error { + // Set MTU using winipcfg + return rm.luid.SetIPInterfaceMTU(uint32(mtu)) +} + +func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error { + // On Windows, routes are managed differently + return nil +} + +func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error { routes := *t.Routes.Load() - foundDefault4 := false - for _, r := range routes { - if len(r.Via) == 0 || !r.Install { - // We don't allow route MTUs so only install routes with a via + if !r.Install { continue } - // Add our unsafe route - // Windows does not support multipath routes natively, so we install only a single route. - // This is not a problem as traffic will always be sent to Nebula which handles the multipath routing internally. - // In effect this provides multipath routing support to windows supporting loadbalancing and redundancy. - err := luid.AddRoute(r.Cidr, r.Via[0].Addr(), uint32(r.Metric)) + route := winipcfg.RouteData{ + Destination: r.Cidr, + Metric: uint32(r.Metric), + } + + if r.MTU > 0 { + // Windows route MTU is not directly supported + t.l.WithField("route", r).Debug("Route MTU is not supported on Windows") + } + + err := rm.luid.AddRoute(route.Destination, route.Destination.Addr(), route.Metric) if err != nil { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) - continue } else { return retErr } } else { t.l.WithField("route", r).Info("Added route") } - - if !foundDefault4 { - if r.Cidr.Bits() == 0 && r.Cidr.Addr().BitLen() == 32 { - foundDefault4 = true - } - } } - ipif, err := luid.IPInterface(windows.AF_INET) - if err != nil { - return fmt.Errorf("failed to get ip interface: %w", err) - } - - ipif.NLMTU = uint32(t.MTU) - if foundDefault4 { - ipif.UseAutomaticMetric = false - ipif.Metric = 0 - } - - if err := ipif.Set(); err != nil { - return fmt.Errorf("failed to set ip interface: %w", err) - } return nil } -func (t *winTun) removeRoutes(routes []Route) error { - luid := winipcfg.LUID(t.tun.LUID()) - +func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) { for _, r := range routes { if !r.Install { continue } - // See comment on luid.AddRoute - err := luid.DeleteRoute(r.Cidr, r.Via[0].Addr()) + err := rm.luid.DeleteRoute(r.Cidr, r.Cidr.Addr()) if err != nil { t.l.WithError(err).WithField("route", r).Error("Failed to remove route") } else { t.l.WithField("route", r).Info("Removed route") } } +} + +func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) { + // Windows doesn't support multi-queue TUN devices + // Return a reader that wraps the same device + return &wgTunReader{ + parent: t, + tunDevice: t.tunDevice, + batchSize: 64, + offset: 0, + l: t.l, + }, nil +} + +func (rm *tun) addIP(t *wgTun, network netip.Prefix) error { + // Add IP address using winipcfg + err := rm.luid.AddIPAddress(network) + if err != nil { + return fmt.Errorf("failed to add IP address %s: %w", network, err) + } return nil } -func (t *winTun) RoutesFor(ip netip.Addr) routing.Gateways { - r, _ := t.routeTree.Load().Lookup(ip) - return r -} +// generateGUIDByDeviceName generates a GUID based on the device name +func generateGUIDByDeviceName(deviceName string) (*windows.GUID, error) { + // Hash the device name to create a deterministic GUID + h := crypto.SHA256.New() + h.Write([]byte(tunGUIDLabel)) + h.Write([]byte(deviceName)) + sum := h.Sum(nil) -func (t *winTun) Networks() []netip.Prefix { - return t.vpnNetworks -} - -func (t *winTun) Name() string { - return t.Device -} - -func (t *winTun) Read(b []byte) (int, error) { - return t.tun.Read(b, 0) -} - -func (t *winTun) Write(b []byte) (int, error) { - return t.tun.Write(b, 0) -} - -func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) { - return nil, fmt.Errorf("TODO: multiqueue not implemented for windows") -} - -func (t *winTun) Close() error { - // It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes, - // so to be certain, just remove everything before destroying. - luid := winipcfg.LUID(t.tun.LUID()) - _ = luid.FlushRoutes(windows.AF_INET) - _ = luid.FlushIPAddresses(windows.AF_INET) - - _ = luid.FlushRoutes(windows.AF_INET6) - _ = luid.FlushIPAddresses(windows.AF_INET6) - - _ = luid.FlushDNS(windows.AF_INET) - _ = luid.FlushDNS(windows.AF_INET6) - - return t.tun.Close() -} - -func generateGUIDByDeviceName(name string) (*windows.GUID, error) { - // GUID is 128 bit - hash := crypto.MD5.New() - - _, err := hash.Write([]byte(tunGUIDLabel)) - if err != nil { - return nil, err + guid := &windows.GUID{ + Data1: binary.LittleEndian.Uint32(sum[0:4]), + Data2: binary.LittleEndian.Uint16(sum[4:6]), + Data3: binary.LittleEndian.Uint16(sum[6:8]), } + copy(guid.Data4[:], sum[8:16]) - _, err = hash.Write([]byte(name)) - if err != nil { - return nil, err - } - - sum := hash.Sum(nil) - - return (*windows.GUID)(unsafe.Pointer(&sum[0])), nil -} - -func checkWinTunExists() error { - myPath, err := os.Executable() - if err != nil { - return err - } - - arch := runtime.GOARCH - switch arch { - case "386": - //NOTE: wintun bundles 386 as x86 - arch = "x86" - } - - _, err = syscall.LoadDLL(filepath.Join(filepath.Dir(myPath), "dist", "windows", "wintun", "bin", arch, "wintun.dll")) - return err + return guid, nil }