claude first stab and first cleanup - ugh

This commit is contained in:
Jay Wren
2025-10-22 16:49:26 -04:00
parent 770147264d
commit 484de41b58
15 changed files with 1415 additions and 2326 deletions

View File

@@ -1,5 +1,5 @@
//go:build !ios && !e2e_testing
// +build !ios,!e2e_testing
//go:build darwin && !ios && !e2e_testing
// +build darwin,!ios,!e2e_testing
package overlay
@@ -8,48 +8,27 @@ import (
"fmt"
"io"
"net/netip"
"os"
"sync/atomic"
"syscall"
"unsafe"
"github.com/gaissmai/bart"
"github.com/sirupsen/logrus"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/util"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
wgtun "golang.zx2c4.com/wireguard/tun"
)
type tun struct {
io.ReadWriteCloser
Device string
vpnNetworks []netip.Prefix
DefaultMTU int
Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr *netroute.LinkAddr
l *logrus.Logger
// cache out buffer since we need to prepend 4 bytes for tun metadata
out []byte
linkAddr *netroute.LinkAddr
}
// ioctl structures for Darwin network configuration
type ifReq struct {
Name [unix.IFNAMSIZ]byte
Flags uint16
pad [8]byte
}
const (
_SIOCAIFADDR_IN6 = 2155899162
_UTUN_OPT_IFNAME = 2
_IN6_IFF_NODAD = 0x0020
_IN6_IFF_SECURED = 0x0400
utunControlName = "com.apple.net.utun_control"
)
type ifreqMTU struct {
Name [16]byte
MTU int32
@@ -79,60 +58,46 @@ type ifreqAlias6 struct {
Lifetime addrLifetime
}
func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
name := c.GetString("tun.dev", "")
ifIndex := -1
if name != "" && name != "utun" {
_, err := fmt.Sscanf(name, "utun%d", &ifIndex)
if err != nil || ifIndex < 0 {
// NOTE: we don't make this error so we don't break existing
// configs that set a name before it was used.
l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring")
ifIndex = -1
}
}
const (
_SIOCAIFADDR_IN6 = 2155899162
_IN6_IFF_NODAD = 0x0020
)
fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL)
func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*wgTun, error) {
return nil, fmt.Errorf("newTunFromFd not supported on Darwin")
}
func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*wgTun, error) {
deviceName := c.GetString("tun.dev", "utun")
mtu := c.GetInt("tun.mtu", DefaultMTU)
// Create WireGuard TUN device
tunDevice, err := wgtun.CreateTUN(deviceName, mtu)
if err != nil {
return nil, fmt.Errorf("system socket: %v", err)
return nil, fmt.Errorf("failed to create TUN device: %w", err)
}
var ctlInfo = &unix.CtlInfo{}
copy(ctlInfo.Name[:], utunControlName)
err = unix.IoctlCtlInfo(fd, ctlInfo)
// Get the actual device name
actualName, err := tunDevice.Name()
if err != nil {
return nil, fmt.Errorf("CTLIOCGINFO: %v", err)
tunDevice.Close()
return nil, fmt.Errorf("failed to get TUN device name: %w", err)
}
err = unix.Connect(fd, &unix.SockaddrCtl{
ID: ctlInfo.Id,
Unit: uint32(ifIndex) + 1,
})
if err != nil {
return nil, fmt.Errorf("SYS_CONNECT: %v", err)
t := &wgTun{
tunDevice: tunDevice,
vpnNetworks: vpnNetworks,
MaxMTU: mtu,
DefaultMTU: mtu,
l: l,
}
name, err = unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME)
if err != nil {
return nil, fmt.Errorf("failed to retrieve tun name: %w", err)
}
err = unix.SetNonblock(fd, true)
if err != nil {
return nil, fmt.Errorf("SetNonblock: %v", err)
}
t := &tun{
ReadWriteCloser: os.NewFile(uintptr(fd), ""),
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
l: l,
}
// Create Darwin-specific route manager
t.routeManager = &tun{}
err = t.reload(c, true)
if err != nil {
tunDevice.Close()
return nil, err
}
@@ -143,215 +108,252 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
}
})
l.WithField("name", actualName).Info("Created WireGuard TUN device")
return t, nil
}
func (t *tun) deviceBytes() (o [16]byte) {
for i, c := range t.Device {
o[i] = byte(c)
}
return
}
func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
return nil, fmt.Errorf("newTunFromFd not supported in Darwin")
}
func (t *tun) Close() error {
if t.ReadWriteCloser != nil {
return t.ReadWriteCloser.Close()
}
return nil
}
func (t *tun) Activate() error {
devName := t.deviceBytes()
s, err := unix.Socket(
unix.AF_INET,
unix.SOCK_DGRAM,
unix.IPPROTO_IP,
)
func (rm *tun) Activate(t *wgTun) error {
name, err := t.tunDevice.Name()
if err != nil {
return err
}
defer unix.Close(s)
fd := uintptr(s)
// Set the MTU on the device
ifm := ifreqMTU{Name: devName, MTU: int32(t.DefaultMTU)}
if err = ioctl(fd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
return fmt.Errorf("failed to set tun mtu: %v", err)
return fmt.Errorf("failed to get device name: %w", err)
}
// Get the device flags
ifrf := ifReq{Name: devName}
if err = ioctl(fd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
return fmt.Errorf("failed to get tun flags: %s", err)
// Set the MTU
rm.SetMTU(t, t.MaxMTU)
// Add IP addresses
for _, network := range t.vpnNetworks {
if err := rm.addIP(t, name, network); err != nil {
return err
}
}
linkAddr, err := getLinkAddr(t.Device)
// Bring up the interface using ioctl
if err := rm.bringUpInterface(name); err != nil {
return fmt.Errorf("failed to bring up interface: %w", err)
}
// Get the link address for routing
linkAddr, err := getLinkAddr(name)
if err != nil {
return err
return fmt.Errorf("failed to get link address: %w", err)
}
if linkAddr == nil {
return fmt.Errorf("unable to discover link_addr for tun interface")
}
t.linkAddr = linkAddr
rm.linkAddr = linkAddr
for _, network := range t.vpnNetworks {
if network.Addr().Is4() {
err = t.activate4(network)
if err != nil {
return err
// Set the routes
if err := rm.AddRoutes(t, false); err != nil {
return err
}
return nil
}
func (rm *tun) bringUpInterface(name string) error {
// Open a socket for ioctl
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
if err != nil {
return fmt.Errorf("failed to create socket: %w", err)
}
defer unix.Close(fd)
// Get current flags
var ifrf ifReq
copy(ifrf.Name[:], name)
if err := ioctl(uintptr(fd), unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
return fmt.Errorf("failed to get interface flags: %w", err)
}
// Set IFF_UP and IFF_RUNNING flags
ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
if err := ioctl(uintptr(fd), unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
return fmt.Errorf("failed to set interface flags: %w", err)
}
return nil
}
func (rm *tun) SetMTU(t *wgTun, mtu int) {
name, err := t.tunDevice.Name()
if err != nil {
t.l.WithError(err).Error("Failed to get device name for MTU set")
return
}
// Open a socket for ioctl
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
if err != nil {
t.l.WithError(err).Error("Failed to create socket for MTU set")
return
}
defer unix.Close(fd)
// Prepare the ioctl request
var ifr ifreqMTU
copy(ifr.Name[:], name)
ifr.MTU = int32(mtu)
// Set the MTU using ioctl
if err := ioctl(uintptr(fd), unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifr))); err != nil {
t.l.WithError(err).Error("Failed to set tun mtu via ioctl")
}
}
func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error {
// On Darwin, routes are set via ifconfig and route commands
return nil
}
func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error {
routes := *t.Routes.Load()
for _, r := range routes {
if !r.Install {
continue
}
err := rm.addRoute(r.Cidr)
if err != nil {
if errors.Is(err, unix.EEXIST) {
t.l.WithField("route", r.Cidr).
Warnf("unable to add unsafe_route, identical route already exists")
} else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
retErr.Log(t.l)
} else {
return retErr
}
}
} else {
err = t.activate6(network)
if err != nil {
return err
}
t.l.WithField("route", r).Info("Added route")
}
}
// Run the interface
ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
return fmt.Errorf("failed to run tun device: %s", err)
}
// Unsafe path routes
return t.addRoutes(false)
return nil
}
func (t *tun) activate4(network netip.Prefix) error {
s, err := unix.Socket(
unix.AF_INET,
unix.SOCK_DGRAM,
unix.IPPROTO_IP,
)
func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) {
for _, r := range routes {
if !r.Install {
continue
}
err := rm.delRoute(r.Cidr)
if err != nil {
t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
} else {
t.l.WithField("route", r).Info("Removed route")
}
}
}
func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) {
// Darwin doesn't support multi-queue TUN devices in the same way as Linux
// Return a reader that wraps the same device
return &wgTunReader{
parent: t,
tunDevice: t.tunDevice,
batchSize: 64,
offset: 0,
l: t.l,
}, nil
}
func (rm *tun) addIP(t *wgTun, name string, network netip.Prefix) error {
addr := network.Addr()
if addr.Is4() {
return rm.addIPv4(name, network)
} else {
return rm.addIPv6(name, network)
}
}
func (rm *tun) addIPv4(name string, network netip.Prefix) error {
// Open an IPv4 socket for ioctl
s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
if err != nil {
return err
return fmt.Errorf("failed to create IPv4 socket: %w", err)
}
defer unix.Close(s)
ifr := ifreqAlias4{
Name: t.deviceBytes(),
Addr: unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET,
Addr: network.Addr().As4(),
},
DstAddr: unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET,
Addr: network.Addr().As4(),
},
MaskAddr: unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET,
Addr: prefixToMask(network).As4(),
},
var ifr ifreqAlias4
copy(ifr.Name[:], name)
// Set the address
ifr.Addr = unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET,
Addr: network.Addr().As4(),
}
// Set the destination address (same as address for point-to-point)
ifr.DstAddr = unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET,
Addr: network.Addr().As4(),
}
// Set the netmask
ifr.MaskAddr = unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET,
Addr: prefixToMask(network).As4(),
}
if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun v4 address: %s", err)
}
err = addRoute(network, t.linkAddr)
if err != nil {
return err
return fmt.Errorf("failed to set IPv4 address via ioctl: %w", err)
}
return nil
}
func (t *tun) activate6(network netip.Prefix) error {
s, err := unix.Socket(
unix.AF_INET6,
unix.SOCK_DGRAM,
unix.IPPROTO_IP,
)
func (rm *tun) addIPv6(name string, network netip.Prefix) error {
// Open an IPv6 socket for ioctl
s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP)
if err != nil {
return err
return fmt.Errorf("failed to create IPv6 socket: %w", err)
}
defer unix.Close(s)
ifr := ifreqAlias6{
Name: t.deviceBytes(),
Addr: unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6,
Addr: network.Addr().As16(),
},
PrefixMask: unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6,
Addr: prefixToMask(network).As16(),
},
Lifetime: addrLifetime{
// never expires
Vltime: 0xffffffff,
Pltime: 0xffffffff,
},
Flags: _IN6_IFF_NODAD,
var ifr ifreqAlias6
copy(ifr.Name[:], name)
// Set the address
ifr.Addr = unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6,
Addr: network.Addr().As16(),
}
// Set the prefix mask
ifr.PrefixMask = unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6,
Addr: prefixToMask(network).As16(),
}
// Set lifetime (never expires)
ifr.Lifetime = addrLifetime{
Vltime: 0xffffffff,
Pltime: 0xffffffff,
}
// Set flags (no DAD - Duplicate Address Detection)
ifr.Flags = _IN6_IFF_NODAD
if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun address: %s", err)
return fmt.Errorf("failed to set IPv6 address via ioctl: %w", err)
}
return nil
}
func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil {
return err
}
if !initial && !change {
return nil
}
routeTree, err := makeRouteTree(t.l, routes, false)
if err != nil {
return err
}
// Teach nebula how to handle the routes before establishing them in the system table
oldRoutes := t.Routes.Swap(&routes)
t.routeTree.Store(routeTree)
if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
}
// Ensure any routes we actually want are installed
err = t.addRoutes(true)
if err != nil {
// Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
}
}
return nil
}
func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
r, ok := t.routeTree.Load().Lookup(ip)
if ok {
return r
}
return routing.Gateways{}
}
// Get the LinkAddr for the interface of the given name
// Is there an easier way to fetch this when we create the interface?
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
func getLinkAddr(name string) (*netroute.LinkAddr, error) {
rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0)
if err != nil {
@@ -377,53 +379,7 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
return nil, nil
}
func (t *tun) addRoutes(logErrors bool) error {
routes := *t.Routes.Load()
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
err := addRoute(r.Cidr, t.linkAddr)
if err != nil {
if errors.Is(err, unix.EEXIST) {
t.l.WithField("route", r.Cidr).
Warnf("unable to add unsafe_route, identical route already exists")
} else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
retErr.Log(t.l)
} else {
return retErr
}
}
} else {
t.l.WithField("route", r).Info("Added route")
}
}
return nil
}
func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes {
if !r.Install {
continue
}
err := delRoute(r.Cidr, t.linkAddr)
if err != nil {
t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
} else {
t.l.WithField("route", r).Info("Removed route")
}
}
return nil
}
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
func (rm *tun) addRoute(prefix netip.Prefix) error {
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
@@ -441,13 +397,13 @@ func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
route.Addrs = []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
unix.RTAX_GATEWAY: gateway,
unix.RTAX_GATEWAY: rm.linkAddr,
}
} else {
route.Addrs = []netroute.Addr{
unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
unix.RTAX_GATEWAY: gateway,
unix.RTAX_GATEWAY: rm.linkAddr,
}
}
@@ -464,7 +420,7 @@ func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
return nil
}
func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
func (rm *tun) delRoute(prefix netip.Prefix) error {
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
@@ -481,13 +437,13 @@ func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
route.Addrs = []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
unix.RTAX_GATEWAY: gateway,
unix.RTAX_GATEWAY: rm.linkAddr,
}
} else {
route.Addrs = []netroute.Addr{
unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
unix.RTAX_GATEWAY: gateway,
unix.RTAX_GATEWAY: rm.linkAddr,
}
}
@@ -495,6 +451,7 @@ func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
if err != nil {
return fmt.Errorf("failed to create route.RouteMessage: %w", err)
}
_, err = unix.Write(sock, data[:])
if err != nil {
return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
@@ -503,52 +460,34 @@ func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
return nil
}
func (t *tun) Read(to []byte) (int, error) {
buf := make([]byte, len(to)+4)
n, err := t.ReadWriteCloser.Read(buf)
copy(to, buf[4:])
return n - 4, err
func ioctl(a1, a2, a3 uintptr) error {
_, _, errno := unix.Syscall(unix.SYS_IOCTL, a1, a2, a3)
if errno != 0 {
return errno
}
return nil
}
// Write is only valid for single threaded use
func (t *tun) Write(from []byte) (int, error) {
buf := t.out
if cap(buf) < len(from)+4 {
buf = make([]byte, len(from)+4)
t.out = buf
}
buf = buf[:len(from)+4]
if len(from) == 0 {
return 0, syscall.EIO
}
// Determine the IP Family for the NULL L2 Header
ipVer := from[0] >> 4
if ipVer == 4 {
buf[3] = syscall.AF_INET
} else if ipVer == 6 {
buf[3] = syscall.AF_INET6
func prefixToMask(prefix netip.Prefix) netip.Addr {
bits := prefix.Bits()
if prefix.Addr().Is4() {
// Create IPv4 netmask from prefix length
mask := ^uint32(0) << (32 - bits)
return netip.AddrFrom4([4]byte{
byte(mask >> 24),
byte(mask >> 16),
byte(mask >> 8),
byte(mask),
})
} else {
return 0, fmt.Errorf("unable to determine IP version from packet")
// Create IPv6 netmask from prefix length
var mask [16]byte
for i := 0; i < bits/8; i++ {
mask[i] = 0xff
}
if bits%8 != 0 {
mask[bits/8] = ^byte(0) << (8 - bits%8)
}
return netip.AddrFrom16(mask)
}
copy(buf[4:], from)
n, err := t.ReadWriteCloser.Write(buf)
return n - 4, err
}
func (t *tun) Networks() []netip.Prefix {
return t.vpnNetworks
}
func (t *tun) Name() string {
return t.Device
}
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
}