remove gso, leave nice pretty GRO

This reverts commit fa098c551a.

bits.go

@@ -5,6 +5,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// TODO: Pretty sure this is just all sorts of racy now, we need it to be atomic
 type Bits struct {
 	length             uint64
 	current            uint64
@@ -43,7 +44,7 @@ func (b *Bits) Check(l logrus.FieldLogger, i uint64) bool {
 	}
 
 	// Not within the window
-	l.Debugf("rejected a packet (top) %d %d\n", b.current, i)
+	l.Error("rejected a packet (top) %d %d\n", b.current, i)
 	return false
 }
 

cmd/gso/gso.go

@@ -0,0 +1,191 @@
+package main
+
+import (
+	"encoding/binary"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"net/netip"
+	"time"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// UDP_SEGMENT enables GSO segmentation
+	UDP_SEGMENT = 103
+	// Maximum GSO segment size (typical MTU - headers)
+	maxGSOSize = 1400
+)
+
+func main() {
+	destAddr := flag.String("dest", "10.4.0.16:4202", "Destination address")
+	gsoSize := flag.Int("gso", 1400, "GSO segment size")
+	totalSize := flag.Int("size", 14000, "Total payload size to send")
+	count := flag.Int("count", 1, "Number of packets to send")
+	flag.Parse()
+
+	if *gsoSize > maxGSOSize {
+		log.Fatalf("GSO size %d exceeds maximum %d", *gsoSize, maxGSOSize)
+	}
+
+	// Resolve destination address
+	_, err := net.ResolveUDPAddr("udp", *destAddr)
+	if err != nil {
+		log.Fatalf("Failed to resolve address: %v", err)
+	}
+
+	// Create a raw UDP socket with GSO support
+	fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
+	if err != nil {
+		log.Fatalf("Failed to create socket: %v", err)
+	}
+	defer unix.Close(fd)
+
+	// Bind to a local address
+	localAddr := &unix.SockaddrInet4{
+		Port: 0, // Let the system choose a port
+	}
+	if err := unix.Bind(fd, localAddr); err != nil {
+		log.Fatalf("Failed to bind socket: %v", err)
+	}
+
+	fmt.Printf("Sending UDP packets with GSO enabled\n")
+	fmt.Printf("Destination: %s\n", *destAddr)
+	fmt.Printf("GSO segment size: %d bytes\n", *gsoSize)
+	fmt.Printf("Total payload size: %d bytes\n", *totalSize)
+	fmt.Printf("Number of packets: %d\n\n", *count)
+
+	// Create payload
+	payload := make([]byte, *totalSize)
+	for i := range payload {
+		payload[i] = byte(i % 256)
+	}
+
+	dest := netip.MustParseAddrPort(*destAddr)
+
+	//if err := unix.SetsockoptInt(fd, unix.SOL_UDP, unix.UDP_SEGMENT, 1400); err != nil {
+	//	panic(err)
+	//}
+
+	for i := 0; i < *count; i++ {
+		err := WriteBatch(fd, payload, dest, uint16(*gsoSize), true)
+		if err != nil {
+			log.Printf("Send error on packet %d: %v", i, err)
+			continue
+		}
+
+		if (i+1)%100 == 0 || i == *count-1 {
+			fmt.Printf("Sent %d packets\n", i+1)
+		}
+	}
+	fmt.Printf("now, let's send without the correct ctrl header\n")
+	time.Sleep(time.Second)
+	for i := 0; i < *count; i++ {
+		err := WriteBatch(fd, payload, dest, uint16(*gsoSize), false)
+		if err != nil {
+			log.Printf("Send error on packet %d: %v", i, err)
+			continue
+		}
+
+		if (i+1)%100 == 0 || i == *count-1 {
+			fmt.Printf("Sent %d packets\n", i+1)
+		}
+	}
+
+}
+
+func WriteBatch(fd int, payload []byte, addr netip.AddrPort, segSize uint16, withHeader bool) error {
+	msgs := make([]rawMessage, 0, 1)
+	iovs := make([]iovec, 0, 1)
+	names := make([][unix.SizeofSockaddrInet6]byte, 0, 1)
+
+	sent := 0
+
+	pkts := []BatchPacket{
+		{
+			Payload: payload,
+			Addr:    addr,
+		},
+	}
+
+	for _, pkt := range pkts {
+		if len(pkt.Payload) == 0 {
+			sent++
+			continue
+		}
+
+		msgs = append(msgs, rawMessage{})
+		iovs = append(iovs, iovec{})
+		names = append(names, [unix.SizeofSockaddrInet6]byte{})
+
+		idx := len(msgs) - 1
+		msg := &msgs[idx]
+		iov := &iovs[idx]
+		name := &names[idx]
+
+		setIovecSlice(iov, pkt.Payload)
+		msg.Hdr.Iov = iov
+		msg.Hdr.Iovlen = 1
+
+		if withHeader {
+			setRawMessageControl(msg, buildGSOControlMessage(segSize)) //
+		} else {
+			setRawMessageControl(msg, nil) //
+		}
+
+		msg.Hdr.Flags = 0
+
+		nameLen, err := encodeSockaddr(name[:], pkt.Addr)
+		if err != nil {
+			return err
+		}
+		msg.Hdr.Name = &name[0]
+		msg.Hdr.Namelen = nameLen
+	}
+
+	if len(msgs) == 0 {
+		return errors.New("nothing to write")
+	}
+
+	offset := 0
+	for offset < len(msgs) {
+		n, _, errno := unix.Syscall6(
+			unix.SYS_SENDMMSG,
+			uintptr(fd),
+			uintptr(unsafe.Pointer(&msgs[offset])),
+			uintptr(len(msgs)-offset),
+			0,
+			0,
+			0,
+		)
+
+		if errno != 0 {
+			if errno == unix.EINTR {
+				continue
+			}
+			return &net.OpError{Op: "sendmmsg", Err: errno}
+		}
+
+		if n == 0 {
+			break
+		}
+		offset += int(n)
+	}
+
+	return nil
+}
+
+func buildGSOControlMessage(segSize uint16) []byte {
+	control := make([]byte, unix.CmsgSpace(2))
+	hdr := (*unix.Cmsghdr)(unsafe.Pointer(&control[0]))
+	hdr.Level = unix.SOL_UDP
+	hdr.Type = unix.UDP_SEGMENT
+	setCmsgLen(hdr, unix.CmsgLen(2))
+	binary.NativeEndian.PutUint16(control[unix.CmsgLen(0):unix.CmsgLen(0)+2], uint16(segSize))
+
+	return control
+}

cmd/gso/helper.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net/netip"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+type iovec struct {
+	Base *byte
+	Len  uint64
+}
+
+type msghdr struct {
+	Name       *byte
+	Namelen    uint32
+	Pad0       [4]byte
+	Iov        *iovec
+	Iovlen     uint64
+	Control    *byte
+	Controllen uint64
+	Flags      int32
+	Pad1       [4]byte
+}
+
+type rawMessage struct {
+	Hdr  msghdr
+	Len  uint32
+	Pad0 [4]byte
+}
+
+type BatchPacket struct {
+	Payload []byte
+	Addr    netip.AddrPort
+}
+
+func encodeSockaddr(dst []byte, addr netip.AddrPort) (uint32, error) {
+	if addr.Addr().Is4() {
+		if !addr.Addr().Is4() {
+			return 0, fmt.Errorf("Listener is IPv4, but writing to IPv6 remote")
+		}
+		var sa unix.RawSockaddrInet4
+		sa.Family = unix.AF_INET
+		sa.Addr = addr.Addr().As4()
+		binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
+		size := unix.SizeofSockaddrInet4
+		copy(dst[:size], (*(*[unix.SizeofSockaddrInet4]byte)(unsafe.Pointer(&sa)))[:])
+		return uint32(size), nil
+	}
+
+	var sa unix.RawSockaddrInet6
+	sa.Family = unix.AF_INET6
+	sa.Addr = addr.Addr().As16()
+	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
+	size := unix.SizeofSockaddrInet6
+	copy(dst[:size], (*(*[unix.SizeofSockaddrInet6]byte)(unsafe.Pointer(&sa)))[:])
+	return uint32(size), nil
+}
+
+func setRawMessageControl(msg *rawMessage, buf []byte) {
+	if len(buf) == 0 {
+		msg.Hdr.Control = nil
+		msg.Hdr.Controllen = 0
+		return
+	}
+	msg.Hdr.Control = &buf[0]
+	msg.Hdr.Controllen = uint64(len(buf))
+}
+
+func setCmsgLen(h *unix.Cmsghdr, l int) {
+	h.Len = uint64(l)
+}
+
+func setIovecSlice(iov *iovec, b []byte) {
+	if len(b) == 0 {
+		iov.Base = nil
+		iov.Len = 0
+		return
+	}
+	iov.Base = &b[0]
+	iov.Len = uint64(len(b))
+}

cmd/nebula-service/main.go

@@ -65,8 +65,16 @@ func main() {
 	}
 
 	if !*configTest {
-		ctrl.Start()
+		wait, err := ctrl.Start()
-		ctrl.ShutdownBlock()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
+		wait()
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

cmd/nebula/main.go

@@ -3,6 +3,9 @@ package main
 import (
 	"flag"
 	"fmt"
+	"log"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 
 	"github.com/sirupsen/logrus"
@@ -58,10 +61,22 @@ func main() {
 		os.Exit(1)
 	}
 
+	go func() {
+		log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
+	}()
+
 	if !*configTest {
-		ctrl.Start()
+		wait, err := ctrl.Start()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
 		notifyReady(l)
-		ctrl.ShutdownBlock()
+		wait()
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

connection_state.go

@@ -13,7 +13,9 @@ import (
 	"github.com/slackhq/nebula/noiseutil"
 )
 
-const ReplayWindow = 1024
+// TODO: In a 5Gbps test, 1024 is not sufficient. With a 1400 MTU this is about 1.4Gbps of window, assuming full packets.
+// 4092 should be sufficient for 5Gbps
+const ReplayWindow = 8192
 
 type ConnectionState struct {
 	eKey           *NebulaCipherState

control.go

@@ -2,9 +2,11 @@ package nebula
 
 import (
 	"context"
+	"errors"
 	"net/netip"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
@@ -13,6 +15,16 @@ import (
 	"github.com/slackhq/nebula/overlay"
 )
 
+type RunState int
+
+const (
+	Stopped  RunState = 0 // The control has yet to be started
+	Started  RunState = 1 // The control has been started
+	Stopping RunState = 2 // The control is stopping
+)
+
+var ErrAlreadyStarted = errors.New("nebula is already started")
+
 // Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching
 // core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc
 
@@ -26,6 +38,9 @@ type controlHostLister interface {
 }
 
 type Control struct {
+	stateLock sync.Mutex
+	state     RunState
+
 	f                      *Interface
 	l                      *logrus.Logger
 	ctx                    context.Context
@@ -49,10 +64,21 @@ type ControlHostInfo struct {
 	CurrentRelaysThroughMe []netip.Addr     `json:"currentRelaysThroughMe"`
 }
 
-// Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock()
+// Start actually runs nebula, this is a nonblocking call.
-func (c *Control) Start() {
+// The returned function can be used to wait for nebula to fully stop.
+func (c *Control) Start() (func(), error) {
+	c.stateLock.Lock()
+	if c.state != Stopped {
+		c.stateLock.Unlock()
+		return nil, ErrAlreadyStarted
+	}
+
 	// Activate the interface
-	c.f.activate()
+	err := c.f.activate()
+	if err != nil {
+		c.stateLock.Unlock()
+		return nil, err
+	}
 
 	// Call all the delayed funcs that waited patiently for the interface to be created.
 	if c.sshStart != nil {
@@ -72,15 +98,33 @@ func (c *Control) Start() {
 	}
 
 	// Start reading packets.
-	c.f.run()
+	c.state = Started
+	c.stateLock.Unlock()
+	return c.f.run(c.ctx)
+}
+
+func (c *Control) State() RunState {
+	c.stateLock.Lock()
+	defer c.stateLock.Unlock()
+	return c.state
 }
 
 func (c *Control) Context() context.Context {
 	return c.ctx
 }
 
-// Stop signals nebula to shutdown and close all tunnels, returns after the shutdown is complete
+// Stop is a non-blocking call that signals nebula to close all tunnels and shut down
 func (c *Control) Stop() {
+	c.stateLock.Lock()
+	if c.state != Started {
+		c.stateLock.Unlock()
+		// We are stopping or stopped already
+		return
+	}
+
+	c.state = Stopping
+	c.stateLock.Unlock()
+
 	// Stop the handshakeManager (and other services), to prevent new tunnels from
 	// being created while we're shutting them all down.
 	c.cancel()
@@ -89,7 +133,7 @@ func (c *Control) Stop() {
 	if err := c.f.Close(); err != nil {
 		c.l.WithError(err).Error("Close interface failed")
 	}
-	c.l.Info("Goodbye")
+	c.state = Stopped
 }
 
 // ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled

firewall_test.go

@@ -392,7 +392,7 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 		c := &cert.CachedCertificate{
 			Certificate: &dummyCert{
 				name:     "nope",
-				networks: []netip.Prefix{netip.MustParsePrefix("fd99::99/128")},
+				networks: []netip.Prefix{netip.MustParsePrefix("fd99:99/128")},
 			},
 			InvertedGroups: map[string]struct{}{"nope": {}},
 		}
@@ -692,50 +692,6 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
 }
 
-func TestFirewall_DropIPSpoofing(t *testing.T) {
-	l := test.NewLogger()
-	ob := &bytes.Buffer{}
-	l.SetOutput(ob)
-
-	c := cert.CachedCertificate{
-		Certificate: &dummyCert{
-			name:     "host-owner",
-			networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
-		},
-	}
-
-	c1 := cert.CachedCertificate{
-		Certificate: &dummyCert{
-			name:           "host",
-			networks:       []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")},
-			unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
-		},
-	}
-	h1 := HostInfo{
-		ConnectionState: &ConnectionState{
-			peerCert: &c1,
-		},
-		vpnAddrs: []netip.Addr{c1.Certificate.Networks()[0].Addr()},
-	}
-	h1.buildNetworks(c1.Certificate.Networks(), c1.Certificate.UnsafeNetworks())
-
-	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
-
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
-	cp := cert.NewCAPool()
-
-	// Packet spoofed by `c1`. Note that the remote addr is not a valid one.
-	p := firewall.Packet{
-		LocalAddr:  netip.MustParseAddr("192.0.2.1"),
-		RemoteAddr: netip.MustParseAddr("192.0.2.3"),
-		LocalPort:  1,
-		RemotePort: 1,
-		Protocol:   firewall.ProtoUDP,
-		Fragment:   false,
-	}
-	assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP)
-}
-
 func BenchmarkLookup(b *testing.B) {
 	ml := func(m map[string]struct{}, a [][]string) {
 		for n := 0; n < b.N; n++ {

go.mod

@@ -8,32 +8,32 @@ require (
 	github.com/armon/go-radix v1.0.0
 	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
 	github.com/flynn/noise v1.1.0
-	github.com/gaissmai/bart v0.25.0
+	github.com/gaissmai/bart v0.20.4
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gopacket v1.1.19
-	github.com/kardianos/service v1.2.4
+	github.com/kardianos/service v1.2.2
-	github.com/miekg/dns v1.1.68
+	github.com/miekg/dns v1.1.65
 	github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b
 	github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
-	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_golang v1.22.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/sirupsen/logrus v1.9.3
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
-	github.com/stretchr/testify v1.11.1
+	github.com/stretchr/testify v1.10.0
 	github.com/vishvananda/netlink v1.3.1
-	golang.org/x/crypto v0.43.0
+	golang.org/x/crypto v0.37.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.45.0
+	golang.org/x/net v0.39.0
-	golang.org/x/sync v0.17.0
+	golang.org/x/sync v0.13.0
-	golang.org/x/sys v0.37.0
+	golang.org/x/sys v0.32.0
-	golang.org/x/term v0.36.0
+	golang.org/x/term v0.31.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
-	golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb
+	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
 	golang.zx2c4.com/wireguard/windows v0.5.3
-	google.golang.org/protobuf v1.36.8
+	google.golang.org/protobuf v1.36.6
 	gopkg.in/yaml.v3 v3.0.1
-	gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c
+	gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
 )
 
 require (
@@ -43,12 +43,11 @@ require (
 	github.com/google/btree v1.1.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/common v0.62.0 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/vishvananda/netns v0.0.5 // indirect
-	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	golang.org/x/mod v0.23.0 // indirect
-	golang.org/x/mod v0.24.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/time v0.7.0 // indirect
+	golang.org/x/tools v0.30.0 // indirect
-	golang.org/x/tools v0.33.0 // indirect
 )

go.sum

@@ -24,8 +24,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
 github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
-github.com/gaissmai/bart v0.25.0 h1:eqiokVPqM3F94vJ0bTHXHtH91S8zkKL+bKh+BsGOsJM=
+github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U=
-github.com/gaissmai/bart v0.25.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
+github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -64,8 +64,8 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kardianos/service v1.2.4 h1:XNlGtZOYNx2u91urOdg/Kfmc+gfmuIo1Dd3rEi2OgBk=
+github.com/kardianos/service v1.2.2 h1:ZvePhAHfvo0A7Mftk/tEzqEZ7Q4lgnR8sGz4xu1YX60=
-github.com/kardianos/service v1.2.4/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=
+github.com/kardianos/service v1.2.2/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
@@ -83,8 +83,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA=
+github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc=
-github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps=
+github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck=
 github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b h1:J/AzCvg5z0Hn1rqZUJjpbzALUmkKX0Zwbc/i4fw7Sfk=
 github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -106,24 +106,24 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
-github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
-github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
-github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
-github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -143,33 +143,29 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
 github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
 github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
 github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
-go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
+golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
-golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
+golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
+golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
-golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -180,8 +176,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM=
+golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
-golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
+golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -189,8 +185,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -201,37 +197,38 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
-golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
+golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
-golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
+golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
+golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
-golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
+golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
-golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb h1:whnFRlWMcXI9d+ZbWg+4sHnLp52d5yiIPUxMBSt4X9A=
+golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
-golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb/go.mod h1:rpwXGsirqLqN2L0JDJQlwOboGHmptD5ZD6T2VmcqhTw=
+golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
 golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
 golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -242,8 +239,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
+google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
-google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -257,5 +254,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c h1:m/r7OM+Y2Ty1sgBQ7Qb27VgIMBW8ZZhT4gLnUyDIhzI=
+gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g=
-gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c/go.mod h1:3r5CMtNQMKIvBlrmM9xWUNamjKBYPOWyXOjmg5Kts3g=
+gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=

hostmap.go

@@ -738,8 +738,7 @@ func (i *HostInfo) buildNetworks(networks, unsafeNetworks []netip.Prefix) {
 
 	i.networks = new(bart.Lite)
 	for _, network := range networks {
-		nprefix := netip.PrefixFrom(network.Addr(), network.Addr().BitLen())
+		i.networks.Insert(network)
-		i.networks.Insert(nprefix)
 	}
 
 	for _, network := range unsafeNetworks {

inside.go

@@ -33,7 +33,8 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		// routes packets from the Nebula addr to the Nebula addr through the Nebula
 		// TUN device.
 		if immediatelyForwardToSelf {
-			if err := f.writeTun(q, packet); err != nil {
+			_, err := f.readers[q].Write(packet)
+			if err != nil {
 				f.l.WithError(err).Error("Failed to forward to tun")
 			}
 		}
@@ -90,7 +91,8 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 		return
 	}
 
-	if err := f.writeTun(q, out); err != nil {
+	_, err := f.readers[q].Write(out)
+	if err != nil {
 		f.l.WithError(err).Error("Failed to write to tun")
 	}
 }

interface.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"io"
 	"net/netip"
-	"os"
 	"runtime"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -18,6 +18,7 @@ import (
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
+	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/udp"
 )
 
@@ -47,7 +48,6 @@ type InterfaceConfig struct {
 	reQueryWait     time.Duration
 
 	ConntrackCacheTimeout time.Duration
-	batchSize             int
 	l                     *logrus.Logger
 }
 
@@ -85,16 +85,20 @@ type Interface struct {
 	version     string
 
 	conntrackCacheTimeout time.Duration
-	batchSize             int
 
 	writers []udp.Conn
 	readers []io.ReadWriteCloser
+	wg      sync.WaitGroup
 
 	metricHandshakes    metrics.Histogram
 	messageMetrics      *MessageMetrics
 	cachedPacketMetrics *cachedPacketMetrics
 
 	l *logrus.Logger
+
+	pktPool  *packet.Pool
+	inbound  chan *packet.Packet
+	outbound chan *packet.Packet
 }
 
 type EncWriter interface {
@@ -112,16 +116,6 @@ type EncWriter interface {
 	GetCertState() *CertState
 }
 
-// BatchReader is an interface for readers that support vectorized packet reading
-type BatchReader interface {
-	BatchRead(buffers [][]byte, sizes []int) (int, error)
-}
-
-// BatchWriter is an interface for writers that support vectorized packet writing
-type BatchWriter interface {
-	BatchWrite([][]byte) (int, error)
-}
-
 type sendRecvErrorConfig uint8
 
 const (
@@ -198,7 +192,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		relayManager:          c.relayManager,
 		connectionManager:     c.connectionManager,
 		conntrackCacheTimeout: c.ConntrackCacheTimeout,
-		batchSize:             c.batchSize,
 
 		metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
 		messageMetrics:   c.MessageMetrics,
@@ -207,9 +200,15 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 			dropped: metrics.GetOrRegisterCounter("hostinfo.cached_packets.dropped", nil),
 		},
 
+		//TODO: configurable size
+		inbound:  make(chan *packet.Packet, 2048),
+		outbound: make(chan *packet.Packet, 2048),
+
 		l: c.l,
 	}
 
+	ifce.pktPool = packet.GetPool()
+
 	ifce.tryPromoteEvery.Store(c.tryPromoteEvery)
 	ifce.reQueryEvery.Store(c.reQueryEvery)
 	ifce.reQueryWait.Store(int64(c.reQueryWait))
@@ -222,7 +221,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 // activate creates the interface on the host. After the interface is created, any
 // other services that want to bind listeners to its IP may do so successfully. However,
 // the interface isn't going to process anything until run() is called.
-func (f *Interface) activate() {
+func (f *Interface) activate() error {
 	// actually turn on tun dev
 
 	addr, err := f.outside.LocalAddr()
@@ -243,33 +242,46 @@ func (f *Interface) activate() {
 		if i > 0 {
 			reader, err = f.inside.NewMultiQueueReader()
 			if err != nil {
-				f.l.Fatal(err)
+				return err
 			}
 		}
 		f.readers[i] = reader
 	}
 
-	if err := f.inside.Activate(); err != nil {
+	if err = f.inside.Activate(); err != nil {
 		f.inside.Close()
-		f.l.Fatal(err)
+		return err
 	}
+
+	return nil
 }
 
-func (f *Interface) run() {
+func (f *Interface) run(c context.Context) (func(), error) {
-	// Launch n queues to read packets from udp
 	for i := 0; i < f.routines; i++ {
+		// read packets from udp and queue to f.inbound
+		f.wg.Add(1)
 		go f.listenOut(i)
+
+		// Launch n queues to read packets from inside tun dev and queue to f.outbound
+		//todo this never stops f.wg.Add(1)
+		go f.listenIn(f.readers[i], i)
+
+		// Launch n workers to process traffic from f.inbound and smash it onto the inside of the tun
+		f.wg.Add(1)
+		go f.workerIn(i, c)
+		f.wg.Add(1)
+		go f.workerIn(i, c)
+
+		// read from f.outbound and write to UDP (outside the tun)
+		f.wg.Add(1)
+		go f.workerOut(i, c)
 	}
 
-	// Launch n queues to read packets from tun dev
+	return f.wg.Wait, nil
-	for i := 0; i < f.routines; i++ {
-		go f.listenIn(f.readers[i], i)
-	}
 }
 
 func (f *Interface) listenOut(i int) {
 	runtime.LockOSThread()
-
 	var li udp.Conn
 	if i > 0 {
 		li = f.writers[i]
@@ -277,124 +289,93 @@ func (f *Interface) listenOut(i int) {
 		li = f.outside
 	}
 
-	ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	err := li.ListenOut(f.pktPool.Get, f.inbound)
-	lhh := f.lightHouse.NewRequestHandler()
+	if err != nil && !f.closed.Load() {
-	plaintext := make([]byte, udp.MTU)
+		f.l.WithError(err).Error("Error while reading packet inbound packet, closing")
-	h := &header.H{}
+		//TODO: Trigger Control to close
-	fwPacket := &firewall.Packet{}
+	}
-	nb := make([]byte, 12)
 
-	li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
+	f.l.Debugf("underlay reader %v is done", i)
-		f.readOutsidePackets(fromUdpAddr, nil, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get(f.l))
+	f.wg.Done()
-	})
 }
 
 func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
 	runtime.LockOSThread()
 
-	// Check if reader supports batch operations
+	for {
-	if batchReader, ok := reader.(BatchReader); ok {
+		p := f.pktPool.Get()
-		err := f.listenInBatch(batchReader, i)
+		n, err := reader.Read(p.Payload)
 		if err != nil {
-			f.l.WithError(err).Error("Fatal error in batch packet reader, exiting goroutine")
+			if !f.closed.Load() {
+				f.l.WithError(err).Error("Error while reading outbound packet, closing")
+				//TODO: Trigger Control to close
+			}
+			break
 		}
-		return
+
+		p.Payload = (p.Payload)[:n]
+		//TODO: nonblocking channel write
+		f.outbound <- p
+		//select {
+		//case f.outbound <- p:
+		//default:
+		//	f.l.Error("Dropped packet from outbound channel")
+		//}
 	}
 
-	// Fall back to single-packet mode
+	f.l.Debugf("overlay reader %v is done", i)
-	packet := make([]byte, mtu)
+	f.wg.Done()
-	out := make([]byte, mtu)
+}
-	fwPacket := &firewall.Packet{}
-	nb := make([]byte, 12, 12)
 
+func (f *Interface) workerIn(i int, ctx context.Context) {
+	lhh := f.lightHouse.NewRequestHandler()
 	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	fwPacket2 := &firewall.Packet{}
+	nb2 := make([]byte, 12, 12)
+	result2 := make([]byte, mtu)
+	h := &header.H{}
 
 	for {
-		n, err := reader.Read(packet)
+		select {
-		if err != nil {
+		case p := <-f.inbound:
-			if errors.Is(err, os.ErrClosed) && f.closed.Load() {
+			if p.SegSize > 0 && p.SegSize < len(p.Payload) {
-				return
+				for offset := 0; offset < len(p.Payload); offset += p.SegSize {
+					end := offset + p.SegSize
+					if end > len(p.Payload) {
+						end = len(p.Payload)
+					}
+					f.readOutsidePackets(p.Addr, nil, result2[:0], p.Payload[offset:end], h, fwPacket2, lhh, nb2, i, conntrackCache.Get(f.l))
+				}
+			} else {
+				f.readOutsidePackets(p.Addr, nil, result2[:0], p.Payload, h, fwPacket2, lhh, nb2, i, conntrackCache.Get(f.l))
 			}
 
-			f.l.WithError(err).Error("Fatal error while reading outbound packet, exiting goroutine")
+			f.pktPool.Put(p)
+		case <-ctx.Done():
+			f.wg.Done()
 			return
 		}
-
-		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get(f.l))
 	}
 }
 
-// listenInBatch handles vectorized packet reading for improved performance
+func (f *Interface) workerOut(i int, ctx context.Context) {
-func (f *Interface) listenInBatch(reader BatchReader, i int) error {
-	// Allocate per-packet state and buffers for batch reading
-	batchSize := f.batchSize
-	if batchSize <= 0 {
-		batchSize = 64 // Fallback to default if not configured
-	}
-	fwPackets := make([]*firewall.Packet, batchSize)
-	outBuffers := make([][]byte, batchSize)
-	nbBuffers := make([][]byte, batchSize)
-	packets := make([][]byte, batchSize)
-	sizes := make([]int, batchSize)
-
-	for j := 0; j < batchSize; j++ {
-		fwPackets[j] = &firewall.Packet{}
-		outBuffers[j] = make([]byte, mtu)
-		nbBuffers[j] = make([]byte, 12)
-		packets[j] = make([]byte, mtu)
-	}
-
 	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	fwPacket1 := &firewall.Packet{}
+	nb1 := make([]byte, 12, 12)
+	result1 := make([]byte, mtu)
 
 	for {
-		n, err := reader.BatchRead(packets, sizes)
+		select {
-		if err != nil {
+		case data := <-f.outbound:
-			if errors.Is(err, os.ErrClosed) && f.closed.Load() {
+			f.consumeInsidePacket(data.Payload, fwPacket1, nb1, result1, i, conntrackCache.Get(f.l))
-				return nil
+			f.pktPool.Put(data)
-			}
+		case <-ctx.Done():
-
+			f.wg.Done()
-			return fmt.Errorf("error while batch reading outbound packets: %w", err)
+			return
-		}
-
-		// Process each packet in the batch
-		cache := conntrackCache.Get(f.l)
-		for idx := 0; idx < n; idx++ {
-			if sizes[idx] > 0 {
-				// Use modulo to reuse fw packet state if batch is larger than our pre-allocated state
-				stateIdx := idx % len(fwPackets)
-				f.consumeInsidePacket(packets[idx][:sizes[idx]], fwPackets[stateIdx], nbBuffers[stateIdx], outBuffers[stateIdx], i, cache)
-			}
 		}
 	}
 }
 
-// writeTunBatch attempts to write multiple packets to the TUN device using batch operations if supported
-func (f *Interface) writeTunBatch(q int, packets [][]byte) error {
-	if len(packets) == 0 {
-		return nil
-	}
-
-	// Check if the reader/writer supports batch operations
-	if batchWriter, ok := f.readers[q].(BatchWriter); ok {
-		_, err := batchWriter.BatchWrite(packets)
-		return err
-	}
-
-	// Fall back to writing packets individually
-	for _, packet := range packets {
-		if _, err := f.readers[q].Write(packet); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// writeTun writes a single packet to the TUN device
-func (f *Interface) writeTun(q int, packet []byte) error {
-	_, err := f.readers[q].Write(packet)
-	return err
-}
-
 func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
 	c.RegisterReloadCallback(f.reloadFirewall)
 	c.RegisterReloadCallback(f.reloadSendRecvError)
@@ -544,6 +525,7 @@ func (f *Interface) GetCertState() *CertState {
 func (f *Interface) Close() error {
 	f.closed.Store(true)
 
+	// Release the udp readers
 	for _, u := range f.writers {
 		err := u.Close()
 		if err != nil {
@@ -551,6 +533,13 @@ func (f *Interface) Close() error {
 		}
 	}
 
-	// Release the tun device
+	// Release the tun readers
-	return f.inside.Close()
+	for _, u := range f.readers {
+		err := u.Close()
+		if err != nil {
+			f.l.WithError(err).Error("Error while closing tun device")
+		}
+	}
+
+	return nil
 }

main.go

@@ -242,7 +242,6 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		relayManager:          NewRelayManager(ctx, l, hostMap, c),
 		punchy:                punchy,
 		ConntrackCacheTimeout: conntrackCacheTimeout,
-		batchSize:             c.GetInt("tun.batch_size", 64),
 		l:                     l,
 	}
 
@@ -285,14 +284,14 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	}
 
 	return &Control{
-		ifce,
+		f:                      ifce,
-		l,
+		l:                      l,
-		ctx,
+		ctx:                    ctx,
-		cancel,
+		cancel:                 cancel,
-		sshStart,
+		sshStart:               sshStart,
-		statsStart,
+		statsStart:             statsStart,
-		dnsStart,
+		dnsStart:               dnsStart,
-		lightHouse.StartUpdateWorker,
+		lighthouseStart:        lightHouse.StartUpdateWorker,
-		connManager.Start,
+		connectionManagerStart: connManager.Start,
 	}, nil
 }

outside.go

@@ -29,7 +29,7 @@ func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []
 		return
 	}
 
-	//l.Error("in packet ", header, packet[HeaderLen:])
+	//f.l.Error("in packet ", h)
 	if ip.IsValid() {
 		if f.myVpnNetworksTable.Contains(ip.Addr()) {
 			if f.l.Level >= logrus.DebugLevel {
@@ -245,6 +245,7 @@ func (f *Interface) handleHostRoaming(hostinfo *HostInfo, udpAddr netip.AddrPort
 			return
 		}
 
+		//TODO: Seems we have a bunch of stuff racing here, since we don't have a lock on hostinfo anymore we announce roaming in bursts
 		hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", udpAddr).
 			Info("Host roamed to new udp ip/port.")
 		hostinfo.lastRoam = time.Now()
@@ -333,13 +334,12 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 			}
 
 			fp.Protocol = uint8(proto)
-			ports := data[offset : offset+4]
 			if incoming {
-				fp.RemotePort = binary.BigEndian.Uint16(ports[0:2])
+				fp.RemotePort = binary.BigEndian.Uint16(data[offset : offset+2])
-				fp.LocalPort = binary.BigEndian.Uint16(ports[2:4])
+				fp.LocalPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
 			} else {
-				fp.LocalPort = binary.BigEndian.Uint16(ports[0:2])
+				fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
-				fp.RemotePort = binary.BigEndian.Uint16(ports[2:4])
+				fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
 			}
 
 			fp.Fragment = false
@@ -471,7 +471,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 
 	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
 	if err != nil {
-		hostinfo.logger(f.l).WithError(err).Error("Failed to decrypt packet")
+		hostinfo.logger(f.l).WithError(err).WithField("fwPacket", fwPacket).Error("Failed to decrypt packet")
 		return false
 	}
 

overlay/route.go

@@ -3,6 +3,7 @@ package overlay
 import (
 	"fmt"
 	"math"
+	"net"
 	"net/netip"
 	"runtime"
 	"strconv"
@@ -304,3 +305,29 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
 
 	return routes, nil
 }
+
+func ipWithin(o *net.IPNet, i *net.IPNet) bool {
+	// Make sure o contains the lowest form of i
+	if !o.Contains(i.IP.Mask(i.Mask)) {
+		return false
+	}
+
+	// Find the max ip in i
+	ip4 := i.IP.To4()
+	if ip4 == nil {
+		return false
+	}
+
+	last := make(net.IP, len(ip4))
+	copy(last, ip4)
+	for x := range ip4 {
+		last[x] |= ^i.Mask[x]
+	}
+
+	// Make sure o contains the max
+	if !o.Contains(last) {
+		return false
+	}
+
+	return true
+}

overlay/route_test.go

@@ -225,7 +225,6 @@ func Test_parseUnsafeRoutes(t *testing.T) {
 	// no mtu
 	c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": "127.0.0.1", "route": "1.0.0.0/8"}}}
 	routes, err = parseUnsafeRoutes(c, []netip.Prefix{n})
-	require.NoError(t, err)
 	assert.Len(t, routes, 1)
 	assert.Equal(t, 0, routes[0].MTU)
 
@@ -319,7 +318,7 @@ func Test_makeRouteTree(t *testing.T) {
 
 	ip, err = netip.ParseAddr("1.1.0.1")
 	require.NoError(t, err)
-	_, ok = routeTree.Lookup(ip)
+	r, ok = routeTree.Lookup(ip)
 	assert.False(t, ok)
 }
 

overlay/tun.go

@@ -1,6 +1,7 @@
 package overlay
 
 import (
+	"net"
 	"net/netip"
 
 	"github.com/sirupsen/logrus"
@@ -70,3 +71,13 @@ func findRemovedRoutes(newRoutes, oldRoutes []Route) []Route {
 
 	return removed
 }
+
+func prefixToMask(prefix netip.Prefix) netip.Addr {
+	pLen := 128
+	if prefix.Addr().Is4() {
+		pLen = 32
+	}
+
+	addr, _ := netip.AddrFromSlice(net.CIDRMask(prefix.Bits(), pLen))
+	return addr
+}

overlay/tun_darwin.go

@@ -1,5 +1,5 @@
-//go:build darwin && !ios && !e2e_testing
+//go:build !ios && !e2e_testing
-// +build darwin,!ios,!e2e_testing
+// +build !ios,!e2e_testing
 
 package overlay
 
@@ -8,27 +8,48 @@ import (
 	"fmt"
 	"io"
 	"net/netip"
+	"os"
+	"sync/atomic"
+	"syscall"
 	"unsafe"
 
+	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
 	"golang.org/x/sys/unix"
-	wgtun "golang.zx2c4.com/wireguard/tun"
 )
 
 type tun struct {
-	linkAddr *netroute.LinkAddr
+	io.ReadWriteCloser
+	Device      string
+	vpnNetworks []netip.Prefix
+	DefaultMTU  int
+	Routes      atomic.Pointer[[]Route]
+	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
+	linkAddr    *netroute.LinkAddr
+	l           *logrus.Logger
+
+	// cache out buffer since we need to prepend 4 bytes for tun metadata
+	out []byte
 }
 
-// ioctl structures for Darwin network configuration
 type ifReq struct {
 	Name  [unix.IFNAMSIZ]byte
 	Flags uint16
 	pad   [8]byte
 }
 
+const (
+	_SIOCAIFADDR_IN6 = 2155899162
+	_UTUN_OPT_IFNAME = 2
+	_IN6_IFF_NODAD   = 0x0020
+	_IN6_IFF_SECURED = 0x0400
+	utunControlName  = "com.apple.net.utun_control"
+)
+
 type ifreqMTU struct {
 	Name [16]byte
 	MTU  int32
@@ -58,61 +79,60 @@ type ifreqAlias6 struct {
 	Lifetime   addrLifetime
 }
 
-const (
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
-	_SIOCAIFADDR_IN6 = 2155899162
-	_IN6_IFF_NODAD   = 0x0020
-)
-
-func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*wgTun, error) {
-	return nil, fmt.Errorf("newTunFromFd not supported on Darwin")
-}
-
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*wgTun, error) {
 	name := c.GetString("tun.dev", "")
-	deviceName := "utun"
+	ifIndex := -1
-
-	// Parse device name to handle utun[0-9]+ format
 	if name != "" && name != "utun" {
-		ifIndex := -1
 		_, err := fmt.Sscanf(name, "utun%d", &ifIndex)
 		if err != nil || ifIndex < 0 {
 			// NOTE: we don't make this error so we don't break existing
 			// configs that set a name before it was used.
 			l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring")
-		} else {
+			ifIndex = -1
-			deviceName = name
 		}
 	}
 
-	mtu := c.GetInt("tun.mtu", DefaultMTU)
+	fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL)
-
-	// Create WireGuard TUN device
-	tunDevice, err := wgtun.CreateTUN(deviceName, mtu)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create TUN device: %w", err)
+		return nil, fmt.Errorf("system socket: %v", err)
 	}
 
-	// Get the actual device name
+	var ctlInfo = &unix.CtlInfo{}
-	actualName, err := tunDevice.Name()
+	copy(ctlInfo.Name[:], utunControlName)
+
+	err = unix.IoctlCtlInfo(fd, ctlInfo)
 	if err != nil {
-		tunDevice.Close()
+		return nil, fmt.Errorf("CTLIOCGINFO: %v", err)
-		return nil, fmt.Errorf("failed to get TUN device name: %w", err)
 	}
 
-	t := &wgTun{
+	err = unix.Connect(fd, &unix.SockaddrCtl{
-		tunDevice:   tunDevice,
+		ID:   ctlInfo.Id,
-		vpnNetworks: vpnNetworks,
+		Unit: uint32(ifIndex) + 1,
-		MaxMTU:      mtu,
+	})
-		DefaultMTU:  mtu,
+	if err != nil {
-		l:           l,
+		return nil, fmt.Errorf("SYS_CONNECT: %v", err)
 	}
 
-	// Create Darwin-specific route manager
+	name, err = unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME)
-	t.routeManager = &tun{}
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve tun name: %w", err)
+	}
+
+	err = unix.SetNonblock(fd, true)
+	if err != nil {
+		return nil, fmt.Errorf("SetNonblock: %v", err)
+	}
+
+	t := &tun{
+		ReadWriteCloser: os.NewFile(uintptr(fd), ""),
+		Device:          name,
+		vpnNetworks:     vpnNetworks,
+		DefaultMTU:      c.GetInt("tun.mtu", DefaultMTU),
+		l:               l,
+	}
 
 	err = t.reload(c, true)
 	if err != nil {
-		tunDevice.Close()
 		return nil, err
 	}
 
@@ -123,251 +143,216 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		}
 	})
 
-	l.WithField("name", actualName).Info("Created WireGuard TUN device")
-
 	return t, nil
 }
 
-func (rm *tun) Activate(t *wgTun) error {
+func (t *tun) deviceBytes() (o [16]byte) {
-	name, err := t.tunDevice.Name()
+	for i, c := range t.Device {
+		o[i] = byte(c)
+	}
+	return
+}
+
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
+	return nil, fmt.Errorf("newTunFromFd not supported in Darwin")
+}
+
+func (t *tun) Close() error {
+	if t.ReadWriteCloser != nil {
+		return t.ReadWriteCloser.Close()
+	}
+	return nil
+}
+
+func (t *tun) Activate() error {
+	devName := t.deviceBytes()
+
+	s, err := unix.Socket(
+		unix.AF_INET,
+		unix.SOCK_DGRAM,
+		unix.IPPROTO_IP,
+	)
 	if err != nil {
-		return fmt.Errorf("failed to get device name: %w", err)
+		return err
+	}
+	defer unix.Close(s)
+
+	fd := uintptr(s)
+
+	// Set the MTU on the device
+	ifm := ifreqMTU{Name: devName, MTU: int32(t.DefaultMTU)}
+	if err = ioctl(fd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
+		return fmt.Errorf("failed to set tun mtu: %v", err)
 	}
 
-	// Set the MTU
+	// Get the device flags
-	rm.SetMTU(t, t.MaxMTU)
+	ifrf := ifReq{Name: devName}
-
+	if err = ioctl(fd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
-	// Add IP addresses
+		return fmt.Errorf("failed to get tun flags: %s", err)
-	for _, network := range t.vpnNetworks {
-		if err := rm.addIP(t, name, network); err != nil {
-			return err
-		}
 	}
 
-	// Bring up the interface using ioctl
+	linkAddr, err := getLinkAddr(t.Device)
-	if err := rm.bringUpInterface(name); err != nil {
-		return fmt.Errorf("failed to bring up interface: %w", err)
-	}
-
-	// Get the link address for routing
-	linkAddr, err := getLinkAddr(name)
 	if err != nil {
-		return fmt.Errorf("failed to get link address: %w", err)
+		return err
 	}
 	if linkAddr == nil {
 		return fmt.Errorf("unable to discover link_addr for tun interface")
 	}
-	rm.linkAddr = linkAddr
+	t.linkAddr = linkAddr
 
-	// Set the routes
+	for _, network := range t.vpnNetworks {
-	if err := rm.AddRoutes(t, false); err != nil {
+		if network.Addr().Is4() {
+			err = t.activate4(network)
+			if err != nil {
+				return err
+			}
+		} else {
+			err = t.activate6(network)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// Run the interface
+	ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
+	if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
+		return fmt.Errorf("failed to run tun device: %s", err)
+	}
+
+	// Unsafe path routes
+	return t.addRoutes(false)
+}
+
+func (t *tun) activate4(network netip.Prefix) error {
+	s, err := unix.Socket(
+		unix.AF_INET,
+		unix.SOCK_DGRAM,
+		unix.IPPROTO_IP,
+	)
+	if err != nil {
+		return err
+	}
+	defer unix.Close(s)
+
+	ifr := ifreqAlias4{
+		Name: t.deviceBytes(),
+		Addr: unix.RawSockaddrInet4{
+			Len:    unix.SizeofSockaddrInet4,
+			Family: unix.AF_INET,
+			Addr:   network.Addr().As4(),
+		},
+		DstAddr: unix.RawSockaddrInet4{
+			Len:    unix.SizeofSockaddrInet4,
+			Family: unix.AF_INET,
+			Addr:   network.Addr().As4(),
+		},
+		MaskAddr: unix.RawSockaddrInet4{
+			Len:    unix.SizeofSockaddrInet4,
+			Family: unix.AF_INET,
+			Addr:   prefixToMask(network).As4(),
+		},
+	}
+
+	if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
+		return fmt.Errorf("failed to set tun v4 address: %s", err)
+	}
+
+	err = addRoute(network, t.linkAddr)
+	if err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (rm *tun) bringUpInterface(name string) error {
+func (t *tun) activate6(network netip.Prefix) error {
-	// Open a socket for ioctl
+	s, err := unix.Socket(
-	fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
+		unix.AF_INET6,
+		unix.SOCK_DGRAM,
+		unix.IPPROTO_IP,
+	)
 	if err != nil {
-		return fmt.Errorf("failed to create socket: %w", err)
+		return err
-	}
-	defer unix.Close(fd)
-
-	// Get current flags
-	var ifrf ifReq
-	copy(ifrf.Name[:], name)
-
-	if err := ioctl(uintptr(fd), unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
-		return fmt.Errorf("failed to get interface flags: %w", err)
-	}
-
-	// Set IFF_UP and IFF_RUNNING flags
-	ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
-
-	if err := ioctl(uintptr(fd), unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
-		return fmt.Errorf("failed to set interface flags: %w", err)
-	}
-
-	return nil
-}
-
-func (rm *tun) SetMTU(t *wgTun, mtu int) {
-	name, err := t.tunDevice.Name()
-	if err != nil {
-		t.l.WithError(err).Error("Failed to get device name for MTU set")
-		return
-	}
-
-	// Open a socket for ioctl
-	fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
-	if err != nil {
-		t.l.WithError(err).Error("Failed to create socket for MTU set")
-		return
-	}
-	defer unix.Close(fd)
-
-	// Prepare the ioctl request
-	var ifr ifreqMTU
-	copy(ifr.Name[:], name)
-	ifr.MTU = int32(mtu)
-
-	// Set the MTU using ioctl
-	if err := ioctl(uintptr(fd), unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifr))); err != nil {
-		t.l.WithError(err).Error("Failed to set tun mtu via ioctl")
-	}
-}
-
-func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error {
-	// On Darwin, routes are set via ifconfig and route commands
-	return nil
-}
-
-func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error {
-	routes := *t.Routes.Load()
-	for _, r := range routes {
-		if !r.Install {
-			continue
-		}
-
-		err := rm.addRoute(r.Cidr)
-		if err != nil {
-			if errors.Is(err, unix.EEXIST) {
-				t.l.WithField("route", r.Cidr).
-					Warnf("unable to add unsafe_route, identical route already exists")
-			} else {
-				retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
-				if logErrors {
-					retErr.Log(t.l)
-				} else {
-					return retErr
-				}
-			}
-		} else {
-			t.l.WithField("route", r).Info("Added route")
-		}
-	}
-
-	return nil
-}
-
-func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) {
-	for _, r := range routes {
-		if !r.Install {
-			continue
-		}
-
-		err := rm.delRoute(r.Cidr)
-		if err != nil {
-			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
-		} else {
-			t.l.WithField("route", r).Info("Removed route")
-		}
-	}
-}
-
-func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) {
-	// Darwin doesn't support multi-queue TUN devices in the same way as Linux
-	// Return a reader that wraps the same device
-	return &wgTunReader{
-		parent:    t,
-		tunDevice: t.tunDevice,
-		offset:    0,
-		l:         t.l,
-	}, nil
-}
-
-func (rm *tun) addIP(t *wgTun, name string, network netip.Prefix) error {
-	addr := network.Addr()
-
-	if addr.Is4() {
-		return rm.addIPv4(name, network)
-	} else {
-		return rm.addIPv6(name, network)
-	}
-}
-
-func (rm *tun) addIPv4(name string, network netip.Prefix) error {
-	// Open an IPv4 socket for ioctl
-	s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-	if err != nil {
-		return fmt.Errorf("failed to create IPv4 socket: %w", err)
 	}
 	defer unix.Close(s)
 
-	var ifr ifreqAlias4
+	ifr := ifreqAlias6{
-	copy(ifr.Name[:], name)
+		Name: t.deviceBytes(),
-
+		Addr: unix.RawSockaddrInet6{
-	// Set the address
+			Len:    unix.SizeofSockaddrInet6,
-	ifr.Addr = unix.RawSockaddrInet4{
+			Family: unix.AF_INET6,
-		Len:    unix.SizeofSockaddrInet4,
+			Addr:   network.Addr().As16(),
-		Family: unix.AF_INET,
+		},
-		Addr:   network.Addr().As4(),
+		PrefixMask: unix.RawSockaddrInet6{
+			Len:    unix.SizeofSockaddrInet6,
+			Family: unix.AF_INET6,
+			Addr:   prefixToMask(network).As16(),
+		},
+		Lifetime: addrLifetime{
+			// never expires
+			Vltime: 0xffffffff,
+			Pltime: 0xffffffff,
+		},
+		//TODO: CERT-V2 should we disable DAD (duplicate address detection) and mark this as a secured address?
+		Flags: _IN6_IFF_NODAD,
 	}
 
-	// Set the destination address (same as address for point-to-point)
-	ifr.DstAddr = unix.RawSockaddrInet4{
-		Len:    unix.SizeofSockaddrInet4,
-		Family: unix.AF_INET,
-		Addr:   network.Addr().As4(),
-	}
-
-	// Set the netmask
-	ifr.MaskAddr = unix.RawSockaddrInet4{
-		Len:    unix.SizeofSockaddrInet4,
-		Family: unix.AF_INET,
-		Addr:   prefixToMask(network).As4(),
-	}
-
-	if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
-		return fmt.Errorf("failed to set IPv4 address via ioctl: %w", err)
-	}
-
-	return nil
-}
-
-func (rm *tun) addIPv6(name string, network netip.Prefix) error {
-	// Open an IPv6 socket for ioctl
-	s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-	if err != nil {
-		return fmt.Errorf("failed to create IPv6 socket: %w", err)
-	}
-	defer unix.Close(s)
-
-	var ifr ifreqAlias6
-	copy(ifr.Name[:], name)
-
-	// Set the address
-	ifr.Addr = unix.RawSockaddrInet6{
-		Len:    unix.SizeofSockaddrInet6,
-		Family: unix.AF_INET6,
-		Addr:   network.Addr().As16(),
-	}
-
-	// Set the prefix mask
-	ifr.PrefixMask = unix.RawSockaddrInet6{
-		Len:    unix.SizeofSockaddrInet6,
-		Family: unix.AF_INET6,
-		Addr:   prefixToMask(network).As16(),
-	}
-
-	// Set lifetime (never expires)
-	ifr.Lifetime = addrLifetime{
-		Vltime: 0xffffffff,
-		Pltime: 0xffffffff,
-	}
-
-	// Set flags (no DAD - Duplicate Address Detection)
-	ifr.Flags = _IN6_IFF_NODAD
-
 	if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
-		return fmt.Errorf("failed to set IPv6 address via ioctl: %w", err)
+		return fmt.Errorf("failed to set tun address: %s", err)
 	}
 
 	return nil
 }
 
+func (t *tun) reload(c *config.C, initial bool) error {
+	change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
+	if err != nil {
+		return err
+	}
+
+	if !initial && !change {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		if err != nil {
+			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
+		}
+
+		// Ensure any routes we actually want are installed
+		err = t.addRoutes(true)
+		if err != nil {
+			// Catch any stray logs
+			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
+		}
+	}
+
+	return nil
+}
+
+func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
+	r, ok := t.routeTree.Load().Lookup(ip)
+	if ok {
+		return r
+	}
+	return routing.Gateways{}
+}
+
+// Get the LinkAddr for the interface of the given name
+// Is there an easier way to fetch this when we create the interface?
+// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
 func getLinkAddr(name string) (*netroute.LinkAddr, error) {
 	rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0)
 	if err != nil {
@@ -393,7 +378,53 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
 	return nil, nil
 }
 
-func (rm *tun) addRoute(prefix netip.Prefix) error {
+func (t *tun) addRoutes(logErrors bool) error {
+	routes := *t.Routes.Load()
+
+	for _, r := range routes {
+		if len(r.Via) == 0 || !r.Install {
+			// We don't allow route MTUs so only install routes with a via
+			continue
+		}
+
+		err := addRoute(r.Cidr, t.linkAddr)
+		if err != nil {
+			if errors.Is(err, unix.EEXIST) {
+				t.l.WithField("route", r.Cidr).
+					Warnf("unable to add unsafe_route, identical route already exists")
+			} else {
+				retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
+				if logErrors {
+					retErr.Log(t.l)
+				} else {
+					return retErr
+				}
+			}
+		} else {
+			t.l.WithField("route", r).Info("Added route")
+		}
+	}
+
+	return nil
+}
+
+func (t *tun) removeRoutes(routes []Route) error {
+	for _, r := range routes {
+		if !r.Install {
+			continue
+		}
+
+		err := delRoute(r.Cidr, t.linkAddr)
+		if err != nil {
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
+		} else {
+			t.l.WithField("route", r).Info("Removed route")
+		}
+	}
+	return nil
+}
+
+func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
 	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
 	if err != nil {
 		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
@@ -411,13 +442,13 @@ func (rm *tun) addRoute(prefix netip.Prefix) error {
 		route.Addrs = []netroute.Addr{
 			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
 			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: rm.linkAddr,
+			unix.RTAX_GATEWAY: gateway,
 		}
 	} else {
 		route.Addrs = []netroute.Addr{
 			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
 			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: rm.linkAddr,
+			unix.RTAX_GATEWAY: gateway,
 		}
 	}
 
@@ -434,7 +465,7 @@ func (rm *tun) addRoute(prefix netip.Prefix) error {
 	return nil
 }
 
-func (rm *tun) delRoute(prefix netip.Prefix) error {
+func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
 	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
 	if err != nil {
 		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
@@ -451,13 +482,13 @@ func (rm *tun) delRoute(prefix netip.Prefix) error {
 		route.Addrs = []netroute.Addr{
 			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
 			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: rm.linkAddr,
+			unix.RTAX_GATEWAY: gateway,
 		}
 	} else {
 		route.Addrs = []netroute.Addr{
 			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
 			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: rm.linkAddr,
+			unix.RTAX_GATEWAY: gateway,
 		}
 	}
 
@@ -465,7 +496,6 @@ func (rm *tun) delRoute(prefix netip.Prefix) error {
 	if err != nil {
 		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
 	}
-
 	_, err = unix.Write(sock, data[:])
 	if err != nil {
 		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
@@ -474,34 +504,52 @@ func (rm *tun) delRoute(prefix netip.Prefix) error {
 	return nil
 }
 
-func ioctl(a1, a2, a3 uintptr) error {
+func (t *tun) Read(to []byte) (int, error) {
-	_, _, errno := unix.Syscall(unix.SYS_IOCTL, a1, a2, a3)
+	buf := make([]byte, len(to)+4)
-	if errno != 0 {
+
-		return errno
+	n, err := t.ReadWriteCloser.Read(buf)
-	}
+
-	return nil
+	copy(to, buf[4:])
+	return n - 4, err
 }
 
-func prefixToMask(prefix netip.Prefix) netip.Addr {
+// Write is only valid for single threaded use
-	bits := prefix.Bits()
+func (t *tun) Write(from []byte) (int, error) {
-	if prefix.Addr().Is4() {
+	buf := t.out
-		// Create IPv4 netmask from prefix length
+	if cap(buf) < len(from)+4 {
-		mask := ^uint32(0) << (32 - bits)
+		buf = make([]byte, len(from)+4)
-		return netip.AddrFrom4([4]byte{
+		t.out = buf
-			byte(mask >> 24),
-			byte(mask >> 16),
-			byte(mask >> 8),
-			byte(mask),
-		})
-	} else {
-		// Create IPv6 netmask from prefix length
-		var mask [16]byte
-		for i := 0; i < bits/8; i++ {
-			mask[i] = 0xff
-		}
-		if bits%8 != 0 {
-			mask[bits/8] = ^byte(0) << (8 - bits%8)
-		}
-		return netip.AddrFrom16(mask)
 	}
+	buf = buf[:len(from)+4]
+
+	if len(from) == 0 {
+		return 0, syscall.EIO
+	}
+
+	// Determine the IP Family for the NULL L2 Header
+	ipVer := from[0] >> 4
+	if ipVer == 4 {
+		buf[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		buf[3] = syscall.AF_INET6
+	} else {
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+
+	copy(buf[4:], from)
+
+	n, err := t.ReadWriteCloser.Write(buf)
+	return n - 4, err
+}
+
+func (t *tun) Networks() []netip.Prefix {
+	return t.vpnNetworks
+}
+
+func (t *tun) Name() string {
+	return t.Device
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
 }

overlay/tun_freebsd.go

@@ -1,77 +1,284 @@
-//go:build freebsd && !e2e_testing
+//go:build !e2e_testing
-// +build freebsd,!e2e_testing
+// +build !e2e_testing
 
 package overlay
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/netip"
-	"os/exec"
+	"sync/atomic"
-	"strconv"
-	"strings"
 	"syscall"
+	"time"
 	"unsafe"
 
+	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
+	netroute "golang.org/x/net/route"
 	"golang.org/x/sys/unix"
-	wgtun "golang.zx2c4.com/wireguard/tun"
 )
 
-type tun struct{}
+const (
+	// FIODGNAME is defined in sys/sys/filio.h on FreeBSD
+	// For 32-bit systems, use FIODGNAME_32 (not defined in this file: 0x80086678)
+	FIODGNAME        = 0x80106678
+	TUNSIFMODE       = 0x8004745e
+	TUNSIFHEAD       = 0x80047460
+	OSIOCAIFADDR_IN6 = 0x8088691b
+	IN6_IFF_NODAD    = 0x0020
+)
+
+type fiodgnameArg struct {
+	length int32
+	pad    [4]byte
+	buf    unsafe.Pointer
+}
 
-// ifreqRename is used for renaming network interfaces on FreeBSD
 type ifreqRename struct {
 	Name [unix.IFNAMSIZ]byte
 	Data uintptr
 }
 
-func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*wgTun, error) {
+type ifreqDestroy struct {
-	return nil, fmt.Errorf("newTunFromFd not supported on FreeBSD")
+	Name [unix.IFNAMSIZ]byte
+	pad  [16]byte
 }
 
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*wgTun, error) {
+type ifReq struct {
-	deviceName := c.GetString("tun.dev", "tun")
+	Name  [unix.IFNAMSIZ]byte
-	mtu := c.GetInt("tun.mtu", DefaultMTU)
+	Flags uint16
+}
 
-	// Create WireGuard TUN device
+type ifreqMTU struct {
-	tunDevice, err := wgtun.CreateTUN(deviceName, mtu)
+	Name [unix.IFNAMSIZ]byte
-	if err != nil {
+	MTU  int32
-		return nil, fmt.Errorf("failed to create TUN device: %w", err)
+}
+
+type addrLifetime struct {
+	Expire    uint64
+	Preferred uint64
+	Vltime    uint32
+	Pltime    uint32
+}
+
+type ifreqAlias4 struct {
+	Name     [unix.IFNAMSIZ]byte
+	Addr     unix.RawSockaddrInet4
+	DstAddr  unix.RawSockaddrInet4
+	MaskAddr unix.RawSockaddrInet4
+	VHid     uint32
+}
+
+type ifreqAlias6 struct {
+	Name       [unix.IFNAMSIZ]byte
+	Addr       unix.RawSockaddrInet6
+	DstAddr    unix.RawSockaddrInet6
+	PrefixMask unix.RawSockaddrInet6
+	Flags      uint32
+	Lifetime   addrLifetime
+	VHid       uint32
+}
+
+type tun struct {
+	Device      string
+	vpnNetworks []netip.Prefix
+	MTU         int
+	Routes      atomic.Pointer[[]Route]
+	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
+	linkAddr    *netroute.LinkAddr
+	l           *logrus.Logger
+	devFd       int
+}
+
+func (t *tun) Read(to []byte) (int, error) {
+	// use readv() to read from the tunnel device, to eliminate the need for copying the buffer
+	if t.devFd < 0 {
+		return -1, syscall.EINVAL
 	}
 
-	// Get the actual device name
+	// first 4 bytes is protocol family, in network byte order
-	actualName, err := tunDevice.Name()
+	head := make([]byte, 4)
+
+	iovecs := []syscall.Iovec{
+		{&head[0], 4},
+		{&to[0], uint64(len(to))},
+	}
+
+	n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+
+	var err error
+	if errno != 0 {
+		err = syscall.Errno(errno)
+	} else {
+		err = nil
+	}
+	// fix bytes read number to exclude header
+	bytesRead := int(n)
+	if bytesRead < 0 {
+		return bytesRead, err
+	} else if bytesRead < 4 {
+		return 0, err
+	} else {
+		return bytesRead - 4, err
+	}
+}
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	// use writev() to write to the tunnel device, to eliminate the need for copying the buffer
+	if t.devFd < 0 {
+		return -1, syscall.EINVAL
+	}
+
+	if len(from) <= 1 {
+		return 0, syscall.EIO
+	}
+	ipVer := from[0] >> 4
+	var head []byte
+	// first 4 bytes is protocol family, in network byte order
+	if ipVer == 4 {
+		head = []byte{0, 0, 0, syscall.AF_INET}
+	} else if ipVer == 6 {
+		head = []byte{0, 0, 0, syscall.AF_INET6}
+	} else {
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+	iovecs := []syscall.Iovec{
+		{&head[0], 4},
+		{&from[0], uint64(len(from))},
+	}
+
+	n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+
+	var err error
+	if errno != 0 {
+		err = syscall.Errno(errno)
+	} else {
+		err = nil
+	}
+
+	return int(n) - 4, err
+}
+
+func (t *tun) Close() error {
+	if t.devFd >= 0 {
+		err := syscall.Close(t.devFd)
+		if err != nil {
+			t.l.WithError(err).Error("Error closing device")
+		}
+		t.devFd = -1
+
+		c := make(chan struct{})
+		go func() {
+			// destroying the interface can block if a read() is still pending. Do this asynchronously.
+			defer close(c)
+			s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+			if err == nil {
+				defer syscall.Close(s)
+				ifreq := ifreqDestroy{Name: t.deviceBytes()}
+				err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
+			}
+			if err != nil {
+				t.l.WithError(err).Error("Error destroying tunnel")
+			}
+		}()
+
+		// wait up to 1 second so we start blocking at the ioctl
+		select {
+		case <-c:
+		case <-time.After(1 * time.Second):
+		}
+	}
+
+	return nil
+}
+
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
+	return nil, fmt.Errorf("newTunFromFd not supported in FreeBSD")
+}
+
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
+	// Try to open existing tun device
+	var fd int
+	var err error
+	deviceName := c.GetString("tun.dev", "")
+	if deviceName != "" {
+		fd, err = syscall.Open("/dev/"+deviceName, syscall.O_RDWR, 0)
+	}
+	if errors.Is(err, fs.ErrNotExist) || deviceName == "" {
+		// If the device doesn't already exist, request a new one and rename it
+		fd, err = syscall.Open("/dev/tun", syscall.O_RDWR, 0)
+	}
 	if err != nil {
-		tunDevice.Close()
+		return nil, err
-		return nil, fmt.Errorf("failed to get TUN device name: %w", err)
+	}
+
+	// Read the name of the interface
+	var name [16]byte
+	arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)}
+	ctrlErr := ioctl(uintptr(fd), FIODGNAME, uintptr(unsafe.Pointer(&arg)))
+
+	if ctrlErr == nil {
+		// set broadcast mode and multicast
+		ifmode := uint32(unix.IFF_BROADCAST | unix.IFF_MULTICAST)
+		ctrlErr = ioctl(uintptr(fd), TUNSIFMODE, uintptr(unsafe.Pointer(&ifmode)))
+	}
+
+	if ctrlErr == nil {
+		// turn on link-layer mode, to support ipv6
+		ifhead := uint32(1)
+		ctrlErr = ioctl(uintptr(fd), TUNSIFHEAD, uintptr(unsafe.Pointer(&ifhead)))
+	}
+
+	if ctrlErr != nil {
+		return nil, err
+	}
+
+	ifName := string(bytes.TrimRight(name[:], "\x00"))
+	if deviceName == "" {
+		deviceName = ifName
 	}
 
 	// If the name doesn't match the desired interface name, rename it now
-	if actualName != deviceName && deviceName != "" && deviceName != "tun" {
+	if ifName != deviceName {
-		if err := renameInterface(actualName, deviceName); err != nil {
+		s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-			tunDevice.Close()
+		if err != nil {
-			return nil, fmt.Errorf("failed to rename interface from %s to %s: %w", actualName, deviceName, err)
+			return nil, err
 		}
-		actualName = deviceName
+		defer syscall.Close(s)
+
+		fd := uintptr(s)
+
+		var fromName [16]byte
+		var toName [16]byte
+		copy(fromName[:], ifName)
+		copy(toName[:], deviceName)
+
+		ifrr := ifreqRename{
+			Name: fromName,
+			Data: uintptr(unsafe.Pointer(&toName)),
+		}
+
+		// Set the device name
+		ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
 	}
 
-	t := &wgTun{
+	t := &tun{
-		tunDevice:   tunDevice,
+		Device:      deviceName,
 		vpnNetworks: vpnNetworks,
-		MaxMTU:      mtu,
+		MTU:         c.GetInt("tun.mtu", DefaultMTU),
-		DefaultMTU:  mtu,
 		l:           l,
+		devFd:       fd,
 	}
 
-	// Create FreeBSD-specific route manager
-	t.routeManager = &tun{}
-
 	err = t.reload(c, true)
 	if err != nil {
-		tunDevice.Close()
 		return nil, err
 	}
 
@@ -82,86 +289,180 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		}
 	})
 
-	l.WithField("name", actualName).Info("Created WireGuard TUN device")
-
 	return t, nil
 }
 
-func (rm *tun) Activate(t *wgTun) error {
+func (t *tun) addIp(cidr netip.Prefix) error {
-	name, err := t.tunDevice.Name()
+	if cidr.Addr().Is4() {
-	if err != nil {
+		ifr := ifreqAlias4{
-		return fmt.Errorf("failed to get device name: %w", err)
+			Name: t.deviceBytes(),
+			Addr: unix.RawSockaddrInet4{
+				Len:    unix.SizeofSockaddrInet4,
+				Family: unix.AF_INET,
+				Addr:   cidr.Addr().As4(),
+			},
+			DstAddr: unix.RawSockaddrInet4{
+				Len:    unix.SizeofSockaddrInet4,
+				Family: unix.AF_INET,
+				Addr:   getBroadcast(cidr).As4(),
+			},
+			MaskAddr: unix.RawSockaddrInet4{
+				Len:    unix.SizeofSockaddrInet4,
+				Family: unix.AF_INET,
+				Addr:   prefixToMask(cidr).As4(),
+			},
+			VHid: 0,
+		}
+		s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+		if err != nil {
+			return err
+		}
+		defer syscall.Close(s)
+		// Note: unix.SIOCAIFADDR corresponds to FreeBSD's OSIOCAIFADDR
+		if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
+			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err)
+		}
+		return nil
 	}
 
-	// Set the MTU
+	if cidr.Addr().Is6() {
-	rm.SetMTU(t, t.MaxMTU)
+		ifr := ifreqAlias6{
+			Name: t.deviceBytes(),
+			Addr: unix.RawSockaddrInet6{
+				Len:    unix.SizeofSockaddrInet6,
+				Family: unix.AF_INET6,
+				Addr:   cidr.Addr().As16(),
+			},
+			PrefixMask: unix.RawSockaddrInet6{
+				Len:    unix.SizeofSockaddrInet6,
+				Family: unix.AF_INET6,
+				Addr:   prefixToMask(cidr).As16(),
+			},
+			Lifetime: addrLifetime{
+				Expire:    0,
+				Preferred: 0,
+				Vltime:    0xffffffff,
+				Pltime:    0xffffffff,
+			},
+			Flags: IN6_IFF_NODAD,
+		}
+		s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+		if err != nil {
+			return err
+		}
+		defer syscall.Close(s)
 
-	// Add IP addresses
+		if err := ioctl(uintptr(s), OSIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
-	for _, network := range t.vpnNetworks {
+			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err)
-		if err := rm.addIP(t, name, network); err != nil {
+		}
+		return nil
+	}
+
+	return fmt.Errorf("unknown address type %v", cidr)
+}
+
+func (t *tun) Activate() error {
+	// Setup our default MTU
+	err := t.setMTU()
+	if err != nil {
+		return err
+	}
+
+	linkAddr, err := getLinkAddr(t.Device)
+	if err != nil {
+		return err
+	}
+	if linkAddr == nil {
+		return fmt.Errorf("unable to discover link_addr for tun interface")
+	}
+	t.linkAddr = linkAddr
+
+	for i := range t.vpnNetworks {
+		err := t.addIp(t.vpnNetworks[i])
+		if err != nil {
 			return err
 		}
 	}
 
-	// Bring up the interface
+	return t.addRoutes(false)
-	if err := runCommandBSD("ifconfig", name, "up"); err != nil {
+}
-		return fmt.Errorf("failed to bring up interface: %w", err)
-	}
 
-	// Set the routes
+func (t *tun) setMTU() error {
-	if err := rm.AddRoutes(t, false); err != nil {
+	// Set the MTU on the device
+	s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+	if err != nil {
+		return err
+	}
+	defer syscall.Close(s)
+
+	ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MTU)}
+	err = ioctl(uintptr(s), unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm)))
+	return err
+}
+
+func (t *tun) reload(c *config.C, initial bool) error {
+	change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
+	if err != nil {
 		return err
 	}
 
+	if !initial && !change {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		if err != nil {
+			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
+		}
+
+		// Ensure any routes we actually want are installed
+		err = t.addRoutes(true)
+		if err != nil {
+			// Catch any stray logs
+			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
+		}
+	}
+
 	return nil
 }
 
-func (rm *tun) SetMTU(t *wgTun, mtu int) {
+func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
-	name, err := t.tunDevice.Name()
+	r, _ := t.routeTree.Load().Lookup(ip)
-	if err != nil {
+	return r
-		t.l.WithError(err).Error("Failed to get device name for MTU set")
-		return
-	}
-
-	if err := runCommandBSD("ifconfig", name, "mtu", strconv.Itoa(mtu)); err != nil {
-		t.l.WithError(err).Error("Failed to set tun mtu")
-	}
 }
 
-func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error {
+func (t *tun) Networks() []netip.Prefix {
-	// On FreeBSD, routes are set via ifconfig and route commands
+	return t.vpnNetworks
-	return nil
 }
 
-func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error {
+func (t *tun) Name() string {
-	name, err := t.tunDevice.Name()
+	return t.Device
-	if err != nil {
+}
-		return fmt.Errorf("failed to get device name: %w", err)
-	}
 
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
+}
+
+func (t *tun) addRoutes(logErrors bool) error {
 	routes := *t.Routes.Load()
 	for _, r := range routes {
-		if !r.Install {
+		if len(r.Via) == 0 || !r.Install {
+			// We don't allow route MTUs so only install routes with a via
 			continue
 		}
 
-		// Add route using route command
+		err := addRoute(r.Cidr, t.linkAddr)
-		args := []string{"add"}
-
-		if r.Cidr.Addr().Is6() {
-			args = append(args, "-inet6")
-		} else {
-			args = append(args, "-inet")
-		}
-
-		args = append(args, r.Cidr.String(), "-interface", name)
-
-		if r.Metric > 0 {
-			// FreeBSD doesn't support route metrics directly like Linux
-			t.l.WithField("route", r).Warn("Route metrics are not fully supported on FreeBSD")
-		}
-
-		err := runCommandBSD("route", args...)
 		if err != nil {
 			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
 			if logErrors {
@@ -177,99 +478,166 @@ func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error {
 	return nil
 }
 
-func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) {
+func (t *tun) removeRoutes(routes []Route) error {
-	name, err := t.tunDevice.Name()
-	if err != nil {
-		t.l.WithError(err).Error("Failed to get device name for route removal")
-		return
-	}
-
 	for _, r := range routes {
 		if !r.Install {
 			continue
 		}
 
-		args := []string{"delete"}
+		err := delRoute(r.Cidr, t.linkAddr)
-
-		if r.Cidr.Addr().Is6() {
-			args = append(args, "-inet6")
-		} else {
-			args = append(args, "-inet")
-		}
-
-		args = append(args, r.Cidr.String(), "-interface", name)
-
-		err := runCommandBSD("route", args...)
 		if err != nil {
 			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
 			t.l.WithField("route", r).Info("Removed route")
 		}
 	}
+	return nil
 }
 
-func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) {
+func (t *tun) deviceBytes() (o [16]byte) {
-	// FreeBSD doesn't support multi-queue TUN devices in the same way as Linux
+	for i, c := range t.Device {
-	// Return a reader that wraps the same device
+		o[i] = byte(c)
-	return &wgTunReader{
+	}
-		parent:    t,
+	return
-		tunDevice: t.tunDevice,
-		offset:    0,
-		l:         t.l,
-	}, nil
 }
 
-func (rm *tun) addIP(t *wgTun, name string, network netip.Prefix) error {
+func flipBytes(b []byte) []byte {
-	addr := network.Addr()
+	for i := 0; i < len(b); i++ {
+		b[i] ^= 0xFF
+	}
+	return b
+}
+func orBytes(a []byte, b []byte) []byte {
+	ret := make([]byte, len(a))
+	for i := 0; i < len(a); i++ {
+		ret[i] = a[i] | b[i]
+	}
+	return ret
+}
 
-	if addr.Is4() {
+func getBroadcast(cidr netip.Prefix) netip.Addr {
-		// For IPv4: ifconfig tun0 10.0.0.1/24
+	broadcast, _ := netip.AddrFromSlice(
-		if err := runCommandBSD("ifconfig", name, network.String()); err != nil {
+		orBytes(
-			return fmt.Errorf("failed to add IPv4 address: %w", err)
+			cidr.Addr().AsSlice(),
+			flipBytes(prefixToMask(cidr).AsSlice()),
+		),
+	)
+	return broadcast
+}
+
+func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
+	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
+	}
+	defer unix.Close(sock)
+
+	route := &netroute.RouteMessage{
+		Version: unix.RTM_VERSION,
+		Type:    unix.RTM_ADD,
+		Flags:   unix.RTF_UP,
+		Seq:     1,
+	}
+
+	if prefix.Addr().Is4() {
+		route.Addrs = []netroute.Addr{
+			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
+			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
+			unix.RTAX_GATEWAY: gateway,
 		}
 	} else {
-		// For IPv6: ifconfig tun0 inet6 add 2001:db8::1/64
+		route.Addrs = []netroute.Addr{
-		if err := runCommandBSD("ifconfig", name, "inet6", "add", network.String()); err != nil {
+			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			return fmt.Errorf("failed to add IPv6 address: %w", err)
+			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
+			unix.RTAX_GATEWAY: gateway,
 		}
 	}
 
-	return nil
+	data, err := route.Marshal()
-}
-
-func runCommandBSD(name string, args ...string) error {
-	cmd := exec.Command(name, args...)
-	output, err := cmd.CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("%s %s failed: %w\nOutput: %s", name, strings.Join(args, " "), err, string(output))
+		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
 	}
-	return nil
-}
 
-func renameInterface(fromName, toName string) error {
+	_, err = unix.Write(sock, data[:])
-	s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
 	if err != nil {
-		return fmt.Errorf("failed to create socket: %w", err)
+		if errors.Is(err, unix.EEXIST) {
-	}
+			// Try to do a change
-	defer syscall.Close(s)
+			route.Type = unix.RTM_CHANGE
-
+			data, err = route.Marshal()
-	fd := uintptr(s)
+			if err != nil {
-
+				return fmt.Errorf("failed to create route.RouteMessage for change: %w", err)
-	var fromNameBytes [unix.IFNAMSIZ]byte
+			}
-	var toNameBytes [unix.IFNAMSIZ]byte
+			_, err = unix.Write(sock, data[:])
-	copy(fromNameBytes[:], fromName)
+			fmt.Println("DOING CHANGE")
-	copy(toNameBytes[:], toName)
+			return err
-
+		}
-	ifrr := ifreqRename{
+		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-		Name: fromNameBytes,
-		Data: uintptr(unsafe.Pointer(&toNameBytes)),
-	}
-
-	// Set the device name using SIOCSIFNAME ioctl
-	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
-	if errno != 0 {
-		return fmt.Errorf("SIOCSIFNAME ioctl failed: %w", errno)
 	}
 
 	return nil
 }
+
+func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
+	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
+	}
+	defer unix.Close(sock)
+
+	route := netroute.RouteMessage{
+		Version: unix.RTM_VERSION,
+		Type:    unix.RTM_DELETE,
+		Seq:     1,
+	}
+
+	if prefix.Addr().Is4() {
+		route.Addrs = []netroute.Addr{
+			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
+			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
+			unix.RTAX_GATEWAY: gateway,
+		}
+	} else {
+		route.Addrs = []netroute.Addr{
+			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
+			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
+			unix.RTAX_GATEWAY: gateway,
+		}
+	}
+
+	data, err := route.Marshal()
+	if err != nil {
+		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
+	}
+	_, err = unix.Write(sock, data[:])
+	if err != nil {
+		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
+	}
+
+	return nil
+}
+
+// getLinkAddr Gets the link address for the interface of the given name
+func getLinkAddr(name string) (*netroute.LinkAddr, error) {
+	rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0)
+	if err != nil {
+		return nil, err
+	}
+	msgs, err := netroute.ParseRIB(unix.NET_RT_IFLIST, rib)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, m := range msgs {
+		switch m := m.(type) {
+		case *netroute.InterfaceMessage:
+			if m.Name == name {
+				sa, ok := m.Addrs[unix.RTAX_IFP].(*netroute.LinkAddr)
+				if ok {
+					return sa, nil
+				}
+			}
+		}
+	}
+
+	return nil, nil
+}

overlay/tun_linux.go

@@ -1,5 +1,5 @@
-//go:build linux && !android && !e2e_testing
+//go:build !android && !e2e_testing
-// +build linux,!android,!e2e_testing
+// +build !android,!e2e_testing
 
 package overlay
 
@@ -9,105 +9,133 @@ import (
 	"net"
 	"net/netip"
 	"os"
+	"strings"
+	"sync/atomic"
 	"time"
 	"unsafe"
 
+	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
-	wgtun "golang.zx2c4.com/wireguard/tun"
 )
 
 type tun struct {
-	deviceIndex               int
+	io.ReadWriteCloser
-	ioctlFd                   uintptr
+	fd          int
-	txQueueLen                int
+	Device      string
+	vpnNetworks []netip.Prefix
+	MaxMTU      int
+	DefaultMTU  int
+	TXQueueLen  int
+	deviceIndex int
+	ioctlFd     uintptr
+
+	Routes                    atomic.Pointer[[]Route]
+	routeTree                 atomic.Pointer[bart.Table[routing.Gateways]]
+	routeChan                 chan struct{}
 	useSystemRoutes           bool
 	useSystemRoutesBufferSize int
+
+	l *logrus.Logger
 }
 
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*wgTun, error) {
+func (t *tun) Networks() []netip.Prefix {
-	deviceName := c.GetString("tun.dev", "")
+	return t.vpnNetworks
-	mtu := c.GetInt("tun.mtu", DefaultMTU)
-
-	// Create WireGuard TUN device
-	tunDevice, err := wgtun.CreateTUN(deviceName, mtu)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create TUN device: %w", err)
-	}
-
-	// Get the actual device name
-	actualName, err := tunDevice.Name()
-	if err != nil {
-		tunDevice.Close()
-		return nil, fmt.Errorf("failed to get TUN device name: %w", err)
-	}
-
-	t := &wgTun{
-		tunDevice:   tunDevice,
-		vpnNetworks: vpnNetworks,
-		MaxMTU:      mtu,
-		DefaultMTU:  mtu,
-		l:           l,
-	}
-
-	// Create Linux-specific route manager
-	routeManager := &tun{
-		txQueueLen:                c.GetInt("tun.tx_queue", 500),
-		useSystemRoutes:           c.GetBool("tun.use_system_route_table", false),
-		useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
-	}
-	t.routeManager = routeManager
-
-	err = t.reload(c, true)
-	if err != nil {
-		tunDevice.Close()
-		return nil, err
-	}
-
-	c.RegisterReloadCallback(func(c *config.C) {
-		err := t.reload(c, false)
-		if err != nil {
-			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
-		}
-	})
-
-	l.WithField("name", actualName).Info("Created WireGuard TUN device")
-
-	return t, nil
 }
 
-func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*wgTun, error) {
+type ifReq struct {
-	// Create TUN device from file descriptor
+	Name  [16]byte
+	Flags uint16
+	pad   [8]byte
+}
+
+type ifreqMTU struct {
+	Name [16]byte
+	MTU  int32
+	pad  [8]byte
+}
+
+type ifreqQLEN struct {
+	Name  [16]byte
+	Value int32
+	pad   [8]byte
+}
+
+func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
 	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
-	mtu := c.GetInt("tun.mtu", DefaultMTU)
+
-	tunDevice, err := wgtun.CreateTUNFromFile(file, mtu)
+	t, err := newTunGeneric(c, l, file, vpnNetworks)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create TUN device from fd: %w", err)
+		return nil, err
 	}
 
-	t := &wgTun{
+	t.Device = "tun0"
-		tunDevice:   tunDevice,
+
-		vpnNetworks: vpnNetworks,
+	return t, nil
-		MaxMTU:      mtu,
+}
-		DefaultMTU:  mtu,
+
-		l:           l,
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
+	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
+	if err != nil {
+		// If /dev/net/tun doesn't exist, try to create it (will happen in docker)
+		if os.IsNotExist(err) {
+			err = os.MkdirAll("/dev/net", 0755)
+			if err != nil {
+				return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
+			}
+			err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200)))
+			if err != nil {
+				return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err)
+			}
+
+			fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0)
+			if err != nil {
+				return nil, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
+			}
+		} else {
+			return nil, err
+		}
 	}
 
-	// Create Linux-specific route manager
+	var req ifReq
-	routeManager := &tun{
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
-		txQueueLen:                c.GetInt("tun.tx_queue", 500),
+	if multiqueue {
+		req.Flags |= unix.IFF_MULTI_QUEUE
+	}
+	copy(req.Name[:], c.GetString("tun.dev", ""))
+	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
+		return nil, err
+	}
+	name := strings.Trim(string(req.Name[:]), "\x00")
+
+	file := os.NewFile(uintptr(fd), "/dev/net/tun")
+	t, err := newTunGeneric(c, l, file, vpnNetworks)
+	if err != nil {
+		return nil, err
+	}
+
+	t.Device = name
+
+	return t, nil
+}
+
+func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []netip.Prefix) (*tun, error) {
+	t := &tun{
+		ReadWriteCloser:           file,
+		fd:                        int(file.Fd()),
+		vpnNetworks:               vpnNetworks,
+		TXQueueLen:                c.GetInt("tun.tx_queue", 500),
 		useSystemRoutes:           c.GetBool("tun.use_system_route_table", false),
 		useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
+		l:                         l,
 	}
-	t.routeManager = routeManager
 
-	err = t.reload(c, true)
+	err := t.reload(c, true)
 	if err != nil {
-		tunDevice.Close()
 		return nil, err
 	}
 
@@ -121,105 +149,273 @@ func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []net
 	return t, nil
 }
 
-func (rm *tun) Activate(t *wgTun) error {
+func (t *tun) reload(c *config.C, initial bool) error {
-	name, err := t.tunDevice.Name()
+	routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
 	if err != nil {
-		return fmt.Errorf("failed to get device name: %w", err)
+		return err
 	}
 
-	if t.routeManager.useSystemRoutes {
+	if !initial && !routeChange && !c.HasChanged("tun.mtu") {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, true)
+	if err != nil {
+		return err
+	}
+
+	oldDefaultMTU := t.DefaultMTU
+	oldMaxMTU := t.MaxMTU
+	newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU)
+	newMaxMTU := newDefaultMTU
+	for i, r := range routes {
+		if r.MTU == 0 {
+			routes[i].MTU = newDefaultMTU
+		}
+
+		if r.MTU > t.MaxMTU {
+			newMaxMTU = r.MTU
+		}
+	}
+
+	t.MaxMTU = newMaxMTU
+	t.DefaultMTU = newDefaultMTU
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		if oldMaxMTU != newMaxMTU {
+			t.setMTU()
+			t.l.Infof("Set max MTU to %v was %v", t.MaxMTU, oldMaxMTU)
+		}
+
+		if oldDefaultMTU != newDefaultMTU {
+			for i := range t.vpnNetworks {
+				err := t.setDefaultRoute(t.vpnNetworks[i])
+				if err != nil {
+					t.l.Warn(err)
+				} else {
+					t.l.Infof("Set default MTU to %v was %v", t.DefaultMTU, oldDefaultMTU)
+				}
+			}
+		}
+
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+
+		// Ensure any routes we actually want are installed
+		err = t.addRoutes(true)
+		if err != nil {
+			// This should never be called since addRoutes should log its own errors in a reload condition
+			util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l)
+		}
+	}
+
+	return nil
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	var req ifReq
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
+	copy(req.Name[:], t.Device)
+	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
+		return nil, err
+	}
+
+	file := os.NewFile(uintptr(fd), "/dev/net/tun")
+
+	return file, nil
+}
+
+func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
+	r, _ := t.routeTree.Load().Lookup(ip)
+	return r
+}
+
+func (t *tun) Write(b []byte) (int, error) {
+	var nn int
+	maximum := len(b)
+
+	for {
+		n, err := unix.Write(t.fd, b[nn:maximum])
+		if n > 0 {
+			nn += n
+		}
+		if nn == len(b) {
+			return nn, err
+		}
+
+		if err != nil {
+			return nn, err
+		}
+
+		if n == 0 {
+			return nn, io.ErrUnexpectedEOF
+		}
+	}
+}
+
+func (t *tun) deviceBytes() (o [16]byte) {
+	for i, c := range t.Device {
+		o[i] = byte(c)
+	}
+	return
+}
+
+func hasNetlinkAddr(al []*netlink.Addr, x netlink.Addr) bool {
+	for i := range al {
+		if al[i].Equal(x) {
+			return true
+		}
+	}
+	return false
+}
+
+// addIPs uses netlink to add all addresses that don't exist, then it removes ones that should not be there
+func (t *tun) addIPs(link netlink.Link) error {
+	newAddrs := make([]*netlink.Addr, len(t.vpnNetworks))
+	for i := range t.vpnNetworks {
+		newAddrs[i] = &netlink.Addr{
+			IPNet: &net.IPNet{
+				IP:   t.vpnNetworks[i].Addr().AsSlice(),
+				Mask: net.CIDRMask(t.vpnNetworks[i].Bits(), t.vpnNetworks[i].Addr().BitLen()),
+			},
+			Label: t.vpnNetworks[i].Addr().Zone(),
+		}
+	}
+
+	//add all new addresses
+	for i := range newAddrs {
+		//AddrReplace still adds new IPs, but if their properties change it will change them as well
+		if err := netlink.AddrReplace(link, newAddrs[i]); err != nil {
+			return err
+		}
+	}
+
+	//iterate over remainder, remove whoever shouldn't be there
+	al, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+	if err != nil {
+		return fmt.Errorf("failed to get tun address list: %s", err)
+	}
+
+	for i := range al {
+		if hasNetlinkAddr(newAddrs, al[i]) {
+			continue
+		}
+		err = netlink.AddrDel(link, &al[i])
+		if err != nil {
+			t.l.WithError(err).Error("failed to remove address from tun address list")
+		} else {
+			t.l.WithField("removed", al[i].String()).Info("removed address not listed in cert(s)")
+		}
+	}
+
+	return nil
+}
+
+func (t *tun) Activate() error {
+	devName := t.deviceBytes()
+
+	if t.useSystemRoutes {
 		t.watchRoutes()
 	}
 
-	// Get the netlink device
-	link, err := netlink.LinkByName(name)
-	if err != nil {
-		return fmt.Errorf("failed to get tun device link: %s", err)
-	}
-
-	rm.deviceIndex = link.Attrs().Index
-
-	// Open socket for ioctl operations
 	s, err := unix.Socket(
-		unix.AF_INET,
+		unix.AF_INET, //because everything we use t.ioctlFd for is address family independent, this is fine
 		unix.SOCK_DGRAM,
 		unix.IPPROTO_IP,
 	)
 	if err != nil {
 		return err
 	}
-	rm.ioctlFd = uintptr(s)
+	t.ioctlFd = uintptr(s)
 
-	rm.SetMTU(t, t.MaxMTU)
+	// Set the device name
+	ifrf := ifReq{Name: devName}
+	if err = ioctl(t.ioctlFd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
+		return fmt.Errorf("failed to set tun device name: %s", err)
+	}
+
+	link, err := netlink.LinkByName(t.Device)
+	if err != nil {
+		return fmt.Errorf("failed to get tun device link: %s", err)
+	}
+
+	t.deviceIndex = link.Attrs().Index
+
+	// Setup our default MTU
+	t.setMTU()
 
 	// Set the transmit queue length
-	devName := deviceBytes(name)
+	ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)}
-	ifrq := ifreqQLEN{Name: devName, Value: int32(rm.txQueueLen)}
+	if err = ioctl(t.ioctlFd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil {
-	if err = ioctl(t.routeManager.ioctlFd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil {
 		// If we can't set the queue length nebula will still work but it may lead to packet loss
 		t.l.WithError(err).Error("Failed to set tun tx queue length")
 	}
 
-	// Disable IPv6 link-local address generation
 	const modeNone = 1
 	if err = netlink.LinkSetIP6AddrGenMode(link, modeNone); err != nil {
 		t.l.WithError(err).Warn("Failed to disable link local address generation")
 	}
 
-	// Add IP addresses
+	if err = t.addIPs(link); err != nil {
-	if err = t.routeManager.addIPs(t, link); err != nil {
 		return err
 	}
 
 	// Bring up the interface
-	if err = netlink.LinkSetUp(link); err != nil {
+	ifrf.Flags = ifrf.Flags | unix.IFF_UP
+	if err = ioctl(t.ioctlFd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
 		return fmt.Errorf("failed to bring the tun device up: %s", err)
 	}
 
-	// Set route MTU
+	//set route MTU
 	for i := range t.vpnNetworks {
-		if err = t.routeManager.SetDefaultRoute(t, t.vpnNetworks[i]); err != nil {
+		if err = t.setDefaultRoute(t.vpnNetworks[i]); err != nil {
 			return fmt.Errorf("failed to set default route MTU: %w", err)
 		}
 	}
 
 	// Set the routes
-	if err = t.routeManager.AddRoutes(t, false); err != nil {
+	if err = t.addRoutes(false); err != nil {
 		return err
 	}
 
+	// Run the interface
+	ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
+	if err = ioctl(t.ioctlFd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
+		return fmt.Errorf("failed to run tun device: %s", err)
+	}
+
 	return nil
 }
 
-func (rm *tun) SetMTU(t *wgTun, mtu int) {
+func (t *tun) setMTU() {
-	name, err := t.tunDevice.Name()
+	// Set the MTU on the device
-	if err != nil {
+	ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)}
-		t.l.WithError(err).Error("Failed to get device name for MTU set")
+	if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
-		return
+		// This is currently a non fatal condition because the route table must have the MTU set appropriately as well
-	}
-
-	link, err := netlink.LinkByName(name)
-	if err != nil {
-		t.l.WithError(err).Error("Failed to get link for MTU set")
-		return
-	}
-
-	if err := netlink.LinkSetMTU(link, mtu); err != nil {
 		t.l.WithError(err).Error("Failed to set tun mtu")
 	}
 }
 
-func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error {
+func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
 	dr := &net.IPNet{
 		IP:   cidr.Masked().Addr().AsSlice(),
 		Mask: net.CIDRMask(cidr.Bits(), cidr.Addr().BitLen()),
 	}
 
 	nr := netlink.Route{
-		LinkIndex: t.routeManager.deviceIndex,
+		LinkIndex: t.deviceIndex,
 		Dst:       dr,
 		MTU:       t.DefaultMTU,
-		AdvMSS:    advMSS(Route{}, t.DefaultMTU, t.MaxMTU),
+		AdvMSS:    t.advMSS(Route{}),
 		Scope:     unix.RT_SCOPE_LINK,
 		Src:       net.IP(cidr.Addr().AsSlice()),
 		Protocol:  unix.RTPROT_KERNEL,
@@ -229,7 +425,7 @@ func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error {
 	err := netlink.RouteReplace(&nr)
 	if err != nil {
 		t.l.WithError(err).WithField("cidr", cidr).Warn("Failed to set default route MTU, retrying")
-		// Retry twice more
+		//retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument`
 		for i := 0; i < 2; i++ {
 			time.Sleep(100 * time.Millisecond)
 			err = netlink.RouteReplace(&nr)
@@ -247,7 +443,8 @@ func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error {
 	return nil
 }
 
-func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error {
+func (t *tun) addRoutes(logErrors bool) error {
+	// Path routes
 	routes := *t.Routes.Load()
 	for _, r := range routes {
 		if !r.Install {
@@ -260,10 +457,10 @@ func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error {
 		}
 
 		nr := netlink.Route{
-			LinkIndex: t.routeManager.deviceIndex,
+			LinkIndex: t.deviceIndex,
 			Dst:       dr,
 			MTU:       r.MTU,
-			AdvMSS:    advMSS(r, t.DefaultMTU, t.MaxMTU),
+			AdvMSS:    t.advMSS(r),
 			Scope:     unix.RT_SCOPE_LINK,
 		}
 
@@ -287,7 +484,7 @@ func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error {
 	return nil
 }
 
-func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) {
+func (t *tun) removeRoutes(routes []Route) {
 	for _, r := range routes {
 		if !r.Install {
 			continue
@@ -299,10 +496,10 @@ func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) {
 		}
 
 		nr := netlink.Route{
-			LinkIndex: t.routeManager.deviceIndex,
+			LinkIndex: t.deviceIndex,
 			Dst:       dr,
 			MTU:       r.MTU,
-			AdvMSS:    advMSS(r, t.DefaultMTU, t.MaxMTU),
+			AdvMSS:    t.advMSS(r),
 			Scope:     unix.RT_SCOPE_LINK,
 		}
 
@@ -319,105 +516,30 @@ func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) {
 	}
 }
 
-func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) {
+func (t *tun) Name() string {
-	// For Linux with WireGuard TUN, we can reuse the same device
+	return t.Device
-	// The vectorized I/O will handle batching
-	return &wgTunReader{
-		parent:    t,
-		tunDevice: t.tunDevice,
-		offset:    0,
-		l:         t.l,
-	}, nil
 }
 
-func deviceBytes(name string) [16]byte {
+func (t *tun) advMSS(r Route) int {
-	var o [16]byte
-	for i, c := range name {
-		if i >= 16 {
-			break
-		}
-		o[i] = byte(c)
-	}
-	return o
-}
-
-func advMSS(r Route, defaultMTU, maxMTU int) int {
 	mtu := r.MTU
 	if r.MTU == 0 {
-		mtu = defaultMTU
+		mtu = t.DefaultMTU
 	}
 
 	// We only need to set advmss if the route MTU does not match the device MTU
-	if mtu != maxMTU {
+	if mtu != t.MaxMTU {
 		return mtu - 40
 	}
 	return 0
 }
 
-type ifreqQLEN struct {
+func (t *tun) watchRoutes() {
-	Name  [16]byte
-	Value int32
-	pad   [8]byte
-}
-
-func hasNetlinkAddr(al []*netlink.Addr, x netlink.Addr) bool {
-	for i := range al {
-		if al[i].Equal(x) {
-			return true
-		}
-	}
-	return false
-}
-
-func (rm *tun) addIPs(t *wgTun, link netlink.Link) error {
-	newAddrs := make([]*netlink.Addr, len(t.vpnNetworks))
-	for i := range t.vpnNetworks {
-		newAddrs[i] = &netlink.Addr{
-			IPNet: &net.IPNet{
-				IP:   t.vpnNetworks[i].Addr().AsSlice(),
-				Mask: net.CIDRMask(t.vpnNetworks[i].Bits(), t.vpnNetworks[i].Addr().BitLen()),
-			},
-			Label: t.vpnNetworks[i].Addr().Zone(),
-		}
-	}
-
-	// Add all new addresses
-	for i := range newAddrs {
-		if err := netlink.AddrReplace(link, newAddrs[i]); err != nil {
-			return err
-		}
-	}
-
-	// Iterate over remainder, remove whoever shouldn't be there
-	al, err := netlink.AddrList(link, netlink.FAMILY_ALL)
-	if err != nil {
-		return fmt.Errorf("failed to get tun address list: %s", err)
-	}
-
-	for i := range al {
-		if hasNetlinkAddr(newAddrs, al[i]) {
-			continue
-		}
-		err = netlink.AddrDel(link, &al[i])
-		if err != nil {
-			t.l.WithError(err).Error("failed to remove address from tun address list")
-		} else {
-			t.l.WithField("removed", al[i].String()).Info("removed address not listed in cert(s)")
-		}
-	}
-
-	return nil
-}
-
-// watchRoutes monitors system route changes
-func (t *wgTun) watchRoutes() {
-
 	rch := make(chan netlink.RouteUpdate)
 	doneChan := make(chan struct{})
 
 	netlinkOptions := netlink.RouteSubscribeOptions{
-		ReceiveBufferSize:      t.routeManager.useSystemRoutesBufferSize,
+		ReceiveBufferSize:      t.useSystemRoutesBufferSize,
-		ReceiveBufferForceSize: t.routeManager.useSystemRoutesBufferSize != 0,
+		ReceiveBufferForceSize: t.useSystemRoutesBufferSize != 0,
 		ErrorCallback:          func(e error) { t.l.WithError(e).Errorf("netlink error") },
 	}
 
@@ -435,19 +557,87 @@ func (t *wgTun) watchRoutes() {
 				if ok {
 					t.updateRoutes(r)
 				} else {
+					// may be should do something here as
+					// netlink stops sending updates
 					return
 				}
 			case <-doneChan:
+				// netlink.RouteSubscriber will close the rch for us
 				return
 			}
 		}
 	}()
 }
 
-func (t *wgTun) updateRoutes(r netlink.RouteUpdate) {
+func (t *tun) isGatewayInVpnNetworks(gwAddr netip.Addr) bool {
-	gateways := t.getGatewaysFromRoute(&r.Route, t.routeManager.deviceIndex)
+	withinNetworks := false
+	for i := range t.vpnNetworks {
+		if t.vpnNetworks[i].Contains(gwAddr) {
+			withinNetworks = true
+			break
+		}
+	}
+
+	return withinNetworks
+}
+
+func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways {
+
+	var gateways routing.Gateways
+
+	link, err := netlink.LinkByName(t.Device)
+	if err != nil {
+		t.l.WithField("Devicename", t.Device).Error("Ignoring route update: failed to get link by name")
+		return gateways
+	}
+
+	// If this route is relevant to our interface and there is a gateway then add it
+	if r.LinkIndex == link.Attrs().Index && len(r.Gw) > 0 {
+		gwAddr, ok := netip.AddrFromSlice(r.Gw)
+		if !ok {
+			t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway address")
+		} else {
+			gwAddr = gwAddr.Unmap()
+
+			if !t.isGatewayInVpnNetworks(gwAddr) {
+				// Gateway isn't in our overlay network, ignore
+				t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
+			} else {
+				gateways = append(gateways, routing.NewGateway(gwAddr, 1))
+			}
+		}
+	}
+
+	for _, p := range r.MultiPath {
+		// If this route is relevant to our interface and there is a gateway then add it
+		if p.LinkIndex == link.Attrs().Index && len(p.Gw) > 0 {
+			gwAddr, ok := netip.AddrFromSlice(p.Gw)
+			if !ok {
+				t.l.WithField("route", r).Debug("Ignoring multipath route update, invalid gateway address")
+			} else {
+				gwAddr = gwAddr.Unmap()
+
+				if !t.isGatewayInVpnNetworks(gwAddr) {
+					// Gateway isn't in our overlay network, ignore
+					t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
+				} else {
+					// p.Hops+1 = weight of the route
+					gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1))
+				}
+			}
+		}
+	}
+
+	routing.CalculateBucketsForGateways(gateways)
+	return gateways
+}
+
+func (t *tun) updateRoutes(r netlink.RouteUpdate) {
+
+	gateways := t.getGatewaysFromRoute(&r.Route)
 
 	if len(gateways) == 0 {
+		// No gateways relevant to our network, no routing changes required.
 		t.l.WithField("route", r).Debug("Ignoring route update, no gateways")
 		return
 	}
@@ -471,6 +661,7 @@ func (t *wgTun) updateRoutes(r netlink.RouteUpdate) {
 	if r.Type == unix.RTM_NEWROUTE {
 		t.l.WithField("destination", dst).WithField("via", gateways).Info("Adding route")
 		newTree.Insert(dst, gateways)
+
 	} else {
 		t.l.WithField("destination", dst).WithField("via", gateways).Info("Removing route")
 		newTree.Delete(dst)
@@ -478,71 +669,18 @@ func (t *wgTun) updateRoutes(r netlink.RouteUpdate) {
 	t.routeTree.Store(newTree)
 }
 
-func (t *wgTun) getGatewaysFromRoute(r *netlink.Route, deviceIndex int) routing.Gateways {
+func (t *tun) Close() error {
-	var gateways routing.Gateways
+	if t.routeChan != nil {
-
+		close(t.routeChan)
-	name, err := t.tunDevice.Name()
-	if err != nil {
-		t.l.Error("Ignoring route update: failed to get device name")
-		return gateways
 	}
 
-	link, err := netlink.LinkByName(name)
+	if t.ReadWriteCloser != nil {
-	if err != nil {
+		_ = t.ReadWriteCloser.Close()
-		t.l.WithField("DeviceName", name).Error("Ignoring route update: failed to get link by name")
-		return gateways
 	}
 
-	// If this route is relevant to our interface and there is a gateway then add it
+	if t.ioctlFd > 0 {
-	if r.LinkIndex == link.Attrs().Index && len(r.Gw) > 0 {
+		_ = os.NewFile(t.ioctlFd, "ioctlFd").Close()
-		gwAddr, ok := netip.AddrFromSlice(r.Gw)
-		if !ok {
-			t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway address")
-		} else {
-			gwAddr = gwAddr.Unmap()
-
-			if !t.isGatewayInVpnNetworks(gwAddr) {
-				t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
-			} else {
-				gateways = append(gateways, routing.NewGateway(gwAddr, 1))
-			}
-		}
 	}
 
-	for _, p := range r.MultiPath {
-		if p.LinkIndex == link.Attrs().Index && len(p.Gw) > 0 {
-			gwAddr, ok := netip.AddrFromSlice(p.Gw)
-			if !ok {
-				t.l.WithField("route", r).Debug("Ignoring multipath route update, invalid gateway address")
-			} else {
-				gwAddr = gwAddr.Unmap()
-
-				if !t.isGatewayInVpnNetworks(gwAddr) {
-					t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
-				} else {
-					gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1))
-				}
-			}
-		}
-	}
-
-	routing.CalculateBucketsForGateways(gateways)
-	return gateways
-}
-
-func (t *wgTun) isGatewayInVpnNetworks(gwAddr netip.Addr) bool {
-	for i := range t.vpnNetworks {
-		if t.vpnNetworks[i].Contains(gwAddr) {
-			return true
-		}
-	}
-	return false
-}
-
-func ioctl(a1, a2, a3 uintptr) error {
-	_, _, errno := unix.Syscall(unix.SYS_IOCTL, a1, a2, a3)
-	if errno != 0 {
-		return errno
-	}
 	return nil
 }

overlay/tun_linux_test.go

@@ -6,27 +6,26 @@ package overlay
 import "testing"
 
 var runAdvMSSTests = []struct {
-	name       string
+	name     string
-	defaultMTU int
+	tun      *tun
-	maxMTU     int
+	r        Route
-	r          Route
+	expected int
-	expected   int
 }{
 	// Standard case, default MTU is the device max MTU
-	{"default", 1440, 1440, Route{}, 0},
+	{"default", &tun{DefaultMTU: 1440, MaxMTU: 1440}, Route{}, 0},
-	{"default-min", 1440, 1440, Route{MTU: 1440}, 0},
+	{"default-min", &tun{DefaultMTU: 1440, MaxMTU: 1440}, Route{MTU: 1440}, 0},
-	{"default-low", 1440, 1440, Route{MTU: 1200}, 1160},
+	{"default-low", &tun{DefaultMTU: 1440, MaxMTU: 1440}, Route{MTU: 1200}, 1160},
 
 	// Case where we have a route MTU set higher than the default
-	{"route", 1440, 8941, Route{}, 1400},
+	{"route", &tun{DefaultMTU: 1440, MaxMTU: 8941}, Route{}, 1400},
-	{"route-min", 1440, 8941, Route{MTU: 1440}, 1400},
+	{"route-min", &tun{DefaultMTU: 1440, MaxMTU: 8941}, Route{MTU: 1440}, 1400},
-	{"route-high", 1440, 8941, Route{MTU: 8941}, 0},
+	{"route-high", &tun{DefaultMTU: 1440, MaxMTU: 8941}, Route{MTU: 8941}, 0},
 }
 
 func TestTunAdvMSS(t *testing.T) {
 	for _, tt := range runAdvMSSTests {
 		t.Run(tt.name, func(t *testing.T) {
-			o := advMSS(tt.r, tt.defaultMTU, tt.maxMTU)
+			o := tt.tun.advMSS(tt.r)
 			if o != tt.expected {
 				t.Errorf("got %d, want %d", o, tt.expected)
 			}

overlay/tun_netbsd.go

@@ -4,12 +4,13 @@
 package overlay
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"net/netip"
 	"os"
+	"os/exec"
 	"regexp"
+	"strconv"
 	"sync/atomic"
 	"syscall"
 	"unsafe"
@@ -19,42 +20,11 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	netroute "golang.org/x/net/route"
-	"golang.org/x/sys/unix"
 )
 
-const (
+type ifreqDestroy struct {
-	SIOCAIFADDR_IN6 = 0x8080696b
+	Name [16]byte
-	TUNSIFHEAD      = 0x80047442
+	pad  [16]byte
-	TUNSIFMODE      = 0x80047458
-)
-
-type ifreqAlias4 struct {
-	Name     [unix.IFNAMSIZ]byte
-	Addr     unix.RawSockaddrInet4
-	DstAddr  unix.RawSockaddrInet4
-	MaskAddr unix.RawSockaddrInet4
-}
-
-type ifreqAlias6 struct {
-	Name       [unix.IFNAMSIZ]byte
-	Addr       unix.RawSockaddrInet6
-	DstAddr    unix.RawSockaddrInet6
-	PrefixMask unix.RawSockaddrInet6
-	Flags      uint32
-	Lifetime   addrLifetime
-}
-
-type ifreq struct {
-	Name [unix.IFNAMSIZ]byte
-	data int
-}
-
-type addrLifetime struct {
-	Expire    uint64
-	Preferred uint64
-	Vltime    uint32
-	Pltime    uint32
 }
 
 type tun struct {
@@ -64,18 +34,40 @@ type tun struct {
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	l           *logrus.Logger
-	f           *os.File
+
-	fd          int
+	io.ReadWriteCloser
 }
 
-var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+func (t *tun) Close() error {
+	if t.ReadWriteCloser != nil {
+		if err := t.ReadWriteCloser.Close(); err != nil {
+			return err
+		}
+
+		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+		if err != nil {
+			return err
+		}
+		defer syscall.Close(s)
+
+		ifreq := ifreqDestroy{Name: t.deviceBytes()}
+
+		err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
+
+		return err
+	}
+	return nil
+}
 
 func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in NetBSD")
 }
 
+var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+
 func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	// Try to open tun device
+	var file *os.File
 	var err error
 	deviceName := c.GetString("tun.dev", "")
 	if deviceName == "" {
@@ -85,23 +77,17 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
 	}
 
-	fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
+	file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
 	}
 
-	err = unix.SetNonblock(fd, true)
-	if err != nil {
-		l.WithError(err).Warn("Failed to set the tun device as nonblocking")
-	}
-
 	t := &tun{
-		f:           os.NewFile(uintptr(fd), ""),
+		ReadWriteCloser: file,
-		fd:          fd,
+		Device:          deviceName,
-		Device:      deviceName,
+		vpnNetworks:     vpnNetworks,
-		vpnNetworks: vpnNetworks,
+		MTU:             c.GetInt("tun.mtu", DefaultMTU),
-		MTU:         c.GetInt("tun.mtu", DefaultMTU),
+		l:               l,
-		l:           l,
 	}
 
 	err = t.reload(c, true)
@@ -119,225 +105,40 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	return t, nil
 }
 
-func (t *tun) Close() error {
-	if t.f != nil {
-		if err := t.f.Close(); err != nil {
-			return fmt.Errorf("error closing tun file: %w", err)
-		}
-
-		// t.f.Close should have handled it for us but let's be extra sure
-		_ = unix.Close(t.fd)
-
-		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-
-		ifr := ifreq{Name: t.deviceBytes()}
-		err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifr)))
-		return err
-	}
-	return nil
-}
-
-func (t *tun) Read(to []byte) (int, error) {
-	rc, err := t.f.SyscallConn()
-	if err != nil {
-		return 0, fmt.Errorf("failed to get syscall conn for tun: %w", err)
-	}
-
-	var errno syscall.Errno
-	var n uintptr
-	err = rc.Read(func(fd uintptr) bool {
-		// first 4 bytes is protocol family, in network byte order
-		head := [4]byte{}
-		iovecs := []syscall.Iovec{
-			{&head[0], 4},
-			{&to[0], uint64(len(to))},
-		}
-
-		n, _, errno = syscall.Syscall(syscall.SYS_READV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-		if errno.Temporary() {
-			// We got an EAGAIN, EINTR, or EWOULDBLOCK, go again
-			return false
-		}
-		return true
-	})
-	if err != nil {
-		if err == syscall.EBADF || err.Error() == "use of closed file" {
-			// Go doesn't export poll.ErrFileClosing but happily reports it to us so here we are
-			// https://github.com/golang/go/blob/master/src/internal/poll/fd_poll_runtime.go#L121
-			return 0, os.ErrClosed
-		}
-		return 0, fmt.Errorf("failed to make read call for tun: %w", err)
-	}
-
-	if errno != 0 {
-		return 0, fmt.Errorf("failed to make inner read call for tun: %w", errno)
-	}
-
-	// fix bytes read number to exclude header
-	bytesRead := int(n)
-	if bytesRead < 0 {
-		return bytesRead, nil
-	} else if bytesRead < 4 {
-		return 0, nil
-	} else {
-		return bytesRead - 4, nil
-	}
-}
-
-// Write is only valid for single threaded use
-func (t *tun) Write(from []byte) (int, error) {
-	if len(from) <= 1 {
-		return 0, syscall.EIO
-	}
-
-	ipVer := from[0] >> 4
-	var head [4]byte
-	// first 4 bytes is protocol family, in network byte order
-	if ipVer == 4 {
-		head[3] = syscall.AF_INET
-	} else if ipVer == 6 {
-		head[3] = syscall.AF_INET6
-	} else {
-		return 0, fmt.Errorf("unable to determine IP version from packet")
-	}
-
-	rc, err := t.f.SyscallConn()
-	if err != nil {
-		return 0, err
-	}
-
-	var errno syscall.Errno
-	var n uintptr
-	err = rc.Write(func(fd uintptr) bool {
-		iovecs := []syscall.Iovec{
-			{&head[0], 4},
-			{&from[0], uint64(len(from))},
-		}
-
-		n, _, errno = syscall.Syscall(syscall.SYS_WRITEV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-		// According to NetBSD documentation for TUN, writes will only return errors in which
-		// this packet will never be delivered so just go on living life.
-		return true
-	})
-	if err != nil {
-		return 0, err
-	}
-
-	if errno != 0 {
-		return 0, errno
-	}
-
-	return int(n) - 4, err
-}
-
 func (t *tun) addIp(cidr netip.Prefix) error {
-	if cidr.Addr().Is4() {
+	var err error
-		var req ifreqAlias4
-		req.Name = t.deviceBytes()
-		req.Addr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   cidr.Addr().As4(),
-		}
-		req.DstAddr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   cidr.Addr().As4(),
-		}
-		req.MaskAddr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   prefixToMask(cidr).As4(),
-		}
 
-		s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+	// TODO use syscalls instead of exec.Command
-		if err != nil {
+	cmd := exec.Command("/sbin/ifconfig", t.Device, cidr.String(), cidr.Addr().String())
-			return err
+	t.l.Debug("command: ", cmd.String())
-		}
+	if err = cmd.Run(); err != nil {
-		defer syscall.Close(s)
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-
-		if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&req))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr(), err)
-		}
-
-		return nil
 	}
 
-	if cidr.Addr().Is6() {
+	cmd = exec.Command("/sbin/route", "-n", "add", "-net", cidr.String(), cidr.Addr().String())
-		var req ifreqAlias6
+	t.l.Debug("command: ", cmd.String())
-		req.Name = t.deviceBytes()
+	if err = cmd.Run(); err != nil {
-		req.Addr = unix.RawSockaddrInet6{
+		return fmt.Errorf("failed to run 'route add': %s", err)
-			Len:    unix.SizeofSockaddrInet6,
-			Family: unix.AF_INET6,
-			Addr:   cidr.Addr().As16(),
-		}
-		req.PrefixMask = unix.RawSockaddrInet6{
-			Len:    unix.SizeofSockaddrInet6,
-			Family: unix.AF_INET6,
-			Addr:   prefixToMask(cidr).As16(),
-		}
-		req.Lifetime = addrLifetime{
-			Vltime: 0xffffffff,
-			Pltime: 0xffffffff,
-		}
-
-		s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-
-		if err := ioctl(uintptr(s), SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&req))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err)
-		}
-		return nil
 	}
 
-	return fmt.Errorf("unknown address type %v", cidr)
+	cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU))
-}
+	t.l.Debug("command: ", cmd.String())
-
+	if err = cmd.Run(); err != nil {
-func (t *tun) Activate() error {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-	mode := int32(unix.IFF_BROADCAST)
-	err := ioctl(uintptr(t.fd), TUNSIFMODE, uintptr(unsafe.Pointer(&mode)))
-	if err != nil {
-		return fmt.Errorf("failed to set tun device mode: %w", err)
-	}
-
-	v := 1
-	err = ioctl(uintptr(t.fd), TUNSIFHEAD, uintptr(unsafe.Pointer(&v)))
-	if err != nil {
-		return fmt.Errorf("failed to set tun device head: %w", err)
-	}
-
-	err = t.doIoctlByName(unix.SIOCSIFMTU, uint32(t.MTU))
-	if err != nil {
-		return fmt.Errorf("failed to set tun mtu: %w", err)
-	}
-
-	for i := range t.vpnNetworks {
-		err = t.addIp(t.vpnNetworks[i])
-		if err != nil {
-			return err
-		}
 	}
 
+	// Unsafe path routes
 	return t.addRoutes(false)
 }
 
-func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
+func (t *tun) Activate() error {
-	s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+	for i := range t.vpnNetworks {
-	if err != nil {
+		err := t.addIp(t.vpnNetworks[i])
-		return err
+		if err != nil {
+			return err
+		}
 	}
-	defer syscall.Close(s)
+	return nil
-
-	ir := ifreq{Name: t.deviceBytes(), data: int(value)}
-	err = ioctl(uintptr(s), ctl, uintptr(unsafe.Pointer(&ir)))
-	return err
 }
 
 func (t *tun) reload(c *config.C, initial bool) error {
@@ -396,23 +197,21 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 
 func (t *tun) addRoutes(logErrors bool) error {
 	routes := *t.Routes.Load()
-
 	for _, r := range routes {
 		if len(r.Via) == 0 || !r.Install {
 			// We don't allow route MTUs so only install routes with a via
 			continue
 		}
 
-		err := addRoute(r.Cidr, t.vpnNetworks)
+		cmd := exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
-		if err != nil {
+		t.l.Debug("command: ", cmd.String())
-			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
+		if err := cmd.Run(); err != nil {
+			retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
 			} else {
 				return retErr
 			}
-		} else {
-			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -425,8 +224,10 @@ func (t *tun) removeRoutes(routes []Route) error {
 			continue
 		}
 
-		err := delRoute(r.Cidr, t.vpnNetworks)
+		//TODO: CERT-V2 is this right?
-		if err != nil {
+		cmd := exec.Command("/sbin/route", "-n", "delete", "-net", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
+		t.l.Debug("command: ", cmd.String())
+		if err := cmd.Run(); err != nil {
 			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
 			t.l.WithField("route", r).Info("Removed route")
@@ -441,147 +242,3 @@ func (t *tun) deviceBytes() (o [16]byte) {
 	}
 	return
 }
-
-func addRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
-	if err != nil {
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
-
-	route := &netroute.RouteMessage{
-		Version: unix.RTM_VERSION,
-		Type:    unix.RTM_ADD,
-		Flags:   unix.RTF_UP | unix.RTF_GATEWAY,
-		Seq:     1,
-	}
-
-	if prefix.Addr().Is4() {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()},
-		}
-	} else {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()},
-		}
-	}
-
-	data, err := route.Marshal()
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
-
-	_, err = unix.Write(sock, data[:])
-	if err != nil {
-		if errors.Is(err, unix.EEXIST) {
-			// Try to do a change
-			route.Type = unix.RTM_CHANGE
-			data, err = route.Marshal()
-			if err != nil {
-				return fmt.Errorf("failed to create route.RouteMessage for change: %w", err)
-			}
-			_, err = unix.Write(sock, data[:])
-			return err
-		}
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
-}
-
-func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
-	if err != nil {
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
-
-	route := netroute.RouteMessage{
-		Version: unix.RTM_VERSION,
-		Type:    unix.RTM_DELETE,
-		Seq:     1,
-	}
-
-	if prefix.Addr().Is4() {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()},
-		}
-	} else {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()},
-		}
-	}
-
-	data, err := route.Marshal()
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
-	_, err = unix.Write(sock, data[:])
-	if err != nil {
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
-}
-
-func ioctl(a1, a2, a3 uintptr) error {
-	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, a1, a2, a3)
-	if errno != 0 {
-		return errno
-	}
-	return nil
-}
-
-func prefixToMask(prefix netip.Prefix) netip.Addr {
-	bits := prefix.Bits()
-	if prefix.Addr().Is4() {
-		mask := ^uint32(0) << (32 - bits)
-		return netip.AddrFrom4([4]byte{
-			byte(mask >> 24),
-			byte(mask >> 16),
-			byte(mask >> 8),
-			byte(mask),
-		})
-	}
-	var mask [16]byte
-	for i := 0; i < bits/8; i++ {
-		mask[i] = 0xff
-	}
-	if bits%8 != 0 {
-		mask[bits/8] = ^byte(0) << (8 - bits%8)
-	}
-	return netip.AddrFrom16(mask)
-}
-
-func selectGateway(prefix netip.Prefix, gateways []netip.Prefix) (netip.Prefix, error) {
-	for _, gw := range gateways {
-		if prefix.Addr().Is4() == gw.Addr().Is4() {
-			return gw, nil
-		}
-	}
-	return netip.Prefix{}, fmt.Errorf("no suitable gateway found for prefix %v", prefix)
-}

overlay/tun_notwin.go

@@ -0,0 +1,14 @@
+//go:build !windows
+// +build !windows
+
+package overlay
+
+import "syscall"
+
+func ioctl(a1, a2, a3 uintptr) error {
+	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, a1, a2, a3)
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}

overlay/tun_openbsd.go

@@ -1,5 +1,5 @@
-//go:build openbsd && !e2e_testing
+//go:build !e2e_testing
-// +build openbsd,!e2e_testing
+// +build !e2e_testing
 
 package overlay
 
@@ -7,53 +7,73 @@ import (
 	"fmt"
 	"io"
 	"net/netip"
+	"os"
 	"os/exec"
+	"regexp"
 	"strconv"
-	"strings"
+	"sync/atomic"
+	"syscall"
 
+	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	wgtun "golang.zx2c4.com/wireguard/tun"
 )
 
-type tun struct{}
+type tun struct {
+	Device      string
+	vpnNetworks []netip.Prefix
+	MTU         int
+	Routes      atomic.Pointer[[]Route]
+	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
+	l           *logrus.Logger
 
-func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*wgTun, error) {
+	io.ReadWriteCloser
-	return nil, fmt.Errorf("newTunFromFd not supported on OpenBSD")
+
+	// cache out buffer since we need to prepend 4 bytes for tun metadata
+	out []byte
 }
 
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*wgTun, error) {
+func (t *tun) Close() error {
-	deviceName := c.GetString("tun.dev", "tun")
+	if t.ReadWriteCloser != nil {
-	mtu := c.GetInt("tun.mtu", DefaultMTU)
+		return t.ReadWriteCloser.Close()
+	}
 
-	// Create WireGuard TUN device
+	return nil
-	tunDevice, err := wgtun.CreateTUN(deviceName, mtu)
+}
+
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
+	return nil, fmt.Errorf("newTunFromFd not supported in OpenBSD")
+}
+
+var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
+	deviceName := c.GetString("tun.dev", "")
+	if deviceName == "" {
+		return nil, fmt.Errorf("a device name in the format of tunN must be specified")
+	}
+
+	if !deviceNameRE.MatchString(deviceName) {
+		return nil, fmt.Errorf("a device name in the format of tunN must be specified")
+	}
+
+	file, err := os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create TUN device: %w", err)
+		return nil, err
 	}
 
-	// Get the actual device name
+	t := &tun{
-	actualName, err := tunDevice.Name()
+		ReadWriteCloser: file,
-	if err != nil {
+		Device:          deviceName,
-		tunDevice.Close()
+		vpnNetworks:     vpnNetworks,
-		return nil, fmt.Errorf("failed to get TUN device name: %w", err)
+		MTU:             c.GetInt("tun.mtu", DefaultMTU),
+		l:               l,
 	}
 
-	t := &wgTun{
-		tunDevice:   tunDevice,
-		vpnNetworks: vpnNetworks,
-		MaxMTU:      mtu,
-		DefaultMTU:  mtu,
-		l:           l,
-	}
-
-	// Create OpenBSD-specific route manager
-	t.routeManager = &tun{}
-
 	err = t.reload(c, true)
 	if err != nil {
-		tunDevice.Close()
 		return nil, err
 	}
 
@@ -64,166 +84,172 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		}
 	})
 
-	l.WithField("name", actualName).Info("Created WireGuard TUN device")
-
 	return t, nil
 }
 
-func (rm *tun) Activate(t *wgTun) error {
+func (t *tun) reload(c *config.C, initial bool) error {
-	name, err := t.tunDevice.Name()
+	change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
 	if err != nil {
-		return fmt.Errorf("failed to get device name: %w", err)
-	}
-
-	// Set the MTU
-	rm.SetMTU(t, t.MaxMTU)
-
-	// Add IP addresses
-	for _, network := range t.vpnNetworks {
-		if err := rm.addIP(t, name, network); err != nil {
-			return err
-		}
-	}
-
-	// Bring up the interface
-	if err := runCommandBSD("ifconfig", name, "up"); err != nil {
-		return fmt.Errorf("failed to bring up interface: %w", err)
-	}
-
-	// Set the routes
-	if err := rm.AddRoutes(t, false); err != nil {
 		return err
 	}
 
+	if !initial && !change {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		if err != nil {
+			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
+		}
+
+		// Ensure any routes we actually want are installed
+		err = t.addRoutes(true)
+		if err != nil {
+			// Catch any stray logs
+			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
+		}
+	}
+
 	return nil
 }
 
-func (rm *tun) SetMTU(t *wgTun, mtu int) {
+func (t *tun) addIp(cidr netip.Prefix) error {
-	name, err := t.tunDevice.Name()
+	var err error
-	if err != nil {
+	// TODO use syscalls instead of exec.Command
-		t.l.WithError(err).Error("Failed to get device name for MTU set")
+	cmd := exec.Command("/sbin/ifconfig", t.Device, cidr.String(), cidr.Addr().String())
-		return
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
 	}
 
-	if err := runCommandBSD("ifconfig", name, "mtu", strconv.Itoa(mtu)); err != nil {
+	cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU))
-		t.l.WithError(err).Error("Failed to set tun mtu")
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
 	}
+
+	cmd = exec.Command("/sbin/route", "-n", "add", "-inet", cidr.String(), cidr.Addr().String())
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run 'route add': %s", err)
+	}
+
+	// Unsafe path routes
+	return t.addRoutes(false)
 }
 
-func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error {
+func (t *tun) Activate() error {
-	// On OpenBSD, routes are set via ifconfig and route commands
+	for i := range t.vpnNetworks {
+		err := t.addIp(t.vpnNetworks[i])
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
-func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error {
+func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
-	name, err := t.tunDevice.Name()
+	r, _ := t.routeTree.Load().Lookup(ip)
-	if err != nil {
+	return r
-		return fmt.Errorf("failed to get device name: %w", err)
+}
-	}
 
+func (t *tun) addRoutes(logErrors bool) error {
 	routes := *t.Routes.Load()
 	for _, r := range routes {
-		if !r.Install {
+		if len(r.Via) == 0 || !r.Install {
+			// We don't allow route MTUs so only install routes with a via
 			continue
 		}
-
+		//TODO: CERT-V2 is this right?
-		// Add route using route command
+		cmd := exec.Command("/sbin/route", "-n", "add", "-inet", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
-		args := []string{"add"}
+		t.l.Debug("command: ", cmd.String())
-
+		if err := cmd.Run(); err != nil {
-		if r.Cidr.Addr().Is6() {
+			retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]any{"route": r}, err)
-			args = append(args, "-inet6")
-		} else {
-			args = append(args, "-inet")
-		}
-
-		args = append(args, r.Cidr.String(), "-interface", name)
-
-		if r.Metric > 0 {
-			// OpenBSD doesn't support route metrics directly like Linux
-			t.l.WithField("route", r).Warn("Route metrics are not fully supported on OpenBSD")
-		}
-
-		err := runCommandBSD("route", args...)
-		if err != nil {
-			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
 			} else {
 				return retErr
 			}
-		} else {
-			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
 	return nil
 }
 
-func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) {
+func (t *tun) removeRoutes(routes []Route) error {
-	name, err := t.tunDevice.Name()
-	if err != nil {
-		t.l.WithError(err).Error("Failed to get device name for route removal")
-		return
-	}
-
 	for _, r := range routes {
 		if !r.Install {
 			continue
 		}
-
+		//TODO: CERT-V2 is this right?
-		args := []string{"delete"}
+		cmd := exec.Command("/sbin/route", "-n", "delete", "-inet", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
-
+		t.l.Debug("command: ", cmd.String())
-		if r.Cidr.Addr().Is6() {
+		if err := cmd.Run(); err != nil {
-			args = append(args, "-inet6")
-		} else {
-			args = append(args, "-inet")
-		}
-
-		args = append(args, r.Cidr.String(), "-interface", name)
-
-		err := runCommandBSD("route", args...)
-		if err != nil {
 			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
 			t.l.WithField("route", r).Info("Removed route")
 		}
 	}
+	return nil
 }
 
-func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) {
+func (t *tun) Networks() []netip.Prefix {
-	// OpenBSD doesn't support multi-queue TUN devices in the same way as Linux
+	return t.vpnNetworks
-	// Return a reader that wraps the same device
-	return &wgTunReader{
-		parent:    t,
-		tunDevice: t.tunDevice,
-		offset:    0,
-		l:         t.l,
-	}, nil
 }
 
-func (rm *tun) addIP(t *wgTun, name string, network netip.Prefix) error {
+func (t *tun) Name() string {
-	addr := network.Addr()
+	return t.Device
+}
 
-	if addr.Is4() {
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-		// For IPv4: ifconfig tun0 10.0.0.1/24
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
-		if err := runCommandBSD("ifconfig", name, network.String()); err != nil {
+}
-			return fmt.Errorf("failed to add IPv4 address: %w", err)
+
-		}
+func (t *tun) Read(to []byte) (int, error) {
+	buf := make([]byte, len(to)+4)
+
+	n, err := t.ReadWriteCloser.Read(buf)
+
+	copy(to, buf[4:])
+	return n - 4, err
+}
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	buf := t.out
+	if cap(buf) < len(from)+4 {
+		buf = make([]byte, len(from)+4)
+		t.out = buf
+	}
+	buf = buf[:len(from)+4]
+
+	if len(from) == 0 {
+		return 0, syscall.EIO
+	}
+
+	// Determine the IP Family for the NULL L2 Header
+	ipVer := from[0] >> 4
+	if ipVer == 4 {
+		buf[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		buf[3] = syscall.AF_INET6
 	} else {
-		// For IPv6: ifconfig tun0 inet6 add 2001:db8::1/64
+		return 0, fmt.Errorf("unable to determine IP version from packet")
-		if err := runCommandBSD("ifconfig", name, "inet6", "add", network.String()); err != nil {
-			return fmt.Errorf("failed to add IPv6 address: %w", err)
-		}
 	}
 
-	return nil
+	copy(buf[4:], from)
-}
 
-func runCommandBSD(name string, args ...string) error {
+	n, err := t.ReadWriteCloser.Write(buf)
-	cmd := exec.Command(name, args...)
+	return n - 4, err
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("%s %s failed: %w\nOutput: %s", name, strings.Join(args, " "), err, string(output))
-	}
-	return nil
 }

overlay/tun_wg.go

@@ -1,242 +0,0 @@
-//go:build !android && !netbsd && !e2e_testing
-// +build !android,!netbsd,!e2e_testing
-
-package overlay
-
-import (
-	"fmt"
-	"io"
-	"net/netip"
-	"sync/atomic"
-
-	"github.com/gaissmai/bart"
-	"github.com/sirupsen/logrus"
-	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/routing"
-	"github.com/slackhq/nebula/util"
-	wgtun "golang.zx2c4.com/wireguard/tun"
-)
-
-// wgTun wraps a WireGuard TUN device and implements the overlay.Device interface
-type wgTun struct {
-	tunDevice   wgtun.Device
-	vpnNetworks []netip.Prefix
-	MaxMTU      int
-	DefaultMTU  int
-
-	Routes    atomic.Pointer[[]Route]
-	routeTree atomic.Pointer[bart.Table[routing.Gateways]]
-	routeChan chan struct{}
-
-	// Platform-specific route management
-	routeManager *tun
-
-	l *logrus.Logger
-}
-
-// BatchReader interface for readers that support vectorized I/O
-type BatchReader interface {
-	BatchRead(buffers [][]byte, sizes []int) (int, error)
-}
-
-// BatchWriter interface for writers that support vectorized I/O
-type BatchWriter interface {
-	BatchWrite(packets [][]byte) (int, error)
-}
-
-// wgTunReader wraps a single TUN queue for multi-queue support
-type wgTunReader struct {
-	parent    *wgTun
-	tunDevice wgtun.Device
-	offset    int
-	l         *logrus.Logger
-}
-
-func (t *wgTun) Networks() []netip.Prefix {
-	return t.vpnNetworks
-}
-
-func (t *wgTun) Name() string {
-	name, err := t.tunDevice.Name()
-	if err != nil {
-		t.l.WithError(err).Error("Failed to get TUN device name")
-		return "unknown"
-	}
-	return name
-}
-
-func (t *wgTun) RoutesFor(ip netip.Addr) routing.Gateways {
-	r, _ := t.routeTree.Load().Lookup(ip)
-	return r
-}
-
-func (t *wgTun) Activate() error {
-	if t.routeManager == nil {
-		return fmt.Errorf("route manager not initialized")
-	}
-	return t.routeManager.Activate(t)
-}
-
-// Read implements single-packet read for backward compatibility
-func (t *wgTun) Read(b []byte) (int, error) {
-	bufs := [][]byte{b}
-	sizes := []int{0}
-	n, err := t.tunDevice.Read(bufs, sizes, 0)
-	if err != nil {
-		return 0, err
-	}
-	if n == 0 {
-		return 0, io.ErrNoProgress
-	}
-	return sizes[0], nil
-}
-
-// Write implements single-packet write for backward compatibility
-func (t *wgTun) Write(b []byte) (int, error) {
-	bufs := [][]byte{b}
-	offset := 0
-
-	// WireGuard TUN expects the packet data to start at offset 0
-	n, err := t.tunDevice.Write(bufs, offset)
-	if err != nil {
-		return 0, err
-	}
-	if n == 0 {
-		return 0, io.ErrShortWrite
-	}
-	return len(b), nil
-}
-
-func (t *wgTun) Close() error {
-	if t.routeChan != nil {
-		close(t.routeChan)
-	}
-
-	if t.tunDevice != nil {
-		return t.tunDevice.Close()
-	}
-
-	return nil
-}
-
-func (t *wgTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	// For WireGuard TUN, we need to create separate TUN device instances for multi-queue
-	// The platform-specific implementation will handle this
-	if t.routeManager == nil {
-		return nil, fmt.Errorf("route manager not initialized for multi-queue reader")
-	}
-
-	return t.routeManager.NewMultiQueueReader(t)
-}
-
-func (t *wgTun) reload(c *config.C, initial bool) error {
-	routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
-	if err != nil {
-		return err
-	}
-
-	if !initial && !routeChange && !c.HasChanged("tun.mtu") {
-		return nil
-	}
-
-	routeTree, err := makeRouteTree(t.l, routes, true)
-	if err != nil {
-		return err
-	}
-
-	oldDefaultMTU := t.DefaultMTU
-	oldMaxMTU := t.MaxMTU
-	newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU)
-	newMaxMTU := newDefaultMTU
-	for i, r := range routes {
-		if r.MTU == 0 {
-			routes[i].MTU = newDefaultMTU
-		}
-
-		if r.MTU > t.MaxMTU {
-			newMaxMTU = r.MTU
-		}
-	}
-
-	t.MaxMTU = newMaxMTU
-	t.DefaultMTU = newDefaultMTU
-
-	// Teach nebula how to handle the routes before establishing them in the system table
-	oldRoutes := t.Routes.Swap(&routes)
-	t.routeTree.Store(routeTree)
-
-	if !initial && t.routeManager != nil {
-		if oldMaxMTU != newMaxMTU {
-			t.routeManager.SetMTU(t, t.MaxMTU)
-			t.l.Infof("Set max MTU to %v was %v", t.MaxMTU, oldMaxMTU)
-		}
-
-		if oldDefaultMTU != newDefaultMTU {
-			for i := range t.vpnNetworks {
-				err := t.routeManager.SetDefaultRoute(t, t.vpnNetworks[i])
-				if err != nil {
-					t.l.Warn(err)
-				} else {
-					t.l.Infof("Set default MTU to %v was %v", t.DefaultMTU, oldDefaultMTU)
-				}
-			}
-		}
-
-		// Remove first, if the system removes a wanted route hopefully it will be re-added next
-		t.routeManager.RemoveRoutes(t, findRemovedRoutes(routes, *oldRoutes))
-
-		// Ensure any routes we actually want are installed
-		err = t.routeManager.AddRoutes(t, true)
-		if err != nil {
-			// This should never be called since AddRoutes should log its own errors in a reload condition
-			util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l)
-		}
-	}
-
-	return nil
-}
-
-// BatchRead reads multiple packets from the TUN device using vectorized I/O
-// The caller provides buffers and sizes slices, and this function returns the number of packets read.
-func (r *wgTunReader) BatchRead(buffers [][]byte, sizes []int) (int, error) {
-	return r.tunDevice.Read(buffers, sizes, r.offset)
-}
-
-// Read implements io.Reader for wgTunReader (single packet for compatibility)
-func (r *wgTunReader) Read(b []byte) (int, error) {
-	bufs := [][]byte{b}
-	sizes := []int{0}
-	n, err := r.tunDevice.Read(bufs, sizes, r.offset)
-	if err != nil {
-		return 0, err
-	}
-	if n == 0 {
-		return 0, io.ErrNoProgress
-	}
-	return sizes[0], nil
-}
-
-// Write implements io.Writer for wgTunReader
-func (r *wgTunReader) Write(b []byte) (int, error) {
-	bufs := [][]byte{b}
-	n, err := r.tunDevice.Write(bufs, r.offset)
-	if err != nil {
-		return 0, err
-	}
-	if n == 0 {
-		return 0, io.ErrShortWrite
-	}
-	return len(b), nil
-}
-
-// BatchWrite writes multiple packets to the TUN device using vectorized I/O
-func (r *wgTunReader) BatchWrite(packets [][]byte) (int, error) {
-	return r.tunDevice.Write(packets, r.offset)
-}
-
-func (r *wgTunReader) Close() error {
-	if r.tunDevice != nil {
-		return r.tunDevice.Close()
-	}
-	return nil
-}

overlay/tun_windows.go

@@ -1,77 +1,84 @@
-//go:build windows && !e2e_testing
+//go:build !e2e_testing
-// +build windows,!e2e_testing
+// +build !e2e_testing
 
 package overlay
 
 import (
 	"crypto"
-	"encoding/binary"
 	"fmt"
 	"io"
 	"net/netip"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync/atomic"
+	"syscall"
+	"unsafe"
 
+	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
+	"github.com/slackhq/nebula/wintun"
 	"golang.org/x/sys/windows"
-	wgtun "golang.zx2c4.com/wireguard/tun"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 )
 
 const tunGUIDLabel = "Fixed Nebula Windows GUID v1"
 
-type tun struct {
+type winTun struct {
-	luid winipcfg.LUID
+	Device      string
+	vpnNetworks []netip.Prefix
+	MTU         int
+	Routes      atomic.Pointer[[]Route]
+	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
+	l           *logrus.Logger
+
+	tun *wintun.NativeTun
 }
 
-func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*wgTun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (Device, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in Windows")
 }
 
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*wgTun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*winTun, error) {
-	deviceName := c.GetString("tun.dev", "Nebula")
+	err := checkWinTunExists()
-	mtu := c.GetInt("tun.mtu", DefaultMTU)
-
-	// Create WireGuard TUN device
-	tunDevice, err := wgtun.CreateTUN(deviceName, mtu)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create TUN device: %w", err)
+		return nil, fmt.Errorf("can not load the wintun driver: %w", err)
 	}
 
-	// Get the actual device name
+	deviceName := c.GetString("tun.dev", "")
-	actualName, err := tunDevice.Name()
+	guid, err := generateGUIDByDeviceName(deviceName)
 	if err != nil {
-		tunDevice.Close()
+		return nil, fmt.Errorf("generate GUID failed: %w", err)
-		return nil, fmt.Errorf("failed to get TUN device name: %w", err)
 	}
 
-	t := &wgTun{
+	t := &winTun{
-		tunDevice:   tunDevice,
+		Device:      deviceName,
 		vpnNetworks: vpnNetworks,
-		MaxMTU:      mtu,
+		MTU:         c.GetInt("tun.mtu", DefaultMTU),
-		DefaultMTU:  mtu,
 		l:           l,
 	}
 
-	// Create Windows-specific route manager
-	rm := &tun{}
-
-	// Get LUID from the TUN device
-	// The WireGuard TUN device on Windows should provide a LUID() method
-	if nativeTun, ok := tunDevice.(interface{ LUID() uint64 }); ok {
-		rm.luid = winipcfg.LUID(nativeTun.LUID())
-	} else {
-		tunDevice.Close()
-		return nil, fmt.Errorf("failed to get LUID from TUN device")
-	}
-	t.routeManager = rm
-
 	err = t.reload(c, true)
 	if err != nil {
-		tunDevice.Close()
 		return nil, err
 	}
 
+	var tunDevice wintun.Device
+	tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU)
+	if err != nil {
+		// Windows 10 has an issue with unclean shutdowns not fully cleaning up the wintun device.
+		// Trying a second time resolves the issue.
+		l.WithError(err).Debug("Failed to create wintun device, retrying")
+		tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU)
+		if err != nil {
+			return nil, fmt.Errorf("create TUN device failed: %w", err)
+		}
+	}
+	t.tun = tunDevice.(*wintun.NativeTun)
+
 	c.RegisterReloadCallback(func(c *config.C) {
 		err := t.reload(c, false)
 		if err != nil {
@@ -79,140 +86,206 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		}
 	})
 
-	l.WithField("name", actualName).Info("Created WireGuard TUN device")
-
 	return t, nil
 }
 
-func (rm *tun) Activate(t *wgTun) error {
+func (t *winTun) reload(c *config.C, initial bool) error {
-	// Set MTU
+	change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
-	err := rm.setMTU(t, t.MaxMTU)
 	if err != nil {
-		return fmt.Errorf("failed to set MTU: %w", err)
+		return err
 	}
 
-	// Add IP addresses
+	if !initial && !change {
-	for _, network := range t.vpnNetworks {
+		return nil
-		if err := rm.addIP(t, network); err != nil {
+	}
-			return err
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		if err != nil {
+			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
+		}
+
+		// Ensure any routes we actually want are installed
+		err = t.addRoutes(true)
+		if err != nil {
+			// Catch any stray logs
+			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
 		}
 	}
 
-	// Add routes
+	return nil
-	if err := rm.AddRoutes(t, false); err != nil {
+}
+
+func (t *winTun) Activate() error {
+	luid := winipcfg.LUID(t.tun.LUID())
+
+	err := luid.SetIPAddresses(t.vpnNetworks)
+	if err != nil {
+		return fmt.Errorf("failed to set address: %w", err)
+	}
+
+	err = t.addRoutes(false)
+	if err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (rm *tun) SetMTU(t *wgTun, mtu int) {
+func (t *winTun) addRoutes(logErrors bool) error {
-	if err := rm.setMTU(t, mtu); err != nil {
+	luid := winipcfg.LUID(t.tun.LUID())
-		t.l.WithError(err).Error("Failed to set MTU")
-	}
-}
-
-func (rm *tun) setMTU(t *wgTun, mtu int) error {
-	// Set MTU using winipcfg
-	// Note: MTU setting on Windows TUN devices may be handled by the driver
-	// For now, we'll skip explicit MTU setting as the WireGuard TUN handles it
-	return nil
-}
-
-func (rm *tun) SetDefaultRoute(t *wgTun, cidr netip.Prefix) error {
-	// On Windows, routes are managed differently
-	return nil
-}
-
-func (rm *tun) AddRoutes(t *wgTun, logErrors bool) error {
 	routes := *t.Routes.Load()
+	foundDefault4 := false
+
 	for _, r := range routes {
-		if !r.Install {
+		if len(r.Via) == 0 || !r.Install {
+			// We don't allow route MTUs so only install routes with a via
 			continue
 		}
 
-		if r.MTU > 0 {
+		// Add our unsafe route
-			// Windows route MTU is not directly supported
+		// Windows does not support multipath routes natively, so we install only a single route.
-			t.l.WithField("route", r).Debug("Route MTU is not supported on Windows")
+		// This is not a problem as traffic will always be sent to Nebula which handles the multipath routing internally.
-		}
+		// In effect this provides multipath routing support to windows supporting loadbalancing and redundancy.
-
+		err := luid.AddRoute(r.Cidr, r.Via[0].Addr(), uint32(r.Metric))
-		// Use winipcfg to add the route
-		// The rm.luid should have the AddRoute method from winipcfg
-		if len(r.Via) == 0 {
-			t.l.WithField("route", r).Warn("Route has no via address, skipping")
-			continue
-		}
-
-		err := rm.luid.AddRoute(r.Cidr, r.Via[0].Addr(), uint32(r.Metric))
 		if err != nil {
 			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
+				continue
 			} else {
 				return retErr
 			}
 		} else {
 			t.l.WithField("route", r).Info("Added route")
 		}
+
+		if !foundDefault4 {
+			if r.Cidr.Bits() == 0 && r.Cidr.Addr().BitLen() == 32 {
+				foundDefault4 = true
+			}
+		}
 	}
 
+	ipif, err := luid.IPInterface(windows.AF_INET)
+	if err != nil {
+		return fmt.Errorf("failed to get ip interface: %w", err)
+	}
+
+	ipif.NLMTU = uint32(t.MTU)
+	if foundDefault4 {
+		ipif.UseAutomaticMetric = false
+		ipif.Metric = 0
+	}
+
+	if err := ipif.Set(); err != nil {
+		return fmt.Errorf("failed to set ip interface: %w", err)
+	}
 	return nil
 }
 
-func (rm *tun) RemoveRoutes(t *wgTun, routes []Route) {
+func (t *winTun) removeRoutes(routes []Route) error {
+	luid := winipcfg.LUID(t.tun.LUID())
+
 	for _, r := range routes {
 		if !r.Install {
 			continue
 		}
 
-		if len(r.Via) == 0 {
+		// See comment on luid.AddRoute
-			continue
+		err := luid.DeleteRoute(r.Cidr, r.Via[0].Addr())
-		}
-
-		err := rm.luid.DeleteRoute(r.Cidr, r.Via[0].Addr())
 		if err != nil {
 			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
 			t.l.WithField("route", r).Info("Removed route")
 		}
 	}
-}
-
-func (rm *tun) NewMultiQueueReader(t *wgTun) (io.ReadWriteCloser, error) {
-	// Windows doesn't support multi-queue TUN devices
-	// Return a reader that wraps the same device
-	return &wgTunReader{
-		parent:    t,
-		tunDevice: t.tunDevice,
-		offset:    0,
-		l:         t.l,
-	}, nil
-}
-
-func (rm *tun) addIP(t *wgTun, network netip.Prefix) error {
-	// Add IP address using winipcfg
-	// SetIPAddresses expects a slice of prefixes
-	err := rm.luid.SetIPAddresses([]netip.Prefix{network})
-	if err != nil {
-		return fmt.Errorf("failed to add IP address %s: %w", network, err)
-	}
 	return nil
 }
 
-// generateGUIDByDeviceName generates a GUID based on the device name
+func (t *winTun) RoutesFor(ip netip.Addr) routing.Gateways {
-func generateGUIDByDeviceName(deviceName string) (*windows.GUID, error) {
+	r, _ := t.routeTree.Load().Lookup(ip)
-	// Hash the device name to create a deterministic GUID
+	return r
-	h := crypto.SHA256.New()
+}
-	h.Write([]byte(tunGUIDLabel))
+
-	h.Write([]byte(deviceName))
+func (t *winTun) Networks() []netip.Prefix {
-	sum := h.Sum(nil)
+	return t.vpnNetworks
-
+}
-	guid := &windows.GUID{
+
-		Data1: binary.LittleEndian.Uint32(sum[0:4]),
+func (t *winTun) Name() string {
-		Data2: binary.LittleEndian.Uint16(sum[4:6]),
+	return t.Device
-		Data3: binary.LittleEndian.Uint16(sum[6:8]),
+}
-	}
+
-	copy(guid.Data4[:], sum[8:16])
+func (t *winTun) Read(b []byte) (int, error) {
-
+	return t.tun.Read(b, 0)
-	return guid, nil
+}
+
+func (t *winTun) Write(b []byte) (int, error) {
+	return t.tun.Write(b, 0)
+}
+
+func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
+}
+
+func (t *winTun) Close() error {
+	// It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes,
+	// so to be certain, just remove everything before destroying.
+	luid := winipcfg.LUID(t.tun.LUID())
+	_ = luid.FlushRoutes(windows.AF_INET)
+	_ = luid.FlushIPAddresses(windows.AF_INET)
+
+	_ = luid.FlushRoutes(windows.AF_INET6)
+	_ = luid.FlushIPAddresses(windows.AF_INET6)
+
+	_ = luid.FlushDNS(windows.AF_INET)
+	_ = luid.FlushDNS(windows.AF_INET6)
+
+	return t.tun.Close()
+}
+
+func generateGUIDByDeviceName(name string) (*windows.GUID, error) {
+	// GUID is 128 bit
+	hash := crypto.MD5.New()
+
+	_, err := hash.Write([]byte(tunGUIDLabel))
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = hash.Write([]byte(name))
+	if err != nil {
+		return nil, err
+	}
+
+	sum := hash.Sum(nil)
+
+	return (*windows.GUID)(unsafe.Pointer(&sum[0])), nil
+}
+
+func checkWinTunExists() error {
+	myPath, err := os.Executable()
+	if err != nil {
+		return err
+	}
+
+	arch := runtime.GOARCH
+	switch arch {
+	case "386":
+		//NOTE: wintun bundles 386 as x86
+		arch = "x86"
+	}
+
+	_, err = syscall.LoadDLL(filepath.Join(filepath.Dir(myPath), "dist", "windows", "wintun", "bin", arch, "wintun.dll"))
+	return err
 }

packet/packet.go

@@ -0,0 +1,45 @@
+package packet
+
+import (
+	"net/netip"
+	"sync"
+
+	"golang.org/x/sys/unix"
+)
+
+const Size = 0xffff
+
+type Packet struct {
+	Payload []byte
+	Control []byte
+	SegSize int
+	Addr    netip.AddrPort
+}
+
+func New() *Packet {
+	return &Packet{
+		Payload: make([]byte, Size),
+		Control: make([]byte, unix.CmsgSpace(2)),
+	}
+}
+
+type Pool struct {
+	pool sync.Pool
+}
+
+var bigPool = &Pool{
+	pool: sync.Pool{New: func() any { return New() }},
+}
+
+func GetPool() *Pool {
+	return bigPool
+}
+
+func (p *Pool) Get() *Packet {
+	return p.pool.Get().(*Packet)
+}
+
+func (p *Pool) Put(x *Packet) {
+	x.Payload = x.Payload[:Size]
+	p.pool.Put(x)
+}

service/service.go

@@ -9,10 +9,13 @@ import (
 	"math"
 	"net"
 	"net/netip"
+	"os"
 	"strings"
 	"sync"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/overlay"
 	"golang.org/x/sync/errgroup"
 	"gvisor.dev/gvisor/pkg/buffer"
@@ -43,8 +46,19 @@ type Service struct {
 	}
 }
 
-func New(control *nebula.Control) (*Service, error) {
+func New(config *config.C) (*Service, error) {
-	control.Start()
+	logger := logrus.New()
+	logger.Out = os.Stdout
+
+	control, err := nebula.Main(config, false, "custom-app", logger, overlay.NewUserDeviceFromConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	wait, err := control.Start()
+	if err != nil {
+		return nil, err
+	}
 
 	ctx := control.Context()
 	eg, ctx := errgroup.WithContext(ctx)
@@ -141,6 +155,12 @@ func New(control *nebula.Control) (*Service, error) {
 		}
 	})
 
+	// Add the nebula wait function to the group
+	eg.Go(func() error {
+		wait()
+		return nil
+	})
+
 	return &s, nil
 }
 

udp/conn.go

@@ -4,19 +4,19 @@ import (
 	"net/netip"
 
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/packet"
 )
 
 const MTU = 9001
 
-type EncReader func(
+type EncReader func(*packet.Packet)
-	addr netip.AddrPort,
+
-	payload []byte,
+type PacketBufferGetter func() *packet.Packet
-)
 
 type Conn interface {
 	Rebind() error
 	LocalAddr() (netip.AddrPort, error)
-	ListenOut(r EncReader)
+	ListenOut(pg PacketBufferGetter, pc chan *packet.Packet) error
 	WriteTo(b []byte, addr netip.AddrPort) error
 	ReloadConfig(c *config.C)
 	Close() error

udp/udp_generic.go

@@ -71,15 +71,14 @@ type rawMessage struct {
 	Len uint32
 }
 
-func (u *GenericConn) ListenOut(r EncReader) {
+func (u *GenericConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	for {
 		// Just read one packet at a time
 		n, rua, err := u.ReadFromUDPAddrPort(buffer)
 		if err != nil {
-			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+			return err
-			return
 		}
 
 		r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n])

udp/udp_linux.go

@@ -9,27 +9,26 @@ import (
 	"net"
 	"net/netip"
 	"syscall"
+	"time"
 	"unsafe"
 
 	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/packet"
 	"golang.org/x/sys/unix"
 )
 
-type StdConn struct {
+var readTimeout = unix.NsecToTimeval(int64(time.Millisecond * 500))
-	sysFd int
-	isV4  bool
-	l     *logrus.Logger
-	batch int
-}
 
-func maybeIPV4(ip net.IP) (net.IP, bool) {
+type StdConn struct {
-	ip4 := ip.To4()
+	sysFd     int
-	if ip4 != nil {
+	isV4      bool
-		return ip4, true
+	l         *logrus.Logger
-	}
+	batch     int
-	return ip, false
+	enableGRO bool
+	enableGSO bool
+	//gso       gsoState
 }
 
 func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
@@ -55,6 +54,11 @@ func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch in
 		}
 	}
 
+	// Set a read timeout
+	if err = unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &readTimeout); err != nil {
+		return nil, fmt.Errorf("unable to set SO_RCVTIMEO: %s", err)
+	}
+
 	var sa unix.Sockaddr
 	if ip.Is4() {
 		sa4 := &unix.SockaddrInet4{Port: port}
@@ -118,10 +122,10 @@ func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
 	}
 }
 
-func (u *StdConn) ListenOut(r EncReader) {
+func (u *StdConn) ListenOut(pg PacketBufferGetter, pc chan *packet.Packet) error {
 	var ip netip.Addr
 
-	msgs, buffers, names := u.PrepareRawMessages(u.batch)
+	msgs, packets, names := u.PrepareRawMessages(u.batch, pg)
 	read := u.ReadMulti
 	if u.batch == 1 {
 		read = u.ReadSingle
@@ -130,22 +134,61 @@ func (u *StdConn) ListenOut(r EncReader) {
 	for {
 		n, err := read(msgs)
 		if err != nil {
-			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+			return err
-			return
 		}
 
 		for i := 0; i < n; i++ {
+			out := packets[i]
+			out.Payload = out.Payload[:msgs[i].Len]
+
 			// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
 			if u.isV4 {
 				ip, _ = netip.AddrFromSlice(names[i][4:8])
 			} else {
 				ip, _ = netip.AddrFromSlice(names[i][8:24])
 			}
-			r(netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4])), buffers[i][:msgs[i].Len])
+			out.Addr = netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4]))
+			ctrlLen := getRawMessageControlLen(&msgs[i])
+			if ctrlLen > 0 {
+				packets[i].SegSize = parseGROControl(packets[i].Control[:ctrlLen])
+			} else {
+				packets[i].SegSize = 0
+			}
+
+			pc <- out
+
+			//rotate this packet out so we don't overwrite it
+			packets[i] = pg()
+			msgs[i].Hdr.Iov.Base = &packets[i].Payload[0]
+			if u.enableGRO {
+				msgs[i].Hdr.Control = &packets[i].Control[0]
+				msgs[i].Hdr.Controllen = uint64(cap(packets[i].Control))
+			}
+
 		}
 	}
 }
 
+func parseGROControl(control []byte) int {
+	if len(control) == 0 {
+		return 0
+	}
+
+	cmsgs, err := unix.ParseSocketControlMessage(control)
+	if err != nil {
+		return 0
+	}
+
+	for _, c := range cmsgs {
+		if c.Header.Level == unix.SOL_UDP && c.Header.Type == unix.UDP_GRO && len(c.Data) >= 2 {
+			segSize := int(binary.LittleEndian.Uint16(c.Data[:2]))
+			return segSize
+		}
+	}
+
+	return 0
+}
+
 func (u *StdConn) ReadSingle(msgs []rawMessage) (int, error) {
 	for {
 		n, _, err := unix.Syscall6(
@@ -159,6 +202,9 @@ func (u *StdConn) ReadSingle(msgs []rawMessage) (int, error) {
 		)
 
 		if err != 0 {
+			if err == unix.EAGAIN || err == unix.EINTR {
+				continue
+			}
 			return 0, &net.OpError{Op: "recvmsg", Err: err}
 		}
 
@@ -180,6 +226,9 @@ func (u *StdConn) ReadMulti(msgs []rawMessage) (int, error) {
 		)
 
 		if err != 0 {
+			if err == unix.EAGAIN || err == unix.EINTR {
+				continue
+			}
 			return 0, &net.OpError{Op: "recvmmsg", Err: err}
 		}
 
@@ -221,7 +270,7 @@ func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
 
 func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
 	if !ip.Addr().Is4() {
-		return ErrInvalidIPv6RemoteForSocket
+		return fmt.Errorf("Listener is IPv4, but writing to IPv6 remote")
 	}
 
 	var rsa unix.RawSockaddrInet4
@@ -294,6 +343,28 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 			u.l.WithError(err).Error("Failed to set listen.so_mark")
 		}
 	}
+	u.configureGRO(true)
+}
+
+func (u *StdConn) configureGRO(enable bool) {
+	if enable == u.enableGRO {
+		return
+	}
+
+	if enable {
+		if err := unix.SetsockoptInt(u.sysFd, unix.SOL_UDP, unix.UDP_GRO, 1); err != nil {
+			u.l.WithError(err).Warn("Failed to enable UDP GRO")
+			return
+		}
+		u.enableGRO = true
+		u.l.Info("UDP GRO enabled")
+		return
+	}
+
+	if err := unix.SetsockoptInt(u.sysFd, unix.SOL_UDP, unix.UDP_GRO, 0); err != nil && err != unix.ENOPROTOOPT {
+		u.l.WithError(err).Warn("Failed to disable UDP GRO")
+	}
+	u.enableGRO = false
 }
 
 func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {

udp/udp_linux_64.go

@@ -7,6 +7,7 @@
 package udp
 
 import (
+	"github.com/slackhq/nebula/packet"
 	"golang.org/x/sys/unix"
 )
 
@@ -33,17 +34,39 @@ type rawMessage struct {
 	Pad0 [4]byte
 }
 
-func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
+func setRawMessageControl(msg *rawMessage, buf []byte) {
+	if len(buf) == 0 {
+		msg.Hdr.Control = nil
+		msg.Hdr.Controllen = 0
+		return
+	}
+	msg.Hdr.Control = &buf[0]
+	msg.Hdr.Controllen = uint64(len(buf))
+}
+
+func getRawMessageControlLen(msg *rawMessage) int {
+	return int(msg.Hdr.Controllen)
+}
+
+func setCmsgLen(h *unix.Cmsghdr, l int) {
+	h.Len = uint64(l)
+}
+
+func (u *StdConn) PrepareRawMessages(n int, pg PacketBufferGetter) ([]rawMessage, []*packet.Packet, [][]byte) {
 	msgs := make([]rawMessage, n)
-	buffers := make([][]byte, n)
 	names := make([][]byte, n)
 
+	packets := make([]*packet.Packet, n)
+	for i := range packets {
+		packets[i] = pg()
+	}
+	//todo?
+
 	for i := range msgs {
-		buffers[i] = make([]byte, MTU)
 		names[i] = make([]byte, unix.SizeofSockaddrInet6)
 
 		vs := []iovec{
-			{Base: &buffers[i][0], Len: uint64(len(buffers[i]))},
+			{Base: &packets[i].Payload[0], Len: uint64(packet.Size)},
 		}
 
 		msgs[i].Hdr.Iov = &vs[0]
@@ -51,7 +74,14 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
 
 		msgs[i].Hdr.Name = &names[i][0]
 		msgs[i].Hdr.Namelen = uint32(len(names[i]))
+		if u.enableGRO {
+			msgs[i].Hdr.Control = &packets[i].Control[0]
+			msgs[i].Hdr.Controllen = uint64(len(packets[i].Control))
+		} else {
+			msgs[i].Hdr.Control = nil
+			msgs[i].Hdr.Controllen = 0
+		}
 	}
 
-	return msgs, buffers, names
+	return msgs, packets, names
 }

udp/udp_rio_windows.go

@@ -134,7 +134,7 @@ func (u *RIOConn) bind(sa windows.Sockaddr) error {
 	return nil
 }
 
-func (u *RIOConn) ListenOut(r EncReader) {
+func (u *RIOConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	for {