tweak defaults and turn on gsogro on linux by default

This reverts commit fa098c551a.

bits.go

@@ -5,6 +5,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// TODO: Pretty sure this is just all sorts of racy now, we need it to be atomic
 type Bits struct {
 	length             uint64
 	current            uint64
@@ -43,7 +44,7 @@ func (b *Bits) Check(l logrus.FieldLogger, i uint64) bool {
 	}
 
 	// Not within the window
-	l.Debugf("rejected a packet (top) %d %d\n", b.current, i)
+	l.Error("rejected a packet (top) %d %d\n", b.current, i)
 	return false
 }
 

cert/pem.go

@@ -1,8 +1,10 @@
 package cert
 
 import (
+	"encoding/hex"
 	"encoding/pem"
 	"fmt"
+	"time"
 
 	"golang.org/x/crypto/ed25519"
 )
@@ -138,6 +140,101 @@ func MarshalSigningPrivateKeyToPEM(curve Curve, b []byte) []byte {
 	}
 }
 
+// Backward compatibility functions for older API
+func MarshalX25519PublicKey(b []byte) []byte {
+	return MarshalPublicKeyToPEM(Curve_CURVE25519, b)
+}
+
+func MarshalX25519PrivateKey(b []byte) []byte {
+	return MarshalPrivateKeyToPEM(Curve_CURVE25519, b)
+}
+
+func MarshalPublicKey(curve Curve, b []byte) []byte {
+	return MarshalPublicKeyToPEM(curve, b)
+}
+
+func MarshalPrivateKey(curve Curve, b []byte) []byte {
+	return MarshalPrivateKeyToPEM(curve, b)
+}
+
+// NebulaCertificate is a compatibility wrapper for the old API
+type NebulaCertificate struct {
+	Details   NebulaCertificateDetails
+	Signature []byte
+	cert      Certificate
+}
+
+// NebulaCertificateDetails is a compatibility wrapper for certificate details
+type NebulaCertificateDetails struct {
+	Name      string
+	NotBefore time.Time
+	NotAfter  time.Time
+	PublicKey []byte
+	IsCA      bool
+	Issuer    []byte
+	Curve     Curve
+}
+
+// UnmarshalNebulaCertificateFromPEM provides backward compatibility with the old API
+func UnmarshalNebulaCertificateFromPEM(b []byte) (*NebulaCertificate, []byte, error) {
+	c, rest, err := UnmarshalCertificateFromPEM(b)
+	if err != nil {
+		return nil, rest, err
+	}
+
+	issuerBytes, err := func() ([]byte, error) {
+		issuer := c.Issuer()
+		if issuer == "" {
+			return nil, nil
+		}
+		decoded, err := hex.DecodeString(issuer)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode issuer fingerprint: %w", err)
+		}
+		return decoded, nil
+	}()
+	if err != nil {
+		return nil, rest, err
+	}
+
+	pubKey := c.PublicKey()
+	if pubKey != nil {
+		pubKey = append([]byte(nil), pubKey...)
+	}
+
+	sig := c.Signature()
+	if sig != nil {
+		sig = append([]byte(nil), sig...)
+	}
+
+	return &NebulaCertificate{
+		Details: NebulaCertificateDetails{
+			Name:      c.Name(),
+			NotBefore: c.NotBefore(),
+			NotAfter:  c.NotAfter(),
+			PublicKey: pubKey,
+			IsCA:      c.IsCA(),
+			Issuer:    issuerBytes,
+			Curve:     c.Curve(),
+		},
+		Signature: sig,
+		cert:      c,
+	}, rest, nil
+}
+
+// IssuerString returns the issuer in hex format for compatibility
+func (n *NebulaCertificate) IssuerString() string {
+	if n.Details.Issuer == nil {
+		return ""
+	}
+	return hex.EncodeToString(n.Details.Issuer)
+}
+
+// Certificate returns the underlying certificate (read-only)
+func (n *NebulaCertificate) Certificate() Certificate {
+	return n.cert
+}
+
 // UnmarshalPrivateKeyFromPEM will try to unmarshal the first pem block in a byte array, returning any non
 // consumed data or an error on failure
 func UnmarshalPrivateKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {

cert_test/cert.go

@@ -114,33 +114,6 @@ func NewTestCert(v cert.Version, curve cert.Curve, ca cert.Certificate, key []by
 	return c, pub, cert.MarshalPrivateKeyToPEM(curve, priv), pem
 }
 
-func NewTestCertDifferentVersion(c cert.Certificate, v cert.Version, ca cert.Certificate, key []byte) (cert.Certificate, []byte) {
-	nc := &cert.TBSCertificate{
-		Version:        v,
-		Curve:          c.Curve(),
-		Name:           c.Name(),
-		Networks:       c.Networks(),
-		UnsafeNetworks: c.UnsafeNetworks(),
-		Groups:         c.Groups(),
-		NotBefore:      time.Unix(c.NotBefore().Unix(), 0),
-		NotAfter:       time.Unix(c.NotAfter().Unix(), 0),
-		PublicKey:      c.PublicKey(),
-		IsCA:           false,
-	}
-
-	c, err := nc.Sign(ca, ca.Curve(), key)
-	if err != nil {
-		panic(err)
-	}
-
-	pem, err := c.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-
-	return c, pem
-}
-
 func X25519Keypair() ([]byte, []byte) {
 	privkey := make([]byte, 32)
 	if _, err := io.ReadFull(rand.Reader, privkey); err != nil {

cmd/nebula-service/main.go

@@ -65,8 +65,16 @@ func main() {
 	}
 
 	if !*configTest {
-		ctrl.Start()
+		wait, err := ctrl.Start()
-		ctrl.ShutdownBlock()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
+		wait()
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

cmd/nebula/main.go

@@ -3,6 +3,9 @@ package main
 import (
 	"flag"
 	"fmt"
+	"log"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 
 	"github.com/sirupsen/logrus"
@@ -58,10 +61,22 @@ func main() {
 		os.Exit(1)
 	}
 
+	go func() {
+		log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
+	}()
+
 	if !*configTest {
-		ctrl.Start()
+		wait, err := ctrl.Start()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
 		notifyReady(l)
-		ctrl.ShutdownBlock()
+		wait()
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

connection_manager.go

@@ -354,6 +354,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 
 		if mainHostInfo {
 			decision = tryRehandshake
+
 		} else {
 			if cm.shouldSwapPrimary(hostinfo) {
 				decision = swapPrimary
@@ -460,10 +461,6 @@ func (cm *connectionManager) shouldSwapPrimary(current *HostInfo) bool {
 	}
 
 	crt := cm.intf.pki.getCertState().getCertificate(current.ConnectionState.myCert.Version())
-	if crt == nil {
-		//my cert was reloaded away. We should definitely swap from this tunnel
-		return true
-	}
 	// If this tunnel is using the latest certificate then we should swap it to primary for a bit and see if things
 	// settle down.
 	return bytes.Equal(current.ConnectionState.myCert.Signature(), crt.Signature())
@@ -478,34 +475,31 @@ func (cm *connectionManager) swapPrimary(current, primary *HostInfo) {
 	cm.hostMap.Unlock()
 }
 
-// isInvalidCertificate decides if we should destroy a tunnel.
+// isInvalidCertificate will check if we should destroy a tunnel if pki.disconnect_invalid is true and
-// returns true if pki.disconnect_invalid is true and the certificate is no longer valid.
+// the certificate is no longer valid. Block listed certificates will skip the pki.disconnect_invalid
-// Blocklisted certificates will skip the pki.disconnect_invalid check and return true.
+// check and return true.
 func (cm *connectionManager) isInvalidCertificate(now time.Time, hostinfo *HostInfo) bool {
 	remoteCert := hostinfo.GetCert()
 	if remoteCert == nil {
-		return false //don't tear down tunnels for handshakes in progress
+		return false
 	}
 
 	caPool := cm.intf.pki.GetCAPool()
 	err := caPool.VerifyCachedCertificate(now, remoteCert)
 	if err == nil {
-		return false //cert is still valid! yay!
-	} else if err == cert.ErrBlockListed { //avoiding errors.Is for speed
-		// Block listed certificates should always be disconnected
-		hostinfo.logger(cm.l).WithError(err).
-			WithField("fingerprint", remoteCert.Fingerprint).
-			Info("Remote certificate is blocked, tearing down the tunnel")
-		return true
-	} else if cm.intf.disconnectInvalid.Load() {
-		hostinfo.logger(cm.l).WithError(err).
-			WithField("fingerprint", remoteCert.Fingerprint).
-			Info("Remote certificate is no longer valid, tearing down the tunnel")
-		return true
-	} else {
-		//if we reach here, the cert is no longer valid, but we're configured to keep tunnels from now-invalid certs open
 		return false
 	}
+
+	if !cm.intf.disconnectInvalid.Load() && err != cert.ErrBlockListed {
+		// Block listed certificates should always be disconnected
+		return false
+	}
+
+	hostinfo.logger(cm.l).WithError(err).
+		WithField("fingerprint", remoteCert.Fingerprint).
+		Info("Remote certificate is no longer valid, tearing down the tunnel")
+
+	return true
 }
 
 func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
@@ -536,45 +530,15 @@ func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
 func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
 	cs := cm.intf.pki.getCertState()
 	curCrt := hostinfo.ConnectionState.myCert
-	curCrtVersion := curCrt.Version()
+	myCrt := cs.getCertificate(curCrt.Version())
-	myCrt := cs.getCertificate(curCrtVersion)
+	if curCrt.Version() >= cs.initiatingVersion && bytes.Equal(curCrt.Signature(), myCrt.Signature()) == true {
-	if myCrt == nil {
+		// The current tunnel is using the latest certificate and version, no need to rehandshake.
-		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-			WithField("version", curCrtVersion).
-			WithField("reason", "local certificate removed").
-			Info("Re-handshaking with remote")
-		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
 		return
 	}
-	peerCrt := hostinfo.ConnectionState.peerCert
-	if peerCrt != nil && curCrtVersion < peerCrt.Certificate.Version() {
-		// if our certificate version is less than theirs, and we have a matching version available, rehandshake?
-		if cs.getCertificate(peerCrt.Certificate.Version()) != nil {
-			cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-				WithField("version", curCrtVersion).
-				WithField("peerVersion", peerCrt.Certificate.Version()).
-				WithField("reason", "local certificate version lower than peer, attempting to correct").
-				Info("Re-handshaking with remote")
-			cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], func(hh *HandshakeHostInfo) {
-				hh.initiatingVersionOverride = peerCrt.Certificate.Version()
-			})
-			return
-		}
-	}
-	if !bytes.Equal(curCrt.Signature(), myCrt.Signature()) {
-		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-			WithField("reason", "local certificate is not current").
-			Info("Re-handshaking with remote")
 
-		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
+	cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-		return
+		WithField("reason", "local certificate is not current").
-	}
+		Info("Re-handshaking with remote")
-	if curCrtVersion < cs.initiatingVersion {
-		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-			WithField("reason", "current cert version < pki.initiatingVersion").
-			Info("Re-handshaking with remote")
 
-		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
+	cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
-		return
-	}
 }

connection_state.go

@@ -13,7 +13,9 @@ import (
 	"github.com/slackhq/nebula/noiseutil"
 )
 
-const ReplayWindow = 1024
+// TODO: In a 5Gbps test, 1024 is not sufficient. With a 1400 MTU this is about 1.4Gbps of window, assuming full packets.
+// 4092 should be sufficient for 5Gbps
+const ReplayWindow = 4096
 
 type ConnectionState struct {
 	eKey           *NebulaCipherState

control.go

@@ -2,9 +2,11 @@ package nebula
 
 import (
 	"context"
+	"errors"
 	"net/netip"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
@@ -13,6 +15,16 @@ import (
 	"github.com/slackhq/nebula/overlay"
 )
 
+type RunState int
+
+const (
+	Stopped  RunState = 0 // The control has yet to be started
+	Started  RunState = 1 // The control has been started
+	Stopping RunState = 2 // The control is stopping
+)
+
+var ErrAlreadyStarted = errors.New("nebula is already started")
+
 // Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching
 // core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc
 
@@ -26,6 +38,9 @@ type controlHostLister interface {
 }
 
 type Control struct {
+	stateLock sync.Mutex
+	state     RunState
+
 	f                      *Interface
 	l                      *logrus.Logger
 	ctx                    context.Context
@@ -49,10 +64,21 @@ type ControlHostInfo struct {
 	CurrentRelaysThroughMe []netip.Addr     `json:"currentRelaysThroughMe"`
 }
 
-// Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock()
+// Start actually runs nebula, this is a nonblocking call.
-func (c *Control) Start() {
+// The returned function can be used to wait for nebula to fully stop.
+func (c *Control) Start() (func(), error) {
+	c.stateLock.Lock()
+	if c.state != Stopped {
+		c.stateLock.Unlock()
+		return nil, ErrAlreadyStarted
+	}
+
 	// Activate the interface
-	c.f.activate()
+	err := c.f.activate()
+	if err != nil {
+		c.stateLock.Unlock()
+		return nil, err
+	}
 
 	// Call all the delayed funcs that waited patiently for the interface to be created.
 	if c.sshStart != nil {
@@ -72,15 +98,33 @@ func (c *Control) Start() {
 	}
 
 	// Start reading packets.
-	c.f.run()
+	c.state = Started
+	c.stateLock.Unlock()
+	return c.f.run(c.ctx)
+}
+
+func (c *Control) State() RunState {
+	c.stateLock.Lock()
+	defer c.stateLock.Unlock()
+	return c.state
 }
 
 func (c *Control) Context() context.Context {
 	return c.ctx
 }
 
-// Stop signals nebula to shutdown and close all tunnels, returns after the shutdown is complete
+// Stop is a non-blocking call that signals nebula to close all tunnels and shut down
 func (c *Control) Stop() {
+	c.stateLock.Lock()
+	if c.state != Started {
+		c.stateLock.Unlock()
+		// We are stopping or stopped already
+		return
+	}
+
+	c.state = Stopping
+	c.stateLock.Unlock()
+
 	// Stop the handshakeManager (and other services), to prevent new tunnels from
 	// being created while we're shutting them all down.
 	c.cancel()
@@ -89,7 +133,7 @@ func (c *Control) Stop() {
 	if err := c.f.Close(); err != nil {
 		c.l.WithError(err).Error("Close interface failed")
 	}
-	c.l.Info("Goodbye")
+	c.state = Stopped
 }
 
 // ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled

e2e/helpers_test.go

@@ -129,109 +129,6 @@ func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name
 	return control, vpnNetworks, udpAddr, c
 }
 
-// newServer creates a nebula instance with fewer assumptions
-func newServer(caCrt []cert.Certificate, certs []cert.Certificate, key []byte, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
-	l := NewTestLogger()
-
-	vpnNetworks := certs[len(certs)-1].Networks()
-
-	var udpAddr netip.AddrPort
-	if vpnNetworks[0].Addr().Is4() {
-		budpIp := vpnNetworks[0].Addr().As4()
-		budpIp[1] -= 128
-		udpAddr = netip.AddrPortFrom(netip.AddrFrom4(budpIp), 4242)
-	} else {
-		budpIp := vpnNetworks[0].Addr().As16()
-		// beef for funsies
-		budpIp[2] = 190
-		budpIp[3] = 239
-		udpAddr = netip.AddrPortFrom(netip.AddrFrom16(budpIp), 4242)
-	}
-
-	caStr := ""
-	for _, ca := range caCrt {
-		x, err := ca.MarshalPEM()
-		if err != nil {
-			panic(err)
-		}
-		caStr += string(x)
-	}
-	certStr := ""
-	for _, c := range certs {
-		x, err := c.MarshalPEM()
-		if err != nil {
-			panic(err)
-		}
-		certStr += string(x)
-	}
-
-	mc := m{
-		"pki": m{
-			"ca":   caStr,
-			"cert": certStr,
-			"key":  string(key),
-		},
-		//"tun": m{"disabled": true},
-		"firewall": m{
-			"outbound": []m{{
-				"proto": "any",
-				"port":  "any",
-				"host":  "any",
-			}},
-			"inbound": []m{{
-				"proto": "any",
-				"port":  "any",
-				"host":  "any",
-			}},
-		},
-		//"handshakes": m{
-		//	"try_interval": "1s",
-		//},
-		"listen": m{
-			"host": udpAddr.Addr().String(),
-			"port": udpAddr.Port(),
-		},
-		"logging": m{
-			"timestamp_format": fmt.Sprintf("%v 15:04:05.000000", certs[0].Name()),
-			"level":            l.Level.String(),
-		},
-		"timers": m{
-			"pending_deletion_interval": 2,
-			"connection_alive_interval": 2,
-		},
-	}
-
-	if overrides != nil {
-		final := m{}
-		err := mergo.Merge(&final, overrides, mergo.WithAppendSlice)
-		if err != nil {
-			panic(err)
-		}
-		err = mergo.Merge(&final, mc, mergo.WithAppendSlice)
-		if err != nil {
-			panic(err)
-		}
-		mc = final
-	}
-
-	cb, err := yaml.Marshal(mc)
-	if err != nil {
-		panic(err)
-	}
-
-	c := config.NewC(l)
-	cStr := string(cb)
-	c.LoadString(cStr)
-
-	control, err := nebula.Main(c, false, "e2e-test", l, nil)
-
-	if err != nil {
-		panic(err)
-	}
-
-	return control, vpnNetworks, udpAddr, c
-}
-
 type doneCb func()
 
 func deadline(t *testing.T, seconds time.Duration) doneCb {

e2e/tunnels_test.go

@@ -4,16 +4,12 @@
 package e2e
 
 import (
-	"fmt"
-	"net/netip"
 	"testing"
 	"time"
 
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/e2e/router"
-	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v3"
 )
 
 func TestDropInactiveTunnels(t *testing.T) {
@@ -59,262 +55,3 @@ func TestDropInactiveTunnels(t *testing.T) {
 	myControl.Stop()
 	theirControl.Stop()
 }
-
-func TestCertUpgrade(t *testing.T) {
-	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
-	// under ideal conditions
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	caB, err := ca.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	ca2B, err := ca2.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	caStr := fmt.Sprintf("%s\n%s", caB, ca2B)
-
-	myCert, _, myPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
-	_, myCert2Pem := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
-
-	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
-	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
-
-	myControl, myVpnIpNet, myUdpAddr, myC := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert}, myPrivKey, m{})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	r.Log("Assert the tunnel between me and them works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	r.Log("yay")
-	//todo ???
-	time.Sleep(1 * time.Second)
-	r.FlushAll()
-
-	mc := m{
-		"pki": m{
-			"ca":   caStr,
-			"cert": string(myCert2Pem),
-			"key":  string(myPrivKey),
-		},
-		//"tun": m{"disabled": true},
-		"firewall": myC.Settings["firewall"],
-		//"handshakes": m{
-		//	"try_interval": "1s",
-		//},
-		"listen":  myC.Settings["listen"],
-		"logging": myC.Settings["logging"],
-		"timers":  myC.Settings["timers"],
-	}
-
-	cb, err := yaml.Marshal(mc)
-	if err != nil {
-		panic(err)
-	}
-
-	r.Logf("reload new v2-only config")
-	err = myC.ReloadConfigString(string(cb))
-	assert.NoError(t, err)
-	r.Log("yay, spin until their sees it")
-	waitStart := time.Now()
-	for {
-		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-		if c == nil {
-			r.Log("nil")
-		} else {
-			version := c.Cert.Version()
-			r.Logf("version %d", version)
-			if version == cert.Version2 {
-				break
-			}
-		}
-		since := time.Since(waitStart)
-		if since > time.Second*10 {
-			t.Fatal("Cert should be new by now")
-		}
-		time.Sleep(time.Second)
-	}
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestCertDowngrade(t *testing.T) {
-	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
-	// under ideal conditions
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	caB, err := ca.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	ca2B, err := ca2.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	caStr := fmt.Sprintf("%s\n%s", caB, ca2B)
-
-	myCert, _, myPrivKey, myCertPem := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
-	myCert2, _ := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
-
-	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
-	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
-
-	myControl, myVpnIpNet, myUdpAddr, myC := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert2}, myPrivKey, m{})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	r.Log("Assert the tunnel between me and them works")
-	//assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
-	//r.Log("yay")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	r.Log("yay")
-	//todo ???
-	time.Sleep(1 * time.Second)
-	r.FlushAll()
-
-	mc := m{
-		"pki": m{
-			"ca":   caStr,
-			"cert": string(myCertPem),
-			"key":  string(myPrivKey),
-		},
-		"firewall": myC.Settings["firewall"],
-		"listen":   myC.Settings["listen"],
-		"logging":  myC.Settings["logging"],
-		"timers":   myC.Settings["timers"],
-	}
-
-	cb, err := yaml.Marshal(mc)
-	if err != nil {
-		panic(err)
-	}
-
-	r.Logf("reload new v1-only config")
-	err = myC.ReloadConfigString(string(cb))
-	assert.NoError(t, err)
-	r.Log("yay, spin until their sees it")
-	waitStart := time.Now()
-	for {
-		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-		c2 := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-		if c == nil || c2 == nil {
-			r.Log("nil")
-		} else {
-			version := c.Cert.Version()
-			theirVersion := c2.Cert.Version()
-			r.Logf("version %d,%d", version, theirVersion)
-			if version == cert.Version1 {
-				break
-			}
-		}
-		since := time.Since(waitStart)
-		if since > time.Second*5 {
-			r.Log("it is unusual that the cert is not new yet, but not a failure yet")
-		}
-		if since > time.Second*10 {
-			r.Log("wtf")
-			t.Fatal("Cert should be new by now")
-		}
-		time.Sleep(time.Second)
-	}
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestCertMismatchCorrection(t *testing.T) {
-	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
-	// under ideal conditions
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	myCert, _, myPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
-	myCert2, _ := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
-
-	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
-	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
-
-	myControl, myVpnIpNet, myUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert2}, myPrivKey, m{})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	r.Log("Assert the tunnel between me and them works")
-	//assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
-	//r.Log("yay")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	r.Log("yay")
-	//todo ???
-	time.Sleep(1 * time.Second)
-	r.FlushAll()
-
-	waitStart := time.Now()
-	for {
-		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-		c2 := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-		if c == nil || c2 == nil {
-			r.Log("nil")
-		} else {
-			version := c.Cert.Version()
-			theirVersion := c2.Cert.Version()
-			r.Logf("version %d,%d", version, theirVersion)
-			if version == theirVersion {
-				break
-			}
-		}
-		since := time.Since(waitStart)
-		if since > time.Second*5 {
-			r.Log("wtf")
-		}
-		if since > time.Second*10 {
-			r.Log("wtf")
-			t.Fatal("Cert should be new by now")
-		}
-		time.Sleep(time.Second)
-	}
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}

examples/config.yml

@@ -132,6 +132,13 @@ listen:
   # Sets the max number of packets to pull from the kernel for each syscall (under systems that support recvmmsg)
   # default is 64, does not support reload
   #batch: 64
+
+# Control batching between UDP and TUN pipelines
+#batch:
+#  inbound_size: 32        # packets to queue from UDP before handing to workers
+#  outbound_size: 32       # packets to queue from TUN before handing to workers
+#  flush_interval: 50us    # flush partially filled batches after this duration
+#  max_outstanding: 1028   # batches buffered per routine on each channel
   # Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
   # Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
   # Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide

firewall_test.go

@@ -392,7 +392,7 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 		c := &cert.CachedCertificate{
 			Certificate: &dummyCert{
 				name:     "nope",
-				networks: []netip.Prefix{netip.MustParsePrefix("fd99::99/128")},
+				networks: []netip.Prefix{netip.MustParsePrefix("fd99:99/128")},
 			},
 			InvertedGroups: map[string]struct{}{"nope": {}},
 		}
@@ -692,50 +692,6 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
 }
 
-func TestFirewall_DropIPSpoofing(t *testing.T) {
-	l := test.NewLogger()
-	ob := &bytes.Buffer{}
-	l.SetOutput(ob)
-
-	c := cert.CachedCertificate{
-		Certificate: &dummyCert{
-			name:     "host-owner",
-			networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
-		},
-	}
-
-	c1 := cert.CachedCertificate{
-		Certificate: &dummyCert{
-			name:           "host",
-			networks:       []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")},
-			unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
-		},
-	}
-	h1 := HostInfo{
-		ConnectionState: &ConnectionState{
-			peerCert: &c1,
-		},
-		vpnAddrs: []netip.Addr{c1.Certificate.Networks()[0].Addr()},
-	}
-	h1.buildNetworks(c1.Certificate.Networks(), c1.Certificate.UnsafeNetworks())
-
-	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
-
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
-	cp := cert.NewCAPool()
-
-	// Packet spoofed by `c1`. Note that the remote addr is not a valid one.
-	p := firewall.Packet{
-		LocalAddr:  netip.MustParseAddr("192.0.2.1"),
-		RemoteAddr: netip.MustParseAddr("192.0.2.3"),
-		LocalPort:  1,
-		RemotePort: 1,
-		Protocol:   firewall.ProtoUDP,
-		Fragment:   false,
-	}
-	assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP)
-}
-
 func BenchmarkLookup(b *testing.B) {
 	ml := func(m map[string]struct{}, a [][]string) {
 		for n := 0; n < b.N; n++ {

go.mod

@@ -8,30 +8,30 @@ require (
 	github.com/armon/go-radix v1.0.0
 	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
 	github.com/flynn/noise v1.1.0
-	github.com/gaissmai/bart v0.25.0
+	github.com/gaissmai/bart v0.20.4
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gopacket v1.1.19
-	github.com/kardianos/service v1.2.4
+	github.com/kardianos/service v1.2.2
-	github.com/miekg/dns v1.1.68
+	github.com/miekg/dns v1.1.65
 	github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b
 	github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
-	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_golang v1.22.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/sirupsen/logrus v1.9.3
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
-	github.com/stretchr/testify v1.11.1
+	github.com/stretchr/testify v1.10.0
 	github.com/vishvananda/netlink v1.3.1
-	golang.org/x/crypto v0.43.0
+	golang.org/x/crypto v0.37.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.45.0
+	golang.org/x/net v0.39.0
-	golang.org/x/sync v0.17.0
+	golang.org/x/sync v0.13.0
-	golang.org/x/sys v0.37.0
+	golang.org/x/sys v0.32.0
-	golang.org/x/term v0.36.0
+	golang.org/x/term v0.31.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
 	golang.zx2c4.com/wireguard/windows v0.5.3
-	google.golang.org/protobuf v1.36.8
+	google.golang.org/protobuf v1.36.6
 	gopkg.in/yaml.v3 v3.0.1
 	gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
 )
@@ -43,12 +43,11 @@ require (
 	github.com/google/btree v1.1.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/common v0.62.0 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/vishvananda/netns v0.0.5 // indirect
-	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	golang.org/x/mod v0.23.0 // indirect
-	golang.org/x/mod v0.24.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.33.0 // indirect
+	golang.org/x/tools v0.30.0 // indirect
 )

go.sum

@@ -24,8 +24,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
 github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
-github.com/gaissmai/bart v0.25.0 h1:eqiokVPqM3F94vJ0bTHXHtH91S8zkKL+bKh+BsGOsJM=
+github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U=
-github.com/gaissmai/bart v0.25.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
+github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -64,8 +64,8 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kardianos/service v1.2.4 h1:XNlGtZOYNx2u91urOdg/Kfmc+gfmuIo1Dd3rEi2OgBk=
+github.com/kardianos/service v1.2.2 h1:ZvePhAHfvo0A7Mftk/tEzqEZ7Q4lgnR8sGz4xu1YX60=
-github.com/kardianos/service v1.2.4/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=
+github.com/kardianos/service v1.2.2/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
@@ -83,8 +83,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA=
+github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc=
-github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps=
+github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck=
 github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b h1:J/AzCvg5z0Hn1rqZUJjpbzALUmkKX0Zwbc/i4fw7Sfk=
 github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -106,24 +106,24 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
-github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
-github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
-github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
-github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -143,33 +143,29 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
 github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
 github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
 github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
-go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
+golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
-golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
+golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
+golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
-golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -180,8 +176,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM=
+golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
-golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
+golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -189,8 +185,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -201,17 +197,18 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
-golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
+golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
-golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
+golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -222,8 +219,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
+golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
-golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
+golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -242,8 +239,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
+google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
-google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

handshake_ix.go

@@ -23,17 +23,13 @@ func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
 		return false
 	}
 
+	// If we're connecting to a v6 address we must use a v2 cert
 	cs := f.pki.getCertState()
 	v := cs.initiatingVersion
-	if hh.initiatingVersionOverride != cert.VersionPre1 {
+	for _, a := range hh.hostinfo.vpnAddrs {
-		v = hh.initiatingVersionOverride
+		if a.Is6() {
-	} else if v < cert.Version2 {
+			v = cert.Version2
-		// If we're connecting to a v6 address we should encourage use of a V2 cert
+			break
-		for _, a := range hh.hostinfo.vpnAddrs {
-			if a.Is6() {
-				v = cert.Version2
-				break
-			}
 		}
 	}
 
@@ -52,7 +48,6 @@ func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).
 			WithField("certVersion", v).
 			Error("Unable to handshake with host because no certificate handshake bytes is available")
-		return false
 	}
 
 	ci, err := NewConnectionState(f.l, cs, crt, true, noise.HandshakeIX)
@@ -108,7 +103,6 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).
 			WithField("certVersion", cs.initiatingVersion).
 			Error("Unable to handshake with host because no certificate is available")
-		return
 	}
 
 	ci, err := NewConnectionState(f.l, cs, crt, false, noise.HandshakeIX)
@@ -149,8 +143,8 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 
 	remoteCert, err := f.pki.GetCAPool().VerifyCertificate(time.Now(), rc)
 	if err != nil {
-		fp, fperr := rc.Fingerprint()
+		fp, err := rc.Fingerprint()
-		if fperr != nil {
+		if err != nil {
 			fp = "<error generating certificate fingerprint>"
 		}
 
@@ -169,19 +163,16 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 
 	if remoteCert.Certificate.Version() != ci.myCert.Version() {
 		// We started off using the wrong certificate version, lets see if we can match the version that was sent to us
-		myCertOtherVersion := cs.getCertificate(remoteCert.Certificate.Version())
+		rc := cs.getCertificate(remoteCert.Certificate.Version())
-		if myCertOtherVersion == nil {
+		if rc == nil {
-			if f.l.Level >= logrus.DebugLevel {
+			f.l.WithError(err).WithField("udpAddr", addr).
-				f.l.WithError(err).WithFields(m{
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cert", remoteCert).
-					"udpAddr":   addr,
+				Info("Unable to handshake with host due to missing certificate version")
-					"handshake": m{"stage": 1, "style": "ix_psk0"},
+			return
-					"cert":      remoteCert,
-				}).Debug("Might be unable to handshake with host due to missing certificate version")
-			}
-		} else {
-			// Record the certificate we are actually using
-			ci.myCert = myCertOtherVersion
 		}
+
+		// Record the certificate we are actually using
+		ci.myCert = rc
 	}
 
 	if len(remoteCert.Certificate.Networks()) == 0 {

handshake_manager.go

@@ -68,12 +68,11 @@ type HandshakeManager struct {
 type HandshakeHostInfo struct {
 	sync.Mutex
 
-	startTime                 time.Time        // Time that we first started trying with this handshake
+	startTime   time.Time        // Time that we first started trying with this handshake
-	ready                     bool             // Is the handshake ready
+	ready       bool             // Is the handshake ready
-	initiatingVersionOverride cert.Version     // Should we use a non-default cert version for this handshake?
+	counter     int64            // How many attempts have we made so far
-	counter                   int64            // How many attempts have we made so far
+	lastRemotes []netip.AddrPort // Remotes that we sent to during the previous attempt
-	lastRemotes               []netip.AddrPort // Remotes that we sent to during the previous attempt
+	packetStore []*cachedPacket  // A set of packets to be transmitted once the handshake completes
-	packetStore               []*cachedPacket  // A set of packets to be transmitted once the handshake completes
 
 	hostinfo *HostInfo
 }

hostmap.go

@@ -738,8 +738,7 @@ func (i *HostInfo) buildNetworks(networks, unsafeNetworks []netip.Prefix) {
 
 	i.networks = new(bart.Lite)
 	for _, network := range networks {
-		nprefix := netip.PrefixFrom(network.Addr(), network.Addr().BitLen())
+		i.networks.Insert(network)
-		i.networks.Insert(nprefix)
 	}
 
 	for _, network := range unsafeNetworks {

inside.go

@@ -11,162 +11,19 @@ import (
 	"github.com/slackhq/nebula/routing"
 )
 
-// consumeInsidePackets processes multiple packets in a batch for improved performance
+func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, queue func(netip.AddrPort, int), q int, localCache firewall.ConntrackCache) bool {
-// packets: slice of packet buffers to process
-// sizes: slice of packet sizes
-// count: number of packets to process
-// outs: slice of output buffers (one per packet) with virtio headroom
-// q: queue index
-// localCache: firewall conntrack cache
-// batchPackets: pre-allocated slice for accumulating encrypted packets
-// batchAddrs: pre-allocated slice for accumulating destination addresses
-func (f *Interface) consumeInsidePackets(packets [][]byte, sizes []int, count int, outs [][]byte, nb []byte, q int, localCache firewall.ConntrackCache, batchPackets *[][]byte, batchAddrs *[]netip.AddrPort) {
-	// Reusable per-packet state
-	fwPacket := &firewall.Packet{}
-
-	// Reset batch accumulation slices (reuse capacity)
-	*batchPackets = (*batchPackets)[:0]
-	*batchAddrs = (*batchAddrs)[:0]
-
-	// Process each packet in the batch
-	for i := 0; i < count; i++ {
-		packet := packets[i][:sizes[i]]
-		out := outs[i]
-
-		// Inline the consumeInsidePacket logic for better performance
-		err := newPacket(packet, false, fwPacket)
-		if err != nil {
-			if f.l.Level >= logrus.DebugLevel {
-				f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
-			}
-			continue
-		}
-
-		// Ignore local broadcast packets
-		if f.dropLocalBroadcast {
-			if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
-				continue
-			}
-		}
-
-		if f.myVpnAddrsTable.Contains(fwPacket.RemoteAddr) {
-			// Immediately forward packets from self to self.
-			if immediatelyForwardToSelf {
-				_, err := f.readers[q].Write(packet)
-				if err != nil {
-					f.l.WithError(err).Error("Failed to forward to tun")
-				}
-			}
-			continue
-		}
-
-		// Ignore multicast packets
-		if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() {
-			continue
-		}
-
-		hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
-			hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
-		})
-
-		if hostinfo == nil {
-			f.rejectInside(packet, out, q)
-			if f.l.Level >= logrus.DebugLevel {
-				f.l.WithField("vpnAddr", fwPacket.RemoteAddr).
-					WithField("fwPacket", fwPacket).
-					Debugln("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks")
-			}
-			continue
-		}
-
-		if !ready {
-			continue
-		}
-
-		dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
-		if dropReason != nil {
-			f.rejectInside(packet, out, q)
-			if f.l.Level >= logrus.DebugLevel {
-				hostinfo.logger(f.l).
-					WithField("fwPacket", fwPacket).
-					WithField("reason", dropReason).
-					Debugln("dropping outbound packet")
-			}
-			continue
-		}
-
-		// Encrypt and prepare packet for batch sending
-		ci := hostinfo.ConnectionState
-		if ci.eKey == nil {
-			continue
-		}
-
-		// Check if this needs relay - if so, send immediately and skip batching
-		useRelay := !hostinfo.remote.IsValid()
-		if useRelay {
-			// Handle relay sends individually (less common path)
-			f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, packet, nb, out, q)
-			continue
-		}
-
-		// Encrypt the packet for batch sending
-		if noiseutil.EncryptLockNeeded {
-			ci.writeLock.Lock()
-		}
-		c := ci.messageCounter.Add(1)
-		out = header.Encode(out, header.Version, header.Message, 0, hostinfo.remoteIndexId, c)
-		f.connectionManager.Out(hostinfo)
-
-		// Query lighthouse if needed
-		if hostinfo.lastRebindCount != f.rebindCount {
-			f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
-			hostinfo.lastRebindCount = f.rebindCount
-			if f.l.Level >= logrus.DebugLevel {
-				f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).Debug("Lighthouse update triggered for punch due to rebind counter")
-			}
-		}
-
-		out, err = ci.eKey.EncryptDanger(out, out, packet, c, nb)
-		if noiseutil.EncryptLockNeeded {
-			ci.writeLock.Unlock()
-		}
-		if err != nil {
-			hostinfo.logger(f.l).WithError(err).
-				WithField("counter", c).
-				Error("Failed to encrypt outgoing packet")
-			continue
-		}
-
-		// Add to batch
-		*batchPackets = append(*batchPackets, out)
-		*batchAddrs = append(*batchAddrs, hostinfo.remote)
-	}
-
-	// Send all accumulated packets in one batch
-	if len(*batchPackets) > 0 {
-		batchSize := len(*batchPackets)
-		f.batchMetrics.udpWriteSize.Update(int64(batchSize))
-
-		n, err := f.writers[q].WriteMulti(*batchPackets, *batchAddrs)
-		if err != nil {
-			f.l.WithError(err).WithField("sent", n).WithField("total", batchSize).Error("Failed to send batch")
-		}
-	}
-}
-
-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
 	err := newPacket(packet, false, fwPacket)
 	if err != nil {
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
 		}
-		return
+		return false
 	}
 
 	// Ignore local broadcast packets
 	if f.dropLocalBroadcast {
 		if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
-			return
+			return false
 		}
 	}
 
@@ -183,12 +40,12 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		}
 		// Otherwise, drop. On linux, we should never see these packets - Linux
 		// routes packets from the nebula addr to the nebula addr through the loopback device.
-		return
+		return false
 	}
 
 	// Ignore multicast packets
 	if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() {
-		return
+		return false
 	}
 
 	hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
@@ -202,26 +59,26 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 				WithField("fwPacket", fwPacket).
 				Debugln("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks")
 		}
-		return
+		return false
 	}
 
 	if !ready {
-		return
+		return false
 	}
 
 	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
-		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
+		return f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, queue, q)
-
-	} else {
-		f.rejectInside(packet, out, q)
-		if f.l.Level >= logrus.DebugLevel {
-			hostinfo.logger(f.l).
-				WithField("fwPacket", fwPacket).
-				WithField("reason", dropReason).
-				Debugln("dropping outbound packet")
-		}
 	}
+
+	f.rejectInside(packet, out, q)
+	if f.l.Level >= logrus.DebugLevel {
+		hostinfo.logger(f.l).
+			WithField("fwPacket", fwPacket).
+			WithField("reason", dropReason).
+			Debugln("dropping outbound packet")
+	}
+	return false
 }
 
 func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
@@ -260,7 +117,7 @@ func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *
 		return
 	}
 
-	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q)
+	_ = f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, nil, q)
 }
 
 // Handshake will attempt to initiate a tunnel with the provided vpn address if it is within our vpn networks. This is a no-op if the tunnel is already established or being established
@@ -371,7 +228,7 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
 		return
 	}
 
-	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, 0)
+	_ = f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, nil, 0)
 }
 
 // SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr
@@ -401,12 +258,12 @@ func (f *Interface) SendMessageToHostInfo(t header.MessageType, st header.Messag
 
 func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p, nb, out []byte) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, nb, out, 0)
+	_ = f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, nb, out, nil, 0)
 }
 
 func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0)
+	_ = f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, nil, 0)
 }
 
 // SendVia sends a payload through a Relay tunnel. No authentication or encryption is done
@@ -474,9 +331,12 @@ func (f *Interface) SendVia(via *HostInfo,
 	f.connectionManager.RelayUsed(relay.LocalIndex)
 }
 
-func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, q int) {
+// sendNoMetrics encrypts and writes/queues an outbound packet. It returns true
+// when the payload has been handed to a caller-provided queue (meaning the
+// caller is responsible for flushing it later).
+func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, queue func(netip.AddrPort, int), q int) bool {
 	if ci.eKey == nil {
-		return
+		return false
 	}
 	useRelay := !remote.IsValid() && !hostinfo.remote.IsValid()
 	fullOut := out
@@ -523,32 +383,39 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 			WithField("udpAddr", remote).WithField("counter", c).
 			WithField("attemptedCounter", c).
 			Error("Failed to encrypt outgoing packet")
-		return
+		return false
 	}
 
-	if remote.IsValid() {
+	dest := remote
-		err = f.writers[q].WriteTo(out, remote)
+	if !dest.IsValid() {
-		if err != nil {
+		dest = hostinfo.remote
-			hostinfo.logger(f.l).WithError(err).
-				WithField("udpAddr", remote).Error("Failed to write outgoing packet")
-		}
-	} else if hostinfo.remote.IsValid() {
-		err = f.writers[q].WriteTo(out, hostinfo.remote)
-		if err != nil {
-			hostinfo.logger(f.l).WithError(err).
-				WithField("udpAddr", remote).Error("Failed to write outgoing packet")
-		}
-	} else {
-		// Try to send via a relay
-		for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
-			relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
-			if err != nil {
-				hostinfo.relayState.DeleteRelay(relayIP)
-				hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetrics failed to find HostInfo")
-				continue
-			}
-			f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
-			break
-		}
 	}
+
+	if dest.IsValid() {
+		if queue != nil {
+			queue(dest, len(out))
+			return true
+		}
+
+		err = f.writers[q].WriteTo(out, dest)
+		if err != nil {
+			hostinfo.logger(f.l).WithError(err).
+				WithField("udpAddr", dest).Error("Failed to write outgoing packet")
+		}
+		return false
+	}
+
+	// Try to send via a relay
+	for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
+		relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
+		if err != nil {
+			hostinfo.relayState.DeleteRelay(relayIP)
+			hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetrics failed to find HostInfo")
+			continue
+		}
+		f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
+		break
+	}
+
+	return false
 }

interface.go

@@ -4,9 +4,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net/netip"
-	"os"
 	"runtime"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -17,11 +18,22 @@ import (
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
+	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/udp"
 )
 
-const mtu = 9001
+const (
-const virtioNetHdrLen = overlay.VirtioNetHdrLen
+	mtu = 9001
+
+	inboundBatchSizeDefault      = 128
+	outboundBatchSizeDefault     = 64
+	batchFlushIntervalDefault    = 12 * time.Microsecond
+	maxOutstandingBatchesDefault = 8
+	sendBatchSizeDefault         = 64
+	maxPendingPacketsDefault     = 32
+	maxPendingBytesDefault       = 64 * 1024
+	maxSendBufPerRoutineDefault  = 16
+)
 
 type InterfaceConfig struct {
 	HostMap            *HostMap
@@ -47,14 +59,18 @@ type InterfaceConfig struct {
 	reQueryWait     time.Duration
 
 	ConntrackCacheTimeout time.Duration
+	BatchConfig           BatchConfig
 	l                     *logrus.Logger
 }
 
-type batchMetrics struct {
+type BatchConfig struct {
-	udpReadSize   metrics.Histogram
+	InboundBatchSize      int
-	tunReadSize   metrics.Histogram
+	OutboundBatchSize     int
-	udpWriteSize  metrics.Histogram
+	FlushInterval         time.Duration
-	tunWriteSize  metrics.Histogram
+	MaxOutstandingPerChan int
+	MaxPendingPackets     int
+	MaxPendingBytes       int
+	MaxSendBuffersPerChan int
 }
 
 type Interface struct {
@@ -93,14 +109,166 @@ type Interface struct {
 	conntrackCacheTimeout time.Duration
 
 	writers []udp.Conn
-	readers []overlay.BatchReadWriter
+	readers []io.ReadWriteCloser
+	wg      sync.WaitGroup
 
 	metricHandshakes    metrics.Histogram
 	messageMetrics      *MessageMetrics
 	cachedPacketMetrics *cachedPacketMetrics
-	batchMetrics        *batchMetrics
 
 	l *logrus.Logger
+
+	inPool  sync.Pool
+	inbound []chan *packetBatch
+
+	outPool  sync.Pool
+	outbound []chan *outboundBatch
+
+	packetBatchPool   sync.Pool
+	outboundBatchPool sync.Pool
+
+	sendPool      sync.Pool
+	sendBufCache  [][]*[]byte
+	sendBatchSize int
+
+	inboundBatchSize      int
+	outboundBatchSize     int
+	batchFlushInterval    time.Duration
+	maxOutstandingPerChan int
+	maxPendingPackets     int
+	maxPendingBytes       int
+	maxSendBufPerRoutine  int
+}
+
+type outboundSend struct {
+	buf    *[]byte
+	length int
+	addr   netip.AddrPort
+}
+
+type packetBatch struct {
+	packets []*packet.Packet
+}
+
+func newPacketBatch(capacity int) *packetBatch {
+	return &packetBatch{
+		packets: make([]*packet.Packet, 0, capacity),
+	}
+}
+
+func (b *packetBatch) add(p *packet.Packet) {
+	b.packets = append(b.packets, p)
+}
+
+func (b *packetBatch) reset() {
+	for i := range b.packets {
+		b.packets[i] = nil
+	}
+	b.packets = b.packets[:0]
+}
+
+func (f *Interface) getPacketBatch() *packetBatch {
+	if v := f.packetBatchPool.Get(); v != nil {
+		b := v.(*packetBatch)
+		b.reset()
+		return b
+	}
+	return newPacketBatch(f.inboundBatchSize)
+}
+
+func (f *Interface) releasePacketBatch(b *packetBatch) {
+	b.reset()
+	f.packetBatchPool.Put(b)
+}
+
+type outboundBatch struct {
+	payloads []*[]byte
+}
+
+func newOutboundBatch(capacity int) *outboundBatch {
+	return &outboundBatch{payloads: make([]*[]byte, 0, capacity)}
+}
+
+func (b *outboundBatch) add(buf *[]byte) {
+	b.payloads = append(b.payloads, buf)
+}
+
+func (b *outboundBatch) reset() {
+	for i := range b.payloads {
+		b.payloads[i] = nil
+	}
+	b.payloads = b.payloads[:0]
+}
+
+func (f *Interface) getOutboundBatch() *outboundBatch {
+	if v := f.outboundBatchPool.Get(); v != nil {
+		b := v.(*outboundBatch)
+		b.reset()
+		return b
+	}
+	return newOutboundBatch(f.outboundBatchSize)
+}
+
+func (f *Interface) releaseOutboundBatch(b *outboundBatch) {
+	b.reset()
+	f.outboundBatchPool.Put(b)
+}
+
+func (f *Interface) getSendBuffer(q int) *[]byte {
+	cache := f.sendBufCache[q]
+	if n := len(cache); n > 0 {
+		buf := cache[n-1]
+		f.sendBufCache[q] = cache[:n-1]
+		*buf = (*buf)[:0]
+		return buf
+	}
+	if v := f.sendPool.Get(); v != nil {
+		buf := v.(*[]byte)
+		*buf = (*buf)[:0]
+		return buf
+	}
+	b := make([]byte, mtu)
+	return &b
+}
+
+func (f *Interface) releaseSendBuffer(q int, buf *[]byte) {
+	if buf == nil {
+		return
+	}
+	*buf = (*buf)[:0]
+	cache := f.sendBufCache[q]
+	if len(cache) < f.maxSendBufPerRoutine {
+		f.sendBufCache[q] = append(cache, buf)
+		return
+	}
+	f.sendPool.Put(buf)
+}
+
+func (f *Interface) flushSendQueue(q int, pending *[]outboundSend, pendingBytes *int) {
+	if len(*pending) == 0 {
+		return
+	}
+
+	batch := make([]udp.BatchPacket, len(*pending))
+	for i, entry := range *pending {
+		batch[i] = udp.BatchPacket{
+			Payload: (*entry.buf)[:entry.length],
+			Addr:    entry.addr,
+		}
+	}
+
+	sent, err := f.writers[q].WriteBatch(batch)
+	if err != nil {
+		f.l.WithError(err).WithField("sent", sent).Error("Failed to batch send packets")
+	}
+
+	for _, entry := range *pending {
+		f.releaseSendBuffer(q, entry.buf)
+	}
+	*pending = (*pending)[:0]
+	if pendingBytes != nil {
+		*pendingBytes = 0
+	}
 }
 
 type EncWriter interface {
@@ -170,6 +338,29 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 	}
 
 	cs := c.pki.getCertState()
+
+	bc := c.BatchConfig
+	if bc.InboundBatchSize <= 0 {
+		bc.InboundBatchSize = inboundBatchSizeDefault
+	}
+	if bc.OutboundBatchSize <= 0 {
+		bc.OutboundBatchSize = outboundBatchSizeDefault
+	}
+	if bc.FlushInterval <= 0 {
+		bc.FlushInterval = batchFlushIntervalDefault
+	}
+	if bc.MaxOutstandingPerChan <= 0 {
+		bc.MaxOutstandingPerChan = maxOutstandingBatchesDefault
+	}
+	if bc.MaxPendingPackets <= 0 {
+		bc.MaxPendingPackets = maxPendingPacketsDefault
+	}
+	if bc.MaxPendingBytes <= 0 {
+		bc.MaxPendingBytes = maxPendingBytesDefault
+	}
+	if bc.MaxSendBuffersPerChan <= 0 {
+		bc.MaxSendBuffersPerChan = maxSendBufPerRoutineDefault
+	}
 	ifce := &Interface{
 		pki:                   c.pki,
 		hostMap:               c.HostMap,
@@ -185,7 +376,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		routines:              c.routines,
 		version:               c.version,
 		writers:               make([]udp.Conn, c.routines),
-		readers:               make([]overlay.BatchReadWriter, c.routines),
+		readers:               make([]io.ReadWriteCloser, c.routines),
 		myVpnNetworks:         cs.myVpnNetworks,
 		myVpnNetworksTable:    cs.myVpnNetworksTable,
 		myVpnAddrs:            cs.myVpnAddrs,
@@ -201,16 +392,50 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 			sent:    metrics.GetOrRegisterCounter("hostinfo.cached_packets.sent", nil),
 			dropped: metrics.GetOrRegisterCounter("hostinfo.cached_packets.dropped", nil),
 		},
-		batchMetrics: &batchMetrics{
+
-			udpReadSize:  metrics.GetOrRegisterHistogram("batch.udp_read_size", nil, metrics.NewUniformSample(1024)),
+		inbound:  make([]chan *packetBatch, c.routines),
-			tunReadSize:  metrics.GetOrRegisterHistogram("batch.tun_read_size", nil, metrics.NewUniformSample(1024)),
+		outbound: make([]chan *outboundBatch, c.routines),
-			udpWriteSize: metrics.GetOrRegisterHistogram("batch.udp_write_size", nil, metrics.NewUniformSample(1024)),
-			tunWriteSize: metrics.GetOrRegisterHistogram("batch.tun_write_size", nil, metrics.NewUniformSample(1024)),
-		},
 
 		l: c.l,
+
+		inboundBatchSize:      bc.InboundBatchSize,
+		outboundBatchSize:     bc.OutboundBatchSize,
+		batchFlushInterval:    bc.FlushInterval,
+		maxOutstandingPerChan: bc.MaxOutstandingPerChan,
+		maxPendingPackets:     bc.MaxPendingPackets,
+		maxPendingBytes:       bc.MaxPendingBytes,
+		maxSendBufPerRoutine:  bc.MaxSendBuffersPerChan,
+		sendBatchSize:         bc.OutboundBatchSize,
 	}
 
+	for i := 0; i < c.routines; i++ {
+		ifce.inbound[i] = make(chan *packetBatch, ifce.maxOutstandingPerChan)
+		ifce.outbound[i] = make(chan *outboundBatch, ifce.maxOutstandingPerChan)
+	}
+
+	ifce.inPool = sync.Pool{New: func() any {
+		return packet.New()
+	}}
+
+	ifce.outPool = sync.Pool{New: func() any {
+		t := make([]byte, mtu)
+		return &t
+	}}
+
+	ifce.packetBatchPool = sync.Pool{New: func() any {
+		return newPacketBatch(ifce.inboundBatchSize)
+	}}
+
+	ifce.outboundBatchPool = sync.Pool{New: func() any {
+		return newOutboundBatch(ifce.outboundBatchSize)
+	}}
+
+	ifce.sendPool = sync.Pool{New: func() any {
+		buf := make([]byte, mtu)
+		return &buf
+	}}
+	ifce.sendBufCache = make([][]*[]byte, c.routines)
+
 	ifce.tryPromoteEvery.Store(c.tryPromoteEvery)
 	ifce.reQueryEvery.Store(c.reQueryEvery)
 	ifce.reQueryWait.Store(int64(c.reQueryWait))
@@ -223,7 +448,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 // activate creates the interface on the host. After the interface is created, any
 // other services that want to bind listeners to its IP may do so successfully. However,
 // the interface isn't going to process anything until run() is called.
-func (f *Interface) activate() {
+func (f *Interface) activate() error {
 	// actually turn on tun dev
 
 	addr, err := f.outside.LocalAddr()
@@ -239,38 +464,49 @@ func (f *Interface) activate() {
 	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
 
 	// Prepare n tun queues
-	var reader overlay.BatchReadWriter = f.inside
+	var reader io.ReadWriteCloser = f.inside
 	for i := 0; i < f.routines; i++ {
 		if i > 0 {
 			reader, err = f.inside.NewMultiQueueReader()
 			if err != nil {
-				f.l.Fatal(err)
+				return err
 			}
 		}
 		f.readers[i] = reader
 	}
 
-	if err := f.inside.Activate(); err != nil {
+	if err = f.inside.Activate(); err != nil {
 		f.inside.Close()
-		f.l.Fatal(err)
+		return err
 	}
+
+	return nil
 }
 
-func (f *Interface) run() {
+func (f *Interface) run(c context.Context) (func(), error) {
-	// Launch n queues to read packets from udp
 	for i := 0; i < f.routines; i++ {
+		// Launch n queues to read packets from udp
+		f.wg.Add(1)
 		go f.listenOut(i)
+
+		// Launch n queues to read packets from tun dev
+		f.wg.Add(1)
+		go f.listenIn(f.readers[i], i)
+
+		// Launch n queues to read packets from tun dev
+		f.wg.Add(1)
+		go f.workerIn(i, c)
+
+		// Launch n queues to read packets from tun dev
+		f.wg.Add(1)
+		go f.workerOut(i, c)
 	}
 
-	// Launch n queues to read packets from tun dev
+	return f.wg.Wait, nil
-	for i := 0; i < f.routines; i++ {
-		go f.listenIn(f.readers[i], i)
-	}
 }
 
 func (f *Interface) listenOut(i int) {
 	runtime.LockOSThread()
-
 	var li udp.Conn
 	if i > 0 {
 		li = f.writers[i]
@@ -278,71 +514,176 @@ func (f *Interface) listenOut(i int) {
 		li = f.outside
 	}
 
-	ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	batch := f.getPacketBatch()
-	lhh := f.lightHouse.NewRequestHandler()
+	lastFlush := time.Now()
 
-	// Pre-allocate output buffers for batch processing
+	flush := func(force bool) {
-	batchSize := li.BatchSize()
+		if len(batch.packets) == 0 {
-	outs := make([][]byte, batchSize)
+			if force {
-	for idx := range outs {
+				f.releasePacketBatch(batch)
-		// Allocate full buffer with virtio header space
-		outs[idx] = make([]byte, virtioNetHdrLen, virtioNetHdrLen+udp.MTU)
-	}
-
-	h := &header.H{}
-	fwPacket := &firewall.Packet{}
-	nb := make([]byte, 12)
-
-	li.ListenOutBatch(func(addrs []netip.AddrPort, payloads [][]byte, count int) {
-		f.readOutsidePacketsBatch(addrs, payloads, count, outs[:count], nb, i, h, fwPacket, lhh, ctCache.Get(f.l))
-	})
-}
-
-func (f *Interface) listenIn(reader overlay.BatchReadWriter, i int) {
-	runtime.LockOSThread()
-
-	batchSize := reader.BatchSize()
-
-	// Allocate buffers for batch reading
-	bufs := make([][]byte, batchSize)
-	for idx := range bufs {
-		bufs[idx] = make([]byte, mtu)
-	}
-	sizes := make([]int, batchSize)
-
-	// Allocate output buffers for batch processing (one per packet)
-	// Each has virtio header headroom to avoid copies on write
-	outs := make([][]byte, batchSize)
-	for idx := range outs {
-		outBuf := make([]byte, virtioNetHdrLen+mtu)
-		outs[idx] = outBuf[virtioNetHdrLen:] // Slice starting after headroom
-	}
-
-	// Pre-allocate batch accumulation buffers for sending
-	batchPackets := make([][]byte, 0, batchSize)
-	batchAddrs := make([]netip.AddrPort, 0, batchSize)
-
-	// Pre-allocate nonce buffer (reused for all encryptions)
-	nb := make([]byte, 12)
-
-	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
-
-	for {
-		n, err := reader.BatchRead(bufs, sizes)
-		if err != nil {
-			if errors.Is(err, os.ErrClosed) && f.closed.Load() {
-				return
 			}
-
+			return
-			f.l.WithError(err).Error("Error while batch reading outbound packets")
-			// This only seems to happen when something fatal happens to the fd, so exit.
-			os.Exit(2)
 		}
 
-		f.batchMetrics.tunReadSize.Update(int64(n))
+		f.inbound[i] <- batch
+		batch = f.getPacketBatch()
+		lastFlush = time.Now()
+	}
 
-		// Process all packets in the batch at once
+	err := li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
-		f.consumeInsidePackets(bufs, sizes, n, outs, nb, i, conntrackCache.Get(f.l), &batchPackets, &batchAddrs)
+		p := f.inPool.Get().(*packet.Packet)
+		p.Payload = p.Payload[:mtu]
+		copy(p.Payload, payload)
+		p.Payload = p.Payload[:len(payload)]
+		p.Addr = fromUdpAddr
+		batch.add(p)
+
+		if len(batch.packets) >= f.inboundBatchSize || time.Since(lastFlush) >= f.batchFlushInterval {
+			flush(false)
+		}
+	})
+
+	if len(batch.packets) > 0 {
+		f.inbound[i] <- batch
+	} else {
+		f.releasePacketBatch(batch)
+	}
+
+	if err != nil && !f.closed.Load() {
+		f.l.WithError(err).Error("Error while reading packet inbound packet, closing")
+		//TODO: Trigger Control to close
+	}
+
+	f.l.Debugf("underlay reader %v is done", i)
+	f.wg.Done()
+}
+
+func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
+	runtime.LockOSThread()
+
+	batch := f.getOutboundBatch()
+	lastFlush := time.Now()
+
+	flush := func(force bool) {
+		if len(batch.payloads) == 0 {
+			if force {
+				f.releaseOutboundBatch(batch)
+			}
+			return
+		}
+
+		f.outbound[i] <- batch
+		batch = f.getOutboundBatch()
+		lastFlush = time.Now()
+	}
+
+	for {
+		p := f.outPool.Get().(*[]byte)
+		*p = (*p)[:mtu]
+		n, err := reader.Read(*p)
+		if err != nil {
+			if !f.closed.Load() {
+				f.l.WithError(err).Error("Error while reading outbound packet, closing")
+				//TODO: Trigger Control to close
+			}
+			break
+		}
+
+		*p = (*p)[:n]
+		batch.add(p)
+
+		if len(batch.payloads) >= f.outboundBatchSize || time.Since(lastFlush) >= f.batchFlushInterval {
+			flush(false)
+		}
+	}
+
+	if len(batch.payloads) > 0 {
+		f.outbound[i] <- batch
+	} else {
+		f.releaseOutboundBatch(batch)
+	}
+
+	f.l.Debugf("overlay reader %v is done", i)
+	f.wg.Done()
+}
+
+func (f *Interface) workerIn(i int, ctx context.Context) {
+	lhh := f.lightHouse.NewRequestHandler()
+	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	fwPacket2 := &firewall.Packet{}
+	nb2 := make([]byte, 12, 12)
+	result2 := make([]byte, mtu)
+	h := &header.H{}
+
+	for {
+		select {
+		case batch := <-f.inbound[i]:
+			for _, p := range batch.packets {
+				f.readOutsidePackets(p.Addr, nil, result2[:0], p.Payload, h, fwPacket2, lhh, nb2, i, conntrackCache.Get(f.l))
+				p.Payload = p.Payload[:mtu]
+				f.inPool.Put(p)
+			}
+			f.releasePacketBatch(batch)
+		case <-ctx.Done():
+			f.wg.Done()
+			return
+		}
+	}
+}
+
+func (f *Interface) workerOut(i int, ctx context.Context) {
+	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	fwPacket1 := &firewall.Packet{}
+	nb1 := make([]byte, 12, 12)
+	pending := make([]outboundSend, 0, f.sendBatchSize)
+	pendingBytes := 0
+	maxPendingPackets := f.maxPendingPackets
+	if maxPendingPackets <= 0 {
+		maxPendingPackets = f.sendBatchSize
+	}
+	maxPendingBytes := f.maxPendingBytes
+	if maxPendingBytes <= 0 {
+		maxPendingBytes = f.sendBatchSize * mtu
+	}
+
+	for {
+		select {
+		case batch := <-f.outbound[i]:
+			for _, data := range batch.payloads {
+				sendBuf := f.getSendBuffer(i)
+				buf := (*sendBuf)[:0]
+				queue := func(addr netip.AddrPort, length int) {
+					if len(pending) >= maxPendingPackets || pendingBytes+length > maxPendingBytes {
+						f.flushSendQueue(i, &pending, &pendingBytes)
+					}
+					pending = append(pending, outboundSend{
+						buf:    sendBuf,
+						length: length,
+						addr:   addr,
+					})
+					pendingBytes += length
+					if len(pending) >= f.sendBatchSize || pendingBytes >= maxPendingBytes {
+						f.flushSendQueue(i, &pending, &pendingBytes)
+					}
+				}
+				sent := f.consumeInsidePacket(*data, fwPacket1, nb1, buf, queue, i, conntrackCache.Get(f.l))
+				if !sent {
+					f.releaseSendBuffer(i, sendBuf)
+				}
+				*data = (*data)[:mtu]
+				f.outPool.Put(data)
+			}
+			f.releaseOutboundBatch(batch)
+			if len(pending) > 0 {
+				f.flushSendQueue(i, &pending, &pendingBytes)
+			}
+		case <-ctx.Done():
+			if len(pending) > 0 {
+				f.flushSendQueue(i, &pending, &pendingBytes)
+			}
+			f.wg.Done()
+			return
+		}
 	}
 }
 
@@ -495,6 +836,7 @@ func (f *Interface) GetCertState() *CertState {
 func (f *Interface) Close() error {
 	f.closed.Store(true)
 
+	// Release the udp readers
 	for _, u := range f.writers {
 		err := u.Close()
 		if err != nil {
@@ -502,6 +844,13 @@ func (f *Interface) Close() error {
 		}
 	}
 
-	// Release the tun device
+	// Release the tun readers
-	return f.inside.Close()
+	for _, u := range f.readers {
+		err := u.Close()
+		if err != nil {
+			f.l.WithError(err).Error("Error while closing tun device")
+		}
+	}
+
+	return nil
 }

lighthouse.go

@@ -1337,19 +1337,12 @@ func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpn
 		}
 	}
 
-	remoteAllowList := lhh.lh.GetRemoteAllowList()
 	for _, a := range n.Details.V4AddrPorts {
-		b := protoV4AddrPortToNetAddrPort(a)
+		punch(protoV4AddrPortToNetAddrPort(a), detailsVpnAddr)
-		if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) {
-			punch(b, detailsVpnAddr)
-		}
 	}
 
 	for _, a := range n.Details.V6AddrPorts {
-		b := protoV6AddrPortToNetAddrPort(a)
+		punch(protoV6AddrPortToNetAddrPort(a), detailsVpnAddr)
-		if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) {
-			punch(b, detailsVpnAddr)
-		}
 	}
 
 	// This sends a nebula test packet to the host trying to contact us. In the case

main.go

@@ -75,8 +75,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	if c.GetBool("sshd.enabled", false) {
 		sshStart, err = configSSH(l, ssh, c)
 		if err != nil {
-			l.WithError(err).Warn("Failed to configure sshd, ssh debugging will not be available")
+			return nil, util.ContextualizeIfNeeded("Error while configuring the sshd", err)
-			sshStart = nil
 		}
 	}
 
@@ -222,6 +221,16 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		}
 	}
 
+	batchCfg := BatchConfig{
+		InboundBatchSize:      c.GetInt("batch.inbound_size", inboundBatchSizeDefault),
+		OutboundBatchSize:     c.GetInt("batch.outbound_size", outboundBatchSizeDefault),
+		FlushInterval:         c.GetDuration("batch.flush_interval", batchFlushIntervalDefault),
+		MaxOutstandingPerChan: c.GetInt("batch.max_outstanding", maxOutstandingBatchesDefault),
+		MaxPendingPackets:     c.GetInt("batch.max_pending_packets", 0),
+		MaxPendingBytes:       c.GetInt("batch.max_pending_bytes", 0),
+		MaxSendBuffersPerChan: c.GetInt("batch.max_send_buffers_per_routine", 0),
+	}
+
 	ifConfig := &InterfaceConfig{
 		HostMap:               hostMap,
 		Inside:                tun,
@@ -243,6 +252,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		relayManager:          NewRelayManager(ctx, l, hostMap, c),
 		punchy:                punchy,
 		ConntrackCacheTimeout: conntrackCacheTimeout,
+		BatchConfig:           batchCfg,
 		l:                     l,
 	}
 
@@ -285,14 +295,14 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	}
 
 	return &Control{
-		ifce,
+		f:                      ifce,
-		l,
+		l:                      l,
-		ctx,
+		ctx:                    ctx,
-		cancel,
+		cancel:                 cancel,
-		sshStart,
+		sshStart:               sshStart,
-		statsStart,
+		statsStart:             statsStart,
-		dnsStart,
+		dnsStart:               dnsStart,
-		lightHouse.StartUpdateWorker,
+		lighthouseStart:        lightHouse.StartUpdateWorker,
-		connManager.Start,
+		connectionManagerStart: connManager.Start,
 	}, nil
 }

outside.go

@@ -29,7 +29,7 @@ func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []
 		return
 	}
 
-	//l.Error("in packet ", header, packet[HeaderLen:])
+	//f.l.Error("in packet ", h)
 	if ip.IsValid() {
 		if f.myVpnNetworksTable.Contains(ip.Addr()) {
 			if f.l.Level >= logrus.DebugLevel {
@@ -95,7 +95,8 @@ func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []
 			switch relay.Type {
 			case TerminalType:
 				// If I am the target of this relay, process the unwrapped packet
-				f.readOutsidePackets(netip.AddrPort{}, &ViaSender{relayHI: hostinfo, remoteIdx: relay.RemoteIndex, relay: relay}, out[:virtioNetHdrLen], signedPayload, h, fwPacket, lhf, nb, q, localCache)
+				// From this recursive point, all these variables are 'burned'. We shouldn't rely on them again.
+				f.readOutsidePackets(netip.AddrPort{}, &ViaSender{relayHI: hostinfo, remoteIdx: relay.RemoteIndex, relay: relay}, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
 				return
 			case ForwardingType:
 				// Find the target HostInfo relay object
@@ -137,7 +138,7 @@ func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []
 			return
 		}
 
-		lhf.HandleRequest(ip, hostinfo.vpnAddrs, d[virtioNetHdrLen:], f)
+		lhf.HandleRequest(ip, hostinfo.vpnAddrs, d, f)
 
 		// Fallthrough to the bottom to record incoming traffic
 
@@ -159,7 +160,7 @@ func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []
 			// This testRequest might be from TryPromoteBest, so we should roam
 			// to the new IP address before responding
 			f.handleHostRoaming(hostinfo, ip)
-			f.send(header.Test, header.TestReply, ci, hostinfo, d[virtioNetHdrLen:], nb, out)
+			f.send(header.Test, header.TestReply, ci, hostinfo, d, nb, out)
 		}
 
 		// Fallthrough to the bottom to record incoming traffic
@@ -202,7 +203,7 @@ func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []
 			return
 		}
 
-		f.relayManager.HandleControlMsg(hostinfo, d[virtioNetHdrLen:], f)
+		f.relayManager.HandleControlMsg(hostinfo, d, f)
 
 	default:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
@@ -244,6 +245,7 @@ func (f *Interface) handleHostRoaming(hostinfo *HostInfo, udpAddr netip.AddrPort
 			return
 		}
 
+		//TODO: Seems we have a bunch of stuff racing here, since we don't have a lock on hostinfo anymore we announce roaming in bursts
 		hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", udpAddr).
 			Info("Host roamed to new udp ip/port.")
 		hostinfo.lastRoam = time.Now()
@@ -469,15 +471,13 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 
 	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
 	if err != nil {
-		hostinfo.logger(f.l).WithError(err).Error("Failed to decrypt packet")
+		hostinfo.logger(f.l).WithError(err).WithField("fwPacket", fwPacket).Error("Failed to decrypt packet")
 		return false
 	}
 
-	packetData := out[virtioNetHdrLen:]
+	err = newPacket(out, true, fwPacket)
-
-	err = newPacket(packetData, true, fwPacket)
 	if err != nil {
-		hostinfo.logger(f.l).WithError(err).WithField("packet", packetData).
+		hostinfo.logger(f.l).WithError(err).WithField("packet", out).
 			Warnf("Error while validating inbound packet")
 		return false
 	}
@@ -492,7 +492,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 	if dropReason != nil {
 		// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
 		// This gives us a buffer to build the reject packet in
-		f.rejectOutside(packetData, hostinfo.ConnectionState, hostinfo, nb, packet, q)
+		f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, nb, packet, q)
 		if f.l.Level >= logrus.DebugLevel {
 			hostinfo.logger(f.l).WithField("fwPacket", fwPacket).
 				WithField("reason", dropReason).
@@ -549,108 +549,3 @@ func (f *Interface) handleRecvError(addr netip.AddrPort, h *header.H) {
 	// We also delete it from pending hostmap to allow for fast reconnect.
 	f.handshakeManager.DeleteHostInfo(hostinfo)
 }
-
-// readOutsidePacketsBatch processes multiple packets received from UDP in a batch
-// and writes all successfully decrypted packets to TUN in a single operation
-func (f *Interface) readOutsidePacketsBatch(addrs []netip.AddrPort, payloads [][]byte, count int, outs [][]byte, nb []byte, q int, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, localCache firewall.ConntrackCache) {
-	// Pre-allocate slice for accumulating successful decryptions
-	tunPackets := make([][]byte, 0, count)
-
-	for i := 0; i < count; i++ {
-		payload := payloads[i]
-		addr := addrs[i]
-		out := outs[i]
-
-		// Parse header
-		err := h.Parse(payload)
-		if err != nil {
-			if len(payload) > 1 {
-				f.l.WithField("packet", payload).Infof("Error while parsing inbound packet from %s: %s", addr, err)
-			}
-			continue
-		}
-
-		if addr.IsValid() {
-			if f.myVpnNetworksTable.Contains(addr.Addr()) {
-				if f.l.Level >= logrus.DebugLevel {
-					f.l.WithField("udpAddr", addr).Debug("Refusing to process double encrypted packet")
-				}
-				continue
-			}
-		}
-
-		var hostinfo *HostInfo
-		if h.Type == header.Message && h.Subtype == header.MessageRelay {
-			hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex)
-		} else {
-			hostinfo = f.hostMap.QueryIndex(h.RemoteIndex)
-		}
-
-		var ci *ConnectionState
-		if hostinfo != nil {
-			ci = hostinfo.ConnectionState
-		}
-
-		switch h.Type {
-		case header.Message:
-			if !f.handleEncrypted(ci, addr, h) {
-				continue
-			}
-
-			switch h.Subtype {
-			case header.MessageNone:
-				// Decrypt packet
-				out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, payload[:header.Len], payload[header.Len:], h.MessageCounter, nb)
-				if err != nil {
-					hostinfo.logger(f.l).WithError(err).Error("Failed to decrypt packet")
-					continue
-				}
-
-				packetData := out[virtioNetHdrLen:]
-
-				err = newPacket(packetData, true, fwPacket)
-				if err != nil {
-					hostinfo.logger(f.l).WithError(err).WithField("packet", packetData).Warnf("Error while validating inbound packet")
-					continue
-				}
-
-				if !hostinfo.ConnectionState.window.Update(f.l, h.MessageCounter) {
-					hostinfo.logger(f.l).WithField("fwPacket", fwPacket).Debugln("dropping out of window packet")
-					continue
-				}
-
-				dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
-				if dropReason != nil {
-					f.rejectOutside(packetData, hostinfo.ConnectionState, hostinfo, nb, payload, q)
-					if f.l.Level >= logrus.DebugLevel {
-						hostinfo.logger(f.l).WithField("fwPacket", fwPacket).WithField("reason", dropReason).Debugln("dropping inbound packet")
-					}
-					continue
-				}
-
-				f.connectionManager.In(hostinfo)
-				// Add to batch for TUN write
-				tunPackets = append(tunPackets, out)
-
-			case header.MessageRelay:
-				// Skip relay packets in batch mode for now (less common path)
-				f.readOutsidePackets(addr, nil, out, payload, h, fwPacket, lhf, nb, q, localCache)
-
-			default:
-				hostinfo.logger(f.l).Debugf("unexpected message subtype %d", h.Subtype)
-			}
-
-		default:
-			// Handle non-Message types using single-packet path
-			f.readOutsidePackets(addr, nil, out, payload, h, fwPacket, lhf, nb, q, localCache)
-		}
-	}
-
-	if len(tunPackets) > 0 {
-		n, err := f.readers[q].WriteBatch(tunPackets, virtioNetHdrLen)
-		if err != nil {
-			f.l.WithError(err).WithField("sent", n).WithField("total", len(tunPackets)).Error("Failed to batch write to tun")
-		}
-		f.batchMetrics.tunWriteSize.Update(int64(len(tunPackets)))
-	}
-}

overlay/device.go

@@ -7,25 +7,11 @@ import (
 	"github.com/slackhq/nebula/routing"
 )
 
-// BatchReadWriter extends io.ReadWriteCloser with batch I/O operations
-type BatchReadWriter interface {
-	io.ReadWriteCloser
-
-	// BatchRead reads multiple packets at once
-	BatchRead(bufs [][]byte, sizes []int) (int, error)
-
-	// WriteBatch writes multiple packets at once
-	WriteBatch(bufs [][]byte, offset int) (int, error)
-
-	// BatchSize returns the optimal batch size for this device
-	BatchSize() int
-}
-
 type Device interface {
-	BatchReadWriter
+	io.ReadWriteCloser
 	Activate() error
 	Networks() []netip.Prefix
 	Name() string
 	RoutesFor(netip.Addr) routing.Gateways
-	NewMultiQueueReader() (BatchReadWriter, error)
+	NewMultiQueueReader() (io.ReadWriteCloser, error)
 }

overlay/tun.go

@@ -1,7 +1,6 @@
 package overlay
 
 import (
-	"fmt"
 	"net"
 	"net/netip"
 
@@ -11,7 +10,6 @@ import (
 )
 
 const DefaultMTU = 1300
-const VirtioNetHdrLen = 10 // Size of virtio_net_hdr structure
 
 // TODO: We may be able to remove routines
 type DeviceFactory func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error)
@@ -83,41 +81,3 @@ func prefixToMask(prefix netip.Prefix) netip.Addr {
 	addr, _ := netip.AddrFromSlice(net.CIDRMask(prefix.Bits(), pLen))
 	return addr
 }
-
-func flipBytes(b []byte) []byte {
-	for i := 0; i < len(b); i++ {
-		b[i] ^= 0xFF
-	}
-	return b
-}
-func orBytes(a []byte, b []byte) []byte {
-	ret := make([]byte, len(a))
-	for i := 0; i < len(a); i++ {
-		ret[i] = a[i] | b[i]
-	}
-	return ret
-}
-
-func getBroadcast(cidr netip.Prefix) netip.Addr {
-	broadcast, _ := netip.AddrFromSlice(
-		orBytes(
-			cidr.Addr().AsSlice(),
-			flipBytes(prefixToMask(cidr).AsSlice()),
-		),
-	)
-	return broadcast
-}
-
-func selectGateway(dest netip.Prefix, gateways []netip.Prefix) (netip.Prefix, error) {
-	for _, gateway := range gateways {
-		if dest.Addr().Is4() && gateway.Addr().Is4() {
-			return gateway, nil
-		}
-
-		if dest.Addr().Is6() && gateway.Addr().Is6() {
-			return gateway, nil
-		}
-	}
-
-	return netip.Prefix{}, fmt.Errorf("no gateway found for %v in the list of vpn networks", dest)
-}

overlay/tun_android.go

@@ -95,29 +95,6 @@ func (t *tun) Name() string {
 	return "android"
 }
 
-func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
 }
-
-func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	n, err := t.Read(bufs[0])
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	return 1, nil
-}
-
-func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	for i, buf := range bufs {
-		_, err := t.Write(buf[offset:])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(bufs), nil
-}
-
-func (t *tun) BatchSize() int {
-	return 1
-}

overlay/tun_darwin.go

@@ -294,6 +294,7 @@ func (t *tun) activate6(network netip.Prefix) error {
 			Vltime: 0xffffffff,
 			Pltime: 0xffffffff,
 		},
+		//TODO: CERT-V2 should we disable DAD (duplicate address detection) and mark this as a secured address?
 		Flags: _IN6_IFF_NODAD,
 	}
 
@@ -549,32 +550,6 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
 }
-
-// BatchRead reads a single packet (batch size 1 for non-Linux platforms)
-func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	n, err := t.Read(bufs[0])
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	return 1, nil
-}
-
-// WriteBatch writes packets individually (no batching for non-Linux platforms)
-func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	for i, buf := range bufs {
-		_, err := t.Write(buf[offset:])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(bufs), nil
-}
-
-// BatchSize returns 1 for non-Linux platforms (no batching)
-func (t *tun) BatchSize() int {
-	return 1
-}

overlay/tun_disabled.go

@@ -105,36 +105,10 @@ func (t *disabledTun) Write(b []byte) (int, error) {
 	return len(b), nil
 }
 
-func (t *disabledTun) NewMultiQueueReader() (BatchReadWriter, error) {
+func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return t, nil
 }
 
-// BatchRead reads a single packet (batch size 1 for disabled tun)
-func (t *disabledTun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	n, err := t.Read(bufs[0])
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	return 1, nil
-}
-
-// WriteBatch writes packets individually (no batching for disabled tun)
-func (t *disabledTun) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	for i, buf := range bufs {
-		_, err := t.Write(buf[offset:])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(bufs), nil
-}
-
-// BatchSize returns 1 for disabled tun (no batching)
-func (t *disabledTun) BatchSize() int {
-	return 1
-}
-
 func (t *disabledTun) Close() error {
 	if t.read != nil {
 		close(t.read)

overlay/tun_freebsd.go

@@ -450,36 +450,10 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
 }
 
-// BatchRead reads a single packet (batch size 1 for FreeBSD)
-func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	n, err := t.Read(bufs[0])
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	return 1, nil
-}
-
-// WriteBatch writes packets individually (no batching for FreeBSD)
-func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	for i, buf := range bufs {
-		_, err := t.Write(buf[offset:])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(bufs), nil
-}
-
-// BatchSize returns 1 for FreeBSD (no batching)
-func (t *tun) BatchSize() int {
-	return 1
-}
-
 func (t *tun) addRoutes(logErrors bool) error {
 	routes := *t.Routes.Load()
 	for _, r := range routes {
@@ -527,6 +501,30 @@ func (t *tun) deviceBytes() (o [16]byte) {
 	return
 }
 
+func flipBytes(b []byte) []byte {
+	for i := 0; i < len(b); i++ {
+		b[i] ^= 0xFF
+	}
+	return b
+}
+func orBytes(a []byte, b []byte) []byte {
+	ret := make([]byte, len(a))
+	for i := 0; i < len(a); i++ {
+		ret[i] = a[i] | b[i]
+	}
+	return ret
+}
+
+func getBroadcast(cidr netip.Prefix) netip.Addr {
+	broadcast, _ := netip.AddrFromSlice(
+		orBytes(
+			cidr.Addr().AsSlice(),
+			flipBytes(prefixToMask(cidr).AsSlice()),
+		),
+	)
+	return broadcast
+}
+
 func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
 	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
 	if err != nil {

overlay/tun_ios.go

@@ -151,29 +151,6 @@ func (t *tun) Name() string {
 	return "iOS"
 }
 
-func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
 }
-
-func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	n, err := t.Read(bufs[0])
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	return 1, nil
-}
-
-func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	for i, buf := range bufs {
-		_, err := t.Write(buf[offset:])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(bufs), nil
-}
-
-func (t *tun) BatchSize() int {
-	return 1
-}

overlay/tun_linux.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"net/netip"
 	"os"
+	"strings"
 	"sync/atomic"
 	"time"
 	"unsafe"
@@ -20,12 +21,10 @@ import (
 	"github.com/slackhq/nebula/util"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
-	wgtun "golang.zx2c4.com/wireguard/tun"
 )
 
 type tun struct {
 	io.ReadWriteCloser
-	wgDevice    wgtun.Device
 	fd          int
 	Device      string
 	vpnNetworks []netip.Prefix
@@ -66,154 +65,59 @@ type ifreqQLEN struct {
 	pad   [8]byte
 }
 
-// wgDeviceWrapper wraps a wireguard Device to implement io.ReadWriteCloser
-// This allows multiqueue readers to use the same wireguard Device batching as the main device
-type wgDeviceWrapper struct {
-	dev wgtun.Device
-	buf []byte // Reusable buffer for single packet reads
-}
-
-func (w *wgDeviceWrapper) Read(b []byte) (int, error) {
-	// Use wireguard Device's batch API for single packet
-	bufs := [][]byte{b}
-	sizes := make([]int, 1)
-	n, err := w.dev.Read(bufs, sizes, 0)
-	if err != nil {
-		return 0, err
-	}
-	if n == 0 {
-		return 0, io.EOF
-	}
-	return sizes[0], nil
-}
-
-func (w *wgDeviceWrapper) Write(b []byte) (int, error) {
-	// Buffer b should have virtio header space (10 bytes) at the beginning
-	// The decrypted packet data starts at offset 10
-	// Pass the full buffer to WireGuard with offset=virtioNetHdrLen
-	bufs := [][]byte{b}
-	n, err := w.dev.Write(bufs, VirtioNetHdrLen)
-	if err != nil {
-		return 0, err
-	}
-	if n == 0 {
-		return 0, io.ErrShortWrite
-	}
-	return len(b), nil
-}
-
-func (w *wgDeviceWrapper) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	// Pass all buffers to WireGuard's batch write
-	return w.dev.Write(bufs, offset)
-}
-
-func (w *wgDeviceWrapper) Close() error {
-	return w.dev.Close()
-}
-
-// BatchRead implements batching for multiqueue readers
-func (w *wgDeviceWrapper) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	// The zero here is offset.
-	return w.dev.Read(bufs, sizes, 0)
-}
-
-// BatchSize returns the optimal batch size
-func (w *wgDeviceWrapper) BatchSize() int {
-	return w.dev.BatchSize()
-}
-
 func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
-	wgDev, name, err := wgtun.CreateUnmonitoredTUNFromFD(deviceFd)
+	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
-	if err != nil {
-		return nil, fmt.Errorf("failed to create TUN from FD: %w", err)
-	}
 
-	file := wgDev.File()
 	t, err := newTunGeneric(c, l, file, vpnNetworks)
 	if err != nil {
-		_ = wgDev.Close()
 		return nil, err
 	}
 
-	t.wgDevice = wgDev
+	t.Device = "tun0"
-	t.Device = name
 
 	return t, nil
 }
 
 func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
-	// Check if /dev/net/tun exists, create if needed (for docker containers)
-	if _, err := os.Stat("/dev/net/tun"); os.IsNotExist(err) {
-		if err := os.MkdirAll("/dev/net", 0755); err != nil {
-			return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
-		}
-		if err := unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200))); err != nil {
-			return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err)
-		}
-	}
-
-	devName := c.GetString("tun.dev", "")
-	mtu := c.GetInt("tun.mtu", DefaultMTU)
-
-	// Create TUN device manually to support multiqueue
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
-		return nil, err
+		// If /dev/net/tun doesn't exist, try to create it (will happen in docker)
+		if os.IsNotExist(err) {
+			err = os.MkdirAll("/dev/net", 0755)
+			if err != nil {
+				return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
+			}
+			err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200)))
+			if err != nil {
+				return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err)
+			}
+
+			fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0)
+			if err != nil {
+				return nil, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
+			}
+		} else {
+			return nil, err
+		}
 	}
 
 	var req ifReq
-	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_VNET_HDR)
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
 	if multiqueue {
 		req.Flags |= unix.IFF_MULTI_QUEUE
 	}
-	copy(req.Name[:], devName)
+	copy(req.Name[:], c.GetString("tun.dev", ""))
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
-		unix.Close(fd)
 		return nil, err
 	}
-
+	name := strings.Trim(string(req.Name[:]), "\x00")
-	// Set nonblocking
-	if err = unix.SetNonblock(fd, true); err != nil {
-		unix.Close(fd)
-		return nil, err
-	}
-
-	// Enable TCP and UDP offload (TSO/GRO) for performance
-	// This allows the kernel to handle segmentation/coalescing
-	const (
-		tunTCPOffloads = unix.TUN_F_CSUM | unix.TUN_F_TSO4 | unix.TUN_F_TSO6
-		tunUDPOffloads = unix.TUN_F_USO4 | unix.TUN_F_USO6
-	)
-	offloads := tunTCPOffloads | tunUDPOffloads
-	if err = unix.IoctlSetInt(fd, unix.TUNSETOFFLOAD, offloads); err != nil {
-		// Log warning but don't fail - offload is optional
-		l.WithError(err).Warn("Failed to enable TUN offload (TSO/GRO), performance may be reduced")
-	}
 
 	file := os.NewFile(uintptr(fd), "/dev/net/tun")
-
-	// Create wireguard device from file descriptor
-	wgDev, err := wgtun.CreateTUNFromFile(file, mtu)
-	if err != nil {
-		file.Close()
-		return nil, fmt.Errorf("failed to create TUN from file: %w", err)
-	}
-
-	name, err := wgDev.Name()
-	if err != nil {
-		_ = wgDev.Close()
-		return nil, fmt.Errorf("failed to get TUN device name: %w", err)
-	}
-
-	// file is now owned by wgDev, get a new reference
-	file = wgDev.File()
 	t, err := newTunGeneric(c, l, file, vpnNetworks)
 	if err != nil {
-		_ = wgDev.Close()
 		return nil, err
 	}
 
-	t.wgDevice = wgDev
 	t.Device = name
 
 	return t, nil
@@ -312,44 +216,22 @@ func (t *tun) reload(c *config.C, initial bool) error {
 	return nil
 }
 
-func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
 	}
 
 	var req ifReq
-	// MUST match the flags used in newTun - includes IFF_VNET_HDR
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
-	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_VNET_HDR | unix.IFF_MULTI_QUEUE)
 	copy(req.Name[:], t.Device)
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
-		unix.Close(fd)
 		return nil, err
 	}
 
-	// Set nonblocking mode - CRITICAL for proper netpoller integration
-	if err = unix.SetNonblock(fd, true); err != nil {
-		unix.Close(fd)
-		return nil, err
-	}
-
-	// Get MTU from main device
-	mtu := t.MaxMTU
-	if mtu == 0 {
-		mtu = DefaultMTU
-	}
-
 	file := os.NewFile(uintptr(fd), "/dev/net/tun")
 
-	// Create wireguard Device from the file descriptor (just like the main device)
+	return file, nil
-	wgDev, err := wgtun.CreateTUNFromFile(file, mtu)
-	if err != nil {
-		file.Close()
-		return nil, fmt.Errorf("failed to create multiqueue TUN device: %w", err)
-	}
-
-	// Return a wrapper that uses the wireguard Device for all I/O
-	return &wgDeviceWrapper{dev: wgDev}, nil
 }
 
 func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
@@ -357,68 +239,7 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	return r
 }
 
-func (t *tun) Read(b []byte) (int, error) {
-	if t.wgDevice != nil {
-		// Use wireguard device which handles virtio headers internally
-		bufs := [][]byte{b}
-		sizes := make([]int, 1)
-		n, err := t.wgDevice.Read(bufs, sizes, 0)
-		if err != nil {
-			return 0, err
-		}
-		if n == 0 {
-			return 0, io.EOF
-		}
-		return sizes[0], nil
-	}
-
-	// Fallback: direct read from file (shouldn't happen in normal operation)
-	return t.ReadWriteCloser.Read(b)
-}
-
-// BatchRead reads multiple packets at once for improved performance
-// bufs: slice of buffers to read into
-// sizes: slice that will be filled with packet sizes
-// Returns number of packets read
-func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	if t.wgDevice != nil {
-		return t.wgDevice.Read(bufs, sizes, 0)
-	}
-
-	// Fallback: single packet read
-	n, err := t.ReadWriteCloser.Read(bufs[0])
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	return 1, nil
-}
-
-// BatchSize returns the optimal number of packets to read/write in a batch
-func (t *tun) BatchSize() int {
-	if t.wgDevice != nil {
-		return t.wgDevice.BatchSize()
-	}
-	return 1
-}
-
 func (t *tun) Write(b []byte) (int, error) {
-	if t.wgDevice != nil {
-		// Buffer b should have virtio header space (10 bytes) at the beginning
-		// The decrypted packet data starts at offset 10
-		// Pass the full buffer to WireGuard with offset=virtioNetHdrLen
-		bufs := [][]byte{b}
-		n, err := t.wgDevice.Write(bufs, VirtioNetHdrLen)
-		if err != nil {
-			return 0, err
-		}
-		if n == 0 {
-			return 0, io.ErrShortWrite
-		}
-		return len(b), nil
-	}
-
-	// Fallback: direct write (shouldn't happen in normal operation)
 	var nn int
 	maximum := len(b)
 
@@ -441,22 +262,6 @@ func (t *tun) Write(b []byte) (int, error) {
 	}
 }
 
-// WriteBatch writes multiple packets to the TUN device in a single syscall
-func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	if t.wgDevice != nil {
-		return t.wgDevice.Write(bufs, offset)
-	}
-
-	// Fallback: write individually (shouldn't happen in normal operation)
-	for i, buf := range bufs {
-		_, err := t.Write(buf)
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(bufs), nil
-}
-
 func (t *tun) deviceBytes() (o [16]byte) {
 	for i, c := range t.Device {
 		o[i] = byte(c)
@@ -869,10 +674,6 @@ func (t *tun) Close() error {
 		close(t.routeChan)
 	}
 
-	if t.wgDevice != nil {
-		_ = t.wgDevice.Close()
-	}
-
 	if t.ReadWriteCloser != nil {
 		_ = t.ReadWriteCloser.Close()
 	}

overlay/tun_netbsd.go

@@ -4,12 +4,13 @@
 package overlay
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"net/netip"
 	"os"
+	"os/exec"
 	"regexp"
+	"strconv"
 	"sync/atomic"
 	"syscall"
 	"unsafe"
@@ -19,42 +20,11 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	netroute "golang.org/x/net/route"
-	"golang.org/x/sys/unix"
 )
 
-const (
+type ifreqDestroy struct {
-	SIOCAIFADDR_IN6 = 0x8080696b
+	Name [16]byte
-	TUNSIFHEAD      = 0x80047442
+	pad  [16]byte
-	TUNSIFMODE      = 0x80047458
-)
-
-type ifreqAlias4 struct {
-	Name     [unix.IFNAMSIZ]byte
-	Addr     unix.RawSockaddrInet4
-	DstAddr  unix.RawSockaddrInet4
-	MaskAddr unix.RawSockaddrInet4
-}
-
-type ifreqAlias6 struct {
-	Name       [unix.IFNAMSIZ]byte
-	Addr       unix.RawSockaddrInet6
-	DstAddr    unix.RawSockaddrInet6
-	PrefixMask unix.RawSockaddrInet6
-	Flags      uint32
-	Lifetime   addrLifetime
-}
-
-type ifreq struct {
-	Name [unix.IFNAMSIZ]byte
-	data int
-}
-
-type addrLifetime struct {
-	Expire    uint64
-	Preferred uint64
-	Vltime    uint32
-	Pltime    uint32
 }
 
 type tun struct {
@@ -64,18 +34,40 @@ type tun struct {
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	l           *logrus.Logger
-	f           *os.File
+
-	fd          int
+	io.ReadWriteCloser
 }
 
-var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+func (t *tun) Close() error {
+	if t.ReadWriteCloser != nil {
+		if err := t.ReadWriteCloser.Close(); err != nil {
+			return err
+		}
+
+		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+		if err != nil {
+			return err
+		}
+		defer syscall.Close(s)
+
+		ifreq := ifreqDestroy{Name: t.deviceBytes()}
+
+		err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
+
+		return err
+	}
+	return nil
+}
 
 func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in NetBSD")
 }
 
+var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+
 func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	// Try to open tun device
+	var file *os.File
 	var err error
 	deviceName := c.GetString("tun.dev", "")
 	if deviceName == "" {
@@ -85,23 +77,17 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
 	}
 
-	fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
+	file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
 	}
 
-	err = unix.SetNonblock(fd, true)
-	if err != nil {
-		l.WithError(err).Warn("Failed to set the tun device as nonblocking")
-	}
-
 	t := &tun{
-		f:           os.NewFile(uintptr(fd), ""),
+		ReadWriteCloser: file,
-		fd:          fd,
+		Device:          deviceName,
-		Device:      deviceName,
+		vpnNetworks:     vpnNetworks,
-		vpnNetworks: vpnNetworks,
+		MTU:             c.GetInt("tun.mtu", DefaultMTU),
-		MTU:         c.GetInt("tun.mtu", DefaultMTU),
+		l:               l,
-		l:           l,
 	}
 
 	err = t.reload(c, true)
@@ -119,225 +105,40 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	return t, nil
 }
 
-func (t *tun) Close() error {
-	if t.f != nil {
-		if err := t.f.Close(); err != nil {
-			return fmt.Errorf("error closing tun file: %w", err)
-		}
-
-		// t.f.Close should have handled it for us but let's be extra sure
-		_ = unix.Close(t.fd)
-
-		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-
-		ifr := ifreq{Name: t.deviceBytes()}
-		err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifr)))
-		return err
-	}
-	return nil
-}
-
-func (t *tun) Read(to []byte) (int, error) {
-	rc, err := t.f.SyscallConn()
-	if err != nil {
-		return 0, fmt.Errorf("failed to get syscall conn for tun: %w", err)
-	}
-
-	var errno syscall.Errno
-	var n uintptr
-	err = rc.Read(func(fd uintptr) bool {
-		// first 4 bytes is protocol family, in network byte order
-		head := [4]byte{}
-		iovecs := []syscall.Iovec{
-			{&head[0], 4},
-			{&to[0], uint64(len(to))},
-		}
-
-		n, _, errno = syscall.Syscall(syscall.SYS_READV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-		if errno.Temporary() {
-			// We got an EAGAIN, EINTR, or EWOULDBLOCK, go again
-			return false
-		}
-		return true
-	})
-	if err != nil {
-		if err == syscall.EBADF || err.Error() == "use of closed file" {
-			// Go doesn't export poll.ErrFileClosing but happily reports it to us so here we are
-			// https://github.com/golang/go/blob/master/src/internal/poll/fd_poll_runtime.go#L121
-			return 0, os.ErrClosed
-		}
-		return 0, fmt.Errorf("failed to make read call for tun: %w", err)
-	}
-
-	if errno != 0 {
-		return 0, fmt.Errorf("failed to make inner read call for tun: %w", errno)
-	}
-
-	// fix bytes read number to exclude header
-	bytesRead := int(n)
-	if bytesRead < 0 {
-		return bytesRead, nil
-	} else if bytesRead < 4 {
-		return 0, nil
-	} else {
-		return bytesRead - 4, nil
-	}
-}
-
-// Write is only valid for single threaded use
-func (t *tun) Write(from []byte) (int, error) {
-	if len(from) <= 1 {
-		return 0, syscall.EIO
-	}
-
-	ipVer := from[0] >> 4
-	var head [4]byte
-	// first 4 bytes is protocol family, in network byte order
-	if ipVer == 4 {
-		head[3] = syscall.AF_INET
-	} else if ipVer == 6 {
-		head[3] = syscall.AF_INET6
-	} else {
-		return 0, fmt.Errorf("unable to determine IP version from packet")
-	}
-
-	rc, err := t.f.SyscallConn()
-	if err != nil {
-		return 0, err
-	}
-
-	var errno syscall.Errno
-	var n uintptr
-	err = rc.Write(func(fd uintptr) bool {
-		iovecs := []syscall.Iovec{
-			{&head[0], 4},
-			{&from[0], uint64(len(from))},
-		}
-
-		n, _, errno = syscall.Syscall(syscall.SYS_WRITEV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-		// According to NetBSD documentation for TUN, writes will only return errors in which
-		// this packet will never be delivered so just go on living life.
-		return true
-	})
-	if err != nil {
-		return 0, err
-	}
-
-	if errno != 0 {
-		return 0, errno
-	}
-
-	return int(n) - 4, err
-}
-
 func (t *tun) addIp(cidr netip.Prefix) error {
-	if cidr.Addr().Is4() {
+	var err error
-		var req ifreqAlias4
-		req.Name = t.deviceBytes()
-		req.Addr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   cidr.Addr().As4(),
-		}
-		req.DstAddr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   cidr.Addr().As4(),
-		}
-		req.MaskAddr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   prefixToMask(cidr).As4(),
-		}
 
-		s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+	// TODO use syscalls instead of exec.Command
-		if err != nil {
+	cmd := exec.Command("/sbin/ifconfig", t.Device, cidr.String(), cidr.Addr().String())
-			return err
+	t.l.Debug("command: ", cmd.String())
-		}
+	if err = cmd.Run(); err != nil {
-		defer syscall.Close(s)
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-
-		if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&req))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr(), err)
-		}
-
-		return nil
 	}
 
-	if cidr.Addr().Is6() {
+	cmd = exec.Command("/sbin/route", "-n", "add", "-net", cidr.String(), cidr.Addr().String())
-		var req ifreqAlias6
+	t.l.Debug("command: ", cmd.String())
-		req.Name = t.deviceBytes()
+	if err = cmd.Run(); err != nil {
-		req.Addr = unix.RawSockaddrInet6{
+		return fmt.Errorf("failed to run 'route add': %s", err)
-			Len:    unix.SizeofSockaddrInet6,
-			Family: unix.AF_INET6,
-			Addr:   cidr.Addr().As16(),
-		}
-		req.PrefixMask = unix.RawSockaddrInet6{
-			Len:    unix.SizeofSockaddrInet6,
-			Family: unix.AF_INET6,
-			Addr:   prefixToMask(cidr).As16(),
-		}
-		req.Lifetime = addrLifetime{
-			Vltime: 0xffffffff,
-			Pltime: 0xffffffff,
-		}
-
-		s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-
-		if err := ioctl(uintptr(s), SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&req))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err)
-		}
-		return nil
 	}
 
-	return fmt.Errorf("unknown address type %v", cidr)
+	cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU))
-}
+	t.l.Debug("command: ", cmd.String())
-
+	if err = cmd.Run(); err != nil {
-func (t *tun) Activate() error {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-	mode := int32(unix.IFF_BROADCAST)
-	err := ioctl(uintptr(t.fd), TUNSIFMODE, uintptr(unsafe.Pointer(&mode)))
-	if err != nil {
-		return fmt.Errorf("failed to set tun device mode: %w", err)
-	}
-
-	v := 1
-	err = ioctl(uintptr(t.fd), TUNSIFHEAD, uintptr(unsafe.Pointer(&v)))
-	if err != nil {
-		return fmt.Errorf("failed to set tun device head: %w", err)
-	}
-
-	err = t.doIoctlByName(unix.SIOCSIFMTU, uint32(t.MTU))
-	if err != nil {
-		return fmt.Errorf("failed to set tun mtu: %w", err)
-	}
-
-	for i := range t.vpnNetworks {
-		err = t.addIp(t.vpnNetworks[i])
-		if err != nil {
-			return err
-		}
 	}
 
+	// Unsafe path routes
 	return t.addRoutes(false)
 }
 
-func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
+func (t *tun) Activate() error {
-	s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+	for i := range t.vpnNetworks {
-	if err != nil {
+		err := t.addIp(t.vpnNetworks[i])
-		return err
+		if err != nil {
+			return err
+		}
 	}
-	defer syscall.Close(s)
+	return nil
-
-	ir := ifreq{Name: t.deviceBytes(), data: int(value)}
-	err = ioctl(uintptr(s), ctl, uintptr(unsafe.Pointer(&ir)))
-	return err
 }
 
 func (t *tun) reload(c *config.C, initial bool) error {
@@ -390,52 +191,27 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd")
 }
 
-func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	n, err := t.Read(bufs[0])
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	return 1, nil
-}
-
-func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	for i, buf := range bufs {
-		_, err := t.Write(buf[offset:])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(bufs), nil
-}
-
-func (t *tun) BatchSize() int {
-	return 1
-}
-
 func (t *tun) addRoutes(logErrors bool) error {
 	routes := *t.Routes.Load()
-
 	for _, r := range routes {
 		if len(r.Via) == 0 || !r.Install {
 			// We don't allow route MTUs so only install routes with a via
 			continue
 		}
 
-		err := addRoute(r.Cidr, t.vpnNetworks)
+		cmd := exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
-		if err != nil {
+		t.l.Debug("command: ", cmd.String())
-			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
+		if err := cmd.Run(); err != nil {
+			retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
 			} else {
 				return retErr
 			}
-		} else {
-			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -448,8 +224,10 @@ func (t *tun) removeRoutes(routes []Route) error {
 			continue
 		}
 
-		err := delRoute(r.Cidr, t.vpnNetworks)
+		//TODO: CERT-V2 is this right?
-		if err != nil {
+		cmd := exec.Command("/sbin/route", "-n", "delete", "-net", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
+		t.l.Debug("command: ", cmd.String())
+		if err := cmd.Run(); err != nil {
 			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
 			t.l.WithField("route", r).Info("Removed route")
@@ -464,109 +242,3 @@ func (t *tun) deviceBytes() (o [16]byte) {
 	}
 	return
 }
-
-func addRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
-	if err != nil {
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
-
-	route := &netroute.RouteMessage{
-		Version: unix.RTM_VERSION,
-		Type:    unix.RTM_ADD,
-		Flags:   unix.RTF_UP | unix.RTF_GATEWAY,
-		Seq:     1,
-	}
-
-	if prefix.Addr().Is4() {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()},
-		}
-	} else {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()},
-		}
-	}
-
-	data, err := route.Marshal()
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
-
-	_, err = unix.Write(sock, data[:])
-	if err != nil {
-		if errors.Is(err, unix.EEXIST) {
-			// Try to do a change
-			route.Type = unix.RTM_CHANGE
-			data, err = route.Marshal()
-			if err != nil {
-				return fmt.Errorf("failed to create route.RouteMessage for change: %w", err)
-			}
-			_, err = unix.Write(sock, data[:])
-			return err
-		}
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
-}
-
-func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
-	if err != nil {
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
-
-	route := netroute.RouteMessage{
-		Version: unix.RTM_VERSION,
-		Type:    unix.RTM_DELETE,
-		Seq:     1,
-	}
-
-	if prefix.Addr().Is4() {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()},
-		}
-	} else {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()},
-		}
-	}
-
-	data, err := route.Marshal()
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
-	_, err = unix.Write(sock, data[:])
-	if err != nil {
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
-}

overlay/tun_openbsd.go

@@ -4,50 +4,23 @@
 package overlay
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"net/netip"
 	"os"
+	"os/exec"
 	"regexp"
+	"strconv"
 	"sync/atomic"
 	"syscall"
-	"unsafe"
 
 	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	netroute "golang.org/x/net/route"
-	"golang.org/x/sys/unix"
 )
 
-const (
-	SIOCAIFADDR_IN6 = 0x8080691a
-)
-
-type ifreqAlias4 struct {
-	Name     [unix.IFNAMSIZ]byte
-	Addr     unix.RawSockaddrInet4
-	DstAddr  unix.RawSockaddrInet4
-	MaskAddr unix.RawSockaddrInet4
-}
-
-type ifreqAlias6 struct {
-	Name       [unix.IFNAMSIZ]byte
-	Addr       unix.RawSockaddrInet6
-	DstAddr    unix.RawSockaddrInet6
-	PrefixMask unix.RawSockaddrInet6
-	Flags      uint32
-	Lifetime   [2]uint32
-}
-
-type ifreq struct {
-	Name [unix.IFNAMSIZ]byte
-	data int
-}
-
 type tun struct {
 	Device      string
 	vpnNetworks []netip.Prefix
@@ -55,46 +28,48 @@ type tun struct {
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	l           *logrus.Logger
-	f           *os.File
+
-	fd          int
+	io.ReadWriteCloser
+
 	// cache out buffer since we need to prepend 4 bytes for tun metadata
 	out []byte
 }
 
-var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+func (t *tun) Close() error {
+	if t.ReadWriteCloser != nil {
+		return t.ReadWriteCloser.Close()
+	}
 
-func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
+	return nil
-	return nil, fmt.Errorf("newTunFromFd not supported in openbsd")
 }
 
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
+	return nil, fmt.Errorf("newTunFromFd not supported in OpenBSD")
+}
+
+var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+
 func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
-	// Try to open tun device
-	var err error
 	deviceName := c.GetString("tun.dev", "")
 	if deviceName == "" {
-		return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
+		return nil, fmt.Errorf("a device name in the format of tunN must be specified")
-	}
-	if !deviceNameRE.MatchString(deviceName) {
-		return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
 	}
 
-	fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
+	if !deviceNameRE.MatchString(deviceName) {
+		return nil, fmt.Errorf("a device name in the format of tunN must be specified")
+	}
+
+	file, err := os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
 	}
 
-	err = unix.SetNonblock(fd, true)
-	if err != nil {
-		l.WithError(err).Warn("Failed to set the tun device as nonblocking")
-	}
-
 	t := &tun{
-		f:           os.NewFile(uintptr(fd), ""),
+		ReadWriteCloser: file,
-		fd:          fd,
+		Device:          deviceName,
-		Device:      deviceName,
+		vpnNetworks:     vpnNetworks,
-		vpnNetworks: vpnNetworks,
+		MTU:             c.GetInt("tun.mtu", DefaultMTU),
-		MTU:         c.GetInt("tun.mtu", DefaultMTU),
+		l:               l,
-		l:           l,
 	}
 
 	err = t.reload(c, true)
@@ -112,154 +87,6 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	return t, nil
 }
 
-func (t *tun) Close() error {
-	if t.f != nil {
-		if err := t.f.Close(); err != nil {
-			return fmt.Errorf("error closing tun file: %w", err)
-		}
-
-		// t.f.Close should have handled it for us but let's be extra sure
-		_ = unix.Close(t.fd)
-	}
-	return nil
-}
-
-func (t *tun) Read(to []byte) (int, error) {
-	buf := make([]byte, len(to)+4)
-
-	n, err := t.f.Read(buf)
-
-	copy(to, buf[4:])
-	return n - 4, err
-}
-
-// Write is only valid for single threaded use
-func (t *tun) Write(from []byte) (int, error) {
-	buf := t.out
-	if cap(buf) < len(from)+4 {
-		buf = make([]byte, len(from)+4)
-		t.out = buf
-	}
-	buf = buf[:len(from)+4]
-
-	if len(from) == 0 {
-		return 0, syscall.EIO
-	}
-
-	// Determine the IP Family for the NULL L2 Header
-	ipVer := from[0] >> 4
-	if ipVer == 4 {
-		buf[3] = syscall.AF_INET
-	} else if ipVer == 6 {
-		buf[3] = syscall.AF_INET6
-	} else {
-		return 0, fmt.Errorf("unable to determine IP version from packet")
-	}
-
-	copy(buf[4:], from)
-
-	n, err := t.f.Write(buf)
-	return n - 4, err
-}
-
-func (t *tun) addIp(cidr netip.Prefix) error {
-	if cidr.Addr().Is4() {
-		var req ifreqAlias4
-		req.Name = t.deviceBytes()
-		req.Addr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   cidr.Addr().As4(),
-		}
-		req.DstAddr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   cidr.Addr().As4(),
-		}
-		req.MaskAddr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   prefixToMask(cidr).As4(),
-		}
-
-		s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-
-		if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&req))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr(), err)
-		}
-
-		err = addRoute(cidr, t.vpnNetworks)
-		if err != nil {
-			return fmt.Errorf("failed to set route for vpn network %v: %w", cidr, err)
-		}
-
-		return nil
-	}
-
-	if cidr.Addr().Is6() {
-		var req ifreqAlias6
-		req.Name = t.deviceBytes()
-		req.Addr = unix.RawSockaddrInet6{
-			Len:    unix.SizeofSockaddrInet6,
-			Family: unix.AF_INET6,
-			Addr:   cidr.Addr().As16(),
-		}
-		req.PrefixMask = unix.RawSockaddrInet6{
-			Len:    unix.SizeofSockaddrInet6,
-			Family: unix.AF_INET6,
-			Addr:   prefixToMask(cidr).As16(),
-		}
-		req.Lifetime[0] = 0xffffffff
-		req.Lifetime[1] = 0xffffffff
-
-		s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-
-		if err := ioctl(uintptr(s), SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&req))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err)
-		}
-
-		return nil
-	}
-
-	return fmt.Errorf("unknown address type %v", cidr)
-}
-
-func (t *tun) Activate() error {
-	err := t.doIoctlByName(unix.SIOCSIFMTU, uint32(t.MTU))
-	if err != nil {
-		return fmt.Errorf("failed to set tun mtu: %w", err)
-	}
-
-	for i := range t.vpnNetworks {
-		err = t.addIp(t.vpnNetworks[i])
-		if err != nil {
-			return err
-		}
-	}
-
-	return t.addRoutes(false)
-}
-
-func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
-	s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-	if err != nil {
-		return err
-	}
-	defer syscall.Close(s)
-
-	ir := ifreq{Name: t.deviceBytes(), data: int(value)}
-	err = ioctl(uintptr(s), ctl, uintptr(unsafe.Pointer(&ir)))
-	return err
-}
-
 func (t *tun) reload(c *config.C, initial bool) error {
 	change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
 	if err != nil {
@@ -297,65 +124,63 @@ func (t *tun) reload(c *config.C, initial bool) error {
 	return nil
 }
 
+func (t *tun) addIp(cidr netip.Prefix) error {
+	var err error
+	// TODO use syscalls instead of exec.Command
+	cmd := exec.Command("/sbin/ifconfig", t.Device, cidr.String(), cidr.Addr().String())
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
+	}
+
+	cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU))
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
+	}
+
+	cmd = exec.Command("/sbin/route", "-n", "add", "-inet", cidr.String(), cidr.Addr().String())
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run 'route add': %s", err)
+	}
+
+	// Unsafe path routes
+	return t.addRoutes(false)
+}
+
+func (t *tun) Activate() error {
+	for i := range t.vpnNetworks {
+		err := t.addIp(t.vpnNetworks[i])
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	r, _ := t.routeTree.Load().Lookup(ip)
 	return r
 }
 
-func (t *tun) Networks() []netip.Prefix {
-	return t.vpnNetworks
-}
-
-func (t *tun) Name() string {
-	return t.Device
-}
-
-func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd")
-}
-
-func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	n, err := t.Read(bufs[0])
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	return 1, nil
-}
-
-func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	for i, buf := range bufs {
-		_, err := t.Write(buf[offset:])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(bufs), nil
-}
-
-func (t *tun) BatchSize() int {
-	return 1
-}
-
 func (t *tun) addRoutes(logErrors bool) error {
 	routes := *t.Routes.Load()
-
 	for _, r := range routes {
 		if len(r.Via) == 0 || !r.Install {
 			// We don't allow route MTUs so only install routes with a via
 			continue
 		}
-
+		//TODO: CERT-V2 is this right?
-		err := addRoute(r.Cidr, t.vpnNetworks)
+		cmd := exec.Command("/sbin/route", "-n", "add", "-inet", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
-		if err != nil {
+		t.l.Debug("command: ", cmd.String())
-			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
+		if err := cmd.Run(); err != nil {
+			retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
 			} else {
 				return retErr
 			}
-		} else {
-			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -367,9 +192,10 @@ func (t *tun) removeRoutes(routes []Route) error {
 		if !r.Install {
 			continue
 		}
-
+		//TODO: CERT-V2 is this right?
-		err := delRoute(r.Cidr, t.vpnNetworks)
+		cmd := exec.Command("/sbin/route", "-n", "delete", "-inet", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
-		if err != nil {
+		t.l.Debug("command: ", cmd.String())
+		if err := cmd.Run(); err != nil {
 			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
 			t.l.WithField("route", r).Info("Removed route")
@@ -378,115 +204,52 @@ func (t *tun) removeRoutes(routes []Route) error {
 	return nil
 }
 
-func (t *tun) deviceBytes() (o [16]byte) {
+func (t *tun) Networks() []netip.Prefix {
-	for i, c := range t.Device {
+	return t.vpnNetworks
-		o[i] = byte(c)
-	}
-	return
 }
 
-func addRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
+func (t *tun) Name() string {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	return t.Device
-	if err != nil {
+}
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
 
-	route := &netroute.RouteMessage{
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-		Version: unix.RTM_VERSION,
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
-		Type:    unix.RTM_ADD,
+}
-		Flags:   unix.RTF_UP | unix.RTF_GATEWAY,
+
-		Seq:     1,
+func (t *tun) Read(to []byte) (int, error) {
+	buf := make([]byte, len(to)+4)
+
+	n, err := t.ReadWriteCloser.Read(buf)
+
+	copy(to, buf[4:])
+	return n - 4, err
+}
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	buf := t.out
+	if cap(buf) < len(from)+4 {
+		buf = make([]byte, len(from)+4)
+		t.out = buf
+	}
+	buf = buf[:len(from)+4]
+
+	if len(from) == 0 {
+		return 0, syscall.EIO
 	}
 
-	if prefix.Addr().Is4() {
+	// Determine the IP Family for the NULL L2 Header
-		gw, err := selectGateway(prefix, gateways)
+	ipVer := from[0] >> 4
-		if err != nil {
+	if ipVer == 4 {
-			return err
+		buf[3] = syscall.AF_INET
-		}
+	} else if ipVer == 6 {
-		route.Addrs = []netroute.Addr{
+		buf[3] = syscall.AF_INET6
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()},
-		}
 	} else {
-		gw, err := selectGateway(prefix, gateways)
+		return 0, fmt.Errorf("unable to determine IP version from packet")
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()},
-		}
 	}
 
-	data, err := route.Marshal()
+	copy(buf[4:], from)
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
 
-	_, err = unix.Write(sock, data[:])
+	n, err := t.ReadWriteCloser.Write(buf)
-	if err != nil {
+	return n - 4, err
-		if errors.Is(err, unix.EEXIST) {
-			// Try to do a change
-			route.Type = unix.RTM_CHANGE
-			data, err = route.Marshal()
-			if err != nil {
-				return fmt.Errorf("failed to create route.RouteMessage for change: %w", err)
-			}
-			_, err = unix.Write(sock, data[:])
-			return err
-		}
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
-}
-
-func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
-	if err != nil {
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
-
-	route := netroute.RouteMessage{
-		Version: unix.RTM_VERSION,
-		Type:    unix.RTM_DELETE,
-		Seq:     1,
-	}
-
-	if prefix.Addr().Is4() {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()},
-		}
-	} else {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()},
-		}
-	}
-
-	data, err := route.Marshal()
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
-	_, err = unix.Write(sock, data[:])
-	if err != nil {
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
 }

overlay/tun_tester.go

@@ -132,29 +132,6 @@ func (t *TestTun) Read(b []byte) (int, error) {
 	return len(p), nil
 }
 
-func (t *TestTun) NewMultiQueueReader() (BatchReadWriter, error) {
+func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented")
 }
-
-func (t *TestTun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	n, err := t.Read(bufs[0])
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	return 1, nil
-}
-
-func (t *TestTun) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	for i, buf := range bufs {
-		_, err := t.Write(buf[offset:])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(bufs), nil
-}
-
-func (t *TestTun) BatchSize() int {
-	return 1
-}

overlay/tun_windows.go

@@ -6,6 +6,7 @@ package overlay
 import (
 	"crypto"
 	"fmt"
+	"io"
 	"net/netip"
 	"os"
 	"path/filepath"
@@ -233,36 +234,10 @@ func (t *winTun) Write(b []byte) (int, error) {
 	return t.tun.Write(b, 0)
 }
 
-func (t *winTun) NewMultiQueueReader() (BatchReadWriter, error) {
+func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
 }
 
-// BatchRead reads a single packet (batch size 1 for Windows)
-func (t *winTun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	n, err := t.Read(bufs[0])
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	return 1, nil
-}
-
-// WriteBatch writes packets individually (no batching for Windows)
-func (t *winTun) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	for i, buf := range bufs {
-		_, err := t.Write(buf[offset:])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(bufs), nil
-}
-
-// BatchSize returns 1 for Windows (no batching)
-func (t *winTun) BatchSize() int {
-	return 1
-}
-
 func (t *winTun) Close() error {
 	// It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes,
 	// so to be certain, just remove everything before destroying.

overlay/user.go

@@ -46,36 +46,10 @@ func (d *UserDevice) RoutesFor(ip netip.Addr) routing.Gateways {
 	return routing.Gateways{routing.NewGateway(ip, 1)}
 }
 
-func (d *UserDevice) NewMultiQueueReader() (BatchReadWriter, error) {
+func (d *UserDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return d, nil
 }
 
-// BatchRead reads a single packet (batch size 1 for UserDevice)
-func (d *UserDevice) BatchRead(bufs [][]byte, sizes []int) (int, error) {
-	n, err := d.Read(bufs[0])
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	return 1, nil
-}
-
-// WriteBatch writes packets individually (no batching for UserDevice)
-func (d *UserDevice) WriteBatch(bufs [][]byte, offset int) (int, error) {
-	for i, buf := range bufs {
-		_, err := d.Write(buf[offset:])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(bufs), nil
-}
-
-// BatchSize returns 1 for UserDevice (no batching)
-func (d *UserDevice) BatchSize() int {
-	return 1
-}
-
 func (d *UserDevice) Pipe() (*io.PipeReader, *io.PipeWriter) {
 	return d.inboundReader, d.outboundWriter
 }

packet/packet.go

@@ -0,0 +1,12 @@
+package packet
+
+import "net/netip"
+
+type Packet struct {
+	Payload []byte
+	Addr    netip.AddrPort
+}
+
+func New() *Packet {
+	return &Packet{Payload: make([]byte, 9001)}
+}

pki.go

@@ -100,62 +100,55 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
 		currentState := p.cs.Load()
 		if newState.v1Cert != nil {
 			if currentState.v1Cert == nil {
-				//adding certs is fine, actually. Networks-in-common confirmed in newCertState().
+				return util.NewContextualError("v1 certificate was added, restart required", nil, err)
-			} else {
-				// did IP in cert change? if so, don't set
-				if !slices.Equal(currentState.v1Cert.Networks(), newState.v1Cert.Networks()) {
-					return util.NewContextualError(
-						"Networks in new cert was different from old",
-						m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks(), "cert_version": cert.Version1},
-						nil,
-					)
-				}
-
-				if currentState.v1Cert.Curve() != newState.v1Cert.Curve() {
-					return util.NewContextualError(
-						"Curve in new v1 cert was different from old",
-						m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve(), "cert_version": cert.Version1},
-						nil,
-					)
-				}
 			}
+
+			// did IP in cert change? if so, don't set
+			if !slices.Equal(currentState.v1Cert.Networks(), newState.v1Cert.Networks()) {
+				return util.NewContextualError(
+					"Networks in new cert was different from old",
+					m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks()},
+					nil,
+				)
+			}
+
+			if currentState.v1Cert.Curve() != newState.v1Cert.Curve() {
+				return util.NewContextualError(
+					"Curve in new cert was different from old",
+					m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve()},
+					nil,
+				)
+			}
+
+		} else if currentState.v1Cert != nil {
+			//TODO: CERT-V2 we should be able to tear this down
+			return util.NewContextualError("v1 certificate was removed, restart required", nil, err)
 		}
 
 		if newState.v2Cert != nil {
 			if currentState.v2Cert == nil {
-				//adding certs is fine, actually
+				return util.NewContextualError("v2 certificate was added, restart required", nil, err)
-			} else {
-				// did IP in cert change? if so, don't set
-				if !slices.Equal(currentState.v2Cert.Networks(), newState.v2Cert.Networks()) {
-					return util.NewContextualError(
-						"Networks in new cert was different from old",
-						m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks(), "cert_version": cert.Version2},
-						nil,
-					)
-				}
-
-				if currentState.v2Cert.Curve() != newState.v2Cert.Curve() {
-					return util.NewContextualError(
-						"Curve in new cert was different from old",
-						m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve(), "cert_version": cert.Version2},
-						nil,
-					)
-				}
 			}
 
-		} else if currentState.v2Cert != nil {
+			// did IP in cert change? if so, don't set
-			//newState.v1Cert is non-nil bc empty certstates aren't permitted
+			if !slices.Equal(currentState.v2Cert.Networks(), newState.v2Cert.Networks()) {
-			if newState.v1Cert == nil {
-				return util.NewContextualError("v1 and v2 certs are nil, this should be impossible", nil, err)
-			}
-			//if we're going to v1-only, we need to make sure we didn't orphan any v2-cert vpnaddrs
-			if !slices.Equal(currentState.v2Cert.Networks(), newState.v1Cert.Networks()) {
 				return util.NewContextualError(
-					"Removing a V2 cert is not permitted unless it has identical networks to the new V1 cert",
+					"Networks in new cert was different from old",
-					m{"new_v1_networks": newState.v1Cert.Networks(), "old_v2_networks": currentState.v2Cert.Networks()},
+					m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks()},
 					nil,
 				)
 			}
+
+			if currentState.v2Cert.Curve() != newState.v2Cert.Curve() {
+				return util.NewContextualError(
+					"Curve in new cert was different from old",
+					m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve()},
+					nil,
+				)
+			}
+
+		} else if currentState.v2Cert != nil {
+			return util.NewContextualError("v2 certificate was removed, restart required", nil, err)
 		}
 
 		// Cipher cant be hot swapped so just leave it at what it was before

service/service.go

@@ -44,7 +44,10 @@ type Service struct {
 }
 
 func New(control *nebula.Control) (*Service, error) {
-	control.Start()
+	wait, err := control.Start()
+	if err != nil {
+		return nil, err
+	}
 
 	ctx := control.Context()
 	eg, ctx := errgroup.WithContext(ctx)
@@ -141,6 +144,12 @@ func New(control *nebula.Control) (*Service, error) {
 		}
 	})
 
+	// Add the nebula wait function to the group
+	eg.Go(func() error {
+		wait()
+		return nil
+	})
+
 	return &s, nil
 }
 

stats.go

@@ -6,7 +6,6 @@ import (
 	"log"
 	"net"
 	"net/http"
-	_ "net/http/pprof"
 	"runtime"
 	"strconv"
 	"time"

udp/conn.go

@@ -13,24 +13,21 @@ type EncReader func(
 	payload []byte,
 )
 
-type EncBatchReader func(
-	addrs []netip.AddrPort,
-	payloads [][]byte,
-	count int,
-)
-
 type Conn interface {
 	Rebind() error
 	LocalAddr() (netip.AddrPort, error)
-	ListenOut(r EncReader)
+	ListenOut(r EncReader) error
-	ListenOutBatch(r EncBatchReader)
 	WriteTo(b []byte, addr netip.AddrPort) error
-	WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error)
+	WriteBatch(pkts []BatchPacket) (int, error)
 	ReloadConfig(c *config.C)
-	BatchSize() int
 	Close() error
 }
 
+type BatchPacket struct {
+	Payload []byte
+	Addr    netip.AddrPort
+}
+
 type NoopConn struct{}
 
 func (NoopConn) Rebind() error {
@@ -39,24 +36,18 @@ func (NoopConn) Rebind() error {
 func (NoopConn) LocalAddr() (netip.AddrPort, error) {
 	return netip.AddrPort{}, nil
 }
-func (NoopConn) ListenOut(_ EncReader) {
+func (NoopConn) ListenOut(_ EncReader) error {
-	return
+	return nil
-}
-func (NoopConn) ListenOutBatch(_ EncBatchReader) {
-	return
 }
 func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
 	return nil
 }
-func (NoopConn) WriteMulti(_ [][]byte, _ []netip.AddrPort) (int, error) {
+func (NoopConn) WriteBatch(_ []BatchPacket) (int, error) {
 	return 0, nil
 }
 func (NoopConn) ReloadConfig(_ *config.C) {
 	return
 }
-func (NoopConn) BatchSize() int {
-	return 1
-}
 func (NoopConn) Close() error {
 	return nil
 }

udp/udp_darwin.go

@@ -140,15 +140,15 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
 	}
 }
 
-// WriteMulti sends multiple packets - fallback implementation without sendmmsg
+func (u *StdConn) WriteBatch(pkts []BatchPacket) (int, error) {
-func (u *StdConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
+	sent := 0
-	for i := range packets {
+	for _, pkt := range pkts {
-		err := u.WriteTo(packets[i], addrs[i])
+		if err := u.WriteTo(pkt.Payload, pkt.Addr); err != nil {
-		if err != nil {
+			return sent, err
-			return i, err
 		}
+		sent++
 	}
-	return len(packets), nil
+	return sent, nil
 }
 
 func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
@@ -176,7 +176,7 @@ func NewUDPStatsEmitter(udpConns []Conn) func() {
 	return func() {}
 }
 
-func (u *StdConn) ListenOut(r EncReader) {
+func (u *StdConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	for {
@@ -185,42 +185,17 @@ func (u *StdConn) ListenOut(r EncReader) {
 		if err != nil {
 			if errors.Is(err, net.ErrClosed) {
 				u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
-				return
+				return err
 			}
 
 			u.l.WithError(err).Error("unexpected udp socket receive error")
+			continue
 		}
 
 		r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n])
 	}
-}
 
-// ListenOutBatch - fallback to single-packet reads for Darwin
+	return nil
-func (u *StdConn) ListenOutBatch(r EncBatchReader) {
-	buffer := make([]byte, MTU)
-	addrs := make([]netip.AddrPort, 1)
-	payloads := make([][]byte, 1)
-
-	for {
-		// Just read one packet at a time and call batch callback with count=1
-		n, rua, err := u.ReadFromUDPAddrPort(buffer)
-		if err != nil {
-			if errors.Is(err, net.ErrClosed) {
-				u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
-				return
-			}
-
-			u.l.WithError(err).Error("unexpected udp socket receive error")
-		}
-
-		addrs[0] = netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port())
-		payloads[0] = buffer[:n]
-		r(addrs, payloads, 1)
-	}
-}
-
-func (u *StdConn) BatchSize() int {
-	return 1
 }
 
 func (u *StdConn) Rebind() error {

udp/udp_generic.go

@@ -42,6 +42,17 @@ func (u *GenericConn) WriteTo(b []byte, addr netip.AddrPort) error {
 	return err
 }
 
+func (u *GenericConn) WriteBatch(pkts []BatchPacket) (int, error) {
+	sent := 0
+	for _, pkt := range pkts {
+		if err := u.WriteTo(pkt.Payload, pkt.Addr); err != nil {
+			return sent, err
+		}
+		sent++
+	}
+	return sent, nil
+}
+
 func (u *GenericConn) LocalAddr() (netip.AddrPort, error) {
 	a := u.UDPConn.LocalAddr()
 
@@ -71,56 +82,16 @@ type rawMessage struct {
 	Len uint32
 }
 
-func (u *GenericConn) ListenOut(r EncReader) {
+func (u *GenericConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	for {
 		// Just read one packet at a time
 		n, rua, err := u.ReadFromUDPAddrPort(buffer)
 		if err != nil {
-			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+			return err
-			return
 		}
 
 		r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n])
 	}
 }
-
-// ListenOutBatch - fallback to single-packet reads for generic platforms
-func (u *GenericConn) ListenOutBatch(r EncBatchReader) {
-	buffer := make([]byte, MTU)
-	addrs := make([]netip.AddrPort, 1)
-	payloads := make([][]byte, 1)
-
-	for {
-		// Just read one packet at a time and call batch callback with count=1
-		n, rua, err := u.ReadFromUDPAddrPort(buffer)
-		if err != nil {
-			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
-			return
-		}
-
-		addrs[0] = netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port())
-		payloads[0] = buffer[:n]
-		r(addrs, payloads, 1)
-	}
-}
-
-// WriteMulti sends multiple packets - fallback implementation
-func (u *GenericConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
-	for i := range packets {
-		err := u.WriteTo(packets[i], addrs[i])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(packets), nil
-}
-
-func (u *GenericConn) BatchSize() int {
-	return 1
-}
-
-func (u *GenericConn) Rebind() error {
-	return nil
-}

udp/udp_linux.go

@@ -5,10 +5,13 @@ package udp
 
 import (
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"net"
 	"net/netip"
+	"sync"
 	"syscall"
+	"time"
 	"unsafe"
 
 	"github.com/rcrowley/go-metrics"
@@ -17,24 +20,40 @@ import (
 	"golang.org/x/sys/unix"
 )
 
+var readTimeout = unix.NsecToTimeval(int64(time.Millisecond * 500))
+
+const (
+	defaultGSOMaxSegments    = 128
+	defaultGSOFlushTimeout   = 80 * time.Microsecond
+	defaultGROReadBufferSize = MTU * defaultGSOMaxSegments
+	maxGSOBatchBytes         = 0xFFFF
+)
+
+var (
+	errGSOFallback = errors.New("udp gso fallback")
+	errGSODisabled = errors.New("udp gso disabled")
+)
+
 type StdConn struct {
 	sysFd int
 	isV4  bool
 	l     *logrus.Logger
 	batch int
 
-	// Pre-allocated buffers for batch writes (sized for IPv6, works for both)
+	enableGRO bool
-	writeMsgs   []rawMessage
+	enableGSO bool
-	writeIovecs []iovec
-	writeNames  [][]byte
-}
 
-func maybeIPV4(ip net.IP) (net.IP, bool) {
+	gsoMu           sync.Mutex
-	ip4 := ip.To4()
+	gsoBuf          []byte
-	if ip4 != nil {
+	gsoAddr         netip.AddrPort
-		return ip4, true
+	gsoSegSize      int
-	}
+	gsoSegments     int
-	return ip, false
+	gsoMaxSegments  int
+	gsoMaxBytes     int
+	gsoFlushTimeout time.Duration
+	gsoTimer        *time.Timer
+
+	groBufSize int
 }
 
 func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
@@ -60,6 +79,11 @@ func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch in
 		}
 	}
 
+	// Set a read timeout
+	if err = unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &readTimeout); err != nil {
+		return nil, fmt.Errorf("unable to set SO_RCVTIMEO: %s", err)
+	}
+
 	var sa unix.Sockaddr
 	if ip.Is4() {
 		sa4 := &unix.SockaddrInet4{Port: port}
@@ -74,26 +98,16 @@ func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch in
 		return nil, fmt.Errorf("unable to bind to socket: %s", err)
 	}
 
-	c := &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}
+	return &StdConn{
-
+		sysFd:           fd,
-	// Pre-allocate write message structures for batching (sized for IPv6, works for both)
+		isV4:            ip.Is4(),
-	c.writeMsgs = make([]rawMessage, batch)
+		l:               l,
-	c.writeIovecs = make([]iovec, batch)
+		batch:           batch,
-	c.writeNames = make([][]byte, batch)
+		gsoMaxSegments:  defaultGSOMaxSegments,
-
+		gsoMaxBytes:     MTU * defaultGSOMaxSegments,
-	for i := range c.writeMsgs {
+		gsoFlushTimeout: defaultGSOFlushTimeout,
-		// Allocate for IPv6 size (larger than IPv4, works for both)
+		groBufSize:      MTU,
-		c.writeNames[i] = make([]byte, unix.SizeofSockaddrInet6)
+	}, err
-
-		// Point to the iovec in the slice
-		c.writeMsgs[i].Hdr.Iov = &c.writeIovecs[i]
-		c.writeMsgs[i].Hdr.Iovlen = 1
-
-		c.writeMsgs[i].Hdr.Name = &c.writeNames[i][0]
-		// Namelen will be set appropriately in writeMulti4/writeMulti6
-	}
-
-	return c, err
 }
 
 func (u *StdConn) Rebind() error {
@@ -142,25 +156,47 @@ func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
 	}
 }
 
-func (u *StdConn) ListenOut(r EncReader) {
+func (u *StdConn) ListenOut(r EncReader) error {
-	var ip netip.Addr
+	var (
+		ip       netip.Addr
+		controls [][]byte
+	)
 
-	msgs, buffers, names := u.PrepareRawMessages(u.batch)
+	bufSize := u.readBufferSize()
+	msgs, buffers, names := u.PrepareRawMessages(u.batch, bufSize)
 	read := u.ReadMulti
 	if u.batch == 1 {
 		read = u.ReadSingle
 	}
 
-	udpBatchHist := metrics.GetOrRegisterHistogram("batch.udp_read_size", nil, metrics.NewUniformSample(1024))
-
 	for {
-		n, err := read(msgs)
+		desired := u.readBufferSize()
-		if err != nil {
+		if len(buffers) == 0 || cap(buffers[0]) < desired {
-			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+			msgs, buffers, names = u.PrepareRawMessages(u.batch, desired)
-			return
+			controls = nil
 		}
 
-		udpBatchHist.Update(int64(n))
+		if u.enableGRO {
+			if controls == nil {
+				controls = make([][]byte, len(msgs))
+				for i := range controls {
+					controls[i] = make([]byte, unix.CmsgSpace(4))
+				}
+			}
+			for i := range msgs {
+				setRawMessageControl(&msgs[i], controls[i])
+			}
+		} else if controls != nil {
+			for i := range msgs {
+				setRawMessageControl(&msgs[i], nil)
+			}
+			controls = nil
+		}
+
+		n, err := read(msgs)
+		if err != nil {
+			return err
+		}
 
 		for i := 0; i < n; i++ {
 			// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
@@ -169,49 +205,80 @@ func (u *StdConn) ListenOut(r EncReader) {
 			} else {
 				ip, _ = netip.AddrFromSlice(names[i][8:24])
 			}
-			r(netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4])), buffers[i][:msgs[i].Len])
+			addr := netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4]))
+			payload := buffers[i][:msgs[i].Len]
+
+			if u.enableGRO && u.l.IsLevelEnabled(logrus.DebugLevel) {
+				ctrlLen := getRawMessageControlLen(&msgs[i])
+				msgFlags := getRawMessageFlags(&msgs[i])
+				u.l.WithFields(logrus.Fields{
+					"tag":         "gro-debug",
+					"stage":       "recv",
+					"payload_len": len(payload),
+					"ctrl_len":    ctrlLen,
+					"msg_flags":   msgFlags,
+				}).Debug("gro batch data")
+				if controls != nil && ctrlLen > 0 {
+					maxDump := ctrlLen
+					if maxDump > 16 {
+						maxDump = 16
+					}
+					u.l.WithFields(logrus.Fields{
+						"tag":         "gro-debug",
+						"stage":       "control-bytes",
+						"control_hex": fmt.Sprintf("%x", controls[i][:maxDump]),
+						"datalen":     ctrlLen,
+					}).Debug("gro control dump")
+				}
+			}
+
+			sawControl := false
+			if controls != nil {
+				if ctrlLen := getRawMessageControlLen(&msgs[i]); ctrlLen > 0 {
+					if segSize, segCount := parseGROControl(controls[i][:ctrlLen]); segSize > 0 {
+						sawControl = true
+						if u.l.IsLevelEnabled(logrus.DebugLevel) {
+							u.l.WithFields(logrus.Fields{
+								"tag":        "gro-debug",
+								"stage":      "control",
+								"seg_size":   segSize,
+								"seg_count":  segCount,
+								"payloadLen": len(payload),
+							}).Debug("gro control parsed")
+						}
+						segSize = normalizeGROSegSize(segSize, segCount, len(payload))
+						if segSize > 0 && segSize < len(payload) {
+							if u.emitGROSegments(r, addr, payload, segSize) {
+								continue
+							}
+						}
+					}
+				}
+			}
+
+			if u.enableGRO && len(payload) > MTU {
+				if !sawControl && u.l.IsLevelEnabled(logrus.DebugLevel) {
+					u.l.WithFields(logrus.Fields{
+						"tag":         "gro-debug",
+						"stage":       "fallback",
+						"payload_len": len(payload),
+					}).Debug("gro control missing; splitting payload by MTU")
+				}
+				if u.emitGROSegments(r, addr, payload, MTU) {
+					continue
+				}
+			}
+
+			r(addr, payload)
 		}
 	}
 }
 
-func (u *StdConn) ListenOutBatch(r EncBatchReader) {
+func (u *StdConn) readBufferSize() int {
-	var ip netip.Addr
+	if u.enableGRO && u.groBufSize > MTU {
-
+		return u.groBufSize
-	msgs, buffers, names := u.PrepareRawMessages(u.batch)
-	read := u.ReadMulti
-	if u.batch == 1 {
-		read = u.ReadSingle
-	}
-
-	udpBatchHist := metrics.GetOrRegisterHistogram("batch.udp_read_size", nil, metrics.NewUniformSample(1024))
-
-	// Pre-allocate slices for batch callback
-	addrs := make([]netip.AddrPort, u.batch)
-	payloads := make([][]byte, u.batch)
-
-	for {
-		n, err := read(msgs)
-		if err != nil {
-			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
-			return
-		}
-
-		udpBatchHist.Update(int64(n))
-
-		// Prepare batch data
-		for i := 0; i < n; i++ {
-			if u.isV4 {
-				ip, _ = netip.AddrFromSlice(names[i][4:8])
-			} else {
-				ip, _ = netip.AddrFromSlice(names[i][8:24])
-			}
-			addrs[i] = netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4]))
-			payloads[i] = buffers[i][:msgs[i].Len]
-		}
-
-		// Call batch callback with all packets
-		r(addrs, payloads, n)
 	}
+	return MTU
 }
 
 func (u *StdConn) ReadSingle(msgs []rawMessage) (int, error) {
@@ -227,6 +294,9 @@ func (u *StdConn) ReadSingle(msgs []rawMessage) (int, error) {
 		)
 
 		if err != 0 {
+			if err == unix.EAGAIN || err == unix.EINTR {
+				continue
+			}
 			return 0, &net.OpError{Op: "recvmsg", Err: err}
 		}
 
@@ -248,6 +318,9 @@ func (u *StdConn) ReadMulti(msgs []rawMessage) (int, error) {
 		)
 
 		if err != 0 {
+			if err == unix.EAGAIN || err == unix.EINTR {
+				continue
+			}
 			return 0, &net.OpError{Op: "recvmmsg", Err: err}
 		}
 
@@ -256,23 +329,130 @@ func (u *StdConn) ReadMulti(msgs []rawMessage) (int, error) {
 }
 
 func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
+	if u.enableGSO && ip.IsValid() {
+		if err := u.queueGSOPacket(b, ip); err == nil {
+			return nil
+		} else if !errors.Is(err, errGSOFallback) {
+			return err
+		}
+	}
+
 	if u.isV4 {
 		return u.writeTo4(b, ip)
 	}
 	return u.writeTo6(b, ip)
 }
 
-func (u *StdConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
+func (u *StdConn) WriteBatch(pkts []BatchPacket) (int, error) {
-	if len(packets) != len(addrs) {
+	if len(pkts) == 0 {
-		return 0, fmt.Errorf("packets and addrs length mismatch")
-	}
-	if len(packets) == 0 {
 		return 0, nil
 	}
-	if u.isV4 {
+
-		return u.writeMulti4(packets, addrs)
+	msgs := make([]rawMessage, 0, len(pkts))
+	iovs := make([]iovec, 0, len(pkts))
+	names := make([][unix.SizeofSockaddrInet6]byte, 0, len(pkts))
+
+	sent := 0
+
+	for _, pkt := range pkts {
+		if len(pkt.Payload) == 0 {
+			sent++
+			continue
+		}
+
+		if u.enableGSO && pkt.Addr.IsValid() {
+			if err := u.queueGSOPacket(pkt.Payload, pkt.Addr); err == nil {
+				sent++
+				continue
+			} else if !errors.Is(err, errGSOFallback) {
+				return sent, err
+			}
+		}
+
+		if !pkt.Addr.IsValid() {
+			if err := u.WriteTo(pkt.Payload, pkt.Addr); err != nil {
+				return sent, err
+			}
+			sent++
+			continue
+		}
+
+		msgs = append(msgs, rawMessage{})
+		iovs = append(iovs, iovec{})
+		names = append(names, [unix.SizeofSockaddrInet6]byte{})
+
+		idx := len(msgs) - 1
+		msg := &msgs[idx]
+		iov := &iovs[idx]
+		name := &names[idx]
+
+		setIovecSlice(iov, pkt.Payload)
+		msg.Hdr.Iov = iov
+		msg.Hdr.Iovlen = 1
+		setRawMessageControl(msg, nil)
+		msg.Hdr.Flags = 0
+
+		nameLen, err := u.encodeSockaddr(name[:], pkt.Addr)
+		if err != nil {
+			return sent, err
+		}
+		msg.Hdr.Name = &name[0]
+		msg.Hdr.Namelen = nameLen
 	}
-	return u.writeMulti6(packets, addrs)
+
+	if len(msgs) == 0 {
+		return sent, nil
+	}
+
+	offset := 0
+	for offset < len(msgs) {
+		n, _, errno := unix.Syscall6(
+			unix.SYS_SENDMMSG,
+			uintptr(u.sysFd),
+			uintptr(unsafe.Pointer(&msgs[offset])),
+			uintptr(len(msgs)-offset),
+			0,
+			0,
+			0,
+		)
+
+		if errno != 0 {
+			if errno == unix.EINTR {
+				continue
+			}
+			return sent + offset, &net.OpError{Op: "sendmmsg", Err: errno}
+		}
+
+		if n == 0 {
+			break
+		}
+		offset += int(n)
+	}
+
+	return sent + len(msgs), nil
+}
+
+func (u *StdConn) encodeSockaddr(dst []byte, addr netip.AddrPort) (uint32, error) {
+	if u.isV4 {
+		if !addr.Addr().Is4() {
+			return 0, fmt.Errorf("Listener is IPv4, but writing to IPv6 remote")
+		}
+		var sa unix.RawSockaddrInet4
+		sa.Family = unix.AF_INET
+		sa.Addr = addr.Addr().As4()
+		binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
+		size := unix.SizeofSockaddrInet4
+		copy(dst[:size], (*(*[unix.SizeofSockaddrInet4]byte)(unsafe.Pointer(&sa)))[:])
+		return uint32(size), nil
+	}
+
+	var sa unix.RawSockaddrInet6
+	sa.Family = unix.AF_INET6
+	sa.Addr = addr.Addr().As16()
+	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
+	size := unix.SizeofSockaddrInet6
+	copy(dst[:size], (*(*[unix.SizeofSockaddrInet6]byte)(unsafe.Pointer(&sa)))[:])
+	return uint32(size), nil
 }
 
 func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
@@ -302,7 +482,7 @@ func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
 
 func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
 	if !ip.Addr().Is4() {
-		return ErrInvalidIPv6RemoteForSocket
+		return fmt.Errorf("Listener is IPv4, but writing to IPv6 remote")
 	}
 
 	var rsa unix.RawSockaddrInet4
@@ -329,123 +509,6 @@ func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
 	}
 }
 
-func (u *StdConn) writeMulti4(packets [][]byte, addrs []netip.AddrPort) (int, error) {
-	sent := 0
-	for sent < len(packets) {
-		// Determine batch size based on remaining packets and buffer capacity
-		batchSize := len(packets) - sent
-		if batchSize > len(u.writeMsgs) {
-			batchSize = len(u.writeMsgs)
-		}
-
-		// Use pre-allocated buffers
-		msgs := u.writeMsgs[:batchSize]
-		iovecs := u.writeIovecs[:batchSize]
-		names := u.writeNames[:batchSize]
-
-		// Setup message structures for this batch
-		for i := 0; i < batchSize; i++ {
-			pktIdx := sent + i
-			if !addrs[pktIdx].Addr().Is4() {
-				return sent + i, ErrInvalidIPv6RemoteForSocket
-			}
-
-			// Setup the packet buffer
-			iovecs[i].Base = &packets[pktIdx][0]
-			iovecs[i].Len = uint(len(packets[pktIdx]))
-
-			// Setup the destination address
-			rsa := (*unix.RawSockaddrInet4)(unsafe.Pointer(&names[i][0]))
-			rsa.Family = unix.AF_INET
-			rsa.Addr = addrs[pktIdx].Addr().As4()
-			binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], addrs[pktIdx].Port())
-
-			// Set the appropriate address length for IPv4
-			msgs[i].Hdr.Namelen = unix.SizeofSockaddrInet4
-		}
-
-		// Send this batch
-		nsent, _, err := unix.Syscall6(
-			unix.SYS_SENDMMSG,
-			uintptr(u.sysFd),
-			uintptr(unsafe.Pointer(&msgs[0])),
-			uintptr(batchSize),
-			0,
-			0,
-			0,
-		)
-
-		if err != 0 {
-			return sent + int(nsent), &net.OpError{Op: "sendmmsg", Err: err}
-		}
-
-		sent += int(nsent)
-		if int(nsent) < batchSize {
-			// Couldn't send all packets in batch, return what we sent
-			return sent, nil
-		}
-	}
-
-	return sent, nil
-}
-
-func (u *StdConn) writeMulti6(packets [][]byte, addrs []netip.AddrPort) (int, error) {
-	sent := 0
-	for sent < len(packets) {
-		// Determine batch size based on remaining packets and buffer capacity
-		batchSize := len(packets) - sent
-		if batchSize > len(u.writeMsgs) {
-			batchSize = len(u.writeMsgs)
-		}
-
-		// Use pre-allocated buffers
-		msgs := u.writeMsgs[:batchSize]
-		iovecs := u.writeIovecs[:batchSize]
-		names := u.writeNames[:batchSize]
-
-		// Setup message structures for this batch
-		for i := 0; i < batchSize; i++ {
-			pktIdx := sent + i
-
-			// Setup the packet buffer
-			iovecs[i].Base = &packets[pktIdx][0]
-			iovecs[i].Len = uint(len(packets[pktIdx]))
-
-			// Setup the destination address
-			rsa := (*unix.RawSockaddrInet6)(unsafe.Pointer(&names[i][0]))
-			rsa.Family = unix.AF_INET6
-			rsa.Addr = addrs[pktIdx].Addr().As16()
-			binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], addrs[pktIdx].Port())
-
-			// Set the appropriate address length for IPv6
-			msgs[i].Hdr.Namelen = unix.SizeofSockaddrInet6
-		}
-
-		// Send this batch
-		nsent, _, err := unix.Syscall6(
-			unix.SYS_SENDMMSG,
-			uintptr(u.sysFd),
-			uintptr(unsafe.Pointer(&msgs[0])),
-			uintptr(batchSize),
-			0,
-			0,
-			0,
-		)
-
-		if err != 0 {
-			return sent + int(nsent), &net.OpError{Op: "sendmmsg", Err: err}
-		}
-
-		sent += int(nsent)
-		if int(nsent) < batchSize {
-			// Couldn't send all packets in batch, return what we sent
-			return sent, nil
-		}
-	}
-
-	return sent, nil
-}
-
 func (u *StdConn) ReloadConfig(c *config.C) {
 	b := c.GetInt("listen.read_buffer", 0)
 	if b > 0 {
@@ -492,6 +555,94 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 			u.l.WithError(err).Error("Failed to set listen.so_mark")
 		}
 	}
+
+	u.configureGRO(c)
+	u.configureGSO(c)
+}
+
+func (u *StdConn) configureGRO(c *config.C) {
+	if c == nil {
+		return
+	}
+
+	enable := c.GetBool("listen.enable_gro", true)
+	if enable == u.enableGRO {
+		if enable {
+			if size := c.GetInt("listen.gro_read_buffer", 0); size > 0 {
+				u.setGROBufferSize(size)
+			}
+		}
+		return
+	}
+
+	if enable {
+		if err := unix.SetsockoptInt(u.sysFd, unix.SOL_UDP, unix.UDP_GRO, 1); err != nil {
+			u.l.WithError(err).Warn("Failed to enable UDP GRO")
+			return
+		}
+		u.enableGRO = true
+		u.setGROBufferSize(c.GetInt("listen.gro_read_buffer", defaultGROReadBufferSize))
+		u.l.WithField("buffer_size", u.groBufSize).Info("UDP GRO enabled")
+		return
+	}
+
+	if err := unix.SetsockoptInt(u.sysFd, unix.SOL_UDP, unix.UDP_GRO, 0); err != nil && err != unix.ENOPROTOOPT {
+		u.l.WithError(err).Warn("Failed to disable UDP GRO")
+	}
+	u.enableGRO = false
+	u.groBufSize = MTU
+}
+
+func (u *StdConn) configureGSO(c *config.C) {
+	enable := c.GetBool("listen.enable_gso", true)
+	if !enable {
+		u.disableGSO()
+	} else {
+		u.enableGSO = true
+	}
+
+	segments := c.GetInt("listen.gso_max_segments", defaultGSOMaxSegments)
+	if segments < 1 {
+		segments = 1
+	}
+	u.gsoMaxSegments = segments
+
+	maxBytes := c.GetInt("listen.gso_max_bytes", 0)
+	if maxBytes <= 0 {
+		maxBytes = MTU * segments
+	}
+	if maxBytes > maxGSOBatchBytes {
+		u.l.WithField("requested", maxBytes).Warn("listen.gso_max_bytes larger than UDP limit; clamping")
+		maxBytes = maxGSOBatchBytes
+	}
+	u.gsoMaxBytes = maxBytes
+
+	timeout := c.GetDuration("listen.gso_flush_timeout", defaultGSOFlushTimeout)
+	if timeout < 0 {
+		timeout = 0
+	}
+	u.gsoFlushTimeout = timeout
+}
+
+func (u *StdConn) setGROBufferSize(size int) {
+	if size < MTU {
+		size = defaultGROReadBufferSize
+	}
+	if size > maxGSOBatchBytes {
+		size = maxGSOBatchBytes
+	}
+	u.groBufSize = size
+}
+
+func (u *StdConn) disableGSO() {
+	u.gsoMu.Lock()
+	defer u.gsoMu.Unlock()
+	u.enableGSO = false
+	_ = u.flushGSOlocked()
+	u.gsoBuf = nil
+	u.gsoSegments = 0
+	u.gsoSegSize = 0
+	u.stopGSOTimerLocked()
 }
 
 func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
@@ -503,11 +654,239 @@ func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
 	return nil
 }
 
-func (u *StdConn) BatchSize() int {
+func (u *StdConn) queueGSOPacket(b []byte, addr netip.AddrPort) error {
-	return u.batch
+	if len(b) == 0 {
+		return nil
+	}
+
+	u.gsoMu.Lock()
+	defer u.gsoMu.Unlock()
+
+	if !u.enableGSO || !addr.IsValid() || len(b) > u.gsoMaxBytes {
+		if err := u.flushGSOlocked(); err != nil {
+			return err
+		}
+		return errGSOFallback
+	}
+
+	if u.gsoSegments == 0 {
+		if cap(u.gsoBuf) < u.gsoMaxBytes {
+			u.gsoBuf = make([]byte, 0, u.gsoMaxBytes)
+		}
+		u.gsoAddr = addr
+		u.gsoSegSize = len(b)
+	} else if addr != u.gsoAddr || len(b) != u.gsoSegSize {
+		if err := u.flushGSOlocked(); err != nil {
+			return err
+		}
+		if cap(u.gsoBuf) < u.gsoMaxBytes {
+			u.gsoBuf = make([]byte, 0, u.gsoMaxBytes)
+		}
+		u.gsoAddr = addr
+		u.gsoSegSize = len(b)
+	}
+
+	if len(u.gsoBuf)+len(b) > u.gsoMaxBytes {
+		if err := u.flushGSOlocked(); err != nil {
+			return err
+		}
+		if cap(u.gsoBuf) < u.gsoMaxBytes {
+			u.gsoBuf = make([]byte, 0, u.gsoMaxBytes)
+		}
+		u.gsoAddr = addr
+		u.gsoSegSize = len(b)
+	}
+
+	u.gsoBuf = append(u.gsoBuf, b...)
+	u.gsoSegments++
+
+	if u.gsoSegments >= u.gsoMaxSegments || u.gsoFlushTimeout <= 0 {
+		return u.flushGSOlocked()
+	}
+
+	u.scheduleGSOFlushLocked()
+	return nil
+}
+
+func (u *StdConn) flushGSOlocked() error {
+	if u.gsoSegments == 0 {
+		u.stopGSOTimerLocked()
+		return nil
+	}
+
+	payload := append([]byte(nil), u.gsoBuf...)
+	addr := u.gsoAddr
+	segSize := u.gsoSegSize
+
+	u.gsoBuf = u.gsoBuf[:0]
+	u.gsoSegments = 0
+	u.gsoSegSize = 0
+	u.stopGSOTimerLocked()
+
+	if segSize <= 0 {
+		return errGSOFallback
+	}
+
+	err := u.sendSegmented(payload, addr, segSize)
+	if errors.Is(err, errGSODisabled) {
+		u.l.WithField("addr", addr).Warn("UDP GSO disabled by kernel, falling back to sendto")
+		u.enableGSO = false
+		return u.sendSegmentsIndividually(payload, addr, segSize)
+	}
+
+	return err
+}
+
+func (u *StdConn) sendSegmented(payload []byte, addr netip.AddrPort, segSize int) error {
+	if len(payload) == 0 {
+		return nil
+	}
+
+	control := make([]byte, unix.CmsgSpace(2))
+	hdr := (*unix.Cmsghdr)(unsafe.Pointer(&control[0]))
+	hdr.Level = unix.SOL_UDP
+	hdr.Type = unix.UDP_SEGMENT
+	setCmsgLen(hdr, unix.CmsgLen(2))
+	binary.NativeEndian.PutUint16(control[unix.CmsgLen(0):unix.CmsgLen(0)+2], uint16(segSize))
+
+	var sa unix.Sockaddr
+	if addr.Addr().Is4() {
+		var sa4 unix.SockaddrInet4
+		sa4.Port = int(addr.Port())
+		sa4.Addr = addr.Addr().As4()
+		sa = &sa4
+	} else {
+		var sa6 unix.SockaddrInet6
+		sa6.Port = int(addr.Port())
+		sa6.Addr = addr.Addr().As16()
+		sa = &sa6
+	}
+
+	if _, err := unix.SendmsgN(u.sysFd, payload, control, sa, 0); err != nil {
+		if errno, ok := err.(syscall.Errno); ok && (errno == unix.EINVAL || errno == unix.ENOTSUP || errno == unix.EOPNOTSUPP) {
+			return errGSODisabled
+		}
+		return &net.OpError{Op: "sendmsg", Err: err}
+	}
+	return nil
+}
+
+func (u *StdConn) sendSegmentsIndividually(buf []byte, addr netip.AddrPort, segSize int) error {
+	if segSize <= 0 {
+		return errGSOFallback
+	}
+
+	for offset := 0; offset < len(buf); offset += segSize {
+		end := offset + segSize
+		if end > len(buf) {
+			end = len(buf)
+		}
+		var err error
+		if u.isV4 {
+			err = u.writeTo4(buf[offset:end], addr)
+		} else {
+			err = u.writeTo6(buf[offset:end], addr)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (u *StdConn) scheduleGSOFlushLocked() {
+	if u.gsoTimer == nil {
+		u.gsoTimer = time.AfterFunc(u.gsoFlushTimeout, u.gsoFlushTimer)
+		return
+	}
+	u.gsoTimer.Reset(u.gsoFlushTimeout)
+}
+
+func (u *StdConn) stopGSOTimerLocked() {
+	if u.gsoTimer != nil {
+		u.gsoTimer.Stop()
+		u.gsoTimer = nil
+	}
+}
+
+func (u *StdConn) gsoFlushTimer() {
+	u.gsoMu.Lock()
+	defer u.gsoMu.Unlock()
+	_ = u.flushGSOlocked()
+}
+
+func parseGROControl(control []byte) (int, int) {
+	if len(control) == 0 {
+		return 0, 0
+	}
+
+	cmsgs, err := unix.ParseSocketControlMessage(control)
+	if err != nil {
+		return 0, 0
+	}
+
+	for _, c := range cmsgs {
+		if c.Header.Level == unix.SOL_UDP && c.Header.Type == unix.UDP_GRO && len(c.Data) >= 2 {
+			segSize := int(binary.NativeEndian.Uint16(c.Data[:2]))
+			segCount := 0
+			if len(c.Data) >= 4 {
+				segCount = int(binary.NativeEndian.Uint16(c.Data[2:4]))
+			}
+			return segSize, segCount
+		}
+	}
+
+	return 0, 0
+}
+
+func (u *StdConn) emitGROSegments(r EncReader, addr netip.AddrPort, payload []byte, segSize int) bool {
+	if segSize <= 0 {
+		return false
+	}
+
+	for offset := 0; offset < len(payload); offset += segSize {
+		end := offset + segSize
+		if end > len(payload) {
+			end = len(payload)
+		}
+		segment := make([]byte, end-offset)
+		copy(segment, payload[offset:end])
+		r(addr, segment)
+	}
+	return true
+}
+
+func normalizeGROSegSize(segSize, segCount, total int) int {
+	if segSize <= 0 || total <= 0 {
+		return segSize
+	}
+
+	if segSize > total && segCount > 0 {
+		segSize = total / segCount
+		if segSize == 0 {
+			segSize = total
+		}
+	}
+
+	if segCount <= 1 && segSize > 0 && total > segSize {
+		calculated := total / segSize
+		if calculated <= 1 {
+			calculated = (total + segSize - 1) / segSize
+		}
+		if calculated > 1 {
+			segCount = calculated
+		}
+	}
+
+	if segSize > MTU {
+		return MTU
+	}
+
+	return segSize
 }
 
 func (u *StdConn) Close() error {
+	u.disableGSO()
 	return syscall.Close(u.sysFd)
 }
 

udp/udp_linux_32.go

@@ -12,7 +12,7 @@ import (
 
 type iovec struct {
 	Base *byte
-	Len  uint
+	Len  uint32
 }
 
 type msghdr struct {
@@ -30,17 +30,20 @@ type rawMessage struct {
 	Len uint32
 }
 
-func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
+func (u *StdConn) PrepareRawMessages(n int, bufSize int) ([]rawMessage, [][]byte, [][]byte) {
+	if bufSize <= 0 {
+		bufSize = MTU
+	}
 	msgs := make([]rawMessage, n)
 	buffers := make([][]byte, n)
 	names := make([][]byte, n)
 
 	for i := range msgs {
-		buffers[i] = make([]byte, MTU)
+		buffers[i] = make([]byte, bufSize)
 		names[i] = make([]byte, unix.SizeofSockaddrInet6)
 
 		vs := []iovec{
-			{Base: &buffers[i][0], Len: uint(len(buffers[i]))},
+			{Base: &buffers[i][0], Len: uint32(len(buffers[i]))},
 		}
 
 		msgs[i].Hdr.Iov = &vs[0]
@@ -52,3 +55,35 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
 
 	return msgs, buffers, names
 }
+
+func setRawMessageControl(msg *rawMessage, buf []byte) {
+	if len(buf) == 0 {
+		msg.Hdr.Control = nil
+		msg.Hdr.Controllen = 0
+		return
+	}
+	msg.Hdr.Control = &buf[0]
+	msg.Hdr.Controllen = uint32(len(buf))
+}
+
+func getRawMessageControlLen(msg *rawMessage) int {
+	return int(msg.Hdr.Controllen)
+}
+
+func getRawMessageFlags(msg *rawMessage) int {
+	return int(msg.Hdr.Flags)
+}
+
+func setCmsgLen(h *unix.Cmsghdr, l int) {
+	h.Len = uint32(l)
+}
+
+func setIovecSlice(iov *iovec, b []byte) {
+	if len(b) == 0 {
+		iov.Base = nil
+		iov.Len = 0
+		return
+	}
+	iov.Base = &b[0]
+	iov.Len = uint32(len(b))
+}

udp/udp_linux_64.go

@@ -12,7 +12,7 @@ import (
 
 type iovec struct {
 	Base *byte
-	Len  uint
+	Len  uint64
 }
 
 type msghdr struct {
@@ -33,17 +33,20 @@ type rawMessage struct {
 	Pad0 [4]byte
 }
 
-func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
+func (u *StdConn) PrepareRawMessages(n int, bufSize int) ([]rawMessage, [][]byte, [][]byte) {
+	if bufSize <= 0 {
+		bufSize = MTU
+	}
 	msgs := make([]rawMessage, n)
 	buffers := make([][]byte, n)
 	names := make([][]byte, n)
 
 	for i := range msgs {
-		buffers[i] = make([]byte, MTU)
+		buffers[i] = make([]byte, bufSize)
 		names[i] = make([]byte, unix.SizeofSockaddrInet6)
 
 		vs := []iovec{
-			{Base: &buffers[i][0], Len: uint(len(buffers[i]))},
+			{Base: &buffers[i][0], Len: uint64(len(buffers[i]))},
 		}
 
 		msgs[i].Hdr.Iov = &vs[0]
@@ -55,3 +58,35 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
 
 	return msgs, buffers, names
 }
+
+func setRawMessageControl(msg *rawMessage, buf []byte) {
+	if len(buf) == 0 {
+		msg.Hdr.Control = nil
+		msg.Hdr.Controllen = 0
+		return
+	}
+	msg.Hdr.Control = &buf[0]
+	msg.Hdr.Controllen = uint64(len(buf))
+}
+
+func getRawMessageControlLen(msg *rawMessage) int {
+	return int(msg.Hdr.Controllen)
+}
+
+func getRawMessageFlags(msg *rawMessage) int {
+	return int(msg.Hdr.Flags)
+}
+
+func setCmsgLen(h *unix.Cmsghdr, l int) {
+	h.Len = uint64(l)
+}
+
+func setIovecSlice(iov *iovec, b []byte) {
+	if len(b) == 0 {
+		iov.Base = nil
+		iov.Len = 0
+		return
+	}
+	iov.Base = &b[0]
+	iov.Len = uint64(len(b))
+}

udp/udp_rio_windows.go

@@ -134,7 +134,7 @@ func (u *RIOConn) bind(sa windows.Sockaddr) error {
 	return nil
 }
 
-func (u *RIOConn) ListenOut(r EncReader) {
+func (u *RIOConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	for {
@@ -304,6 +304,17 @@ func (u *RIOConn) WriteTo(buf []byte, ip netip.AddrPort) error {
 	return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
 }
 
+func (u *RIOConn) WriteBatch(pkts []BatchPacket) (int, error) {
+	sent := 0
+	for _, pkt := range pkts {
+		if err := u.WriteTo(pkt.Payload, pkt.Addr); err != nil {
+			return sent, err
+		}
+		sent++
+	}
+	return sent, nil
+}
+
 func (u *RIOConn) LocalAddr() (netip.AddrPort, error) {
 	sa, err := windows.Getsockname(u.sock)
 	if err != nil {

udp/udp_tester.go

@@ -106,6 +106,17 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
 	return nil
 }
 
+func (u *TesterConn) WriteBatch(pkts []BatchPacket) (int, error) {
+	sent := 0
+	for _, pkt := range pkts {
+		if err := u.WriteTo(pkt.Payload, pkt.Addr); err != nil {
+			return sent, err
+		}
+		sent++
+	}
+	return sent, nil
+}
+
 func (u *TesterConn) ListenOut(r EncReader) {
 	for {
 		p, ok := <-u.RxPackets
@@ -116,31 +127,6 @@ func (u *TesterConn) ListenOut(r EncReader) {
 	}
 }
 
-func (u *TesterConn) ListenOutBatch(r EncBatchReader) {
-	addrs := make([]netip.AddrPort, 1)
-	payloads := make([][]byte, 1)
-
-	for {
-		p, ok := <-u.RxPackets
-		if !ok {
-			return
-		}
-		addrs[0] = p.From
-		payloads[0] = p.Data
-		r(addrs, payloads, 1)
-	}
-}
-
-func (u *TesterConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
-	for i := range packets {
-		err := u.WriteTo(packets[i], addrs[i])
-		if err != nil {
-			return i, err
-		}
-	}
-	return len(packets), nil
-}
-
 func (u *TesterConn) ReloadConfig(*config.C) {}
 
 func NewUDPStatsEmitter(_ []Conn) func() {
@@ -152,10 +138,6 @@ func (u *TesterConn) LocalAddr() (netip.AddrPort, error) {
 	return u.Addr, nil
 }
 
-func (u *TesterConn) BatchSize() int {
-	return 1
-}
-
 func (u *TesterConn) Rebind() error {
 	return nil
 }