remove gso, leave nice pretty GRO

This reverts commit fa098c551a.

bits.go

@@ -5,6 +5,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// TODO: Pretty sure this is just all sorts of racy now, we need it to be atomic
 type Bits struct {
 	length             uint64
 	current            uint64
@@ -43,7 +44,7 @@ func (b *Bits) Check(l logrus.FieldLogger, i uint64) bool {
 	}
 
 	// Not within the window
-	l.Debugf("rejected a packet (top) %d %d\n", b.current, i)
+	l.Error("rejected a packet (top) %d %d\n", b.current, i)
 	return false
 }
 

cert/cert.go

@@ -58,6 +58,9 @@ type Certificate interface {
 	// PublicKey is the raw bytes to be used in asymmetric cryptographic operations.
 	PublicKey() []byte
 
+	// MarshalPublicKeyPEM is the value of PublicKey marshalled to PEM
+	MarshalPublicKeyPEM() []byte
+
 	// Curve identifies which curve was used for the PublicKey and Signature.
 	Curve() Curve
 

cert/cert_v1.go

@@ -83,6 +83,10 @@ func (c *certificateV1) PublicKey() []byte {
 	return c.details.publicKey
 }
 
+func (c *certificateV1) MarshalPublicKeyPEM() []byte {
+	return marshalCertPublicKeyToPEM(c)
+}
+
 func (c *certificateV1) Signature() []byte {
 	return c.signature
 }

cert/cert_v1_test.go

@@ -1,6 +1,7 @@
 package cert
 
 import (
+	"crypto/ed25519"
 	"fmt"
 	"net/netip"
 	"testing"
@@ -13,6 +14,7 @@ import (
 )
 
 func TestCertificateV1_Marshal(t *testing.T) {
+	t.Parallel()
 	before := time.Now().Add(time.Second * -60).Round(time.Second)
 	after := time.Now().Add(time.Second * 60).Round(time.Second)
 	pubKey := []byte("1234567890abcedfghij1234567890ab")
@@ -60,6 +62,58 @@ func TestCertificateV1_Marshal(t *testing.T) {
 	assert.Equal(t, nc.Groups(), nc2.Groups())
 }
 
+func TestCertificateV1_PublicKeyPem(t *testing.T) {
+	t.Parallel()
+	before := time.Now().Add(time.Second * -60).Round(time.Second)
+	after := time.Now().Add(time.Second * 60).Round(time.Second)
+	pubKey := ed25519.PublicKey("1234567890abcedfghij1234567890ab")
+
+	nc := certificateV1{
+		details: detailsV1{
+			name:           "testing",
+			networks:       []netip.Prefix{},
+			unsafeNetworks: []netip.Prefix{},
+			groups:         []string{"test-group1", "test-group2", "test-group3"},
+			notBefore:      before,
+			notAfter:       after,
+			publicKey:      pubKey,
+			isCA:           false,
+			issuer:         "1234567890abcedfghij1234567890ab",
+		},
+		signature: []byte("1234567890abcedfghij1234567890ab"),
+	}
+
+	assert.Equal(t, Version1, nc.Version())
+	assert.Equal(t, Curve_CURVE25519, nc.Curve())
+	pubPem := "-----BEGIN NEBULA X25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA X25519 PUBLIC KEY-----\n"
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
+	assert.False(t, nc.IsCA())
+
+	nc.details.isCA = true
+	assert.Equal(t, Curve_CURVE25519, nc.Curve())
+	pubPem = "-----BEGIN NEBULA ED25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA ED25519 PUBLIC KEY-----\n"
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
+	assert.True(t, nc.IsCA())
+
+	pubP256KeyPem := []byte(`-----BEGIN NEBULA P256 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAA=
+-----END NEBULA P256 PUBLIC KEY-----
+`)
+	pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem)
+	require.NoError(t, err)
+	nc.details.curve = Curve_P256
+	nc.details.publicKey = pubP256Key
+	assert.Equal(t, Curve_P256, nc.Curve())
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
+	assert.True(t, nc.IsCA())
+
+	nc.details.isCA = false
+	assert.Equal(t, Curve_P256, nc.Curve())
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
+	assert.False(t, nc.IsCA())
+}
+
 func TestCertificateV1_Expired(t *testing.T) {
 	nc := certificateV1{
 		details: detailsV1{

cert/cert_v2.go

@@ -114,6 +114,10 @@ func (c *certificateV2) PublicKey() []byte {
 	return c.publicKey
 }
 
+func (c *certificateV2) MarshalPublicKeyPEM() []byte {
+	return marshalCertPublicKeyToPEM(c)
+}
+
 func (c *certificateV2) Signature() []byte {
 	return c.signature
 }

cert/cert_v2_test.go

@@ -15,6 +15,7 @@ import (
 )
 
 func TestCertificateV2_Marshal(t *testing.T) {
+	t.Parallel()
 	before := time.Now().Add(time.Second * -60).Round(time.Second)
 	after := time.Now().Add(time.Second * 60).Round(time.Second)
 	pubKey := []byte("1234567890abcedfghij1234567890ab")
@@ -75,6 +76,58 @@ func TestCertificateV2_Marshal(t *testing.T) {
 	assert.Equal(t, nc.Groups(), nc2.Groups())
 }
 
+func TestCertificateV2_PublicKeyPem(t *testing.T) {
+	t.Parallel()
+	before := time.Now().Add(time.Second * -60).Round(time.Second)
+	after := time.Now().Add(time.Second * 60).Round(time.Second)
+	pubKey := ed25519.PublicKey("1234567890abcedfghij1234567890ab")
+
+	nc := certificateV2{
+		details: detailsV2{
+			name:           "testing",
+			networks:       []netip.Prefix{},
+			unsafeNetworks: []netip.Prefix{},
+			groups:         []string{"test-group1", "test-group2", "test-group3"},
+			notBefore:      before,
+			notAfter:       after,
+			isCA:           false,
+			issuer:         "1234567890abcedfghij1234567890ab",
+		},
+		publicKey: pubKey,
+		signature: []byte("1234567890abcedfghij1234567890ab"),
+	}
+
+	assert.Equal(t, Version2, nc.Version())
+	assert.Equal(t, Curve_CURVE25519, nc.Curve())
+	pubPem := "-----BEGIN NEBULA X25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA X25519 PUBLIC KEY-----\n"
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
+	assert.False(t, nc.IsCA())
+
+	nc.details.isCA = true
+	assert.Equal(t, Curve_CURVE25519, nc.Curve())
+	pubPem = "-----BEGIN NEBULA ED25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA ED25519 PUBLIC KEY-----\n"
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
+	assert.True(t, nc.IsCA())
+
+	pubP256KeyPem := []byte(`-----BEGIN NEBULA P256 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAA=
+-----END NEBULA P256 PUBLIC KEY-----
+`)
+	pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem)
+	require.NoError(t, err)
+	nc.curve = Curve_P256
+	nc.publicKey = pubP256Key
+	assert.Equal(t, Curve_P256, nc.Curve())
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
+	assert.True(t, nc.IsCA())
+
+	nc.details.isCA = false
+	assert.Equal(t, Curve_P256, nc.Curve())
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
+	assert.False(t, nc.IsCA())
+}
+
 func TestCertificateV2_Expired(t *testing.T) {
 	nc := certificateV2{
 		details: detailsV2{

cert/pem.go

@@ -7,19 +7,26 @@ import (
 	"golang.org/x/crypto/ed25519"
 )
 
-const (
+const ( //cert banners
 	CertificateBanner   = "NEBULA CERTIFICATE"
 	CertificateV2Banner = "NEBULA CERTIFICATE V2"
+)
+
+const ( //key-agreement-key banners
 	X25519PrivateKeyBanner = "NEBULA X25519 PRIVATE KEY"
 	X25519PublicKeyBanner  = "NEBULA X25519 PUBLIC KEY"
+	P256PrivateKeyBanner   = "NEBULA P256 PRIVATE KEY"
+	P256PublicKeyBanner    = "NEBULA P256 PUBLIC KEY"
+)
+
+/* including "ECDSA" in the P256 banners is a clue that these keys should be used only for signing */
+const ( //signing key banners
+	EncryptedECDSAP256PrivateKeyBanner = "NEBULA ECDSA P256 ENCRYPTED PRIVATE KEY"
+	ECDSAP256PrivateKeyBanner          = "NEBULA ECDSA P256 PRIVATE KEY"
+	ECDSAP256PublicKeyBanner           = "NEBULA ECDSA P256 PUBLIC KEY"
 	EncryptedEd25519PrivateKeyBanner   = "NEBULA ED25519 ENCRYPTED PRIVATE KEY"
 	Ed25519PrivateKeyBanner            = "NEBULA ED25519 PRIVATE KEY"
 	Ed25519PublicKeyBanner             = "NEBULA ED25519 PUBLIC KEY"
-
-	P256PrivateKeyBanner               = "NEBULA P256 PRIVATE KEY"
-	P256PublicKeyBanner                = "NEBULA P256 PUBLIC KEY"
-	EncryptedECDSAP256PrivateKeyBanner = "NEBULA ECDSA P256 ENCRYPTED PRIVATE KEY"
-	ECDSAP256PrivateKeyBanner          = "NEBULA ECDSA P256 PRIVATE KEY"
 )
 
 // UnmarshalCertificateFromPEM will try to unmarshal the first pem block in a byte array, returning any non consumed
@@ -51,6 +58,16 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
 
 }
 
+func marshalCertPublicKeyToPEM(c Certificate) []byte {
+	if c.IsCA() {
+		return MarshalSigningPublicKeyToPEM(c.Curve(), c.PublicKey())
+	} else {
+		return MarshalPublicKeyToPEM(c.Curve(), c.PublicKey())
+	}
+}
+
+// MarshalPublicKeyToPEM returns a PEM representation of a public key used for ECDH.
+// if your public key came from a certificate, prefer Certificate.PublicKeyPEM() if possible, to avoid mistakes!
 func MarshalPublicKeyToPEM(curve Curve, b []byte) []byte {
 	switch curve {
 	case Curve_CURVE25519:
@@ -62,6 +79,19 @@ func MarshalPublicKeyToPEM(curve Curve, b []byte) []byte {
 	}
 }
 
+// MarshalSigningPublicKeyToPEM returns a PEM representation of a public key used for signing.
+// if your public key came from a certificate, prefer Certificate.PublicKeyPEM() if possible, to avoid mistakes!
+func MarshalSigningPublicKeyToPEM(curve Curve, b []byte) []byte {
+	switch curve {
+	case Curve_CURVE25519:
+		return pem.EncodeToMemory(&pem.Block{Type: Ed25519PublicKeyBanner, Bytes: b})
+	case Curve_P256:
+		return pem.EncodeToMemory(&pem.Block{Type: P256PublicKeyBanner, Bytes: b})
+	default:
+		return nil
+	}
+}
+
 func UnmarshalPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {
 	k, r := pem.Decode(b)
 	if k == nil {
@@ -73,7 +103,7 @@ func UnmarshalPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {
 	case X25519PublicKeyBanner, Ed25519PublicKeyBanner:
 		expectedLen = 32
 		curve = Curve_CURVE25519
-	case P256PublicKeyBanner:
+	case P256PublicKeyBanner, ECDSAP256PublicKeyBanner:
 		// Uncompressed
 		expectedLen = 65
 		curve = Curve_P256

cert/pem_test.go

@@ -177,6 +177,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 }
 
 func TestUnmarshalPublicKeyFromPEM(t *testing.T) {
+	t.Parallel()
 	pubKey := []byte(`# A good key
 -----BEGIN NEBULA ED25519 PUBLIC KEY-----
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
@@ -230,6 +231,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 }
 
 func TestUnmarshalX25519PublicKey(t *testing.T) {
+	t.Parallel()
 	pubKey := []byte(`# A good key
 -----BEGIN NEBULA X25519 PUBLIC KEY-----
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
@@ -240,6 +242,12 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 AAAAAAAAAAAAAAAAAAAAAAA=
 -----END NEBULA P256 PUBLIC KEY-----
+`)
+	oldPubP256Key := []byte(`# A good key
+-----BEGIN NEBULA ECDSA P256 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAA=
+-----END NEBULA ECDSA P256 PUBLIC KEY-----
 `)
 	shortKey := []byte(`# A short key
 -----BEGIN NEBULA X25519 PUBLIC KEY-----
@@ -256,15 +264,22 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 -END NEBULA X25519 PUBLIC KEY-----`)
 
-	keyBundle := appendByteSlices(pubKey, pubP256Key, shortKey, invalidBanner, invalidPem)
+	keyBundle := appendByteSlices(pubKey, pubP256Key, oldPubP256Key, shortKey, invalidBanner, invalidPem)
 
 	// Success test case
 	k, rest, curve, err := UnmarshalPublicKeyFromPEM(keyBundle)
 	assert.Len(t, k, 32)
 	require.NoError(t, err)
-	assert.Equal(t, rest, appendByteSlices(pubP256Key, shortKey, invalidBanner, invalidPem))
+	assert.Equal(t, rest, appendByteSlices(pubP256Key, oldPubP256Key, shortKey, invalidBanner, invalidPem))
 	assert.Equal(t, Curve_CURVE25519, curve)
 
+	// Success test case
+	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
+	assert.Len(t, k, 65)
+	require.NoError(t, err)
+	assert.Equal(t, rest, appendByteSlices(oldPubP256Key, shortKey, invalidBanner, invalidPem))
+	assert.Equal(t, Curve_P256, curve)
+
 	// Success test case
 	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Len(t, k, 65)

cert_test/cert.go

@@ -114,33 +114,6 @@ func NewTestCert(v cert.Version, curve cert.Curve, ca cert.Certificate, key []by
 	return c, pub, cert.MarshalPrivateKeyToPEM(curve, priv), pem
 }
 
-func NewTestCertDifferentVersion(c cert.Certificate, v cert.Version, ca cert.Certificate, key []byte) (cert.Certificate, []byte) {
-	nc := &cert.TBSCertificate{
-		Version:        v,
-		Curve:          c.Curve(),
-		Name:           c.Name(),
-		Networks:       c.Networks(),
-		UnsafeNetworks: c.UnsafeNetworks(),
-		Groups:         c.Groups(),
-		NotBefore:      time.Unix(c.NotBefore().Unix(), 0),
-		NotAfter:       time.Unix(c.NotAfter().Unix(), 0),
-		PublicKey:      c.PublicKey(),
-		IsCA:           false,
-	}
-
-	c, err := nc.Sign(ca, ca.Curve(), key)
-	if err != nil {
-		panic(err)
-	}
-
-	pem, err := c.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-
-	return c, pem
-}
-
 func X25519Keypair() ([]byte, []byte) {
 	privkey := make([]byte, 32)
 	if _, err := io.ReadFull(rand.Reader, privkey); err != nil {

cmd/gso/gso.go

@@ -0,0 +1,191 @@
+package main
+
+import (
+	"encoding/binary"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"net/netip"
+	"time"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// UDP_SEGMENT enables GSO segmentation
+	UDP_SEGMENT = 103
+	// Maximum GSO segment size (typical MTU - headers)
+	maxGSOSize = 1400
+)
+
+func main() {
+	destAddr := flag.String("dest", "10.4.0.16:4202", "Destination address")
+	gsoSize := flag.Int("gso", 1400, "GSO segment size")
+	totalSize := flag.Int("size", 14000, "Total payload size to send")
+	count := flag.Int("count", 1, "Number of packets to send")
+	flag.Parse()
+
+	if *gsoSize > maxGSOSize {
+		log.Fatalf("GSO size %d exceeds maximum %d", *gsoSize, maxGSOSize)
+	}
+
+	// Resolve destination address
+	_, err := net.ResolveUDPAddr("udp", *destAddr)
+	if err != nil {
+		log.Fatalf("Failed to resolve address: %v", err)
+	}
+
+	// Create a raw UDP socket with GSO support
+	fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
+	if err != nil {
+		log.Fatalf("Failed to create socket: %v", err)
+	}
+	defer unix.Close(fd)
+
+	// Bind to a local address
+	localAddr := &unix.SockaddrInet4{
+		Port: 0, // Let the system choose a port
+	}
+	if err := unix.Bind(fd, localAddr); err != nil {
+		log.Fatalf("Failed to bind socket: %v", err)
+	}
+
+	fmt.Printf("Sending UDP packets with GSO enabled\n")
+	fmt.Printf("Destination: %s\n", *destAddr)
+	fmt.Printf("GSO segment size: %d bytes\n", *gsoSize)
+	fmt.Printf("Total payload size: %d bytes\n", *totalSize)
+	fmt.Printf("Number of packets: %d\n\n", *count)
+
+	// Create payload
+	payload := make([]byte, *totalSize)
+	for i := range payload {
+		payload[i] = byte(i % 256)
+	}
+
+	dest := netip.MustParseAddrPort(*destAddr)
+
+	//if err := unix.SetsockoptInt(fd, unix.SOL_UDP, unix.UDP_SEGMENT, 1400); err != nil {
+	//	panic(err)
+	//}
+
+	for i := 0; i < *count; i++ {
+		err := WriteBatch(fd, payload, dest, uint16(*gsoSize), true)
+		if err != nil {
+			log.Printf("Send error on packet %d: %v", i, err)
+			continue
+		}
+
+		if (i+1)%100 == 0 || i == *count-1 {
+			fmt.Printf("Sent %d packets\n", i+1)
+		}
+	}
+	fmt.Printf("now, let's send without the correct ctrl header\n")
+	time.Sleep(time.Second)
+	for i := 0; i < *count; i++ {
+		err := WriteBatch(fd, payload, dest, uint16(*gsoSize), false)
+		if err != nil {
+			log.Printf("Send error on packet %d: %v", i, err)
+			continue
+		}
+
+		if (i+1)%100 == 0 || i == *count-1 {
+			fmt.Printf("Sent %d packets\n", i+1)
+		}
+	}
+
+}
+
+func WriteBatch(fd int, payload []byte, addr netip.AddrPort, segSize uint16, withHeader bool) error {
+	msgs := make([]rawMessage, 0, 1)
+	iovs := make([]iovec, 0, 1)
+	names := make([][unix.SizeofSockaddrInet6]byte, 0, 1)
+
+	sent := 0
+
+	pkts := []BatchPacket{
+		{
+			Payload: payload,
+			Addr:    addr,
+		},
+	}
+
+	for _, pkt := range pkts {
+		if len(pkt.Payload) == 0 {
+			sent++
+			continue
+		}
+
+		msgs = append(msgs, rawMessage{})
+		iovs = append(iovs, iovec{})
+		names = append(names, [unix.SizeofSockaddrInet6]byte{})
+
+		idx := len(msgs) - 1
+		msg := &msgs[idx]
+		iov := &iovs[idx]
+		name := &names[idx]
+
+		setIovecSlice(iov, pkt.Payload)
+		msg.Hdr.Iov = iov
+		msg.Hdr.Iovlen = 1
+
+		if withHeader {
+			setRawMessageControl(msg, buildGSOControlMessage(segSize)) //
+		} else {
+			setRawMessageControl(msg, nil) //
+		}
+
+		msg.Hdr.Flags = 0
+
+		nameLen, err := encodeSockaddr(name[:], pkt.Addr)
+		if err != nil {
+			return err
+		}
+		msg.Hdr.Name = &name[0]
+		msg.Hdr.Namelen = nameLen
+	}
+
+	if len(msgs) == 0 {
+		return errors.New("nothing to write")
+	}
+
+	offset := 0
+	for offset < len(msgs) {
+		n, _, errno := unix.Syscall6(
+			unix.SYS_SENDMMSG,
+			uintptr(fd),
+			uintptr(unsafe.Pointer(&msgs[offset])),
+			uintptr(len(msgs)-offset),
+			0,
+			0,
+			0,
+		)
+
+		if errno != 0 {
+			if errno == unix.EINTR {
+				continue
+			}
+			return &net.OpError{Op: "sendmmsg", Err: errno}
+		}
+
+		if n == 0 {
+			break
+		}
+		offset += int(n)
+	}
+
+	return nil
+}
+
+func buildGSOControlMessage(segSize uint16) []byte {
+	control := make([]byte, unix.CmsgSpace(2))
+	hdr := (*unix.Cmsghdr)(unsafe.Pointer(&control[0]))
+	hdr.Level = unix.SOL_UDP
+	hdr.Type = unix.UDP_SEGMENT
+	setCmsgLen(hdr, unix.CmsgLen(2))
+	binary.NativeEndian.PutUint16(control[unix.CmsgLen(0):unix.CmsgLen(0)+2], uint16(segSize))
+
+	return control
+}

cmd/gso/helper.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net/netip"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+type iovec struct {
+	Base *byte
+	Len  uint64
+}
+
+type msghdr struct {
+	Name       *byte
+	Namelen    uint32
+	Pad0       [4]byte
+	Iov        *iovec
+	Iovlen     uint64
+	Control    *byte
+	Controllen uint64
+	Flags      int32
+	Pad1       [4]byte
+}
+
+type rawMessage struct {
+	Hdr  msghdr
+	Len  uint32
+	Pad0 [4]byte
+}
+
+type BatchPacket struct {
+	Payload []byte
+	Addr    netip.AddrPort
+}
+
+func encodeSockaddr(dst []byte, addr netip.AddrPort) (uint32, error) {
+	if addr.Addr().Is4() {
+		if !addr.Addr().Is4() {
+			return 0, fmt.Errorf("Listener is IPv4, but writing to IPv6 remote")
+		}
+		var sa unix.RawSockaddrInet4
+		sa.Family = unix.AF_INET
+		sa.Addr = addr.Addr().As4()
+		binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
+		size := unix.SizeofSockaddrInet4
+		copy(dst[:size], (*(*[unix.SizeofSockaddrInet4]byte)(unsafe.Pointer(&sa)))[:])
+		return uint32(size), nil
+	}
+
+	var sa unix.RawSockaddrInet6
+	sa.Family = unix.AF_INET6
+	sa.Addr = addr.Addr().As16()
+	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
+	size := unix.SizeofSockaddrInet6
+	copy(dst[:size], (*(*[unix.SizeofSockaddrInet6]byte)(unsafe.Pointer(&sa)))[:])
+	return uint32(size), nil
+}
+
+func setRawMessageControl(msg *rawMessage, buf []byte) {
+	if len(buf) == 0 {
+		msg.Hdr.Control = nil
+		msg.Hdr.Controllen = 0
+		return
+	}
+	msg.Hdr.Control = &buf[0]
+	msg.Hdr.Controllen = uint64(len(buf))
+}
+
+func setCmsgLen(h *unix.Cmsghdr, l int) {
+	h.Len = uint64(l)
+}
+
+func setIovecSlice(iov *iovec, b []byte) {
+	if len(b) == 0 {
+		iov.Base = nil
+		iov.Len = 0
+		return
+	}
+	iov.Base = &b[0]
+	iov.Len = uint64(len(b))
+}

cmd/nebula-service/main.go

@@ -65,8 +65,16 @@ func main() {
 	}
 
 	if !*configTest {
-		ctrl.Start()
-		ctrl.ShutdownBlock()
+		wait, err := ctrl.Start()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
+		wait()
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

cmd/nebula/main.go

@@ -3,6 +3,9 @@ package main
 import (
 	"flag"
 	"fmt"
+	"log"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 
 	"github.com/sirupsen/logrus"
@@ -58,10 +61,22 @@ func main() {
 		os.Exit(1)
 	}
 
+	go func() {
+		log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
+	}()
+
 	if !*configTest {
-		ctrl.Start()
+		wait, err := ctrl.Start()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
 		notifyReady(l)
-		ctrl.ShutdownBlock()
+		wait()
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

connection_manager.go

@@ -354,6 +354,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 
 		if mainHostInfo {
 			decision = tryRehandshake
+
 		} else {
 			if cm.shouldSwapPrimary(hostinfo) {
 				decision = swapPrimary
@@ -460,10 +461,6 @@ func (cm *connectionManager) shouldSwapPrimary(current *HostInfo) bool {
 	}
 
 	crt := cm.intf.pki.getCertState().getCertificate(current.ConnectionState.myCert.Version())
-	if crt == nil {
-		//my cert was reloaded away. We should definitely swap from this tunnel
-		return true
-	}
 	// If this tunnel is using the latest certificate then we should swap it to primary for a bit and see if things
 	// settle down.
 	return bytes.Equal(current.ConnectionState.myCert.Signature(), crt.Signature())
@@ -478,34 +475,31 @@ func (cm *connectionManager) swapPrimary(current, primary *HostInfo) {
 	cm.hostMap.Unlock()
 }
 
-// isInvalidCertificate decides if we should destroy a tunnel.
-// returns true if pki.disconnect_invalid is true and the certificate is no longer valid.
-// Blocklisted certificates will skip the pki.disconnect_invalid check and return true.
+// isInvalidCertificate will check if we should destroy a tunnel if pki.disconnect_invalid is true and
+// the certificate is no longer valid. Block listed certificates will skip the pki.disconnect_invalid
+// check and return true.
 func (cm *connectionManager) isInvalidCertificate(now time.Time, hostinfo *HostInfo) bool {
 	remoteCert := hostinfo.GetCert()
 	if remoteCert == nil {
-		return false //don't tear down tunnels for handshakes in progress
+		return false
 	}
 
 	caPool := cm.intf.pki.GetCAPool()
 	err := caPool.VerifyCachedCertificate(now, remoteCert)
 	if err == nil {
-		return false //cert is still valid! yay!
-	} else if err == cert.ErrBlockListed { //avoiding errors.Is for speed
+		return false
+	}
+
+	if !cm.intf.disconnectInvalid.Load() && err != cert.ErrBlockListed {
 		// Block listed certificates should always be disconnected
-		hostinfo.logger(cm.l).WithError(err).
-			WithField("fingerprint", remoteCert.Fingerprint).
-			Info("Remote certificate is blocked, tearing down the tunnel")
-		return true
-	} else if cm.intf.disconnectInvalid.Load() {
+		return false
+	}
+
 	hostinfo.logger(cm.l).WithError(err).
 		WithField("fingerprint", remoteCert.Fingerprint).
 		Info("Remote certificate is no longer valid, tearing down the tunnel")
+
 	return true
-	} else {
-		//if we reach here, the cert is no longer valid, but we're configured to keep tunnels from now-invalid certs open
-		return false
-	}
 }
 
 func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
@@ -536,45 +530,15 @@ func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
 func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
 	cs := cm.intf.pki.getCertState()
 	curCrt := hostinfo.ConnectionState.myCert
-	curCrtVersion := curCrt.Version()
-	myCrt := cs.getCertificate(curCrtVersion)
-	if myCrt == nil {
-		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-			WithField("version", curCrtVersion).
-			WithField("reason", "local certificate removed").
-			Info("Re-handshaking with remote")
-		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
+	myCrt := cs.getCertificate(curCrt.Version())
+	if curCrt.Version() >= cs.initiatingVersion && bytes.Equal(curCrt.Signature(), myCrt.Signature()) == true {
+		// The current tunnel is using the latest certificate and version, no need to rehandshake.
 		return
 	}
-	peerCrt := hostinfo.ConnectionState.peerCert
-	if peerCrt != nil && curCrtVersion < peerCrt.Certificate.Version() {
-		// if our certificate version is less than theirs, and we have a matching version available, rehandshake?
-		if cs.getCertificate(peerCrt.Certificate.Version()) != nil {
-			cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-				WithField("version", curCrtVersion).
-				WithField("peerVersion", peerCrt.Certificate.Version()).
-				WithField("reason", "local certificate version lower than peer, attempting to correct").
-				Info("Re-handshaking with remote")
-			cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], func(hh *HandshakeHostInfo) {
-				hh.initiatingVersionOverride = peerCrt.Certificate.Version()
-			})
-			return
-		}
-	}
-	if !bytes.Equal(curCrt.Signature(), myCrt.Signature()) {
+
 	cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
 		WithField("reason", "local certificate is not current").
 		Info("Re-handshaking with remote")
 
 	cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
-		return
-	}
-	if curCrtVersion < cs.initiatingVersion {
-		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-			WithField("reason", "current cert version < pki.initiatingVersion").
-			Info("Re-handshaking with remote")
-
-		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
-		return
-	}
 }

connection_manager_test.go

@@ -446,6 +446,10 @@ func (d *dummyCert) PublicKey() []byte {
 	return d.publicKey
 }
 
+func (d *dummyCert) MarshalPublicKeyPEM() []byte {
+	return cert.MarshalPublicKeyToPEM(d.curve, d.publicKey)
+}
+
 func (d *dummyCert) Signature() []byte {
 	return d.signature
 }

connection_state.go

@@ -13,7 +13,9 @@ import (
 	"github.com/slackhq/nebula/noiseutil"
 )
 
-const ReplayWindow = 1024
+// TODO: In a 5Gbps test, 1024 is not sufficient. With a 1400 MTU this is about 1.4Gbps of window, assuming full packets.
+// 4092 should be sufficient for 5Gbps
+const ReplayWindow = 8192
 
 type ConnectionState struct {
 	eKey           *NebulaCipherState

control.go

@@ -2,9 +2,11 @@ package nebula
 
 import (
 	"context"
+	"errors"
 	"net/netip"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
@@ -13,6 +15,16 @@ import (
 	"github.com/slackhq/nebula/overlay"
 )
 
+type RunState int
+
+const (
+	Stopped  RunState = 0 // The control has yet to be started
+	Started  RunState = 1 // The control has been started
+	Stopping RunState = 2 // The control is stopping
+)
+
+var ErrAlreadyStarted = errors.New("nebula is already started")
+
 // Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching
 // core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc
 
@@ -26,6 +38,9 @@ type controlHostLister interface {
 }
 
 type Control struct {
+	stateLock sync.Mutex
+	state     RunState
+
 	f                      *Interface
 	l                      *logrus.Logger
 	ctx                    context.Context
@@ -49,10 +64,21 @@ type ControlHostInfo struct {
 	CurrentRelaysThroughMe []netip.Addr     `json:"currentRelaysThroughMe"`
 }
 
-// Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock()
-func (c *Control) Start() {
+// Start actually runs nebula, this is a nonblocking call.
+// The returned function can be used to wait for nebula to fully stop.
+func (c *Control) Start() (func(), error) {
+	c.stateLock.Lock()
+	if c.state != Stopped {
+		c.stateLock.Unlock()
+		return nil, ErrAlreadyStarted
+	}
+
 	// Activate the interface
-	c.f.activate()
+	err := c.f.activate()
+	if err != nil {
+		c.stateLock.Unlock()
+		return nil, err
+	}
 
 	// Call all the delayed funcs that waited patiently for the interface to be created.
 	if c.sshStart != nil {
@@ -72,15 +98,33 @@ func (c *Control) Start() {
 	}
 
 	// Start reading packets.
-	c.f.run()
+	c.state = Started
+	c.stateLock.Unlock()
+	return c.f.run(c.ctx)
+}
+
+func (c *Control) State() RunState {
+	c.stateLock.Lock()
+	defer c.stateLock.Unlock()
+	return c.state
 }
 
 func (c *Control) Context() context.Context {
 	return c.ctx
 }
 
-// Stop signals nebula to shutdown and close all tunnels, returns after the shutdown is complete
+// Stop is a non-blocking call that signals nebula to close all tunnels and shut down
 func (c *Control) Stop() {
+	c.stateLock.Lock()
+	if c.state != Started {
+		c.stateLock.Unlock()
+		// We are stopping or stopped already
+		return
+	}
+
+	c.state = Stopping
+	c.stateLock.Unlock()
+
 	// Stop the handshakeManager (and other services), to prevent new tunnels from
 	// being created while we're shutting them all down.
 	c.cancel()
@@ -89,7 +133,7 @@ func (c *Control) Stop() {
 	if err := c.f.Close(); err != nil {
 		c.l.WithError(err).Error("Close interface failed")
 	}
-	c.l.Info("Goodbye")
+	c.state = Stopped
 }
 
 // ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled

e2e/helpers_test.go

@@ -29,6 +29,8 @@ type m = map[string]any
 
 // newSimpleServer creates a nebula instance with many assumptions
 func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
+	l := NewTestLogger()
+
 	var vpnNetworks []netip.Prefix
 	for _, sn := range strings.Split(sVpnNetworks, ",") {
 		vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn))
@@ -54,25 +56,6 @@ func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name
 		budpIp[3] = 239
 		udpAddr = netip.AddrPortFrom(netip.AddrFrom16(budpIp), 4242)
 	}
-	return newSimpleServerWithUdp(v, caCrt, caKey, name, sVpnNetworks, udpAddr, overrides)
-}
-
-func newSimpleServerWithUdp(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, udpAddr netip.AddrPort, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
-	l := NewTestLogger()
-
-	var vpnNetworks []netip.Prefix
-	for _, sn := range strings.Split(sVpnNetworks, ",") {
-		vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn))
-		if err != nil {
-			panic(err)
-		}
-		vpnNetworks = append(vpnNetworks, vpnIpNet)
-	}
-
-	if len(vpnNetworks) == 0 {
-		panic("no vpn networks")
-	}
-
 	_, _, myPrivKey, myPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, caCrt, caKey, name, time.Now(), time.Now().Add(5*time.Minute), vpnNetworks, nil, []string{})
 
 	caB, err := caCrt.MarshalPEM()
@@ -146,109 +129,6 @@ func newSimpleServerWithUdp(v cert.Version, caCrt cert.Certificate, caKey []byte
 	return control, vpnNetworks, udpAddr, c
 }
 
-// newServer creates a nebula instance with fewer assumptions
-func newServer(caCrt []cert.Certificate, certs []cert.Certificate, key []byte, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
-	l := NewTestLogger()
-
-	vpnNetworks := certs[len(certs)-1].Networks()
-
-	var udpAddr netip.AddrPort
-	if vpnNetworks[0].Addr().Is4() {
-		budpIp := vpnNetworks[0].Addr().As4()
-		budpIp[1] -= 128
-		udpAddr = netip.AddrPortFrom(netip.AddrFrom4(budpIp), 4242)
-	} else {
-		budpIp := vpnNetworks[0].Addr().As16()
-		// beef for funsies
-		budpIp[2] = 190
-		budpIp[3] = 239
-		udpAddr = netip.AddrPortFrom(netip.AddrFrom16(budpIp), 4242)
-	}
-
-	caStr := ""
-	for _, ca := range caCrt {
-		x, err := ca.MarshalPEM()
-		if err != nil {
-			panic(err)
-		}
-		caStr += string(x)
-	}
-	certStr := ""
-	for _, c := range certs {
-		x, err := c.MarshalPEM()
-		if err != nil {
-			panic(err)
-		}
-		certStr += string(x)
-	}
-
-	mc := m{
-		"pki": m{
-			"ca":   caStr,
-			"cert": certStr,
-			"key":  string(key),
-		},
-		//"tun": m{"disabled": true},
-		"firewall": m{
-			"outbound": []m{{
-				"proto": "any",
-				"port":  "any",
-				"host":  "any",
-			}},
-			"inbound": []m{{
-				"proto": "any",
-				"port":  "any",
-				"host":  "any",
-			}},
-		},
-		//"handshakes": m{
-		//	"try_interval": "1s",
-		//},
-		"listen": m{
-			"host": udpAddr.Addr().String(),
-			"port": udpAddr.Port(),
-		},
-		"logging": m{
-			"timestamp_format": fmt.Sprintf("%v 15:04:05.000000", certs[0].Name()),
-			"level":            l.Level.String(),
-		},
-		"timers": m{
-			"pending_deletion_interval": 2,
-			"connection_alive_interval": 2,
-		},
-	}
-
-	if overrides != nil {
-		final := m{}
-		err := mergo.Merge(&final, overrides, mergo.WithAppendSlice)
-		if err != nil {
-			panic(err)
-		}
-		err = mergo.Merge(&final, mc, mergo.WithAppendSlice)
-		if err != nil {
-			panic(err)
-		}
-		mc = final
-	}
-
-	cb, err := yaml.Marshal(mc)
-	if err != nil {
-		panic(err)
-	}
-
-	c := config.NewC(l)
-	cStr := string(cb)
-	c.LoadString(cStr)
-
-	control, err := nebula.Main(c, false, "e2e-test", l, nil)
-
-	if err != nil {
-		panic(err)
-	}
-
-	return control, vpnNetworks, udpAddr, c
-}
-
 type doneCb func()
 
 func deadline(t *testing.T, seconds time.Duration) doneCb {

e2e/tunnels_test.go

@@ -4,16 +4,12 @@
 package e2e
 
 import (
-	"fmt"
-	"net/netip"
 	"testing"
 	"time"
 
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/e2e/router"
-	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v3"
 )
 
 func TestDropInactiveTunnels(t *testing.T) {
@@ -59,309 +55,3 @@ func TestDropInactiveTunnels(t *testing.T) {
 	myControl.Stop()
 	theirControl.Stop()
 }
-
-func TestCertUpgrade(t *testing.T) {
-	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
-	// under ideal conditions
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	caB, err := ca.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	ca2B, err := ca2.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	caStr := fmt.Sprintf("%s\n%s", caB, ca2B)
-
-	myCert, _, myPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
-	_, myCert2Pem := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
-
-	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
-	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
-
-	myControl, myVpnIpNet, myUdpAddr, myC := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert}, myPrivKey, m{})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	r.Log("Assert the tunnel between me and them works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	r.Log("yay")
-	//todo ???
-	time.Sleep(1 * time.Second)
-	r.FlushAll()
-
-	mc := m{
-		"pki": m{
-			"ca":   caStr,
-			"cert": string(myCert2Pem),
-			"key":  string(myPrivKey),
-		},
-		//"tun": m{"disabled": true},
-		"firewall": myC.Settings["firewall"],
-		//"handshakes": m{
-		//	"try_interval": "1s",
-		//},
-		"listen":  myC.Settings["listen"],
-		"logging": myC.Settings["logging"],
-		"timers":  myC.Settings["timers"],
-	}
-
-	cb, err := yaml.Marshal(mc)
-	if err != nil {
-		panic(err)
-	}
-
-	r.Logf("reload new v2-only config")
-	err = myC.ReloadConfigString(string(cb))
-	assert.NoError(t, err)
-	r.Log("yay, spin until their sees it")
-	waitStart := time.Now()
-	for {
-		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-		if c == nil {
-			r.Log("nil")
-		} else {
-			version := c.Cert.Version()
-			r.Logf("version %d", version)
-			if version == cert.Version2 {
-				break
-			}
-		}
-		since := time.Since(waitStart)
-		if since > time.Second*10 {
-			t.Fatal("Cert should be new by now")
-		}
-		time.Sleep(time.Second)
-	}
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestCertDowngrade(t *testing.T) {
-	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
-	// under ideal conditions
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	caB, err := ca.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	ca2B, err := ca2.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	caStr := fmt.Sprintf("%s\n%s", caB, ca2B)
-
-	myCert, _, myPrivKey, myCertPem := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
-	myCert2, _ := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
-
-	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
-	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
-
-	myControl, myVpnIpNet, myUdpAddr, myC := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert2}, myPrivKey, m{})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	r.Log("Assert the tunnel between me and them works")
-	//assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
-	//r.Log("yay")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	r.Log("yay")
-	//todo ???
-	time.Sleep(1 * time.Second)
-	r.FlushAll()
-
-	mc := m{
-		"pki": m{
-			"ca":   caStr,
-			"cert": string(myCertPem),
-			"key":  string(myPrivKey),
-		},
-		"firewall": myC.Settings["firewall"],
-		"listen":   myC.Settings["listen"],
-		"logging":  myC.Settings["logging"],
-		"timers":   myC.Settings["timers"],
-	}
-
-	cb, err := yaml.Marshal(mc)
-	if err != nil {
-		panic(err)
-	}
-
-	r.Logf("reload new v1-only config")
-	err = myC.ReloadConfigString(string(cb))
-	assert.NoError(t, err)
-	r.Log("yay, spin until their sees it")
-	waitStart := time.Now()
-	for {
-		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-		c2 := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-		if c == nil || c2 == nil {
-			r.Log("nil")
-		} else {
-			version := c.Cert.Version()
-			theirVersion := c2.Cert.Version()
-			r.Logf("version %d,%d", version, theirVersion)
-			if version == cert.Version1 {
-				break
-			}
-		}
-		since := time.Since(waitStart)
-		if since > time.Second*5 {
-			r.Log("wtf")
-		}
-		if since > time.Second*10 {
-			r.Log("wtf")
-			t.Fatal("Cert should be new by now")
-		}
-		time.Sleep(time.Second)
-	}
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestCertMismatchCorrection(t *testing.T) {
-	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
-	// under ideal conditions
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	myCert, _, myPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
-	myCert2, _ := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
-
-	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
-	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
-
-	myControl, myVpnIpNet, myUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert2}, myPrivKey, m{})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	r.Log("Assert the tunnel between me and them works")
-	//assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
-	//r.Log("yay")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	r.Log("yay")
-	//todo ???
-	time.Sleep(1 * time.Second)
-	r.FlushAll()
-
-	waitStart := time.Now()
-	for {
-		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-		c2 := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-		if c == nil || c2 == nil {
-			r.Log("nil")
-		} else {
-			version := c.Cert.Version()
-			theirVersion := c2.Cert.Version()
-			r.Logf("version %d,%d", version, theirVersion)
-			if version == theirVersion {
-				break
-			}
-		}
-		since := time.Since(waitStart)
-		if since > time.Second*5 {
-			r.Log("wtf")
-		}
-		if since > time.Second*10 {
-			r.Log("wtf")
-			t.Fatal("Cert should be new by now")
-		}
-		time.Sleep(time.Second)
-	}
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestCrossStackRelaysWork(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24,fc00::1/64", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay  ", "10.128.0.128/24,fc00::128/64", m{"relay": m{"am_relay": true}})
-	theirUdp := netip.MustParseAddrPort("10.0.0.2:4242")
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, "them   ", "fc00::2/64", theirUdp, m{"relay": m{"use_relays": true}})
-
-	//myVpnV4 := myVpnIpNet[0]
-	myVpnV6 := myVpnIpNet[1]
-	relayVpnV4 := relayVpnIpNet[0]
-	relayVpnV6 := relayVpnIpNet[1]
-	theirVpnV6 := theirVpnIpNet[0]
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnV4.Addr(), relayUdpAddr)
-	myControl.InjectLightHouseAddr(relayVpnV6.Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnV6.Addr(), []netip.Addr{relayVpnV6.Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnV6.Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me"))
-
-	p := r.RouteForAllUntilTxTun(theirControl)
-	r.Log("Assert the tunnel works")
-	assertUdpPacket(t, []byte("Hi from me"), p, myVpnV6.Addr(), theirVpnV6.Addr(), 80, 80)
-
-	t.Log("reply?")
-	theirControl.InjectTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them"))
-	p = r.RouteForAllUntilTxTun(myControl)
-	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnV6.Addr(), myVpnV6.Addr(), 80, 80)
-
-	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
-	//t.Log("finish up")
-	//myControl.Stop()
-	//theirControl.Stop()
-	//relayControl.Stop()
-}

handshake_ix.go

@@ -23,19 +23,15 @@ func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
 		return false
 	}
 
+	// If we're connecting to a v6 address we must use a v2 cert
 	cs := f.pki.getCertState()
 	v := cs.initiatingVersion
-	if hh.initiatingVersionOverride != cert.VersionPre1 {
-		v = hh.initiatingVersionOverride
-	} else if v < cert.Version2 {
-		// If we're connecting to a v6 address we must use a v2 cert
 	for _, a := range hh.hostinfo.vpnAddrs {
 		if a.Is6() {
 			v = cert.Version2
 			break
 		}
 	}
-	}
 
 	crt := cs.getCertificate(v)
 	if crt == nil {
@@ -167,19 +163,16 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 
 	if remoteCert.Certificate.Version() != ci.myCert.Version() {
 		// We started off using the wrong certificate version, lets see if we can match the version that was sent to us
-		myCertOtherVersion := cs.getCertificate(remoteCert.Certificate.Version())
-		if myCertOtherVersion == nil {
-			if f.l.Level >= logrus.DebugLevel {
-				f.l.WithError(err).WithFields(m{
-					"udpAddr":   addr,
-					"handshake": m{"stage": 1, "style": "ix_psk0"},
-					"cert":      remoteCert,
-				}).Debug("Might be unable to handshake with host due to missing certificate version")
+		rc := cs.getCertificate(remoteCert.Certificate.Version())
+		if rc == nil {
+			f.l.WithError(err).WithField("udpAddr", addr).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cert", remoteCert).
+				Info("Unable to handshake with host due to missing certificate version")
+			return
 		}
-		} else {
+
 		// Record the certificate we are actually using
-			ci.myCert = myCertOtherVersion
-		}
+		ci.myCert = rc
 	}
 
 	if len(remoteCert.Certificate.Networks()) == 0 {

handshake_manager.go

@@ -70,7 +70,6 @@ type HandshakeHostInfo struct {
 
 	startTime   time.Time        // Time that we first started trying with this handshake
 	ready       bool             // Is the handshake ready
-	initiatingVersionOverride cert.Version     // Should we use a non-default cert version for this handshake?
 	counter     int64            // How many attempts have we made so far
 	lastRemotes []netip.AddrPort // Remotes that we sent to during the previous attempt
 	packetStore []*cachedPacket  // A set of packets to be transmitted once the handshake completes
@@ -300,8 +299,6 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 						InitiatorRelayIndex: idx,
 					}
 
-					relayFrom := hm.f.myVpnAddrs[0]
-
 					switch relayHostInfo.GetCert().Certificate.Version() {
 					case cert.Version1:
 						if !hm.f.myVpnAddrs[0].Is4() {
@@ -319,13 +316,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 						b = vpnIp.As4()
 						m.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 					case cert.Version2:
-						if vpnIp.Is4() {
-							relayFrom = hm.f.myVpnAddrs[0]
-						} else {
-							//todo do this smarter
-							relayFrom = hm.f.myVpnAddrs[len(hm.f.myVpnAddrs)-1]
-						}
-						m.RelayFromAddr = netAddrToProtoAddr(relayFrom)
+						m.RelayFromAddr = netAddrToProtoAddr(hm.f.myVpnAddrs[0])
 						m.RelayToAddr = netAddrToProtoAddr(vpnIp)
 					default:
 						hostinfo.logger(hm.l).Error("Unknown certificate version found while creating relay")
@@ -340,7 +331,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 					} else {
 						hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
 						hm.l.WithFields(logrus.Fields{
-							"relayFrom":           relayFrom,
+							"relayFrom":           hm.f.myVpnAddrs[0],
 							"relayTo":             vpnIp,
 							"initiatorRelayIndex": idx,
 							"relay":               relay}).
@@ -366,8 +357,6 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 					InitiatorRelayIndex: existingRelay.LocalIndex,
 				}
 
-				relayFrom := hm.f.myVpnAddrs[0]
-
 				switch relayHostInfo.GetCert().Certificate.Version() {
 				case cert.Version1:
 					if !hm.f.myVpnAddrs[0].Is4() {
@@ -385,14 +374,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 					b = vpnIp.As4()
 					m.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 				case cert.Version2:
-					if vpnIp.Is4() {
-						relayFrom = hm.f.myVpnAddrs[0]
-					} else {
-						//todo do this smarter
-						relayFrom = hm.f.myVpnAddrs[len(hm.f.myVpnAddrs)-1]
-					}
-
-					m.RelayFromAddr = netAddrToProtoAddr(relayFrom)
+					m.RelayFromAddr = netAddrToProtoAddr(hm.f.myVpnAddrs[0])
 					m.RelayToAddr = netAddrToProtoAddr(vpnIp)
 				default:
 					hostinfo.logger(hm.l).Error("Unknown certificate version found while creating relay")
@@ -407,7 +389,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 					// This must send over the hostinfo, not over hm.Hosts[ip]
 					hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
 					hm.l.WithFields(logrus.Fields{
-						"relayFrom":           relayFrom,
+						"relayFrom":           hm.f.myVpnAddrs[0],
 						"relayTo":             vpnIp,
 						"initiatorRelayIndex": existingRelay.LocalIndex,
 						"relay":               relay}).

hostmap.go

@@ -2,7 +2,6 @@ package nebula
 
 import (
 	"errors"
-	"fmt"
 	"net"
 	"net/netip"
 	"slices"
@@ -522,7 +521,6 @@ func (hm *HostMap) QueryVpnAddrsRelayFor(targetIps []netip.Addr, relayHostIp net
 		return nil, nil, errors.New("unable to find host")
 	}
 
-	lastH := h
 	for h != nil {
 		for _, targetIp := range targetIps {
 			r, ok := h.relayState.QueryRelayForByIp(targetIp)
@@ -530,12 +528,10 @@ func (hm *HostMap) QueryVpnAddrsRelayFor(targetIps []netip.Addr, relayHostIp net
 				return h, r, nil
 			}
 		}
-		lastH = h
 		h = h.next
 	}
 
-	//todo no merge
-	return nil, nil, fmt.Errorf("unable to find host with relay: %v", lastH)
+	return nil, nil, errors.New("unable to find host with relay")
 }
 
 func (hm *HostMap) unlockedDisestablishVpnAddrRelayFor(hi *HostInfo) {

interface.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"io"
 	"net/netip"
-	"os"
 	"runtime"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -18,6 +18,7 @@ import (
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
+	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/udp"
 )
 
@@ -87,12 +88,17 @@ type Interface struct {
 
 	writers []udp.Conn
 	readers []io.ReadWriteCloser
+	wg      sync.WaitGroup
 
 	metricHandshakes    metrics.Histogram
 	messageMetrics      *MessageMetrics
 	cachedPacketMetrics *cachedPacketMetrics
 
 	l *logrus.Logger
+
+	pktPool  *packet.Pool
+	inbound  chan *packet.Packet
+	outbound chan *packet.Packet
 }
 
 type EncWriter interface {
@@ -194,9 +200,15 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 			dropped: metrics.GetOrRegisterCounter("hostinfo.cached_packets.dropped", nil),
 		},
 
+		//TODO: configurable size
+		inbound:  make(chan *packet.Packet, 2048),
+		outbound: make(chan *packet.Packet, 2048),
+
 		l: c.l,
 	}
 
+	ifce.pktPool = packet.GetPool()
+
 	ifce.tryPromoteEvery.Store(c.tryPromoteEvery)
 	ifce.reQueryEvery.Store(c.reQueryEvery)
 	ifce.reQueryWait.Store(int64(c.reQueryWait))
@@ -209,7 +221,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 // activate creates the interface on the host. After the interface is created, any
 // other services that want to bind listeners to its IP may do so successfully. However,
 // the interface isn't going to process anything until run() is called.
-func (f *Interface) activate() {
+func (f *Interface) activate() error {
 	// actually turn on tun dev
 
 	addr, err := f.outside.LocalAddr()
@@ -230,33 +242,46 @@ func (f *Interface) activate() {
 		if i > 0 {
 			reader, err = f.inside.NewMultiQueueReader()
 			if err != nil {
-				f.l.Fatal(err)
+				return err
 			}
 		}
 		f.readers[i] = reader
 	}
 
-	if err := f.inside.Activate(); err != nil {
+	if err = f.inside.Activate(); err != nil {
 		f.inside.Close()
-		f.l.Fatal(err)
-	}
+		return err
 	}
 
-func (f *Interface) run() {
-	// Launch n queues to read packets from udp
+	return nil
+}
+
+func (f *Interface) run(c context.Context) (func(), error) {
 	for i := 0; i < f.routines; i++ {
+		// read packets from udp and queue to f.inbound
+		f.wg.Add(1)
 		go f.listenOut(i)
+
+		// Launch n queues to read packets from inside tun dev and queue to f.outbound
+		//todo this never stops f.wg.Add(1)
+		go f.listenIn(f.readers[i], i)
+
+		// Launch n workers to process traffic from f.inbound and smash it onto the inside of the tun
+		f.wg.Add(1)
+		go f.workerIn(i, c)
+		f.wg.Add(1)
+		go f.workerIn(i, c)
+
+		// read from f.outbound and write to UDP (outside the tun)
+		f.wg.Add(1)
+		go f.workerOut(i, c)
 	}
 
-	// Launch n queues to read packets from tun dev
-	for i := 0; i < f.routines; i++ {
-		go f.listenIn(f.readers[i], i)
-	}
+	return f.wg.Wait, nil
 }
 
 func (f *Interface) listenOut(i int) {
 	runtime.LockOSThread()
-
 	var li udp.Conn
 	if i > 0 {
 		li = f.writers[i]
@@ -264,41 +289,90 @@ func (f *Interface) listenOut(i int) {
 		li = f.outside
 	}
 
-	ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
-	lhh := f.lightHouse.NewRequestHandler()
-	plaintext := make([]byte, udp.MTU)
-	h := &header.H{}
-	fwPacket := &firewall.Packet{}
-	nb := make([]byte, 12, 12)
+	err := li.ListenOut(f.pktPool.Get, f.inbound)
+	if err != nil && !f.closed.Load() {
+		f.l.WithError(err).Error("Error while reading packet inbound packet, closing")
+		//TODO: Trigger Control to close
+	}
 
-	li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
-		f.readOutsidePackets(fromUdpAddr, nil, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get(f.l))
-	})
+	f.l.Debugf("underlay reader %v is done", i)
+	f.wg.Done()
 }
 
 func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
 	runtime.LockOSThread()
 
-	packet := make([]byte, mtu)
-	out := make([]byte, mtu)
-	fwPacket := &firewall.Packet{}
-	nb := make([]byte, 12, 12)
+	for {
+		p := f.pktPool.Get()
+		n, err := reader.Read(p.Payload)
+		if err != nil {
+			if !f.closed.Load() {
+				f.l.WithError(err).Error("Error while reading outbound packet, closing")
+				//TODO: Trigger Control to close
+			}
+			break
+		}
 
+		p.Payload = (p.Payload)[:n]
+		//TODO: nonblocking channel write
+		f.outbound <- p
+		//select {
+		//case f.outbound <- p:
+		//default:
+		//	f.l.Error("Dropped packet from outbound channel")
+		//}
+	}
+
+	f.l.Debugf("overlay reader %v is done", i)
+	f.wg.Done()
+}
+
+func (f *Interface) workerIn(i int, ctx context.Context) {
+	lhh := f.lightHouse.NewRequestHandler()
 	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	fwPacket2 := &firewall.Packet{}
+	nb2 := make([]byte, 12, 12)
+	result2 := make([]byte, mtu)
+	h := &header.H{}
 
 	for {
-		n, err := reader.Read(packet)
-		if err != nil {
-			if errors.Is(err, os.ErrClosed) && f.closed.Load() {
+		select {
+		case p := <-f.inbound:
+			if p.SegSize > 0 && p.SegSize < len(p.Payload) {
+				for offset := 0; offset < len(p.Payload); offset += p.SegSize {
+					end := offset + p.SegSize
+					if end > len(p.Payload) {
+						end = len(p.Payload)
+					}
+					f.readOutsidePackets(p.Addr, nil, result2[:0], p.Payload[offset:end], h, fwPacket2, lhh, nb2, i, conntrackCache.Get(f.l))
+				}
+			} else {
+				f.readOutsidePackets(p.Addr, nil, result2[:0], p.Payload, h, fwPacket2, lhh, nb2, i, conntrackCache.Get(f.l))
+			}
+
+			f.pktPool.Put(p)
+		case <-ctx.Done():
+			f.wg.Done()
 			return
 		}
-
-			f.l.WithError(err).Error("Error while reading outbound packet")
-			// This only seems to happen when something fatal happens to the fd, so exit.
-			os.Exit(2)
+	}
 }
 
-		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get(f.l))
+func (f *Interface) workerOut(i int, ctx context.Context) {
+	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	fwPacket1 := &firewall.Packet{}
+	nb1 := make([]byte, 12, 12)
+	result1 := make([]byte, mtu)
+
+	for {
+		select {
+		case data := <-f.outbound:
+			f.consumeInsidePacket(data.Payload, fwPacket1, nb1, result1, i, conntrackCache.Get(f.l))
+			f.pktPool.Put(data)
+		case <-ctx.Done():
+			f.wg.Done()
+			return
+		}
 	}
 }
 
@@ -451,6 +525,7 @@ func (f *Interface) GetCertState() *CertState {
 func (f *Interface) Close() error {
 	f.closed.Store(true)
 
+	// Release the udp readers
 	for _, u := range f.writers {
 		err := u.Close()
 		if err != nil {
@@ -458,6 +533,13 @@ func (f *Interface) Close() error {
 		}
 	}
 
-	// Release the tun device
-	return f.inside.Close()
+	// Release the tun readers
+	for _, u := range f.readers {
+		err := u.Close()
+		if err != nil {
+			f.l.WithError(err).Error("Error while closing tun device")
+		}
+	}
+
+	return nil
 }

main.go

@@ -284,14 +284,14 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	}
 
 	return &Control{
-		ifce,
-		l,
-		ctx,
-		cancel,
-		sshStart,
-		statsStart,
-		dnsStart,
-		lightHouse.StartUpdateWorker,
-		connManager.Start,
+		f:                      ifce,
+		l:                      l,
+		ctx:                    ctx,
+		cancel:                 cancel,
+		sshStart:               sshStart,
+		statsStart:             statsStart,
+		dnsStart:               dnsStart,
+		lighthouseStart:        lightHouse.StartUpdateWorker,
+		connectionManagerStart: connManager.Start,
 	}, nil
 }

outside.go

@@ -29,7 +29,7 @@ func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []
 		return
 	}
 
-	//l.Error("in packet ", header, packet[HeaderLen:])
+	//f.l.Error("in packet ", h)
 	if ip.IsValid() {
 		if f.myVpnNetworksTable.Contains(ip.Addr()) {
 			if f.l.Level >= logrus.DebugLevel {
@@ -245,6 +245,7 @@ func (f *Interface) handleHostRoaming(hostinfo *HostInfo, udpAddr netip.AddrPort
 			return
 		}
 
+		//TODO: Seems we have a bunch of stuff racing here, since we don't have a lock on hostinfo anymore we announce roaming in bursts
 		hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", udpAddr).
 			Info("Host roamed to new udp ip/port.")
 		hostinfo.lastRoam = time.Now()
@@ -470,7 +471,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 
 	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
 	if err != nil {
-		hostinfo.logger(f.l).WithError(err).Error("Failed to decrypt packet")
+		hostinfo.logger(f.l).WithError(err).WithField("fwPacket", fwPacket).Error("Failed to decrypt packet")
 		return false
 	}
 

overlay/tun.go

@@ -1,6 +1,7 @@
 package overlay
 
 import (
+	"net"
 	"net/netip"
 
 	"github.com/sirupsen/logrus"
@@ -70,3 +71,13 @@ func findRemovedRoutes(newRoutes, oldRoutes []Route) []Route {
 
 	return removed
 }
+
+func prefixToMask(prefix netip.Prefix) netip.Addr {
+	pLen := 128
+	if prefix.Addr().Is4() {
+		pLen = 32
+	}
+
+	addr, _ := netip.AddrFromSlice(net.CIDRMask(prefix.Bits(), pLen))
+	return addr
+}

overlay/tun_darwin.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
 	"net/netip"
 	"os"
 	"sync/atomic"
@@ -554,13 +553,3 @@ func (t *tun) Name() string {
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
 }
-
-func prefixToMask(prefix netip.Prefix) netip.Addr {
-	pLen := 128
-	if prefix.Addr().Is4() {
-		pLen = 32
-	}
-
-	addr, _ := netip.AddrFromSlice(net.CIDRMask(prefix.Bits(), pLen))
-	return addr
-}

overlay/tun_freebsd.go

@@ -10,11 +10,9 @@ import (
 	"io"
 	"io/fs"
 	"net/netip"
-	"os"
-	"os/exec"
-	"strconv"
 	"sync/atomic"
 	"syscall"
+	"time"
 	"unsafe"
 
 	"github.com/gaissmai/bart"
@@ -22,12 +20,18 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
+	netroute "golang.org/x/net/route"
+	"golang.org/x/sys/unix"
 )
 
 const (
 	// FIODGNAME is defined in sys/sys/filio.h on FreeBSD
 	// For 32-bit systems, use FIODGNAME_32 (not defined in this file: 0x80086678)
 	FIODGNAME        = 0x80106678
+	TUNSIFMODE       = 0x8004745e
+	TUNSIFHEAD       = 0x80047460
+	OSIOCAIFADDR_IN6 = 0x8088691b
+	IN6_IFF_NODAD    = 0x0020
 )
 
 type fiodgnameArg struct {
@@ -37,43 +41,159 @@ type fiodgnameArg struct {
 }
 
 type ifreqRename struct {
-	Name [16]byte
+	Name [unix.IFNAMSIZ]byte
 	Data uintptr
 }
 
 type ifreqDestroy struct {
-	Name [16]byte
+	Name [unix.IFNAMSIZ]byte
 	pad  [16]byte
 }
 
+type ifReq struct {
+	Name  [unix.IFNAMSIZ]byte
+	Flags uint16
+}
+
+type ifreqMTU struct {
+	Name [unix.IFNAMSIZ]byte
+	MTU  int32
+}
+
+type addrLifetime struct {
+	Expire    uint64
+	Preferred uint64
+	Vltime    uint32
+	Pltime    uint32
+}
+
+type ifreqAlias4 struct {
+	Name     [unix.IFNAMSIZ]byte
+	Addr     unix.RawSockaddrInet4
+	DstAddr  unix.RawSockaddrInet4
+	MaskAddr unix.RawSockaddrInet4
+	VHid     uint32
+}
+
+type ifreqAlias6 struct {
+	Name       [unix.IFNAMSIZ]byte
+	Addr       unix.RawSockaddrInet6
+	DstAddr    unix.RawSockaddrInet6
+	PrefixMask unix.RawSockaddrInet6
+	Flags      uint32
+	Lifetime   addrLifetime
+	VHid       uint32
+}
+
 type tun struct {
 	Device      string
 	vpnNetworks []netip.Prefix
 	MTU         int
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
+	linkAddr    *netroute.LinkAddr
 	l           *logrus.Logger
+	devFd       int
+}
 
-	io.ReadWriteCloser
+func (t *tun) Read(to []byte) (int, error) {
+	// use readv() to read from the tunnel device, to eliminate the need for copying the buffer
+	if t.devFd < 0 {
+		return -1, syscall.EINVAL
+	}
+
+	// first 4 bytes is protocol family, in network byte order
+	head := make([]byte, 4)
+
+	iovecs := []syscall.Iovec{
+		{&head[0], 4},
+		{&to[0], uint64(len(to))},
+	}
+
+	n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+
+	var err error
+	if errno != 0 {
+		err = syscall.Errno(errno)
+	} else {
+		err = nil
+	}
+	// fix bytes read number to exclude header
+	bytesRead := int(n)
+	if bytesRead < 0 {
+		return bytesRead, err
+	} else if bytesRead < 4 {
+		return 0, err
+	} else {
+		return bytesRead - 4, err
+	}
+}
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	// use writev() to write to the tunnel device, to eliminate the need for copying the buffer
+	if t.devFd < 0 {
+		return -1, syscall.EINVAL
+	}
+
+	if len(from) <= 1 {
+		return 0, syscall.EIO
+	}
+	ipVer := from[0] >> 4
+	var head []byte
+	// first 4 bytes is protocol family, in network byte order
+	if ipVer == 4 {
+		head = []byte{0, 0, 0, syscall.AF_INET}
+	} else if ipVer == 6 {
+		head = []byte{0, 0, 0, syscall.AF_INET6}
+	} else {
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+	iovecs := []syscall.Iovec{
+		{&head[0], 4},
+		{&from[0], uint64(len(from))},
+	}
+
+	n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+
+	var err error
+	if errno != 0 {
+		err = syscall.Errno(errno)
+	} else {
+		err = nil
+	}
+
+	return int(n) - 4, err
 }
 
 func (t *tun) Close() error {
-	if t.ReadWriteCloser != nil {
-		if err := t.ReadWriteCloser.Close(); err != nil {
-			return err
-		}
-
-		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+	if t.devFd >= 0 {
+		err := syscall.Close(t.devFd)
 		if err != nil {
-			return err
+			t.l.WithError(err).Error("Error closing device")
 		}
+		t.devFd = -1
+
+		c := make(chan struct{})
+		go func() {
+			// destroying the interface can block if a read() is still pending. Do this asynchronously.
+			defer close(c)
+			s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+			if err == nil {
 				defer syscall.Close(s)
-
 				ifreq := ifreqDestroy{Name: t.deviceBytes()}
-
-		// Destroy the interface
 				err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
-		return err
+			}
+			if err != nil {
+				t.l.WithError(err).Error("Error destroying tunnel")
+			}
+		}()
+
+		// wait up to 1 second so we start blocking at the ioctl
+		select {
+		case <-c:
+		case <-time.After(1 * time.Second):
+		}
 	}
 
 	return nil
@@ -85,32 +205,37 @@ func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun,
 
 func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	// Try to open existing tun device
-	var file *os.File
+	var fd int
 	var err error
 	deviceName := c.GetString("tun.dev", "")
 	if deviceName != "" {
-		file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0)
+		fd, err = syscall.Open("/dev/"+deviceName, syscall.O_RDWR, 0)
 	}
 	if errors.Is(err, fs.ErrNotExist) || deviceName == "" {
 		// If the device doesn't already exist, request a new one and rename it
-		file, err = os.OpenFile("/dev/tun", os.O_RDWR, 0)
+		fd, err = syscall.Open("/dev/tun", syscall.O_RDWR, 0)
 	}
 	if err != nil {
 		return nil, err
 	}
 
-	rawConn, err := file.SyscallConn()
-	if err != nil {
-		return nil, fmt.Errorf("SyscallConn: %v", err)
+	// Read the name of the interface
+	var name [16]byte
+	arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)}
+	ctrlErr := ioctl(uintptr(fd), FIODGNAME, uintptr(unsafe.Pointer(&arg)))
+
+	if ctrlErr == nil {
+		// set broadcast mode and multicast
+		ifmode := uint32(unix.IFF_BROADCAST | unix.IFF_MULTICAST)
+		ctrlErr = ioctl(uintptr(fd), TUNSIFMODE, uintptr(unsafe.Pointer(&ifmode)))
+	}
+
+	if ctrlErr == nil {
+		// turn on link-layer mode, to support ipv6
+		ifhead := uint32(1)
+		ctrlErr = ioctl(uintptr(fd), TUNSIFHEAD, uintptr(unsafe.Pointer(&ifhead)))
 	}
 
-	var name [16]byte
-	var ctrlErr error
-	rawConn.Control(func(fd uintptr) {
-		// Read the name of the interface
-		arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)}
-		ctrlErr = ioctl(fd, FIODGNAME, uintptr(unsafe.Pointer(&arg)))
-	})
 	if ctrlErr != nil {
 		return nil, err
 	}
@@ -122,11 +247,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 
 	// If the name doesn't match the desired interface name, rename it now
 	if ifName != deviceName {
-		s, err := syscall.Socket(
-			syscall.AF_INET,
-			syscall.SOCK_DGRAM,
-			syscall.IPPROTO_IP,
-		)
+		s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
 		if err != nil {
 			return nil, err
 		}
@@ -149,11 +270,11 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	}
 
 	t := &tun{
-		ReadWriteCloser: file,
 		Device:      deviceName,
 		vpnNetworks: vpnNetworks,
 		MTU:         c.GetInt("tun.mtu", DefaultMTU),
 		l:           l,
+		devFd:       fd,
 	}
 
 	err = t.reload(c, true)
@@ -172,38 +293,111 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 }
 
 func (t *tun) addIp(cidr netip.Prefix) error {
-	var err error
-	// TODO use syscalls instead of exec.Command
-	cmd := exec.Command("/sbin/ifconfig", t.Device, cidr.String(), cidr.Addr().String())
-	t.l.Debug("command: ", cmd.String())
-	if err = cmd.Run(); err != nil {
-		return fmt.Errorf("failed to run 'ifconfig': %s", err)
+	if cidr.Addr().Is4() {
+		ifr := ifreqAlias4{
+			Name: t.deviceBytes(),
+			Addr: unix.RawSockaddrInet4{
+				Len:    unix.SizeofSockaddrInet4,
+				Family: unix.AF_INET,
+				Addr:   cidr.Addr().As4(),
+			},
+			DstAddr: unix.RawSockaddrInet4{
+				Len:    unix.SizeofSockaddrInet4,
+				Family: unix.AF_INET,
+				Addr:   getBroadcast(cidr).As4(),
+			},
+			MaskAddr: unix.RawSockaddrInet4{
+				Len:    unix.SizeofSockaddrInet4,
+				Family: unix.AF_INET,
+				Addr:   prefixToMask(cidr).As4(),
+			},
+			VHid: 0,
+		}
+		s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+		if err != nil {
+			return err
+		}
+		defer syscall.Close(s)
+		// Note: unix.SIOCAIFADDR corresponds to FreeBSD's OSIOCAIFADDR
+		if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
+			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err)
+		}
+		return nil
 	}
 
-	cmd = exec.Command("/sbin/route", "-n", "add", "-net", cidr.String(), "-interface", t.Device)
-	t.l.Debug("command: ", cmd.String())
-	if err = cmd.Run(); err != nil {
-		return fmt.Errorf("failed to run 'route add': %s", err)
+	if cidr.Addr().Is6() {
+		ifr := ifreqAlias6{
+			Name: t.deviceBytes(),
+			Addr: unix.RawSockaddrInet6{
+				Len:    unix.SizeofSockaddrInet6,
+				Family: unix.AF_INET6,
+				Addr:   cidr.Addr().As16(),
+			},
+			PrefixMask: unix.RawSockaddrInet6{
+				Len:    unix.SizeofSockaddrInet6,
+				Family: unix.AF_INET6,
+				Addr:   prefixToMask(cidr).As16(),
+			},
+			Lifetime: addrLifetime{
+				Expire:    0,
+				Preferred: 0,
+				Vltime:    0xffffffff,
+				Pltime:    0xffffffff,
+			},
+			Flags: IN6_IFF_NODAD,
+		}
+		s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+		if err != nil {
+			return err
+		}
+		defer syscall.Close(s)
+
+		if err := ioctl(uintptr(s), OSIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
+			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err)
+		}
+		return nil
 	}
 
-	cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU))
-	t.l.Debug("command: ", cmd.String())
-	if err = cmd.Run(); err != nil {
-		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-	}
-
-	// Unsafe path routes
-	return t.addRoutes(false)
+	return fmt.Errorf("unknown address type %v", cidr)
 }
 
 func (t *tun) Activate() error {
+	// Setup our default MTU
+	err := t.setMTU()
+	if err != nil {
+		return err
+	}
+
+	linkAddr, err := getLinkAddr(t.Device)
+	if err != nil {
+		return err
+	}
+	if linkAddr == nil {
+		return fmt.Errorf("unable to discover link_addr for tun interface")
+	}
+	t.linkAddr = linkAddr
+
 	for i := range t.vpnNetworks {
 		err := t.addIp(t.vpnNetworks[i])
 		if err != nil {
 			return err
 		}
 	}
-	return nil
+
+	return t.addRoutes(false)
+}
+
+func (t *tun) setMTU() error {
+	// Set the MTU on the device
+	s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+	if err != nil {
+		return err
+	}
+	defer syscall.Close(s)
+
+	ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MTU)}
+	err = ioctl(uintptr(s), unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm)))
+	return err
 }
 
 func (t *tun) reload(c *config.C, initial bool) error {
@@ -268,15 +462,16 @@ func (t *tun) addRoutes(logErrors bool) error {
 			continue
 		}
 
-		cmd := exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), "-interface", t.Device)
-		t.l.Debug("command: ", cmd.String())
-		if err := cmd.Run(); err != nil {
-			retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]any{"route": r}, err)
+		err := addRoute(r.Cidr, t.linkAddr)
+		if err != nil {
+			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
 			} else {
 				return retErr
 			}
+		} else {
+			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -289,9 +484,8 @@ func (t *tun) removeRoutes(routes []Route) error {
 			continue
 		}
 
-		cmd := exec.Command("/sbin/route", "-n", "delete", "-net", r.Cidr.String(), "-interface", t.Device)
-		t.l.Debug("command: ", cmd.String())
-		if err := cmd.Run(); err != nil {
+		err := delRoute(r.Cidr, t.linkAddr)
+		if err != nil {
 			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
 			t.l.WithField("route", r).Info("Removed route")
@@ -306,3 +500,144 @@ func (t *tun) deviceBytes() (o [16]byte) {
 	}
 	return
 }
+
+func flipBytes(b []byte) []byte {
+	for i := 0; i < len(b); i++ {
+		b[i] ^= 0xFF
+	}
+	return b
+}
+func orBytes(a []byte, b []byte) []byte {
+	ret := make([]byte, len(a))
+	for i := 0; i < len(a); i++ {
+		ret[i] = a[i] | b[i]
+	}
+	return ret
+}
+
+func getBroadcast(cidr netip.Prefix) netip.Addr {
+	broadcast, _ := netip.AddrFromSlice(
+		orBytes(
+			cidr.Addr().AsSlice(),
+			flipBytes(prefixToMask(cidr).AsSlice()),
+		),
+	)
+	return broadcast
+}
+
+func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
+	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
+	}
+	defer unix.Close(sock)
+
+	route := &netroute.RouteMessage{
+		Version: unix.RTM_VERSION,
+		Type:    unix.RTM_ADD,
+		Flags:   unix.RTF_UP,
+		Seq:     1,
+	}
+
+	if prefix.Addr().Is4() {
+		route.Addrs = []netroute.Addr{
+			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
+			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
+			unix.RTAX_GATEWAY: gateway,
+		}
+	} else {
+		route.Addrs = []netroute.Addr{
+			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
+			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
+			unix.RTAX_GATEWAY: gateway,
+		}
+	}
+
+	data, err := route.Marshal()
+	if err != nil {
+		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
+	}
+
+	_, err = unix.Write(sock, data[:])
+	if err != nil {
+		if errors.Is(err, unix.EEXIST) {
+			// Try to do a change
+			route.Type = unix.RTM_CHANGE
+			data, err = route.Marshal()
+			if err != nil {
+				return fmt.Errorf("failed to create route.RouteMessage for change: %w", err)
+			}
+			_, err = unix.Write(sock, data[:])
+			fmt.Println("DOING CHANGE")
+			return err
+		}
+		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
+	}
+
+	return nil
+}
+
+func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
+	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
+	}
+	defer unix.Close(sock)
+
+	route := netroute.RouteMessage{
+		Version: unix.RTM_VERSION,
+		Type:    unix.RTM_DELETE,
+		Seq:     1,
+	}
+
+	if prefix.Addr().Is4() {
+		route.Addrs = []netroute.Addr{
+			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
+			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
+			unix.RTAX_GATEWAY: gateway,
+		}
+	} else {
+		route.Addrs = []netroute.Addr{
+			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
+			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
+			unix.RTAX_GATEWAY: gateway,
+		}
+	}
+
+	data, err := route.Marshal()
+	if err != nil {
+		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
+	}
+	_, err = unix.Write(sock, data[:])
+	if err != nil {
+		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
+	}
+
+	return nil
+}
+
+// getLinkAddr Gets the link address for the interface of the given name
+func getLinkAddr(name string) (*netroute.LinkAddr, error) {
+	rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0)
+	if err != nil {
+		return nil, err
+	}
+	msgs, err := netroute.ParseRIB(unix.NET_RT_IFLIST, rib)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, m := range msgs {
+		switch m := m.(type) {
+		case *netroute.InterfaceMessage:
+			if m.Name == name {
+				sa, ok := m.Addrs[unix.RTAX_IFP].(*netroute.LinkAddr)
+				if ok {
+					return sa, nil
+				}
+			}
+		}
+	}
+
+	return nil, nil
+}

packet/packet.go

@@ -0,0 +1,45 @@
+package packet
+
+import (
+	"net/netip"
+	"sync"
+
+	"golang.org/x/sys/unix"
+)
+
+const Size = 0xffff
+
+type Packet struct {
+	Payload []byte
+	Control []byte
+	SegSize int
+	Addr    netip.AddrPort
+}
+
+func New() *Packet {
+	return &Packet{
+		Payload: make([]byte, Size),
+		Control: make([]byte, unix.CmsgSpace(2)),
+	}
+}
+
+type Pool struct {
+	pool sync.Pool
+}
+
+var bigPool = &Pool{
+	pool: sync.Pool{New: func() any { return New() }},
+}
+
+func GetPool() *Pool {
+	return bigPool
+}
+
+func (p *Pool) Get() *Packet {
+	return p.pool.Get().(*Packet)
+}
+
+func (p *Pool) Put(x *Packet) {
+	x.Payload = x.Payload[:Size]
+	p.pool.Put(x)
+}

pki.go

@@ -100,36 +100,41 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
 		currentState := p.cs.Load()
 		if newState.v1Cert != nil {
 			if currentState.v1Cert == nil {
-				//adding certs is fine, actually. Networks-in-common confirmed in newCertState().
-			} else {
+				return util.NewContextualError("v1 certificate was added, restart required", nil, err)
+			}
+
 			// did IP in cert change? if so, don't set
 			if !slices.Equal(currentState.v1Cert.Networks(), newState.v1Cert.Networks()) {
 				return util.NewContextualError(
 					"Networks in new cert was different from old",
-						m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks(), "cert_version": cert.Version1},
+					m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks()},
 					nil,
 				)
 			}
 
 			if currentState.v1Cert.Curve() != newState.v1Cert.Curve() {
 				return util.NewContextualError(
-						"Curve in new v1 cert was different from old",
-						m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve(), "cert_version": cert.Version1},
+					"Curve in new cert was different from old",
+					m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve()},
 					nil,
 				)
 			}
-			}
+
+		} else if currentState.v1Cert != nil {
+			//TODO: CERT-V2 we should be able to tear this down
+			return util.NewContextualError("v1 certificate was removed, restart required", nil, err)
 		}
 
 		if newState.v2Cert != nil {
 			if currentState.v2Cert == nil {
-				//adding certs is fine, actually
-			} else {
+				return util.NewContextualError("v2 certificate was added, restart required", nil, err)
+			}
+
 			// did IP in cert change? if so, don't set
 			if !slices.Equal(currentState.v2Cert.Networks(), newState.v2Cert.Networks()) {
 				return util.NewContextualError(
 					"Networks in new cert was different from old",
-						m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks(), "cert_version": cert.Version2},
+					m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks()},
 					nil,
 				)
 			}
@@ -137,25 +142,13 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
 			if currentState.v2Cert.Curve() != newState.v2Cert.Curve() {
 				return util.NewContextualError(
 					"Curve in new cert was different from old",
-						m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve(), "cert_version": cert.Version2},
+					m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve()},
 					nil,
 				)
 			}
-			}
 
 		} else if currentState.v2Cert != nil {
-			//newState.v1Cert is non-nil bc empty certstates aren't permitted
-			if newState.v1Cert == nil {
-				return util.NewContextualError("v1 and v2 certs are nil, this should be impossible", nil, err)
-			}
-			//if we're going to v1-only, we need to make sure we didn't orphan any v2-cert vpnaddrs
-			if !slices.Equal(currentState.v2Cert.Networks(), newState.v1Cert.Networks()) {
-				return util.NewContextualError(
-					"Removing a V2 cert is not permitted unless it has identical networks to the new V1 cert",
-					m{"new_v1_networks": newState.v1Cert.Networks(), "old_v2_networks": currentState.v2Cert.Networks()},
-					nil,
-				)
-			}
+			return util.NewContextualError("v2 certificate was removed, restart required", nil, err)
 		}
 
 		// Cipher cant be hot swapped so just leave it at what it was before

relay_manager.go

@@ -190,7 +190,6 @@ func (rm *relayManager) handleCreateRelayResponse(v cert.Version, h *HostInfo, f
 			InitiatorRelayIndex: peerRelay.RemoteIndex,
 		}
 
-		relayFrom := h.vpnAddrs[0]
 		if v == cert.Version1 {
 			peer := peerHostInfo.vpnAddrs[0]
 			if !peer.Is4() {
@@ -208,13 +207,7 @@ func (rm *relayManager) handleCreateRelayResponse(v cert.Version, h *HostInfo, f
 			b = targetAddr.As4()
 			resp.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 		} else {
-			if targetAddr.Is4() {
-				relayFrom = h.vpnAddrs[0]
-			} else {
-				//todo do this smarter
-				relayFrom = h.vpnAddrs[len(h.vpnAddrs)-1]
-			}
-			resp.RelayFromAddr = netAddrToProtoAddr(relayFrom)
+			resp.RelayFromAddr = netAddrToProtoAddr(peerHostInfo.vpnAddrs[0])
 			resp.RelayToAddr = target
 		}
 
@@ -367,7 +360,7 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 			Type:                NebulaControl_CreateRelayRequest,
 			InitiatorRelayIndex: index,
 		}
-		relayFrom := h.vpnAddrs[0]
+
 		if v == cert.Version1 {
 			if !h.vpnAddrs[0].Is4() {
 				rm.l.WithField("relayFrom", h.vpnAddrs[0]).
@@ -384,13 +377,7 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 			b = target.As4()
 			req.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 		} else {
-			if target.Is4() {
-				relayFrom = h.vpnAddrs[0]
-			} else {
-				//todo do this smarter
-				relayFrom = h.vpnAddrs[len(h.vpnAddrs)-1]
-			}
-			req.RelayFromAddr = netAddrToProtoAddr(relayFrom)
+			req.RelayFromAddr = netAddrToProtoAddr(h.vpnAddrs[0])
 			req.RelayToAddr = netAddrToProtoAddr(target)
 		}
 
@@ -401,7 +388,7 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 		} else {
 			f.SendMessageToHostInfo(header.Control, 0, peer, msg, make([]byte, 12), make([]byte, mtu))
 			rm.l.WithFields(logrus.Fields{
-				"relayFrom":           relayFrom,
+				"relayFrom":           h.vpnAddrs[0],
 				"relayTo":             target,
 				"initiatorRelayIndex": req.InitiatorRelayIndex,
 				"responderRelayIndex": req.ResponderRelayIndex,

service/service.go

@@ -9,10 +9,13 @@ import (
 	"math"
 	"net"
 	"net/netip"
+	"os"
 	"strings"
 	"sync"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/overlay"
 	"golang.org/x/sync/errgroup"
 	"gvisor.dev/gvisor/pkg/buffer"
@@ -43,8 +46,19 @@ type Service struct {
 	}
 }
 
-func New(control *nebula.Control) (*Service, error) {
-	control.Start()
+func New(config *config.C) (*Service, error) {
+	logger := logrus.New()
+	logger.Out = os.Stdout
+
+	control, err := nebula.Main(config, false, "custom-app", logger, overlay.NewUserDeviceFromConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	wait, err := control.Start()
+	if err != nil {
+		return nil, err
+	}
 
 	ctx := control.Context()
 	eg, ctx := errgroup.WithContext(ctx)
@@ -141,6 +155,12 @@ func New(control *nebula.Control) (*Service, error) {
 		}
 	})
 
+	// Add the nebula wait function to the group
+	eg.Go(func() error {
+		wait()
+		return nil
+	})
+
 	return &s, nil
 }
 

udp/conn.go

@@ -4,19 +4,19 @@ import (
 	"net/netip"
 
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/packet"
 )
 
 const MTU = 9001
 
-type EncReader func(
-	addr netip.AddrPort,
-	payload []byte,
-)
+type EncReader func(*packet.Packet)
+
+type PacketBufferGetter func() *packet.Packet
 
 type Conn interface {
 	Rebind() error
 	LocalAddr() (netip.AddrPort, error)
-	ListenOut(r EncReader)
+	ListenOut(pg PacketBufferGetter, pc chan *packet.Packet) error
 	WriteTo(b []byte, addr netip.AddrPort) error
 	ReloadConfig(c *config.C)
 	Close() error

udp/udp_generic.go

@@ -71,15 +71,14 @@ type rawMessage struct {
 	Len uint32
 }
 
-func (u *GenericConn) ListenOut(r EncReader) {
+func (u *GenericConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	for {
 		// Just read one packet at a time
 		n, rua, err := u.ReadFromUDPAddrPort(buffer)
 		if err != nil {
-			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
-			return
+			return err
 		}
 
 		r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n])

udp/udp_linux.go

@@ -9,27 +9,26 @@ import (
 	"net"
 	"net/netip"
 	"syscall"
+	"time"
 	"unsafe"
 
 	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/packet"
 	"golang.org/x/sys/unix"
 )
 
+var readTimeout = unix.NsecToTimeval(int64(time.Millisecond * 500))
+
 type StdConn struct {
 	sysFd     int
 	isV4      bool
 	l         *logrus.Logger
 	batch     int
-}
-
-func maybeIPV4(ip net.IP) (net.IP, bool) {
-	ip4 := ip.To4()
-	if ip4 != nil {
-		return ip4, true
-	}
-	return ip, false
+	enableGRO bool
+	enableGSO bool
+	//gso       gsoState
 }
 
 func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
@@ -55,6 +54,11 @@ func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch in
 		}
 	}
 
+	// Set a read timeout
+	if err = unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &readTimeout); err != nil {
+		return nil, fmt.Errorf("unable to set SO_RCVTIMEO: %s", err)
+	}
+
 	var sa unix.Sockaddr
 	if ip.Is4() {
 		sa4 := &unix.SockaddrInet4{Port: port}
@@ -118,10 +122,10 @@ func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
 	}
 }
 
-func (u *StdConn) ListenOut(r EncReader) {
+func (u *StdConn) ListenOut(pg PacketBufferGetter, pc chan *packet.Packet) error {
 	var ip netip.Addr
 
-	msgs, buffers, names := u.PrepareRawMessages(u.batch)
+	msgs, packets, names := u.PrepareRawMessages(u.batch, pg)
 	read := u.ReadMulti
 	if u.batch == 1 {
 		read = u.ReadSingle
@@ -130,22 +134,61 @@ func (u *StdConn) ListenOut(r EncReader) {
 	for {
 		n, err := read(msgs)
 		if err != nil {
-			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
-			return
+			return err
 		}
 
 		for i := 0; i < n; i++ {
+			out := packets[i]
+			out.Payload = out.Payload[:msgs[i].Len]
+
 			// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
 			if u.isV4 {
 				ip, _ = netip.AddrFromSlice(names[i][4:8])
 			} else {
 				ip, _ = netip.AddrFromSlice(names[i][8:24])
 			}
-			r(netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4])), buffers[i][:msgs[i].Len])
+			out.Addr = netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4]))
+			ctrlLen := getRawMessageControlLen(&msgs[i])
+			if ctrlLen > 0 {
+				packets[i].SegSize = parseGROControl(packets[i].Control[:ctrlLen])
+			} else {
+				packets[i].SegSize = 0
+			}
+
+			pc <- out
+
+			//rotate this packet out so we don't overwrite it
+			packets[i] = pg()
+			msgs[i].Hdr.Iov.Base = &packets[i].Payload[0]
+			if u.enableGRO {
+				msgs[i].Hdr.Control = &packets[i].Control[0]
+				msgs[i].Hdr.Controllen = uint64(cap(packets[i].Control))
+			}
+
 		}
 	}
 }
 
+func parseGROControl(control []byte) int {
+	if len(control) == 0 {
+		return 0
+	}
+
+	cmsgs, err := unix.ParseSocketControlMessage(control)
+	if err != nil {
+		return 0
+	}
+
+	for _, c := range cmsgs {
+		if c.Header.Level == unix.SOL_UDP && c.Header.Type == unix.UDP_GRO && len(c.Data) >= 2 {
+			segSize := int(binary.LittleEndian.Uint16(c.Data[:2]))
+			return segSize
+		}
+	}
+
+	return 0
+}
+
 func (u *StdConn) ReadSingle(msgs []rawMessage) (int, error) {
 	for {
 		n, _, err := unix.Syscall6(
@@ -159,6 +202,9 @@ func (u *StdConn) ReadSingle(msgs []rawMessage) (int, error) {
 		)
 
 		if err != 0 {
+			if err == unix.EAGAIN || err == unix.EINTR {
+				continue
+			}
 			return 0, &net.OpError{Op: "recvmsg", Err: err}
 		}
 
@@ -180,6 +226,9 @@ func (u *StdConn) ReadMulti(msgs []rawMessage) (int, error) {
 		)
 
 		if err != 0 {
+			if err == unix.EAGAIN || err == unix.EINTR {
+				continue
+			}
 			return 0, &net.OpError{Op: "recvmmsg", Err: err}
 		}
 
@@ -221,7 +270,7 @@ func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
 
 func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
 	if !ip.Addr().Is4() {
-		return ErrInvalidIPv6RemoteForSocket
+		return fmt.Errorf("Listener is IPv4, but writing to IPv6 remote")
 	}
 
 	var rsa unix.RawSockaddrInet4
@@ -294,6 +343,28 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 			u.l.WithError(err).Error("Failed to set listen.so_mark")
 		}
 	}
+	u.configureGRO(true)
+}
+
+func (u *StdConn) configureGRO(enable bool) {
+	if enable == u.enableGRO {
+		return
+	}
+
+	if enable {
+		if err := unix.SetsockoptInt(u.sysFd, unix.SOL_UDP, unix.UDP_GRO, 1); err != nil {
+			u.l.WithError(err).Warn("Failed to enable UDP GRO")
+			return
+		}
+		u.enableGRO = true
+		u.l.Info("UDP GRO enabled")
+		return
+	}
+
+	if err := unix.SetsockoptInt(u.sysFd, unix.SOL_UDP, unix.UDP_GRO, 0); err != nil && err != unix.ENOPROTOOPT {
+		u.l.WithError(err).Warn("Failed to disable UDP GRO")
+	}
+	u.enableGRO = false
 }
 
 func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {

udp/udp_linux_64.go

@@ -7,6 +7,7 @@
 package udp
 
 import (
+	"github.com/slackhq/nebula/packet"
 	"golang.org/x/sys/unix"
 )
 
@@ -33,17 +34,39 @@ type rawMessage struct {
 	Pad0 [4]byte
 }
 
-func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
+func setRawMessageControl(msg *rawMessage, buf []byte) {
+	if len(buf) == 0 {
+		msg.Hdr.Control = nil
+		msg.Hdr.Controllen = 0
+		return
+	}
+	msg.Hdr.Control = &buf[0]
+	msg.Hdr.Controllen = uint64(len(buf))
+}
+
+func getRawMessageControlLen(msg *rawMessage) int {
+	return int(msg.Hdr.Controllen)
+}
+
+func setCmsgLen(h *unix.Cmsghdr, l int) {
+	h.Len = uint64(l)
+}
+
+func (u *StdConn) PrepareRawMessages(n int, pg PacketBufferGetter) ([]rawMessage, []*packet.Packet, [][]byte) {
 	msgs := make([]rawMessage, n)
-	buffers := make([][]byte, n)
 	names := make([][]byte, n)
 
+	packets := make([]*packet.Packet, n)
+	for i := range packets {
+		packets[i] = pg()
+	}
+	//todo?
+
 	for i := range msgs {
-		buffers[i] = make([]byte, MTU)
 		names[i] = make([]byte, unix.SizeofSockaddrInet6)
 
 		vs := []iovec{
-			{Base: &buffers[i][0], Len: uint64(len(buffers[i]))},
+			{Base: &packets[i].Payload[0], Len: uint64(packet.Size)},
 		}
 
 		msgs[i].Hdr.Iov = &vs[0]
@@ -51,7 +74,14 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
 
 		msgs[i].Hdr.Name = &names[i][0]
 		msgs[i].Hdr.Namelen = uint32(len(names[i]))
+		if u.enableGRO {
+			msgs[i].Hdr.Control = &packets[i].Control[0]
+			msgs[i].Hdr.Controllen = uint64(len(packets[i].Control))
+		} else {
+			msgs[i].Hdr.Control = nil
+			msgs[i].Hdr.Controllen = 0
+		}
 	}
 
-	return msgs, buffers, names
+	return msgs, packets, names
 }

udp/udp_rio_windows.go

@@ -134,7 +134,7 @@ func (u *RIOConn) bind(sa windows.Sockaddr) error {
 	return nil
 }
 
-func (u *RIOConn) ListenOut(r EncReader) {
+func (u *RIOConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	for {