test and stupid fix

cert/cert.go

@@ -58,9 +58,6 @@ type Certificate interface {
 	// PublicKey is the raw bytes to be used in asymmetric cryptographic operations.
 	PublicKey() []byte
 
-	// MarshalPublicKeyPEM is the value of PublicKey marshalled to PEM
-	MarshalPublicKeyPEM() []byte
-
 	// Curve identifies which curve was used for the PublicKey and Signature.
 	Curve() Curve
 

cert/cert_v1.go

@@ -83,10 +83,6 @@ func (c *certificateV1) PublicKey() []byte {
 	return c.details.publicKey
 }
 
-func (c *certificateV1) MarshalPublicKeyPEM() []byte {
-	return marshalCertPublicKeyToPEM(c)
-}
-
 func (c *certificateV1) Signature() []byte {
 	return c.signature
 }

cert/cert_v1_test.go

@@ -1,7 +1,6 @@
 package cert
 
 import (
-	"crypto/ed25519"
 	"fmt"
 	"net/netip"
 	"testing"
@@ -14,7 +13,6 @@ import (
 )
 
 func TestCertificateV1_Marshal(t *testing.T) {
-	t.Parallel()
 	before := time.Now().Add(time.Second * -60).Round(time.Second)
 	after := time.Now().Add(time.Second * 60).Round(time.Second)
 	pubKey := []byte("1234567890abcedfghij1234567890ab")
@@ -62,58 +60,6 @@ func TestCertificateV1_Marshal(t *testing.T) {
 	assert.Equal(t, nc.Groups(), nc2.Groups())
 }
 
-func TestCertificateV1_PublicKeyPem(t *testing.T) {
-	t.Parallel()
-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
-	pubKey := ed25519.PublicKey("1234567890abcedfghij1234567890ab")
-
-	nc := certificateV1{
-		details: detailsV1{
-			name:           "testing",
-			networks:       []netip.Prefix{},
-			unsafeNetworks: []netip.Prefix{},
-			groups:         []string{"test-group1", "test-group2", "test-group3"},
-			notBefore:      before,
-			notAfter:       after,
-			publicKey:      pubKey,
-			isCA:           false,
-			issuer:         "1234567890abcedfghij1234567890ab",
-		},
-		signature: []byte("1234567890abcedfghij1234567890ab"),
-	}
-
-	assert.Equal(t, Version1, nc.Version())
-	assert.Equal(t, Curve_CURVE25519, nc.Curve())
-	pubPem := "-----BEGIN NEBULA X25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA X25519 PUBLIC KEY-----\n"
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
-	assert.False(t, nc.IsCA())
-
-	nc.details.isCA = true
-	assert.Equal(t, Curve_CURVE25519, nc.Curve())
-	pubPem = "-----BEGIN NEBULA ED25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA ED25519 PUBLIC KEY-----\n"
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
-	assert.True(t, nc.IsCA())
-
-	pubP256KeyPem := []byte(`-----BEGIN NEBULA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA P256 PUBLIC KEY-----
-`)
-	pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem)
-	require.NoError(t, err)
-	nc.details.curve = Curve_P256
-	nc.details.publicKey = pubP256Key
-	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
-	assert.True(t, nc.IsCA())
-
-	nc.details.isCA = false
-	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
-	assert.False(t, nc.IsCA())
-}
-
 func TestCertificateV1_Expired(t *testing.T) {
 	nc := certificateV1{
 		details: detailsV1{

cert/cert_v2.go

@@ -114,10 +114,6 @@ func (c *certificateV2) PublicKey() []byte {
 	return c.publicKey
 }
 
-func (c *certificateV2) MarshalPublicKeyPEM() []byte {
-	return marshalCertPublicKeyToPEM(c)
-}
-
 func (c *certificateV2) Signature() []byte {
 	return c.signature
 }

cert/cert_v2_test.go

@@ -15,7 +15,6 @@ import (
 )
 
 func TestCertificateV2_Marshal(t *testing.T) {
-	t.Parallel()
 	before := time.Now().Add(time.Second * -60).Round(time.Second)
 	after := time.Now().Add(time.Second * 60).Round(time.Second)
 	pubKey := []byte("1234567890abcedfghij1234567890ab")
@@ -76,58 +75,6 @@ func TestCertificateV2_Marshal(t *testing.T) {
 	assert.Equal(t, nc.Groups(), nc2.Groups())
 }
 
-func TestCertificateV2_PublicKeyPem(t *testing.T) {
-	t.Parallel()
-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
-	pubKey := ed25519.PublicKey("1234567890abcedfghij1234567890ab")
-
-	nc := certificateV2{
-		details: detailsV2{
-			name:           "testing",
-			networks:       []netip.Prefix{},
-			unsafeNetworks: []netip.Prefix{},
-			groups:         []string{"test-group1", "test-group2", "test-group3"},
-			notBefore:      before,
-			notAfter:       after,
-			isCA:           false,
-			issuer:         "1234567890abcedfghij1234567890ab",
-		},
-		publicKey: pubKey,
-		signature: []byte("1234567890abcedfghij1234567890ab"),
-	}
-
-	assert.Equal(t, Version2, nc.Version())
-	assert.Equal(t, Curve_CURVE25519, nc.Curve())
-	pubPem := "-----BEGIN NEBULA X25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA X25519 PUBLIC KEY-----\n"
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
-	assert.False(t, nc.IsCA())
-
-	nc.details.isCA = true
-	assert.Equal(t, Curve_CURVE25519, nc.Curve())
-	pubPem = "-----BEGIN NEBULA ED25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA ED25519 PUBLIC KEY-----\n"
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
-	assert.True(t, nc.IsCA())
-
-	pubP256KeyPem := []byte(`-----BEGIN NEBULA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA P256 PUBLIC KEY-----
-`)
-	pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem)
-	require.NoError(t, err)
-	nc.curve = Curve_P256
-	nc.publicKey = pubP256Key
-	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
-	assert.True(t, nc.IsCA())
-
-	nc.details.isCA = false
-	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
-	assert.False(t, nc.IsCA())
-}
-
 func TestCertificateV2_Expired(t *testing.T) {
 	nc := certificateV2{
 		details: detailsV2{

cert/pem.go

@@ -7,26 +7,19 @@ import (
 	"golang.org/x/crypto/ed25519"
 )
 
-const ( //cert banners
+const (
 	CertificateBanner                = "NEBULA CERTIFICATE"
 	CertificateV2Banner              = "NEBULA CERTIFICATE V2"
-)
-
-const ( //key-agreement-key banners
 	X25519PrivateKeyBanner           = "NEBULA X25519 PRIVATE KEY"
 	X25519PublicKeyBanner            = "NEBULA X25519 PUBLIC KEY"
-	P256PrivateKeyBanner   = "NEBULA P256 PRIVATE KEY"
-	P256PublicKeyBanner    = "NEBULA P256 PUBLIC KEY"
-)
-
-/* including "ECDSA" in the P256 banners is a clue that these keys should be used only for signing */
-const ( //signing key banners
-	EncryptedECDSAP256PrivateKeyBanner = "NEBULA ECDSA P256 ENCRYPTED PRIVATE KEY"
-	ECDSAP256PrivateKeyBanner          = "NEBULA ECDSA P256 PRIVATE KEY"
-	ECDSAP256PublicKeyBanner           = "NEBULA ECDSA P256 PUBLIC KEY"
 	EncryptedEd25519PrivateKeyBanner = "NEBULA ED25519 ENCRYPTED PRIVATE KEY"
 	Ed25519PrivateKeyBanner          = "NEBULA ED25519 PRIVATE KEY"
 	Ed25519PublicKeyBanner           = "NEBULA ED25519 PUBLIC KEY"
+
+	P256PrivateKeyBanner               = "NEBULA P256 PRIVATE KEY"
+	P256PublicKeyBanner                = "NEBULA P256 PUBLIC KEY"
+	EncryptedECDSAP256PrivateKeyBanner = "NEBULA ECDSA P256 ENCRYPTED PRIVATE KEY"
+	ECDSAP256PrivateKeyBanner          = "NEBULA ECDSA P256 PRIVATE KEY"
 )
 
 // UnmarshalCertificateFromPEM will try to unmarshal the first pem block in a byte array, returning any non consumed
@@ -58,16 +51,6 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
 
 }
 
-func marshalCertPublicKeyToPEM(c Certificate) []byte {
-	if c.IsCA() {
-		return MarshalSigningPublicKeyToPEM(c.Curve(), c.PublicKey())
-	} else {
-		return MarshalPublicKeyToPEM(c.Curve(), c.PublicKey())
-	}
-}
-
-// MarshalPublicKeyToPEM returns a PEM representation of a public key used for ECDH.
-// if your public key came from a certificate, prefer Certificate.PublicKeyPEM() if possible, to avoid mistakes!
 func MarshalPublicKeyToPEM(curve Curve, b []byte) []byte {
 	switch curve {
 	case Curve_CURVE25519:
@@ -79,19 +62,6 @@ func MarshalPublicKeyToPEM(curve Curve, b []byte) []byte {
 	}
 }
 
-// MarshalSigningPublicKeyToPEM returns a PEM representation of a public key used for signing.
-// if your public key came from a certificate, prefer Certificate.PublicKeyPEM() if possible, to avoid mistakes!
-func MarshalSigningPublicKeyToPEM(curve Curve, b []byte) []byte {
-	switch curve {
-	case Curve_CURVE25519:
-		return pem.EncodeToMemory(&pem.Block{Type: Ed25519PublicKeyBanner, Bytes: b})
-	case Curve_P256:
-		return pem.EncodeToMemory(&pem.Block{Type: P256PublicKeyBanner, Bytes: b})
-	default:
-		return nil
-	}
-}
-
 func UnmarshalPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {
 	k, r := pem.Decode(b)
 	if k == nil {
@@ -103,7 +73,7 @@ func UnmarshalPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {
 	case X25519PublicKeyBanner, Ed25519PublicKeyBanner:
 		expectedLen = 32
 		curve = Curve_CURVE25519
-	case P256PublicKeyBanner, ECDSAP256PublicKeyBanner:
+	case P256PublicKeyBanner:
 		// Uncompressed
 		expectedLen = 65
 		curve = Curve_P256

cert/pem_test.go

@@ -177,7 +177,6 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 }
 
 func TestUnmarshalPublicKeyFromPEM(t *testing.T) {
-	t.Parallel()
 	pubKey := []byte(`# A good key
 -----BEGIN NEBULA ED25519 PUBLIC KEY-----
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
@@ -231,7 +230,6 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 }
 
 func TestUnmarshalX25519PublicKey(t *testing.T) {
-	t.Parallel()
 	pubKey := []byte(`# A good key
 -----BEGIN NEBULA X25519 PUBLIC KEY-----
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
@@ -242,12 +240,6 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 AAAAAAAAAAAAAAAAAAAAAAA=
 -----END NEBULA P256 PUBLIC KEY-----
-`)
-	oldPubP256Key := []byte(`# A good key
------BEGIN NEBULA ECDSA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA ECDSA P256 PUBLIC KEY-----
 `)
 	shortKey := []byte(`# A short key
 -----BEGIN NEBULA X25519 PUBLIC KEY-----
@@ -264,22 +256,15 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 -END NEBULA X25519 PUBLIC KEY-----`)
 
-	keyBundle := appendByteSlices(pubKey, pubP256Key, oldPubP256Key, shortKey, invalidBanner, invalidPem)
+	keyBundle := appendByteSlices(pubKey, pubP256Key, shortKey, invalidBanner, invalidPem)
 
 	// Success test case
 	k, rest, curve, err := UnmarshalPublicKeyFromPEM(keyBundle)
 	assert.Len(t, k, 32)
 	require.NoError(t, err)
-	assert.Equal(t, rest, appendByteSlices(pubP256Key, oldPubP256Key, shortKey, invalidBanner, invalidPem))
+	assert.Equal(t, rest, appendByteSlices(pubP256Key, shortKey, invalidBanner, invalidPem))
 	assert.Equal(t, Curve_CURVE25519, curve)
 
-	// Success test case
-	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
-	assert.Len(t, k, 65)
-	require.NoError(t, err)
-	assert.Equal(t, rest, appendByteSlices(oldPubP256Key, shortKey, invalidBanner, invalidPem))
-	assert.Equal(t, Curve_P256, curve)
-
 	// Success test case
 	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Len(t, k, 65)

cert_test/cert.go

@@ -114,6 +114,33 @@ func NewTestCert(v cert.Version, curve cert.Curve, ca cert.Certificate, key []by
 	return c, pub, cert.MarshalPrivateKeyToPEM(curve, priv), pem
 }
 
+func NewTestCertDifferentVersion(c cert.Certificate, v cert.Version, ca cert.Certificate, key []byte) (cert.Certificate, []byte) {
+	nc := &cert.TBSCertificate{
+		Version:        v,
+		Curve:          c.Curve(),
+		Name:           c.Name(),
+		Networks:       c.Networks(),
+		UnsafeNetworks: c.UnsafeNetworks(),
+		Groups:         c.Groups(),
+		NotBefore:      time.Unix(c.NotBefore().Unix(), 0),
+		NotAfter:       time.Unix(c.NotAfter().Unix(), 0),
+		PublicKey:      c.PublicKey(),
+		IsCA:           false,
+	}
+
+	c, err := nc.Sign(ca, ca.Curve(), key)
+	if err != nil {
+		panic(err)
+	}
+
+	pem, err := c.MarshalPEM()
+	if err != nil {
+		panic(err)
+	}
+
+	return c, pem
+}
+
 func X25519Keypair() ([]byte, []byte) {
 	privkey := make([]byte, 32)
 	if _, err := io.ReadFull(rand.Reader, privkey); err != nil {

connection_manager.go

@@ -354,7 +354,6 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 
 		if mainHostInfo {
 			decision = tryRehandshake
-
 		} else {
 			if cm.shouldSwapPrimary(hostinfo) {
 				decision = swapPrimary
@@ -461,6 +460,10 @@ func (cm *connectionManager) shouldSwapPrimary(current *HostInfo) bool {
 	}
 
 	crt := cm.intf.pki.getCertState().getCertificate(current.ConnectionState.myCert.Version())
+	if crt == nil {
+		//my cert was reloaded away. We should definitely swap from this tunnel
+		return true
+	}
 	// If this tunnel is using the latest certificate then we should swap it to primary for a bit and see if things
 	// settle down.
 	return bytes.Equal(current.ConnectionState.myCert.Signature(), crt.Signature())
@@ -475,31 +478,34 @@ func (cm *connectionManager) swapPrimary(current, primary *HostInfo) {
 	cm.hostMap.Unlock()
 }
 
-// isInvalidCertificate will check if we should destroy a tunnel if pki.disconnect_invalid is true and
+// isInvalidCertificate decides if we should destroy a tunnel.
-// the certificate is no longer valid. Block listed certificates will skip the pki.disconnect_invalid
+// returns true if pki.disconnect_invalid is true and the certificate is no longer valid.
-// check and return true.
+// Blocklisted certificates will skip the pki.disconnect_invalid check and return true.
 func (cm *connectionManager) isInvalidCertificate(now time.Time, hostinfo *HostInfo) bool {
 	remoteCert := hostinfo.GetCert()
 	if remoteCert == nil {
-		return false
+		return false //don't tear down tunnels for handshakes in progress
 	}
 
 	caPool := cm.intf.pki.GetCAPool()
 	err := caPool.VerifyCachedCertificate(now, remoteCert)
 	if err == nil {
-		return false
+		return false //cert is still valid! yay!
-	}
+	} else if err == cert.ErrBlockListed { //avoiding errors.Is for speed
-
-	if !cm.intf.disconnectInvalid.Load() && err != cert.ErrBlockListed {
 		// Block listed certificates should always be disconnected
-		return false
+		hostinfo.logger(cm.l).WithError(err).
-	}
+			WithField("fingerprint", remoteCert.Fingerprint).
-
+			Info("Remote certificate is blocked, tearing down the tunnel")
+		return true
+	} else if cm.intf.disconnectInvalid.Load() {
 		hostinfo.logger(cm.l).WithError(err).
 			WithField("fingerprint", remoteCert.Fingerprint).
 			Info("Remote certificate is no longer valid, tearing down the tunnel")
-
 		return true
+	} else {
+		//if we reach here, the cert is no longer valid, but we're configured to keep tunnels from now-invalid certs open
+		return false
+	}
 }
 
 func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
@@ -530,15 +536,45 @@ func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
 func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
 	cs := cm.intf.pki.getCertState()
 	curCrt := hostinfo.ConnectionState.myCert
-	myCrt := cs.getCertificate(curCrt.Version())
+	curCrtVersion := curCrt.Version()
-	if curCrt.Version() >= cs.initiatingVersion && bytes.Equal(curCrt.Signature(), myCrt.Signature()) == true {
+	myCrt := cs.getCertificate(curCrtVersion)
-		// The current tunnel is using the latest certificate and version, no need to rehandshake.
+	if myCrt == nil {
+		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
+			WithField("version", curCrtVersion).
+			WithField("reason", "local certificate removed").
+			Info("Re-handshaking with remote")
+		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
 		return
 	}
-
+	peerCrt := hostinfo.ConnectionState.peerCert
+	if peerCrt != nil && curCrtVersion < peerCrt.Certificate.Version() {
+		// if our certificate version is less than theirs, and we have a matching version available, rehandshake?
+		if cs.getCertificate(peerCrt.Certificate.Version()) != nil {
+			cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
+				WithField("version", curCrtVersion).
+				WithField("peerVersion", peerCrt.Certificate.Version()).
+				WithField("reason", "local certificate version lower than peer, attempting to correct").
+				Info("Re-handshaking with remote")
+			cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], func(hh *HandshakeHostInfo) {
+				hh.initiatingVersionOverride = peerCrt.Certificate.Version()
+			})
+			return
+		}
+	}
+	if !bytes.Equal(curCrt.Signature(), myCrt.Signature()) {
 		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
 			WithField("reason", "local certificate is not current").
 			Info("Re-handshaking with remote")
 
 		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
+		return
+	}
+	if curCrtVersion < cs.initiatingVersion {
+		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
+			WithField("reason", "current cert version < pki.initiatingVersion").
+			Info("Re-handshaking with remote")
+
+		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
+		return
+	}
 }

connection_manager_test.go

@@ -446,10 +446,6 @@ func (d *dummyCert) PublicKey() []byte {
 	return d.publicKey
 }
 
-func (d *dummyCert) MarshalPublicKeyPEM() []byte {
-	return cert.MarshalPublicKeyToPEM(d.curve, d.publicKey)
-}
-
 func (d *dummyCert) Signature() []byte {
 	return d.signature
 }

e2e/helpers_test.go

@@ -146,6 +146,109 @@ func newSimpleServerWithUdp(v cert.Version, caCrt cert.Certificate, caKey []byte
 	return control, vpnNetworks, udpAddr, c
 }
 
+// newServer creates a nebula instance with fewer assumptions
+func newServer(caCrt []cert.Certificate, certs []cert.Certificate, key []byte, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
+	l := NewTestLogger()
+
+	vpnNetworks := certs[len(certs)-1].Networks()
+
+	var udpAddr netip.AddrPort
+	if vpnNetworks[0].Addr().Is4() {
+		budpIp := vpnNetworks[0].Addr().As4()
+		budpIp[1] -= 128
+		udpAddr = netip.AddrPortFrom(netip.AddrFrom4(budpIp), 4242)
+	} else {
+		budpIp := vpnNetworks[0].Addr().As16()
+		// beef for funsies
+		budpIp[2] = 190
+		budpIp[3] = 239
+		udpAddr = netip.AddrPortFrom(netip.AddrFrom16(budpIp), 4242)
+	}
+
+	caStr := ""
+	for _, ca := range caCrt {
+		x, err := ca.MarshalPEM()
+		if err != nil {
+			panic(err)
+		}
+		caStr += string(x)
+	}
+	certStr := ""
+	for _, c := range certs {
+		x, err := c.MarshalPEM()
+		if err != nil {
+			panic(err)
+		}
+		certStr += string(x)
+	}
+
+	mc := m{
+		"pki": m{
+			"ca":   caStr,
+			"cert": certStr,
+			"key":  string(key),
+		},
+		//"tun": m{"disabled": true},
+		"firewall": m{
+			"outbound": []m{{
+				"proto": "any",
+				"port":  "any",
+				"host":  "any",
+			}},
+			"inbound": []m{{
+				"proto": "any",
+				"port":  "any",
+				"host":  "any",
+			}},
+		},
+		//"handshakes": m{
+		//	"try_interval": "1s",
+		//},
+		"listen": m{
+			"host": udpAddr.Addr().String(),
+			"port": udpAddr.Port(),
+		},
+		"logging": m{
+			"timestamp_format": fmt.Sprintf("%v 15:04:05.000000", certs[0].Name()),
+			"level":            l.Level.String(),
+		},
+		"timers": m{
+			"pending_deletion_interval": 2,
+			"connection_alive_interval": 2,
+		},
+	}
+
+	if overrides != nil {
+		final := m{}
+		err := mergo.Merge(&final, overrides, mergo.WithAppendSlice)
+		if err != nil {
+			panic(err)
+		}
+		err = mergo.Merge(&final, mc, mergo.WithAppendSlice)
+		if err != nil {
+			panic(err)
+		}
+		mc = final
+	}
+
+	cb, err := yaml.Marshal(mc)
+	if err != nil {
+		panic(err)
+	}
+
+	c := config.NewC(l)
+	cStr := string(cb)
+	c.LoadString(cStr)
+
+	control, err := nebula.Main(c, false, "e2e-test", l, nil)
+
+	if err != nil {
+		panic(err)
+	}
+
+	return control, vpnNetworks, udpAddr, c
+}
+
 type doneCb func()
 
 func deadline(t *testing.T, seconds time.Duration) doneCb {

e2e/tunnels_test.go

@@ -4,6 +4,7 @@
 package e2e
 
 import (
+	"fmt"
 	"net/netip"
 	"testing"
 	"time"
@@ -11,6 +12,8 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/e2e/router"
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v3"
 )
 
 func TestDropInactiveTunnels(t *testing.T) {
@@ -57,6 +60,265 @@ func TestDropInactiveTunnels(t *testing.T) {
 	theirControl.Stop()
 }
 
+func TestCertUpgrade(t *testing.T) {
+	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
+	// under ideal conditions
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	caB, err := ca.MarshalPEM()
+	if err != nil {
+		panic(err)
+	}
+	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	ca2B, err := ca2.MarshalPEM()
+	if err != nil {
+		panic(err)
+	}
+	caStr := fmt.Sprintf("%s\n%s", caB, ca2B)
+
+	myCert, _, myPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
+	_, myCert2Pem := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
+
+	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
+	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
+
+	myControl, myVpnIpNet, myUdpAddr, myC := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert}, myPrivKey, m{})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
+
+	// Share our underlay information
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	// Start the servers
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	r.Log("Assert the tunnel between me and them works")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+	r.Log("yay")
+	//todo ???
+	time.Sleep(1 * time.Second)
+	r.FlushAll()
+
+	mc := m{
+		"pki": m{
+			"ca":   caStr,
+			"cert": string(myCert2Pem),
+			"key":  string(myPrivKey),
+		},
+		//"tun": m{"disabled": true},
+		"firewall": myC.Settings["firewall"],
+		//"handshakes": m{
+		//	"try_interval": "1s",
+		//},
+		"listen":  myC.Settings["listen"],
+		"logging": myC.Settings["logging"],
+		"timers":  myC.Settings["timers"],
+	}
+
+	cb, err := yaml.Marshal(mc)
+	if err != nil {
+		panic(err)
+	}
+
+	r.Logf("reload new v2-only config")
+	err = myC.ReloadConfigString(string(cb))
+	assert.NoError(t, err)
+	r.Log("yay, spin until their sees it")
+	waitStart := time.Now()
+	for {
+		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+		if c == nil {
+			r.Log("nil")
+		} else {
+			version := c.Cert.Version()
+			r.Logf("version %d", version)
+			if version == cert.Version2 {
+				break
+			}
+		}
+		since := time.Since(waitStart)
+		if since > time.Second*10 {
+			t.Fatal("Cert should be new by now")
+		}
+		time.Sleep(time.Second)
+	}
+
+	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestCertDowngrade(t *testing.T) {
+	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
+	// under ideal conditions
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	caB, err := ca.MarshalPEM()
+	if err != nil {
+		panic(err)
+	}
+	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	ca2B, err := ca2.MarshalPEM()
+	if err != nil {
+		panic(err)
+	}
+	caStr := fmt.Sprintf("%s\n%s", caB, ca2B)
+
+	myCert, _, myPrivKey, myCertPem := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
+	myCert2, _ := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
+
+	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
+	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
+
+	myControl, myVpnIpNet, myUdpAddr, myC := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert2}, myPrivKey, m{})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
+
+	// Share our underlay information
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	// Start the servers
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	r.Log("Assert the tunnel between me and them works")
+	//assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
+	//r.Log("yay")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+	r.Log("yay")
+	//todo ???
+	time.Sleep(1 * time.Second)
+	r.FlushAll()
+
+	mc := m{
+		"pki": m{
+			"ca":   caStr,
+			"cert": string(myCertPem),
+			"key":  string(myPrivKey),
+		},
+		"firewall": myC.Settings["firewall"],
+		"listen":   myC.Settings["listen"],
+		"logging":  myC.Settings["logging"],
+		"timers":   myC.Settings["timers"],
+	}
+
+	cb, err := yaml.Marshal(mc)
+	if err != nil {
+		panic(err)
+	}
+
+	r.Logf("reload new v1-only config")
+	err = myC.ReloadConfigString(string(cb))
+	assert.NoError(t, err)
+	r.Log("yay, spin until their sees it")
+	waitStart := time.Now()
+	for {
+		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+		c2 := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
+		if c == nil || c2 == nil {
+			r.Log("nil")
+		} else {
+			version := c.Cert.Version()
+			theirVersion := c2.Cert.Version()
+			r.Logf("version %d,%d", version, theirVersion)
+			if version == cert.Version1 {
+				break
+			}
+		}
+		since := time.Since(waitStart)
+		if since > time.Second*5 {
+			r.Log("wtf")
+		}
+		if since > time.Second*10 {
+			r.Log("wtf")
+			t.Fatal("Cert should be new by now")
+		}
+		time.Sleep(time.Second)
+	}
+
+	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestCertMismatchCorrection(t *testing.T) {
+	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
+	// under ideal conditions
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	myCert, _, myPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
+	myCert2, _ := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
+
+	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
+	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
+
+	myControl, myVpnIpNet, myUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert2}, myPrivKey, m{})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
+
+	// Share our underlay information
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	// Start the servers
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	r.Log("Assert the tunnel between me and them works")
+	//assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
+	//r.Log("yay")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+	r.Log("yay")
+	//todo ???
+	time.Sleep(1 * time.Second)
+	r.FlushAll()
+
+	waitStart := time.Now()
+	for {
+		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+		c2 := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
+		if c == nil || c2 == nil {
+			r.Log("nil")
+		} else {
+			version := c.Cert.Version()
+			theirVersion := c2.Cert.Version()
+			r.Logf("version %d,%d", version, theirVersion)
+			if version == theirVersion {
+				break
+			}
+		}
+		since := time.Since(waitStart)
+		if since > time.Second*5 {
+			r.Log("wtf")
+		}
+		if since > time.Second*10 {
+			r.Log("wtf")
+			t.Fatal("Cert should be new by now")
+		}
+		time.Sleep(time.Second)
+	}
+
+	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
 func TestCrossStackRelaysWork(t *testing.T) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24,fc00::1/64", m{"relay": m{"use_relays": true}})

firewall.go

@@ -417,8 +417,6 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 	return nil
 }
 
-var ErrUnknownNetworkType = errors.New("unknown network type")
-var ErrPeerRejected = errors.New("remote IP is not within a subnet that we handle")
 var ErrInvalidRemoteIP = errors.New("remote IP is not in remote certificate subnets")
 var ErrInvalidLocalIP = errors.New("local IP is not in list of handled local IPs")
 var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
@@ -431,31 +429,18 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
 		return nil
 	}
 
-	// Make sure remote address matches nebula certificate, and determine how to treat it
+	// Make sure remote address matches nebula certificate
-	if h.networks == nil {
+	if h.networks != nil {
-		// Simple case: Certificate has one address and no unsafe networks
+		if !h.networks.Contains(fp.RemoteAddr) {
-		if h.vpnAddrs[0] != fp.RemoteAddr {
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
 			return ErrInvalidRemoteIP
 		}
 	} else {
-		nwType, ok := h.networks.Lookup(fp.RemoteAddr)
+		// Simple case: Certificate has one address and no unsafe networks
-		if !ok {
+		if h.vpnAddrs[0] != fp.RemoteAddr {
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
 			return ErrInvalidRemoteIP
 		}
-		switch nwType {
-		case NetworkTypeVPN:
-			break // nothing special
-		case NetworkTypeVPNPeer:
-			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-			return ErrPeerRejected // reject for now, one day this may have different FW rules
-		case NetworkTypeUnsafe:
-			break // nothing special, one day this may have different FW rules
-		default:
-			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-			return ErrUnknownNetworkType //should never happen
-		}
 	}
 
 	// Make sure we are supposed to be handling this local ip address

firewall_test.go

@@ -8,7 +8,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
@@ -150,8 +149,7 @@ func TestFirewall_Drop(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
+
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("1.2.3.4"),
 		RemoteAddr: netip.MustParseAddr("1.2.3.4"),
@@ -176,7 +174,7 @@ func TestFirewall_Drop(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
 	}
-	h.buildNetworks(myVpnNetworksTable, c.networks, c.unsafeNetworks)
+	h.buildNetworks(c.networks, c.unsafeNetworks)
 
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
@@ -228,9 +226,6 @@ func TestFirewall_DropV6(t *testing.T) {
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
 
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("fd00::/7"))
-
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("fd12::34"),
 		RemoteAddr: netip.MustParseAddr("fd12::34"),
@@ -255,7 +250,7 @@ func TestFirewall_DropV6(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{netip.MustParseAddr("fd12::34")},
 	}
-	h.buildNetworks(myVpnNetworksTable, c.networks, c.unsafeNetworks)
+	h.buildNetworks(c.networks, c.unsafeNetworks)
 
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
@@ -397,7 +392,7 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 		c := &cert.CachedCertificate{
 			Certificate: &dummyCert{
 				name:     "nope",
-				networks: []netip.Prefix{netip.MustParsePrefix("fd99::99/128")},
+				networks: []netip.Prefix{netip.MustParsePrefix("fd99:99/128")},
 			},
 			InvertedGroups: map[string]struct{}{"nope": {}},
 		}
@@ -458,8 +453,6 @@ func TestFirewall_Drop2(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("1.2.3.4"),
@@ -485,7 +478,7 @@ func TestFirewall_Drop2(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h.buildNetworks(myVpnNetworksTable, c.Certificate.Networks(), c.Certificate.UnsafeNetworks())
+	h.buildNetworks(c.Certificate.Networks(), c.Certificate.UnsafeNetworks())
 
 	c1 := cert.CachedCertificate{
 		Certificate: &dummyCert{
@@ -500,7 +493,7 @@ func TestFirewall_Drop2(t *testing.T) {
 			peerCert: &c1,
 		},
 	}
-	h1.buildNetworks(myVpnNetworksTable, c1.Certificate.Networks(), c1.Certificate.UnsafeNetworks())
+	h1.buildNetworks(c1.Certificate.Networks(), c1.Certificate.UnsafeNetworks())
 
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group", "test-group"}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
@@ -517,8 +510,6 @@ func TestFirewall_Drop3(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("1.2.3.4"),
@@ -550,7 +541,7 @@ func TestFirewall_Drop3(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h1.buildNetworks(myVpnNetworksTable, c1.Certificate.Networks(), c1.Certificate.UnsafeNetworks())
+	h1.buildNetworks(c1.Certificate.Networks(), c1.Certificate.UnsafeNetworks())
 
 	c2 := cert.CachedCertificate{
 		Certificate: &dummyCert{
@@ -565,7 +556,7 @@ func TestFirewall_Drop3(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h2.buildNetworks(myVpnNetworksTable, c2.Certificate.Networks(), c2.Certificate.UnsafeNetworks())
+	h2.buildNetworks(c2.Certificate.Networks(), c2.Certificate.UnsafeNetworks())
 
 	c3 := cert.CachedCertificate{
 		Certificate: &dummyCert{
@@ -580,7 +571,7 @@ func TestFirewall_Drop3(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h3.buildNetworks(myVpnNetworksTable, c3.Certificate.Networks(), c3.Certificate.UnsafeNetworks())
+	h3.buildNetworks(c3.Certificate.Networks(), c3.Certificate.UnsafeNetworks())
 
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "host1", netip.Prefix{}, netip.Prefix{}, "", ""))
@@ -606,8 +597,6 @@ func TestFirewall_Drop3V6(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("fd00::/7"))
 
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("fd12::34"),
@@ -631,7 +620,7 @@ func TestFirewall_Drop3V6(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h.buildNetworks(myVpnNetworksTable, c.Certificate.Networks(), c.Certificate.UnsafeNetworks())
+	h.buildNetworks(c.Certificate.Networks(), c.Certificate.UnsafeNetworks())
 
 	// Test a remote address match
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -644,8 +633,6 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("1.2.3.4"),
@@ -672,7 +659,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h.buildNetworks(myVpnNetworksTable, c.Certificate.Networks(), c.Certificate.UnsafeNetworks())
+	h.buildNetworks(c.Certificate.Networks(), c.Certificate.UnsafeNetworks())
 
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
@@ -705,52 +692,6 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
 }
 
-func TestFirewall_DropIPSpoofing(t *testing.T) {
-	l := test.NewLogger()
-	ob := &bytes.Buffer{}
-	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
-
-	c := cert.CachedCertificate{
-		Certificate: &dummyCert{
-			name:     "host-owner",
-			networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
-		},
-	}
-
-	c1 := cert.CachedCertificate{
-		Certificate: &dummyCert{
-			name:           "host",
-			networks:       []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")},
-			unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
-		},
-	}
-	h1 := HostInfo{
-		ConnectionState: &ConnectionState{
-			peerCert: &c1,
-		},
-		vpnAddrs: []netip.Addr{c1.Certificate.Networks()[0].Addr()},
-	}
-	h1.buildNetworks(myVpnNetworksTable, c1.Certificate.Networks(), c1.Certificate.UnsafeNetworks())
-
-	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
-
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
-	cp := cert.NewCAPool()
-
-	// Packet spoofed by `c1`. Note that the remote addr is not a valid one.
-	p := firewall.Packet{
-		LocalAddr:  netip.MustParseAddr("192.0.2.1"),
-		RemoteAddr: netip.MustParseAddr("192.0.2.3"),
-		LocalPort:  1,
-		RemotePort: 1,
-		Protocol:   firewall.ProtoUDP,
-		Fragment:   false,
-	}
-	assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP)
-}
-
 func BenchmarkLookup(b *testing.B) {
 	ml := func(m map[string]struct{}, a [][]string) {
 		for n := 0; n < b.N; n++ {

go.mod

@@ -8,30 +8,30 @@ require (
 	github.com/armon/go-radix v1.0.0
 	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
 	github.com/flynn/noise v1.1.0
-	github.com/gaissmai/bart v0.25.0
+	github.com/gaissmai/bart v0.20.4
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gopacket v1.1.19
-	github.com/kardianos/service v1.2.4
+	github.com/kardianos/service v1.2.2
-	github.com/miekg/dns v1.1.68
+	github.com/miekg/dns v1.1.65
 	github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b
 	github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
-	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_golang v1.22.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/sirupsen/logrus v1.9.3
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
-	github.com/stretchr/testify v1.11.1
+	github.com/stretchr/testify v1.10.0
 	github.com/vishvananda/netlink v1.3.1
-	golang.org/x/crypto v0.43.0
+	golang.org/x/crypto v0.37.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.45.0
+	golang.org/x/net v0.39.0
-	golang.org/x/sync v0.17.0
+	golang.org/x/sync v0.13.0
-	golang.org/x/sys v0.37.0
+	golang.org/x/sys v0.32.0
-	golang.org/x/term v0.36.0
+	golang.org/x/term v0.31.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
 	golang.zx2c4.com/wireguard/windows v0.5.3
-	google.golang.org/protobuf v1.36.8
+	google.golang.org/protobuf v1.36.6
 	gopkg.in/yaml.v3 v3.0.1
 	gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
 )
@@ -43,12 +43,11 @@ require (
 	github.com/google/btree v1.1.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/common v0.62.0 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/vishvananda/netns v0.0.5 // indirect
-	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	golang.org/x/mod v0.23.0 // indirect
-	golang.org/x/mod v0.24.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.33.0 // indirect
+	golang.org/x/tools v0.30.0 // indirect
 )

go.sum

@@ -24,8 +24,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
 github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
-github.com/gaissmai/bart v0.25.0 h1:eqiokVPqM3F94vJ0bTHXHtH91S8zkKL+bKh+BsGOsJM=
+github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U=
-github.com/gaissmai/bart v0.25.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
+github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -64,8 +64,8 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kardianos/service v1.2.4 h1:XNlGtZOYNx2u91urOdg/Kfmc+gfmuIo1Dd3rEi2OgBk=
+github.com/kardianos/service v1.2.2 h1:ZvePhAHfvo0A7Mftk/tEzqEZ7Q4lgnR8sGz4xu1YX60=
-github.com/kardianos/service v1.2.4/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=
+github.com/kardianos/service v1.2.2/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
@@ -83,8 +83,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA=
+github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc=
-github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps=
+github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck=
 github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b h1:J/AzCvg5z0Hn1rqZUJjpbzALUmkKX0Zwbc/i4fw7Sfk=
 github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -106,24 +106,24 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
-github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
-github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
-github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
-github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -143,33 +143,29 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
 github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
 github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
 github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
-go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
+golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
-golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
+golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
+golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
-golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -180,8 +176,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM=
+golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
-golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
+golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -189,8 +185,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -201,17 +197,18 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
-golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
+golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
-golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
+golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -222,8 +219,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
+golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
-golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
+golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -242,8 +239,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
+google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
-google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

handshake_ix.go

@@ -23,15 +23,19 @@ func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
 		return false
 	}
 
-	// If we're connecting to a v6 address we must use a v2 cert
 	cs := f.pki.getCertState()
 	v := cs.initiatingVersion
+	if hh.initiatingVersionOverride != cert.VersionPre1 {
+		v = hh.initiatingVersionOverride
+	} else if v < cert.Version2 {
+		// If we're connecting to a v6 address we must use a v2 cert
 		for _, a := range hh.hostinfo.vpnAddrs {
 			if a.Is6() {
 				v = cert.Version2
 				break
 			}
 		}
+	}
 
 	crt := cs.getCertificate(v)
 	if crt == nil {
@@ -163,16 +167,19 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 
 	if remoteCert.Certificate.Version() != ci.myCert.Version() {
 		// We started off using the wrong certificate version, lets see if we can match the version that was sent to us
-		rc := cs.getCertificate(remoteCert.Certificate.Version())
+		myCertOtherVersion := cs.getCertificate(remoteCert.Certificate.Version())
-		if rc == nil {
+		if myCertOtherVersion == nil {
-			f.l.WithError(err).WithField("udpAddr", addr).
+			if f.l.Level >= logrus.DebugLevel {
-				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cert", remoteCert).
+				f.l.WithError(err).WithFields(m{
-				Info("Unable to handshake with host due to missing certificate version")
+					"udpAddr":   addr,
-			return
+					"handshake": m{"stage": 1, "style": "ix_psk0"},
+					"cert":      remoteCert,
+				}).Debug("Might be unable to handshake with host due to missing certificate version")
 			}
-
+		} else {
 			// Record the certificate we are actually using
-		ci.myCert = rc
+			ci.myCert = myCertOtherVersion
+		}
 	}
 
 	if len(remoteCert.Certificate.Networks()) == 0 {
@@ -183,18 +190,17 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 		return
 	}
 
+	var vpnAddrs []netip.Addr
+	var filteredNetworks []netip.Prefix
 	certName := remoteCert.Certificate.Name()
 	certVersion := remoteCert.Certificate.Version()
 	fingerprint := remoteCert.Fingerprint
 	issuer := remoteCert.Certificate.Issuer()
-	vpnNetworks := remoteCert.Certificate.Networks()
 
-	anyVpnAddrsInCommon := false
+	for _, network := range remoteCert.Certificate.Networks() {
-	vpnAddrs := make([]netip.Addr, len(vpnNetworks))
-	for i, network := range vpnNetworks {
 		vpnAddr := network.Addr()
 		if f.myVpnAddrsTable.Contains(vpnAddr) {
-			f.l.WithField("vpnNetworks", vpnNetworks).WithField("udpAddr", addr).
+			f.l.WithField("vpnAddr", vpnAddr).WithField("udpAddr", addr).
 				WithField("certName", certName).
 				WithField("certVersion", certVersion).
 				WithField("fingerprint", fingerprint).
@@ -202,10 +208,24 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Refusing to handshake with myself")
 			return
 		}
-		vpnAddrs[i] = network.Addr()
+
-		if f.myVpnNetworksTable.Contains(vpnAddr) {
+		// vpnAddrs outside our vpn networks are of no use to us, filter them out
-			anyVpnAddrsInCommon = true
+		if !f.myVpnNetworksTable.Contains(vpnAddr) {
+			continue
 		}
+
+		filteredNetworks = append(filteredNetworks, network)
+		vpnAddrs = append(vpnAddrs, vpnAddr)
+	}
+
+	if len(vpnAddrs) == 0 {
+		f.l.WithError(err).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("No usable vpn addresses from host, refusing handshake")
+		return
 	}
 
 	if addr.IsValid() {
@@ -242,30 +262,26 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 		},
 	}
 
-	msgRxL := f.l.WithFields(m{
+	f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
-		"vpnAddrs":       vpnAddrs,
+		WithField("certName", certName).
-		"udpAddr":        addr,
+		WithField("certVersion", certVersion).
-		"certName":       certName,
+		WithField("fingerprint", fingerprint).
-		"certVersion":    certVersion,
+		WithField("issuer", issuer).
-		"fingerprint":    fingerprint,
+		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-		"issuer":         issuer,
+		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-		"initiatorIndex": hs.Details.InitiatorIndex,
+		Info("Handshake message received")
-		"responderIndex": hs.Details.ResponderIndex,
-		"remoteIndex":    h.RemoteIndex,
-		"handshake":      m{"stage": 1, "style": "ix_psk0"},
-	})
-
-	if anyVpnAddrsInCommon {
-		msgRxL.Info("Handshake message received")
-	} else {
-		//todo warn if not lighthouse or relay?
-		msgRxL.Info("Handshake message received, but no vpnNetworks in common.")
-	}
 
 	hs.Details.ResponderIndex = myIndex
 	hs.Details.Cert = cs.getHandshakeBytes(ci.myCert.Version())
 	if hs.Details.Cert == nil {
-		msgRxL.WithField("myCertVersion", ci.myCert.Version()).
+		f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			WithField("certVersion", ci.myCert.Version()).
 			Error("Unable to handshake with host because no certificate handshake bytes is available")
 		return
 	}
@@ -323,7 +339,7 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 
 	hostinfo.remotes = f.lightHouse.QueryCache(vpnAddrs)
 	hostinfo.SetRemote(addr)
-	hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate.Networks(), remoteCert.Certificate.UnsafeNetworks())
+	hostinfo.buildNetworks(filteredNetworks, remoteCert.Certificate.UnsafeNetworks())
 
 	existing, err := f.handshakeManager.CheckAndComplete(hostinfo, 0, f)
 	if err != nil {
@@ -564,17 +580,30 @@ func ixHandshakeStage2(f *Interface, addr netip.AddrPort, via *ViaSender, hh *Ha
 		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
 	}
 
-	anyVpnAddrsInCommon := false
+	var vpnAddrs []netip.Addr
-	vpnAddrs := make([]netip.Addr, len(vpnNetworks))
+	var filteredNetworks []netip.Prefix
-	for i, network := range vpnNetworks {
+	for _, network := range vpnNetworks {
-		vpnAddrs[i] = network.Addr()
+		// vpnAddrs outside our vpn networks are of no use to us, filter them out
-		if f.myVpnNetworksTable.Contains(network.Addr()) {
+		vpnAddr := network.Addr()
-			anyVpnAddrsInCommon = true
+		if !f.myVpnNetworksTable.Contains(vpnAddr) {
+			continue
 		}
+
+		filteredNetworks = append(filteredNetworks, network)
+		vpnAddrs = append(vpnAddrs, vpnAddr)
+	}
+
+	if len(vpnAddrs) == 0 {
+		f.l.WithError(err).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).Error("No usable vpn addresses from host, refusing handshake")
+		return true
 	}
 
 	// Ensure the right host responded
-	// todo is it more correct to see if any of hostinfo.vpnAddrs are in the cert? it should have len==1, but one day it might not?
 	if !slices.Contains(vpnAddrs, hostinfo.vpnAddrs[0]) {
 		f.l.WithField("intendedVpnAddrs", hostinfo.vpnAddrs).WithField("haveVpnNetworks", vpnNetworks).
 			WithField("udpAddr", addr).
@@ -587,7 +616,6 @@ func ixHandshakeStage2(f *Interface, addr netip.AddrPort, via *ViaSender, hh *Ha
 		f.handshakeManager.DeleteHostInfo(hostinfo)
 
 		// Create a new hostinfo/handshake for the intended vpn ip
-		//TODO is hostinfo.vpnAddrs[0] always the address to use?
 		f.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) {
 			// Block the current used address
 			newHH.hostinfo.remotes = hostinfo.remotes
@@ -614,7 +642,7 @@ func ixHandshakeStage2(f *Interface, addr netip.AddrPort, via *ViaSender, hh *Ha
 	ci.window.Update(f.l, 2)
 
 	duration := time.Since(hh.startTime).Nanoseconds()
-	msgRxL := f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+	f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 		WithField("certName", certName).
 		WithField("certVersion", certVersion).
 		WithField("fingerprint", fingerprint).
@@ -622,17 +650,12 @@ func ixHandshakeStage2(f *Interface, addr netip.AddrPort, via *ViaSender, hh *Ha
 		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
 		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 		WithField("durationNs", duration).
-		WithField("sentCachedPackets", len(hh.packetStore))
+		WithField("sentCachedPackets", len(hh.packetStore)).
-	if anyVpnAddrsInCommon {
+		Info("Handshake message received")
-		msgRxL.Info("Handshake message received")
-	} else {
-		//todo warn if not lighthouse or relay?
-		msgRxL.Info("Handshake message received, but no vpnNetworks in common.")
-	}
 
 	// Build up the radix for the firewall if we have subnets in the cert
 	hostinfo.vpnAddrs = vpnAddrs
-	hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate.Networks(), remoteCert.Certificate.UnsafeNetworks())
+	hostinfo.buildNetworks(filteredNetworks, remoteCert.Certificate.UnsafeNetworks())
 
 	// Complete our handshake and update metrics, this will replace any existing tunnels for the vpnAddrs here
 	f.handshakeManager.Complete(hostinfo, f)

handshake_manager.go

@@ -70,6 +70,7 @@ type HandshakeHostInfo struct {
 
 	startTime                 time.Time        // Time that we first started trying with this handshake
 	ready                     bool             // Is the handshake ready
+	initiatingVersionOverride cert.Version     // Should we use a non-default cert version for this handshake?
 	counter                   int64            // How many attempts have we made so far
 	lastRemotes               []netip.AddrPort // Remotes that we sent to during the previous attempt
 	packetStore               []*cachedPacket  // A set of packets to be transmitted once the handshake completes
@@ -299,6 +300,8 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 						InitiatorRelayIndex: idx,
 					}
 
+					relayFrom := hm.f.myVpnAddrs[0]
+
 					switch relayHostInfo.GetCert().Certificate.Version() {
 					case cert.Version1:
 						if !hm.f.myVpnAddrs[0].Is4() {
@@ -316,7 +319,13 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 						b = vpnIp.As4()
 						m.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 					case cert.Version2:
-						m.RelayFromAddr = netAddrToProtoAddr(hm.f.myVpnAddrs[0])
+						if vpnIp.Is4() {
+							relayFrom = hm.f.myVpnAddrs[0]
+						} else {
+							//todo do this smarter
+							relayFrom = hm.f.myVpnAddrs[len(hm.f.myVpnAddrs)-1]
+						}
+						m.RelayFromAddr = netAddrToProtoAddr(relayFrom)
 						m.RelayToAddr = netAddrToProtoAddr(vpnIp)
 					default:
 						hostinfo.logger(hm.l).Error("Unknown certificate version found while creating relay")
@@ -331,7 +340,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 					} else {
 						hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
 						hm.l.WithFields(logrus.Fields{
-							"relayFrom":           hm.f.myVpnAddrs[0],
+							"relayFrom":           relayFrom,
 							"relayTo":             vpnIp,
 							"initiatorRelayIndex": idx,
 							"relay":               relay}).
@@ -357,6 +366,8 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 					InitiatorRelayIndex: existingRelay.LocalIndex,
 				}
 
+				relayFrom := hm.f.myVpnAddrs[0]
+
 				switch relayHostInfo.GetCert().Certificate.Version() {
 				case cert.Version1:
 					if !hm.f.myVpnAddrs[0].Is4() {
@@ -374,7 +385,14 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 					b = vpnIp.As4()
 					m.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 				case cert.Version2:
-					m.RelayFromAddr = netAddrToProtoAddr(hm.f.myVpnAddrs[0])
+					if vpnIp.Is4() {
+						relayFrom = hm.f.myVpnAddrs[0]
+					} else {
+						//todo do this smarter
+						relayFrom = hm.f.myVpnAddrs[len(hm.f.myVpnAddrs)-1]
+					}
+
+					m.RelayFromAddr = netAddrToProtoAddr(relayFrom)
 					m.RelayToAddr = netAddrToProtoAddr(vpnIp)
 				default:
 					hostinfo.logger(hm.l).Error("Unknown certificate version found while creating relay")
@@ -389,7 +407,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 					// This must send over the hostinfo, not over hm.Hosts[ip]
 					hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
 					hm.l.WithFields(logrus.Fields{
-						"relayFrom":           hm.f.myVpnAddrs[0],
+						"relayFrom":           relayFrom,
 						"relayTo":             vpnIp,
 						"initiatorRelayIndex": existingRelay.LocalIndex,
 						"relay":               relay}).

hostmap.go

@@ -2,6 +2,7 @@ package nebula
 
 import (
 	"errors"
+	"fmt"
 	"net"
 	"net/netip"
 	"slices"
@@ -212,18 +213,6 @@ func (rs *RelayState) InsertRelay(ip netip.Addr, idx uint32, r *Relay) {
 	rs.relayForByIdx[idx] = r
 }
 
-type NetworkType uint8
-
-const (
-	NetworkTypeUnknown NetworkType = iota
-	// NetworkTypeVPN is a network that overlaps one or more of the vpnNetworks in our certificate
-	NetworkTypeVPN
-	// NetworkTypeVPNPeer is a network that does not overlap one of our networks
-	NetworkTypeVPNPeer
-	// NetworkTypeUnsafe is a network from Certificate.UnsafeNetworks()
-	NetworkTypeUnsafe
-)
-
 type HostInfo struct {
 	remote          netip.AddrPort
 	remotes         *RemoteList
@@ -232,11 +221,13 @@ type HostInfo struct {
 	remoteIndexId   uint32
 	localIndexId    uint32
 
-	// vpnAddrs is a list of vpn addresses assigned to this host
+	// vpnAddrs is a list of vpn addresses assigned to this host that are within our own vpn networks
+	// The host may have other vpn addresses that are outside our
+	// vpn networks but were removed because they are not usable
 	vpnAddrs []netip.Addr
 
-	// networks is a combination of specific vpn addresses (not prefixes!) and full unsafe networks assigned to this host.
+	// networks are both all vpn and unsafe networks assigned to this host
-	networks   *bart.Table[NetworkType]
+	networks   *bart.Lite
 	relayState RelayState
 
 	// HandshakePacket records the packets used to create this hostinfo
@@ -531,6 +522,7 @@ func (hm *HostMap) QueryVpnAddrsRelayFor(targetIps []netip.Addr, relayHostIp net
 		return nil, nil, errors.New("unable to find host")
 	}
 
+	lastH := h
 	for h != nil {
 		for _, targetIp := range targetIps {
 			r, ok := h.relayState.QueryRelayForByIp(targetIp)
@@ -538,10 +530,12 @@ func (hm *HostMap) QueryVpnAddrsRelayFor(targetIps []netip.Addr, relayHostIp net
 				return h, r, nil
 			}
 		}
+		lastH = h
 		h = h.next
 	}
 
-	return nil, nil, errors.New("unable to find host with relay")
+	//todo no merge
+	return nil, nil, fmt.Errorf("unable to find host with relay: %v", lastH)
 }
 
 func (hm *HostMap) unlockedDisestablishVpnAddrRelayFor(hi *HostInfo) {
@@ -740,27 +734,19 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, newRemote netip.AddrPort) b
 	return false
 }
 
-func (i *HostInfo) buildNetworks(myVpnNetworksTable *bart.Lite, networks, unsafeNetworks []netip.Prefix) {
+func (i *HostInfo) buildNetworks(networks, unsafeNetworks []netip.Prefix) {
 	if len(networks) == 1 && len(unsafeNetworks) == 0 {
-		if myVpnNetworksTable.Contains(networks[0].Addr()) {
+		// Simple case, no CIDRTree needed
-			return // Simple case, no CIDRTree needed
+		return
-		}
 	}
 
-	i.networks = new(bart.Table[NetworkType])
+	i.networks = new(bart.Lite)
 	for _, network := range networks {
-		var nwType NetworkType
+		i.networks.Insert(network)
-		if myVpnNetworksTable.Contains(network.Addr()) {
-			nwType = NetworkTypeVPN
-		} else {
-			nwType = NetworkTypeVPNPeer
-		}
-		nprefix := netip.PrefixFrom(network.Addr(), network.Addr().BitLen())
-		i.networks.Insert(nprefix, nwType)
 	}
 
 	for _, network := range unsafeNetworks {
-		i.networks.Insert(network, NetworkTypeUnsafe)
+		i.networks.Insert(network)
 	}
 }
 

lighthouse.go

@@ -1017,17 +1017,17 @@ func (lhh *LightHouseHandler) resetMeta() *NebulaMeta {
 	return lhh.meta
 }
 
-func (lhh *LightHouseHandler) HandleRequest(rAddr netip.AddrPort, hostinfo *HostInfo, p []byte, w EncWriter) {
+func (lhh *LightHouseHandler) HandleRequest(rAddr netip.AddrPort, fromVpnAddrs []netip.Addr, p []byte, w EncWriter) {
 	n := lhh.resetMeta()
 	err := n.Unmarshal(p)
 	if err != nil {
-		lhh.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", rAddr).
+		lhh.l.WithError(err).WithField("vpnAddrs", fromVpnAddrs).WithField("udpAddr", rAddr).
 			Error("Failed to unmarshal lighthouse packet")
 		return
 	}
 
 	if n.Details == nil {
-		lhh.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", rAddr).
+		lhh.l.WithField("vpnAddrs", fromVpnAddrs).WithField("udpAddr", rAddr).
 			Error("Invalid lighthouse update")
 		return
 	}
@@ -1036,24 +1036,24 @@ func (lhh *LightHouseHandler) HandleRequest(rAddr netip.AddrPort, hostinfo *Host
 
 	switch n.Type {
 	case NebulaMeta_HostQuery:
-		lhh.handleHostQuery(n, hostinfo, rAddr, w)
+		lhh.handleHostQuery(n, fromVpnAddrs, rAddr, w)
 
 	case NebulaMeta_HostQueryReply:
-		lhh.handleHostQueryReply(n, hostinfo.vpnAddrs)
+		lhh.handleHostQueryReply(n, fromVpnAddrs)
 
 	case NebulaMeta_HostUpdateNotification:
-		lhh.handleHostUpdateNotification(n, hostinfo, w)
+		lhh.handleHostUpdateNotification(n, fromVpnAddrs, w)
 
 	case NebulaMeta_HostMovedNotification:
 	case NebulaMeta_HostPunchNotification:
-		lhh.handleHostPunchNotification(n, hostinfo.vpnAddrs, w)
+		lhh.handleHostPunchNotification(n, fromVpnAddrs, w)
 
 	case NebulaMeta_HostUpdateNotificationAck:
 		// noop
 	}
 }
 
-func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, hostinfo *HostInfo, addr netip.AddrPort, w EncWriter) {
+func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []netip.Addr, addr netip.AddrPort, w EncWriter) {
 	// Exit if we don't answer queries
 	if !lhh.lh.amLighthouse {
 		if lhh.l.Level >= logrus.DebugLevel {
@@ -1065,7 +1065,7 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, hostinfo *HostInfo,
 	queryVpnAddr, useVersion, err := n.Details.GetVpnAddrAndVersion()
 	if err != nil {
 		if lhh.l.Level >= logrus.DebugLevel {
-			lhh.l.WithField("from", hostinfo.vpnAddrs).WithField("details", n.Details).
+			lhh.l.WithField("from", fromVpnAddrs).WithField("details", n.Details).
 				Debugln("Dropping malformed HostQuery")
 		}
 		return
@@ -1073,7 +1073,7 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, hostinfo *HostInfo,
 	if useVersion == cert.Version1 && queryVpnAddr.Is6() {
 		// this case really shouldn't be possible to represent, but reject it anyway.
 		if lhh.l.Level >= logrus.DebugLevel {
-			lhh.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("queryVpnAddr", queryVpnAddr).
+			lhh.l.WithField("vpnAddrs", fromVpnAddrs).WithField("queryVpnAddr", queryVpnAddr).
 				Debugln("invalid vpn addr for v1 handleHostQuery")
 		}
 		return
@@ -1099,14 +1099,14 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, hostinfo *HostInfo,
 	}
 
 	if err != nil {
-		lhh.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).Error("Failed to marshal lighthouse host query reply")
+		lhh.l.WithError(err).WithField("vpnAddrs", fromVpnAddrs).Error("Failed to marshal lighthouse host query reply")
 		return
 	}
 
 	lhh.lh.metricTx(NebulaMeta_HostQueryReply, 1)
-	w.SendMessageToHostInfo(header.LightHouse, 0, hostinfo, lhh.pb[:ln], lhh.nb, lhh.out[:0])
+	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.nb, lhh.out[:0])
 
-	lhh.sendHostPunchNotification(n, hostinfo.vpnAddrs, queryVpnAddr, w)
+	lhh.sendHostPunchNotification(n, fromVpnAddrs, queryVpnAddr, w)
 }
 
 // sendHostPunchNotification signals the other side to punch some zero byte udp packets
@@ -1115,34 +1115,20 @@ func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAd
 	found, ln, err := lhh.lh.queryAndPrepMessage(whereToPunch, func(c *cache) (int, error) {
 		n = lhh.resetMeta()
 		n.Type = NebulaMeta_HostPunchNotification
-		punchNotifDestHI := lhh.lh.ifce.GetHostInfo(punchNotifDest)
+		targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest)
 		var useVersion cert.Version
-		if punchNotifDestHI == nil {
+		if targetHI == nil {
 			useVersion = lhh.lh.ifce.GetCertState().initiatingVersion
 		} else {
+			crt := targetHI.GetCert().Certificate
+			useVersion = crt.Version()
 			// we can only retarget if we have a hostinfo
-			punchNotifDestCrt := punchNotifDestHI.GetCert().Certificate
+			newDest, ok := findNetworkUnion(crt.Networks(), fromVpnAddrs)
-			useVersion = punchNotifDestCrt.Version()
-			punchNotifDestNetworks := punchNotifDestCrt.Networks()
-
-			//if we (the lighthouse) don't have a network in common with punchNotifDest, try to find one
-			if !lhh.lh.myVpnNetworksTable.Contains(punchNotifDest) {
-				newPunchNotifDest, ok := findNetworkUnion(lhh.lh.myVpnNetworks, punchNotifDestHI.vpnAddrs)
 			if ok {
-					punchNotifDest = newPunchNotifDest
+				whereToPunch = newDest
 			} else {
 				if lhh.l.Level >= logrus.DebugLevel {
-						lhh.l.WithField("to", punchNotifDestNetworks).Debugln("unable to notify host to host, no addresses in common")
+					lhh.l.WithField("to", crt.Networks()).Debugln("unable to punch to host, no addresses in common")
-					}
-				}
-			}
-
-			newWhereToPunch, ok := findNetworkUnion(punchNotifDestNetworks, fromVpnAddrs)
-			if ok {
-				whereToPunch = newWhereToPunch
-			} else {
-				if lhh.l.Level >= logrus.DebugLevel {
-					lhh.l.WithFields(m{"from": fromVpnAddrs, "to": punchNotifDestNetworks}).Debugln("unable to punch to host, no addresses in common with requestor")
 				}
 			}
 		}
@@ -1248,8 +1234,7 @@ func (lhh *LightHouseHandler) handleHostQueryReply(n *NebulaMeta, fromVpnAddrs [
 	}
 }
 
-func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, hostinfo *HostInfo, w EncWriter) {
+func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, w EncWriter) {
-	fromVpnAddrs := hostinfo.vpnAddrs
 	if !lhh.lh.amLighthouse {
 		if lhh.l.Level >= logrus.DebugLevel {
 			lhh.l.Debugln("I am not a lighthouse, do not take host updates: ", fromVpnAddrs)
@@ -1317,7 +1302,7 @@ func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, hostin
 	}
 
 	lhh.lh.metricTx(NebulaMeta_HostUpdateNotificationAck, 1)
-	w.SendMessageToHostInfo(header.LightHouse, 0, hostinfo, lhh.pb[:ln], lhh.nb, lhh.out[:0])
+	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.nb, lhh.out[:0])
 }
 
 func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, w EncWriter) {

lighthouse_test.go

@@ -132,13 +132,8 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) {
 	)
 
 	mw := &mockEncWriter{}
-	hostinfo := &HostInfo{
+
-		ConnectionState: &ConnectionState{
+	hi := []netip.Addr{vpnIp2}
-			eKey: nil,
-			dKey: nil,
-		},
-		vpnAddrs: []netip.Addr{vpnIp2},
-	}
 	b.Run("notfound", func(b *testing.B) {
 		lhh := lh.NewRequestHandler()
 		req := &NebulaMeta{
@@ -151,7 +146,7 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) {
 		p, err := req.Marshal()
 		require.NoError(b, err)
 		for n := 0; n < b.N; n++ {
-			lhh.HandleRequest(rAddr, hostinfo, p, mw)
+			lhh.HandleRequest(rAddr, hi, p, mw)
 		}
 	})
 	b.Run("found", func(b *testing.B) {
@@ -167,7 +162,7 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) {
 		require.NoError(b, err)
 
 		for n := 0; n < b.N; n++ {
-			lhh.HandleRequest(rAddr, hostinfo, p, mw)
+			lhh.HandleRequest(rAddr, hi, p, mw)
 		}
 	})
 }
@@ -331,14 +326,7 @@ func newLHHostRequest(fromAddr netip.AddrPort, myVpnIp, queryVpnIp netip.Addr, l
 	w := &testEncWriter{
 		metaFilter: &filter,
 	}
-	hostinfo := &HostInfo{
+	lhh.HandleRequest(fromAddr, []netip.Addr{myVpnIp}, b, w)
-		ConnectionState: &ConnectionState{
-			eKey: nil,
-			dKey: nil,
-		},
-		vpnAddrs: []netip.Addr{myVpnIp},
-	}
-	lhh.HandleRequest(fromAddr, hostinfo, b, w)
 	return w.lastReply
 }
 
@@ -367,15 +355,9 @@ func newLHHostUpdate(fromAddr netip.AddrPort, vpnIp netip.Addr, addrs []netip.Ad
 	if err != nil {
 		panic(err)
 	}
-	hostinfo := &HostInfo{
+
-		ConnectionState: &ConnectionState{
-			eKey: nil,
-			dKey: nil,
-		},
-		vpnAddrs: []netip.Addr{vpnIp},
-	}
 	w := &testEncWriter{}
-	lhh.HandleRequest(fromAddr, hostinfo, b, w)
+	lhh.HandleRequest(fromAddr, []netip.Addr{vpnIp}, b, w)
 }
 
 type testLhReply struct {

outside.go

@@ -138,7 +138,7 @@ func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []
 			return
 		}
 
-		lhf.HandleRequest(ip, hostinfo, d, f)
+		lhf.HandleRequest(ip, hostinfo.vpnAddrs, d, f)
 
 		// Fallthrough to the bottom to record incoming traffic
 

overlay/tun.go

@@ -1,8 +1,6 @@
 package overlay
 
 import (
-	"fmt"
-	"net"
 	"net/netip"
 
 	"github.com/sirupsen/logrus"
@@ -72,51 +70,3 @@ func findRemovedRoutes(newRoutes, oldRoutes []Route) []Route {
 
 	return removed
 }
-
-func prefixToMask(prefix netip.Prefix) netip.Addr {
-	pLen := 128
-	if prefix.Addr().Is4() {
-		pLen = 32
-	}
-
-	addr, _ := netip.AddrFromSlice(net.CIDRMask(prefix.Bits(), pLen))
-	return addr
-}
-
-func flipBytes(b []byte) []byte {
-	for i := 0; i < len(b); i++ {
-		b[i] ^= 0xFF
-	}
-	return b
-}
-func orBytes(a []byte, b []byte) []byte {
-	ret := make([]byte, len(a))
-	for i := 0; i < len(a); i++ {
-		ret[i] = a[i] | b[i]
-	}
-	return ret
-}
-
-func getBroadcast(cidr netip.Prefix) netip.Addr {
-	broadcast, _ := netip.AddrFromSlice(
-		orBytes(
-			cidr.Addr().AsSlice(),
-			flipBytes(prefixToMask(cidr).AsSlice()),
-		),
-	)
-	return broadcast
-}
-
-func selectGateway(dest netip.Prefix, gateways []netip.Prefix) (netip.Prefix, error) {
-	for _, gateway := range gateways {
-		if dest.Addr().Is4() && gateway.Addr().Is4() {
-			return gateway, nil
-		}
-
-		if dest.Addr().Is6() && gateway.Addr().Is6() {
-			return gateway, nil
-		}
-	}
-
-	return netip.Prefix{}, fmt.Errorf("no gateway found for %v in the list of vpn networks", dest)
-}

overlay/tun_darwin.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/netip"
 	"os"
 	"sync/atomic"
@@ -294,6 +295,7 @@ func (t *tun) activate6(network netip.Prefix) error {
 			Vltime: 0xffffffff,
 			Pltime: 0xffffffff,
 		},
+		//TODO: CERT-V2 should we disable DAD (duplicate address detection) and mark this as a secured address?
 		Flags: _IN6_IFF_NODAD,
 	}
 
@@ -552,3 +554,13 @@ func (t *tun) Name() string {
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
 }
+
+func prefixToMask(prefix netip.Prefix) netip.Addr {
+	pLen := 128
+	if prefix.Addr().Is4() {
+		pLen = 32
+	}
+
+	addr, _ := netip.AddrFromSlice(net.CIDRMask(prefix.Bits(), pLen))
+	return addr
+}

overlay/tun_freebsd.go

@@ -10,9 +10,11 @@ import (
 	"io"
 	"io/fs"
 	"net/netip"
+	"os"
+	"os/exec"
+	"strconv"
 	"sync/atomic"
 	"syscall"
-	"time"
 	"unsafe"
 
 	"github.com/gaissmai/bart"
@@ -20,18 +22,12 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	netroute "golang.org/x/net/route"
-	"golang.org/x/sys/unix"
 )
 
 const (
 	// FIODGNAME is defined in sys/sys/filio.h on FreeBSD
 	// For 32-bit systems, use FIODGNAME_32 (not defined in this file: 0x80086678)
 	FIODGNAME = 0x80106678
-	TUNSIFMODE       = 0x8004745e
-	TUNSIFHEAD       = 0x80047460
-	OSIOCAIFADDR_IN6 = 0x8088691b
-	IN6_IFF_NODAD    = 0x0020
 )
 
 type fiodgnameArg struct {
@@ -41,159 +37,43 @@ type fiodgnameArg struct {
 }
 
 type ifreqRename struct {
-	Name [unix.IFNAMSIZ]byte
+	Name [16]byte
 	Data uintptr
 }
 
 type ifreqDestroy struct {
-	Name [unix.IFNAMSIZ]byte
+	Name [16]byte
 	pad  [16]byte
 }
 
-type ifReq struct {
-	Name  [unix.IFNAMSIZ]byte
-	Flags uint16
-}
-
-type ifreqMTU struct {
-	Name [unix.IFNAMSIZ]byte
-	MTU  int32
-}
-
-type addrLifetime struct {
-	Expire    uint64
-	Preferred uint64
-	Vltime    uint32
-	Pltime    uint32
-}
-
-type ifreqAlias4 struct {
-	Name     [unix.IFNAMSIZ]byte
-	Addr     unix.RawSockaddrInet4
-	DstAddr  unix.RawSockaddrInet4
-	MaskAddr unix.RawSockaddrInet4
-	VHid     uint32
-}
-
-type ifreqAlias6 struct {
-	Name       [unix.IFNAMSIZ]byte
-	Addr       unix.RawSockaddrInet6
-	DstAddr    unix.RawSockaddrInet6
-	PrefixMask unix.RawSockaddrInet6
-	Flags      uint32
-	Lifetime   addrLifetime
-	VHid       uint32
-}
-
 type tun struct {
 	Device      string
 	vpnNetworks []netip.Prefix
 	MTU         int
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	linkAddr    *netroute.LinkAddr
 	l           *logrus.Logger
-	devFd       int
-}
 
-func (t *tun) Read(to []byte) (int, error) {
+	io.ReadWriteCloser
-	// use readv() to read from the tunnel device, to eliminate the need for copying the buffer
-	if t.devFd < 0 {
-		return -1, syscall.EINVAL
-	}
-
-	// first 4 bytes is protocol family, in network byte order
-	head := make([]byte, 4)
-
-	iovecs := []syscall.Iovec{
-		{&head[0], 4},
-		{&to[0], uint64(len(to))},
-	}
-
-	n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-
-	var err error
-	if errno != 0 {
-		err = syscall.Errno(errno)
-	} else {
-		err = nil
-	}
-	// fix bytes read number to exclude header
-	bytesRead := int(n)
-	if bytesRead < 0 {
-		return bytesRead, err
-	} else if bytesRead < 4 {
-		return 0, err
-	} else {
-		return bytesRead - 4, err
-	}
-}
-
-// Write is only valid for single threaded use
-func (t *tun) Write(from []byte) (int, error) {
-	// use writev() to write to the tunnel device, to eliminate the need for copying the buffer
-	if t.devFd < 0 {
-		return -1, syscall.EINVAL
-	}
-
-	if len(from) <= 1 {
-		return 0, syscall.EIO
-	}
-	ipVer := from[0] >> 4
-	var head []byte
-	// first 4 bytes is protocol family, in network byte order
-	if ipVer == 4 {
-		head = []byte{0, 0, 0, syscall.AF_INET}
-	} else if ipVer == 6 {
-		head = []byte{0, 0, 0, syscall.AF_INET6}
-	} else {
-		return 0, fmt.Errorf("unable to determine IP version from packet")
-	}
-	iovecs := []syscall.Iovec{
-		{&head[0], 4},
-		{&from[0], uint64(len(from))},
-	}
-
-	n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-
-	var err error
-	if errno != 0 {
-		err = syscall.Errno(errno)
-	} else {
-		err = nil
-	}
-
-	return int(n) - 4, err
 }
 
 func (t *tun) Close() error {
-	if t.devFd >= 0 {
+	if t.ReadWriteCloser != nil {
-		err := syscall.Close(t.devFd)
+		if err := t.ReadWriteCloser.Close(); err != nil {
-		if err != nil {
+			return err
-			t.l.WithError(err).Error("Error closing device")
 		}
-		t.devFd = -1
 
-		c := make(chan struct{})
-		go func() {
-			// destroying the interface can block if a read() is still pending. Do this asynchronously.
-			defer close(c)
 		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
-			if err == nil {
-				defer syscall.Close(s)
-				ifreq := ifreqDestroy{Name: t.deviceBytes()}
-				err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
-			}
 		if err != nil {
-				t.l.WithError(err).Error("Error destroying tunnel")
+			return err
 		}
-		}()
+		defer syscall.Close(s)
 
-		// wait up to 1 second so we start blocking at the ioctl
+		ifreq := ifreqDestroy{Name: t.deviceBytes()}
-		select {
+
-		case <-c:
+		// Destroy the interface
-		case <-time.After(1 * time.Second):
+		err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
-		}
+		return err
 	}
 
 	return nil
@@ -205,37 +85,32 @@ func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun,
 
 func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	// Try to open existing tun device
-	var fd int
+	var file *os.File
 	var err error
 	deviceName := c.GetString("tun.dev", "")
 	if deviceName != "" {
-		fd, err = syscall.Open("/dev/"+deviceName, syscall.O_RDWR, 0)
+		file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0)
 	}
 	if errors.Is(err, fs.ErrNotExist) || deviceName == "" {
 		// If the device doesn't already exist, request a new one and rename it
-		fd, err = syscall.Open("/dev/tun", syscall.O_RDWR, 0)
+		file, err = os.OpenFile("/dev/tun", os.O_RDWR, 0)
 	}
 	if err != nil {
 		return nil, err
 	}
 
-	// Read the name of the interface
+	rawConn, err := file.SyscallConn()
+	if err != nil {
+		return nil, fmt.Errorf("SyscallConn: %v", err)
+	}
+
 	var name [16]byte
+	var ctrlErr error
+	rawConn.Control(func(fd uintptr) {
+		// Read the name of the interface
 		arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)}
-	ctrlErr := ioctl(uintptr(fd), FIODGNAME, uintptr(unsafe.Pointer(&arg)))
+		ctrlErr = ioctl(fd, FIODGNAME, uintptr(unsafe.Pointer(&arg)))
-
+	})
-	if ctrlErr == nil {
-		// set broadcast mode and multicast
-		ifmode := uint32(unix.IFF_BROADCAST | unix.IFF_MULTICAST)
-		ctrlErr = ioctl(uintptr(fd), TUNSIFMODE, uintptr(unsafe.Pointer(&ifmode)))
-	}
-
-	if ctrlErr == nil {
-		// turn on link-layer mode, to support ipv6
-		ifhead := uint32(1)
-		ctrlErr = ioctl(uintptr(fd), TUNSIFHEAD, uintptr(unsafe.Pointer(&ifhead)))
-	}
-
 	if ctrlErr != nil {
 		return nil, err
 	}
@@ -247,7 +122,11 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 
 	// If the name doesn't match the desired interface name, rename it now
 	if ifName != deviceName {
-		s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+		s, err := syscall.Socket(
+			syscall.AF_INET,
+			syscall.SOCK_DGRAM,
+			syscall.IPPROTO_IP,
+		)
 		if err != nil {
 			return nil, err
 		}
@@ -270,11 +149,11 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	}
 
 	t := &tun{
+		ReadWriteCloser: file,
 		Device:          deviceName,
 		vpnNetworks:     vpnNetworks,
 		MTU:             c.GetInt("tun.mtu", DefaultMTU),
 		l:               l,
-		devFd:       fd,
 	}
 
 	err = t.reload(c, true)
@@ -293,111 +172,38 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 }
 
 func (t *tun) addIp(cidr netip.Prefix) error {
-	if cidr.Addr().Is4() {
+	var err error
-		ifr := ifreqAlias4{
+	// TODO use syscalls instead of exec.Command
-			Name: t.deviceBytes(),
+	cmd := exec.Command("/sbin/ifconfig", t.Device, cidr.String(), cidr.Addr().String())
-			Addr: unix.RawSockaddrInet4{
+	t.l.Debug("command: ", cmd.String())
-				Len:    unix.SizeofSockaddrInet4,
+	if err = cmd.Run(); err != nil {
-				Family: unix.AF_INET,
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-				Addr:   cidr.Addr().As4(),
-			},
-			DstAddr: unix.RawSockaddrInet4{
-				Len:    unix.SizeofSockaddrInet4,
-				Family: unix.AF_INET,
-				Addr:   getBroadcast(cidr).As4(),
-			},
-			MaskAddr: unix.RawSockaddrInet4{
-				Len:    unix.SizeofSockaddrInet4,
-				Family: unix.AF_INET,
-				Addr:   prefixToMask(cidr).As4(),
-			},
-			VHid: 0,
-		}
-		s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-		// Note: unix.SIOCAIFADDR corresponds to FreeBSD's OSIOCAIFADDR
-		if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err)
-		}
-		return nil
 	}
 
-	if cidr.Addr().Is6() {
+	cmd = exec.Command("/sbin/route", "-n", "add", "-net", cidr.String(), "-interface", t.Device)
-		ifr := ifreqAlias6{
+	t.l.Debug("command: ", cmd.String())
-			Name: t.deviceBytes(),
+	if err = cmd.Run(); err != nil {
-			Addr: unix.RawSockaddrInet6{
+		return fmt.Errorf("failed to run 'route add': %s", err)
-				Len:    unix.SizeofSockaddrInet6,
-				Family: unix.AF_INET6,
-				Addr:   cidr.Addr().As16(),
-			},
-			PrefixMask: unix.RawSockaddrInet6{
-				Len:    unix.SizeofSockaddrInet6,
-				Family: unix.AF_INET6,
-				Addr:   prefixToMask(cidr).As16(),
-			},
-			Lifetime: addrLifetime{
-				Expire:    0,
-				Preferred: 0,
-				Vltime:    0xffffffff,
-				Pltime:    0xffffffff,
-			},
-			Flags: IN6_IFF_NODAD,
-		}
-		s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-
-		if err := ioctl(uintptr(s), OSIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err)
-		}
-		return nil
 	}
 
-	return fmt.Errorf("unknown address type %v", cidr)
+	cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU))
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
+	}
+
+	// Unsafe path routes
+	return t.addRoutes(false)
 }
 
 func (t *tun) Activate() error {
-	// Setup our default MTU
-	err := t.setMTU()
-	if err != nil {
-		return err
-	}
-
-	linkAddr, err := getLinkAddr(t.Device)
-	if err != nil {
-		return err
-	}
-	if linkAddr == nil {
-		return fmt.Errorf("unable to discover link_addr for tun interface")
-	}
-	t.linkAddr = linkAddr
-
 	for i := range t.vpnNetworks {
 		err := t.addIp(t.vpnNetworks[i])
 		if err != nil {
 			return err
 		}
 	}
-
+	return nil
-	return t.addRoutes(false)
-}
-
-func (t *tun) setMTU() error {
-	// Set the MTU on the device
-	s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-	if err != nil {
-		return err
-	}
-	defer syscall.Close(s)
-
-	ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MTU)}
-	err = ioctl(uintptr(s), unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm)))
-	return err
 }
 
 func (t *tun) reload(c *config.C, initial bool) error {
@@ -462,16 +268,15 @@ func (t *tun) addRoutes(logErrors bool) error {
 			continue
 		}
 
-		err := addRoute(r.Cidr, t.linkAddr)
+		cmd := exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), "-interface", t.Device)
-		if err != nil {
+		t.l.Debug("command: ", cmd.String())
-			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
+		if err := cmd.Run(); err != nil {
+			retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
 			} else {
 				return retErr
 			}
-		} else {
-			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -484,8 +289,9 @@ func (t *tun) removeRoutes(routes []Route) error {
 			continue
 		}
 
-		err := delRoute(r.Cidr, t.linkAddr)
+		cmd := exec.Command("/sbin/route", "-n", "delete", "-net", r.Cidr.String(), "-interface", t.Device)
-		if err != nil {
+		t.l.Debug("command: ", cmd.String())
+		if err := cmd.Run(); err != nil {
 			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
 			t.l.WithField("route", r).Info("Removed route")
@@ -500,120 +306,3 @@ func (t *tun) deviceBytes() (o [16]byte) {
 	}
 	return
 }
-
-func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
-	if err != nil {
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
-
-	route := &netroute.RouteMessage{
-		Version: unix.RTM_VERSION,
-		Type:    unix.RTM_ADD,
-		Flags:   unix.RTF_UP,
-		Seq:     1,
-	}
-
-	if prefix.Addr().Is4() {
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: gateway,
-		}
-	} else {
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: gateway,
-		}
-	}
-
-	data, err := route.Marshal()
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
-
-	_, err = unix.Write(sock, data[:])
-	if err != nil {
-		if errors.Is(err, unix.EEXIST) {
-			// Try to do a change
-			route.Type = unix.RTM_CHANGE
-			data, err = route.Marshal()
-			if err != nil {
-				return fmt.Errorf("failed to create route.RouteMessage for change: %w", err)
-			}
-			_, err = unix.Write(sock, data[:])
-			fmt.Println("DOING CHANGE")
-			return err
-		}
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
-}
-
-func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
-	if err != nil {
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
-
-	route := netroute.RouteMessage{
-		Version: unix.RTM_VERSION,
-		Type:    unix.RTM_DELETE,
-		Seq:     1,
-	}
-
-	if prefix.Addr().Is4() {
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: gateway,
-		}
-	} else {
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: gateway,
-		}
-	}
-
-	data, err := route.Marshal()
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
-	_, err = unix.Write(sock, data[:])
-	if err != nil {
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
-}
-
-// getLinkAddr Gets the link address for the interface of the given name
-func getLinkAddr(name string) (*netroute.LinkAddr, error) {
-	rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0)
-	if err != nil {
-		return nil, err
-	}
-	msgs, err := netroute.ParseRIB(unix.NET_RT_IFLIST, rib)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, m := range msgs {
-		switch m := m.(type) {
-		case *netroute.InterfaceMessage:
-			if m.Name == name {
-				sa, ok := m.Addrs[unix.RTAX_IFP].(*netroute.LinkAddr)
-				if ok {
-					return sa, nil
-				}
-			}
-		}
-	}
-
-	return nil, nil
-}

overlay/tun_netbsd.go

@@ -4,12 +4,13 @@
 package overlay
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"net/netip"
 	"os"
+	"os/exec"
 	"regexp"
+	"strconv"
 	"sync/atomic"
 	"syscall"
 	"unsafe"
@@ -19,42 +20,11 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	netroute "golang.org/x/net/route"
-	"golang.org/x/sys/unix"
 )
 
-const (
+type ifreqDestroy struct {
-	SIOCAIFADDR_IN6 = 0x8080696b
+	Name [16]byte
-	TUNSIFHEAD      = 0x80047442
+	pad  [16]byte
-	TUNSIFMODE      = 0x80047458
-)
-
-type ifreqAlias4 struct {
-	Name     [unix.IFNAMSIZ]byte
-	Addr     unix.RawSockaddrInet4
-	DstAddr  unix.RawSockaddrInet4
-	MaskAddr unix.RawSockaddrInet4
-}
-
-type ifreqAlias6 struct {
-	Name       [unix.IFNAMSIZ]byte
-	Addr       unix.RawSockaddrInet6
-	DstAddr    unix.RawSockaddrInet6
-	PrefixMask unix.RawSockaddrInet6
-	Flags      uint32
-	Lifetime   addrLifetime
-}
-
-type ifreq struct {
-	Name [unix.IFNAMSIZ]byte
-	data int
-}
-
-type addrLifetime struct {
-	Expire    uint64
-	Preferred uint64
-	Vltime    uint32
-	Pltime    uint32
 }
 
 type tun struct {
@@ -64,18 +34,40 @@ type tun struct {
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	l           *logrus.Logger
-	f           *os.File
+
-	fd          int
+	io.ReadWriteCloser
 }
 
-var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+func (t *tun) Close() error {
+	if t.ReadWriteCloser != nil {
+		if err := t.ReadWriteCloser.Close(); err != nil {
+			return err
+		}
+
+		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+		if err != nil {
+			return err
+		}
+		defer syscall.Close(s)
+
+		ifreq := ifreqDestroy{Name: t.deviceBytes()}
+
+		err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
+
+		return err
+	}
+	return nil
+}
 
 func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in NetBSD")
 }
 
+var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+
 func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	// Try to open tun device
+	var file *os.File
 	var err error
 	deviceName := c.GetString("tun.dev", "")
 	if deviceName == "" {
@@ -85,19 +77,13 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
 	}
 
-	fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
+	file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
 	}
 
-	err = unix.SetNonblock(fd, true)
-	if err != nil {
-		l.WithError(err).Warn("Failed to set the tun device as nonblocking")
-	}
-
 	t := &tun{
-		f:           os.NewFile(uintptr(fd), ""),
+		ReadWriteCloser: file,
-		fd:          fd,
 		Device:          deviceName,
 		vpnNetworks:     vpnNetworks,
 		MTU:             c.GetInt("tun.mtu", DefaultMTU),
@@ -119,225 +105,40 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	return t, nil
 }
 
-func (t *tun) Close() error {
-	if t.f != nil {
-		if err := t.f.Close(); err != nil {
-			return fmt.Errorf("error closing tun file: %w", err)
-		}
-
-		// t.f.Close should have handled it for us but let's be extra sure
-		_ = unix.Close(t.fd)
-
-		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-
-		ifr := ifreq{Name: t.deviceBytes()}
-		err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifr)))
-		return err
-	}
-	return nil
-}
-
-func (t *tun) Read(to []byte) (int, error) {
-	rc, err := t.f.SyscallConn()
-	if err != nil {
-		return 0, fmt.Errorf("failed to get syscall conn for tun: %w", err)
-	}
-
-	var errno syscall.Errno
-	var n uintptr
-	err = rc.Read(func(fd uintptr) bool {
-		// first 4 bytes is protocol family, in network byte order
-		head := [4]byte{}
-		iovecs := []syscall.Iovec{
-			{&head[0], 4},
-			{&to[0], uint64(len(to))},
-		}
-
-		n, _, errno = syscall.Syscall(syscall.SYS_READV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-		if errno.Temporary() {
-			// We got an EAGAIN, EINTR, or EWOULDBLOCK, go again
-			return false
-		}
-		return true
-	})
-	if err != nil {
-		if err == syscall.EBADF || err.Error() == "use of closed file" {
-			// Go doesn't export poll.ErrFileClosing but happily reports it to us so here we are
-			// https://github.com/golang/go/blob/master/src/internal/poll/fd_poll_runtime.go#L121
-			return 0, os.ErrClosed
-		}
-		return 0, fmt.Errorf("failed to make read call for tun: %w", err)
-	}
-
-	if errno != 0 {
-		return 0, fmt.Errorf("failed to make inner read call for tun: %w", errno)
-	}
-
-	// fix bytes read number to exclude header
-	bytesRead := int(n)
-	if bytesRead < 0 {
-		return bytesRead, nil
-	} else if bytesRead < 4 {
-		return 0, nil
-	} else {
-		return bytesRead - 4, nil
-	}
-}
-
-// Write is only valid for single threaded use
-func (t *tun) Write(from []byte) (int, error) {
-	if len(from) <= 1 {
-		return 0, syscall.EIO
-	}
-
-	ipVer := from[0] >> 4
-	var head [4]byte
-	// first 4 bytes is protocol family, in network byte order
-	if ipVer == 4 {
-		head[3] = syscall.AF_INET
-	} else if ipVer == 6 {
-		head[3] = syscall.AF_INET6
-	} else {
-		return 0, fmt.Errorf("unable to determine IP version from packet")
-	}
-
-	rc, err := t.f.SyscallConn()
-	if err != nil {
-		return 0, err
-	}
-
-	var errno syscall.Errno
-	var n uintptr
-	err = rc.Write(func(fd uintptr) bool {
-		iovecs := []syscall.Iovec{
-			{&head[0], 4},
-			{&from[0], uint64(len(from))},
-		}
-
-		n, _, errno = syscall.Syscall(syscall.SYS_WRITEV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-		// According to NetBSD documentation for TUN, writes will only return errors in which
-		// this packet will never be delivered so just go on living life.
-		return true
-	})
-	if err != nil {
-		return 0, err
-	}
-
-	if errno != 0 {
-		return 0, errno
-	}
-
-	return int(n) - 4, err
-}
-
 func (t *tun) addIp(cidr netip.Prefix) error {
-	if cidr.Addr().Is4() {
+	var err error
-		var req ifreqAlias4
+
-		req.Name = t.deviceBytes()
+	// TODO use syscalls instead of exec.Command
-		req.Addr = unix.RawSockaddrInet4{
+	cmd := exec.Command("/sbin/ifconfig", t.Device, cidr.String(), cidr.Addr().String())
-			Len:    unix.SizeofSockaddrInet4,
+	t.l.Debug("command: ", cmd.String())
-			Family: unix.AF_INET,
+	if err = cmd.Run(); err != nil {
-			Addr:   cidr.Addr().As4(),
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-		}
-		req.DstAddr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   cidr.Addr().As4(),
-		}
-		req.MaskAddr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   prefixToMask(cidr).As4(),
 	}
 
-		s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+	cmd = exec.Command("/sbin/route", "-n", "add", "-net", cidr.String(), cidr.Addr().String())
-		if err != nil {
+	t.l.Debug("command: ", cmd.String())
-			return err
+	if err = cmd.Run(); err != nil {
-		}
+		return fmt.Errorf("failed to run 'route add': %s", err)
-		defer syscall.Close(s)
-
-		if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&req))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr(), err)
 	}
 
-		return nil
+	cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU))
-	}
+	t.l.Debug("command: ", cmd.String())
-
+	if err = cmd.Run(); err != nil {
-	if cidr.Addr().Is6() {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-		var req ifreqAlias6
-		req.Name = t.deviceBytes()
-		req.Addr = unix.RawSockaddrInet6{
-			Len:    unix.SizeofSockaddrInet6,
-			Family: unix.AF_INET6,
-			Addr:   cidr.Addr().As16(),
-		}
-		req.PrefixMask = unix.RawSockaddrInet6{
-			Len:    unix.SizeofSockaddrInet6,
-			Family: unix.AF_INET6,
-			Addr:   prefixToMask(cidr).As16(),
-		}
-		req.Lifetime = addrLifetime{
-			Vltime: 0xffffffff,
-			Pltime: 0xffffffff,
-		}
-
-		s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-
-		if err := ioctl(uintptr(s), SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&req))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err)
-		}
-		return nil
-	}
-
-	return fmt.Errorf("unknown address type %v", cidr)
-}
-
-func (t *tun) Activate() error {
-	mode := int32(unix.IFF_BROADCAST)
-	err := ioctl(uintptr(t.fd), TUNSIFMODE, uintptr(unsafe.Pointer(&mode)))
-	if err != nil {
-		return fmt.Errorf("failed to set tun device mode: %w", err)
-	}
-
-	v := 1
-	err = ioctl(uintptr(t.fd), TUNSIFHEAD, uintptr(unsafe.Pointer(&v)))
-	if err != nil {
-		return fmt.Errorf("failed to set tun device head: %w", err)
-	}
-
-	err = t.doIoctlByName(unix.SIOCSIFMTU, uint32(t.MTU))
-	if err != nil {
-		return fmt.Errorf("failed to set tun mtu: %w", err)
-	}
-
-	for i := range t.vpnNetworks {
-		err = t.addIp(t.vpnNetworks[i])
-		if err != nil {
-			return err
-		}
 	}
 
+	// Unsafe path routes
 	return t.addRoutes(false)
 }
 
-func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
+func (t *tun) Activate() error {
-	s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
+	for i := range t.vpnNetworks {
+		err := t.addIp(t.vpnNetworks[i])
 		if err != nil {
 			return err
 		}
-	defer syscall.Close(s)
+	}
-
+	return nil
-	ir := ifreq{Name: t.deviceBytes(), data: int(value)}
-	err = ioctl(uintptr(s), ctl, uintptr(unsafe.Pointer(&ir)))
-	return err
 }
 
 func (t *tun) reload(c *config.C, initial bool) error {
@@ -396,23 +197,21 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 
 func (t *tun) addRoutes(logErrors bool) error {
 	routes := *t.Routes.Load()
-
 	for _, r := range routes {
 		if len(r.Via) == 0 || !r.Install {
 			// We don't allow route MTUs so only install routes with a via
 			continue
 		}
 
-		err := addRoute(r.Cidr, t.vpnNetworks)
+		cmd := exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
-		if err != nil {
+		t.l.Debug("command: ", cmd.String())
-			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
+		if err := cmd.Run(); err != nil {
+			retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
 			} else {
 				return retErr
 			}
-		} else {
-			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -425,8 +224,10 @@ func (t *tun) removeRoutes(routes []Route) error {
 			continue
 		}
 
-		err := delRoute(r.Cidr, t.vpnNetworks)
+		//TODO: CERT-V2 is this right?
-		if err != nil {
+		cmd := exec.Command("/sbin/route", "-n", "delete", "-net", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
+		t.l.Debug("command: ", cmd.String())
+		if err := cmd.Run(); err != nil {
 			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
 			t.l.WithField("route", r).Info("Removed route")
@@ -441,109 +242,3 @@ func (t *tun) deviceBytes() (o [16]byte) {
 	}
 	return
 }
-
-func addRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
-	if err != nil {
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
-
-	route := &netroute.RouteMessage{
-		Version: unix.RTM_VERSION,
-		Type:    unix.RTM_ADD,
-		Flags:   unix.RTF_UP | unix.RTF_GATEWAY,
-		Seq:     1,
-	}
-
-	if prefix.Addr().Is4() {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()},
-		}
-	} else {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()},
-		}
-	}
-
-	data, err := route.Marshal()
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
-
-	_, err = unix.Write(sock, data[:])
-	if err != nil {
-		if errors.Is(err, unix.EEXIST) {
-			// Try to do a change
-			route.Type = unix.RTM_CHANGE
-			data, err = route.Marshal()
-			if err != nil {
-				return fmt.Errorf("failed to create route.RouteMessage for change: %w", err)
-			}
-			_, err = unix.Write(sock, data[:])
-			return err
-		}
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
-}
-
-func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
-	if err != nil {
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
-
-	route := netroute.RouteMessage{
-		Version: unix.RTM_VERSION,
-		Type:    unix.RTM_DELETE,
-		Seq:     1,
-	}
-
-	if prefix.Addr().Is4() {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()},
-		}
-	} else {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()},
-		}
-	}
-
-	data, err := route.Marshal()
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
-	_, err = unix.Write(sock, data[:])
-	if err != nil {
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
-}

overlay/tun_openbsd.go

@@ -4,50 +4,23 @@
 package overlay
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"net/netip"
 	"os"
+	"os/exec"
 	"regexp"
+	"strconv"
 	"sync/atomic"
 	"syscall"
-	"unsafe"
 
 	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	netroute "golang.org/x/net/route"
-	"golang.org/x/sys/unix"
 )
 
-const (
-	SIOCAIFADDR_IN6 = 0x8080691a
-)
-
-type ifreqAlias4 struct {
-	Name     [unix.IFNAMSIZ]byte
-	Addr     unix.RawSockaddrInet4
-	DstAddr  unix.RawSockaddrInet4
-	MaskAddr unix.RawSockaddrInet4
-}
-
-type ifreqAlias6 struct {
-	Name       [unix.IFNAMSIZ]byte
-	Addr       unix.RawSockaddrInet6
-	DstAddr    unix.RawSockaddrInet6
-	PrefixMask unix.RawSockaddrInet6
-	Flags      uint32
-	Lifetime   [2]uint32
-}
-
-type ifreq struct {
-	Name [unix.IFNAMSIZ]byte
-	data int
-}
-
 type tun struct {
 	Device      string
 	vpnNetworks []netip.Prefix
@@ -55,42 +28,44 @@ type tun struct {
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	l           *logrus.Logger
-	f           *os.File
+
-	fd          int
+	io.ReadWriteCloser
+
 	// cache out buffer since we need to prepend 4 bytes for tun metadata
 	out []byte
 }
 
-var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+func (t *tun) Close() error {
+	if t.ReadWriteCloser != nil {
+		return t.ReadWriteCloser.Close()
+	}
+
+	return nil
+}
 
 func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
-	return nil, fmt.Errorf("newTunFromFd not supported in openbsd")
+	return nil, fmt.Errorf("newTunFromFd not supported in OpenBSD")
 }
 
+var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+
 func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
-	// Try to open tun device
-	var err error
 	deviceName := c.GetString("tun.dev", "")
 	if deviceName == "" {
-		return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
+		return nil, fmt.Errorf("a device name in the format of tunN must be specified")
-	}
-	if !deviceNameRE.MatchString(deviceName) {
-		return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
 	}
 
-	fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
+	if !deviceNameRE.MatchString(deviceName) {
+		return nil, fmt.Errorf("a device name in the format of tunN must be specified")
+	}
+
+	file, err := os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
 	}
 
-	err = unix.SetNonblock(fd, true)
-	if err != nil {
-		l.WithError(err).Warn("Failed to set the tun device as nonblocking")
-	}
-
 	t := &tun{
-		f:           os.NewFile(uintptr(fd), ""),
+		ReadWriteCloser: file,
-		fd:          fd,
 		Device:          deviceName,
 		vpnNetworks:     vpnNetworks,
 		MTU:             c.GetInt("tun.mtu", DefaultMTU),
@@ -112,154 +87,6 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	return t, nil
 }
 
-func (t *tun) Close() error {
-	if t.f != nil {
-		if err := t.f.Close(); err != nil {
-			return fmt.Errorf("error closing tun file: %w", err)
-		}
-
-		// t.f.Close should have handled it for us but let's be extra sure
-		_ = unix.Close(t.fd)
-	}
-	return nil
-}
-
-func (t *tun) Read(to []byte) (int, error) {
-	buf := make([]byte, len(to)+4)
-
-	n, err := t.f.Read(buf)
-
-	copy(to, buf[4:])
-	return n - 4, err
-}
-
-// Write is only valid for single threaded use
-func (t *tun) Write(from []byte) (int, error) {
-	buf := t.out
-	if cap(buf) < len(from)+4 {
-		buf = make([]byte, len(from)+4)
-		t.out = buf
-	}
-	buf = buf[:len(from)+4]
-
-	if len(from) == 0 {
-		return 0, syscall.EIO
-	}
-
-	// Determine the IP Family for the NULL L2 Header
-	ipVer := from[0] >> 4
-	if ipVer == 4 {
-		buf[3] = syscall.AF_INET
-	} else if ipVer == 6 {
-		buf[3] = syscall.AF_INET6
-	} else {
-		return 0, fmt.Errorf("unable to determine IP version from packet")
-	}
-
-	copy(buf[4:], from)
-
-	n, err := t.f.Write(buf)
-	return n - 4, err
-}
-
-func (t *tun) addIp(cidr netip.Prefix) error {
-	if cidr.Addr().Is4() {
-		var req ifreqAlias4
-		req.Name = t.deviceBytes()
-		req.Addr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   cidr.Addr().As4(),
-		}
-		req.DstAddr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   cidr.Addr().As4(),
-		}
-		req.MaskAddr = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Addr:   prefixToMask(cidr).As4(),
-		}
-
-		s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-
-		if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&req))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr(), err)
-		}
-
-		err = addRoute(cidr, t.vpnNetworks)
-		if err != nil {
-			return fmt.Errorf("failed to set route for vpn network %v: %w", cidr, err)
-		}
-
-		return nil
-	}
-
-	if cidr.Addr().Is6() {
-		var req ifreqAlias6
-		req.Name = t.deviceBytes()
-		req.Addr = unix.RawSockaddrInet6{
-			Len:    unix.SizeofSockaddrInet6,
-			Family: unix.AF_INET6,
-			Addr:   cidr.Addr().As16(),
-		}
-		req.PrefixMask = unix.RawSockaddrInet6{
-			Len:    unix.SizeofSockaddrInet6,
-			Family: unix.AF_INET6,
-			Addr:   prefixToMask(cidr).As16(),
-		}
-		req.Lifetime[0] = 0xffffffff
-		req.Lifetime[1] = 0xffffffff
-
-		s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-		if err != nil {
-			return err
-		}
-		defer syscall.Close(s)
-
-		if err := ioctl(uintptr(s), SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&req))); err != nil {
-			return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err)
-		}
-
-		return nil
-	}
-
-	return fmt.Errorf("unknown address type %v", cidr)
-}
-
-func (t *tun) Activate() error {
-	err := t.doIoctlByName(unix.SIOCSIFMTU, uint32(t.MTU))
-	if err != nil {
-		return fmt.Errorf("failed to set tun mtu: %w", err)
-	}
-
-	for i := range t.vpnNetworks {
-		err = t.addIp(t.vpnNetworks[i])
-		if err != nil {
-			return err
-		}
-	}
-
-	return t.addRoutes(false)
-}
-
-func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
-	s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP)
-	if err != nil {
-		return err
-	}
-	defer syscall.Close(s)
-
-	ir := ifreq{Name: t.deviceBytes(), data: int(value)}
-	err = ioctl(uintptr(s), ctl, uintptr(unsafe.Pointer(&ir)))
-	return err
-}
-
 func (t *tun) reload(c *config.C, initial bool) error {
 	change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
 	if err != nil {
@@ -297,42 +124,63 @@ func (t *tun) reload(c *config.C, initial bool) error {
 	return nil
 }
 
+func (t *tun) addIp(cidr netip.Prefix) error {
+	var err error
+	// TODO use syscalls instead of exec.Command
+	cmd := exec.Command("/sbin/ifconfig", t.Device, cidr.String(), cidr.Addr().String())
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
+	}
+
+	cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU))
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
+	}
+
+	cmd = exec.Command("/sbin/route", "-n", "add", "-inet", cidr.String(), cidr.Addr().String())
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run 'route add': %s", err)
+	}
+
+	// Unsafe path routes
+	return t.addRoutes(false)
+}
+
+func (t *tun) Activate() error {
+	for i := range t.vpnNetworks {
+		err := t.addIp(t.vpnNetworks[i])
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	r, _ := t.routeTree.Load().Lookup(ip)
 	return r
 }
 
-func (t *tun) Networks() []netip.Prefix {
-	return t.vpnNetworks
-}
-
-func (t *tun) Name() string {
-	return t.Device
-}
-
-func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd")
-}
-
 func (t *tun) addRoutes(logErrors bool) error {
 	routes := *t.Routes.Load()
-
 	for _, r := range routes {
 		if len(r.Via) == 0 || !r.Install {
 			// We don't allow route MTUs so only install routes with a via
 			continue
 		}
-
+		//TODO: CERT-V2 is this right?
-		err := addRoute(r.Cidr, t.vpnNetworks)
+		cmd := exec.Command("/sbin/route", "-n", "add", "-inet", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
-		if err != nil {
+		t.l.Debug("command: ", cmd.String())
-			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
+		if err := cmd.Run(); err != nil {
+			retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]any{"route": r}, err)
 			if logErrors {
 				retErr.Log(t.l)
 			} else {
 				return retErr
 			}
-		} else {
-			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -344,9 +192,10 @@ func (t *tun) removeRoutes(routes []Route) error {
 		if !r.Install {
 			continue
 		}
-
+		//TODO: CERT-V2 is this right?
-		err := delRoute(r.Cidr, t.vpnNetworks)
+		cmd := exec.Command("/sbin/route", "-n", "delete", "-inet", r.Cidr.String(), t.vpnNetworks[0].Addr().String())
-		if err != nil {
+		t.l.Debug("command: ", cmd.String())
+		if err := cmd.Run(); err != nil {
 			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
 			t.l.WithField("route", r).Info("Removed route")
@@ -355,115 +204,52 @@ func (t *tun) removeRoutes(routes []Route) error {
 	return nil
 }
 
-func (t *tun) deviceBytes() (o [16]byte) {
+func (t *tun) Networks() []netip.Prefix {
-	for i, c := range t.Device {
+	return t.vpnNetworks
-		o[i] = byte(c)
-	}
-	return
 }
 
-func addRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
+func (t *tun) Name() string {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	return t.Device
-	if err != nil {
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
-
-	route := &netroute.RouteMessage{
-		Version: unix.RTM_VERSION,
-		Type:    unix.RTM_ADD,
-		Flags:   unix.RTF_UP | unix.RTF_GATEWAY,
-		Seq:     1,
 }
 
-	if prefix.Addr().Is4() {
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-		gw, err := selectGateway(prefix, gateways)
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
-		if err != nil {
-			return err
 }
-		route.Addrs = []netroute.Addr{
+
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
+func (t *tun) Read(to []byte) (int, error) {
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
+	buf := make([]byte, len(to)+4)
-			unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()},
+
+	n, err := t.ReadWriteCloser.Read(buf)
+
+	copy(to, buf[4:])
+	return n - 4, err
 }
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	buf := t.out
+	if cap(buf) < len(from)+4 {
+		buf = make([]byte, len(from)+4)
+		t.out = buf
+	}
+	buf = buf[:len(from)+4]
+
+	if len(from) == 0 {
+		return 0, syscall.EIO
+	}
+
+	// Determine the IP Family for the NULL L2 Header
+	ipVer := from[0] >> 4
+	if ipVer == 4 {
+		buf[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		buf[3] = syscall.AF_INET6
 	} else {
-		gw, err := selectGateway(prefix, gateways)
+		return 0, fmt.Errorf("unable to determine IP version from packet")
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()},
-		}
 	}
 
-	data, err := route.Marshal()
+	copy(buf[4:], from)
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
 
-	_, err = unix.Write(sock, data[:])
+	n, err := t.ReadWriteCloser.Write(buf)
-	if err != nil {
+	return n - 4, err
-		if errors.Is(err, unix.EEXIST) {
-			// Try to do a change
-			route.Type = unix.RTM_CHANGE
-			data, err = route.Marshal()
-			if err != nil {
-				return fmt.Errorf("failed to create route.RouteMessage for change: %w", err)
-			}
-			_, err = unix.Write(sock, data[:])
-			return err
-		}
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
-}
-
-func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
-	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
-	if err != nil {
-		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
-	}
-	defer unix.Close(sock)
-
-	route := netroute.RouteMessage{
-		Version: unix.RTM_VERSION,
-		Type:    unix.RTM_DELETE,
-		Seq:     1,
-	}
-
-	if prefix.Addr().Is4() {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()},
-			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()},
-			unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()},
-		}
-	} else {
-		gw, err := selectGateway(prefix, gateways)
-		if err != nil {
-			return err
-		}
-		route.Addrs = []netroute.Addr{
-			unix.RTAX_DST:     &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()},
-			unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()},
-			unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()},
-		}
-	}
-
-	data, err := route.Marshal()
-	if err != nil {
-		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
-	}
-	_, err = unix.Write(sock, data[:])
-	if err != nil {
-		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-	}
-
-	return nil
 }

pki.go

@@ -100,41 +100,36 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
 		currentState := p.cs.Load()
 		if newState.v1Cert != nil {
 			if currentState.v1Cert == nil {
-				return util.NewContextualError("v1 certificate was added, restart required", nil, err)
+				//adding certs is fine, actually. Networks-in-common confirmed in newCertState().
-			}
+			} else {
-
 				// did IP in cert change? if so, don't set
 				if !slices.Equal(currentState.v1Cert.Networks(), newState.v1Cert.Networks()) {
 					return util.NewContextualError(
 						"Networks in new cert was different from old",
-					m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks()},
+						m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks(), "cert_version": cert.Version1},
 						nil,
 					)
 				}
 
 				if currentState.v1Cert.Curve() != newState.v1Cert.Curve() {
 					return util.NewContextualError(
-					"Curve in new cert was different from old",
+						"Curve in new v1 cert was different from old",
-					m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve()},
+						m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve(), "cert_version": cert.Version1},
 						nil,
 					)
 				}
-
+			}
-		} else if currentState.v1Cert != nil {
-			//TODO: CERT-V2 we should be able to tear this down
-			return util.NewContextualError("v1 certificate was removed, restart required", nil, err)
 		}
 
 		if newState.v2Cert != nil {
 			if currentState.v2Cert == nil {
-				return util.NewContextualError("v2 certificate was added, restart required", nil, err)
+				//adding certs is fine, actually
-			}
+			} else {
-
 				// did IP in cert change? if so, don't set
 				if !slices.Equal(currentState.v2Cert.Networks(), newState.v2Cert.Networks()) {
 					return util.NewContextualError(
 						"Networks in new cert was different from old",
-					m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks()},
+						m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks(), "cert_version": cert.Version2},
 						nil,
 					)
 				}
@@ -142,13 +137,25 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
 				if currentState.v2Cert.Curve() != newState.v2Cert.Curve() {
 					return util.NewContextualError(
 						"Curve in new cert was different from old",
-					m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve()},
+						m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve(), "cert_version": cert.Version2},
 						nil,
 					)
 				}
+			}
 
 		} else if currentState.v2Cert != nil {
-			return util.NewContextualError("v2 certificate was removed, restart required", nil, err)
+			//newState.v1Cert is non-nil bc empty certstates aren't permitted
+			if newState.v1Cert == nil {
+				return util.NewContextualError("v1 and v2 certs are nil, this should be impossible", nil, err)
+			}
+			//if we're going to v1-only, we need to make sure we didn't orphan any v2-cert vpnaddrs
+			if !slices.Equal(currentState.v2Cert.Networks(), newState.v1Cert.Networks()) {
+				return util.NewContextualError(
+					"Removing a V2 cert is not permitted unless it has identical networks to the new V1 cert",
+					m{"new_v1_networks": newState.v1Cert.Networks(), "old_v2_networks": currentState.v2Cert.Networks()},
+					nil,
+				)
+			}
 		}
 
 		// Cipher cant be hot swapped so just leave it at what it was before

relay_manager.go

@@ -190,6 +190,7 @@ func (rm *relayManager) handleCreateRelayResponse(v cert.Version, h *HostInfo, f
 			InitiatorRelayIndex: peerRelay.RemoteIndex,
 		}
 
+		relayFrom := h.vpnAddrs[0]
 		if v == cert.Version1 {
 			peer := peerHostInfo.vpnAddrs[0]
 			if !peer.Is4() {
@@ -207,7 +208,13 @@ func (rm *relayManager) handleCreateRelayResponse(v cert.Version, h *HostInfo, f
 			b = targetAddr.As4()
 			resp.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 		} else {
-			resp.RelayFromAddr = netAddrToProtoAddr(peerHostInfo.vpnAddrs[0])
+			if targetAddr.Is4() {
+				relayFrom = h.vpnAddrs[0]
+			} else {
+				//todo do this smarter
+				relayFrom = h.vpnAddrs[len(h.vpnAddrs)-1]
+			}
+			resp.RelayFromAddr = netAddrToProtoAddr(relayFrom)
 			resp.RelayToAddr = target
 		}
 
@@ -360,7 +367,7 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 			Type:                NebulaControl_CreateRelayRequest,
 			InitiatorRelayIndex: index,
 		}
-
+		relayFrom := h.vpnAddrs[0]
 		if v == cert.Version1 {
 			if !h.vpnAddrs[0].Is4() {
 				rm.l.WithField("relayFrom", h.vpnAddrs[0]).
@@ -377,7 +384,13 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 			b = target.As4()
 			req.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 		} else {
-			req.RelayFromAddr = netAddrToProtoAddr(h.vpnAddrs[0])
+			if target.Is4() {
+				relayFrom = h.vpnAddrs[0]
+			} else {
+				//todo do this smarter
+				relayFrom = h.vpnAddrs[len(h.vpnAddrs)-1]
+			}
+			req.RelayFromAddr = netAddrToProtoAddr(relayFrom)
 			req.RelayToAddr = netAddrToProtoAddr(target)
 		}
 
@@ -388,7 +401,7 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 		} else {
 			f.SendMessageToHostInfo(header.Control, 0, peer, msg, make([]byte, 12), make([]byte, mtu))
 			rm.l.WithFields(logrus.Fields{
-				"relayFrom":           h.vpnAddrs[0],
+				"relayFrom":           relayFrom,
 				"relayTo":             target,
 				"initiatorRelayIndex": req.InitiatorRelayIndex,
 				"responderRelayIndex": req.ResponderRelayIndex,