weird line breaks

e2e/helpers_test.go

@@ -29,8 +29,6 @@ type m = map[string]any
 
 // newSimpleServer creates a nebula instance with many assumptions
 func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
-	l := NewTestLogger()
-
 	var vpnNetworks []netip.Prefix
 	for _, sn := range strings.Split(sVpnNetworks, ",") {
 		vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn))
@@ -56,6 +54,25 @@ func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name
 		budpIp[3] = 239
 		udpAddr = netip.AddrPortFrom(netip.AddrFrom16(budpIp), 4242)
 	}
+	return newSimpleServerWithUdp(v, caCrt, caKey, name, sVpnNetworks, udpAddr, overrides)
+}
+
+func newSimpleServerWithUdp(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, udpAddr netip.AddrPort, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
+	l := NewTestLogger()
+
+	var vpnNetworks []netip.Prefix
+	for _, sn := range strings.Split(sVpnNetworks, ",") {
+		vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn))
+		if err != nil {
+			panic(err)
+		}
+		vpnNetworks = append(vpnNetworks, vpnIpNet)
+	}
+
+	if len(vpnNetworks) == 0 {
+		panic("no vpn networks")
+	}
+
 	_, _, myPrivKey, myPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, caCrt, caKey, name, time.Now(), time.Now().Add(5*time.Minute), vpnNetworks, nil, []string{})
 
 	caB, err := caCrt.MarshalPEM()

e2e/tunnels_test.go

@@ -4,6 +4,7 @@
 package e2e
 
 import (
+	"net/netip"
 	"testing"
 	"time"
 
@@ -55,3 +56,50 @@ func TestDropInactiveTunnels(t *testing.T) {
 	myControl.Stop()
 	theirControl.Stop()
 }
+
+func TestCrossStackRelaysWork(t *testing.T) {
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24,fc00::1/64", m{"relay": m{"use_relays": true}})
+	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay  ", "10.128.0.128/24,fc00::128/64", m{"relay": m{"am_relay": true}})
+	theirUdp := netip.MustParseAddrPort("10.0.0.2:4242")
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, "them   ", "fc00::2/64", theirUdp, m{"relay": m{"use_relays": true}})
+
+	//myVpnV4 := myVpnIpNet[0]
+	myVpnV6 := myVpnIpNet[1]
+	relayVpnV4 := relayVpnIpNet[0]
+	relayVpnV6 := relayVpnIpNet[1]
+	theirVpnV6 := theirVpnIpNet[0]
+
+	// Teach my how to get to the relay and that their can be reached via the relay
+	myControl.InjectLightHouseAddr(relayVpnV4.Addr(), relayUdpAddr)
+	myControl.InjectLightHouseAddr(relayVpnV6.Addr(), relayUdpAddr)
+	myControl.InjectRelays(theirVpnV6.Addr(), []netip.Addr{relayVpnV6.Addr()})
+	relayControl.InjectLightHouseAddr(theirVpnV6.Addr(), theirUdpAddr)
+
+	// Build a router so we don't have to reason who gets which packet
+	r := router.NewR(t, myControl, relayControl, theirControl)
+	defer r.RenderFlow()
+
+	// Start the servers
+	myControl.Start()
+	relayControl.Start()
+	theirControl.Start()
+
+	t.Log("Trigger a handshake from me to them via the relay")
+	myControl.InjectTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me"))
+
+	p := r.RouteForAllUntilTxTun(theirControl)
+	r.Log("Assert the tunnel works")
+	assertUdpPacket(t, []byte("Hi from me"), p, myVpnV6.Addr(), theirVpnV6.Addr(), 80, 80)
+
+	t.Log("reply?")
+	theirControl.InjectTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them"))
+	p = r.RouteForAllUntilTxTun(myControl)
+	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnV6.Addr(), myVpnV6.Addr(), 80, 80)
+
+	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
+	//t.Log("finish up")
+	//myControl.Stop()
+	//theirControl.Stop()
+	//relayControl.Stop()
+}

handshake_manager.go

@@ -292,6 +292,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 					idx, err := AddRelay(hm.l, relayHostInfo, hm.mainHostMap, vpnIp, nil, TerminalType, Requested)
 					if err != nil {
 						hostinfo.logger(hm.l).WithField("relay", relay.String()).WithError(err).Info("Failed to add relay to hostmap")
+						continue
 					}
 
 					m := NebulaControl{
@@ -301,37 +302,25 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 
 					switch relayHostInfo.GetCert().Certificate.Version() {
 					case cert.Version1:
-						if !hm.f.myVpnAddrs[0].Is4() {
+						err = buildRelayInfoCertV1(&m, hm.f.myVpnNetworks, vpnIp)
-							hostinfo.logger(hm.l).Error("can not establish v1 relay with a v6 network because the relay is not running a current nebula version")
-							continue
-						}
-
-						if !vpnIp.Is4() {
-							hostinfo.logger(hm.l).Error("can not establish v1 relay with a v6 remote network because the relay is not running a current nebula version")
-							continue
-						}
-
-						b := hm.f.myVpnAddrs[0].As4()
-						m.OldRelayFromAddr = binary.BigEndian.Uint32(b[:])
-						b = vpnIp.As4()
-						m.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 					case cert.Version2:
-						m.RelayFromAddr = netAddrToProtoAddr(hm.f.myVpnAddrs[0])
+						err = buildRelayInfoCertV2(&m, hm.f.myVpnNetworks, vpnIp)
-						m.RelayToAddr = netAddrToProtoAddr(vpnIp)
 					default:
-						hostinfo.logger(hm.l).Error("Unknown certificate version found while creating relay")
+						err = errors.New("unknown certificate version found while creating relay")
+					}
+					if err != nil {
+						hostinfo.logger(hm.l).WithError(err).Error("Refusing to relay")
 						continue
 					}
 
 					msg, err := m.Marshal()
 					if err != nil {
-						hostinfo.logger(hm.l).
+						hostinfo.logger(hm.l).WithError(err).
-							WithError(err).
 							Error("Failed to marshal Control message to create relay")
 					} else {
 						hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
 						hm.l.WithFields(logrus.Fields{
-							"relayFrom":           hm.f.myVpnAddrs[0],
+							"relayFrom":           m.GetRelayFrom(),
 							"relayTo":             vpnIp,
 							"initiatorRelayIndex": idx,
 							"relay":               relay}).
@@ -357,39 +346,27 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 					InitiatorRelayIndex: existingRelay.LocalIndex,
 				}
 
+				var err error
 				switch relayHostInfo.GetCert().Certificate.Version() {
 				case cert.Version1:
-					if !hm.f.myVpnAddrs[0].Is4() {
+					err = buildRelayInfoCertV1(&m, hm.f.myVpnNetworks, vpnIp)
-						hostinfo.logger(hm.l).Error("can not establish v1 relay with a v6 network because the relay is not running a current nebula version")
-						continue
-					}
-
-					if !vpnIp.Is4() {
-						hostinfo.logger(hm.l).Error("can not establish v1 relay with a v6 remote network because the relay is not running a current nebula version")
-						continue
-					}
-
-					b := hm.f.myVpnAddrs[0].As4()
-					m.OldRelayFromAddr = binary.BigEndian.Uint32(b[:])
-					b = vpnIp.As4()
-					m.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 				case cert.Version2:
-					m.RelayFromAddr = netAddrToProtoAddr(hm.f.myVpnAddrs[0])
+					err = buildRelayInfoCertV2(&m, hm.f.myVpnNetworks, vpnIp)
-					m.RelayToAddr = netAddrToProtoAddr(vpnIp)
 				default:
-					hostinfo.logger(hm.l).Error("Unknown certificate version found while creating relay")
+					err = errors.New("unknown certificate version found while creating relay")
+				}
+				if err != nil {
+					hostinfo.logger(hm.l).WithError(err).Error("Refusing to relay")
 					continue
 				}
 				msg, err := m.Marshal()
 				if err != nil {
-					hostinfo.logger(hm.l).
+					hostinfo.logger(hm.l).WithError(err).Error("Failed to marshal Control message to create relay")
-						WithError(err).
-						Error("Failed to marshal Control message to create relay")
 				} else {
 					// This must send over the hostinfo, not over hm.Hosts[ip]
 					hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
 					hm.l.WithFields(logrus.Fields{
-						"relayFrom":           hm.f.myVpnAddrs[0],
+						"relayFrom":           m.GetRelayFrom(),
 						"relayTo":             vpnIp,
 						"initiatorRelayIndex": existingRelay.LocalIndex,
 						"relay":               relay}).
@@ -724,3 +701,32 @@ func generateIndex(l *logrus.Logger) (uint32, error) {
 func hsTimeout(tries int64, interval time.Duration) time.Duration {
 	return time.Duration(tries / 2 * ((2 * int64(interval)) + (tries-1)*int64(interval)))
 }
+
+var errNoRelayTooOld = errors.New("can not establish v1 relay with a v6 network because the relay is not running a current nebula version")
+
+func buildRelayInfoCertV1(m *NebulaControl, myVpnNetworks []netip.Prefix, peerVpnIp netip.Addr) error {
+	relayFrom := myVpnNetworks[0].Addr()
+	if !relayFrom.Is4() {
+		return errNoRelayTooOld
+	}
+	if !peerVpnIp.Is4() {
+		return errNoRelayTooOld
+	}
+
+	b := relayFrom.As4()
+	m.OldRelayFromAddr = binary.BigEndian.Uint32(b[:])
+	b = peerVpnIp.As4()
+	m.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
+	return nil
+}
+
+func buildRelayInfoCertV2(m *NebulaControl, myVpnNetworks []netip.Prefix, peerVpnIp netip.Addr) error {
+	for i := range myVpnNetworks {
+		if myVpnNetworks[i].Contains(peerVpnIp) {
+			m.RelayFromAddr = netAddrToProtoAddr(myVpnNetworks[i].Addr())
+			m.RelayToAddr = netAddrToProtoAddr(peerVpnIp)
+			return nil
+		}
+	}
+	return errors.New("cannot establish relay, no networks in common")
+}

hostmap.go

@@ -512,13 +512,16 @@ func (hm *HostMap) QueryVpnAddr(vpnIp netip.Addr) *HostInfo {
 	return hm.queryVpnAddr(vpnIp, nil)
 }
 
+var errUnableToFindHost = errors.New("unable to find host")
+var errUnableToFindHostWithRelay = errors.New("unable to find host with relay")
+
 func (hm *HostMap) QueryVpnAddrsRelayFor(targetIps []netip.Addr, relayHostIp netip.Addr) (*HostInfo, *Relay, error) {
 	hm.RLock()
 	defer hm.RUnlock()
 
 	h, ok := hm.Hosts[relayHostIp]
 	if !ok {
-		return nil, nil, errors.New("unable to find host")
+		return nil, nil, errUnableToFindHost
 	}
 
 	for h != nil {
@@ -531,7 +534,7 @@ func (hm *HostMap) QueryVpnAddrsRelayFor(targetIps []netip.Addr, relayHostIp net
 		h = h.next
 	}
 
-	return nil, nil, errors.New("unable to find host with relay")
+	return nil, nil, errUnableToFindHostWithRelay
 }
 
 func (hm *HostMap) unlockedDisestablishVpnAddrRelayFor(hi *HostInfo) {

lighthouse.go

@@ -1425,7 +1425,7 @@ func (d *NebulaMetaDetails) GetRelays() []netip.Addr {
 	return relays
 }
 
-// FindNetworkUnion returns the first netip.Addr contained in the list of provided netip.Prefix, if able
+// findNetworkUnion returns the first netip.Addr of addrs contained in the list of provided netip.Prefix, if able
 func findNetworkUnion(prefixes []netip.Prefix, addrs []netip.Addr) (netip.Addr, bool) {
 	for i := range prefixes {
 		for j := range addrs {
@@ -1450,3 +1450,13 @@ func (d *NebulaMetaDetails) GetVpnAddrAndVersion() (netip.Addr, cert.Version, er
 		return netip.Addr{}, cert.Version1, ErrBadDetailsVpnAddr
 	}
 }
+
+func (d *NebulaControl) GetRelayFrom() netip.Addr {
+	if d.OldRelayFromAddr != 0 {
+		b := [4]byte{}
+		binary.BigEndian.PutUint32(b[:], d.OldRelayFromAddr)
+		return netip.AddrFrom4(b)
+	} else {
+		return protoAddrToNetAddr(d.RelayFromAddr)
+	}
+}

relay_manager.go

@@ -155,6 +155,8 @@ func (rm *relayManager) handleCreateRelayResponse(v cert.Version, h *HostInfo, f
 		"vpnAddrs":            h.vpnAddrs}).
 		Info("handleCreateRelayResponse")
 
+	//peer == relayFrom
+	//target == relayTo
 	target := m.RelayToAddr
 	targetAddr := protoAddrToNetAddr(target)
 
@@ -190,11 +192,12 @@ func (rm *relayManager) handleCreateRelayResponse(v cert.Version, h *HostInfo, f
 			InitiatorRelayIndex: peerRelay.RemoteIndex,
 		}
 
+		relayFrom := h.vpnAddrs[0]
 		if v == cert.Version1 {
 			peer := peerHostInfo.vpnAddrs[0]
 			if !peer.Is4() {
 				rm.l.WithField("relayFrom", peer).
-					WithField("relayTo", target).
+					WithField("relayTo", targetAddr).
 					WithField("initiatorRelayIndex", resp.InitiatorRelayIndex).
 					WithField("responderRelayIndex", resp.ResponderRelayIndex).
 					WithField("vpnAddrs", peerHostInfo.vpnAddrs).
@@ -207,7 +210,22 @@ func (rm *relayManager) handleCreateRelayResponse(v cert.Version, h *HostInfo, f
 			b = targetAddr.As4()
 			resp.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 		} else {
-			resp.RelayFromAddr = netAddrToProtoAddr(peerHostInfo.vpnAddrs[0])
+			ok = false
+			peerNetworks := h.GetCert().Certificate.Networks()
+			for i := range peerNetworks {
+				if peerNetworks[i].Contains(targetAddr) {
+					relayFrom = peerNetworks[i].Addr()
+					ok = true
+					break
+				}
+			}
+			if !ok {
+				rm.l.WithFields(logrus.Fields{"from": f.myVpnNetworks, "to": targetAddr}).
+					Error("cannot establish relay, no networks in common")
+				return
+			}
+
+			resp.RelayFromAddr = netAddrToProtoAddr(relayFrom)
 			resp.RelayToAddr = target
 		}
 
@@ -218,8 +236,8 @@ func (rm *relayManager) handleCreateRelayResponse(v cert.Version, h *HostInfo, f
 		} else {
 			f.SendMessageToHostInfo(header.Control, 0, peerHostInfo, msg, make([]byte, 12), make([]byte, mtu))
 			rm.l.WithFields(logrus.Fields{
-				"relayFrom":           resp.RelayFromAddr,
+				"relayFrom":           relayFrom,
-				"relayTo":             resp.RelayToAddr,
+				"relayTo":             targetAddr,
 				"initiatorRelayIndex": resp.InitiatorRelayIndex,
 				"responderRelayIndex": resp.ResponderRelayIndex,
 				"vpnAddrs":            peerHostInfo.vpnAddrs}).
@@ -313,8 +331,7 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 
 		msg, err := resp.Marshal()
 		if err != nil {
-			logMsg.
+			logMsg.WithError(err).Error("relayManager Failed to marshal Control CreateRelayResponse message to create relay")
-				WithError(err).Error("relayManager Failed to marshal Control CreateRelayResponse message to create relay")
 		} else {
 			f.SendMessageToHostInfo(header.Control, 0, h, msg, make([]byte, 12), make([]byte, mtu))
 			rm.l.WithFields(logrus.Fields{
@@ -360,10 +377,10 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 			Type:                NebulaControl_CreateRelayRequest,
 			InitiatorRelayIndex: index,
 		}
-
+		relayFrom := h.vpnAddrs[0]
 		if v == cert.Version1 {
-			if !h.vpnAddrs[0].Is4() {
+			if !relayFrom.Is4() {
-				rm.l.WithField("relayFrom", h.vpnAddrs[0]).
+				rm.l.WithField("relayFrom", relayFrom).
 					WithField("relayTo", target).
 					WithField("initiatorRelayIndex", req.InitiatorRelayIndex).
 					WithField("responderRelayIndex", req.ResponderRelayIndex).
@@ -372,23 +389,37 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 				return
 			}
 
-			b := h.vpnAddrs[0].As4()
+			b := relayFrom.As4()
 			req.OldRelayFromAddr = binary.BigEndian.Uint32(b[:])
 			b = target.As4()
 			req.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
 		} else {
-			req.RelayFromAddr = netAddrToProtoAddr(h.vpnAddrs[0])
+			ok = false
+			peerNetworks := h.GetCert().Certificate.Networks()
+			for i := range peerNetworks {
+				if peerNetworks[i].Contains(target) {
+					relayFrom = peerNetworks[i].Addr()
+					ok = true
+					break
+				}
+			}
+			if !ok {
+				rm.l.WithFields(logrus.Fields{"from": f.myVpnNetworks, "to": target}).
+					Error("cannot establish relay, no networks in common")
+				return
+			}
+
+			req.RelayFromAddr = netAddrToProtoAddr(relayFrom)
 			req.RelayToAddr = netAddrToProtoAddr(target)
 		}
 
 		msg, err := req.Marshal()
 		if err != nil {
-			logMsg.
+			logMsg.WithError(err).Error("relayManager Failed to marshal Control message to create relay")
-				WithError(err).Error("relayManager Failed to marshal Control message to create relay")
 		} else {
 			f.SendMessageToHostInfo(header.Control, 0, peer, msg, make([]byte, 12), make([]byte, mtu))
 			rm.l.WithFields(logrus.Fields{
-				"relayFrom":           h.vpnAddrs[0],
+				"relayFrom":           relayFrom,
 				"relayTo":             target,
 				"initiatorRelayIndex": req.InitiatorRelayIndex,
 				"responderRelayIndex": req.ResponderRelayIndex,
@@ -401,8 +432,7 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 		if !ok {
 			_, err := AddRelay(rm.l, h, f.hostMap, target, &m.InitiatorRelayIndex, ForwardingType, PeerRequested)
 			if err != nil {
-				logMsg.
+				logMsg.WithError(err).Error("relayManager Failed to allocate a local index for relay")
-					WithError(err).Error("relayManager Failed to allocate a local index for relay")
 				return
 			}
 		}