Experimenting

2026-05-16 04:47:38 +02:00 · 2026-05-11 11:51:46 -05:00
parent b7e9939e92
commit 86cef88744
33 changed files with 691 additions and 560 deletions
--- a/inside.go
+++ b/inside.go
@@ -8,12 +8,13 @@ import (
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
-	"github.com/slackhq/nebula/noiseutil"
 	"github.com/slackhq/nebula/routing"
 )

-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
-	err := newPacket(packet, false, fwPacket)
+func (f *Interface) consumeInsidePacket(buf *WireBuffer, q int, localCache firewall.ConntrackCache) {
+	packet := buf.IPPacket()
+
+	err := newPacket(packet, false, buf.FwPacket)
 	if err != nil {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			f.l.Debug("Error while validating outbound packet",
@@ -26,12 +27,12 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet

 	// Ignore local broadcast packets
 	if f.dropLocalBroadcast {
-		if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
+		if f.myBroadcastAddrsTable.Contains(buf.FwPacket.RemoteAddr) {
 			return
 		}
 	}

-	if f.myVpnAddrsTable.Contains(fwPacket.RemoteAddr) {
+	if f.myVpnAddrsTable.Contains(buf.FwPacket.RemoteAddr) {
 		// Immediately forward packets from self to self.
 		// This should only happen on Darwin-based and FreeBSD hosts, which
 		// routes packets from the Nebula addr to the Nebula addr through the Nebula
@@ -48,20 +49,20 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 	}

 	// Ignore multicast packets
-	if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() {
+	if f.dropMulticast && buf.FwPacket.RemoteAddr.IsMulticast() {
 		return
 	}

-	hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
+	hostinfo, ready := f.getOrHandshakeConsiderRouting(buf.FwPacket, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
 	})

 	if hostinfo == nil {
-		f.rejectInside(packet, out, q)
+		f.rejectInside(packet, buf.Out, q)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			f.l.Debug("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks",
-				"vpnAddr", fwPacket.RemoteAddr,
-				"fwPacket", fwPacket,
+				"vpnAddr", buf.FwPacket.RemoteAddr,
+				"fwPacket", buf.FwPacket,
 			)
 		}
 		return
@@ -71,15 +72,15 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}

-	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*buf.FwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
-		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
+		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, buf, q)

 	} else {
-		f.rejectInside(packet, out, q)
+		f.rejectInside(packet, buf.Out, q)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			hostinfo.logger(f.l).Debug("dropping outbound packet",
-				"fwPacket", fwPacket,
+				"fwPacket", buf.FwPacket,
 				"reason", dropReason,
 			)
 		}
@@ -102,27 +103,27 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 	}
 }

-func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, out []byte, q int) {
+func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, scratch []byte, buf *WireBuffer, q int) {
 	if !f.firewall.OutSendReject {
 		return
 	}

-	out = iputil.CreateRejectPacket(packet, out)
-	if len(out) == 0 {
+	rejectIP := iputil.CreateRejectPacket(packet, scratch)
+	if len(rejectIP) == 0 {
 		return
 	}

-	if len(out) > iputil.MaxRejectPacketSize {
+	if len(rejectIP) > iputil.MaxRejectPacketSize {
 		if f.l.Enabled(context.Background(), slog.LevelInfo) {
 			f.l.Info("rejectOutside: packet too big, not sending",
 				"packet", packet,
-				"outPacket", out,
+				"outPacket", rejectIP,
 			)
 		}
 		return
 	}

-	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q)
+	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, rejectIP, buf, q)
 }

 // Handshake will attempt to initiate a tunnel with the provided vpn address. This is a no-op if the tunnel is already established or being established
@@ -215,7 +216,7 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac

 }

-func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
+func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p []byte, buf *WireBuffer) {
 	fp := &firewall.Packet{}
 	err := newPacket(p, false, fp)
 	if err != nil {
@@ -235,12 +236,12 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
 		return
 	}

-	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, 0)
+	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, buf, 0)
 }

 // SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr.
 // This function ignores myVpnNetworksTable, and will always attempt to treat the address as a vpnAddr
-func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte) {
+func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p []byte, buf *WireBuffer) {
 	hostInfo, ready := f.handshakeManager.GetOrHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, t, st, p, f.SendMessageToHostInfo, f.cachedPacketMetrics)
 	})
@@ -258,113 +259,73 @@ func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.Message
 		return
 	}

-	f.SendMessageToHostInfo(t, st, hostInfo, p, nb, out)
+	f.SendMessageToHostInfo(t, st, hostInfo, p, buf)
 }

-func (f *Interface) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hi *HostInfo, p, nb, out []byte) {
-	f.send(t, st, hi.ConnectionState, hi, p, nb, out)
+func (f *Interface) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hi *HostInfo, p []byte, buf *WireBuffer) {
+	f.send(t, st, hi.ConnectionState, hi, p, buf)
 }

-func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p, nb, out []byte) {
+func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p []byte, buf *WireBuffer) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, nb, out, 0)
+	f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, buf, 0)
 }

-func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte) {
+func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p []byte, buf *WireBuffer) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0)
+	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, buf, 0)
 }

 // SendVia sends a payload through a Relay tunnel. No authentication or encryption is done
 // to the payload for the ultimate target host, making this a useful method for sending
 // handshake messages to peers through relay tunnels.
-// via is the HostInfo through which the message is relayed.
-// ad is the plaintext data to authenticate, but not encrypt
-// nb is a buffer used to store the nonce value, re-used for performance reasons.
-// out is a buffer used to store the result of the Encrypt operation
-// q indicates which writer to use to send the packet.
-func (f *Interface) SendVia(via *HostInfo,
-	relay *Relay,
-	ad,
-	nb,
-	out []byte,
-	nocopy bool,
-) {
-	if noiseutil.EncryptLockNeeded {
-		// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
-		via.ConnectionState.writeLock.Lock()
-	}
-	c := via.ConnectionState.messageCounter.Add(1)
-
-	out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
-	f.connectionManager.Out(via)
-
-	// Authenticate the header and payload, but do not encrypt for this message type.
-	// The payload consists of the inner, unencrypted Nebula header, as well as the end-to-end encrypted payload.
-	if len(out)+len(ad)+via.ConnectionState.eKey.Overhead() > cap(out) {
-		if noiseutil.EncryptLockNeeded {
-			via.ConnectionState.writeLock.Unlock()
-		}
+//
+// via is the HostInfo through which the message is relayed. ad is staged into
+// the inner-payload slot of buf and then AAD-only sealed under via's key by
+// SealRelayInPlace. The sendNoMetrics relay-forward path skips this entry
+// point and calls sendViaInPlace directly because its inner ciphertext is
+// already in place from the encrypt step.
+func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad []byte, buf *WireBuffer) {
+	if header.Len+len(ad)+via.ConnectionState.eKey.Overhead() > cap(buf.Out) {
 		via.logger(f.l).Error("SendVia out buffer not large enough for relay",
-			"outCap", cap(out),
+			"outCap", cap(buf.Out),
 			"payloadLen", len(ad),
-			"headerLen", len(out),
+			"headerLen", header.Len,
 			"cipherOverhead", via.ConnectionState.eKey.Overhead(),
 		)
 		return
 	}
+	buf.StageRelayInner(ad)
+	f.sendViaInPlace(via, relay, len(ad), buf)
+}

-	// The header bytes are written to the 'out' slice; Grow the slice to hold the header and associated data payload.
-	offset := len(out)
-	out = out[:offset+len(ad)]
-
-	// In one call path, the associated data _is_ already stored in out. In other call paths, the associated data must
-	// be copied into 'out'.
-	if !nocopy {
-		copy(out[offset:], ad)
-	}
-
-	var err error
-	out, err = via.ConnectionState.eKey.EncryptDanger(out, out, nil, c, nb)
-	if noiseutil.EncryptLockNeeded {
-		via.ConnectionState.writeLock.Unlock()
-	}
+// sendViaInPlace stamps the outer relay header, AAD-seals over the [outer
+// header | inner-already-staged] region, and writes the result to via.remote.
+// Called from SendVia (after staging ad) and from sendNoMetrics' relay-forward
+// path (where the inner ciphertext is already in place from SealForRelay).
+func (f *Interface) sendViaInPlace(via *HostInfo, relay *Relay, innerLen int, buf *WireBuffer) {
+	f.connectionManager.Out(via)
+	out, err := buf.SealRelayInPlace(via.ConnectionState, relay.RemoteIndex, innerLen)
 	if err != nil {
 		via.logger(f.l).Info("Failed to EncryptDanger in sendVia", "error", err)
 		return
 	}
-	err = f.writers[0].WriteTo(out, via.remote)
-	if err != nil {
+	if err := f.writers[0].WriteTo(out, via.remote); err != nil {
 		via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err)
 	}
 	f.connectionManager.RelayUsed(relay.LocalIndex)
 }

-func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, q int) {
+// sendNoMetrics encrypts and writes one outbound nebula packet (data, control,
+// lighthouse, etc) using buf as the per-call wire scratch. When the hostinfo
+// has no direct remote we encrypt into the relay-reserved slot via
+// SealForRelay so sendViaInPlace can wrap it without an extra copy.
+func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p []byte, buf *WireBuffer, q int) {
 	if ci.eKey == nil {
 		return
 	}
 	useRelay := !remote.IsValid() && !hostinfo.remote.IsValid()
-	fullOut := out

-	if useRelay {
-		if len(out) < header.Len {
-			// out always has a capacity of mtu, but not always a length greater than the header.Len.
-			// Grow it to make sure the next operation works.
-			out = out[:header.Len]
-		}
-		// Save a header's worth of data at the front of the 'out' buffer.
-		out = out[header.Len:]
-	}
-
-	if noiseutil.EncryptLockNeeded {
-		// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
-		ci.writeLock.Lock()
-	}
-	c := ci.messageCounter.Add(1)
-
-	//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
-	out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
 	f.connectionManager.Out(hostinfo)

 	// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against
@@ -381,50 +342,42 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 		}
 	}

+	var out []byte
 	var err error
-	out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
-	if noiseutil.EncryptLockNeeded {
-		ci.writeLock.Unlock()
+	if useRelay {
+		out, err = buf.SealForRelay(ci, t, st, hostinfo.remoteIndexId, p)
+	} else {
+		out, err = buf.Seal(ci, t, st, hostinfo.remoteIndexId, p)
 	}
 	if err != nil {
 		hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet",
 			"error", err,
 			"udpAddr", remote,
-			"counter", c,
-			"attemptedCounter", c,
 		)
 		return
 	}

-	if remote.IsValid() {
-		err = f.writers[q].WriteTo(out, remote)
-		if err != nil {
-			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
-				"error", err,
-				"udpAddr", remote,
-			)
+	switch {
+	case remote.IsValid():
+		if err := f.writers[q].WriteTo(out, remote); err != nil {
+			hostinfo.logger(f.l).Error("Failed to write outgoing packet", "error", err, "udpAddr", remote)
 		}
-	} else if hostinfo.remote.IsValid() {
-		err = f.writers[q].WriteTo(out, hostinfo.remote)
-		if err != nil {
-			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
-				"error", err,
-				"udpAddr", remote,
-			)
+	case hostinfo.remote.IsValid():
+		if err := f.writers[q].WriteTo(out, hostinfo.remote); err != nil {
+			hostinfo.logger(f.l).Error("Failed to write outgoing packet", "error", err, "udpAddr", hostinfo.remote)
 		}
-	} else {
-		// Try to send via a relay
+	default:
+		// SealForRelay placed the inner ciphertext at buf.Out[header.Len:],
+		// so sendViaInPlace can wrap it with the outer relay header without
+		// an extra copy.
 		for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
 			relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
 			if err != nil {
 				hostinfo.relayState.DeleteRelay(relayIP)
-				hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo",
-					"relay", relayIP,
-					"error", err,
-				)
+				hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo", "relay", relayIP, "error", err)
 				continue
 			}
-			f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
+			f.sendViaInPlace(relayHostInfo, relay, len(out), buf)
 			break
 		}
 	}