the definitive tun offloads branch (#1704)

2026-10-08 12:07:55 +02:00 · 2026-08-21 11:57:38 -05:00
parent b8b159a486
commit edc3c5e018
111 changed files with 12830 additions and 853 deletions
@@ -2,6 +2,8 @@ package nebula

 import (
 	"context"
+	"fmt"
+	"io"
 	"log/slog"
 	"net/netip"

@@ -9,10 +11,24 @@ import (
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
 	"github.com/slackhq/nebula/noiseutil"
+	"github.com/slackhq/nebula/overlay/batch"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )

-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
+	// borrowed: pkt.Bytes is owned by the originating tio.Queue and is
+	// only valid until the next Read on that queue. Every consumer below
+	// (parse, self-forward, handshake cache, sendInsideMessage) reads it
+	// synchronously; do not retain pkt outside this call. If a future
+	// caller needs to keep the packet, use pkt.Clone() to detach it from
+	// the borrow.
+	//
+	// pkt.Bytes is either one IP datagram (GSO zero) or a TSO/USO
+	// superpacket. In both cases the L3+L4 headers at the start describe
+	// the same 5-tuple every segment will share, so a single newPacket /
+	// firewall check covers the whole superpacket.
+	packet := pkt.Bytes
 	err := newPacket(packet, false, fwPacket)
 	if err != nil {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
@@ -37,7 +53,14 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		// routes packets from the Nebula addr to the Nebula addr through the Nebula
 		// TUN device.
 		if immediatelyForwardToSelf {
-			_, err := f.readers[q].Write(packet)
+			// Write copies into the kernel queue synchronously, so seg's lifetime ends at return.
+			// A self-forwarded superpacket would be re-handed to the
+			// kernel as one giant blob; segment first so the loopback
+			// path sees one IP datagram per Write.
+			err := tio.SegmentSuperpacket(pkt, func(seg []byte) error {
+				_, werr := f.queues[q].Write(seg)
+				return werr
+			})
 			if err != nil {
 				f.l.Error("Failed to forward to tun", "error", err)
 			}
@@ -52,12 +75,24 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}

-	hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
-		hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
+	hostinfo, ready := f.getOrHandshakeConsiderRouting(&fwPacket.Packet, func(hh *HandshakeHostInfo) {
+		// borrowed: SegmentSuperpacket builds each segment in the kernel-supplied pkt
+		// bytes underneath. cachePacket explicitly copies its argument (handshake_manager.go cachePacket),
+		// so retaining segments past the loop is safe.
+		err := tio.SegmentSuperpacket(pkt, func(seg []byte) error {
+			hh.cachePacket(f.l, header.Message, 0, seg, f.sendMessageNow, f.cachedPacketMetrics)
+			return nil
+		})
+		if err != nil && f.l.Enabled(context.Background(), slog.LevelDebug) {
+			f.l.Debug("Failed to segment superpacket for handshake cache",
+				"error", err,
+				"vpnAddr", fwPacket.RemoteAddr,
+			)
+		}
 	})

 	if hostinfo == nil {
-		f.rejectInside(packet, out, q)
+		f.rejectInside(packet, rejectBuf, q)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			f.l.Debug("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks",
 				"vpnAddr", fwPacket.RemoteAddr,
@@ -71,12 +106,11 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}

-	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(fwPacket.Packet, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
-		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
-
+		f.sendInsideMessage(hostinfo, pkt, nb, sendBatch)
 	} else {
-		f.rejectInside(packet, out, q)
+		f.rejectInside(packet, rejectBuf, q)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			hostinfo.logger(f.l).Debug("dropping outbound packet",
 				"fwPacket", fwPacket,
@@ -86,6 +120,125 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 	}
 }

+func (f *Interface) sendInsideEncrypt(hostinfo *HostInfo, ci *ConnectionState, seg, scratch, nb []byte) []byte {
+	if noiseutil.EncryptLockNeeded {
+		ci.writeLock.Lock()
+	}
+	c := ci.messageCounter.Add(1)
+
+	out := header.Encode(scratch, header.Version, header.Message, 0, hostinfo.remoteIndexId, c)
+
+	out, encErr := ci.eKey.EncryptDanger(out, out, seg, c, nb)
+	if noiseutil.EncryptLockNeeded {
+		ci.writeLock.Unlock()
+	}
+	if encErr != nil {
+		hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet",
+			"error", encErr,
+			"udpAddr", hostinfo.GetRemote(),
+			"counter", c,
+		)
+		// Skip this segment; the rest of the superpacket can still go out. TCP will retransmit anything we drop here.
+		return nil
+	}
+
+	return out
+}
+
+// sendInsideMessage encrypts a firewall-approved inside packet (or every
+// segment of a TSO/USO superpacket) into the caller's batch slot for
+// later sendmmsg flush. Segmentation is fused with encryption here so the
+// kernel-supplied superpacket bytes never get written into a separate
+// scratch arena: SegmentSuperpacket builds each segment's plaintext in
+// segScratch[:segLen] in turn, and we encrypt directly into a fresh SendBatch slot.
+func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt tio.Packet, nb []byte, sendBatch *batch.SendBatch) {
+	ci := hostinfo.ConnectionState
+	if ci.eKey == nil {
+		return
+	}
+
+	// One traffic-out mark covers every segment of the superpacket; doing it
+	// per segment in sendInsideEncrypt paid an atomic store up to ~45 extra
+	// times per TSO packet, inside writeLock under boring crypto.
+	f.connectionManager.Out(hostinfo)
+
+	remote := hostinfo.GetRemote()
+	if hostinfo.lastRebindCount != f.rebindCount {
+		//NOTE: there is an update hole if a tunnel isn't used and exactly 256 rebinds occur before the tunnel is
+		// finally used again. This tunnel would eventually be torn down and recreated if this action didn't help.
+		f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
+		hostinfo.lastRebindCount = f.rebindCount
+		if f.l.Enabled(context.Background(), slog.LevelDebug) {
+			hostinfo.logger(f.l).Debug("Lighthouse update triggered for punch due to rebind counter",
+				"vpnAddrs", hostinfo.vpnAddrs,
+			)
+		}
+	}
+
+	if !remote.IsValid() { //the relay path
+		//first, find our relay hostinfo:
+		var relayHostInfo *HostInfo
+		var relay *Relay
+		var err error
+		for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
+			relayHostInfo, relay, err = f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
+			if err != nil {
+				hostinfo.relayState.DeleteRelay(relayIP)
+				hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo",
+					"relay", relayIP,
+					"error", err,
+				)
+				continue
+			}
+			break
+		}
+		if relayHostInfo == nil || relay == nil {
+			//failure already logged
+			return
+		}
+
+		err = tio.SegmentSuperpacket(pkt, func(seg []byte) error {
+			//relay header + header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305) + relay tag
+			scratch := sendBatch.Reserve(header.Len + header.Len + len(seg) + 16 + 16)
+
+			innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb)
+			if innerPacket == nil {
+				return nil
+			}
+
+			//now we need to do a relay-encrypt:
+			toSend, err := f.prepareSendVia(relayHostInfo, relay, innerPacket, nb, scratch, true)
+			if err != nil {
+				//already logged
+				return nil
+			}
+
+			sendBatch.Commit(toSend, relayHostInfo.GetRemote())
+			return nil
+		})
+		if err != nil {
+			hostinfo.logger(f.l).Error("Failed to segment superpacket for relay send", "error", err)
+		}
+		return
+	}
+
+	err := tio.SegmentSuperpacket(pkt, func(seg []byte) error {
+		// header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305)
+		scratch := sendBatch.Reserve(header.Len + len(seg) + 16)
+
+		out := f.sendInsideEncrypt(hostinfo, ci, seg, scratch, nb)
+		if out == nil {
+			return nil
+		}
+
+		sendBatch.Commit(out, remote)
+		return nil
+	})
+	if err != nil {
+		hostinfo.logger(f.l).Error("Failed to segment superpacket for send", "error", err)
+	}
+}
+
 func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 	if !f.firewall.OutboundSendReject {
 		return
@@ -96,33 +249,36 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 		return
 	}

-	_, err := f.readers[q].Write(out)
+	_, err := f.queues[q].Write(out)
 	if err != nil {
 		f.l.Error("Failed to write to tun", "error", err)
 	}
 }

-func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, out []byte, q int) {
+func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, rejectBuf []byte, q int) {
 	if !f.firewall.InboundSendReject {
 		return
 	}

-	out = iputil.CreateRejectPacket(packet, out)
+	// split rejectBuf to make sure we have room to write the plaintext rejection, then encrypt it, without trampling anything
+	// we can't re-use packet, if we need to send an icmp reject, it won't be long enough.
+	half := len(rejectBuf) / 2
+	encryptBuf := rejectBuf[0:0:half] //the first half of rejectBuf's capacity, len set to 0
+	buildBuf := rejectBuf[half:]
+
+	out := iputil.CreateRejectPacket(packet, buildBuf)
 	if len(out) == 0 {
 		return
 	}

 	if len(out) > iputil.MaxRejectPacketSize {
 		if f.l.Enabled(context.Background(), slog.LevelInfo) {
-			f.l.Info("rejectOutside: packet too big, not sending",
-				"packet", packet,
-				"outPacket", out,
-			)
+			f.l.Info("rejectOutside: packet too big, not sending", "packet", packet, "outPacket", out)
 		}
 		return
 	}

-	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q)
+	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, encryptBuf, q)
 }

 // Handshake will attempt to initiate a tunnel with the provided vpn address. This is a no-op if the tunnel is already established or being established
@@ -216,7 +372,7 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac
 }

 func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
-	fp := &firewall.Packet{}
+	fp := &firewall.ParsedPacket{}
 	err := newPacket(p, false, fp)
 	if err != nil {
 		f.l.Warn("error while parsing outgoing packet for firewall check", "error", err)
@@ -224,7 +380,7 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
 	}

 	// check if packet is in outbound fw rules
-	dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil)
+	dropReason := f.firewall.Drop(fp.Packet, false, hostinfo, f.pki.GetCAPool(), nil)
 	if dropReason != nil {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			f.l.Debug("dropping cached packet",
@@ -283,21 +439,13 @@ func (f *Interface) dropExhausted(hostinfo *HostInfo, c uint64, msg string) {
 	}
 }

-// SendVia sends a payload through a Relay tunnel. No authentication or encryption is done
-// to the payload for the ultimate target host, making this a useful method for sending
-// handshake messages to peers through relay tunnels.
-// via is the HostInfo through which the message is relayed.
-// ad is the plaintext data to authenticate, but not encrypt
-// nb is a buffer used to store the nonce value, re-used for performance reasons.
-// out is a buffer used to store the result of the Encrypt operation
-// q indicates which writer to use to send the packet.
-func (f *Interface) SendVia(via *HostInfo,
+func (f *Interface) prepareSendVia(via *HostInfo,
 	relay *Relay,
 	ad,
 	nb,
 	out []byte,
 	nocopy bool,
-) {
+) ([]byte, error) {
 	if noiseutil.EncryptLockNeeded {
 		// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
 		via.ConnectionState.writeLock.Lock()
@@ -308,7 +456,7 @@ func (f *Interface) SendVia(via *HostInfo,
 			via.ConnectionState.writeLock.Unlock()
 		}
 		f.dropExhausted(via, c, "Dropping outbound relay packets, tunnel message counter is exhausted")
-		return
+		return nil, fmt.Errorf("tunnel message counter is exhausted")
 	}

 	out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
@@ -326,7 +474,7 @@ func (f *Interface) SendVia(via *HostInfo,
 			"headerLen", len(out),
 			"cipherOverhead", via.ConnectionState.eKey.Overhead(),
 		)
-		return
+		return nil, io.ErrShortBuffer
 	}

 	// The header bytes are written to the 'out' slice; Grow the slice to hold the header and associated data payload.
@@ -346,13 +494,31 @@ func (f *Interface) SendVia(via *HostInfo,
 	}
 	if err != nil {
 		via.logger(f.l).Info("Failed to EncryptDanger in sendVia", "error", err)
+		return nil, err
+	}
+	f.connectionManager.RelayUsed(relay.LocalIndex)
+	return out, nil
+}
+
+// SendVia sends a payload through a Relay tunnel. No authentication or encryption is done
+// to the payload for the ultimate target host, making this a useful method for sending
+// handshake messages to peers through relay tunnels.
+// via is the HostInfo through which the message is relayed.
+// ad is the plaintext data to authenticate, but not encrypt
+// nb is a buffer used to store the nonce value, re-used for performance reasons.
+// out is a buffer used to store the result of the Encrypt operation
+// q indicates which writer to use to send the packet.
+func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, nocopy bool, q int) {
+	toSend, err := f.prepareSendVia(via, relay, ad, nb, out, nocopy)
+	if err != nil {
+		// already logged by prepareSendVia
 		return
 	}
-	err = f.writers[0].WriteTo(out, via.GetRemote())
+
+	err = f.writers[q].WriteTo(toSend, via.GetRemote())
 	if err != nil {
 		via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err)
 	}
-	f.connectionManager.RelayUsed(relay.LocalIndex)
 }

 func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, q int) {
@@ -445,7 +611,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 				)
 				continue
 			}
-			f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
+			f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true, q)
 			break
 		}
 	}