the definitive tun offloads branch (#1704)

connection_state.go

@@ -14,7 +14,7 @@ import (
 )
 
 const (
-	ReplayWindow = 1024
+	ReplayWindow = 8192
 
 	// RehandshakeAfterMessages rolls keys inside the AES-GCM data-volume margin (~2^-36 advantage at 64KB frames).
 	RehandshakeAfterMessages = uint64(1) << 34
@@ -26,6 +26,14 @@ const (
 // RehandshakeAfterMessages must stay below RejectAfterMessages so tunnels roll before the hard send stop.
 const _ = RejectAfterMessages - RehandshakeAfterMessages
 
+// sessionEpoch hands out a receiver-local ordinal to every ConnectionState at creation. The RX
+// staging sort (overlay/batch) orders packets by (epoch, message counter). A re-handshake never
+// rekeys an existing tunnel; it brings up a new hostinfo and ConnectionState with a counter space
+// starting near zero, while the old tunnel keeps decrypting until torn down. During that cutover
+// one flush batch can hold packets from both tunnels, and the epoch keeps the old tunnel's
+// packets sorted first.
+var sessionEpoch atomic.Uint64
+
 type ConnectionState struct {
 	eKey           noiseutil.CipherState
 	dKey           noiseutil.CipherState
@@ -36,6 +44,8 @@ type ConnectionState struct {
 	window         *Bits
 	decryptLock    sync.Mutex
 	writeLock      sync.Mutex
+	// epoch is this session's sessionEpoch ordinal. Immutable after creation.
+	epoch uint64
 }
 
 // newConnectionStateFromResult builds a fully-populated ConnectionState from a
@@ -55,6 +65,7 @@ func newConnectionStateFromResult(r *handshake.Result) (*ConnectionState, error)
 		eKey:      noiseutil.NewCipherState(r.EKey, r.Cipher),
 		dKey:      noiseutil.NewCipherState(r.DKey, r.Cipher),
 		window:    NewBits(ReplayWindow),
+		epoch:     sessionEpoch.Add(1),
 	}
 	ci.messageCounter.Add(r.MessageIndex)
 	for i := uint64(1); i <= r.MessageIndex; i++ {
@@ -85,8 +96,7 @@ func (cs *ConnectionState) Curve() cert.Curve {
 	return cs.myCert.Curve()
 }
 
-func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, out []byte, packet []byte, nb []byte) ([]byte, error) {
-	var err error
+func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, packet []byte, nb []byte) ([]byte, error) {
 	cs.decryptLock.Lock()
 	result := cs.window.Check(l, messageCounter)
 	cs.decryptLock.Unlock()
@@ -94,7 +104,7 @@ func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, out []
 		return nil, ErrAlreadySeen
 	}
 
-	out, err = cs.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
+	out, err := cs.dKey.DecryptDanger(packet[header.Len:header.Len], packet[:header.Len], packet[header.Len:], messageCounter, nb)
 	if err != nil {
 		return nil, err
 	}
@@ -108,7 +118,6 @@ func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, out []
 	return out, nil
 }
 
-// VerifyRelay verifies AEAD protected (but not encrypted) relay frames. packet must be length-checked by the caller.
 func (cs *ConnectionState) VerifyRelay(l *slog.Logger, messageCounter uint64, packet []byte, nb []byte) error {
 	cs.decryptLock.Lock()
 	result := cs.window.Check(l, messageCounter)
@@ -117,6 +126,11 @@ func (cs *ConnectionState) VerifyRelay(l *slog.Logger, messageCounter uint64, pa
 		return ErrAlreadySeen
 	}
 
+	// The entire body is sent as AD, not encrypted.
+	// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
+	// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
+	// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
+	// which will gracefully fail in the DecryptDanger call.
 	signedPayload := packet[:len(packet)-cs.dKey.Overhead()]
 	signatureValue := packet[len(packet)-cs.dKey.Overhead():]
 	_, err := cs.dKey.DecryptDanger(nil, signedPayload, signatureValue, messageCounter, nb)
@@ -130,6 +144,5 @@ func (cs *ConnectionState) VerifyRelay(l *slog.Logger, messageCounter uint64, pa
 	if !result {
 		return ErrAlreadySeen
 	}
-
 	return nil
 }