Enforce outbound message counter limits to prevent nonce reuse (#1841) (#1850)

(cherry picked from commit f15d10fc54)

Co-authored-by: Nate Brown <nbrown.us@gmail.com>

noiseutil/aesgcm.go

@@ -25,6 +25,9 @@ func (s *CipherStateAESGCM) EncryptDanger(out, ad, plaintext []byte, n uint64, n
 	if s == nil {
 		return nil, errors.New("no cipher state available to encrypt")
 	}
+	if n >= RejectAfterMessages {
+		return nil, ErrMessageCounterExhausted
+	}
 	nb[0] = 0
 	nb[1] = 0
 	nb[2] = 0

noiseutil/chachapoly.go

@@ -24,6 +24,9 @@ func (s *CipherStateChaChaPoly) EncryptDanger(out, ad, plaintext []byte, n uint6
 	if s == nil {
 		return nil, errors.New("no cipher state available to encrypt")
 	}
+	if n >= RejectAfterMessages {
+		return nil, ErrMessageCounterExhausted
+	}
 	nb[0] = 0
 	nb[1] = 0
 	nb[2] = 0

noiseutil/cipher_state.go

@@ -1,11 +1,22 @@
 package noiseutil
 
 import (
+	"errors"
 	"fmt"
+	"math"
 
 	"github.com/flynn/noise"
 )
 
+// RejectHeadroom is the wrap gap for senders racing the counter, sized large enough for any routine count.
+const RejectHeadroom = uint64(1) << 40
+
+// RejectAfterMessages is the nonce ceiling: encrypting stops RejectHeadroom short of the wrap.
+const RejectAfterMessages = math.MaxUint64 - RejectHeadroom
+
+// ErrMessageCounterExhausted is returned by EncryptDanger once the nonce reaches RejectAfterMessages.
+var ErrMessageCounterExhausted = errors.New("message counter exhausted")
+
 // CipherState is the post-handshake AEAD cipher used for the data plane.
 // Each supported cipher has its own concrete implementation in this package with the nonce endianness hardcoded,
 // so the encrypt/decrypt fast path avoids interface dispatch on the byte order.

noiseutil/cipher_state_test.go

@@ -1,6 +1,7 @@
 package noiseutil
 
 import (
+	"math"
 	"testing"
 
 	"github.com/flynn/noise"
@@ -89,6 +90,24 @@ func roundtrip(t *testing.T, enc, dec CipherState) {
 	assert.Equal(t, 16, enc.Overhead())
 }
 
+func TestEncryptRejectsExhaustedCounter(t *testing.T) {
+	// Pin the headroom below the uint64 wrap so a typo can't silently move the ceiling.
+	require.Equal(t, uint64(1)<<40, RejectHeadroom)
+	require.Equal(t, math.MaxUint64-RejectHeadroom, RejectAfterMessages)
+
+	encA, _ := buildCipherStates(t, CipherAESGCM)
+	encC, _ := buildCipherStates(t, noise.CipherChaChaPoly)
+	nb := make([]byte, 12)
+
+	for _, cs := range []CipherState{NewCipherStateAESGCM(encA), NewCipherStateChaChaPoly(encC)} {
+		_, err := cs.EncryptDanger(nil, nil, []byte("x"), RejectAfterMessages-1, nb)
+		require.NoError(t, err)
+
+		_, err = cs.EncryptDanger(nil, nil, []byte("x"), RejectAfterMessages, nb)
+		require.ErrorIs(t, err, ErrMessageCounterExhausted)
+	}
+}
+
 func BenchmarkCipherStateEncryptAESGCM(b *testing.B) {
 	enc, _ := buildCipherStatesB(b, CipherAESGCM)
 	benchEncryptCipherState(b, NewCipherState(enc, CipherAESGCM))