Enforce outbound message counter limits to prevent nonce reuse (#1841) (#1850)

(cherry picked from commit f15d10fc54)

Co-authored-by: Nate Brown <nbrown.us@gmail.com>

noiseutil/cipher_state_test.go

@@ -1,6 +1,7 @@
 package noiseutil
 
 import (
+	"math"
 	"testing"
 
 	"github.com/flynn/noise"
@@ -89,6 +90,24 @@ func roundtrip(t *testing.T, enc, dec CipherState) {
 	assert.Equal(t, 16, enc.Overhead())
 }
 
+func TestEncryptRejectsExhaustedCounter(t *testing.T) {
+	// Pin the headroom below the uint64 wrap so a typo can't silently move the ceiling.
+	require.Equal(t, uint64(1)<<40, RejectHeadroom)
+	require.Equal(t, math.MaxUint64-RejectHeadroom, RejectAfterMessages)
+
+	encA, _ := buildCipherStates(t, CipherAESGCM)
+	encC, _ := buildCipherStates(t, noise.CipherChaChaPoly)
+	nb := make([]byte, 12)
+
+	for _, cs := range []CipherState{NewCipherStateAESGCM(encA), NewCipherStateChaChaPoly(encC)} {
+		_, err := cs.EncryptDanger(nil, nil, []byte("x"), RejectAfterMessages-1, nb)
+		require.NoError(t, err)
+
+		_, err = cs.EncryptDanger(nil, nil, []byte("x"), RejectAfterMessages, nb)
+		require.ErrorIs(t, err, ErrMessageCounterExhausted)
+	}
+}
+
 func BenchmarkCipherStateEncryptAESGCM(b *testing.B) {
 	enc, _ := buildCipherStatesB(b, CipherAESGCM)
 	benchEncryptCipherState(b, NewCipherState(enc, CipherAESGCM))