firewall: Keep conntrack entries per peer and recheck them when a peer reconnects

firewall/cache.go

@@ -3,15 +3,23 @@ package firewall
 import (
 	"context"
 	"log/slog"
+	"net/netip"
 	"sync/atomic"
 	"time"
 
 	"github.com/slackhq/nebula/logging"
 )
 
+// ConntrackKey identifies a tracked flow by its packet and the peer it was allowed for, so peers that route
+// the same unsafe network never share a conntrack entry.
+type ConntrackKey struct {
+	Peer netip.Addr
+	Packet
+}
+
 // ConntrackCache is used as a local routine cache to know if a given flow
 // has been seen in the conntrack table.
-type ConntrackCache map[Packet]struct{}
+type ConntrackCache map[ConntrackKey]struct{}
 
 type ConntrackCacheTicker struct {
 	cacheV    uint64

firewall/cache_test.go

@@ -23,7 +23,7 @@ func newFixedTicker(t *testing.T, l *slog.Logger, cacheLen int) *ConntrackCacheT
 		cache: make(ConntrackCache, cacheLen),
 	}
 	for i := range cacheLen {
-		c.cache[Packet{LocalPort: uint16(i) + 1}] = struct{}{}
+		c.cache[ConntrackKey{Packet: Packet{LocalPort: uint16(i) + 1}}] = struct{}{}
 	}
 	c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path
 	return c

firewall/packet.go

@@ -26,6 +26,10 @@ type Packet struct {
 	RemotePort uint16
 	Protocol   uint8
 	Fragment   bool
+
+	// pad fills what would otherwise be trailing padding so maps hash and compare a ConntrackKey as one
+	// block of memory instead of field by field. It must always be zero.
+	pad [2]byte
 }
 
 func (fp *Packet) Copy() *Packet {