Compare commits

...
8 changed files with 432 additions and 38 deletions

No files matched your search

+101 -34
View File
@@ -32,16 +32,21 @@ type conn struct {
Expires time.Time // Time when this conntrack entry will expire Expires time.Time // Time when this conntrack entry will expire
// record why the original connection passed the firewall, so we can re-validate // record why the original connection passed the firewall, so we can re-validate
// after ruleset changes. Note, rulesVersion is a uint16 so that these two // after ruleset changes or when the peer handshakes again, which may present a
// fields pack for free after the uint32 above // cert with different groups. These pack into the word after Expires.
incoming bool incoming bool
rulesVersion uint16 rulesVersion uint16
localIndex uint32
} }
// TODO: need conntrack max tracked connections handling // TODO: need conntrack max tracked connections handling
type Firewall struct { type Firewall struct {
Conntrack *FirewallConntrack Conntrack *FirewallConntrack
// siblings finds the other gateways we route an unsafe network through, so a reply that comes back through a
// different gateway than the flow went out through can be matched to it. Nil turns that off.
siblings gatewaySiblings
InRules *FirewallTable InRules *FirewallTable
OutRules *FirewallTable OutRules *FirewallTable
@@ -82,8 +87,8 @@ type firewallMetrics struct {
type FirewallConntrack struct { type FirewallConntrack struct {
sync.Mutex sync.Mutex
Conns map[firewall.Packet]*conn Conns map[firewall.ConntrackKey]*conn
TimerWheel *TimerWheel[firewall.Packet] TimerWheel *TimerWheel[firewall.ConntrackKey]
} }
// FirewallTable is the entry point for a rule, the evaluation order is: // FirewallTable is the entry point for a rule, the evaluation order is:
@@ -167,8 +172,8 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
return &Firewall{ return &Firewall{
Conntrack: &FirewallConntrack{ Conntrack: &FirewallConntrack{
Conns: make(map[firewall.Packet]*conn), Conns: make(map[firewall.ConntrackKey]*conn),
TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax), TimerWheel: NewTimerWheel[firewall.ConntrackKey](tmin, tmax),
}, },
InRules: newFirewallTable(), InRules: newFirewallTable(),
OutRules: newFirewallTable(), OutRules: newFirewallTable(),
@@ -425,6 +430,7 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
// returns nil if the packet should not be dropped. // returns nil if the packet should not be dropped.
func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error { func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
// Make sure remote address matches nebula certificate, and determine how to treat it // Make sure remote address matches nebula certificate, and determine how to treat it
unsafe := false
if h.networks == nil { if h.networks == nil {
// Simple case: Certificate has one address and no unsafe networks // Simple case: Certificate has one address and no unsafe networks
if h.vpnAddrs[0] != fp.RemoteAddr { if h.vpnAddrs[0] != fp.RemoteAddr {
@@ -444,7 +450,7 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
f.metrics(incoming).droppedRemoteAddr.Inc(1) f.metrics(incoming).droppedRemoteAddr.Inc(1)
return ErrPeerRejected // reject for now, one day this may have different FW rules return ErrPeerRejected // reject for now, one day this may have different FW rules
case NetworkTypeUnsafe: case NetworkTypeUnsafe:
break // nothing special, one day this may have different FW rules unsafe = true // nothing else special, one day this may have different FW rules
default: default:
f.metrics(incoming).droppedRemoteAddr.Inc(1) f.metrics(incoming).droppedRemoteAddr.Inc(1)
return ErrUnknownNetworkType //should never happen return ErrUnknownNetworkType //should never happen
@@ -457,8 +463,11 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
return ErrInvalidLocalIP return ErrInvalidLocalIP
} }
// The first vpn address identifies the peer, as it does elsewhere. A host's v1 and v2 certs must agree on it
key := firewall.ConntrackKey{Packet: fp, Peer: h.vpnAddrs[0]}
// Check if we spoke to this tuple, if we did then allow this packet // Check if we spoke to this tuple, if we did then allow this packet
if f.inConns(fp, h, caPool, localCache) { if f.inConns(key, h, caPool, localCache) {
return nil return nil
} }
@@ -469,12 +478,15 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
// We now know which firewall table to check against // We now know which firewall table to check against
if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) { if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
if unsafe && f.inheritConn(key, incoming, h, caPool) {
return nil
}
f.metrics(incoming).droppedNoRule.Inc(1) f.metrics(incoming).droppedNoRule.Inc(1)
return ErrNoMatchingRule return ErrNoMatchingRule
} }
// We always want to conntrack since it is a faster operation // We always want to conntrack since it is a faster operation
f.addConn(fp, incoming) f.addConn(key, incoming, h)
return nil return nil
} }
@@ -503,9 +515,9 @@ func (f *Firewall) EmitStats() {
metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV())) metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
} }
func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool { func (f *Firewall) inConns(key firewall.ConntrackKey, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
if localCache != nil { if localCache != nil {
if _, ok := localCache[fp]; ok { if _, ok := localCache[key]; ok {
return true return true
} }
} }
@@ -518,49 +530,52 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
f.evict(ep) f.evict(ep)
} }
c, ok := conntrack.Conns[fp] c, ok := conntrack.Conns[key]
if !ok { if !ok {
conntrack.Unlock() conntrack.Unlock()
return false return false
} }
if c.rulesVersion != f.rulesVersion { if c.rulesVersion != f.rulesVersion || c.localIndex != h.localIndexId {
// This conntrack entry was for an older rule set, validate // This conntrack entry was allowed by an older rule set or for an earlier tunnel to the peer, validate
// it still passes with the current rule set // it still passes with the current rule set and the peer's current certificate
table := f.OutRules table := f.OutRules
if c.incoming { if c.incoming {
table = f.InRules table = f.InRules
} }
// We now know which firewall table to check against // We now know which firewall table to check against
if !table.match(fp, c.incoming, h.ConnectionState.peerCert, caPool) { if !table.match(key.Packet, c.incoming, h.ConnectionState.peerCert, caPool) {
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
h.logger(f.l).Debug("dropping old conntrack entry, does not match new ruleset", h.logger(f.l).Debug("dropping old conntrack entry, does not match current ruleset and certificate",
"fwPacket", fp, "fwPacket", key.Packet,
"incoming", c.incoming, "incoming", c.incoming,
"rulesVersion", f.rulesVersion, "rulesVersion", f.rulesVersion,
"oldRulesVersion", c.rulesVersion, "oldRulesVersion", c.rulesVersion,
"oldLocalIndex", c.localIndex,
) )
} }
delete(conntrack.Conns, fp) delete(conntrack.Conns, key)
conntrack.Unlock() conntrack.Unlock()
return false return false
} }
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
h.logger(f.l).Debug("keeping old conntrack entry, does match new ruleset", h.logger(f.l).Debug("keeping old conntrack entry, does match current ruleset and certificate",
"fwPacket", fp, "fwPacket", key.Packet,
"incoming", c.incoming, "incoming", c.incoming,
"rulesVersion", f.rulesVersion, "rulesVersion", f.rulesVersion,
"oldRulesVersion", c.rulesVersion, "oldRulesVersion", c.rulesVersion,
"oldLocalIndex", c.localIndex,
) )
} }
c.rulesVersion = f.rulesVersion c.rulesVersion = f.rulesVersion
c.localIndex = h.localIndexId
} }
switch fp.Protocol { switch key.Protocol {
case iputil.IPProtocolTCP: case iputil.IPProtocolTCP:
c.Expires = time.Now().Add(f.TCPTimeout) c.Expires = time.Now().Add(f.TCPTimeout)
case iputil.IPProtocolUDP: case iputil.IPProtocolUDP:
@@ -572,17 +587,17 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
conntrack.Unlock() conntrack.Unlock()
if localCache != nil { if localCache != nil {
localCache[fp] = struct{}{} localCache[key] = struct{}{}
} }
return true return true
} }
func (f *Firewall) addConn(fp firewall.Packet, incoming bool) { func (f *Firewall) addConn(key firewall.ConntrackKey, incoming bool, h *HostInfo) {
var timeout time.Duration var timeout time.Duration
c := &conn{} c := &conn{}
switch fp.Protocol { switch key.Protocol {
case iputil.IPProtocolTCP: case iputil.IPProtocolTCP:
timeout = f.TCPTimeout timeout = f.TCPTimeout
case iputil.IPProtocolUDP: case iputil.IPProtocolUDP:
@@ -593,26 +608,78 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
conntrack := f.Conntrack conntrack := f.Conntrack
conntrack.Lock() conntrack.Lock()
if _, ok := conntrack.Conns[fp]; !ok { if _, ok := conntrack.Conns[key]; !ok {
conntrack.TimerWheel.Advance(time.Now()) conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(fp, timeout) conntrack.TimerWheel.Add(key, timeout)
} }
// Record which rulesVersion allowed this connection, so we can retest after // Record which rulesVersion and tunnel allowed this connection, so we can retest after
// firewall reload // firewall reload or when the peer handshakes again
c.incoming = incoming c.incoming = incoming
c.rulesVersion = f.rulesVersion c.rulesVersion = f.rulesVersion
c.localIndex = h.localIndexId
c.Expires = time.Now().Add(timeout) c.Expires = time.Now().Add(timeout)
conntrack.Conns[fp] = c conntrack.Conns[key] = c
conntrack.Unlock() conntrack.Unlock()
} }
// gatewaySiblings finds the other gateways we route an unsafe network through
type gatewaySiblings interface {
// siblingPeers appends to buf the conntrack peer of every other gateway we route remote through, if h is one of
// those gateways
siblingPeers(remote netip.Addr, h *HostInfo, buf []netip.Addr) []netip.Addr
}
// inheritConn lets h carry a flow that another of our gateways for the same unsafe network already carries. We
// spread the flows we open across the gateways, but the hosts behind them answer through whichever gateway their own
// routing picks, so a reply can come back through a different gateway than the flow went out through. The flow is
// checked against the rules that allowed it in the first place with h's certificate, the same check an entry gets
// when its peer reconnects. Peers that aren't one of our gateways for the network never inherit a flow.
func (f *Firewall) inheritConn(key firewall.ConntrackKey, incoming bool, h *HostInfo, caPool *cert.CAPool) bool {
if f.siblings == nil {
return false
}
var buf [4]netip.Addr
now := time.Now()
for _, peer := range f.siblings.siblingPeers(key.RemoteAddr, h, buf[:0]) {
sk := key
sk.Peer = peer
conntrack := f.Conntrack
conntrack.Lock()
c, ok := conntrack.Conns[sk]
live := ok && c.Expires.After(now)
var allowedIncoming bool
if live {
allowedIncoming = c.incoming
}
conntrack.Unlock()
// The rules for this packet's own direction already said no, only a flow allowed the other way can help
if !live || allowedIncoming == incoming {
continue
}
table := f.OutRules
if allowedIncoming {
table = f.InRules
}
if !table.match(key.Packet, allowedIncoming, h.ConnectionState.peerCert, caPool) {
continue
}
f.addConn(key, allowedIncoming, h)
return true
}
return false
}
// Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel // Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel
// Caller must own the connMutex lock! // Caller must own the connMutex lock!
func (f *Firewall) evict(p firewall.Packet) { func (f *Firewall) evict(key firewall.ConntrackKey) {
// Are we still tracking this conn? // Are we still tracking this conn?
conntrack := f.Conntrack conntrack := f.Conntrack
t, ok := conntrack.Conns[p] t, ok := conntrack.Conns[key]
if !ok { if !ok {
return return
} }
@@ -622,12 +689,12 @@ func (f *Firewall) evict(p firewall.Packet) {
// Timeout is in the future, re-add the timer // Timeout is in the future, re-add the timer
if newT > 0 { if newT > 0 {
conntrack.TimerWheel.Advance(time.Now()) conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(p, newT) conntrack.TimerWheel.Add(key, newT)
return return
} }
// This conn is done // This conn is done
delete(conntrack.Conns, p) delete(conntrack.Conns, key)
} }
func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool { func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
+9 -1
View File
@@ -3,15 +3,23 @@ package firewall
import ( import (
"context" "context"
"log/slog" "log/slog"
"net/netip"
"sync/atomic" "sync/atomic"
"time" "time"
"github.com/slackhq/nebula/logging" "github.com/slackhq/nebula/logging"
) )
// ConntrackKey identifies a tracked flow by its packet and the peer it was allowed for, so peers that route
// the same unsafe network never share a conntrack entry.
type ConntrackKey struct {
Peer netip.Addr
Packet
}
// ConntrackCache is used as a local routine cache to know if a given flow // ConntrackCache is used as a local routine cache to know if a given flow
// has been seen in the conntrack table. // has been seen in the conntrack table.
type ConntrackCache map[Packet]struct{} type ConntrackCache map[ConntrackKey]struct{}
type ConntrackCacheTicker struct { type ConntrackCacheTicker struct {
cacheV uint64 cacheV uint64
+1 -1
View File
@@ -23,7 +23,7 @@ func newFixedTicker(t *testing.T, l *slog.Logger, cacheLen int) *ConntrackCacheT
cache: make(ConntrackCache, cacheLen), cache: make(ConntrackCache, cacheLen),
} }
for i := range cacheLen { for i := range cacheLen {
c.cache[Packet{LocalPort: uint16(i) + 1}] = struct{}{} c.cache[ConntrackKey{Packet: Packet{LocalPort: uint16(i) + 1}}] = struct{}{}
} }
c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path
return c return c
+4
View File
@@ -26,6 +26,10 @@ type Packet struct {
RemotePort uint16 RemotePort uint16
Protocol uint8 Protocol uint8
Fragment bool Fragment bool
// pad fills what would otherwise be trailing padding so maps hash and compare a ConntrackKey as one
// block of memory instead of field by field. It must always be zero.
pad [2]byte
} }
func (fp *Packet) Copy() *Packet { func (fp *Packet) Copy() *Packet {
+248 -1
View File
@@ -3,6 +3,7 @@ package nebula
import ( import (
"bytes" "bytes"
"errors" "errors"
"fmt"
"log/slog" "log/slog"
"math" "math"
"net/netip" "net/netip"
@@ -979,6 +980,252 @@ func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) {
"SECURITY: attacker spoofed victim's overlay source IP (192.0.2.2) by reusing an existing conntrack 4-tuple; Drop returned %v instead of rejecting", got) "SECURITY: attacker spoofed victim's overlay source IP (192.0.2.2) by reusing an existing conntrack 4-tuple; Drop returned %v instead of rejecting", got)
} }
func TestFirewall_ConntrackUnsafeNetworkPeers(t *testing.T) {
l := test.NewLoggerWithOutput(&bytes.Buffer{})
myVpnNetworksTable := new(bart.Lite)
myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
owner := &dummyCert{
name: "owner",
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
}
// Two routers for the same unsafe network, only one of them is trusted inbound
router := func(name string, addr string, groups ...string) *HostInfo {
inverted := map[string]struct{}{}
for _, g := range groups {
inverted[g] = struct{}{}
}
c := &cert.CachedCertificate{
Certificate: &dummyCert{
name: name,
groups: groups,
networks: []netip.Prefix{netip.MustParsePrefix(addr + "/24")},
unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
},
InvertedGroups: inverted,
}
h := &HostInfo{
ConnectionState: &ConnectionState{peerCert: c},
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
}
h.buildNetworks(myVpnNetworksTable, c.Certificate)
return h
}
trusted := router("trusted", "192.0.2.2", "trusted")
other := router("other", "192.0.2.3")
cp := cert.NewCAPool()
flow := firewall.Packet{
LocalAddr: netip.MustParseAddr("192.0.2.1"),
RemoteAddr: netip.MustParseAddr("198.51.100.5"),
LocalPort: 443,
RemotePort: 55000,
Protocol: iputil.IPProtocolTCP,
}
for _, useCache := range []bool{false, true} {
var cache firewall.ConntrackCache
if useCache {
cache = firewall.ConntrackCache{}
}
t.Run(fmt.Sprintf("inbound flow allowed for one peer/cache=%v", useCache), func(t *testing.T) {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"trusted"}, "", "", "", "", ""))
clear(cache)
require.NoError(t, fw.Drop(flow, true, trusted, cp, cache))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, other, cp, cache))
require.NoError(t, fw.Drop(flow, true, trusted, cp, cache))
})
t.Run(fmt.Sprintf("reply to a flow opened via one peer/cache=%v", useCache), func(t *testing.T) {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
clear(cache)
require.NoError(t, fw.Drop(flow, false, trusted, cp, cache))
require.NoError(t, fw.Drop(flow, true, trusted, cp, cache))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, other, cp, cache))
})
}
}
// fakeSiblings maps each gateway to the conntrack peers of the gateways it shares a route with
type fakeSiblings map[*HostInfo][]netip.Addr
func (s fakeSiblings) siblingPeers(_ netip.Addr, h *HostInfo, buf []netip.Addr) []netip.Addr {
return append(buf, s[h]...)
}
func TestFirewall_ConntrackGatewaySiblings(t *testing.T) {
l := test.NewLoggerWithOutput(&bytes.Buffer{})
myVpnNetworksTable := new(bart.Lite)
myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
owner := &dummyCert{
name: "owner",
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
}
// Routers for the same unsafe network. a and b are both our gateways for it, c only claims it in its cert
router := func(name string, addr string, groups ...string) *HostInfo {
inverted := map[string]struct{}{}
for _, g := range groups {
inverted[g] = struct{}{}
}
c := &cert.CachedCertificate{
Certificate: &dummyCert{
name: name,
groups: groups,
networks: []netip.Prefix{netip.MustParsePrefix(addr + "/24")},
unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
},
InvertedGroups: inverted,
}
h := &HostInfo{
ConnectionState: &ConnectionState{peerCert: c},
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
}
h.buildNetworks(myVpnNetworksTable, c.Certificate)
return h
}
a := router("a", "192.0.2.2", "trusted")
b := router("b", "192.0.2.3", "trusted")
c := router("c", "192.0.2.4", "trusted")
siblings := fakeSiblings{a: {b.vpnAddrs[0]}, b: {a.vpnAddrs[0]}}
cp := cert.NewCAPool()
flow := firewall.Packet{
LocalAddr: netip.MustParseAddr("192.0.2.1"),
RemoteAddr: netip.MustParseAddr("198.51.100.5"),
LocalPort: 55000,
RemotePort: 443,
Protocol: iputil.IPProtocolTCP,
}
// We open flows through our gateways, nothing is allowed in on its own
newFirewall := func(outGroup string) *Firewall {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{outGroup}, "", "", "", "", ""))
fw.siblings = siblings
return fw
}
t.Run("reply through the other gateway", func(t *testing.T) {
fw := newFirewall("trusted")
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
require.NoError(t, fw.Drop(flow, true, a, cp, nil))
// a has its own entry now, and keeps the flow when b's is gone
delete(fw.Conntrack.Conns, firewall.ConntrackKey{Packet: flow, Peer: b.vpnAddrs[0]})
require.NoError(t, fw.Drop(flow, true, a, cp, nil))
})
t.Run("not our gateway", func(t *testing.T) {
fw := newFirewall("trusted")
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, c, cp, nil))
})
t.Run("off without siblings", func(t *testing.T) {
fw := newFirewall("trusted")
fw.siblings = nil
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, a, cp, nil))
})
t.Run("expired sibling entry", func(t *testing.T) {
fw := newFirewall("trusted")
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
fw.Conntrack.Conns[firewall.ConntrackKey{Packet: flow, Peer: b.vpnAddrs[0]}].Expires = time.Now().Add(-time.Second)
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, a, cp, nil))
})
t.Run("the original rules must allow the new gateway", func(t *testing.T) {
fw := newFirewall("b-only")
b.ConnectionState.peerCert.InvertedGroups["b-only"] = struct{}{}
defer delete(b.ConnectionState.peerCert.InvertedGroups, "b-only")
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, a, cp, nil))
})
}
func TestFirewall_ConntrackPeerRehandshake(t *testing.T) {
l := test.NewLoggerWithOutput(&bytes.Buffer{})
myVpnNetworksTable := new(bart.Lite)
myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
owner := &dummyCert{
name: "owner",
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
}
// Each call is a new tunnel to the same peer, presenting a cert with the given groups
tunnel := func(localIndex uint32, groups ...string) *HostInfo {
inverted := map[string]struct{}{}
for _, g := range groups {
inverted[g] = struct{}{}
}
c := &cert.CachedCertificate{
Certificate: &dummyCert{
name: "peer",
groups: groups,
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24"), netip.MustParsePrefix("2001:db8::2/64")},
},
InvertedGroups: inverted,
}
h := &HostInfo{
ConnectionState: &ConnectionState{peerCert: c},
vpnAddrs: []netip.Addr{netip.MustParseAddr("192.0.2.2"), netip.MustParseAddr("2001:db8::2")},
localIndexId: localIndex,
}
h.buildNetworks(myVpnNetworksTable, c.Certificate)
return h
}
cp := cert.NewCAPool()
flow := firewall.Packet{
LocalAddr: netip.MustParseAddr("192.0.2.1"),
RemoteAddr: netip.MustParseAddr("192.0.2.2"),
LocalPort: 443,
RemotePort: 55000,
Protocol: iputil.IPProtocolTCP,
}
key := firewall.ConntrackKey{Packet: flow, Peer: netip.MustParseAddr("192.0.2.2")}
t.Run("inbound flow follows the peer's groups", func(t *testing.T) {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"trusted"}, "", "", "", "", ""))
require.NoError(t, fw.Drop(flow, true, tunnel(1, "trusted"), cp, nil))
// Same groups on a new tunnel keeps the entry and moves it to the new tunnel
require.NoError(t, fw.Drop(flow, true, tunnel(2, "trusted"), cp, nil))
require.Contains(t, fw.Conntrack.Conns, key)
assert.Equal(t, uint32(2), fw.Conntrack.Conns[key].localIndex)
// Losing the group on a new tunnel removes the entry
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, tunnel(3), cp, nil))
assert.NotContains(t, fw.Conntrack.Conns, key)
})
t.Run("reply to a flow we opened survives a new tunnel", func(t *testing.T) {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
require.NoError(t, fw.Drop(flow, false, tunnel(1, "trusted"), cp, nil))
require.NoError(t, fw.Drop(flow, true, tunnel(2), cp, nil))
assert.Equal(t, uint32(2), fw.Conntrack.Conns[key].localIndex)
})
}
// BenchmarkFirewallDropConntrackHit measures Drop on an already-established flow // BenchmarkFirewallDropConntrackHit measures Drop on an already-established flow
// (a conntrack hit). This is the fast path that the source-IP<->cert binding // (a conntrack hit). This is the fast path that the source-IP<->cert binding
// reordering adds work to, so it quantifies the cost of moving the address checks // reordering adds work to, so it quantifies the cost of moving the address checks
@@ -1742,6 +1989,6 @@ func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, end
func resetConntrack(fw *Firewall) { func resetConntrack(fw *Firewall) {
fw.Conntrack.Lock() fw.Conntrack.Lock()
fw.Conntrack.Conns = map[firewall.Packet]*conn{} fw.Conntrack.Conns = map[firewall.ConntrackKey]*conn{}
fw.Conntrack.Unlock() fw.Conntrack.Unlock()
} }
+26
View File
@@ -25,6 +25,7 @@ import (
"github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/ratelimit" "github.com/slackhq/nebula/ratelimit"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
) )
@@ -268,6 +269,9 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
ifce.reQueryWait.Store(int64(c.reQueryWait)) ifce.reQueryWait.Store(int64(c.reQueryWait))
ifce.connectionManager.intf = ifce ifce.connectionManager.intf = ifce
if ifce.firewall != nil {
ifce.firewall.siblings = ifce
}
// Held until Close so waiting on the interface blocks until the resources are actually released // Held until Close so waiting on the interface blocks until the resources are actually released
ifce.wg.Add(1) ifce.wg.Add(1)
@@ -554,6 +558,7 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
} }
oldFw := f.firewall oldFw := f.firewall
fw.siblings = f
conntrack := oldFw.Conntrack conntrack := oldFw.Conntrack
conntrack.Lock() conntrack.Lock()
defer conntrack.Unlock() defer conntrack.Unlock()
@@ -711,3 +716,24 @@ func (f *Interface) Close() error {
f.wg.Done() f.wg.Done()
return errors.Join(errs...) return errors.Join(errs...)
} }
// siblingPeers implements gatewaySiblings from the routes and tunnels we have now. Routes name a gateway by any of its
// vpn addresses, conntrack keys a peer by its first one, so each gateway is resolved through its tunnel. A gateway
// we have no tunnel to can't be carrying a live flow.
func (f *Interface) siblingPeers(remote netip.Addr, h *HostInfo, buf []netip.Addr) []netip.Addr {
gateways := f.inside.RoutesFor(remote)
isGateway := func(g routing.Gateway) bool { return slices.Contains(h.vpnAddrs, g.Addr()) }
if len(gateways) < 2 || !slices.ContainsFunc(gateways, isGateway) {
return buf
}
for _, g := range gateways {
if isGateway(g) {
continue
}
if s := f.hostMap.QueryVpnAddr(g.Addr()); s != nil && s != h {
buf = append(buf, s.vpnAddrs[0])
}
}
return buf
}
+1 -1
View File
@@ -39,7 +39,7 @@ func Test_statsEmitter_setsGauges(t *testing.T) {
hostMap: hostMap, hostMap: hostMap,
inside: &overlaytest.NoopTun{}, inside: &overlaytest.NoopTun{},
outside: &udp.NoopConn{}, outside: &udp.NoopConn{},
firewall: &Firewall{Conntrack: &FirewallConntrack{Conns: map[firewall.Packet]*conn{}}}, firewall: &Firewall{Conntrack: &FirewallConntrack{Conns: map[firewall.ConntrackKey]*conn{}}},
lightHouse: lh, lightHouse: lh,
pki: &PKI{}, pki: &PKI{},
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig), handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
+42
View File
@@ -4,8 +4,11 @@ import (
"net/netip" "net/netip"
"testing" "testing"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/overlay/overlaytest"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -140,3 +143,42 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
assert.True(t, res.Failed) assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall) assert.Same(t, fw, f.firewall)
} }
// routesTun is a NoopTun with a route table
type routesTun struct {
overlaytest.NoopTun
routes *bart.Table[routing.Gateways]
}
func (t routesTun) RoutesFor(ip netip.Addr) routing.Gateways {
r, _ := t.routes.Lookup(ip)
return r
}
func TestInterface_SiblingPeers(t *testing.T) {
addr := netip.MustParseAddr
routes := new(bart.Table[routing.Gateways])
// a and b carry 198.51.100.0/24, the route names b by its v6 address
routes.Insert(netip.MustParsePrefix("198.51.100.0/24"), routing.Gateways{
routing.NewGateway(addr("192.0.2.2"), 1),
routing.NewGateway(addr("fd00::3"), 1),
routing.NewGateway(addr("192.0.2.5"), 1), // no tunnel to this one
})
routes.Insert(netip.MustParsePrefix("203.0.113.0/24"), routing.Gateways{routing.NewGateway(addr("192.0.2.2"), 1)})
hm := newHostMap(test.NewLogger())
a := &HostInfo{vpnAddrs: []netip.Addr{addr("192.0.2.2"), addr("fd00::2")}, localIndexId: 1}
b := &HostInfo{vpnAddrs: []netip.Addr{addr("192.0.2.3"), addr("fd00::3")}, localIndexId: 2}
c := &HostInfo{vpnAddrs: []netip.Addr{addr("192.0.2.4")}, localIndexId: 3}
for _, h := range []*HostInfo{a, b, c} {
hm.unlockedAddHostInfo(h, &Interface{})
}
f := &Interface{inside: routesTun{routes: routes}, hostMap: hm}
// Siblings come back as their first vpn address, the one conntrack keys on, whichever address the route names
assert.Equal(t, []netip.Addr{addr("192.0.2.3")}, f.siblingPeers(addr("198.51.100.5"), a, nil))
assert.Equal(t, []netip.Addr{addr("192.0.2.2")}, f.siblingPeers(addr("198.51.100.5"), b, nil))
assert.Empty(t, f.siblingPeers(addr("198.51.100.5"), c, nil), "not one of our gateways for the network")
assert.Empty(t, f.siblingPeers(addr("203.0.113.5"), a, nil), "a single gateway route has no siblings")
assert.Empty(t, f.siblingPeers(addr("192.0.2.3"), a, nil), "no route")
}