Compare commits

..
Author SHA1 Message Date
JackDoan b4158b3855 Skip the inbound firewall for the rest of a flow's run within one receive batch 2026-10-01 12:01:00 -05:00
JackDoan 905d01aee5 conntrack: lock-free sync.Map keyed by firewall.Packet with a coarse clock, drop the per-routine cache 2026-10-01 12:00:52 -05:00
Nate Brown d5062dacf0 lighthouse: Never block in QueryServer, drop queries when the channel is full (#1910) 2026-09-30 15:26:34 -05:00
Jack Doan 7cfa47d569 tweaks to error handling in iputil.IPv6FindUpperProtocol (#1893) 2026-09-22 15:45:01 -05:00
Caleb JasikandNate Brown e20569ce51 Let the darwin tun adopt an fd the host already configured (#1890)
newTunFromFd was a stub on darwin, so an embedder handed a utun by the OS had
no way to use it. A darwin network extension is given exactly that:
NEPacketTunnelProvider creates the device and applies addressing, mtu and
routes from its network settings, and a sandboxed extension cannot change them
afterwards. Adopting that fd means everything about moving packets is shared
with newTun and only the setup differs, so the tun tracks whether the host owns
it and leaves the system routing table and interface config alone when it does.

Co-authored-by: Nate Brown <nbrown.us@gmail.com>
2026-09-22 11:54:30 -05:00
Caleb JasikandNate Brown 9651ffdb0d Cancel the context when Main hands back no Control (#1889)
The goroutines Main starts stop only when its context does, and only a caller
holding the Control can arrange that. Cancelling solely on error left a config
test, which returns no Control and no error, with the lighthouse query worker
and a hostname resolver per dns named static host running for the life of the
process. Every embedder that validates a config in process rather than by
exec'ing pays for that on each config load, dnclient and the apple clients
included.

Co-authored-by: Nate Brown <nbrown.us@gmail.com>
2026-09-22 11:52:28 -05:00
Jack Doan 3a45c01656 Advertise underlay addresses on Android (#1881) 2026-09-21 14:59:56 -05:00
Jack Doan 019cc46a7f smoke: Prove windows honors NlMtu with a 9000 byte ping (#1873) 2026-09-21 12:59:31 -05:00
Nate Brown cf3f315cd3 IPv6FindUpperProtocol: fail closed when the extension header walk runs out of budget (#1883) 2026-09-21 12:09:51 -05:00
32 changed files with 971 additions and 505 deletions

No files matched your search

+14
View File
@@ -263,6 +263,20 @@ try {
} }
Write-Host "OK: WSL peer -> windows lighthouse over v6" Write-Host "OK: WSL peer -> windows lighthouse over v6"
# The NlMtu check above proves the value was written, not that windows honors it. A payload this size only
# crosses the tunnel as fragments cut at NlMtu on the way out of the adapter. Left at the adapter default it
# reaches nebula whole, overflows the udp.MTU (9001) write buffer and is dropped, so the ping never answers.
# Anything smaller would ride out as one oversized datagram and survive on IP fragmentation of the underlay.
$BigPayload = 9000
foreach ($target in @(@{ Family = 'v4'; Ip = $Ip2 }, @{ Family = 'v6'; Ip = $Ip6_2 })) {
Wait-Until -TimeoutSec 30 -What "$($target.Family) ping with a $BigPayload byte payload from windows lighthouse to WSL peer ($($target.Ip))" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the large $($target.Family) ping succeeded" }
$null = & ping.exe -n 1 -w 1000 -l $BigPayload $target.Ip
$LASTEXITCODE -eq 0
}
Write-Host "OK: windows lighthouse -> WSL peer, $BigPayload byte $($target.Family) payload"
}
Write-Host '' Write-Host ''
Write-Host 'All smoke checks passed.' Write-Host 'All smoke checks passed.'
} }
+3
View File
@@ -242,11 +242,14 @@ test-cov-html:
go test -coverprofile=coverage.out go test -coverprofile=coverage.out
go tool cover -html=coverage.out go tool cover -html=coverage.out
# The package builds only compile. The final line links an android binary so a linker-only failure
# cannot pass CI.
build-test-mobile: build-test-mobile:
GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build -o /dev/null ${NEBULA_CMD_PATH}
bench: bench:
go test -bench=. go test -bench=.
+2 -3
View File
@@ -8,6 +8,5 @@ If you believe you have found a security vulnerability with Nebula, please let
us know right away. We will investigate all reports and do our best to quickly us know right away. We will investigate all reports and do our best to quickly
fix valid issues. fix valid issues.
For critical reports, if you are interested in a bug bounty: You can submit your report on [HackerOne](https://hackerone.com/slack). You can submit your report on [HackerOne](https://hackerone.com/slack) and our
security team will respond as soon as possible.
For other reports: You can [submit a report via GitHub](https://github.com/slackhq/nebula/security/advisories/new).
+3
View File
@@ -7,6 +7,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/overlay/overlaytest" "github.com/slackhq/nebula/overlay/overlaytest"
@@ -21,6 +22,8 @@ func newTestLighthouse() *LightHouse {
l: test.NewLogger(), l: test.NewLogger(),
addrMap: map[netip.Addr]*RemoteList{}, addrMap: map[netip.Addr]*RemoteList{},
queryChan: make(chan netip.Addr, 10), queryChan: make(chan netip.Addr, 10),
// Nothing drains queryChan here, so an 11th query is dropped.
queryDropped: metrics.NilCounter{},
} }
lighthouses := []netip.Addr{} lighthouses := []netip.Addr{}
staticList := map[netip.Addr]struct{}{} staticList := map[netip.Addr]struct{}{}
+2 -2
View File
@@ -410,8 +410,8 @@ logging:
#try_interval: 100ms #try_interval: 100ms
#retries: 20 #retries: 20
# query_buffer is the size of the buffer channel for querying lighthouses # query_buffer is the size of the buffer channel for querying lighthouses, drops are recorded in the lighthouse.query.dropped metric.
#query_buffer: 64 #query_buffer: 1024
# trigger_buffer is the size of the buffer channel for quickly sending handshakes # trigger_buffer is the size of the buffer channel for quickly sending handshakes
# after receiving the response for lighthouse queries # after receiving the response for lighthouse queries
+98 -111
View File
@@ -14,6 +14,7 @@ import (
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"sync/atomic"
"time" "time"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
@@ -28,16 +29,23 @@ type FirewallInterface interface {
AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
} }
// conn is a conntrack entry. Only expires changes after it is stored, a rules version change
// stores a replacement entry instead.
type conn struct { type conn struct {
Expires time.Time // Time when this conntrack entry will expire expires atomic.Uint32 // conntrack clock second when this entry expires, see FirewallConntrack.now
// record why the original connection passed the firewall, so we can re-validate // record why the original connection passed the firewall, so we can re-validate
// after ruleset changes. Note, rulesVersion is a uint16 so that these two // after ruleset changes
// fields pack for free after the uint32 above
incoming bool incoming bool
rulesVersion uint16 rulesVersion uint16
} }
func newConn(expires uint32, incoming bool, rulesVersion uint16) *conn {
c := &conn{incoming: incoming, rulesVersion: rulesVersion}
c.expires.Store(expires)
return c
}
// TODO: need conntrack max tracked connections handling // TODO: need conntrack max tracked connections handling
type Firewall struct { type Firewall struct {
Conntrack *FirewallConntrack Conntrack *FirewallConntrack
@@ -79,11 +87,69 @@ type firewallMetrics struct {
droppedNoRule metrics.Counter droppedNoRule metrics.Counter
} }
// FirewallConntrack is safe for concurrent use by every routine without a lock.
type FirewallConntrack struct { type FirewallConntrack struct {
sync.Mutex conns sync.Map // firewall.Packet -> *conn
count atomic.Int64 // entries in conns, which sync.Map can't tell us
now atomic.Uint32 // the conntrack clock: whole seconds since epoch, kept current by Run
epoch time.Time
}
Conns map[firewall.Packet]*conn func newFirewallConntrack() *FirewallConntrack {
TimerWheel *TimerWheel[firewall.Packet] return &FirewallConntrack{epoch: time.Now()}
}
// Run advances the conntrack clock every second and sweeps out expired entries every minute,
// until ctx is done. Reading a shared clock is much cheaper than asking the OS on every packet.
func (ct *FirewallConntrack) Run(ctx context.Context) {
t := time.NewTicker(time.Second)
defer t.Stop()
var nextSweep uint32
for {
select {
case <-ctx.Done():
return
case <-t.C:
// Measured from epoch rather than counted, so a late tick can't slow the clock down
now := uint32(time.Since(ct.epoch) / time.Second)
ct.now.Store(now)
if now >= nextSweep {
ct.sweep(now)
nextSweep = now + 60
}
}
}
}
// sweep deletes entries that expired at or before now.
func (ct *FirewallConntrack) sweep(now uint32) {
ct.conns.Range(func(k, v any) bool {
if v.(*conn).expires.Load() <= now {
ct.delete(k.(firewall.Packet), v.(*conn))
}
return true
})
}
func (ct *FirewallConntrack) store(fp firewall.Packet, c *conn) {
if _, replaced := ct.conns.Swap(fp, c); !replaced {
ct.count.Add(1)
}
}
// delete removes fp's entry if it is still c, so a newer entry stored by another routine survives
func (ct *FirewallConntrack) delete(fp firewall.Packet, c *conn) {
if ct.conns.CompareAndDelete(fp, c) {
ct.count.Add(-1)
}
}
// clear deletes every entry
func (ct *FirewallConntrack) clear() {
ct.conns.Range(func(k, v any) bool {
ct.delete(k.(firewall.Packet), v.(*conn))
return true
})
} }
// FirewallTable is the entry point for a rule, the evaluation order is: // FirewallTable is the entry point for a rule, the evaluation order is:
@@ -132,26 +198,10 @@ type firewallLocalCIDR struct {
LocalCIDR *bart.Lite LocalCIDR *bart.Lite
} }
// NewFirewall creates a new Firewall object. A TimerWheel is created for you from the provided timeouts. // NewFirewall creates a new Firewall object.
// The certificate provided should be the highest version loaded in memory. // The certificate provided should be the highest version loaded in memory.
func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c cert.Certificate) *Firewall { func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c cert.Certificate) *Firewall {
//TODO: error on 0 duration //TODO: error on 0 duration
var tmin, tmax time.Duration
if tcpTimeout < UDPTimeout {
tmin = tcpTimeout
tmax = UDPTimeout
} else {
tmin = UDPTimeout
tmax = tcpTimeout
}
if defaultTimeout < tmin {
tmin = defaultTimeout
} else if defaultTimeout > tmax {
tmax = defaultTimeout
}
routableNetworks := new(bart.Lite) routableNetworks := new(bart.Lite)
var assignedNetworks []netip.Prefix var assignedNetworks []netip.Prefix
for _, network := range c.Networks() { for _, network := range c.Networks() {
@@ -166,10 +216,7 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
} }
return &Firewall{ return &Firewall{
Conntrack: &FirewallConntrack{ Conntrack: newFirewallConntrack(),
Conns: make(map[firewall.Packet]*conn),
TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax),
},
InRules: newFirewallTable(), InRules: newFirewallTable(),
OutRules: newFirewallTable(), OutRules: newFirewallTable(),
TCPTimeout: tcpTimeout, TCPTimeout: tcpTimeout,
@@ -423,7 +470,7 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
// Drop returns an error if the packet should be dropped, explaining why. It // Drop returns an error if the packet should be dropped, explaining why. It
// returns nil if the packet should not be dropped. // returns nil if the packet should not be dropped.
func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error { func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool) error {
// Make sure remote address matches nebula certificate, and determine how to treat it // Make sure remote address matches nebula certificate, and determine how to treat it
if h.networks == nil { if h.networks == nil {
// Simple case: Certificate has one address and no unsafe networks // Simple case: Certificate has one address and no unsafe networks
@@ -458,7 +505,7 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
} }
// Check if we spoke to this tuple, if we did then allow this packet // Check if we spoke to this tuple, if we did then allow this packet
if f.inConns(fp, h, caPool, localCache) { if f.inConns(fp, h, caPool) {
return nil return nil
} }
@@ -494,36 +541,19 @@ func (f *Firewall) Destroy() {
} }
func (f *Firewall) EmitStats() { func (f *Firewall) EmitStats() {
conntrack := f.Conntrack metrics.GetOrRegisterGauge("firewall.conntrack.count", nil).Update(f.Conntrack.count.Load())
conntrack.Lock()
conntrackCount := len(conntrack.Conns)
conntrack.Unlock()
metrics.GetOrRegisterGauge("firewall.conntrack.count", nil).Update(int64(conntrackCount))
metrics.GetOrRegisterGauge("firewall.rules.version", nil).Update(int64(f.rulesVersion)) metrics.GetOrRegisterGauge("firewall.rules.version", nil).Update(int64(f.rulesVersion))
metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV())) metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
} }
func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool { func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool) bool {
if localCache != nil {
if _, ok := localCache[fp]; ok {
return true
}
}
conntrack := f.Conntrack conntrack := f.Conntrack
conntrack.Lock() v, ok := conntrack.conns.Load(fp)
// Purge every time we test
ep, has := conntrack.TimerWheel.Purge()
if has {
f.evict(ep)
}
c, ok := conntrack.Conns[fp]
if !ok { if !ok {
conntrack.Unlock()
return false return false
} }
c := v.(*conn)
expires := conntrack.now.Load() + f.timeout(fp.Protocol)
if c.rulesVersion != f.rulesVersion { if c.rulesVersion != f.rulesVersion {
// This conntrack entry was for an older rule set, validate // This conntrack entry was for an older rule set, validate
@@ -543,8 +573,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
"oldRulesVersion", c.rulesVersion, "oldRulesVersion", c.rulesVersion,
) )
} }
delete(conntrack.Conns, fp) conntrack.delete(fp, c)
conntrack.Unlock()
return false return false
} }
@@ -557,77 +586,35 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
) )
} }
c.rulesVersion = f.rulesVersion conntrack.store(fp, newConn(expires, c.incoming, f.rulesVersion))
return true
} }
switch fp.Protocol { // Most packets land in the same second as the one before, skip writing an unchanged expiry
case iputil.IPProtocolTCP: if c.expires.Load() != expires {
c.Expires = time.Now().Add(f.TCPTimeout) c.expires.Store(expires)
case iputil.IPProtocolUDP:
c.Expires = time.Now().Add(f.UDPTimeout)
default:
c.Expires = time.Now().Add(f.DefaultTimeout)
}
conntrack.Unlock()
if localCache != nil {
localCache[fp] = struct{}{}
} }
return true return true
} }
func (f *Firewall) addConn(fp firewall.Packet, incoming bool) { func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
var timeout time.Duration
c := &conn{}
switch fp.Protocol {
case iputil.IPProtocolTCP:
timeout = f.TCPTimeout
case iputil.IPProtocolUDP:
timeout = f.UDPTimeout
default:
timeout = f.DefaultTimeout
}
conntrack := f.Conntrack
conntrack.Lock()
if _, ok := conntrack.Conns[fp]; !ok {
conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(fp, timeout)
}
// Record which rulesVersion allowed this connection, so we can retest after // Record which rulesVersion allowed this connection, so we can retest after
// firewall reload // firewall reload
c.incoming = incoming conntrack := f.Conntrack
c.rulesVersion = f.rulesVersion conntrack.store(fp, newConn(conntrack.now.Load()+f.timeout(fp.Protocol), incoming, f.rulesVersion))
c.Expires = time.Now().Add(timeout)
conntrack.Conns[fp] = c
conntrack.Unlock()
} }
// Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel // timeout returns how many seconds a conntrack entry for proto lives past its last packet
// Caller must own the connMutex lock! func (f *Firewall) timeout(proto uint8) uint32 {
func (f *Firewall) evict(p firewall.Packet) { switch proto {
// Are we still tracking this conn? case iputil.IPProtocolTCP:
conntrack := f.Conntrack return uint32(f.TCPTimeout / time.Second)
t, ok := conntrack.Conns[p] case iputil.IPProtocolUDP:
if !ok { return uint32(f.UDPTimeout / time.Second)
return default:
return uint32(f.DefaultTimeout / time.Second)
} }
newT := t.Expires.Sub(time.Now())
// Timeout is in the future, re-add the timer
if newT > 0 {
conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(p, newT)
return
}
// This conn is done
delete(conntrack.Conns, p)
} }
func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool { func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
-69
View File
@@ -1,69 +0,0 @@
package firewall
import (
"context"
"log/slog"
"sync/atomic"
"time"
"github.com/slackhq/nebula/logging"
)
// ConntrackCache is used as a local routine cache to know if a given flow
// has been seen in the conntrack table.
type ConntrackCache map[Packet]struct{}
type ConntrackCacheTicker struct {
cacheV uint64
cacheTick atomic.Uint64
l *slog.Logger
cache ConntrackCache
}
func NewConntrackCacheTicker(ctx context.Context, l *slog.Logger, d time.Duration) *ConntrackCacheTicker {
if d == 0 {
return nil
}
c := &ConntrackCacheTicker{
l: l,
cache: ConntrackCache{},
}
go c.tick(ctx, d)
return c
}
func (c *ConntrackCacheTicker) tick(ctx context.Context, d time.Duration) {
t := time.NewTicker(d)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
c.cacheTick.Add(1)
}
}
}
// Get checks if the cache ticker has moved to the next version before returning
// the map. If it has moved, we reset the map.
func (c *ConntrackCacheTicker) Get() ConntrackCache {
if c == nil {
return nil
}
if tick := c.cacheTick.Load(); tick != c.cacheV {
c.cacheV = tick
if ll := len(c.cache); ll > 0 {
if c.l.Enabled(context.Background(), logging.LevelTrace) {
c.l.Log(context.Background(), logging.LevelTrace, "resetting conntrack cache", "len", ll)
}
c.cache = make(ConntrackCache, ll)
}
}
return c.cache
}
-70
View File
@@ -1,70 +0,0 @@
package firewall
import (
"bytes"
"log/slog"
"strings"
"testing"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
)
// The tests below pin the log format produced by ConntrackCacheTicker.Get
// so changes cannot silently break what operators are grepping for. The
// ticker's internal state (cache + cacheTick) is poked directly to avoid
// racing a goroutine-driven tick in tests.
func newFixedTicker(t *testing.T, l *slog.Logger, cacheLen int) *ConntrackCacheTicker {
t.Helper()
c := &ConntrackCacheTicker{
l: l,
cache: make(ConntrackCache, cacheLen),
}
for i := range cacheLen {
c.cache[Packet{LocalPort: uint16(i) + 1}] = struct{}{}
}
c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path
return c
}
func TestConntrackCacheTicker_Get_TextFormat(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewLoggerWithOutputAndLevel(buf, logging.LevelTrace)
c := newFixedTicker(t, l, 3)
c.Get()
assert.Equal(t, "level=DEBUG-4 msg=\"resetting conntrack cache\" len=3\n", buf.String())
}
func TestConntrackCacheTicker_Get_JSONFormat(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewJSONLoggerWithOutput(buf, logging.LevelTrace)
c := newFixedTicker(t, l, 2)
c.Get()
assert.JSONEq(t, `{"level":"DEBUG-4","msg":"resetting conntrack cache","len":2}`, strings.TrimSpace(buf.String()))
}
func TestConntrackCacheTicker_Get_QuietBelowTrace(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewLoggerWithOutputAndLevel(buf, slog.LevelDebug)
c := newFixedTicker(t, l, 5)
c.Get()
assert.Empty(t, buf.String())
}
func TestConntrackCacheTicker_Get_QuietWhenCacheEmpty(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewLoggerWithOutputAndLevel(buf, logging.LevelTrace)
c := newFixedTicker(t, l, 0)
c.Get()
assert.Empty(t, buf.String())
}
+4
View File
@@ -26,6 +26,10 @@ type Packet struct {
RemotePort uint16 RemotePort uint16
Protocol uint8 Protocol uint8
Fragment bool Fragment bool
// pad fills what would otherwise be compiler padding. A Packet with no padding hashes as one
// block of memory, which makes it a much faster map key. Always zero.
pad [2]byte
} }
func (fp *Packet) Copy() *Packet { func (fp *Packet) Copy() *Packet {
+126 -100
View File
@@ -2,6 +2,7 @@ package nebula
import ( import (
"bytes" "bytes"
"context"
"errors" "errors"
"log/slog" "log/slog"
"math" "math"
@@ -23,39 +24,12 @@ func TestNewFirewall(t *testing.T) {
l := test.NewLogger() l := test.NewLogger()
c := &dummyCert{} c := &dummyCert{}
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c) fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c)
conntrack := fw.Conntrack assert.NotNil(t, fw.Conntrack)
assert.NotNil(t, conntrack)
assert.NotNil(t, conntrack.Conns)
assert.NotNil(t, conntrack.TimerWheel)
assert.NotNil(t, fw.InRules) assert.NotNil(t, fw.InRules)
assert.NotNil(t, fw.OutRules) assert.NotNil(t, fw.OutRules)
assert.Equal(t, time.Second, fw.TCPTimeout) assert.Equal(t, time.Second, fw.TCPTimeout)
assert.Equal(t, time.Minute, fw.UDPTimeout) assert.Equal(t, time.Minute, fw.UDPTimeout)
assert.Equal(t, time.Hour, fw.DefaultTimeout) assert.Equal(t, time.Hour, fw.DefaultTimeout)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
fw = NewFirewall(l, time.Second, time.Hour, time.Minute, c)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
fw = NewFirewall(l, time.Hour, time.Second, time.Minute, c)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
fw = NewFirewall(l, time.Hour, time.Minute, time.Second, c)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
fw = NewFirewall(l, time.Minute, time.Hour, time.Second, c)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
fw = NewFirewall(l, time.Minute, time.Second, time.Hour, c)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
} }
func TestFirewall_AddRule(t *testing.T) { func TestFirewall_AddRule(t *testing.T) {
@@ -212,44 +186,44 @@ func TestFirewall_Drop(t *testing.T) {
cp := cert.NewCAPool() cp := cert.NewCAPool()
// Drop outbound // Drop outbound
assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil)) assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp))
// Allow inbound // Allow inbound
resetConntrack(fw) resetConntrack(fw)
require.NoError(t, fw.Drop(p, true, &h, cp, nil)) require.NoError(t, fw.Drop(p, true, &h, cp))
// Allow outbound because conntrack // Allow outbound because conntrack
require.NoError(t, fw.Drop(p, false, &h, cp, nil)) require.NoError(t, fw.Drop(p, false, &h, cp))
// test remote mismatch // test remote mismatch
oldRemote := p.RemoteAddr oldRemote := p.RemoteAddr
p.RemoteAddr = netip.MustParseAddr("1.2.3.10") p.RemoteAddr = netip.MustParseAddr("1.2.3.10")
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP) assert.Equal(t, fw.Drop(p, false, &h, cp), ErrInvalidRemoteIP)
p.RemoteAddr = oldRemote p.RemoteAddr = oldRemote
// ensure signer doesn't get in the way of group checks // ensure signer doesn't get in the way of group checks
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(p, true, &h, cp), ErrNoMatchingRule)
// test caSha doesn't drop on match // test caSha doesn't drop on match
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.Drop(p, true, &h, cp, nil)) require.NoError(t, fw.Drop(p, true, &h, cp))
// ensure ca name doesn't get in the way of group checks // ensure ca name doesn't get in the way of group checks
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(p, true, &h, cp), ErrNoMatchingRule)
// test caName doesn't drop on match // test caName doesn't drop on match
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.Drop(p, true, &h, cp, nil)) require.NoError(t, fw.Drop(p, true, &h, cp))
} }
func TestFirewall_DropV6(t *testing.T) { func TestFirewall_DropV6(t *testing.T) {
@@ -290,44 +264,44 @@ func TestFirewall_DropV6(t *testing.T) {
cp := cert.NewCAPool() cp := cert.NewCAPool()
// Drop outbound // Drop outbound
assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil)) assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp))
// Allow inbound // Allow inbound
resetConntrack(fw) resetConntrack(fw)
require.NoError(t, fw.Drop(p, true, &h, cp, nil)) require.NoError(t, fw.Drop(p, true, &h, cp))
// Allow outbound because conntrack // Allow outbound because conntrack
require.NoError(t, fw.Drop(p, false, &h, cp, nil)) require.NoError(t, fw.Drop(p, false, &h, cp))
// test remote mismatch // test remote mismatch
oldRemote := p.RemoteAddr oldRemote := p.RemoteAddr
p.RemoteAddr = netip.MustParseAddr("fd12::56") p.RemoteAddr = netip.MustParseAddr("fd12::56")
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP) assert.Equal(t, fw.Drop(p, false, &h, cp), ErrInvalidRemoteIP)
p.RemoteAddr = oldRemote p.RemoteAddr = oldRemote
// ensure signer doesn't get in the way of group checks // ensure signer doesn't get in the way of group checks
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(p, true, &h, cp), ErrNoMatchingRule)
// test caSha doesn't drop on match // test caSha doesn't drop on match
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.Drop(p, true, &h, cp, nil)) require.NoError(t, fw.Drop(p, true, &h, cp))
// ensure ca name doesn't get in the way of group checks // ensure ca name doesn't get in the way of group checks
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(p, true, &h, cp), ErrNoMatchingRule)
// test caName doesn't drop on match // test caName doesn't drop on match
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.Drop(p, true, &h, cp, nil)) require.NoError(t, fw.Drop(p, true, &h, cp))
} }
func BenchmarkFirewallTable_match(b *testing.B) { func BenchmarkFirewallTable_match(b *testing.B) {
@@ -534,10 +508,10 @@ func TestFirewall_Drop2(t *testing.T) {
cp := cert.NewCAPool() cp := cert.NewCAPool()
// h1/c1 lacks the proper groups // h1/c1 lacks the proper groups
require.ErrorIs(t, fw.Drop(p, true, &h1, cp, nil), ErrNoMatchingRule) require.ErrorIs(t, fw.Drop(p, true, &h1, cp), ErrNoMatchingRule)
// c has the proper groups // c has the proper groups
resetConntrack(fw) resetConntrack(fw)
require.NoError(t, fw.Drop(p, true, &h, cp, nil)) require.NoError(t, fw.Drop(p, true, &h, cp))
} }
func TestFirewall_Drop3(t *testing.T) { func TestFirewall_Drop3(t *testing.T) {
@@ -614,18 +588,18 @@ func TestFirewall_Drop3(t *testing.T) {
cp := cert.NewCAPool() cp := cert.NewCAPool()
// c1 should pass because host match // c1 should pass because host match
require.NoError(t, fw.Drop(p, true, &h1, cp, nil)) require.NoError(t, fw.Drop(p, true, &h1, cp))
// c2 should pass because ca sha match // c2 should pass because ca sha match
resetConntrack(fw) resetConntrack(fw)
require.NoError(t, fw.Drop(p, true, &h2, cp, nil)) require.NoError(t, fw.Drop(p, true, &h2, cp))
// c3 should fail because no match // c3 should fail because no match
resetConntrack(fw) resetConntrack(fw)
assert.Equal(t, fw.Drop(p, true, &h3, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(p, true, &h3, cp), ErrNoMatchingRule)
// Test a remote address match // Test a remote address match
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "1.2.3.4/24", "", "", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "1.2.3.4/24", "", "", ""))
require.NoError(t, fw.Drop(p, true, &h1, cp, nil)) require.NoError(t, fw.Drop(p, true, &h1, cp))
} }
func TestFirewall_Drop3V6(t *testing.T) { func TestFirewall_Drop3V6(t *testing.T) {
@@ -662,7 +636,7 @@ func TestFirewall_Drop3V6(t *testing.T) {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
cp := cert.NewCAPool() cp := cert.NewCAPool()
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "fd12::34/120", "", "", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "fd12::34/120", "", "", ""))
require.NoError(t, fw.Drop(p, true, &h, cp, nil)) require.NoError(t, fw.Drop(p, true, &h, cp))
} }
func TestFirewall_DropConntrackReload(t *testing.T) { func TestFirewall_DropConntrackReload(t *testing.T) {
@@ -703,12 +677,12 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
cp := cert.NewCAPool() cp := cert.NewCAPool()
// Drop outbound // Drop outbound
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound // Allow inbound
resetConntrack(fw) resetConntrack(fw)
require.NoError(t, fw.Drop(p, true, &h, cp, nil)) require.NoError(t, fw.Drop(p, true, &h, cp))
// Allow outbound because conntrack // Allow outbound because conntrack
require.NoError(t, fw.Drop(p, false, &h, cp, nil)) require.NoError(t, fw.Drop(p, false, &h, cp))
oldFw := fw oldFw := fw
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -717,7 +691,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
fw.rulesVersion = oldFw.rulesVersion + 1 fw.rulesVersion = oldFw.rulesVersion + 1
// Allow outbound because conntrack and new rules allow port 10 // Allow outbound because conntrack and new rules allow port 10
require.NoError(t, fw.Drop(p, false, &h, cp, nil)) require.NoError(t, fw.Drop(p, false, &h, cp))
oldFw = fw oldFw = fw
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -726,7 +700,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
fw.rulesVersion = oldFw.rulesVersion + 1 fw.rulesVersion = oldFw.rulesVersion + 1
// Drop outbound because conntrack doesn't match new ruleset // Drop outbound because conntrack doesn't match new ruleset
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(p, false, &h, cp), ErrNoMatchingRule)
} }
func TestFirewall_ICMPPortBehavior(t *testing.T) { func TestFirewall_ICMPPortBehavior(t *testing.T) {
@@ -771,12 +745,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0 p.LocalPort = 0
p.RemotePort = 0 p.RemotePort = 0
// Drop outbound // Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound // Allow inbound
resetConntrack(fw) resetConntrack(fw)
require.NoError(t, fw.Drop(*p, true, &h, cp, nil)) require.NoError(t, fw.Drop(*p, true, &h, cp))
//now also allow outbound //now also allow outbound
require.NoError(t, fw.Drop(*p, false, &h, cp, nil)) require.NoError(t, fw.Drop(*p, false, &h, cp))
}) })
t.Run("nonzero ports", func(t *testing.T) { t.Run("nonzero ports", func(t *testing.T) {
@@ -784,12 +758,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0xabcd p.LocalPort = 0xabcd
p.RemotePort = 0x1234 p.RemotePort = 0x1234
// Drop outbound // Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound // Allow inbound
resetConntrack(fw) resetConntrack(fw)
require.NoError(t, fw.Drop(*p, true, &h, cp, nil)) require.NoError(t, fw.Drop(*p, true, &h, cp))
//now also allow outbound //now also allow outbound
require.NoError(t, fw.Drop(*p, false, &h, cp, nil)) require.NoError(t, fw.Drop(*p, false, &h, cp))
}) })
}) })
@@ -801,12 +775,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0 p.LocalPort = 0
p.RemotePort = 0 p.RemotePort = 0
// Drop outbound // Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound // Allow inbound
resetConntrack(fw) resetConntrack(fw)
assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, true, &h, cp), ErrNoMatchingRule)
//now also allow outbound //now also allow outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
}) })
t.Run("nonzero ports, still blocked", func(t *testing.T) { t.Run("nonzero ports, still blocked", func(t *testing.T) {
@@ -814,12 +788,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0xabcd p.LocalPort = 0xabcd
p.RemotePort = 0x1234 p.RemotePort = 0x1234
// Drop outbound // Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound // Allow inbound
resetConntrack(fw) resetConntrack(fw)
assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, true, &h, cp), ErrNoMatchingRule)
//now also allow outbound //now also allow outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
}) })
t.Run("nonzero, matching ports, still blocked", func(t *testing.T) { t.Run("nonzero, matching ports, still blocked", func(t *testing.T) {
@@ -827,12 +801,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 80 p.LocalPort = 80
p.RemotePort = 80 p.RemotePort = 80
// Drop outbound // Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound // Allow inbound
resetConntrack(fw) resetConntrack(fw)
assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, true, &h, cp), ErrNoMatchingRule)
//now also allow outbound //now also allow outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
}) })
}) })
t.Run("Any proto, any port", func(t *testing.T) { t.Run("Any proto, any port", func(t *testing.T) {
@@ -844,12 +818,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0 p.LocalPort = 0
p.RemotePort = 0 p.RemotePort = 0
// Drop outbound // Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound // Allow inbound
resetConntrack(fw) resetConntrack(fw)
require.NoError(t, fw.Drop(*p, true, &h, cp, nil)) require.NoError(t, fw.Drop(*p, true, &h, cp))
//now also allow outbound //now also allow outbound
require.NoError(t, fw.Drop(*p, false, &h, cp, nil)) require.NoError(t, fw.Drop(*p, false, &h, cp))
}) })
t.Run("nonzero ports, allowed", func(t *testing.T) { t.Run("nonzero ports, allowed", func(t *testing.T) {
@@ -858,15 +832,15 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0xabcd p.LocalPort = 0xabcd
p.RemotePort = 0x1234 p.RemotePort = 0x1234
// Drop outbound // Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound // Allow inbound
resetConntrack(fw) resetConntrack(fw)
require.NoError(t, fw.Drop(*p, true, &h, cp, nil)) require.NoError(t, fw.Drop(*p, true, &h, cp))
//now also allow outbound //now also allow outbound
require.NoError(t, fw.Drop(*p, false, &h, cp, nil)) require.NoError(t, fw.Drop(*p, false, &h, cp))
//different ID is blocked //different ID is blocked
p.RemotePort++ p.RemotePort++
require.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) require.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
}) })
}) })
@@ -914,7 +888,7 @@ func TestFirewall_DropIPSpoofing(t *testing.T) {
Protocol: iputil.IPProtocolUDP, Protocol: iputil.IPProtocolUDP,
Fragment: false, Fragment: false,
} }
assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP) assert.Equal(t, fw.Drop(p, true, &h1, cp), ErrInvalidRemoteIP)
} }
func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) { func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) {
@@ -965,20 +939,84 @@ func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) {
Protocol: iputil.IPProtocolUDP, Protocol: iputil.IPProtocolUDP,
} }
require.NoError(t, fw.Drop(flow, true, &victimHI, cp, nil), require.NoError(t, fw.Drop(flow, true, &victimHI, cp),
"victim's own traffic from its own overlay IP must be allowed") "victim's own traffic from its own overlay IP must be allowed")
unseen := flow unseen := flow
unseen.RemotePort = 55001 unseen.RemotePort = 55001
assert.Equal(t, ErrInvalidRemoteIP, fw.Drop(unseen, true, &attackerHI, cp, nil), assert.Equal(t, ErrInvalidRemoteIP, fw.Drop(unseen, true, &attackerHI, cp),
"sanity: attacker forging victim's source IP must be rejected when no conntrack entry exists") "sanity: attacker forging victim's source IP must be rejected when no conntrack entry exists")
got := fw.Drop(flow, true, &attackerHI, cp, nil) got := fw.Drop(flow, true, &attackerHI, cp)
t.Logf("attacker replaying victim's 4-tuple: Drop returned %v (nil == packet ALLOWED == spoof succeeded)", got) t.Logf("attacker replaying victim's 4-tuple: Drop returned %v (nil == packet ALLOWED == spoof succeeded)", got)
assert.Equal(t, ErrInvalidRemoteIP, got, assert.Equal(t, ErrInvalidRemoteIP, got,
"SECURITY: attacker spoofed victim's overlay source IP (192.0.2.2) by reusing an existing conntrack 4-tuple; Drop returned %v instead of rejecting", got) "SECURITY: attacker spoofed victim's overlay source IP (192.0.2.2) by reusing an existing conntrack 4-tuple; Drop returned %v instead of rejecting", got)
} }
func TestFirewallConntrack_sweep(t *testing.T) {
ct := newFirewallConntrack()
live := firewall.Packet{LocalPort: 1}
dead := firewall.Packet{LocalPort: 2}
ct.store(live, newConn(11, true, 0))
ct.store(dead, newConn(10, true, 0))
assert.Equal(t, int64(2), ct.count.Load())
ct.sweep(10)
_, ok := ct.conns.Load(live)
assert.True(t, ok)
_, ok = ct.conns.Load(dead)
assert.False(t, ok)
assert.Equal(t, int64(1), ct.count.Load())
}
func TestFirewallConntrack_storeAndDeleteCount(t *testing.T) {
ct := newFirewallConntrack()
fp := firewall.Packet{LocalPort: 1}
first := newConn(10, true, 0)
ct.store(fp, first)
// Replacing an entry doesn't add to the count
second := newConn(20, true, 1)
ct.store(fp, second)
assert.Equal(t, int64(1), ct.count.Load())
// Deleting a stale entry is a no-op, the newer one stays
ct.delete(fp, first)
v, ok := ct.conns.Load(fp)
assert.True(t, ok)
assert.Same(t, second, v)
assert.Equal(t, int64(1), ct.count.Load())
ct.delete(fp, second)
_, ok = ct.conns.Load(fp)
assert.False(t, ok)
assert.Equal(t, int64(0), ct.count.Load())
ct.store(fp, first)
ct.store(firewall.Packet{LocalPort: 2}, newConn(10, true, 0))
ct.clear()
assert.Equal(t, int64(0), ct.count.Load())
}
func TestFirewallConntrack_Run(t *testing.T) {
ct := newFirewallConntrack()
ct.epoch = time.Now().Add(-time.Hour)
ct.store(firewall.Packet{LocalPort: 1}, newConn(10, true, 0))
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
ct.Run(ctx)
close(done)
}()
// The first tick sets the clock to an hour in and sweeps the long expired entry
require.Eventually(t, func() bool { return ct.count.Load() == 0 }, 5*time.Second, 10*time.Millisecond)
assert.GreaterOrEqual(t, ct.now.Load(), uint32(3600))
cancel()
<-done
}
// BenchmarkFirewallDropConntrackHit measures Drop on an already-established flow // BenchmarkFirewallDropConntrackHit measures Drop on an already-established flow
// (a conntrack hit). This is the fast path that the source-IP<->cert binding // (a conntrack hit). This is the fast path that the source-IP<->cert binding
// reordering adds work to, so it quantifies the cost of moving the address checks // reordering adds work to, so it quantifies the cost of moving the address checks
@@ -987,8 +1025,6 @@ func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) {
// so the remote-address check is a single netip.Addr compare. // so the remote-address check is a single netip.Addr compare.
// - complex: peer cert has unsafe networks (h.networks populated), so the // - complex: peer cert has unsafe networks (h.networks populated), so the
// remote-address check is a BART lookup. // remote-address check is a BART lookup.
// - noCache/localCache: whether a per-batch ConntrackCache is supplied, which in
// the original code let the fast path skip straight past the address checks.
func BenchmarkFirewallDropConntrackHit(b *testing.B) { func BenchmarkFirewallDropConntrackHit(b *testing.B) {
l := test.NewLoggerWithOutput(&bytes.Buffer{}) l := test.NewLoggerWithOutput(&bytes.Buffer{})
@@ -1038,12 +1074,9 @@ func BenchmarkFirewallDropConntrackHit(b *testing.B) {
cases := []struct { cases := []struct {
name string name string
host *HostInfo host *HostInfo
useCache bool
}{ }{
{"simple/noCache", simpleHost, false}, {"simple", simpleHost},
{"simple/localCache", simpleHost, true}, {"complex", complexHost},
{"complex/noCache", complexHost, false},
{"complex/localCache", complexHost, true},
} }
for _, tc := range cases { for _, tc := range cases {
@@ -1052,17 +1085,12 @@ func BenchmarkFirewallDropConntrackHit(b *testing.B) {
require.NoError(b, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", "")) require.NoError(b, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
// Establish the conntrack entry so every benchmarked Drop is a hit. // Establish the conntrack entry so every benchmarked Drop is a hit.
require.NoError(b, fw.Drop(flow, true, tc.host, cp, nil)) require.NoError(b, fw.Drop(flow, true, tc.host, cp))
var cache firewall.ConntrackCache
if tc.useCache {
cache = firewall.ConntrackCache{}
}
b.ReportAllocs() b.ReportAllocs()
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
if err := fw.Drop(flow, true, tc.host, cp, cache); err != nil { if err := fw.Drop(flow, true, tc.host, cp); err != nil {
b.Fatal(err) b.Fatal(err)
} }
} }
@@ -1550,7 +1578,7 @@ func (c *testcase) Test(t *testing.T, fw *Firewall) {
t.Helper() t.Helper()
cp := cert.NewCAPool() cp := cert.NewCAPool()
resetConntrack(fw) resetConntrack(fw)
err := fw.Drop(c.p, true, c.h, cp, nil) err := fw.Drop(c.p, true, c.h, cp)
if c.err == nil { if c.err == nil {
require.NoError(t, err, "failed to not drop remote address %s", c.p.RemoteAddr) require.NoError(t, err, "failed to not drop remote address %s", c.p.RemoteAddr)
} else { } else {
@@ -1741,7 +1769,5 @@ func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, end
} }
func resetConntrack(fw *Firewall) { func resetConntrack(fw *Firewall) {
fw.Conntrack.Lock() fw.Conntrack.clear()
fw.Conntrack.Conns = map[firewall.Packet]*conn{}
fw.Conntrack.Unlock()
} }
+1 -1
View File
@@ -5,6 +5,7 @@ go 1.26.0
require ( require (
dario.cat/mergo v1.0.2 dario.cat/mergo v1.0.2
filippo.io/bigmod v0.1.0 filippo.io/bigmod v0.1.0
github.com/DefinedNet/netlink v1.3.2-dn.1
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
github.com/armon/go-radix v1.0.0 github.com/armon/go-radix v1.0.0
github.com/flynn/noise v1.1.0 github.com/flynn/noise v1.1.0
@@ -20,7 +21,6 @@ require (
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
github.com/stretchr/testify v1.12.1 github.com/stretchr/testify v1.12.1
github.com/vishvananda/netlink v1.3.1
go.uber.org/goleak v1.3.0 go.uber.org/goleak v1.3.0
go.yaml.in/yaml/v3 v3.0.5 go.yaml.in/yaml/v3 v3.0.5
golang.org/x/crypto v0.57.0 golang.org/x/crypto v0.57.0
+2 -4
View File
@@ -3,6 +3,8 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8= filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8=
filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI= filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI=
github.com/DefinedNet/netlink v1.3.2-dn.1 h1:Md2PsUE1Xuy1Zxs/Bm6356n1kt5bq2HspXlPJ6pEspg=
github.com/DefinedNet/netlink v1.3.2-dn.1/go.mod h1:o4YSn8Jw7zoDcYXDmV77TRd4eS5sSLuE+vu2zAN67C8=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -138,8 +140,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -195,8 +195,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-58
View File
@@ -6,7 +6,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net"
"net/netip" "net/netip"
"slices" "slices"
"sync" "sync"
@@ -18,7 +17,6 @@ import (
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging"
) )
const defaultPromoteEvery = 1000 // Count of packets sent before we try moving a tunnel to a preferred underlay ip address const defaultPromoteEvery = 1000 // Count of packets sent before we try moving a tunnel to a preferred underlay ip address
@@ -930,59 +928,3 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
return li return li
} }
// Utility functions
func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
//FIXME: This function is pretty garbage
var finalAddrs []netip.Addr
ifaces, _ := net.Interfaces()
for _, i := range ifaces {
allow := allowList.AllowName(i.Name)
if l.Enabled(context.Background(), logging.LevelTrace) {
l.Log(context.Background(), logging.LevelTrace, "localAllowList.AllowName",
"interfaceName", i.Name,
"allow", allow,
)
}
if !allow {
continue
}
addrs, _ := i.Addrs()
for _, rawAddr := range addrs {
var addr netip.Addr
switch v := rawAddr.(type) {
case *net.IPNet:
//continue
addr, _ = netip.AddrFromSlice(v.IP)
case *net.IPAddr:
addr, _ = netip.AddrFromSlice(v.IP)
}
if !addr.IsValid() {
if l.Enabled(context.Background(), slog.LevelDebug) {
l.Debug("addr was invalid", "localAddr", rawAddr)
}
continue
}
addr = addr.Unmap()
if addr.IsLoopback() == false && addr.IsLinkLocalUnicast() == false {
isAllowed := allowList.Allow(addr)
if l.Enabled(context.Background(), logging.LevelTrace) {
l.Log(context.Background(), logging.LevelTrace, "localAllowList.Allow",
"localAddr", addr,
"allowed", isAllowed,
)
}
if !isAllowed {
continue
}
finalAddrs = append(finalAddrs, addr)
}
}
}
return finalAddrs
}
+3 -3
View File
@@ -16,7 +16,7 @@ import (
"github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/routing"
) )
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) { func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int) {
// borrowed: pkt.Bytes is owned by the originating tio.Queue and is // borrowed: pkt.Bytes is owned by the originating tio.Queue and is
// only valid until the next Read on that queue. Every consumer below // only valid until the next Read on that queue. Every consumer below
// (parse, self-forward, handshake cache, sendInsideMessage) reads it // (parse, self-forward, handshake cache, sendInsideMessage) reads it
@@ -109,7 +109,7 @@ func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.Parse
return return
} }
dropReason := f.firewall.Drop(fwPacket.Packet, false, hostinfo, f.pki.GetCAPool(), localCache) dropReason := f.firewall.Drop(fwPacket.Packet, false, hostinfo, f.pki.GetCAPool())
if dropReason == nil { if dropReason == nil {
f.sendInsideMessage(hostinfo, pkt, nb, sendBatch) f.sendInsideMessage(hostinfo, pkt, nb, sendBatch)
} else { } else {
@@ -380,7 +380,7 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
} }
// check if packet is in outbound fw rules // check if packet is in outbound fw rules
dropReason := f.firewall.Drop(fp.Packet, false, hostinfo, f.pki.GetCAPool(), nil) dropReason := f.firewall.Drop(fp.Packet, false, hostinfo, f.pki.GetCAPool())
if dropReason != nil { if dropReason != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
f.l.Debug("dropping cached packet", f.l.Debug("dropping cached packet",
+1 -1
View File
@@ -50,7 +50,7 @@ func newSelfForwardInterface(myAddrs ...netip.Addr) (*Interface, *capturingTun)
} }
func consumeInside(f *Interface, packet []byte) { func consumeInside(f *Interface, packet []byte) {
f.consumeInsidePacket(tio.Packet{Bytes: packet}, &firewall.ParsedPacket{}, make([]byte, 12), nil, make([]byte, mtu), 0, nil) f.consumeInsidePacket(tio.Packet{Bytes: packet}, &firewall.ParsedPacket{}, make([]byte, 12), nil, make([]byte, mtu), 0)
} }
// l4Proto describes one upper-layer header for these tests: its IP next-header // l4Proto describes one upper-layer header for these tests: its IP next-header
+17 -16
View File
@@ -52,8 +52,6 @@ type InterfaceConfig struct {
reQueryEvery uint32 reQueryEvery uint32
reQueryWait time.Duration reQueryWait time.Duration
ConntrackCacheTimeout time.Duration
// CpuAffinity, when non-empty, names the CPUs each TUN reader goroutine // CpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
// should pin to. Queue i pins to CpuAffinity[i % len(CpuAffinity)] — // should pin to. Queue i pins to CpuAffinity[i % len(CpuAffinity)] —
// shorter lists than `routines` cycle. Empty list keeps the default // shorter lists than `routines` cycle. Empty list keeps the default
@@ -111,8 +109,6 @@ type Interface struct {
rebindEpoch atomic.Uint32 rebindEpoch atomic.Uint32
version string version string
conntrackCacheTimeout time.Duration
ctx context.Context ctx context.Context
writers []udp.Conn writers []udp.Conn
queues []tio.Queue queues []tio.Queue
@@ -227,7 +223,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
myBroadcastAddrsTable: cs.myVpnBroadcastAddrsTable, myBroadcastAddrsTable: cs.myVpnBroadcastAddrsTable,
relayManager: c.relayManager, relayManager: c.relayManager,
connectionManager: c.connectionManager, connectionManager: c.connectionManager,
conntrackCacheTimeout: c.ConntrackCacheTimeout,
cpuAffinity: c.CpuAffinity, cpuAffinity: c.CpuAffinity,
pinThreads: c.PinThreads, pinThreads: c.PinThreads,
@@ -361,8 +356,19 @@ type rxContext struct {
h *header.H h *header.H
fwPacket *firewall.ParsedPacket fwPacket *firewall.ParsedPacket
hostmapCache map[uint32]*HostInfo hostmapCache map[uint32]*HostInfo
// lastAllowed is the last flow the inbound firewall let in during this batch. UDP GRO hands us segments
// in runs from the same flow, so the rest of a run can skip the firewall. Cleared by endBatch.
lastAllowed firewall.Packet
lastAllowedHost *HostInfo
lhh *LightHouseHandler lhh *LightHouseHandler
ctCache *firewall.ConntrackCacheTicker }
// endBatch forgets what was learned during a batch, so nothing outlives a hostmap or firewall change by more than one batch
func (rxc *rxContext) endBatch() {
clear(rxc.hostmapCache)
rxc.lastAllowed = firewall.Packet{}
rxc.lastAllowedHost = nil
} }
func newRxContext(f *Interface, q int) *rxContext { func newRxContext(f *Interface, q int) *rxContext {
@@ -374,7 +380,6 @@ func newRxContext(f *Interface, q int) *rxContext {
fwPacket: &firewall.ParsedPacket{}, fwPacket: &firewall.ParsedPacket{},
hostmapCache: map[uint32]*HostInfo{}, hostmapCache: map[uint32]*HostInfo{},
lhh: f.lightHouse.NewRequestHandler(), lhh: f.lightHouse.NewRequestHandler(),
ctCache: firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout),
} }
} }
@@ -396,7 +401,7 @@ func (f *Interface) listenOut(i int) {
if err := f.batchers[i].Flush(); err != nil { if err := f.batchers[i].Flush(); err != nil {
f.l.Error("Failed to flush tun coalescer", "error", err) f.l.Error("Failed to flush tun coalescer", "error", err)
} }
clear(rxc.hostmapCache) rxc.endBatch()
} }
err := li.ListenOut(listener, flusher) err := li.ListenOut(listener, flusher)
@@ -444,8 +449,6 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
fwPacket := &firewall.ParsedPacket{} fwPacket := &firewall.ParsedPacket{}
nb := make([]byte, 12, 12) nb := make([]byte, 12, 12)
conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
for { for {
pkts, err := queue.Read() pkts, err := queue.Read()
if err != nil { if err != nil {
@@ -458,7 +461,7 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
} }
for _, pkt := range pkts { for _, pkt := range pkts {
f.consumeInsidePacket(pkt, fwPacket, nb, sb, rejectBuf, i, conntrackCache.Get()) f.consumeInsidePacket(pkt, fwPacket, nb, sb, rejectBuf, i)
// Flush incrementally once a full sendmmsg batch has // Flush incrementally once a full sendmmsg batch has
// accumulated so the first packets of a deep read drain // accumulated so the first packets of a deep read drain
// hit the wire while the rest are still being encrypted. // hit the wire while the rest are still being encrypted.
@@ -531,9 +534,8 @@ func (f *Interface) reloadFirewall(c *config.C) {
} }
oldFw := f.firewall oldFw := f.firewall
conntrack := oldFw.Conntrack // Conntrack lives as long as the interface, keep using the running one
conntrack.Lock() fw.Conntrack = oldFw.Conntrack
defer conntrack.Unlock()
fw.rulesVersion = oldFw.rulesVersion + 1 fw.rulesVersion = oldFw.rulesVersion + 1
// If rulesVersion is back to zero, we have wrapped all the way around. Be // If rulesVersion is back to zero, we have wrapped all the way around. Be
@@ -544,8 +546,7 @@ func (f *Interface) reloadFirewall(c *config.C) {
"oldFirewallHashes", oldFw.GetRuleHashes(), "oldFirewallHashes", oldFw.GetRuleHashes(),
"rulesVersion", fw.rulesVersion, "rulesVersion", fw.rulesVersion,
) )
} else { fw.Conntrack.clear()
fw.Conntrack = conntrack
} }
f.firewall = fw f.firewall = fw
+1 -2
View File
@@ -10,7 +10,6 @@ import (
"github.com/rcrowley/go-metrics" "github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/overlay/overlaytest" "github.com/slackhq/nebula/overlay/overlaytest"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
@@ -43,7 +42,7 @@ func Test_emitStats_primesGauges(t *testing.T) {
hostMap: hostMap, hostMap: hostMap,
inside: &overlaytest.NoopTun{}, inside: &overlaytest.NoopTun{},
outside: &udp.NoopConn{}, outside: &udp.NoopConn{},
firewall: &Firewall{Conntrack: &FirewallConntrack{Conns: map[firewall.Packet]*conn{}}}, firewall: &Firewall{Conntrack: newFirewallConntrack()},
lightHouse: lh, lightHouse: lh,
pki: &PKI{}, pki: &PKI{},
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig), handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
+3 -13
View File
@@ -45,8 +45,7 @@ func setTransportChecksum4(packet []byte) {
if ihl < ipv4.HeaderLen || end < ihl || end > len(packet) { if ihl < ipv4.HeaderLen || end < ihl || end > len(packet) {
return return
} }
// The checksum covers the whole datagram, which a fragment (MF set or a // do not attempt to set checksums for fragments
// non-zero offset) does not carry.
if binary.BigEndian.Uint16(packet[6:8])&0x3fff != 0 { if binary.BigEndian.Uint16(packet[6:8])&0x3fff != 0 {
return return
} }
@@ -68,10 +67,7 @@ func setTransportChecksum6(packet []byte) {
return return
} }
// The checksum covers the whole datagram, which a fragment does not carry. // we must find the transport header in order to write a checksum
// An unknown extension header hides where the transport header starts. A
// chain longer than the walk's budget ends it early, at an offset that was
// never checked against the packet.
proto, offset, _, anyFragment, err := IPv6FindUpperProtocol(packet[:end]) proto, offset, _, anyFragment, err := IPv6FindUpperProtocol(packet[:end])
if err != nil || anyFragment || offset >= end { if err != nil || anyFragment || offset >= end {
return return
@@ -109,8 +105,7 @@ func transportExtent(transport []byte, proto uint8) ([]byte, bool) {
// writeTransportChecksum stores the checksum of transport, taken over the // writeTransportChecksum stores the checksum of transport, taken over the
// pseudo-header sum csum, in the header's checksum field. A UDP checksum that // pseudo-header sum csum, in the header's checksum field. A UDP checksum that
// computes to zero goes on the wire as 0xffff: zero means no checksum was // computes to zero goes on the wire as 0xffff: zero means no checksum was
// computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200 // computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200 section 8.1).
// section 8.1).
func writeTransportChecksum(transport []byte, proto uint8, csum uint32) { func writeTransportChecksum(transport []byte, proto uint8, csum uint32) {
var at, minLen int var at, minLen int
switch proto { switch proto {
@@ -133,11 +128,6 @@ func writeTransportChecksum(transport []byte, proto uint8, csum uint32) {
binary.BigEndian.PutUint16(transport[at:], sum) binary.BigEndian.PutUint16(transport[at:], sum)
} }
// fold reduces a pseudo-header sum to the 16 bit seed Checksum takes. Carrying
// the high half back into the low half is what keeps the reduction lossless, so
// the seed sums exactly as the wider value would; 0xffff is its fixed point.
// Every term of that sum comes from a 16 bit field, so it stays far below the
// width at which the accumulator would wrap.
func fold(csum uint32) uint16 { func fold(csum uint32) uint16 {
for csum > 0xffff { for csum > 0xffff {
csum = (csum >> 16) + (csum & 0xffff) csum = (csum >> 16) + (csum & 0xffff)
+13 -16
View File
@@ -345,14 +345,11 @@ func ipv6CreateRejectTCPPacket(packet []byte, out []byte, offset int) []byte {
return out return out
} }
// IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the // IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the offset it begins at, and whether the packet is a non-first fragment.
// offset it begins at, and whether the packet is a non-first fragment. Only the RFC 8200 and IANA extension // Only the RFC 8200 and IANA extension headers below are walked.
// headers below are walked. Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental // Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental 253/254, and real upper layer protocols like SCTP or GRE, is terminal.
// 253/254, and real upper layer protocols like SCTP or GRE, is terminal. Walking those as extension headers // For a non-first fragment the returned protocol is the fragmented protocol and offset points at the fragment header, there is no transport header to locate.
// is a firewall bypass, so they fail closed. For a non-first fragment the returned protocol is the fragmented // Returns ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header, or the chain is truncated, or if more than 8 extension headers are present.
// protocol and offset points at the fragment header, there is no transport header to locate. Returns
// ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header or the chain is truncated before a
// terminal protocol is reached.
func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool, anyFragment bool, err error) { func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool, anyFragment bool, err error) {
const maxIPv6ExtHeaders = 8 const maxIPv6ExtHeaders = 8
if len(packet) < ipv6.HeaderLen { if len(packet) < ipv6.HeaderLen {
@@ -365,14 +362,14 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm
switch nextHeader { switch nextHeader {
case 0, 43, 60: // Hop-by-Hop, Routing, Destination case 0, 43, 60: // Hop-by-Hop, Routing, Destination
if len(packet) < offset+2 { if len(packet) < offset+2 {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload return 0, 0, false, false, ErrIPv6CouldNotFindPayload
} }
nextHeader = packet[offset] nextHeader = packet[offset]
offset += (int(packet[offset+1]) + 1) << 3 offset += (int(packet[offset+1]) + 1) << 3
case 44: // Fragment case 44: // Fragment
if len(packet) < offset+8 { if len(packet) < offset+8 {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload return 0, 0, false, false, ErrIPv6CouldNotFindPayload
} }
anyFragment = true anyFragment = true
// Non-first fragments carry no transport header, report the fragmented protocol and stop // Non-first fragments carry no transport header, report the fragmented protocol and stop
@@ -384,21 +381,21 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm
case 51: // AH case 51: // AH
if len(packet) < offset+2 { if len(packet) < offset+2 {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload return 0, 0, false, false, ErrIPv6CouldNotFindPayload
} }
nextHeader = packet[offset] nextHeader = packet[offset]
offset += (int(packet[offset+1]) + 2) << 2 offset += (int(packet[offset+1]) + 2) << 2
default: default:
// A prior extension header can declare a length that advances offset past the packet. The terminal
// protocol's header isn't actually here, so treat the chain as truncated rather than classifying it.
if offset > len(packet) { if offset > len(packet) {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload // A prior extension header can declare a length that advances offset past the packet, reject this.
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
} }
return nextHeader, offset, isFragment, anyFragment, nil return nextHeader, offset, isFragment, anyFragment, nil //success!
} }
} }
return nextHeader, offset, isFragment, anyFragment, nil // Fail closed
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
} }
func CreateICMPEchoResponse(packet, out []byte) []byte { func CreateICMPEchoResponse(packet, out []byte) []byte {
+17 -5
View File
@@ -529,6 +529,17 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
firstFragToUDP := []byte{17, 0, 0, 1, 0, 0, 0, 1} // frag offset 0, M=1, next = UDP firstFragToUDP := []byte{17, 0, 0, 1, 0, 0, 0, 1} // frag offset 0, M=1, next = UDP
nonFirstFrag := []byte{17, 0, 0, 9, 0, 0, 0, 1} // frag offset non-zero, next = UDP nonFirstFrag := []byte{17, 0, 0, 9, 0, 0, 0, 1} // frag offset non-zero, next = UDP
transport := []byte{0, 80, 1, 187, 0, 0, 0, 0} // stand-in bytes, IPv6FindUpperProtocol never reads ports transport := []byte{0, 80, 1, 187, 0, 0, 0, 0} // stand-in bytes, IPv6FindUpperProtocol never reads ports
extToHbH := []byte{0, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = Hop-by-Hop
// 8 walked headers spend the budget before the 9th slot's terminal is ever classified. This is the
// report 4040546 shape: without failing closed the walk hands back offset 104, and a caller reads ports
// from the attacker's bytes there.
var budgetChain []byte
for range 7 {
budgetChain = append(budgetChain, extToHbH...)
}
budgetChain = append(budgetChain, extToTCP...)
budgetChain = append(budgetChain, transport...)
tests := []struct { tests := []struct {
name string name string
@@ -549,9 +560,10 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
{"first fragment walks to transport", 44, append(firstFragToUDP, transport...), 17, ipv6.HeaderLen + 8, false, true, nil}, {"first fragment walks to transport", 44, append(firstFragToUDP, transport...), 17, ipv6.HeaderLen + 8, false, true, nil},
{"non-first fragment stops", 44, append(nonFirstFrag, transport...), 17, ipv6.HeaderLen, true, true, nil}, {"non-first fragment stops", 44, append(nonFirstFrag, transport...), 17, ipv6.HeaderLen, true, true, nil},
{"unknown protocol is terminal", 132, transport, 132, ipv6.HeaderLen, false, false, nil}, // SCTP {"unknown protocol is terminal", 132, transport, 132, ipv6.HeaderLen, false, false, nil}, // SCTP
{"truncated extension header", 0, nil, 0, ipv6.HeaderLen, false, false, ErrIPv6CouldNotFindPayload}, {"truncated extension header", 0, nil, 0, 0, false, false, ErrIPv6CouldNotFindPayload},
// Destination Options with a declared length (255+1)*8 = 2048 that runs past the 48 byte buffer, next = SCTP // Destination Options with a declared length (255+1)*8 = 2048 that runs past the 48 byte buffer, next = SCTP
{"extension length past buffer", 60, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 132, ipv6.HeaderLen + 2048, false, false, ErrIPv6CouldNotFindPayload}, {"extension length past buffer", 0, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 0, 0, false, false, ErrIPv6CouldNotFindPayload},
{"budget exhausted fails closed", 0, budgetChain, 0, 0, false, false, ErrIPv6CouldNotFindPayload},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -559,10 +571,10 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
packet := makeIPv6Packet(src, dst, tt.nextHeader, tt.payload) packet := makeIPv6Packet(src, dst, tt.nextHeader, tt.payload)
proto, offset, isFragment, anyFragment, err := IPv6FindUpperProtocol(packet) proto, offset, isFragment, anyFragment, err := IPv6FindUpperProtocol(packet)
if tt.wantErr != nil { if tt.wantErr != nil {
assert.ErrorIs(t, err, tt.wantErr) require.ErrorIs(t, err, tt.wantErr)
return } else {
}
require.NoError(t, err) require.NoError(t, err)
}
assert.Equal(t, tt.wantProto, proto) assert.Equal(t, tt.wantProto, proto)
assert.Equal(t, tt.wantOffset, offset) assert.Equal(t, tt.wantOffset, offset)
assert.Equal(t, tt.wantFragment, isFragment) assert.Equal(t, tt.wantFragment, isFragment)
+40 -4
View File
@@ -15,10 +15,12 @@ import (
"time" "time"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging" "github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/nicfinder"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
) )
@@ -42,6 +44,10 @@ type LightHouse struct {
// addresses rather than whatever this machine's NICs happen to be. Set it before Start. // addresses rather than whatever this machine's NICs happen to be. Set it before Start.
localAddrsFn func(*LocalAllowList) []netip.Addr localAddrsFn func(*LocalAllowList) []netip.Addr
// lastLocalAddrsErr is the previous localAddrsFn failure. Enumeration runs on every update, so an
// unchanged failure is demoted to Debug rather than warning every lighthouse.interval forever.
lastLocalAddrsErr atomic.Pointer[string]
// Local cache of answers from light houses // Local cache of answers from light houses
// map of vpn addr to answers // map of vpn addr to answers
addrMap map[netip.Addr]*RemoteList addrMap map[netip.Addr]*RemoteList
@@ -76,6 +82,8 @@ type LightHouse struct {
updateTrigger chan struct{} updateTrigger chan struct{}
queryChan chan netip.Addr queryChan chan netip.Addr
// queryDropped counts queries QueryServer dropped because queryChan was full.
queryDropped metrics.Counter
calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote
@@ -111,11 +119,14 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
nebulaPort: nebulaPort, nebulaPort: nebulaPort,
punchy: p, punchy: p,
updateTrigger: make(chan struct{}, 1), updateTrigger: make(chan struct{}, 1),
queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)), queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 1024)),
queryDropped: metrics.GetOrRegisterCounter("lighthouse.query.dropped", nil),
l: l, l: l,
} }
h.localAddrsFn = func(al *LocalAllowList) []netip.Addr { h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
return localAddrs(h.l, al) addrs, err := nicfinder.Find(ctx, h.l, al)
h.logLocalAddrsErr(err)
return addrs
} }
lighthouses := make([]netip.Addr, 0) lighthouses := make([]netip.Addr, 0)
@@ -505,14 +516,19 @@ func (lh *LightHouse) Query(vpnAddr netip.Addr) *RemoteList {
return nil return nil
} }
// QueryServer is asynchronous so no reply should be expected // QueryServer is asynchronous and non-blocking, no reply should be expected.
// If the query channel is full, the query is dropped.
func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) { func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) {
// Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses // Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses
if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) { if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) {
return return
} }
lh.queryChan <- vpnAddr select {
case lh.queryChan <- vpnAddr:
default:
lh.queryDropped.Inc(1)
}
} }
func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList { func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList {
@@ -916,6 +932,26 @@ func (lh *LightHouse) TriggerUpdate() {
} }
} }
// logLocalAddrsErr reports a localAddrs failure at Warn the first time it is seen and at Debug while
// it persists unchanged, so a permanent failure does not warn on every update forever.
func (lh *LightHouse) logLocalAddrsErr(err error) {
if err == nil {
lh.lastLocalAddrsErr.Store(nil)
return
}
msg := err.Error()
prev := lh.lastLocalAddrsErr.Swap(&msg)
if prev != nil && *prev == msg {
if lh.l.Enabled(context.Background(), slog.LevelDebug) {
lh.l.Debug("Failed to collect local addresses to advertise", "error", err)
}
return
}
lh.l.Warn("Failed to collect local addresses to advertise", "error", err)
}
func (lh *LightHouse) SendUpdate() { func (lh *LightHouse) SendUpdate() {
var v4 []*V4AddrPort var v4 []*V4AddrPort
var v6 []*V6AddrPort var v6 []*V6AddrPort
+85
View File
@@ -1,10 +1,14 @@
package nebula package nebula
import ( import (
"bytes"
"encoding/binary" "encoding/binary"
"errors"
"fmt" "fmt"
"net/netip" "net/netip"
"sync"
"testing" "testing"
"time"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
@@ -764,3 +768,84 @@ func TestLighthouse_DeletesWork(t *testing.T) {
out = lh.Query(testHost) out = lh.Query(testHost)
assert.Nil(t, out) assert.Nil(t, out)
} }
func TestLightHouse_logLocalAddrsErr(t *testing.T) {
out := &bytes.Buffer{}
lh := &LightHouse{l: test.NewLoggerWithOutput(out)}
// The first sighting of a failure warns.
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "permission denied")
// Repeating unchanged does not warn again, which is what keeps a permanent failure from warning
// on every lighthouse.interval for the life of the process.
out.Reset()
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.NotContains(t, out.String(), "level=WARN")
// A different failure is a new event and warns.
out.Reset()
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "something else")
// Recovering resets, so the same failure returning later warns again.
out.Reset()
lh.logLocalAddrsErr(nil)
assert.Empty(t, out.String())
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
}
// heldEncWriter holds the query worker in its first GetHostInfo until release closes.
type heldEncWriter struct {
mockEncWriter
once sync.Once
held chan struct{}
release chan struct{}
}
func (w *heldEncWriter) GetHostInfo(netip.Addr) *HostInfo {
w.once.Do(func() { close(w.held) })
<-w.release
return nil
}
// QueryServer never waits on the query worker, since the packet readers call it and nothing drains the channel at
// shutdown; a query that finds the channel full is dropped and counted. The worker is held on its first query so the
// channel fills.
func TestLighthouse_QueryServerDoesNotBlock(t *testing.T) {
l := test.NewLogger()
cs := testCertState(netip.MustParsePrefix("10.128.0.1/16"))
lh1 := "10.128.0.2"
c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}}
c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
c.Settings["handshakes"] = map[string]any{"query_buffer": 1}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err)
w := &heldEncWriter{held: make(chan struct{}), release: make(chan struct{})}
lh.ifce = w
t.Cleanup(func() { close(w.release) })
lh.QueryServer(netip.MustParseAddr("10.128.0.10"))
select {
case <-w.held:
case <-time.After(5 * time.Second):
t.Fatal("the query worker never took the first query")
}
dropped := lh.queryDropped.Count()
done := make(chan struct{})
go func() {
lh.QueryServer(netip.MustParseAddr("10.128.0.11"))
lh.QueryServer(netip.MustParseAddr("10.128.0.12"))
close(done)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("QueryServer blocked on a full query channel")
}
assert.EqualValues(t, 1, lh.queryDropped.Count()-dropped)
}
+6 -15
View File
@@ -32,9 +32,12 @@ const _ = noiseutil.RejectHeadroom - 4*maxRoutines
func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) { func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
// Automatically cancel the context if Main returns an error, to signal all created goroutines to quit. // The goroutines started below stop only when this context does, and only a caller holding the
// Control can arrange that. Cancel whenever we are not handing one back, which covers an error
// and a config test alike: a config test used to leave the lighthouse query worker, and a
// hostname resolver per dns named static host, running for the life of the process.
defer func() { defer func() {
if reterr != nil { if retcon == nil {
cancel() cancel()
} }
}() }()
@@ -110,18 +113,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
l.Info("Using multiple routines", "routines", routines) l.Info("Using multiple routines", "routines", routines)
} }
// EXPERIMENTAL
// Intentionally not documented yet while we do more testing and determine
// a good default value.
conntrackCacheTimeout := c.GetDuration("firewall.conntrack.routine_cache_timeout", 0)
if routines > 1 && !c.IsSet("firewall.conntrack.routine_cache_timeout") {
// Use a different default if we are running with multiple routines
conntrackCacheTimeout = 1 * time.Second
}
if conntrackCacheTimeout > 0 {
l.Info("Using routine-local conntrack cache", "duration", conntrackCacheTimeout)
}
var tun overlay.Device var tun overlay.Device
if !configTest { if !configTest {
c.CatchHUP(ctx) c.CatchHUP(ctx)
@@ -290,7 +281,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
version: buildVersion, version: buildVersion,
relayManager: NewRelayManager(ctx, l, hostMap, c), relayManager: NewRelayManager(ctx, l, hostMap, c),
punchy: punchy, punchy: punchy,
ConntrackCacheTimeout: conntrackCacheTimeout,
CpuAffinity: cpuAffinity, CpuAffinity: cpuAffinity,
PinThreads: pinThreads, PinThreads: pinThreads,
l: l, l: l,
@@ -313,6 +303,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
handshakeManager.f = ifce handshakeManager.f = ifce
go handshakeManager.Run(ctx) go handshakeManager.Run(ctx)
go ifce.firewall.Conntrack.Run(ctx)
punchy.Start(ctx, ifce, hostMap, lightHouse) punchy.Start(ctx, ifce, hostMap, lightHouse)
} }
+74
View File
@@ -1,12 +1,21 @@
package nebula package nebula
import ( import (
"fmt"
"net/netip"
"os"
"path/filepath"
"testing" "testing"
"time"
"github.com/slackhq/nebula/cert"
cert_test "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
) )
func TestParseCpuAffinity(t *testing.T) { func TestParseCpuAffinity(t *testing.T) {
@@ -49,3 +58,68 @@ func TestParseCpuAffinity(t *testing.T) {
assert.Nil(t, parseCpuAffinity(newConfig([]any{1 << 20}), l, 1)) assert.Nil(t, parseCpuAffinity(newConfig([]any{1 << 20}), l, 1))
} }
} }
// TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running
// when it hands back a Control to stop them with.
//
// A config test gets no Control, so anything it started had nothing to stop it: the lighthouse
// query worker, and a hostname resolver per dns named static host, ran for the life of the
// process. That matters to every embedder that validates a config in process rather than by
// exec'ing, dnclient and the apple clients included, because they do it on each config load and
// the leak accumulates.
func TestMain_ConfigTestReleasesItsGoroutines(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
l := test.NewLogger()
dir := t.TempDir()
before := time.Now().Add(-time.Hour)
after := time.Now().Add(time.Hour)
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
_, _, keyPEM, certPEM := cert_test.NewTestCert(
cert.Version2, cert.Curve_CURVE25519, ca, caKey, "config-test", before, after, networks, nil, nil)
caPath := filepath.Join(dir, "ca.pem")
certPath := filepath.Join(dir, "cert.pem")
keyPath := filepath.Join(dir, "key.pem")
require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
// A static host by address, not by name: the query worker is the goroutine under test and a
// hostname would drag a real dns lookup into a unit test.
configBody := fmt.Sprintf(`
pki:
ca: %s
cert: %s
key: %s
static_host_map:
"10.0.0.2": ["192.0.2.1:4242"]
lighthouse:
hosts:
- "10.0.0.2"
listen:
host: 127.0.0.1
port: 0
tun:
disabled: true
firewall:
outbound:
- port: any
proto: any
host: any
inbound:
- port: any
proto: any
host: any
`, caPath, certPath, keyPath)
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
c := config.NewC(l)
require.NoError(t, c.Load(dir))
ctrl, err := Main(c, true, "config-test", l, nil)
require.NoError(t, err)
require.Nil(t, ctrl, "a config test hands back nothing to stop, so it must stop itself")
}
+39
View File
@@ -0,0 +1,39 @@
//go:build !android
package nicfinder
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/netip"
)
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
ifaces, err := net.Interfaces()
if err != nil {
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
}
var out []netip.Addr
var errs []error
for _, i := range ifaces {
if !allowName(ctx, l, filter, i.Name) {
continue
}
addrs, err := i.Addrs()
if err != nil {
errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err))
continue
}
for _, rawAddr := range addrs {
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
out = append(out, addr)
}
}
}
return out, errors.Join(errs...)
}
+104
View File
@@ -0,0 +1,104 @@
//go:build android
package nicfinder
import (
"context"
"errors"
"fmt"
"log/slog"
"net/netip"
"os"
"github.com/DefinedNet/netlink"
"github.com/DefinedNet/netlink/nl"
"golang.org/x/sys/unix"
)
// Android denies bind() on netlink route sockets and RTM_GETLINK, so net.Interfaces fails with EACCES.
// Enumerate with an RTM_GETADDR dump on an unbound socket instead.
// localInterfaces returns every interface that has an address, with its addresses. It lists
// addresses over a netlink socket that is never bound and names each index with SIOCGIFNAME, the
// same operations Bionic's getifaddrs uses for app UIDs, which the sandbox permits.
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
var errs []error
addrs, err := netlinkAddrList()
if err != nil {
if !errors.Is(err, netlink.ErrDumpInterrupted) {
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
}
// The address table kept changing under the dump and netlink gave up retrying. What it
// returned is still worth advertising.
errs = append(errs, fmt.Errorf("local addresses may be incomplete: %w", err))
}
// Any socket will do for SIOCGIFNAME.
sfd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
defer unix.Close(sfd)
var out []netip.Addr
nameAllowed := make(map[int]bool) // by interface index, once named
for _, a := range addrs {
allow, named := nameAllowed[a.LinkIndex]
if (named && !allow) || a.IPNet == nil {
continue
}
addr, ok := allowedAddr(ctx, l, filter, a.IPNet)
if !ok {
continue
}
if !named {
name, err := interfaceNameByIndex(sfd, a.LinkIndex)
// ENODEV means the interface went away after the dump. Any other failure is reported
// and the interface skipped, so the rest are still returned.
if err != nil && !errors.Is(err, unix.ENODEV) {
errs = append(errs, fmt.Errorf("failed to resolve name of interface %d: %w", a.LinkIndex, err))
}
allow = err == nil && allowName(ctx, l, filter, name)
nameAllowed[a.LinkIndex] = allow
}
if allow {
out = append(out, addr)
}
}
return out, errors.Join(errs...)
}
// netlinkAddrList is netlink.AddrList over a socket that is never bound. The kernel autobinds it
// on the first send, which the sandbox permits.
func netlinkAddrList() ([]netlink.Addr, error) {
fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
s, err := nl.NewNetlinkSocketFromFd(fd)
if err != nil {
unix.Close(fd)
return nil, err
}
h := netlink.NewHandleFromSockets(
map[int]*nl.SocketHandle{unix.NETLINK_ROUTE: {Socket: s}},
netlink.HandleOptions{RetryInterrupted: true},
)
defer h.Close()
if err := h.SetSocketTimeout(netlink.GetSocketTimeout()); err != nil {
return nil, err
}
return h.AddrList(nil, netlink.FAMILY_ALL)
}
// interfaceNameByIndex resolves an interface index with SIOCGIFNAME.
func interfaceNameByIndex(fd int, index int) (string, error) {
var ifr unix.Ifreq
ifr.SetUint32(uint32(index)) // ifr_ifindex is in the ifr_ifru union
if err := unix.IoctlIfreq(fd, unix.SIOCGIFNAME, &ifr); err != nil {
return "", err
}
return ifr.Name(), nil
}
+59
View File
@@ -0,0 +1,59 @@
package nicfinder
import (
"context"
"log/slog"
"net"
"net/netip"
"github.com/slackhq/nebula/logging"
)
// Filter decides which interfaces, by name, and which of their addresses Find returns.
type Filter interface {
AllowName(string) bool
Allow(netip.Addr) bool
}
// Find returns the addresses of this host's network interfaces that allowList accepts, with
// loopback and link-local addresses left out. When only some interfaces could be read it returns
// their addresses alongside an error for the rest. allowList must not be nil; it is called as
// given, so a nil pointer of a type whose methods accept a nil receiver is fine.
func Find(ctx context.Context, l *slog.Logger, allowList Filter) ([]netip.Addr, error) {
return localAddrs(ctx, l, allowList)
}
// allowName asks filter about an interface name and traces the answer.
func allowName(ctx context.Context, l *slog.Logger, filter Filter, name string) bool {
allow := filter.AllowName(name)
if l.Enabled(ctx, logging.LevelTrace) {
l.Log(ctx, logging.LevelTrace, "localAllowList.AllowName", "interfaceName", name, "allow", allow)
}
return allow
}
// allowedAddr converts one address a platform reports and says whether to advertise it: it must
// be a *net.IPNet or *net.IPAddr carrying an IP, not be loopback or link-local, and be accepted by
// filter, whose answer is traced. IPv4 comes back unmapped.
func allowedAddr(ctx context.Context, l *slog.Logger, filter Filter, rawAddr net.Addr) (netip.Addr, bool) {
var ip net.IP
switch v := rawAddr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
addr, ok := netip.AddrFromSlice(ip)
if !ok {
return netip.Addr{}, false
}
addr = addr.Unmap()
if addr.IsLoopback() || addr.IsLinkLocalUnicast() {
return netip.Addr{}, false
}
isAllowed := filter.Allow(addr)
if l.Enabled(ctx, logging.LevelTrace) {
l.Log(ctx, logging.LevelTrace, "localAllowList.Allow", "localAddr", addr, "allowed", isAllowed)
}
return addr, isAllowed
}
+117
View File
@@ -0,0 +1,117 @@
package nicfinder
import (
"context"
"net"
"net/netip"
"regexp"
"testing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
)
type allowAll struct{}
func (allowAll) Allow(netip.Addr) bool { return true }
func (allowAll) AllowName(string) bool { return true }
type testAllowList struct {
bannedName *regexp.Regexp
bannedAddr netip.Addr
}
func (l *testAllowList) Allow(addr netip.Addr) bool {
return addr != l.bannedAddr
}
func (l *testAllowList) AllowName(n string) bool {
return !l.bannedName.MatchString(n)
}
func newTestAllowList(bannedName string) *testAllowList {
return &testAllowList{
bannedName: regexp.MustCompile(bannedName),
}
}
func TestAllowedAddr(t *testing.T) {
ctx, l := context.Background(), test.NewLogger()
allowed := func(filter Filter, addrs []net.Addr) []netip.Addr {
var out []netip.Addr
for _, rawAddr := range addrs {
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
out = append(out, addr)
}
}
return out
}
addrs := []net.Addr{
&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)},
&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
&net.IPNet{IP: net.ParseIP("10.0.0.5"), Mask: net.CIDRMask(24, 32)},
&net.IPNet{IP: net.ParseIP("169.254.10.10"), Mask: net.CIDRMask(16, 32)},
&net.IPNet{IP: net.ParseIP("fe80::1"), Mask: net.CIDRMask(64, 128)},
&net.IPAddr{IP: net.ParseIP("fd00::5")},
&net.IPNet{},
otherAddr{},
nil,
}
// Loopback, link local and anything without an IP are dropped, and IPv4 comes back unmapped
// even though net.ParseIP hands over a 16 byte slice.
assert.Equal(t, []netip.Addr{
netip.MustParseAddr("10.0.0.5"),
netip.MustParseAddr("fd00::5"),
}, allowed(allowAll{}, addrs))
// The filter sees the unmapped form and can drop a single address.
al := newTestAllowList("none")
al.bannedAddr = netip.MustParseAddr("10.0.0.5")
assert.Equal(t, []netip.Addr{netip.MustParseAddr("fd00::5")}, allowed(al, addrs))
}
// otherAddr is a net.Addr that is neither *net.IPNet nor *net.IPAddr. netlink.Addr is one such
// type: it satisfies net.Addr through an embedded *net.IPNet, and passing it directly once
// silently produced no addresses.
type otherAddr struct{}
func (otherAddr) Network() string { return "other" }
func (otherAddr) String() string { return "other" }
// hostFilter counts AllowName calls and can reject every name or every address.
type hostFilter struct {
rejectNames bool
rejectAddrs bool
asked map[string]int
}
func (f *hostFilter) Allow(netip.Addr) bool { return !f.rejectAddrs }
func (f *hostFilter) AllowName(name string) bool {
f.asked[name]++
return !f.rejectNames
}
// Runs this platform's localAddrs against the host and checks it honors the whole filter.
func TestLocalAddrsAppliesFilter(t *testing.T) {
ctx, l := context.Background(), test.NewLogger()
f := &hostFilter{asked: map[string]int{}}
all, err := localAddrs(ctx, l, f)
if len(f.asked) == 0 {
t.Skipf("no interfaces to enumerate here: %v", err)
}
for name, n := range f.asked {
assert.Equal(t, 1, n, "AllowName asked %d times for %s", n, name)
}
// Loopback and link-local never come back, whatever the filter says.
for _, addr := range all {
assert.False(t, addr.IsLoopback() || addr.IsLinkLocalUnicast(), "%s", addr)
}
none, _ := localAddrs(ctx, l, &hostFilter{rejectNames: true, asked: map[string]int{}})
assert.Empty(t, none, "addresses got past a filter that rejects every interface name")
none, _ = localAddrs(ctx, l, &hostFilter{rejectAddrs: true, asked: map[string]int{}})
assert.Empty(t, none, "addresses got past a filter that rejects every address")
}
+19 -2
View File
@@ -354,7 +354,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
// and cannot drift into misreading an unknown protocol (SCTP, GRE, etc.) as a forged transport. // and cannot drift into misreading an unknown protocol (SCTP, GRE, etc.) as a forged transport.
proto, offset, isFragment, anyFragment, err := iputil.IPv6FindUpperProtocol(data) proto, offset, isFragment, anyFragment, err := iputil.IPv6FindUpperProtocol(data)
if err != nil { if err != nil {
return ErrIPv6PacketTooShort return err
} }
fp.Protocol = proto fp.Protocol = proto
@@ -477,7 +477,7 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounte
return return
} }
dropReason := f.firewall.Drop(rxc.fwPacket.Packet, true, hostinfo, f.pki.GetCAPool(), rxc.ctCache.Get()) dropReason := f.allowInbound(hostinfo, rxc)
if dropReason != nil { if dropReason != nil {
f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, rxc.nb, rxc.scratch, rxc.q) f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, rxc.nb, rxc.scratch, rxc.q)
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
@@ -492,6 +492,23 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounte
} }
} }
// allowInbound runs the inbound firewall check on rxc.fwPacket. It skips the check when the packet belongs to
// the same flow from the same host as the last one allowed in this batch.
func (f *Interface) allowInbound(hostinfo *HostInfo, rxc *rxContext) error {
fp := rxc.fwPacket.Packet
if hostinfo == rxc.lastAllowedHost && fp == rxc.lastAllowed {
return nil
}
if err := f.firewall.Drop(fp, true, hostinfo, f.pki.GetCAPool()); err != nil {
return err
}
rxc.lastAllowed = fp
rxc.lastAllowedHost = hostinfo
return nil
}
func (f *Interface) maybeSendRecvError(endpoint netip.AddrPort, index uint32) { func (f *Interface) maybeSendRecvError(endpoint netip.AddrPort, index uint32) {
if f.sendRecvErrorConfig.ShouldRecvError(endpoint) { if f.sendRecvErrorConfig.ShouldRecvError(endpoint) {
f.sendRecvError(endpoint, index) f.sendRecvError(endpoint, index)
+57 -7
View File
@@ -6,12 +6,16 @@ import (
"net" "net"
"net/netip" "net/netip"
"testing" "testing"
"time"
"github.com/gaissmai/bart"
"github.com/google/gopacket" "github.com/google/gopacket"
"github.com/google/gopacket/layers" "github.com/google/gopacket/layers"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"golang.org/x/net/ipv4" "golang.org/x/net/ipv4"
@@ -117,7 +121,7 @@ func Test_newPacket_v6(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
err = newPacket(buffer.Bytes(), true, p) err = newPacket(buffer.Bytes(), true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort) require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
// A v6 packet with a hop-by-hop extension // A v6 packet with a hop-by-hop extension
// ICMPv6 Payload (Echo Request) // ICMPv6 Payload (Echo Request)
@@ -151,12 +155,12 @@ func Test_newPacket_v6(t *testing.T) {
// A full IPv6 header and 1 byte in the first extension, but missing // A full IPv6 header and 1 byte in the first extension, but missing
// the length byte. // the length byte.
err = newPacket(buffer.Bytes()[:41], true, p) err = newPacket(buffer.Bytes()[:41], true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort) require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
// A full IPv6 header plus 1 full extension, but only 1 byte of the // A full IPv6 header plus 1 full extension, but only 1 byte of the
// next layer, missing length byte // next layer, missing length byte
err = newPacket(buffer.Bytes()[:49], true, p) err = newPacket(buffer.Bytes()[:49], true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort) require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
err = nil err = nil
// A good ICMP packet // A good ICMP packet
@@ -354,12 +358,12 @@ func Test_newPacket_v6(t *testing.T) {
// Ensure buffer bounds checking during processing, a truncated AH header can't reach the payload // Ensure buffer bounds checking during processing, a truncated AH header can't reach the payload
err = newPacket(b[:41], true, p) err = newPacket(b[:41], true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort) require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
// Invalid AH header // Invalid AH header
b = buffer.Bytes() b = buffer.Bytes()
err = newPacket(b, true, p) err = newPacket(b, true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort) require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
} }
func Test_newPacket_ipv6Fragment(t *testing.T) { func Test_newPacket_ipv6Fragment(t *testing.T) {
@@ -466,7 +470,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
// Too short of a fragment packet // Too short of a fragment packet
err = newPacket(secondFrag[:len(secondFrag)-10], false, p) err = newPacket(secondFrag[:len(secondFrag)-10], false, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort) require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
} }
func BenchmarkParseV6(b *testing.B) { func BenchmarkParseV6(b *testing.B) {
@@ -707,7 +711,7 @@ func Test_newPacket_v6ExtHeaderPastBuffer(t *testing.T) {
pkt[40] = byte(layers.IPProtocolSCTP) // Dest Options next header = SCTP pkt[40] = byte(layers.IPProtocolSCTP) // Dest Options next header = SCTP
pkt[41] = 255 // declared length (255+1)*8 = 2048, past the 48 byte buffer pkt[41] = 255 // declared length (255+1)*8 = 2048, past the 48 byte buffer
require.ErrorIs(t, newPacket(pkt, true, p), ErrIPv6PacketTooShort) require.ErrorIs(t, newPacket(pkt, true, p), iputil.ErrIPv6CouldNotFindPayload)
} }
// Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized // Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized
@@ -841,3 +845,49 @@ func Test_newPacket_parsedFields(t *testing.T) {
assert.True(t, p.Fragment) assert.True(t, p.Fragment)
assert.True(t, p.FragAny) assert.True(t, p.FragAny)
} }
func TestInterface_allowInbound(t *testing.T) {
l := test.NewLoggerWithOutput(&bytes.Buffer{})
myNetworks := new(bart.Lite)
myNetworks.Insert(netip.MustParsePrefix("192.0.2.1/24"))
me := &dummyCert{name: "me", networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")}}
peer := &cert.CachedCertificate{Certificate: &dummyCert{name: "peer", networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")}}}
h := &HostInfo{ConnectionState: &ConnectionState{peerCert: peer}, vpnAddrs: []netip.Addr{netip.MustParseAddr("192.0.2.2")}}
h.buildNetworks(myNetworks, peer.Certificate)
allowAll := NewFirewall(l, time.Minute, time.Minute, time.Minute, me)
require.NoError(t, allowAll.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
allowNone := NewFirewall(l, time.Minute, time.Minute, time.Minute, me)
f := &Interface{firewall: allowAll, pki: &PKI{}, l: l}
rxc := &rxContext{fwPacket: &firewall.ParsedPacket{}, hostmapCache: map[uint32]*HostInfo{}}
flow := firewall.Packet{
LocalAddr: netip.MustParseAddr("192.0.2.1"),
RemoteAddr: netip.MustParseAddr("192.0.2.2"),
LocalPort: 443,
RemotePort: 55000,
Protocol: iputil.IPProtocolUDP,
}
rxc.fwPacket.Packet = flow
require.NoError(t, f.allowInbound(h, rxc))
// With a firewall that allows nothing, the rest of the flow's run in this batch still passes
f.firewall = allowNone
require.NoError(t, f.allowInbound(h, rxc))
// A different flow is checked
other := flow
other.RemotePort++
rxc.fwPacket.Packet = other
require.ErrorIs(t, f.allowInbound(h, rxc), ErrNoMatchingRule)
// So is the same flow from a different host
rxc.fwPacket.Packet = flow
h2 := &HostInfo{ConnectionState: h.ConnectionState, vpnAddrs: h.vpnAddrs}
h2.buildNetworks(myNetworks, peer.Certificate)
require.ErrorIs(t, f.allowInbound(h2, rxc), ErrNoMatchingRule)
// And the same flow in the next batch
rxc.endBatch()
require.ErrorIs(t, f.allowInbound(h, rxc), ErrNoMatchingRule)
}
+60 -2
View File
@@ -29,6 +29,9 @@ type tun struct {
Routes atomic.Pointer[[]Route] Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]] routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr *netroute.LinkAddr linkAddr *netroute.LinkAddr
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
// and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool
l *slog.Logger l *slog.Logger
} }
@@ -149,8 +152,48 @@ func (t *tun) deviceBytes() (o [16]byte) {
return return
} }
func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { // newTunFromFd adopts a utun the host already created and configured, which is how a darwin
return nil, fmt.Errorf("newTunFromFd not supported in Darwin") // network extension is handed its device. Everything about moving packets is shared with newTun,
// only the setup differs: the host owns addressing and routing here.
func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
if err := unix.SetNonblock(deviceFd, true); err != nil {
// We own the fd from the moment it is handed to us
_ = unix.Close(deviceFd)
return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err)
}
file := os.NewFile(uintptr(deviceFd), "/dev/tun")
t := &tun{
f: file,
Device: utunNameFromFd(deviceFd),
vpnNetworks: vpnNetworks,
DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
hostOwned: true,
l: l,
}
if err := t.reload(c, true); err != nil {
_ = file.Close()
return nil, err
}
c.RegisterReloadCallback(func(c *config.C) {
if err := t.reload(c, false); err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
}
})
return t, nil
}
// utunNameFromFd asks the socket what interface it is, for logs. A blank name is not worth
// failing a tunnel over, so an error just leaves it empty.
func utunNameFromFd(fd int) string {
name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME)
if err != nil {
return ""
}
return name
} }
func (t *tun) Close() error { func (t *tun) Close() error {
@@ -161,6 +204,12 @@ func (t *tun) Close() error {
} }
func (t *tun) Activate() error { func (t *tun) Activate() error {
// The host handed us a configured device. Its addresses, mtu and routes come from the network
// settings it applied, and a sandboxed extension cannot change them anyway.
if t.hostOwned {
return nil
}
devName := t.deviceBytes() devName := t.deviceBytes()
s, err := unix.Socket( s, err := unix.Socket(
@@ -374,6 +423,11 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
} }
func (t *tun) addRoutes(logErrors bool) error { func (t *tun) addRoutes(logErrors bool) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -403,6 +457,10 @@ func (t *tun) addRoutes(logErrors bool) error {
} }
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
if t.hostOwned {
return nil
}
for _, r := range routes { for _, r := range routes {
if !r.Install { if !r.Install {
continue continue
+1 -1
View File
@@ -15,12 +15,12 @@ import (
"time" "time"
"unsafe" "unsafe"
"github.com/DefinedNet/netlink"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )