Compare commits

...
Author SHA1 Message Date
JackDoan b4158b3855 Skip the inbound firewall for the rest of a flow's run within one receive batch 2026-10-01 12:01:00 -05:00
JackDoan 905d01aee5 conntrack: lock-free sync.Map keyed by firewall.Packet with a coarse clock, drop the per-routine cache 2026-10-01 12:00:52 -05:00
Nate Brown d5062dacf0 lighthouse: Never block in QueryServer, drop queries when the channel is full (#1910) 2026-09-30 15:26:34 -05:00
Jack Doan 7cfa47d569 tweaks to error handling in iputil.IPv6FindUpperProtocol (#1893) 2026-09-22 15:45:01 -05:00
Caleb JasikandNate Brown e20569ce51 Let the darwin tun adopt an fd the host already configured (#1890)
newTunFromFd was a stub on darwin, so an embedder handed a utun by the OS had
no way to use it. A darwin network extension is given exactly that:
NEPacketTunnelProvider creates the device and applies addressing, mtu and
routes from its network settings, and a sandboxed extension cannot change them
afterwards. Adopting that fd means everything about moving packets is shared
with newTun and only the setup differs, so the tun tracks whether the host owns
it and leaves the system routing table and interface config alone when it does.

Co-authored-by: Nate Brown <nbrown.us@gmail.com>
2026-09-22 11:54:30 -05:00
Caleb JasikandNate Brown 9651ffdb0d Cancel the context when Main hands back no Control (#1889)
The goroutines Main starts stop only when its context does, and only a caller
holding the Control can arrange that. Cancelling solely on error left a config
test, which returns no Control and no error, with the lighthouse query worker
and a hostname resolver per dns named static host running for the life of the
process. Every embedder that validates a config in process rather than by
exec'ing pays for that on each config load, dnclient and the apple clients
included.

Co-authored-by: Nate Brown <nbrown.us@gmail.com>
2026-09-22 11:52:28 -05:00
Jack Doan 3a45c01656 Advertise underlay addresses on Android (#1881) 2026-09-21 14:59:56 -05:00
Jack Doan 019cc46a7f smoke: Prove windows honors NlMtu with a 9000 byte ping (#1873) 2026-09-21 12:59:31 -05:00
Nate Brown cf3f315cd3 IPv6FindUpperProtocol: fail closed when the extension header walk runs out of budget (#1883) 2026-09-21 12:09:51 -05:00
Wade Simmons 3fc627e08d Add script to backport a merged PR to a release branch (#1851)
* Add script to backport a merged PR to a release branch

.github/backport.sh cherry-picks a merged PR's commit onto a release
branch and opens a backport PR mirroring the shape of #1842.

- Infers the target release from the PR's milestone (v1.12.0 -> release-1.11);
  an explicit target-version argument overrides it.
- Requires a clean working tree (untracked files allowed) before starting.
- Prompts with the exact push and gh pr create commands before running them.
- Sets the new PR's milestone to the highest open v<target>.* milestone.
- Removes the needs-backport label from the original PR once the backport opens.
- Supports --continue to resume after resolving cherry-pick conflicts.

* cleanup

* just fetch all of origin

We need to ensure we have MERGE_SHA available too
2026-09-14 16:14:43 -04:00
32 changed files with 1137 additions and 528 deletions

No files matched your search

+142
View File
@@ -0,0 +1,142 @@
#!/usr/bin/env bash
#
# Backport a merged PR to a release branch.
#
# Cherry-picks the merge commit of a merged PR onto release-1.11 and opens a
# backport PR, mirroring the shape of #1842.
#
# Usage: ./backport.sh <pr-number> [target-version] [--continue]
# pr-number The merged PR to backport
# target-version Release version to target (default: 1.11)
# --continue Resume after resolving cherry-pick conflicts: assumes the
# fixes are staged, runs `git cherry-pick --continue`, and
# proceeds to push and open the backport PR.
#
# Requires: gh (authenticated), git, jq.
set -euo pipefail
PR_NUMBER=${1:?usage: backport.sh <pr-number> [target-version] [--continue]}
shift
CONTINUE=0
TARGET_VERSION=""
for arg in "$@"; do
case "$arg" in
--continue) CONTINUE=1 ;;
*) TARGET_VERSION=$arg ;;
esac
done
for cmd in gh git jq; do
command -v "$cmd" >/dev/null || { echo "error: $cmd is required" >&2; exit 1; }
done
# Pull the source PR's metadata. Refuse to backport a PR that never merged.
pr_json=$(gh pr view "$PR_NUMBER" --json title,body,mergedAt,mergeCommit,milestone)
PR_TITLE=$(jq -r '.title' <<<"$pr_json")
PR_BODY=$(jq -r '.body' <<<"$pr_json")
MERGE_SHA=$(jq -r '.mergeCommit.oid // empty' <<<"$pr_json")
PR_MILESTONE=$(jq -r '.milestone.title // empty' <<<"$pr_json")
if [ "$(jq -r '.mergedAt // empty' <<<"$pr_json")" = "" ] || [ -z "$MERGE_SHA" ]; then
echo "error: PR #${PR_NUMBER} is not merged (no merge commit to cherry-pick)" >&2
exit 1
fi
# Default the target to one minor below the PR's milestone: a PR landing in
# v1.12.0 backports to release-1.11. An explicit target-version arg overrides.
if [ -z "$TARGET_VERSION" ]; then
if [[ $PR_MILESTONE =~ ^v?([0-9]+)\.([0-9]+) ]]; then
TARGET_VERSION="${BASH_REMATCH[1]}.$(( BASH_REMATCH[2] - 1 ))"
else
echo "error: PR #${PR_NUMBER} has no v<major>.<minor>.* milestone to infer the target from." >&2
echo "Pass the target version explicitly, e.g. $0 ${PR_NUMBER} 1.11" >&2
exit 1
fi
fi
TARGET_BRANCH="release-${TARGET_VERSION}"
# Slug from the PR title: lower-case, non-alphanumerics to dashes, trimmed.
slug=$(printf '%s' "$PR_TITLE" \
| tr '[:upper:]' '[:lower:]' \
| sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//' \
| cut -c1-50 \
| sed -E 's/-+$//')
BRANCH="backport-${TARGET_VERSION//./-}-${slug}"
if [ "$CONTINUE" -eq 1 ]; then
# Resuming: the branch already exists and a cherry-pick is mid-conflict.
current=$(git rev-parse --abbrev-ref HEAD)
if [ "$current" != "$BRANCH" ]; then
echo "error: --continue expects to be on ${BRANCH}, but HEAD is ${current}" >&2
exit 1
fi
echo "Resuming backport of #${PR_NUMBER} on ${BRANCH}"
# Conflicts assumed resolved and staged; core.editor=true keeps the commit message.
git -c core.editor=true cherry-pick --continue
else
# A fresh run switches branches and cherry-picks, so tracked changes must be
# clean. Untracked files are fine, and --continue is exempt (its resolved
# conflicts are meant to be staged).
if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
echo "error: working tree has uncommitted changes to tracked files; commit or stash them first." >&2
exit 1
fi
echo "Backporting #${PR_NUMBER} (${MERGE_SHA}) onto ${TARGET_BRANCH} as ${BRANCH}"
git fetch origin
git checkout -b "$BRANCH" "origin/${TARGET_BRANCH}"
# -m 1 handles a real merge commit; plain cherry-pick handles a squash merge.
if ! { git cherry-pick -x -m 1 "$MERGE_SHA" 2>/dev/null || git cherry-pick -x "$MERGE_SHA"; }; then
echo "error: ${MERGE_SHA} did not cherry-pick cleanly onto ${TARGET_BRANCH}." >&2
echo "Resolve the conflicts, 'git add' them, then re-run:" >&2
echo " $0 ${PR_NUMBER} ${TARGET_VERSION} --continue" >&2
exit 1
fi
fi
title="backport v${TARGET_VERSION}: ${PR_TITLE}"
body=$(printf 'Backport from #%s to release v%s\n\n---\n\n%s' \
"$PR_NUMBER" "$TARGET_VERSION" "$PR_BODY")
# Highest open milestone matching v<version>.* (e.g. v1.11.1), if any.
MILESTONE=$(gh api "repos/{owner}/{repo}/milestones?state=open" \
--jq ".[].title | select(startswith(\"v${TARGET_VERSION}.\"))" \
| sort -V | tail -n1)
milestone_args=()
if [ -n "$MILESTONE" ]; then
milestone_args=(--milestone "$MILESTONE")
else
echo "warning: no open milestone matching v${TARGET_VERSION}.* found; PR will have no milestone" >&2
fi
echo
echo "About to run:"
printf ' git push -u --force-with-lease origin %q\n' "$BRANCH"
printf ' gh pr create --base %q --head %q --title %q --body %q' \
"$TARGET_BRANCH" "$BRANCH" "$title" "$body"
[ -n "$MILESTONE" ] && printf ' --milestone %q' "$MILESTONE"
printf '\n'
echo
read -r -p "Open this pull request? [y/N] " reply
case "$reply" in
[yY] | [yY][eE][sS]) ;;
*) echo "Aborted. The cherry-pick is on local branch ${BRANCH}; push it manually if desired."; exit 0 ;;
esac
git push -u --force-with-lease origin "$BRANCH"
gh pr create \
--base "$TARGET_BRANCH" \
--head "$BRANCH" \
--title "$title" \
--body "$body" \
${milestone_args[@]+"${milestone_args[@]}"}
# The backport now exists, so drop the label that flagged this PR for one.
gh pr edit "$PR_NUMBER" --remove-label needs-backport
# Switch back to the original branch we were on before doing the backport
git checkout -
+14
View File
@@ -263,6 +263,20 @@ try {
}
Write-Host "OK: WSL peer -> windows lighthouse over v6"
# The NlMtu check above proves the value was written, not that windows honors it. A payload this size only
# crosses the tunnel as fragments cut at NlMtu on the way out of the adapter. Left at the adapter default it
# reaches nebula whole, overflows the udp.MTU (9001) write buffer and is dropped, so the ping never answers.
# Anything smaller would ride out as one oversized datagram and survive on IP fragmentation of the underlay.
$BigPayload = 9000
foreach ($target in @(@{ Family = 'v4'; Ip = $Ip2 }, @{ Family = 'v6'; Ip = $Ip6_2 })) {
Wait-Until -TimeoutSec 30 -What "$($target.Family) ping with a $BigPayload byte payload from windows lighthouse to WSL peer ($($target.Ip))" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the large $($target.Family) ping succeeded" }
$null = & ping.exe -n 1 -w 1000 -l $BigPayload $target.Ip
$LASTEXITCODE -eq 0
}
Write-Host "OK: windows lighthouse -> WSL peer, $BigPayload byte $($target.Family) payload"
}
Write-Host ''
Write-Host 'All smoke checks passed.'
}
+3
View File
@@ -242,11 +242,14 @@ test-cov-html:
go test -coverprofile=coverage.out
go tool cover -html=coverage.out
# The package builds only compile. The final line links an android binary so a linker-only failure
# cannot pass CI.
build-test-mobile:
GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build -o /dev/null ${NEBULA_CMD_PATH}
bench:
go test -bench=.
+3
View File
@@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/overlay/overlaytest"
@@ -21,6 +22,8 @@ func newTestLighthouse() *LightHouse {
l: test.NewLogger(),
addrMap: map[netip.Addr]*RemoteList{},
queryChan: make(chan netip.Addr, 10),
// Nothing drains queryChan here, so an 11th query is dropped.
queryDropped: metrics.NilCounter{},
}
lighthouses := []netip.Addr{}
staticList := map[netip.Addr]struct{}{}
+2 -2
View File
@@ -410,8 +410,8 @@ logging:
#try_interval: 100ms
#retries: 20
# query_buffer is the size of the buffer channel for querying lighthouses
#query_buffer: 64
# query_buffer is the size of the buffer channel for querying lighthouses, drops are recorded in the lighthouse.query.dropped metric.
#query_buffer: 1024
# trigger_buffer is the size of the buffer channel for quickly sending handshakes
# after receiving the response for lighthouse queries
+98 -111
View File
@@ -14,6 +14,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/gaissmai/bart"
@@ -28,16 +29,23 @@ type FirewallInterface interface {
AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
}
// conn is a conntrack entry. Only expires changes after it is stored, a rules version change
// stores a replacement entry instead.
type conn struct {
Expires time.Time // Time when this conntrack entry will expire
expires atomic.Uint32 // conntrack clock second when this entry expires, see FirewallConntrack.now
// record why the original connection passed the firewall, so we can re-validate
// after ruleset changes. Note, rulesVersion is a uint16 so that these two
// fields pack for free after the uint32 above
// after ruleset changes
incoming bool
rulesVersion uint16
}
func newConn(expires uint32, incoming bool, rulesVersion uint16) *conn {
c := &conn{incoming: incoming, rulesVersion: rulesVersion}
c.expires.Store(expires)
return c
}
// TODO: need conntrack max tracked connections handling
type Firewall struct {
Conntrack *FirewallConntrack
@@ -79,11 +87,69 @@ type firewallMetrics struct {
droppedNoRule metrics.Counter
}
// FirewallConntrack is safe for concurrent use by every routine without a lock.
type FirewallConntrack struct {
sync.Mutex
conns sync.Map // firewall.Packet -> *conn
count atomic.Int64 // entries in conns, which sync.Map can't tell us
now atomic.Uint32 // the conntrack clock: whole seconds since epoch, kept current by Run
epoch time.Time
}
Conns map[firewall.Packet]*conn
TimerWheel *TimerWheel[firewall.Packet]
func newFirewallConntrack() *FirewallConntrack {
return &FirewallConntrack{epoch: time.Now()}
}
// Run advances the conntrack clock every second and sweeps out expired entries every minute,
// until ctx is done. Reading a shared clock is much cheaper than asking the OS on every packet.
func (ct *FirewallConntrack) Run(ctx context.Context) {
t := time.NewTicker(time.Second)
defer t.Stop()
var nextSweep uint32
for {
select {
case <-ctx.Done():
return
case <-t.C:
// Measured from epoch rather than counted, so a late tick can't slow the clock down
now := uint32(time.Since(ct.epoch) / time.Second)
ct.now.Store(now)
if now >= nextSweep {
ct.sweep(now)
nextSweep = now + 60
}
}
}
}
// sweep deletes entries that expired at or before now.
func (ct *FirewallConntrack) sweep(now uint32) {
ct.conns.Range(func(k, v any) bool {
if v.(*conn).expires.Load() <= now {
ct.delete(k.(firewall.Packet), v.(*conn))
}
return true
})
}
func (ct *FirewallConntrack) store(fp firewall.Packet, c *conn) {
if _, replaced := ct.conns.Swap(fp, c); !replaced {
ct.count.Add(1)
}
}
// delete removes fp's entry if it is still c, so a newer entry stored by another routine survives
func (ct *FirewallConntrack) delete(fp firewall.Packet, c *conn) {
if ct.conns.CompareAndDelete(fp, c) {
ct.count.Add(-1)
}
}
// clear deletes every entry
func (ct *FirewallConntrack) clear() {
ct.conns.Range(func(k, v any) bool {
ct.delete(k.(firewall.Packet), v.(*conn))
return true
})
}
// FirewallTable is the entry point for a rule, the evaluation order is:
@@ -132,26 +198,10 @@ type firewallLocalCIDR struct {
LocalCIDR *bart.Lite
}
// NewFirewall creates a new Firewall object. A TimerWheel is created for you from the provided timeouts.
// NewFirewall creates a new Firewall object.
// The certificate provided should be the highest version loaded in memory.
func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c cert.Certificate) *Firewall {
//TODO: error on 0 duration
var tmin, tmax time.Duration
if tcpTimeout < UDPTimeout {
tmin = tcpTimeout
tmax = UDPTimeout
} else {
tmin = UDPTimeout
tmax = tcpTimeout
}
if defaultTimeout < tmin {
tmin = defaultTimeout
} else if defaultTimeout > tmax {
tmax = defaultTimeout
}
routableNetworks := new(bart.Lite)
var assignedNetworks []netip.Prefix
for _, network := range c.Networks() {
@@ -166,10 +216,7 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
}
return &Firewall{
Conntrack: &FirewallConntrack{
Conns: make(map[firewall.Packet]*conn),
TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax),
},
Conntrack: newFirewallConntrack(),
InRules: newFirewallTable(),
OutRules: newFirewallTable(),
TCPTimeout: tcpTimeout,
@@ -423,7 +470,7 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
// Drop returns an error if the packet should be dropped, explaining why. It
// returns nil if the packet should not be dropped.
func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool) error {
// Make sure remote address matches nebula certificate, and determine how to treat it
if h.networks == nil {
// Simple case: Certificate has one address and no unsafe networks
@@ -458,7 +505,7 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
}
// Check if we spoke to this tuple, if we did then allow this packet
if f.inConns(fp, h, caPool, localCache) {
if f.inConns(fp, h, caPool) {
return nil
}
@@ -494,36 +541,19 @@ func (f *Firewall) Destroy() {
}
func (f *Firewall) EmitStats() {
conntrack := f.Conntrack
conntrack.Lock()
conntrackCount := len(conntrack.Conns)
conntrack.Unlock()
metrics.GetOrRegisterGauge("firewall.conntrack.count", nil).Update(int64(conntrackCount))
metrics.GetOrRegisterGauge("firewall.conntrack.count", nil).Update(f.Conntrack.count.Load())
metrics.GetOrRegisterGauge("firewall.rules.version", nil).Update(int64(f.rulesVersion))
metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
}
func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
if localCache != nil {
if _, ok := localCache[fp]; ok {
return true
}
}
func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool) bool {
conntrack := f.Conntrack
conntrack.Lock()
// Purge every time we test
ep, has := conntrack.TimerWheel.Purge()
if has {
f.evict(ep)
}
c, ok := conntrack.Conns[fp]
v, ok := conntrack.conns.Load(fp)
if !ok {
conntrack.Unlock()
return false
}
c := v.(*conn)
expires := conntrack.now.Load() + f.timeout(fp.Protocol)
if c.rulesVersion != f.rulesVersion {
// This conntrack entry was for an older rule set, validate
@@ -543,8 +573,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
"oldRulesVersion", c.rulesVersion,
)
}
delete(conntrack.Conns, fp)
conntrack.Unlock()
conntrack.delete(fp, c)
return false
}
@@ -557,77 +586,35 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
)
}
c.rulesVersion = f.rulesVersion
conntrack.store(fp, newConn(expires, c.incoming, f.rulesVersion))
return true
}
switch fp.Protocol {
case iputil.IPProtocolTCP:
c.Expires = time.Now().Add(f.TCPTimeout)
case iputil.IPProtocolUDP:
c.Expires = time.Now().Add(f.UDPTimeout)
default:
c.Expires = time.Now().Add(f.DefaultTimeout)
}
conntrack.Unlock()
if localCache != nil {
localCache[fp] = struct{}{}
// Most packets land in the same second as the one before, skip writing an unchanged expiry
if c.expires.Load() != expires {
c.expires.Store(expires)
}
return true
}
func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
var timeout time.Duration
c := &conn{}
switch fp.Protocol {
case iputil.IPProtocolTCP:
timeout = f.TCPTimeout
case iputil.IPProtocolUDP:
timeout = f.UDPTimeout
default:
timeout = f.DefaultTimeout
}
conntrack := f.Conntrack
conntrack.Lock()
if _, ok := conntrack.Conns[fp]; !ok {
conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(fp, timeout)
}
// Record which rulesVersion allowed this connection, so we can retest after
// firewall reload
c.incoming = incoming
c.rulesVersion = f.rulesVersion
c.Expires = time.Now().Add(timeout)
conntrack.Conns[fp] = c
conntrack.Unlock()
conntrack := f.Conntrack
conntrack.store(fp, newConn(conntrack.now.Load()+f.timeout(fp.Protocol), incoming, f.rulesVersion))
}
// Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel
// Caller must own the connMutex lock!
func (f *Firewall) evict(p firewall.Packet) {
// Are we still tracking this conn?
conntrack := f.Conntrack
t, ok := conntrack.Conns[p]
if !ok {
return
// timeout returns how many seconds a conntrack entry for proto lives past its last packet
func (f *Firewall) timeout(proto uint8) uint32 {
switch proto {
case iputil.IPProtocolTCP:
return uint32(f.TCPTimeout / time.Second)
case iputil.IPProtocolUDP:
return uint32(f.UDPTimeout / time.Second)
default:
return uint32(f.DefaultTimeout / time.Second)
}
newT := t.Expires.Sub(time.Now())
// Timeout is in the future, re-add the timer
if newT > 0 {
conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(p, newT)
return
}
// This conn is done
delete(conntrack.Conns, p)
}
func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
-69
View File
@@ -1,69 +0,0 @@
package firewall
import (
"context"
"log/slog"
"sync/atomic"
"time"
"github.com/slackhq/nebula/logging"
)
// ConntrackCache is used as a local routine cache to know if a given flow
// has been seen in the conntrack table.
type ConntrackCache map[Packet]struct{}
type ConntrackCacheTicker struct {
cacheV uint64
cacheTick atomic.Uint64
l *slog.Logger
cache ConntrackCache
}
func NewConntrackCacheTicker(ctx context.Context, l *slog.Logger, d time.Duration) *ConntrackCacheTicker {
if d == 0 {
return nil
}
c := &ConntrackCacheTicker{
l: l,
cache: ConntrackCache{},
}
go c.tick(ctx, d)
return c
}
func (c *ConntrackCacheTicker) tick(ctx context.Context, d time.Duration) {
t := time.NewTicker(d)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
c.cacheTick.Add(1)
}
}
}
// Get checks if the cache ticker has moved to the next version before returning
// the map. If it has moved, we reset the map.
func (c *ConntrackCacheTicker) Get() ConntrackCache {
if c == nil {
return nil
}
if tick := c.cacheTick.Load(); tick != c.cacheV {
c.cacheV = tick
if ll := len(c.cache); ll > 0 {
if c.l.Enabled(context.Background(), logging.LevelTrace) {
c.l.Log(context.Background(), logging.LevelTrace, "resetting conntrack cache", "len", ll)
}
c.cache = make(ConntrackCache, ll)
}
}
return c.cache
}
-70
View File
@@ -1,70 +0,0 @@
package firewall
import (
"bytes"
"log/slog"
"strings"
"testing"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
)
// The tests below pin the log format produced by ConntrackCacheTicker.Get
// so changes cannot silently break what operators are grepping for. The
// ticker's internal state (cache + cacheTick) is poked directly to avoid
// racing a goroutine-driven tick in tests.
func newFixedTicker(t *testing.T, l *slog.Logger, cacheLen int) *ConntrackCacheTicker {
t.Helper()
c := &ConntrackCacheTicker{
l: l,
cache: make(ConntrackCache, cacheLen),
}
for i := range cacheLen {
c.cache[Packet{LocalPort: uint16(i) + 1}] = struct{}{}
}
c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path
return c
}
func TestConntrackCacheTicker_Get_TextFormat(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewLoggerWithOutputAndLevel(buf, logging.LevelTrace)
c := newFixedTicker(t, l, 3)
c.Get()
assert.Equal(t, "level=DEBUG-4 msg=\"resetting conntrack cache\" len=3\n", buf.String())
}
func TestConntrackCacheTicker_Get_JSONFormat(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewJSONLoggerWithOutput(buf, logging.LevelTrace)
c := newFixedTicker(t, l, 2)
c.Get()
assert.JSONEq(t, `{"level":"DEBUG-4","msg":"resetting conntrack cache","len":2}`, strings.TrimSpace(buf.String()))
}
func TestConntrackCacheTicker_Get_QuietBelowTrace(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewLoggerWithOutputAndLevel(buf, slog.LevelDebug)
c := newFixedTicker(t, l, 5)
c.Get()
assert.Empty(t, buf.String())
}
func TestConntrackCacheTicker_Get_QuietWhenCacheEmpty(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewLoggerWithOutputAndLevel(buf, logging.LevelTrace)
c := newFixedTicker(t, l, 0)
c.Get()
assert.Empty(t, buf.String())
}
+4
View File
@@ -26,6 +26,10 @@ type Packet struct {
RemotePort uint16
Protocol uint8
Fragment bool
// pad fills what would otherwise be compiler padding. A Packet with no padding hashes as one
// block of memory, which makes it a much faster map key. Always zero.
pad [2]byte
}
func (fp *Packet) Copy() *Packet {
+128 -102
View File
@@ -2,6 +2,7 @@ package nebula
import (
"bytes"
"context"
"errors"
"log/slog"
"math"
@@ -23,39 +24,12 @@ func TestNewFirewall(t *testing.T) {
l := test.NewLogger()
c := &dummyCert{}
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c)
conntrack := fw.Conntrack
assert.NotNil(t, conntrack)
assert.NotNil(t, conntrack.Conns)
assert.NotNil(t, conntrack.TimerWheel)
assert.NotNil(t, fw.Conntrack)
assert.NotNil(t, fw.InRules)
assert.NotNil(t, fw.OutRules)
assert.Equal(t, time.Second, fw.TCPTimeout)
assert.Equal(t, time.Minute, fw.UDPTimeout)
assert.Equal(t, time.Hour, fw.DefaultTimeout)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
fw = NewFirewall(l, time.Second, time.Hour, time.Minute, c)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
fw = NewFirewall(l, time.Hour, time.Second, time.Minute, c)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
fw = NewFirewall(l, time.Hour, time.Minute, time.Second, c)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
fw = NewFirewall(l, time.Minute, time.Hour, time.Second, c)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
fw = NewFirewall(l, time.Minute, time.Second, time.Hour, c)
assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
assert.Equal(t, 3602, conntrack.TimerWheel.wheelLen)
}
func TestFirewall_AddRule(t *testing.T) {
@@ -212,44 +186,44 @@ func TestFirewall_Drop(t *testing.T) {
cp := cert.NewCAPool()
// Drop outbound
assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp))
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp))
// Allow outbound because conntrack
require.NoError(t, fw.Drop(p, false, &h, cp, nil))
require.NoError(t, fw.Drop(p, false, &h, cp))
// test remote mismatch
oldRemote := p.RemoteAddr
p.RemoteAddr = netip.MustParseAddr("1.2.3.10")
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP)
assert.Equal(t, fw.Drop(p, false, &h, cp), ErrInvalidRemoteIP)
p.RemoteAddr = oldRemote
// ensure signer doesn't get in the way of group checks
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, true, &h, cp), ErrNoMatchingRule)
// test caSha doesn't drop on match
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp))
// ensure ca name doesn't get in the way of group checks
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, true, &h, cp), ErrNoMatchingRule)
// test caName doesn't drop on match
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp))
}
func TestFirewall_DropV6(t *testing.T) {
@@ -290,44 +264,44 @@ func TestFirewall_DropV6(t *testing.T) {
cp := cert.NewCAPool()
// Drop outbound
assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp))
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp))
// Allow outbound because conntrack
require.NoError(t, fw.Drop(p, false, &h, cp, nil))
require.NoError(t, fw.Drop(p, false, &h, cp))
// test remote mismatch
oldRemote := p.RemoteAddr
p.RemoteAddr = netip.MustParseAddr("fd12::56")
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP)
assert.Equal(t, fw.Drop(p, false, &h, cp), ErrInvalidRemoteIP)
p.RemoteAddr = oldRemote
// ensure signer doesn't get in the way of group checks
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, true, &h, cp), ErrNoMatchingRule)
// test caSha doesn't drop on match
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp))
// ensure ca name doesn't get in the way of group checks
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, true, &h, cp), ErrNoMatchingRule)
// test caName doesn't drop on match
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp))
}
func BenchmarkFirewallTable_match(b *testing.B) {
@@ -534,10 +508,10 @@ func TestFirewall_Drop2(t *testing.T) {
cp := cert.NewCAPool()
// h1/c1 lacks the proper groups
require.ErrorIs(t, fw.Drop(p, true, &h1, cp, nil), ErrNoMatchingRule)
require.ErrorIs(t, fw.Drop(p, true, &h1, cp), ErrNoMatchingRule)
// c has the proper groups
resetConntrack(fw)
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp))
}
func TestFirewall_Drop3(t *testing.T) {
@@ -614,18 +588,18 @@ func TestFirewall_Drop3(t *testing.T) {
cp := cert.NewCAPool()
// c1 should pass because host match
require.NoError(t, fw.Drop(p, true, &h1, cp, nil))
require.NoError(t, fw.Drop(p, true, &h1, cp))
// c2 should pass because ca sha match
resetConntrack(fw)
require.NoError(t, fw.Drop(p, true, &h2, cp, nil))
require.NoError(t, fw.Drop(p, true, &h2, cp))
// c3 should fail because no match
resetConntrack(fw)
assert.Equal(t, fw.Drop(p, true, &h3, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, true, &h3, cp), ErrNoMatchingRule)
// Test a remote address match
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "1.2.3.4/24", "", "", ""))
require.NoError(t, fw.Drop(p, true, &h1, cp, nil))
require.NoError(t, fw.Drop(p, true, &h1, cp))
}
func TestFirewall_Drop3V6(t *testing.T) {
@@ -662,7 +636,7 @@ func TestFirewall_Drop3V6(t *testing.T) {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
cp := cert.NewCAPool()
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "fd12::34/120", "", "", ""))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp))
}
func TestFirewall_DropConntrackReload(t *testing.T) {
@@ -703,12 +677,12 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
cp := cert.NewCAPool()
// Drop outbound
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp))
// Allow outbound because conntrack
require.NoError(t, fw.Drop(p, false, &h, cp, nil))
require.NoError(t, fw.Drop(p, false, &h, cp))
oldFw := fw
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -717,7 +691,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
fw.rulesVersion = oldFw.rulesVersion + 1
// Allow outbound because conntrack and new rules allow port 10
require.NoError(t, fw.Drop(p, false, &h, cp, nil))
require.NoError(t, fw.Drop(p, false, &h, cp))
oldFw = fw
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -726,7 +700,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
fw.rulesVersion = oldFw.rulesVersion + 1
// Drop outbound because conntrack doesn't match new ruleset
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, false, &h, cp), ErrNoMatchingRule)
}
func TestFirewall_ICMPPortBehavior(t *testing.T) {
@@ -771,12 +745,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0
p.RemotePort = 0
// Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
require.NoError(t, fw.Drop(*p, true, &h, cp))
//now also allow outbound
require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
require.NoError(t, fw.Drop(*p, false, &h, cp))
})
t.Run("nonzero ports", func(t *testing.T) {
@@ -784,12 +758,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0xabcd
p.RemotePort = 0x1234
// Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
require.NoError(t, fw.Drop(*p, true, &h, cp))
//now also allow outbound
require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
require.NoError(t, fw.Drop(*p, false, &h, cp))
})
})
@@ -801,12 +775,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0
p.RemotePort = 0
// Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, true, &h, cp), ErrNoMatchingRule)
//now also allow outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
})
t.Run("nonzero ports, still blocked", func(t *testing.T) {
@@ -814,12 +788,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0xabcd
p.RemotePort = 0x1234
// Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, true, &h, cp), ErrNoMatchingRule)
//now also allow outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
})
t.Run("nonzero, matching ports, still blocked", func(t *testing.T) {
@@ -827,12 +801,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 80
p.RemotePort = 80
// Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, true, &h, cp), ErrNoMatchingRule)
//now also allow outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
})
})
t.Run("Any proto, any port", func(t *testing.T) {
@@ -844,12 +818,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0
p.RemotePort = 0
// Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
require.NoError(t, fw.Drop(*p, true, &h, cp))
//now also allow outbound
require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
require.NoError(t, fw.Drop(*p, false, &h, cp))
})
t.Run("nonzero ports, allowed", func(t *testing.T) {
@@ -858,15 +832,15 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0xabcd
p.RemotePort = 0x1234
// Drop outbound
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
require.NoError(t, fw.Drop(*p, true, &h, cp))
//now also allow outbound
require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
require.NoError(t, fw.Drop(*p, false, &h, cp))
//different ID is blocked
p.RemotePort++
require.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
require.Equal(t, fw.Drop(*p, false, &h, cp), ErrNoMatchingRule)
})
})
@@ -914,7 +888,7 @@ func TestFirewall_DropIPSpoofing(t *testing.T) {
Protocol: iputil.IPProtocolUDP,
Fragment: false,
}
assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP)
assert.Equal(t, fw.Drop(p, true, &h1, cp), ErrInvalidRemoteIP)
}
func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) {
@@ -965,20 +939,84 @@ func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) {
Protocol: iputil.IPProtocolUDP,
}
require.NoError(t, fw.Drop(flow, true, &victimHI, cp, nil),
require.NoError(t, fw.Drop(flow, true, &victimHI, cp),
"victim's own traffic from its own overlay IP must be allowed")
unseen := flow
unseen.RemotePort = 55001
assert.Equal(t, ErrInvalidRemoteIP, fw.Drop(unseen, true, &attackerHI, cp, nil),
assert.Equal(t, ErrInvalidRemoteIP, fw.Drop(unseen, true, &attackerHI, cp),
"sanity: attacker forging victim's source IP must be rejected when no conntrack entry exists")
got := fw.Drop(flow, true, &attackerHI, cp, nil)
got := fw.Drop(flow, true, &attackerHI, cp)
t.Logf("attacker replaying victim's 4-tuple: Drop returned %v (nil == packet ALLOWED == spoof succeeded)", got)
assert.Equal(t, ErrInvalidRemoteIP, got,
"SECURITY: attacker spoofed victim's overlay source IP (192.0.2.2) by reusing an existing conntrack 4-tuple; Drop returned %v instead of rejecting", got)
}
func TestFirewallConntrack_sweep(t *testing.T) {
ct := newFirewallConntrack()
live := firewall.Packet{LocalPort: 1}
dead := firewall.Packet{LocalPort: 2}
ct.store(live, newConn(11, true, 0))
ct.store(dead, newConn(10, true, 0))
assert.Equal(t, int64(2), ct.count.Load())
ct.sweep(10)
_, ok := ct.conns.Load(live)
assert.True(t, ok)
_, ok = ct.conns.Load(dead)
assert.False(t, ok)
assert.Equal(t, int64(1), ct.count.Load())
}
func TestFirewallConntrack_storeAndDeleteCount(t *testing.T) {
ct := newFirewallConntrack()
fp := firewall.Packet{LocalPort: 1}
first := newConn(10, true, 0)
ct.store(fp, first)
// Replacing an entry doesn't add to the count
second := newConn(20, true, 1)
ct.store(fp, second)
assert.Equal(t, int64(1), ct.count.Load())
// Deleting a stale entry is a no-op, the newer one stays
ct.delete(fp, first)
v, ok := ct.conns.Load(fp)
assert.True(t, ok)
assert.Same(t, second, v)
assert.Equal(t, int64(1), ct.count.Load())
ct.delete(fp, second)
_, ok = ct.conns.Load(fp)
assert.False(t, ok)
assert.Equal(t, int64(0), ct.count.Load())
ct.store(fp, first)
ct.store(firewall.Packet{LocalPort: 2}, newConn(10, true, 0))
ct.clear()
assert.Equal(t, int64(0), ct.count.Load())
}
func TestFirewallConntrack_Run(t *testing.T) {
ct := newFirewallConntrack()
ct.epoch = time.Now().Add(-time.Hour)
ct.store(firewall.Packet{LocalPort: 1}, newConn(10, true, 0))
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
ct.Run(ctx)
close(done)
}()
// The first tick sets the clock to an hour in and sweeps the long expired entry
require.Eventually(t, func() bool { return ct.count.Load() == 0 }, 5*time.Second, 10*time.Millisecond)
assert.GreaterOrEqual(t, ct.now.Load(), uint32(3600))
cancel()
<-done
}
// BenchmarkFirewallDropConntrackHit measures Drop on an already-established flow
// (a conntrack hit). This is the fast path that the source-IP<->cert binding
// reordering adds work to, so it quantifies the cost of moving the address checks
@@ -987,8 +1025,6 @@ func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) {
// so the remote-address check is a single netip.Addr compare.
// - complex: peer cert has unsafe networks (h.networks populated), so the
// remote-address check is a BART lookup.
// - noCache/localCache: whether a per-batch ConntrackCache is supplied, which in
// the original code let the fast path skip straight past the address checks.
func BenchmarkFirewallDropConntrackHit(b *testing.B) {
l := test.NewLoggerWithOutput(&bytes.Buffer{})
@@ -1036,14 +1072,11 @@ func BenchmarkFirewallDropConntrackHit(b *testing.B) {
}
cases := []struct {
name string
host *HostInfo
useCache bool
name string
host *HostInfo
}{
{"simple/noCache", simpleHost, false},
{"simple/localCache", simpleHost, true},
{"complex/noCache", complexHost, false},
{"complex/localCache", complexHost, true},
{"simple", simpleHost},
{"complex", complexHost},
}
for _, tc := range cases {
@@ -1052,17 +1085,12 @@ func BenchmarkFirewallDropConntrackHit(b *testing.B) {
require.NoError(b, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
// Establish the conntrack entry so every benchmarked Drop is a hit.
require.NoError(b, fw.Drop(flow, true, tc.host, cp, nil))
var cache firewall.ConntrackCache
if tc.useCache {
cache = firewall.ConntrackCache{}
}
require.NoError(b, fw.Drop(flow, true, tc.host, cp))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := fw.Drop(flow, true, tc.host, cp, cache); err != nil {
if err := fw.Drop(flow, true, tc.host, cp); err != nil {
b.Fatal(err)
}
}
@@ -1550,7 +1578,7 @@ func (c *testcase) Test(t *testing.T, fw *Firewall) {
t.Helper()
cp := cert.NewCAPool()
resetConntrack(fw)
err := fw.Drop(c.p, true, c.h, cp, nil)
err := fw.Drop(c.p, true, c.h, cp)
if c.err == nil {
require.NoError(t, err, "failed to not drop remote address %s", c.p.RemoteAddr)
} else {
@@ -1741,7 +1769,5 @@ func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, end
}
func resetConntrack(fw *Firewall) {
fw.Conntrack.Lock()
fw.Conntrack.Conns = map[firewall.Packet]*conn{}
fw.Conntrack.Unlock()
fw.Conntrack.clear()
}
+1 -1
View File
@@ -5,6 +5,7 @@ go 1.26.0
require (
dario.cat/mergo v1.0.2
filippo.io/bigmod v0.1.0
github.com/DefinedNet/netlink v1.3.2-dn.1
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
github.com/armon/go-radix v1.0.0
github.com/flynn/noise v1.1.0
@@ -20,7 +21,6 @@ require (
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
github.com/stretchr/testify v1.12.1
github.com/vishvananda/netlink v1.3.1
go.uber.org/goleak v1.3.0
go.yaml.in/yaml/v3 v3.0.5
golang.org/x/crypto v0.57.0
+2 -4
View File
@@ -3,6 +3,8 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8=
filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI=
github.com/DefinedNet/netlink v1.3.2-dn.1 h1:Md2PsUE1Xuy1Zxs/Bm6356n1kt5bq2HspXlPJ6pEspg=
github.com/DefinedNet/netlink v1.3.2-dn.1/go.mod h1:o4YSn8Jw7zoDcYXDmV77TRd4eS5sSLuE+vu2zAN67C8=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -138,8 +140,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -195,8 +195,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-58
View File
@@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"log/slog"
"net"
"net/netip"
"slices"
"sync"
@@ -18,7 +17,6 @@ import (
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging"
)
const defaultPromoteEvery = 1000 // Count of packets sent before we try moving a tunnel to a preferred underlay ip address
@@ -930,59 +928,3 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
return li
}
// Utility functions
func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
//FIXME: This function is pretty garbage
var finalAddrs []netip.Addr
ifaces, _ := net.Interfaces()
for _, i := range ifaces {
allow := allowList.AllowName(i.Name)
if l.Enabled(context.Background(), logging.LevelTrace) {
l.Log(context.Background(), logging.LevelTrace, "localAllowList.AllowName",
"interfaceName", i.Name,
"allow", allow,
)
}
if !allow {
continue
}
addrs, _ := i.Addrs()
for _, rawAddr := range addrs {
var addr netip.Addr
switch v := rawAddr.(type) {
case *net.IPNet:
//continue
addr, _ = netip.AddrFromSlice(v.IP)
case *net.IPAddr:
addr, _ = netip.AddrFromSlice(v.IP)
}
if !addr.IsValid() {
if l.Enabled(context.Background(), slog.LevelDebug) {
l.Debug("addr was invalid", "localAddr", rawAddr)
}
continue
}
addr = addr.Unmap()
if addr.IsLoopback() == false && addr.IsLinkLocalUnicast() == false {
isAllowed := allowList.Allow(addr)
if l.Enabled(context.Background(), logging.LevelTrace) {
l.Log(context.Background(), logging.LevelTrace, "localAllowList.Allow",
"localAddr", addr,
"allowed", isAllowed,
)
}
if !isAllowed {
continue
}
finalAddrs = append(finalAddrs, addr)
}
}
}
return finalAddrs
}
+3 -3
View File
@@ -16,7 +16,7 @@ import (
"github.com/slackhq/nebula/routing"
)
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int) {
// borrowed: pkt.Bytes is owned by the originating tio.Queue and is
// only valid until the next Read on that queue. Every consumer below
// (parse, self-forward, handshake cache, sendInsideMessage) reads it
@@ -109,7 +109,7 @@ func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.Parse
return
}
dropReason := f.firewall.Drop(fwPacket.Packet, false, hostinfo, f.pki.GetCAPool(), localCache)
dropReason := f.firewall.Drop(fwPacket.Packet, false, hostinfo, f.pki.GetCAPool())
if dropReason == nil {
f.sendInsideMessage(hostinfo, pkt, nb, sendBatch)
} else {
@@ -380,7 +380,7 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
}
// check if packet is in outbound fw rules
dropReason := f.firewall.Drop(fp.Packet, false, hostinfo, f.pki.GetCAPool(), nil)
dropReason := f.firewall.Drop(fp.Packet, false, hostinfo, f.pki.GetCAPool())
if dropReason != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
f.l.Debug("dropping cached packet",
+1 -1
View File
@@ -50,7 +50,7 @@ func newSelfForwardInterface(myAddrs ...netip.Addr) (*Interface, *capturingTun)
}
func consumeInside(f *Interface, packet []byte) {
f.consumeInsidePacket(tio.Packet{Bytes: packet}, &firewall.ParsedPacket{}, make([]byte, 12), nil, make([]byte, mtu), 0, nil)
f.consumeInsidePacket(tio.Packet{Bytes: packet}, &firewall.ParsedPacket{}, make([]byte, 12), nil, make([]byte, mtu), 0)
}
// l4Proto describes one upper-layer header for these tests: its IP next-header
+18 -17
View File
@@ -52,8 +52,6 @@ type InterfaceConfig struct {
reQueryEvery uint32
reQueryWait time.Duration
ConntrackCacheTimeout time.Duration
// CpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
// should pin to. Queue i pins to CpuAffinity[i % len(CpuAffinity)] —
// shorter lists than `routines` cycle. Empty list keeps the default
@@ -111,8 +109,6 @@ type Interface struct {
rebindEpoch atomic.Uint32
version string
conntrackCacheTimeout time.Duration
ctx context.Context
writers []udp.Conn
queues []tio.Queue
@@ -227,7 +223,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
myBroadcastAddrsTable: cs.myVpnBroadcastAddrsTable,
relayManager: c.relayManager,
connectionManager: c.connectionManager,
conntrackCacheTimeout: c.ConntrackCacheTimeout,
cpuAffinity: c.CpuAffinity,
pinThreads: c.PinThreads,
@@ -361,8 +356,19 @@ type rxContext struct {
h *header.H
fwPacket *firewall.ParsedPacket
hostmapCache map[uint32]*HostInfo
lhh *LightHouseHandler
ctCache *firewall.ConntrackCacheTicker
// lastAllowed is the last flow the inbound firewall let in during this batch. UDP GRO hands us segments
// in runs from the same flow, so the rest of a run can skip the firewall. Cleared by endBatch.
lastAllowed firewall.Packet
lastAllowedHost *HostInfo
lhh *LightHouseHandler
}
// endBatch forgets what was learned during a batch, so nothing outlives a hostmap or firewall change by more than one batch
func (rxc *rxContext) endBatch() {
clear(rxc.hostmapCache)
rxc.lastAllowed = firewall.Packet{}
rxc.lastAllowedHost = nil
}
func newRxContext(f *Interface, q int) *rxContext {
@@ -374,7 +380,6 @@ func newRxContext(f *Interface, q int) *rxContext {
fwPacket: &firewall.ParsedPacket{},
hostmapCache: map[uint32]*HostInfo{},
lhh: f.lightHouse.NewRequestHandler(),
ctCache: firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout),
}
}
@@ -396,7 +401,7 @@ func (f *Interface) listenOut(i int) {
if err := f.batchers[i].Flush(); err != nil {
f.l.Error("Failed to flush tun coalescer", "error", err)
}
clear(rxc.hostmapCache)
rxc.endBatch()
}
err := li.ListenOut(listener, flusher)
@@ -444,8 +449,6 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
fwPacket := &firewall.ParsedPacket{}
nb := make([]byte, 12, 12)
conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
for {
pkts, err := queue.Read()
if err != nil {
@@ -458,7 +461,7 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
}
for _, pkt := range pkts {
f.consumeInsidePacket(pkt, fwPacket, nb, sb, rejectBuf, i, conntrackCache.Get())
f.consumeInsidePacket(pkt, fwPacket, nb, sb, rejectBuf, i)
// Flush incrementally once a full sendmmsg batch has
// accumulated so the first packets of a deep read drain
// hit the wire while the rest are still being encrypted.
@@ -531,9 +534,8 @@ func (f *Interface) reloadFirewall(c *config.C) {
}
oldFw := f.firewall
conntrack := oldFw.Conntrack
conntrack.Lock()
defer conntrack.Unlock()
// Conntrack lives as long as the interface, keep using the running one
fw.Conntrack = oldFw.Conntrack
fw.rulesVersion = oldFw.rulesVersion + 1
// If rulesVersion is back to zero, we have wrapped all the way around. Be
@@ -544,8 +546,7 @@ func (f *Interface) reloadFirewall(c *config.C) {
"oldFirewallHashes", oldFw.GetRuleHashes(),
"rulesVersion", fw.rulesVersion,
)
} else {
fw.Conntrack = conntrack
fw.Conntrack.clear()
}
f.firewall = fw
+1 -2
View File
@@ -10,7 +10,6 @@ import (
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/overlay/overlaytest"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
@@ -43,7 +42,7 @@ func Test_emitStats_primesGauges(t *testing.T) {
hostMap: hostMap,
inside: &overlaytest.NoopTun{},
outside: &udp.NoopConn{},
firewall: &Firewall{Conntrack: &FirewallConntrack{Conns: map[firewall.Packet]*conn{}}},
firewall: &Firewall{Conntrack: newFirewallConntrack()},
lightHouse: lh,
pki: &PKI{},
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
+3 -13
View File
@@ -45,8 +45,7 @@ func setTransportChecksum4(packet []byte) {
if ihl < ipv4.HeaderLen || end < ihl || end > len(packet) {
return
}
// The checksum covers the whole datagram, which a fragment (MF set or a
// non-zero offset) does not carry.
// do not attempt to set checksums for fragments
if binary.BigEndian.Uint16(packet[6:8])&0x3fff != 0 {
return
}
@@ -68,10 +67,7 @@ func setTransportChecksum6(packet []byte) {
return
}
// The checksum covers the whole datagram, which a fragment does not carry.
// An unknown extension header hides where the transport header starts. A
// chain longer than the walk's budget ends it early, at an offset that was
// never checked against the packet.
// we must find the transport header in order to write a checksum
proto, offset, _, anyFragment, err := IPv6FindUpperProtocol(packet[:end])
if err != nil || anyFragment || offset >= end {
return
@@ -109,8 +105,7 @@ func transportExtent(transport []byte, proto uint8) ([]byte, bool) {
// writeTransportChecksum stores the checksum of transport, taken over the
// pseudo-header sum csum, in the header's checksum field. A UDP checksum that
// computes to zero goes on the wire as 0xffff: zero means no checksum was
// computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200
// section 8.1).
// computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200 section 8.1).
func writeTransportChecksum(transport []byte, proto uint8, csum uint32) {
var at, minLen int
switch proto {
@@ -133,11 +128,6 @@ func writeTransportChecksum(transport []byte, proto uint8, csum uint32) {
binary.BigEndian.PutUint16(transport[at:], sum)
}
// fold reduces a pseudo-header sum to the 16 bit seed Checksum takes. Carrying
// the high half back into the low half is what keeps the reduction lossless, so
// the seed sums exactly as the wider value would; 0xffff is its fixed point.
// Every term of that sum comes from a 16 bit field, so it stays far below the
// width at which the accumulator would wrap.
func fold(csum uint32) uint16 {
for csum > 0xffff {
csum = (csum >> 16) + (csum & 0xffff)
+13 -16
View File
@@ -345,14 +345,11 @@ func ipv6CreateRejectTCPPacket(packet []byte, out []byte, offset int) []byte {
return out
}
// IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the
// offset it begins at, and whether the packet is a non-first fragment. Only the RFC 8200 and IANA extension
// headers below are walked. Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental
// 253/254, and real upper layer protocols like SCTP or GRE, is terminal. Walking those as extension headers
// is a firewall bypass, so they fail closed. For a non-first fragment the returned protocol is the fragmented
// protocol and offset points at the fragment header, there is no transport header to locate. Returns
// ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header or the chain is truncated before a
// terminal protocol is reached.
// IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the offset it begins at, and whether the packet is a non-first fragment.
// Only the RFC 8200 and IANA extension headers below are walked.
// Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental 253/254, and real upper layer protocols like SCTP or GRE, is terminal.
// For a non-first fragment the returned protocol is the fragmented protocol and offset points at the fragment header, there is no transport header to locate.
// Returns ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header, or the chain is truncated, or if more than 8 extension headers are present.
func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool, anyFragment bool, err error) {
const maxIPv6ExtHeaders = 8
if len(packet) < ipv6.HeaderLen {
@@ -365,14 +362,14 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm
switch nextHeader {
case 0, 43, 60: // Hop-by-Hop, Routing, Destination
if len(packet) < offset+2 {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
}
nextHeader = packet[offset]
offset += (int(packet[offset+1]) + 1) << 3
case 44: // Fragment
if len(packet) < offset+8 {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
}
anyFragment = true
// Non-first fragments carry no transport header, report the fragmented protocol and stop
@@ -384,21 +381,21 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm
case 51: // AH
if len(packet) < offset+2 {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
}
nextHeader = packet[offset]
offset += (int(packet[offset+1]) + 2) << 2
default:
// A prior extension header can declare a length that advances offset past the packet. The terminal
// protocol's header isn't actually here, so treat the chain as truncated rather than classifying it.
if offset > len(packet) {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
// A prior extension header can declare a length that advances offset past the packet, reject this.
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
}
return nextHeader, offset, isFragment, anyFragment, nil
return nextHeader, offset, isFragment, anyFragment, nil //success!
}
}
return nextHeader, offset, isFragment, anyFragment, nil
// Fail closed
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
}
func CreateICMPEchoResponse(packet, out []byte) []byte {
+17 -5
View File
@@ -529,6 +529,17 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
firstFragToUDP := []byte{17, 0, 0, 1, 0, 0, 0, 1} // frag offset 0, M=1, next = UDP
nonFirstFrag := []byte{17, 0, 0, 9, 0, 0, 0, 1} // frag offset non-zero, next = UDP
transport := []byte{0, 80, 1, 187, 0, 0, 0, 0} // stand-in bytes, IPv6FindUpperProtocol never reads ports
extToHbH := []byte{0, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = Hop-by-Hop
// 8 walked headers spend the budget before the 9th slot's terminal is ever classified. This is the
// report 4040546 shape: without failing closed the walk hands back offset 104, and a caller reads ports
// from the attacker's bytes there.
var budgetChain []byte
for range 7 {
budgetChain = append(budgetChain, extToHbH...)
}
budgetChain = append(budgetChain, extToTCP...)
budgetChain = append(budgetChain, transport...)
tests := []struct {
name string
@@ -549,9 +560,10 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
{"first fragment walks to transport", 44, append(firstFragToUDP, transport...), 17, ipv6.HeaderLen + 8, false, true, nil},
{"non-first fragment stops", 44, append(nonFirstFrag, transport...), 17, ipv6.HeaderLen, true, true, nil},
{"unknown protocol is terminal", 132, transport, 132, ipv6.HeaderLen, false, false, nil}, // SCTP
{"truncated extension header", 0, nil, 0, ipv6.HeaderLen, false, false, ErrIPv6CouldNotFindPayload},
{"truncated extension header", 0, nil, 0, 0, false, false, ErrIPv6CouldNotFindPayload},
// Destination Options with a declared length (255+1)*8 = 2048 that runs past the 48 byte buffer, next = SCTP
{"extension length past buffer", 60, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 132, ipv6.HeaderLen + 2048, false, false, ErrIPv6CouldNotFindPayload},
{"extension length past buffer", 0, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 0, 0, false, false, ErrIPv6CouldNotFindPayload},
{"budget exhausted fails closed", 0, budgetChain, 0, 0, false, false, ErrIPv6CouldNotFindPayload},
}
for _, tt := range tests {
@@ -559,10 +571,10 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
packet := makeIPv6Packet(src, dst, tt.nextHeader, tt.payload)
proto, offset, isFragment, anyFragment, err := IPv6FindUpperProtocol(packet)
if tt.wantErr != nil {
assert.ErrorIs(t, err, tt.wantErr)
return
require.ErrorIs(t, err, tt.wantErr)
} else {
require.NoError(t, err)
}
require.NoError(t, err)
assert.Equal(t, tt.wantProto, proto)
assert.Equal(t, tt.wantOffset, offset)
assert.Equal(t, tt.wantFragment, isFragment)
+40 -4
View File
@@ -15,10 +15,12 @@ import (
"time"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/nicfinder"
"github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util"
)
@@ -42,6 +44,10 @@ type LightHouse struct {
// addresses rather than whatever this machine's NICs happen to be. Set it before Start.
localAddrsFn func(*LocalAllowList) []netip.Addr
// lastLocalAddrsErr is the previous localAddrsFn failure. Enumeration runs on every update, so an
// unchanged failure is demoted to Debug rather than warning every lighthouse.interval forever.
lastLocalAddrsErr atomic.Pointer[string]
// Local cache of answers from light houses
// map of vpn addr to answers
addrMap map[netip.Addr]*RemoteList
@@ -76,6 +82,8 @@ type LightHouse struct {
updateTrigger chan struct{}
queryChan chan netip.Addr
// queryDropped counts queries QueryServer dropped because queryChan was full.
queryDropped metrics.Counter
calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote
@@ -111,11 +119,14 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
nebulaPort: nebulaPort,
punchy: p,
updateTrigger: make(chan struct{}, 1),
queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 1024)),
queryDropped: metrics.GetOrRegisterCounter("lighthouse.query.dropped", nil),
l: l,
}
h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
return localAddrs(h.l, al)
addrs, err := nicfinder.Find(ctx, h.l, al)
h.logLocalAddrsErr(err)
return addrs
}
lighthouses := make([]netip.Addr, 0)
@@ -505,14 +516,19 @@ func (lh *LightHouse) Query(vpnAddr netip.Addr) *RemoteList {
return nil
}
// QueryServer is asynchronous so no reply should be expected
// QueryServer is asynchronous and non-blocking, no reply should be expected.
// If the query channel is full, the query is dropped.
func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) {
// Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses
if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) {
return
}
lh.queryChan <- vpnAddr
select {
case lh.queryChan <- vpnAddr:
default:
lh.queryDropped.Inc(1)
}
}
func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList {
@@ -916,6 +932,26 @@ func (lh *LightHouse) TriggerUpdate() {
}
}
// logLocalAddrsErr reports a localAddrs failure at Warn the first time it is seen and at Debug while
// it persists unchanged, so a permanent failure does not warn on every update forever.
func (lh *LightHouse) logLocalAddrsErr(err error) {
if err == nil {
lh.lastLocalAddrsErr.Store(nil)
return
}
msg := err.Error()
prev := lh.lastLocalAddrsErr.Swap(&msg)
if prev != nil && *prev == msg {
if lh.l.Enabled(context.Background(), slog.LevelDebug) {
lh.l.Debug("Failed to collect local addresses to advertise", "error", err)
}
return
}
lh.l.Warn("Failed to collect local addresses to advertise", "error", err)
}
func (lh *LightHouse) SendUpdate() {
var v4 []*V4AddrPort
var v6 []*V6AddrPort
+85
View File
@@ -1,10 +1,14 @@
package nebula
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"net/netip"
"sync"
"testing"
"time"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert"
@@ -764,3 +768,84 @@ func TestLighthouse_DeletesWork(t *testing.T) {
out = lh.Query(testHost)
assert.Nil(t, out)
}
func TestLightHouse_logLocalAddrsErr(t *testing.T) {
out := &bytes.Buffer{}
lh := &LightHouse{l: test.NewLoggerWithOutput(out)}
// The first sighting of a failure warns.
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "permission denied")
// Repeating unchanged does not warn again, which is what keeps a permanent failure from warning
// on every lighthouse.interval for the life of the process.
out.Reset()
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.NotContains(t, out.String(), "level=WARN")
// A different failure is a new event and warns.
out.Reset()
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "something else")
// Recovering resets, so the same failure returning later warns again.
out.Reset()
lh.logLocalAddrsErr(nil)
assert.Empty(t, out.String())
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
}
// heldEncWriter holds the query worker in its first GetHostInfo until release closes.
type heldEncWriter struct {
mockEncWriter
once sync.Once
held chan struct{}
release chan struct{}
}
func (w *heldEncWriter) GetHostInfo(netip.Addr) *HostInfo {
w.once.Do(func() { close(w.held) })
<-w.release
return nil
}
// QueryServer never waits on the query worker, since the packet readers call it and nothing drains the channel at
// shutdown; a query that finds the channel full is dropped and counted. The worker is held on its first query so the
// channel fills.
func TestLighthouse_QueryServerDoesNotBlock(t *testing.T) {
l := test.NewLogger()
cs := testCertState(netip.MustParsePrefix("10.128.0.1/16"))
lh1 := "10.128.0.2"
c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}}
c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
c.Settings["handshakes"] = map[string]any{"query_buffer": 1}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err)
w := &heldEncWriter{held: make(chan struct{}), release: make(chan struct{})}
lh.ifce = w
t.Cleanup(func() { close(w.release) })
lh.QueryServer(netip.MustParseAddr("10.128.0.10"))
select {
case <-w.held:
case <-time.After(5 * time.Second):
t.Fatal("the query worker never took the first query")
}
dropped := lh.queryDropped.Count()
done := make(chan struct{})
go func() {
lh.QueryServer(netip.MustParseAddr("10.128.0.11"))
lh.QueryServer(netip.MustParseAddr("10.128.0.12"))
close(done)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("QueryServer blocked on a full query channel")
}
assert.EqualValues(t, 1, lh.queryDropped.Count()-dropped)
}
+28 -37
View File
@@ -32,9 +32,12 @@ const _ = noiseutil.RejectHeadroom - 4*maxRoutines
func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
ctx, cancel := context.WithCancel(context.Background())
// Automatically cancel the context if Main returns an error, to signal all created goroutines to quit.
// The goroutines started below stop only when this context does, and only a caller holding the
// Control can arrange that. Cancel whenever we are not handing one back, which covers an error
// and a config test alike: a config test used to leave the lighthouse query worker, and a
// hostname resolver per dns named static host, running for the life of the process.
defer func() {
if reterr != nil {
if retcon == nil {
cancel()
}
}()
@@ -110,18 +113,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
l.Info("Using multiple routines", "routines", routines)
}
// EXPERIMENTAL
// Intentionally not documented yet while we do more testing and determine
// a good default value.
conntrackCacheTimeout := c.GetDuration("firewall.conntrack.routine_cache_timeout", 0)
if routines > 1 && !c.IsSet("firewall.conntrack.routine_cache_timeout") {
// Use a different default if we are running with multiple routines
conntrackCacheTimeout = 1 * time.Second
}
if conntrackCacheTimeout > 0 {
l.Info("Using routine-local conntrack cache", "duration", conntrackCacheTimeout)
}
var tun overlay.Device
if !configTest {
c.CatchHUP(ctx)
@@ -271,29 +262,28 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
}
ifConfig := &InterfaceConfig{
HostMap: hostMap,
Inside: tun,
Outside: udpConns[0],
pki: pki,
Firewall: fw,
DnsServer: ds,
HandshakeManager: handshakeManager,
connectionManager: connManager,
lightHouse: lightHouse,
tryPromoteEvery: c.GetUint32("counters.try_promote", defaultPromoteEvery),
reQueryEvery: c.GetUint32("counters.requery_every_packets", defaultReQueryEvery),
reQueryWait: c.GetDuration("timers.requery_wait_duration", defaultReQueryWait),
DropLocalBroadcast: c.GetBool("tun.drop_local_broadcast", false),
DropMulticast: c.GetBool("tun.drop_multicast", false),
routines: routines,
MessageMetrics: messageMetrics,
version: buildVersion,
relayManager: NewRelayManager(ctx, l, hostMap, c),
punchy: punchy,
ConntrackCacheTimeout: conntrackCacheTimeout,
CpuAffinity: cpuAffinity,
PinThreads: pinThreads,
l: l,
HostMap: hostMap,
Inside: tun,
Outside: udpConns[0],
pki: pki,
Firewall: fw,
DnsServer: ds,
HandshakeManager: handshakeManager,
connectionManager: connManager,
lightHouse: lightHouse,
tryPromoteEvery: c.GetUint32("counters.try_promote", defaultPromoteEvery),
reQueryEvery: c.GetUint32("counters.requery_every_packets", defaultReQueryEvery),
reQueryWait: c.GetDuration("timers.requery_wait_duration", defaultReQueryWait),
DropLocalBroadcast: c.GetBool("tun.drop_local_broadcast", false),
DropMulticast: c.GetBool("tun.drop_multicast", false),
routines: routines,
MessageMetrics: messageMetrics,
version: buildVersion,
relayManager: NewRelayManager(ctx, l, hostMap, c),
punchy: punchy,
CpuAffinity: cpuAffinity,
PinThreads: pinThreads,
l: l,
}
var ifce *Interface
@@ -313,6 +303,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
handshakeManager.f = ifce
go handshakeManager.Run(ctx)
go ifce.firewall.Conntrack.Run(ctx)
punchy.Start(ctx, ifce, hostMap, lightHouse)
}
+74
View File
@@ -1,12 +1,21 @@
package nebula
import (
"fmt"
"net/netip"
"os"
"path/filepath"
"testing"
"time"
"github.com/slackhq/nebula/cert"
cert_test "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
func TestParseCpuAffinity(t *testing.T) {
@@ -49,3 +58,68 @@ func TestParseCpuAffinity(t *testing.T) {
assert.Nil(t, parseCpuAffinity(newConfig([]any{1 << 20}), l, 1))
}
}
// TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running
// when it hands back a Control to stop them with.
//
// A config test gets no Control, so anything it started had nothing to stop it: the lighthouse
// query worker, and a hostname resolver per dns named static host, ran for the life of the
// process. That matters to every embedder that validates a config in process rather than by
// exec'ing, dnclient and the apple clients included, because they do it on each config load and
// the leak accumulates.
func TestMain_ConfigTestReleasesItsGoroutines(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
l := test.NewLogger()
dir := t.TempDir()
before := time.Now().Add(-time.Hour)
after := time.Now().Add(time.Hour)
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
_, _, keyPEM, certPEM := cert_test.NewTestCert(
cert.Version2, cert.Curve_CURVE25519, ca, caKey, "config-test", before, after, networks, nil, nil)
caPath := filepath.Join(dir, "ca.pem")
certPath := filepath.Join(dir, "cert.pem")
keyPath := filepath.Join(dir, "key.pem")
require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
// A static host by address, not by name: the query worker is the goroutine under test and a
// hostname would drag a real dns lookup into a unit test.
configBody := fmt.Sprintf(`
pki:
ca: %s
cert: %s
key: %s
static_host_map:
"10.0.0.2": ["192.0.2.1:4242"]
lighthouse:
hosts:
- "10.0.0.2"
listen:
host: 127.0.0.1
port: 0
tun:
disabled: true
firewall:
outbound:
- port: any
proto: any
host: any
inbound:
- port: any
proto: any
host: any
`, caPath, certPath, keyPath)
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
c := config.NewC(l)
require.NoError(t, c.Load(dir))
ctrl, err := Main(c, true, "config-test", l, nil)
require.NoError(t, err)
require.Nil(t, ctrl, "a config test hands back nothing to stop, so it must stop itself")
}
+39
View File
@@ -0,0 +1,39 @@
//go:build !android
package nicfinder
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/netip"
)
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
ifaces, err := net.Interfaces()
if err != nil {
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
}
var out []netip.Addr
var errs []error
for _, i := range ifaces {
if !allowName(ctx, l, filter, i.Name) {
continue
}
addrs, err := i.Addrs()
if err != nil {
errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err))
continue
}
for _, rawAddr := range addrs {
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
out = append(out, addr)
}
}
}
return out, errors.Join(errs...)
}
+104
View File
@@ -0,0 +1,104 @@
//go:build android
package nicfinder
import (
"context"
"errors"
"fmt"
"log/slog"
"net/netip"
"os"
"github.com/DefinedNet/netlink"
"github.com/DefinedNet/netlink/nl"
"golang.org/x/sys/unix"
)
// Android denies bind() on netlink route sockets and RTM_GETLINK, so net.Interfaces fails with EACCES.
// Enumerate with an RTM_GETADDR dump on an unbound socket instead.
// localInterfaces returns every interface that has an address, with its addresses. It lists
// addresses over a netlink socket that is never bound and names each index with SIOCGIFNAME, the
// same operations Bionic's getifaddrs uses for app UIDs, which the sandbox permits.
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
var errs []error
addrs, err := netlinkAddrList()
if err != nil {
if !errors.Is(err, netlink.ErrDumpInterrupted) {
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
}
// The address table kept changing under the dump and netlink gave up retrying. What it
// returned is still worth advertising.
errs = append(errs, fmt.Errorf("local addresses may be incomplete: %w", err))
}
// Any socket will do for SIOCGIFNAME.
sfd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
defer unix.Close(sfd)
var out []netip.Addr
nameAllowed := make(map[int]bool) // by interface index, once named
for _, a := range addrs {
allow, named := nameAllowed[a.LinkIndex]
if (named && !allow) || a.IPNet == nil {
continue
}
addr, ok := allowedAddr(ctx, l, filter, a.IPNet)
if !ok {
continue
}
if !named {
name, err := interfaceNameByIndex(sfd, a.LinkIndex)
// ENODEV means the interface went away after the dump. Any other failure is reported
// and the interface skipped, so the rest are still returned.
if err != nil && !errors.Is(err, unix.ENODEV) {
errs = append(errs, fmt.Errorf("failed to resolve name of interface %d: %w", a.LinkIndex, err))
}
allow = err == nil && allowName(ctx, l, filter, name)
nameAllowed[a.LinkIndex] = allow
}
if allow {
out = append(out, addr)
}
}
return out, errors.Join(errs...)
}
// netlinkAddrList is netlink.AddrList over a socket that is never bound. The kernel autobinds it
// on the first send, which the sandbox permits.
func netlinkAddrList() ([]netlink.Addr, error) {
fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
s, err := nl.NewNetlinkSocketFromFd(fd)
if err != nil {
unix.Close(fd)
return nil, err
}
h := netlink.NewHandleFromSockets(
map[int]*nl.SocketHandle{unix.NETLINK_ROUTE: {Socket: s}},
netlink.HandleOptions{RetryInterrupted: true},
)
defer h.Close()
if err := h.SetSocketTimeout(netlink.GetSocketTimeout()); err != nil {
return nil, err
}
return h.AddrList(nil, netlink.FAMILY_ALL)
}
// interfaceNameByIndex resolves an interface index with SIOCGIFNAME.
func interfaceNameByIndex(fd int, index int) (string, error) {
var ifr unix.Ifreq
ifr.SetUint32(uint32(index)) // ifr_ifindex is in the ifr_ifru union
if err := unix.IoctlIfreq(fd, unix.SIOCGIFNAME, &ifr); err != nil {
return "", err
}
return ifr.Name(), nil
}
+59
View File
@@ -0,0 +1,59 @@
package nicfinder
import (
"context"
"log/slog"
"net"
"net/netip"
"github.com/slackhq/nebula/logging"
)
// Filter decides which interfaces, by name, and which of their addresses Find returns.
type Filter interface {
AllowName(string) bool
Allow(netip.Addr) bool
}
// Find returns the addresses of this host's network interfaces that allowList accepts, with
// loopback and link-local addresses left out. When only some interfaces could be read it returns
// their addresses alongside an error for the rest. allowList must not be nil; it is called as
// given, so a nil pointer of a type whose methods accept a nil receiver is fine.
func Find(ctx context.Context, l *slog.Logger, allowList Filter) ([]netip.Addr, error) {
return localAddrs(ctx, l, allowList)
}
// allowName asks filter about an interface name and traces the answer.
func allowName(ctx context.Context, l *slog.Logger, filter Filter, name string) bool {
allow := filter.AllowName(name)
if l.Enabled(ctx, logging.LevelTrace) {
l.Log(ctx, logging.LevelTrace, "localAllowList.AllowName", "interfaceName", name, "allow", allow)
}
return allow
}
// allowedAddr converts one address a platform reports and says whether to advertise it: it must
// be a *net.IPNet or *net.IPAddr carrying an IP, not be loopback or link-local, and be accepted by
// filter, whose answer is traced. IPv4 comes back unmapped.
func allowedAddr(ctx context.Context, l *slog.Logger, filter Filter, rawAddr net.Addr) (netip.Addr, bool) {
var ip net.IP
switch v := rawAddr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
addr, ok := netip.AddrFromSlice(ip)
if !ok {
return netip.Addr{}, false
}
addr = addr.Unmap()
if addr.IsLoopback() || addr.IsLinkLocalUnicast() {
return netip.Addr{}, false
}
isAllowed := filter.Allow(addr)
if l.Enabled(ctx, logging.LevelTrace) {
l.Log(ctx, logging.LevelTrace, "localAllowList.Allow", "localAddr", addr, "allowed", isAllowed)
}
return addr, isAllowed
}
+117
View File
@@ -0,0 +1,117 @@
package nicfinder
import (
"context"
"net"
"net/netip"
"regexp"
"testing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
)
type allowAll struct{}
func (allowAll) Allow(netip.Addr) bool { return true }
func (allowAll) AllowName(string) bool { return true }
type testAllowList struct {
bannedName *regexp.Regexp
bannedAddr netip.Addr
}
func (l *testAllowList) Allow(addr netip.Addr) bool {
return addr != l.bannedAddr
}
func (l *testAllowList) AllowName(n string) bool {
return !l.bannedName.MatchString(n)
}
func newTestAllowList(bannedName string) *testAllowList {
return &testAllowList{
bannedName: regexp.MustCompile(bannedName),
}
}
func TestAllowedAddr(t *testing.T) {
ctx, l := context.Background(), test.NewLogger()
allowed := func(filter Filter, addrs []net.Addr) []netip.Addr {
var out []netip.Addr
for _, rawAddr := range addrs {
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
out = append(out, addr)
}
}
return out
}
addrs := []net.Addr{
&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)},
&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
&net.IPNet{IP: net.ParseIP("10.0.0.5"), Mask: net.CIDRMask(24, 32)},
&net.IPNet{IP: net.ParseIP("169.254.10.10"), Mask: net.CIDRMask(16, 32)},
&net.IPNet{IP: net.ParseIP("fe80::1"), Mask: net.CIDRMask(64, 128)},
&net.IPAddr{IP: net.ParseIP("fd00::5")},
&net.IPNet{},
otherAddr{},
nil,
}
// Loopback, link local and anything without an IP are dropped, and IPv4 comes back unmapped
// even though net.ParseIP hands over a 16 byte slice.
assert.Equal(t, []netip.Addr{
netip.MustParseAddr("10.0.0.5"),
netip.MustParseAddr("fd00::5"),
}, allowed(allowAll{}, addrs))
// The filter sees the unmapped form and can drop a single address.
al := newTestAllowList("none")
al.bannedAddr = netip.MustParseAddr("10.0.0.5")
assert.Equal(t, []netip.Addr{netip.MustParseAddr("fd00::5")}, allowed(al, addrs))
}
// otherAddr is a net.Addr that is neither *net.IPNet nor *net.IPAddr. netlink.Addr is one such
// type: it satisfies net.Addr through an embedded *net.IPNet, and passing it directly once
// silently produced no addresses.
type otherAddr struct{}
func (otherAddr) Network() string { return "other" }
func (otherAddr) String() string { return "other" }
// hostFilter counts AllowName calls and can reject every name or every address.
type hostFilter struct {
rejectNames bool
rejectAddrs bool
asked map[string]int
}
func (f *hostFilter) Allow(netip.Addr) bool { return !f.rejectAddrs }
func (f *hostFilter) AllowName(name string) bool {
f.asked[name]++
return !f.rejectNames
}
// Runs this platform's localAddrs against the host and checks it honors the whole filter.
func TestLocalAddrsAppliesFilter(t *testing.T) {
ctx, l := context.Background(), test.NewLogger()
f := &hostFilter{asked: map[string]int{}}
all, err := localAddrs(ctx, l, f)
if len(f.asked) == 0 {
t.Skipf("no interfaces to enumerate here: %v", err)
}
for name, n := range f.asked {
assert.Equal(t, 1, n, "AllowName asked %d times for %s", n, name)
}
// Loopback and link-local never come back, whatever the filter says.
for _, addr := range all {
assert.False(t, addr.IsLoopback() || addr.IsLinkLocalUnicast(), "%s", addr)
}
none, _ := localAddrs(ctx, l, &hostFilter{rejectNames: true, asked: map[string]int{}})
assert.Empty(t, none, "addresses got past a filter that rejects every interface name")
none, _ = localAddrs(ctx, l, &hostFilter{rejectAddrs: true, asked: map[string]int{}})
assert.Empty(t, none, "addresses got past a filter that rejects every address")
}
+19 -2
View File
@@ -354,7 +354,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
// and cannot drift into misreading an unknown protocol (SCTP, GRE, etc.) as a forged transport.
proto, offset, isFragment, anyFragment, err := iputil.IPv6FindUpperProtocol(data)
if err != nil {
return ErrIPv6PacketTooShort
return err
}
fp.Protocol = proto
@@ -477,7 +477,7 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounte
return
}
dropReason := f.firewall.Drop(rxc.fwPacket.Packet, true, hostinfo, f.pki.GetCAPool(), rxc.ctCache.Get())
dropReason := f.allowInbound(hostinfo, rxc)
if dropReason != nil {
f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, rxc.nb, rxc.scratch, rxc.q)
if f.l.Enabled(context.Background(), slog.LevelDebug) {
@@ -492,6 +492,23 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounte
}
}
// allowInbound runs the inbound firewall check on rxc.fwPacket. It skips the check when the packet belongs to
// the same flow from the same host as the last one allowed in this batch.
func (f *Interface) allowInbound(hostinfo *HostInfo, rxc *rxContext) error {
fp := rxc.fwPacket.Packet
if hostinfo == rxc.lastAllowedHost && fp == rxc.lastAllowed {
return nil
}
if err := f.firewall.Drop(fp, true, hostinfo, f.pki.GetCAPool()); err != nil {
return err
}
rxc.lastAllowed = fp
rxc.lastAllowedHost = hostinfo
return nil
}
func (f *Interface) maybeSendRecvError(endpoint netip.AddrPort, index uint32) {
if f.sendRecvErrorConfig.ShouldRecvError(endpoint) {
f.sendRecvError(endpoint, index)
+57 -7
View File
@@ -6,12 +6,16 @@ import (
"net"
"net/netip"
"testing"
"time"
"github.com/gaissmai/bart"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/ipv4"
@@ -117,7 +121,7 @@ func Test_newPacket_v6(t *testing.T) {
require.NoError(t, err)
err = newPacket(buffer.Bytes(), true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
// A v6 packet with a hop-by-hop extension
// ICMPv6 Payload (Echo Request)
@@ -151,12 +155,12 @@ func Test_newPacket_v6(t *testing.T) {
// A full IPv6 header and 1 byte in the first extension, but missing
// the length byte.
err = newPacket(buffer.Bytes()[:41], true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
// A full IPv6 header plus 1 full extension, but only 1 byte of the
// next layer, missing length byte
err = newPacket(buffer.Bytes()[:49], true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
err = nil
// A good ICMP packet
@@ -354,12 +358,12 @@ func Test_newPacket_v6(t *testing.T) {
// Ensure buffer bounds checking during processing, a truncated AH header can't reach the payload
err = newPacket(b[:41], true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
// Invalid AH header
b = buffer.Bytes()
err = newPacket(b, true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
}
func Test_newPacket_ipv6Fragment(t *testing.T) {
@@ -466,7 +470,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
// Too short of a fragment packet
err = newPacket(secondFrag[:len(secondFrag)-10], false, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
}
func BenchmarkParseV6(b *testing.B) {
@@ -707,7 +711,7 @@ func Test_newPacket_v6ExtHeaderPastBuffer(t *testing.T) {
pkt[40] = byte(layers.IPProtocolSCTP) // Dest Options next header = SCTP
pkt[41] = 255 // declared length (255+1)*8 = 2048, past the 48 byte buffer
require.ErrorIs(t, newPacket(pkt, true, p), ErrIPv6PacketTooShort)
require.ErrorIs(t, newPacket(pkt, true, p), iputil.ErrIPv6CouldNotFindPayload)
}
// Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized
@@ -841,3 +845,49 @@ func Test_newPacket_parsedFields(t *testing.T) {
assert.True(t, p.Fragment)
assert.True(t, p.FragAny)
}
func TestInterface_allowInbound(t *testing.T) {
l := test.NewLoggerWithOutput(&bytes.Buffer{})
myNetworks := new(bart.Lite)
myNetworks.Insert(netip.MustParsePrefix("192.0.2.1/24"))
me := &dummyCert{name: "me", networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")}}
peer := &cert.CachedCertificate{Certificate: &dummyCert{name: "peer", networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")}}}
h := &HostInfo{ConnectionState: &ConnectionState{peerCert: peer}, vpnAddrs: []netip.Addr{netip.MustParseAddr("192.0.2.2")}}
h.buildNetworks(myNetworks, peer.Certificate)
allowAll := NewFirewall(l, time.Minute, time.Minute, time.Minute, me)
require.NoError(t, allowAll.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
allowNone := NewFirewall(l, time.Minute, time.Minute, time.Minute, me)
f := &Interface{firewall: allowAll, pki: &PKI{}, l: l}
rxc := &rxContext{fwPacket: &firewall.ParsedPacket{}, hostmapCache: map[uint32]*HostInfo{}}
flow := firewall.Packet{
LocalAddr: netip.MustParseAddr("192.0.2.1"),
RemoteAddr: netip.MustParseAddr("192.0.2.2"),
LocalPort: 443,
RemotePort: 55000,
Protocol: iputil.IPProtocolUDP,
}
rxc.fwPacket.Packet = flow
require.NoError(t, f.allowInbound(h, rxc))
// With a firewall that allows nothing, the rest of the flow's run in this batch still passes
f.firewall = allowNone
require.NoError(t, f.allowInbound(h, rxc))
// A different flow is checked
other := flow
other.RemotePort++
rxc.fwPacket.Packet = other
require.ErrorIs(t, f.allowInbound(h, rxc), ErrNoMatchingRule)
// So is the same flow from a different host
rxc.fwPacket.Packet = flow
h2 := &HostInfo{ConnectionState: h.ConnectionState, vpnAddrs: h.vpnAddrs}
h2.buildNetworks(myNetworks, peer.Certificate)
require.ErrorIs(t, f.allowInbound(h2, rxc), ErrNoMatchingRule)
// And the same flow in the next batch
rxc.endBatch()
require.ErrorIs(t, f.allowInbound(h, rxc), ErrNoMatchingRule)
}
+61 -3
View File
@@ -29,7 +29,10 @@ type tun struct {
Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr *netroute.LinkAddr
l *slog.Logger
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
// and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool
l *slog.Logger
}
type ifReq struct {
@@ -149,8 +152,48 @@ func (t *tun) deviceBytes() (o [16]byte) {
return
}
func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
return nil, fmt.Errorf("newTunFromFd not supported in Darwin")
// newTunFromFd adopts a utun the host already created and configured, which is how a darwin
// network extension is handed its device. Everything about moving packets is shared with newTun,
// only the setup differs: the host owns addressing and routing here.
func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
if err := unix.SetNonblock(deviceFd, true); err != nil {
// We own the fd from the moment it is handed to us
_ = unix.Close(deviceFd)
return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err)
}
file := os.NewFile(uintptr(deviceFd), "/dev/tun")
t := &tun{
f: file,
Device: utunNameFromFd(deviceFd),
vpnNetworks: vpnNetworks,
DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
hostOwned: true,
l: l,
}
if err := t.reload(c, true); err != nil {
_ = file.Close()
return nil, err
}
c.RegisterReloadCallback(func(c *config.C) {
if err := t.reload(c, false); err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
}
})
return t, nil
}
// utunNameFromFd asks the socket what interface it is, for logs. A blank name is not worth
// failing a tunnel over, so an error just leaves it empty.
func utunNameFromFd(fd int) string {
name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME)
if err != nil {
return ""
}
return name
}
func (t *tun) Close() error {
@@ -161,6 +204,12 @@ func (t *tun) Close() error {
}
func (t *tun) Activate() error {
// The host handed us a configured device. Its addresses, mtu and routes come from the network
// settings it applied, and a sandboxed extension cannot change them anyway.
if t.hostOwned {
return nil
}
devName := t.deviceBytes()
s, err := unix.Socket(
@@ -374,6 +423,11 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
}
func (t *tun) addRoutes(logErrors bool) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
routes := *t.Routes.Load()
for _, r := range routes {
@@ -403,6 +457,10 @@ func (t *tun) addRoutes(logErrors bool) error {
}
func (t *tun) removeRoutes(routes []Route) error {
if t.hostOwned {
return nil
}
for _, r := range routes {
if !r.Install {
continue
+1 -1
View File
@@ -15,12 +15,12 @@ import (
"time"
"unsafe"
"github.com/DefinedNet/netlink"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/util"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
)