Compare commits

..
Author SHA1 Message Date
JackDoan 04e80f0dcf improvements to config parsing 2026-10-01 10:18:30 -05:00
JackDoan ed8ae57fee checkpt 2026-09-30 15:06:35 -05:00
JackDoan e99e48cca7 checkpt 2026-09-30 14:57:19 -05:00
JackDoan 027c728702 checkpt 2026-09-30 14:52:14 -05:00
JackDoan 1cfed59214 checkpt 2026-09-30 14:46:44 -05:00
JackDoan c4b9df20ac u64->u8 2026-09-30 14:13:21 -05:00
JackDoan 92bad7460b checkpt 2026-09-30 14:04:05 -05:00
JackDoan cac7586c0b checkpt 2026-09-30 14:03:26 -05:00
JackDoan 9aab46c1ad no bart for cidr 2026-09-30 12:24:01 -05:00
JackDoan f3b76703ae maybe better 2026-09-29 19:51:08 -05:00
JackDoan fc0dbfec4c maybe better 2026-09-29 19:49:27 -05:00
JackDoan ca69a0613b memory oh baby 2026-09-29 19:38:53 -05:00
JackDoan 84a26846f4 refactor checkpt 2026-09-29 18:41:01 -05:00
JackDoan 317388e37d refactor checkpt 2026-09-29 18:36:48 -05:00
JackDoan 06fd191367 stuff 2026-09-29 18:04:53 -05:00
JackDoan 3f392883ca stuff 2026-09-29 17:52:13 -05:00
JackDoan 30cf2bbe6c Don't check port any firewall rules twice for packets without ports 2026-09-29 17:38:49 -05:00
JackDoan 11b04517f3 Fold proto any firewall rules into each protocol's rules so a packet checks one table 2026-09-29 17:38:19 -05:00
JackDoan 610ba57105 Allow firewall rules on any IP protocol number 2026-09-29 17:24:09 -05:00
JackDoan 9e96894bd7 Pass firewall packets by pointer through conntrack and rule matching 2026-09-29 17:18:11 -05:00
JackDoan 14f80450fb allow more protos to have ports parsed 2026-09-29 15:51:59 -05:00
JackDoan 641d1fb703 Clarify which traffic port rules match 2026-09-29 13:17:21 -05:00
JackDoan 11a4298c36 Only read IPv4 ports from TCP and UDP 2026-09-29 13:02:30 -05:00
108 changed files with 2940 additions and 6476 deletions

No files matched your search

+3 -3
View File
@@ -14,7 +14,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -40,7 +40,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -83,7 +83,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
# GitHub holds ARNs, not credentials, and ARNs outlive a rotation # GitHub holds ARNs, not credentials, and ARNs outlive a rotation
+3 -3
View File
@@ -34,7 +34,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -66,7 +66,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -92,7 +92,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own # WSL2 + Ubuntu so the smoke can run a real linux peer with its own
+2 -2
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Smoke Docker - name: Smoke Docker
@@ -51,7 +51,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: build - name: build
+4 -4
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Install goimports - name: Install goimports
@@ -42,7 +42,7 @@ jobs:
- name: golangci-lint - name: golangci-lint
uses: golangci/golangci-lint-action@v9 uses: golangci/golangci-lint-action@v9
with: with:
version: v2.14 version: v2.12
test: test:
name: Test ${{ matrix.name }} name: Test ${{ matrix.name }}
@@ -87,7 +87,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -132,7 +132,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build ${{ matrix.name }} - name: Build ${{ matrix.name }}
+12 -9
View File
@@ -7,19 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
### Removed ### Changed
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and - Firewall rules are held as a flat list with a bitset of rule ids per port, instead of a tree of maps copied
`mutex-profile-fraction` commands, and `sshd.sandbox_dir`, which is now ignored if set. into every port of a range. A rule like `port: 1024-65535` now costs kilobytes rather than tens of megabytes,
and matching is faster on every path.
- A firewall rule's `proto` can be `udplite`, `dccp`, `sctp`, `icmpv6`, or any IP protocol number. A rule with
a port or port range on a protocol that has no ports, such as `proto: 47`, is rejected rather than silently
never matching. `port: any` and `port: fragment` apply to every protocol.
- An ICMP non-first fragment is matched against `port: fragment` rules like a fragment of any other protocol.
It was only matched against `port: any` rules before.
### Fixed ### Fixed
- Signing a v2 certificate with an empty name, a name longer than 253 bytes, or an empty group now fails. - A firewall rule with `group: []`, `groups:` with no value, or a `groups` entry that isn't a string, such as
These certificates were signed successfully but could not be loaded. `groups: [1, 2]`, no longer crashes Nebula on startup or reload. An empty `group` array is rejected, a null
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula `groups` means no groups, and non-string entries are read as strings.
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
the handshake starts, so a pending hostinfo never exists without one.
## [1.11.1] - 2026-08-21 ## [1.11.1] - 2026-08-21
+1 -10
View File
@@ -389,16 +389,7 @@ func (c *certificateV2) fromTBSCertificate(t *TBSCertificate) error {
} }
func (c *certificateV2) validate() error { func (c *certificateV2) validate() error {
// Mirror the name and group rules in unmarshalDetails so signing fails instead of producing an unparseable certificate // Empty names are allowed
if c.details.name == "" || len(c.details.name) > MaxNameLength {
return NewErrInvalidCertificateProperties("name must be between 1 and %d bytes", MaxNameLength)
}
for _, group := range c.details.groups {
if group == "" {
return NewErrInvalidCertificateProperties("groups may not be empty")
}
}
if len(c.publicKey) == 0 { if len(c.publicKey) == 0 {
return ErrInvalidPublicKey return ErrInvalidPublicKey
-45
View File
@@ -6,7 +6,6 @@ import (
"crypto/elliptic" "crypto/elliptic"
"crypto/rand" "crypto/rand"
"net/netip" "net/netip"
"strings"
"testing" "testing"
"time" "time"
@@ -136,47 +135,3 @@ func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
assert.True(t, normie) assert.True(t, normie)
} }
} }
func TestCertificateV2_SignRejectsUnparseable(t *testing.T) {
_, priv, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
var invalid *ErrInvalidCertificateProperties
newTBS := func() TBSCertificate {
return TBSCertificate{
Version: Version2,
Name: "testing",
Networks: []netip.Prefix{mustParsePrefixUnmapped("10.1.1.1/24")},
Groups: []string{"test-group1"},
NotBefore: time.Now().Add(-time.Minute),
NotAfter: time.Now().Add(time.Minute),
PublicKey: []byte("1234567890abcedfghij1234567890ab"),
Curve: Curve_CURVE25519,
IsCA: true,
}
}
tbs := newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength)
c, err := tbs.Sign(nil, Curve_CURVE25519, priv)
require.NoError(t, err)
b, err := c.Marshal()
require.NoError(t, err)
_, err = unmarshalCertificateV2(b, nil, Curve_CURVE25519)
require.NoError(t, err)
tbs = newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength+1)
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Name = ""
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Groups = []string{"test-group1", ""}
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
}
+1 -2
View File
@@ -85,10 +85,9 @@ func main() {
fmt.Printf("failed to apply logging config: %s", err) fmt.Printf("failed to apply logging config: %s", err)
os.Exit(1) os.Exit(1)
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
l.Error("Failed to reconfigure logger on reload", "error", err) l.Error("Failed to reconfigure logger on reload", "error", err)
r.Failed()
} }
}) })
+1 -2
View File
@@ -34,10 +34,9 @@ func (p *program) Start(s service.Service) error {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
return fmt.Errorf("failed to apply logging config: %s", err) return fmt.Errorf("failed to apply logging config: %s", err)
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
l.Error("Failed to reconfigure logger on reload", "error", err) l.Error("Failed to reconfigure logger on reload", "error", err)
r.Failed()
} }
}) })
+1 -2
View File
@@ -71,10 +71,9 @@ func main() {
fmt.Printf("failed to apply logging config: %s", err) fmt.Printf("failed to apply logging config: %s", err)
os.Exit(1) os.Exit(1)
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
l.Error("Failed to reconfigure logger on reload", "error", err) l.Error("Failed to reconfigure logger on reload", "error", err)
r.Failed()
} }
}) })
+31 -40
View File
@@ -26,12 +26,9 @@ type C struct {
files []string files []string
Settings map[string]any Settings map[string]any
oldSettings map[string]any oldSettings map[string]any
callbacks []func(*C, *Reload) callbacks []func(*C)
l *slog.Logger l *slog.Logger
reloadLock sync.Mutex reloadLock sync.Mutex
// Set by the first reload that needs a restart and never cleared. Guarded by reloadLock
needsRestart bool
} }
func NewC(l *slog.Logger) *C { func NewC(l *slog.Logger) *C {
@@ -75,9 +72,8 @@ func (c *C) LoadString(raw string) error {
// RegisterReloadCallback stores a function to be called when a config reload is triggered. The functions registered // RegisterReloadCallback stores a function to be called when a config reload is triggered. The functions registered
// here should decide if they need to make a change to the current process before making the change. HasChanged can be // here should decide if they need to make a change to the current process before making the change. HasChanged can be
// used to help decide if a change is necessary. // used to help decide if a change is necessary.
// These functions should return quickly or spawn their own go routine if they will take a while. // These functions should return quickly or spawn their own go routine if they will take a while
// Use r to report anything that failed or needs a restart to apply. func (c *C) RegisterReloadCallback(f func(*C)) {
func (c *C) RegisterReloadCallback(f func(c *C, r *Reload)) {
c.callbacks = append(c.callbacks, f) c.callbacks = append(c.callbacks, f)
} }
@@ -148,55 +144,50 @@ func (c *C) CatchHUP(ctx context.Context) {
return return
case <-ch: case <-ch:
c.l.Info("Caught HUP, reloading config") c.l.Info("Caught HUP, reloading config")
c.ReloadConfig().Log(c.l) c.ReloadConfig()
} }
} }
}() }()
} }
// ReloadConfig reloads the config from the path provided to Load and runs every registered callback. A config that func (c *C) ReloadConfig() {
// fails to load is logged here, like any other failure, and the result carries the error for the summary.
func (c *C) ReloadConfig() ReloadResult {
c.reloadLock.Lock() c.reloadLock.Lock()
defer c.reloadLock.Unlock() defer c.reloadLock.Unlock()
c.oldSettings = make(map[string]any) c.oldSettings = make(map[string]any)
maps.Copy(c.oldSettings, c.Settings) maps.Copy(c.oldSettings, c.Settings)
if err := c.Load(c.path); err != nil { err := c.Load(c.path)
c.l.Error("Error occurred while reloading config", "config_path", c.path, "error", err) if err != nil {
return ReloadResult{Err: err, RestartRequired: c.needsRestart} c.l.Error("Error occurred while reloading config",
"config_path", c.path,
"error", err,
)
return
} }
return c.runCallbacks()
}
// ReloadConfigString is ReloadConfig for a config held in memory rather than on disk.
func (c *C) ReloadConfigString(raw string) ReloadResult {
c.reloadLock.Lock()
defer c.reloadLock.Unlock()
c.oldSettings = make(map[string]any)
maps.Copy(c.oldSettings, c.Settings)
if err := c.LoadString(raw); err != nil {
c.l.Error("Error occurred while reloading config", "error", err)
return ReloadResult{Err: err, RestartRequired: c.needsRestart}
}
return c.runCallbacks()
}
func (c *C) runCallbacks() ReloadResult {
r := &Reload{c: c}
for _, v := range c.callbacks { for _, v := range c.callbacks {
v(c, r) v(c)
}
}
func (c *C) ReloadConfigString(raw string) error {
c.reloadLock.Lock()
defer c.reloadLock.Unlock()
c.oldSettings = make(map[string]any)
maps.Copy(c.oldSettings, c.Settings)
err := c.LoadString(raw)
if err != nil {
return err
} }
if r.restart { for _, v := range c.callbacks {
c.needsRestart = true v(c)
} }
return ReloadResult{Failed: r.failed, RestartRequired: c.needsRestart, RestartKeys: r.keys}
return nil
} }
// GetString will get the string for k or return the default d if not found or invalid // GetString will get the string for k or return the default d if not found or invalid
@@ -400,7 +391,7 @@ func (c *C) parse() error {
var nm map[string]any var nm map[string]any
err = yaml.Unmarshal(b, &nm) err = yaml.Unmarshal(b, &nm)
if err != nil { if err != nil {
return fmt.Errorf("%s: %w", path, err) return err
} }
// We need to use WithAppendSlice so that firewall rules in separate // We need to use WithAppendSlice so that firewall rules in separate
+2 -2
View File
@@ -19,7 +19,7 @@ func TestConfig_Load(t *testing.T) {
// invalid yaml // invalid yaml
c := NewC(l) c := NewC(l)
os.WriteFile(filepath.Join(dir, "01.yaml"), []byte(" invalid yaml"), 0644) os.WriteFile(filepath.Join(dir, "01.yaml"), []byte(" invalid yaml"), 0644)
require.EqualError(t, c.Load(dir), filepath.Join(dir, "01.yaml")+": yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into map[string]interface {}") require.EqualError(t, c.Load(dir), "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into map[string]interface {}")
// simple multi config merge // simple multi config merge
c = NewC(l) c = NewC(l)
@@ -129,7 +129,7 @@ func TestConfig_ReloadConfig(t *testing.T) {
os.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n inner: ho"), 0644) os.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n inner: ho"), 0644)
c.RegisterReloadCallback(func(c *C, _ *Reload) { c.RegisterReloadCallback(func(c *C) {
done <- true done <- true
}) })
-98
View File
@@ -1,98 +0,0 @@
package config
import (
"context"
"fmt"
"log/slog"
"slices"
)
// ReloadResult describes how a reload went. Problems are logged where they happen, Log writes a one line summary.
type ReloadResult struct {
// Err is set when the new config could not be read or parsed, no callbacks ran
Err error
// Failed is true when a callback could not apply part of the new config
Failed bool
// RestartRequired is true once any reload has needed a restart, and stays true until the process restarts
RestartRequired bool
// RestartKeys are the keys this reload changed that only apply on a restart
RestartKeys []string
}
// Summary returns a one line summary of the reload and the level to log it at.
func (r ReloadResult) Summary() (slog.Level, string) {
switch {
case r.Err != nil && r.RestartRequired:
return slog.LevelError, "Config reload failed, the new config did not load and a restart is still required"
case r.Err != nil:
return slog.LevelError, "Config reload failed, the new config did not load"
case r.Failed && r.RestartRequired:
return slog.LevelError, "Config reload completed with errors and requires a restart to fully apply, review the logs"
case r.Failed:
return slog.LevelError, "Config reload completed with errors, review the logs"
case r.RestartRequired:
return slog.LevelWarn, "Config reload was completed but requires a restart to fully apply"
default:
return slog.LevelInfo, "Config reload complete"
}
}
// String is the summary with Err and RestartKeys included, for output other than the log.
func (r ReloadResult) String() string {
_, msg := r.Summary()
if r.Err != nil {
msg = fmt.Sprintf("%s: %v", msg, r.Err)
}
if len(r.RestartKeys) > 0 {
msg = fmt.Sprintf("%s, keys %v", msg, r.RestartKeys)
}
return msg
}
// Log writes Summary to l.
func (r ReloadResult) Log(l *slog.Logger) {
level, msg := r.Summary()
if len(r.RestartKeys) > 0 {
l.Log(context.Background(), level, msg, "keys", r.RestartKeys)
return
}
l.Log(context.Background(), level, msg)
}
// Reload is passed to each reload callback to report how applying the new config went.
type Reload struct {
c *C
failed bool
restart bool
// Changed keys that only apply on a restart, from every callback
keys []string
}
// Failed marks the reload as failed, the caller logs why. Safe to call on a nil Reload, which is what the initial load
// passes.
func (r *Reload) Failed() {
if r != nil {
r.failed = true
}
}
// RestartRequired marks the reload as needing a restart, the caller logs why. Safe to call on a nil Reload.
func (r *Reload) RestartRequired() {
if r != nil {
r.restart = true
}
}
// RestartRequiredIfChanged marks the reload as needing a restart if any of keys changed, and adds them to RestartKeys.
// Use it for keys that are only read at startup. Safe to call on a nil Reload.
func (r *Reload) RestartRequiredIfChanged(keys ...string) {
if r == nil {
return
}
for _, k := range keys {
if r.c.HasChanged(k) && !slices.Contains(r.keys, k) {
r.keys = append(r.keys, k)
r.restart = true
}
}
}
-108
View File
@@ -1,108 +0,0 @@
package config
import (
"errors"
"log/slog"
"testing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReloadConfigString(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1"))
fail := false
ran := 0
c.RegisterReloadCallback(func(c *C, r *Reload) {
ran++
if fail {
r.Failed()
}
})
assert.Equal(t, ReloadResult{}, c.ReloadConfigString("a: 2"))
fail = true
assert.Equal(t, ReloadResult{Failed: true}, c.ReloadConfigString("a: 3"))
assert.Equal(t, 2, ran)
// A config that fails to load runs no callbacks and returns the error
res := c.ReloadConfigString("a: [")
require.Error(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, 2, ran)
assert.Equal(t, 3, c.GetInt("a", 0))
}
func TestReloadRestartRequiredLatches(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1\nb: 1"))
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a", "b") })
// Nothing that needs a restart changed
assert.Equal(t, ReloadResult{}, c.ReloadConfigString("a: 1\nb: 1\nc: 1"))
assert.Equal(t, ReloadResult{RestartRequired: true, RestartKeys: []string{"a"}}, c.ReloadConfigString("a: 2\nb: 1\nc: 1"))
// The restart sticks through a reload with no changes, reverting the key and a config that fails to load. Only the
// reload that changed the key lists it
assert.Equal(t, ReloadResult{RestartRequired: true}, c.ReloadConfigString("a: 2\nb: 1\nc: 1"))
assert.Equal(t, ReloadResult{RestartRequired: true, RestartKeys: []string{"a"}}, c.ReloadConfigString("a: 1\nb: 1\nc: 1"))
res := c.ReloadConfigString("a: [")
require.Error(t, res.Err)
assert.True(t, res.RestartRequired)
}
// A key reported by more than one callback is listed once
func TestReloadRestartKeysDeduped(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1\nb: 1"))
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a") })
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a", "b") })
assert.Equal(t, []string{"a", "b"}, c.ReloadConfigString("a: 2\nb: 2").RestartKeys)
}
// A callback can require a restart without a key changing, a file the config points at may have
func TestReloadRestartRequiredWithoutAChange(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1"))
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequired() })
assert.Equal(t, ReloadResult{RestartRequired: true}, c.ReloadConfigString("a: 1"))
}
func TestReloadResultSummary(t *testing.T) {
cases := []struct {
res ReloadResult
level slog.Level
msg string
}{
{ReloadResult{}, slog.LevelInfo, "Config reload complete"},
{ReloadResult{RestartRequired: true}, slog.LevelWarn, "Config reload was completed but requires a restart to fully apply"},
{ReloadResult{Failed: true}, slog.LevelError, "Config reload completed with errors, review the logs"},
{ReloadResult{Failed: true, RestartRequired: true}, slog.LevelError, "Config reload completed with errors and requires a restart to fully apply, review the logs"},
{ReloadResult{Err: errors.New("bad yaml")}, slog.LevelError, "Config reload failed, the new config did not load"},
{ReloadResult{Err: errors.New("bad yaml"), RestartRequired: true}, slog.LevelError, "Config reload failed, the new config did not load and a restart is still required"},
}
for _, tc := range cases {
level, msg := tc.res.Summary()
assert.Equal(t, tc.level, level, tc.msg)
assert.Equal(t, tc.msg, msg)
}
res := ReloadResult{RestartRequired: true, RestartKeys: []string{"a", "b"}}
assert.Equal(t, "Config reload was completed but requires a restart to fully apply, keys [a b]", res.String())
// String includes the error
assert.Equal(t, "Config reload failed, the new config did not load: bad yaml", ReloadResult{Err: errors.New("bad yaml")}.String())
}
// The initial load passes a nil Reload
func TestReloadNilIsSafe(t *testing.T) {
var r *Reload
r.Failed()
r.RestartRequired()
r.RestartRequiredIfChanged("a")
}
+5 -4
View File
@@ -57,8 +57,7 @@ func newConnectionManagerFromConfig(l *slog.Logger, c *config.C, hm *HostMap, p
} }
cm.reload(c, true) cm.reload(c, true)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("timers.connection_alive_interval", "timers.pending_deletion_interval")
cm.reload(c, false) cm.reload(c, false)
}) })
@@ -175,8 +174,10 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
switch decision { switch decision {
case deleteTunnel: case deleteTunnel:
// Local delete only, clearing the lighthouse cache if this was the last hostinfo for the vpn addr if cm.hostMap.DeleteHostInfo(hostinfo) {
cm.intf.closeTunnel(hostinfo) // Only clearing the lighthouse cache if this is the last hostinfo for this vpn ip in the hostmap
cm.intf.lightHouse.DeleteVpnAddrs(hostinfo.vpnAddrs)
}
case closeTunnel: case closeTunnel:
cm.intf.sendCloseTunnel(hostinfo) cm.intf.sendCloseTunnel(hostinfo)
-3
View File
@@ -7,7 +7,6 @@ import (
"testing" "testing"
"time" "time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/overlay/overlaytest" "github.com/slackhq/nebula/overlay/overlaytest"
@@ -22,8 +21,6 @@ func newTestLighthouse() *LightHouse {
l: test.NewLogger(), l: test.NewLogger(),
addrMap: map[netip.Addr]*RemoteList{}, addrMap: map[netip.Addr]*RemoteList{},
queryChan: make(chan netip.Addr, 10), queryChan: make(chan netip.Addr, 10),
// Nothing drains queryChan here, so an 11th query is dropped.
queryDropped: metrics.NilCounter{},
} }
lighthouses := []netip.Addr{} lighthouses := []netip.Addr{}
staticList := map[netip.Addr]struct{}{} staticList := map[netip.Addr]struct{}{}
-119
View File
@@ -2,8 +2,6 @@ package nebula
import ( import (
"net/netip" "net/netip"
"strconv"
"sync"
"testing" "testing"
"time" "time"
@@ -13,7 +11,6 @@ import (
ct "github.com/slackhq/nebula/cert_test" ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake" "github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -150,122 +147,6 @@ func TestSendNoMetricsCloseTunnelKeepsRebindEpoch(t *testing.T) {
assert.True(t, hostinfo.markOut(1), "a CloseTunnel send must not consume the rebind epoch") assert.True(t, hostinfo.markOut(1), "a CloseTunnel send must not consume the rebind epoch")
} }
// capturingConn is a udp.Conn that records every WriteTo.
type capturingConn struct {
udp.NoopConn
writes [][]byte
addrs []netip.AddrPort
}
func (c *capturingConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.writes = append(c.writes, append([]byte(nil), b...))
c.addrs = append(c.addrs, addr)
return nil
}
// TestSendNoMetricsViaRelay sends through a relay from the mtu sized buffer the cached packet flush uses. Payloads up
// to overlay.MaxMTU go out intact, anything bigger is dropped rather than outgrowing the buffer, which once panicked.
func TestSendNoMetricsViaRelay(t *testing.T) {
tests := []struct {
n int
sent bool
}{
{0, true},
{1300, true},
{overlay.MaxMTU, true},
{overlay.MaxMTU + 1, false}, // too big for the relay's tag
{mtu - 47, false}, // too big for the inner tag, this panicked
{9000, false}, // a full packet at tun.mtu 9000
}
for _, tt := range tests {
t.Run(strconv.Itoa(tt.n), func(t *testing.T) {
// One tunnel to the target, carried end to end, and one to the relay that wraps it
targetInit, targetResp := runTestHandshake(t)
relayInit, relayResp := runTestHandshake(t)
ci, err := newConnectionStateFromResult(targetInit)
require.NoError(t, err)
relayCI, err := newConnectionStateFromResult(relayInit)
require.NoError(t, err)
targetAddr := netip.MustParseAddr("10.0.0.2")
relayAddr := netip.MustParseAddr("10.0.0.3")
relayRemote := netip.MustParseAddrPort("192.0.2.1:4242")
relay := &Relay{Type: TerminalType, State: Established, LocalIndex: 5, RemoteIndex: 6, PeerAddr: targetAddr}
relayHI := &HostInfo{
vpnAddrs: []netip.Addr{relayAddr},
ConnectionState: relayCI,
relayState: RelayState{
relayForByAddr: map[netip.Addr]*Relay{targetAddr: relay},
relayForByIdx: map[uint32]*Relay{relay.LocalIndex: relay},
},
}
relayHI.remote.Store(&relayRemote)
// No remote of its own, so the send has to go via the relay
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{targetAddr},
ConnectionState: ci,
remoteIndexId: 7,
relayState: RelayState{
relays: []netip.Addr{relayAddr},
relayForByAddr: map[netip.Addr]*Relay{},
relayForByIdx: map[uint32]*Relay{},
},
}
l := test.NewLogger()
hm := newHostMap(l)
hm.Hosts[relayAddr] = relayHI
conn := &capturingConn{}
f := &Interface{
l: l,
hostMap: hm,
messageMetrics: &MessageMetrics{txExhausted: metrics.NewCounter()},
writers: []udp.Conn{conn},
}
f.connectionManager = &connectionManager{intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{}}
payload := make([]byte, tt.n)
for i := range payload {
payload[i] = byte(i)
}
counter := ci.messageCounter.Load()
f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, payload, make([]byte, 12), make([]byte, mtu), 0)
if !tt.sent {
assert.Empty(t, conn.writes)
assert.Equal(t, counter, ci.messageCounter.Load(), "a dropped packet must not spend a counter")
return
}
require.Len(t, conn.writes, 1)
assert.Equal(t, relayRemote, conn.addrs[0])
pkt := conn.writes[0]
// The relay authenticates the whole packet under its own header
outer := &header.H{}
require.NoError(t, outer.Parse(pkt))
assert.Equal(t, header.MessageRelay, outer.Subtype)
assert.Equal(t, relay.RemoteIndex, outer.RemoteIndex)
relayPeer, err := newConnectionStateFromResult(relayResp)
require.NoError(t, err)
require.NoError(t, relayPeer.VerifyRelay(l, outer.MessageCounter, pkt, make([]byte, 12)))
// Inside it, the target decrypts the end to end packet back to the payload
inner := pkt[header.Len : len(pkt)-relayPeer.dKey.Overhead()]
h := &header.H{}
require.NoError(t, h.Parse(inner))
assert.Equal(t, header.Message, h.Type)
assert.Equal(t, hostinfo.remoteIndexId, h.RemoteIndex)
targetPeer, err := newConnectionStateFromResult(targetResp)
require.NoError(t, err)
got, err := targetPeer.Decrypt(l, h.MessageCounter, inner, make([]byte, 12))
require.NoError(t, err)
assert.Equal(t, payload, got)
})
}
}
func TestNewConnectionStateFromResult(t *testing.T) { func TestNewConnectionStateFromResult(t *testing.T) {
initR, respR := runTestHandshake(t) initR, respR := runTestHandshake(t)
+3 -3
View File
@@ -151,9 +151,9 @@ func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) { func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
return len(bufs), nil return len(bufs), nil
} }
func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {} func (c *fakeConn) ReloadConfig(_ *config.C) {}
func (c *fakeConn) SupportsMultipleReaders() bool { return true } func (c *fakeConn) SupportsMultipleReaders() bool { return true }
func (c *fakeConn) Close() error { c.closed = true; return nil } func (c *fakeConn) Close() error { c.closed = true; return nil }
type multiqueueDevice struct { type multiqueueDevice struct {
*fakeDevice *fakeDevice
+1 -2
View File
@@ -69,10 +69,9 @@ func newDnsServerFromConfig(ctx context.Context, l *slog.Logger, pki *PKI, hostM
ds.mux = dns.NewServeMux() ds.mux = dns.NewServeMux()
ds.mux.HandleFunc(".", ds.handleDnsRequest) ds.mux.HandleFunc(".", ds.handleDnsRequest)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
if err := ds.reload(c, false); err != nil { if err := ds.reload(c, false); err != nil {
ds.l.Error("Failed to reload DNS responder from config", "error", err) ds.l.Error("Failed to reload DNS responder from config", "error", err)
r.Failed()
} }
}) })
+1 -1
View File
@@ -1513,7 +1513,7 @@ func TestLighthouseUpdateOnReload(t *testing.T) {
// Reload the config. The lighthouse.hosts change triggers TriggerUpdate, // Reload the config. The lighthouse.hosts change triggers TriggerUpdate,
// which wakes the update worker. It calls SendUpdate, initiating a // which wakes the update worker. It calls SendUpdate, initiating a
// handshake to the new lighthouse and caching the HostUpdateNotification. // handshake to the new lighthouse and caching the HostUpdateNotification.
require.NoError(t, myConfig.ReloadConfigString(string(newCfg)).Err) require.NoError(t, myConfig.ReloadConfigString(string(newCfg)))
// Route until the lighthouse receives the HostUpdateNotification. // Route until the lighthouse receives the HostUpdateNotification.
// This covers: handshake stage 1, stage 2, then the cached update. // This covers: handshake stage 1, stage 2, then the cached update.
+2 -2
View File
@@ -129,7 +129,7 @@ func TestCertUpgrade(t *testing.T) {
} }
r.Logf("reload new v2-only config") r.Logf("reload new v2-only config")
err = myC.ReloadConfigString(string(cb)).Err err = myC.ReloadConfigString(string(cb))
assert.NoError(t, err) assert.NoError(t, err)
r.Log("yay, spin until their sees it") r.Log("yay, spin until their sees it")
waitStart := time.Now() waitStart := time.Now()
@@ -222,7 +222,7 @@ func TestCertDowngrade(t *testing.T) {
} }
r.Logf("reload new v1-only config") r.Logf("reload new v1-only config")
err = myC.ReloadConfigString(string(cb)).Err err = myC.ReloadConfigString(string(cb))
assert.NoError(t, err) assert.NoError(t, err)
r.Log("yay, spin until their sees it") r.Log("yay, spin until their sees it")
waitStart := time.Now() waitStart := time.Now()
+25 -27
View File
@@ -8,7 +8,6 @@ pki:
cert: /etc/nebula/host.crt cert: /etc/nebula/host.crt
key: /etc/nebula/host.key key: /etc/nebula/host.key
# blocklist is a list of certificate fingerprints that we will refuse to talk to # blocklist is a list of certificate fingerprints that we will refuse to talk to
# Entries are the hex fingerprints shown by `nebula-cert print`, an invalid entry fails startup and reload.
#blocklist: #blocklist:
# - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72 # - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72
# disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid. # disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid.
@@ -44,7 +43,7 @@ static_host_map:
lighthouse: lighthouse:
# am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes # am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
# you have configured to be lighthouses in your network. Not reloadable. # you have configured to be lighthouses in your network
am_lighthouse: false am_lighthouse: false
# serve_dns optionally starts a dns listener that responds to various queries and can even be # serve_dns optionally starts a dns listener that responds to various queries and can even be
# delegated to for resolution # delegated to for resolution
@@ -127,9 +126,8 @@ lighthouse:
# Port Nebula will be listening on. The default here is 4242. For a lighthouse node, the port should be defined, # Port Nebula will be listening on. The default here is 4242. For a lighthouse node, the port should be defined,
# however using port 0 will dynamically assign a port and is recommended for roaming nodes. # however using port 0 will dynamically assign a port and is recommended for roaming nodes.
listen: listen:
# To listen on only ipv4, use "0.0.0.0". Not reloadable. # To listen on only ipv4, use "0.0.0.0"
host: "::" host: "::"
# Not reloadable.
port: 4242 port: 4242
# Sets the max number of packets to pull from the kernel for each syscall (under systems that support recvmmsg) # Sets the max number of packets to pull from the kernel for each syscall (under systems that support recvmmsg)
# default is 64, does not support reload # default is 64, does not support reload
@@ -137,13 +135,12 @@ listen:
# superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the # superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the
# default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts. # default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts.
#batch: 64 #batch: 64
# Linux only. Reloadable, but removing a value leaves the socket at the old size until Nebula restarts.
# Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel # Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
# Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default) # Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
# Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide # Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide
# max, net.core.rmem_max and net.core.wmem_max # max, net.core.rmem_max and net.core.wmem_max
read_buffer: 3412512 #read_buffer: 10485760
write_buffer: 3412512 #write_buffer: 10485760
# On Windows only # On Windows only
# When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port. # When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port.
@@ -182,7 +179,7 @@ listen:
# Currently, this defaults to 1 which means we have 1 tun queue reader and 1 # Currently, this defaults to 1 which means we have 1 tun queue reader and 1
# UDP queue reader. Setting this above one will set IFF_MULTI_QUEUE on the tun # UDP queue reader. Setting this above one will set IFF_MULTI_QUEUE on the tun
# device and SO_REUSEPORT on the UDP socket to allow multiple queues. # device and SO_REUSEPORT on the UDP socket to allow multiple queues.
# This option is only supported on Linux. Not reloadable. # This option is only supported on Linux.
#routines: 1 #routines: 1
punchy: punchy:
@@ -206,7 +203,6 @@ punchy:
# Cipher allows you to choose between the available ciphers for your network. Options are chachapoly or aes # Cipher allows you to choose between the available ciphers for your network. Options are chachapoly or aes
# IMPORTANT: this value must be identical on ALL NODES/LIGHTHOUSES. We do not/will not support use of different ciphers simultaneously! # IMPORTANT: this value must be identical on ALL NODES/LIGHTHOUSES. We do not/will not support use of different ciphers simultaneously!
# Not reloadable.
#cipher: aes #cipher: aes
# Preferred ranges is used to define a hint about the local network ranges, which speeds up discovering the fastest # Preferred ranges is used to define a hint about the local network ranges, which speeds up discovering the fastest
@@ -233,6 +229,12 @@ punchy:
# Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access. # Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
#trusted_cas: #trusted_cas:
#- "ssh public key string" #- "ssh public key string"
# sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
# save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
# and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
# The directory is NOT automatically created.
# Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
#sandbox_dir: /var/tmp/nebula-debug
# EXPERIMENTAL: relay support for networks that can't establish direct connections. # EXPERIMENTAL: relay support for networks that can't establish direct connections.
relay: relay:
@@ -251,25 +253,18 @@ relay:
# Configure the private interface. Note: addr is baked into the nebula certificate # Configure the private interface. Note: addr is baked into the nebula certificate
tun: tun:
# When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root) # When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root)
# Not reloadable.
disabled: false disabled: false
# Name of the device. If not set, a default will be chosen by the OS. # Name of the device. If not set, a default will be chosen by the OS.
# For macOS: if set, must be in the form `utun[0-9]+`. # For macOS: if set, must be in the form `utun[0-9]+`.
# For OpenBSD and NetBSD: if set, must be in the form `tun[0-9]+`, otherwise the first free one is used. # For NetBSD: Required to be set, must be in the form `tun[0-9]+`
# Not reloadable.
dev: nebula1 dev: nebula1
# Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert # Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert
# Not reloadable.
drop_local_broadcast: false drop_local_broadcast: false
# Toggles forwarding of multicast packets. Not reloadable. # Toggles forwarding of multicast packets
drop_multicast: false drop_multicast: false
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 5000 # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500
# Not reloadable. tx_queue: 500
tx_queue: 5000
# Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
# Anything above 8937, here or on a route below, is capped to 8937 with a warning
# Reloadable on Linux only. Where the tun is handed to nebula by a host app (iOS, Android, macOS network extensions)
# the host app owns the MTU.
mtu: 1300 mtu: 1300
# the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable. # the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable.
@@ -362,7 +357,7 @@ tun:
#use_system_route_table: false #use_system_route_table: false
# Buffer size for reading routes updates. 0 means default system buffer size. (/proc/sys/net/core/rmem_default). # Buffer size for reading routes updates. 0 means default system buffer size. (/proc/sys/net/core/rmem_default).
# If using massive routes updates, for example BGP, you may need to increase this value to avoid packet loss. # If using massive routes updates, for example BGP, you may need to increase this value to avoid packet loss.
# SO_RCVBUFFORCE is used to avoid having to raise the system wide max. Not reloadable. # SO_RCVBUFFORCE is used to avoid having to raise the system wide max
#use_system_route_table_buffer_size: 0 #use_system_route_table_buffer_size: 0
# Configure logging level # Configure logging level
@@ -412,16 +407,14 @@ logging:
# Handshakes are sent to all known addresses at each interval with a linear backoff, # Handshakes are sent to all known addresses at each interval with a linear backoff,
# Wait try_interval after the 1st attempt, 2 * try_interval after the 2nd, etc, until the handshake is older than timeout # Wait try_interval after the 1st attempt, 2 * try_interval after the 2nd, etc, until the handshake is older than timeout
# A 100ms interval with the default 10 retries will give a handshake 5.5 seconds to resolve before timing out # A 100ms interval with the default 10 retries will give a handshake 5.5 seconds to resolve before timing out
# Not reloadable.
#try_interval: 100ms #try_interval: 100ms
#retries: 20 #retries: 20
# query_buffer is the size of the buffer channel for querying lighthouses, drops are recorded in the lighthouse.query.dropped metric. # query_buffer is the size of the buffer channel for querying lighthouses
# Not reloadable. #query_buffer: 64
#query_buffer: 1024
# trigger_buffer is the size of the buffer channel for quickly sending handshakes # trigger_buffer is the size of the buffer channel for quickly sending handshakes
# after receiving the response for lighthouse queries. Not reloadable. # after receiving the response for lighthouse queries
#trigger_buffer: 64 #trigger_buffer: 64
# Tunnel manager settings # Tunnel manager settings
@@ -456,7 +449,9 @@ firewall:
#default_local_cidr_any: false #default_local_cidr_any: false
conntrack: conntrack:
# tcp_timeout also applies to sctp and dccp
tcp_timeout: 12m tcp_timeout: 12m
# udp_timeout also applies to udplite
udp_timeout: 3m udp_timeout: 3m
default_timeout: 10m default_timeout: 10m
@@ -464,8 +459,11 @@ firewall:
# Rules are comprised of a protocol, port, and one or more of host, group, or CIDR # Rules are comprised of a protocol, port, and one or more of host, group, or CIDR
# Logical evaluation is roughly: port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND (local cidr) # Logical evaluation is roughly: port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND (local cidr)
# - port: Takes `0` or `any` as any, a single number `80`, a range `200-901`, or `fragment` to match second and further fragments of fragmented packets (since there is no port available). # - port: Takes `0` or `any` as any, a single number `80`, a range `200-901`, or `fragment` to match second and further fragments of fragmented packets (since there is no port available).
# proto: `any`, `tcp`, `udp`, or `icmp` # proto: `any`, `tcp`, `udp`, `udplite`, `dccp`, `sctp`, `icmp`, or any other IP protocol number from 1 to 255, ie `47` for gre
# a port specification is ignored if proto is `icmp` # a port specification is ignored if proto is `icmp`
# `icmp`, `1`, and `58` are the same, each matches both ICMP and ICMPv6
# Nebula only reads ports from tcp, udp, udplite, dccp, and sctp, so a proto `any` rule with a specific port or range only matches those protocols.
# Rules for other protocols (gre, esp, etc) need port `any`.
# host: `any` or a literal hostname, ie `test-host` # host: `any` or a literal hostname, ie `test-host`
# group: `any` or a literal group name, ie `default-group` # group: `any` or a literal group name, ie `default-group`
# groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass # groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass
+38 -757
View File
@@ -2,17 +2,10 @@ package nebula
import ( import (
"context" "context"
"crypto/sha256"
"encoding/hex"
"errors" "errors"
"fmt"
"hash/fnv"
"log/slog" "log/slog"
"net/netip" "net/netip"
"reflect"
"slices"
"strconv" "strconv"
"strings"
"sync" "sync"
"time" "time"
@@ -24,10 +17,6 @@ import (
"github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/iputil"
) )
type FirewallInterface interface {
AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
}
type conn struct { type conn struct {
Expires time.Time // Time when this conntrack entry will expire Expires time.Time // Time when this conntrack entry will expire
@@ -42,8 +31,7 @@ type conn struct {
type Firewall struct { type Firewall struct {
Conntrack *FirewallConntrack Conntrack *FirewallConntrack
InRules *FirewallTable rules firewall.Rules
OutRules *FirewallTable
InboundSendReject bool InboundSendReject bool
OutboundSendReject bool OutboundSendReject bool
@@ -63,12 +51,10 @@ type Firewall struct {
// unsafeNetworks is the list of unsafe networks issued to us in the certificate // unsafeNetworks is the list of unsafe networks issued to us in the certificate
unsafeNetworks []netip.Prefix unsafeNetworks []netip.Prefix
rules string
rulesVersion uint16 rulesVersion uint16
defaultLocalCIDRAny bool incomingMetrics firewallMetrics
incomingMetrics firewallMetrics outgoingMetrics firewallMetrics
outgoingMetrics firewallMetrics
l *slog.Logger l *slog.Logger
} }
@@ -86,52 +72,6 @@ type FirewallConntrack struct {
TimerWheel *TimerWheel[firewall.Packet] TimerWheel *TimerWheel[firewall.Packet]
} }
// FirewallTable is the entry point for a rule, the evaluation order is:
// Proto AND port AND (CA SHA or CA name) AND local CIDR AND (group OR groups OR name OR remote CIDR)
type FirewallTable struct {
TCP firewallPort
UDP firewallPort
ICMP firewallPort
AnyProto firewallPort
}
func newFirewallTable() *FirewallTable {
return &FirewallTable{
TCP: firewallPort{},
UDP: firewallPort{},
ICMP: firewallPort{},
AnyProto: firewallPort{},
}
}
type FirewallCA struct {
Any *FirewallRule
CANames map[string]*FirewallRule
CAShas map[string]*FirewallRule
}
type FirewallRule struct {
// Any makes Hosts, Groups, and CIDR irrelevant
Any *firewallLocalCIDR
Hosts map[string]*firewallLocalCIDR
Groups []*firewallGroups
CIDR *bart.Table[*firewallLocalCIDR]
}
type firewallGroups struct {
Groups []string
LocalCIDR *firewallLocalCIDR
}
// Even though ports are uint16, int32 maps are faster for lookup
// Plus we can use `-1` for fragment rules
type firewallPort map[int32]*FirewallCA
type firewallLocalCIDR struct {
Any bool
LocalCIDR *bart.Lite
}
// NewFirewall creates a new Firewall object. A TimerWheel is created for you from the provided timeouts. // NewFirewall creates a new Firewall object. A TimerWheel is created for you from the provided timeouts.
// The certificate provided should be the highest version loaded in memory. // The certificate provided should be the highest version loaded in memory.
func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c cert.Certificate) *Firewall { func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c cert.Certificate) *Firewall {
@@ -170,8 +110,8 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
Conns: make(map[firewall.Packet]*conn), Conns: make(map[firewall.Packet]*conn),
TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax), TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax),
}, },
InRules: newFirewallTable(), // Nothing is allowed until there are rules
OutRules: newFirewallTable(), rules: firewall.NewRulesBuilder(l).Build(assignedNetworks, unsafeNetworks),
TCPTimeout: tcpTimeout, TCPTimeout: tcpTimeout,
UDPTimeout: UDPTimeout, UDPTimeout: UDPTimeout,
DefaultTimeout: defaultTimeout, DefaultTimeout: defaultTimeout,
@@ -212,8 +152,6 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
//TODO: max_connections //TODO: max_connections
) )
fw.defaultLocalCIDRAny = c.GetBool("firewall.default_local_cidr_any", false)
inboundAction := c.GetString("firewall.inbound_action", "drop") inboundAction := c.GetString("firewall.inbound_action", "drop")
switch inboundAction { switch inboundAction {
case "reject": case "reject":
@@ -236,80 +174,23 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
fw.OutboundSendReject = false fw.OutboundSendReject = false
} }
err := AddFirewallRulesFromConfig(l, false, c, fw) rules, err := firewall.RulesFromConfig(l, c, fw.assignedNetworks, fw.unsafeNetworks)
if err != nil {
return nil, err
}
err = AddFirewallRulesFromConfig(l, true, c, fw)
if err != nil { if err != nil {
return nil, err return nil, err
} }
fw.rules = rules
return fw, nil return fw, nil
} }
// AddRule properly creates the in memory rule structure for a firewall table.
func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
var (
ft *FirewallTable
fp firewallPort
)
if incoming {
ft = f.InRules
} else {
ft = f.OutRules
}
switch proto {
case iputil.IPProtocolTCP:
fp = ft.TCP
case iputil.IPProtocolUDP:
fp = ft.UDP
case iputil.IPProtocolICMP, iputil.IPProtocolICMPv6:
//ICMP traffic doesn't have ports, so we always coerce to "any", even if a value is provided
if startPort != firewall.PortAny {
f.l.Warn("ignoring port specification for ICMP firewall rule", "startPort", startPort)
}
startPort = firewall.PortAny
endPort = firewall.PortAny
fp = ft.ICMP
case firewall.ProtoAny:
fp = ft.AnyProto
default:
return fmt.Errorf("unknown protocol %v", proto)
}
// We need this rule string because we generate a hash. Removing this will break firewall reload.
ruleString := fmt.Sprintf(
"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s",
incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha,
)
f.rules += ruleString + "\n"
direction := "incoming"
if !incoming {
direction = "outgoing"
}
f.l.Info("Firewall rule added",
"firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha},
)
return fp.addRule(f, startPort, endPort, groups, host, cidr, localCidr, caName, caSha)
}
// GetRuleHash returns a hash representation of all inbound and outbound rules // GetRuleHash returns a hash representation of all inbound and outbound rules
func (f *Firewall) GetRuleHash() string { func (f *Firewall) GetRuleHash() string {
sum := sha256.Sum256([]byte(f.rules)) return f.rules.Hash
return hex.EncodeToString(sum[:])
} }
// GetRuleHashFNV returns a uint32 FNV-1 hash representation the rules, for use as a metric value // GetRuleHashFNV returns a uint32 FNV-1 hash representation the rules, for use as a metric value
func (f *Firewall) GetRuleHashFNV() uint32 { func (f *Firewall) GetRuleHashFNV() uint32 {
h := fnv.New32a() return f.rules.HashFNV
h.Write([]byte(f.rules))
return h.Sum32()
} }
// GetRuleHashes returns both the sha256 and FNV-1 hashes, suitable for logging // GetRuleHashes returns both the sha256 and FNV-1 hashes, suitable for logging
@@ -317,104 +198,6 @@ func (f *Firewall) GetRuleHashes() string {
return "SHA:" + f.GetRuleHash() + ",FNV:" + strconv.FormatUint(uint64(f.GetRuleHashFNV()), 10) return "SHA:" + f.GetRuleHash() + ",FNV:" + strconv.FormatUint(uint64(f.GetRuleHashFNV()), 10)
} }
func AddFirewallRulesFromConfig(l *slog.Logger, inbound bool, c *config.C, fw FirewallInterface) error {
var table string
if inbound {
table = "firewall.inbound"
} else {
table = "firewall.outbound"
}
r := c.Get(table)
if r == nil {
return nil
}
rs, ok := r.([]any)
if !ok {
return fmt.Errorf("%s failed to parse, should be an array of rules", table)
}
for i, t := range rs {
r, err := convertRule(l, t, table, i)
if err != nil {
return fmt.Errorf("%s rule #%v; %s", table, i, err)
}
if r.Code != "" && r.Port != "" {
return fmt.Errorf("%s rule #%v; only one of port or code should be provided", table, i)
}
if r.Host == "" && len(r.Groups) == 0 && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
return fmt.Errorf("%s rule #%v; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided", table, i)
}
var sPort, errPort string
if r.Code != "" {
errPort = "code"
sPort = r.Code
} else {
errPort = "port"
sPort = r.Port
}
var proto uint8
var startPort, endPort int32
switch r.Proto {
case "any":
proto = firewall.ProtoAny
startPort, endPort, err = parsePort(sPort)
case "tcp":
proto = iputil.IPProtocolTCP
startPort, endPort, err = parsePort(sPort)
case "udp":
proto = iputil.IPProtocolUDP
startPort, endPort, err = parsePort(sPort)
case "icmp":
proto = iputil.IPProtocolICMP
startPort = firewall.PortAny
endPort = firewall.PortAny
if sPort != "" {
l.Warn("ignoring port specification for ICMP firewall rule", "port", sPort)
}
default:
return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
}
if err != nil {
return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err)
}
if r.Cidr != "" && r.Cidr != "any" {
_, err = netip.ParsePrefix(r.Cidr)
if err != nil {
return fmt.Errorf("%s rule #%v; cidr did not parse; %s", table, i, err)
}
}
if r.LocalCidr != "" && r.LocalCidr != "any" {
_, err = netip.ParsePrefix(r.LocalCidr)
if err != nil {
return fmt.Errorf("%s rule #%v; local_cidr did not parse; %s", table, i, err)
}
}
if warning := r.sanity(); warning != nil {
l.Warn("firewall rule sanity check",
"table", table,
"rule", i,
"warning", warning,
)
}
err = fw.AddRule(inbound, proto, startPort, endPort, r.Groups, r.Host, r.Cidr, r.LocalCidr, r.CAName, r.CASha)
if err != nil {
return fmt.Errorf("%s rule #%v; `%s`", table, i, err)
}
}
return nil
}
var ErrUnknownNetworkType = errors.New("unknown network type") var ErrUnknownNetworkType = errors.New("unknown network type")
var ErrPeerRejected = errors.New("remote address is not within a network that we handle") var ErrPeerRejected = errors.New("remote address is not within a network that we handle")
var ErrInvalidRemoteIP = errors.New("remote address is not in remote certificate networks") var ErrInvalidRemoteIP = errors.New("remote address is not in remote certificate networks")
@@ -458,23 +241,17 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
} }
// Check if we spoke to this tuple, if we did then allow this packet // Check if we spoke to this tuple, if we did then allow this packet
if f.inConns(fp, h, caPool, localCache) { if f.inConns(&fp, h, caPool, localCache) {
return nil return nil
} }
table := f.OutRules if !f.rules.Match(&fp, incoming, h.ConnectionState.peerCert, caPool) {
if incoming {
table = f.InRules
}
// We now know which firewall table to check against
if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
f.metrics(incoming).droppedNoRule.Inc(1) f.metrics(incoming).droppedNoRule.Inc(1)
return ErrNoMatchingRule return ErrNoMatchingRule
} }
// We always want to conntrack since it is a faster operation // We always want to conntrack since it is a faster operation
f.addConn(fp, incoming) f.addConn(&fp, incoming)
return nil return nil
} }
@@ -503,9 +280,9 @@ func (f *Firewall) EmitStats() {
metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV())) metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
} }
func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool { func (f *Firewall) inConns(fp *firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
if localCache != nil { if localCache != nil {
if _, ok := localCache[fp]; ok { if _, ok := localCache[*fp]; ok {
return true return true
} }
} }
@@ -518,7 +295,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
f.evict(ep) f.evict(ep)
} }
c, ok := conntrack.Conns[fp] c, ok := conntrack.Conns[*fp]
if !ok { if !ok {
conntrack.Unlock() conntrack.Unlock()
@@ -528,29 +305,23 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
if c.rulesVersion != f.rulesVersion { if c.rulesVersion != f.rulesVersion {
// This conntrack entry was for an older rule set, validate // This conntrack entry was for an older rule set, validate
// it still passes with the current rule set // it still passes with the current rule set
table := f.OutRules if !f.rules.Match(fp, c.incoming, h.ConnectionState.peerCert, caPool) {
if c.incoming {
table = f.InRules
}
// We now know which firewall table to check against
if !table.match(fp, c.incoming, h.ConnectionState.peerCert, caPool) {
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
h.logger(f.l).Debug("dropping old conntrack entry, does not match new ruleset", h.logger(f.l).Debug("dropping old conntrack entry, does not match new ruleset",
"fwPacket", fp, "fwPacket", *fp,
"incoming", c.incoming, "incoming", c.incoming,
"rulesVersion", f.rulesVersion, "rulesVersion", f.rulesVersion,
"oldRulesVersion", c.rulesVersion, "oldRulesVersion", c.rulesVersion,
) )
} }
delete(conntrack.Conns, fp) delete(conntrack.Conns, *fp)
conntrack.Unlock() conntrack.Unlock()
return false return false
} }
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
h.logger(f.l).Debug("keeping old conntrack entry, does match new ruleset", h.logger(f.l).Debug("keeping old conntrack entry, does match new ruleset",
"fwPacket", fp, "fwPacket", *fp,
"incoming", c.incoming, "incoming", c.incoming,
"rulesVersion", f.rulesVersion, "rulesVersion", f.rulesVersion,
"oldRulesVersion", c.rulesVersion, "oldRulesVersion", c.rulesVersion,
@@ -560,42 +331,39 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
c.rulesVersion = f.rulesVersion c.rulesVersion = f.rulesVersion
} }
switch fp.Protocol { c.Expires = time.Now().Add(f.conntrackTimeout(fp.Protocol))
case iputil.IPProtocolTCP:
c.Expires = time.Now().Add(f.TCPTimeout)
case iputil.IPProtocolUDP:
c.Expires = time.Now().Add(f.UDPTimeout)
default:
c.Expires = time.Now().Add(f.DefaultTimeout)
}
conntrack.Unlock() conntrack.Unlock()
if localCache != nil { if localCache != nil {
localCache[fp] = struct{}{} localCache[*fp] = struct{}{}
} }
return true return true
} }
func (f *Firewall) addConn(fp firewall.Packet, incoming bool) { // conntrackTimeout returns how long a conntrack entry for proto lives without traffic.
var timeout time.Duration // Connection oriented protocols share the tcp timeout, datagram protocols share the udp timeout.
c := &conn{} func (f *Firewall) conntrackTimeout(proto uint8) time.Duration {
switch proto {
switch fp.Protocol { case iputil.IPProtocolTCP, iputil.IPProtocolSCTP, iputil.IPProtocolDCCP:
case iputil.IPProtocolTCP: return f.TCPTimeout
timeout = f.TCPTimeout case iputil.IPProtocolUDP, iputil.IPProtocolUDPLite:
case iputil.IPProtocolUDP: return f.UDPTimeout
timeout = f.UDPTimeout
default: default:
timeout = f.DefaultTimeout return f.DefaultTimeout
} }
}
func (f *Firewall) addConn(fp *firewall.Packet, incoming bool) {
timeout := f.conntrackTimeout(fp.Protocol)
c := &conn{}
conntrack := f.Conntrack conntrack := f.Conntrack
conntrack.Lock() conntrack.Lock()
if _, ok := conntrack.Conns[fp]; !ok { if _, ok := conntrack.Conns[*fp]; !ok {
conntrack.TimerWheel.Advance(time.Now()) conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(fp, timeout) conntrack.TimerWheel.Add(*fp, timeout)
} }
// Record which rulesVersion allowed this connection, so we can retest after // Record which rulesVersion allowed this connection, so we can retest after
@@ -603,7 +371,7 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
c.incoming = incoming c.incoming = incoming
c.rulesVersion = f.rulesVersion c.rulesVersion = f.rulesVersion
c.Expires = time.Now().Add(timeout) c.Expires = time.Now().Add(timeout)
conntrack.Conns[fp] = c conntrack.Conns[*fp] = c
conntrack.Unlock() conntrack.Unlock()
} }
@@ -629,490 +397,3 @@ func (f *Firewall) evict(p firewall.Packet) {
// This conn is done // This conn is done
delete(conntrack.Conns, p) delete(conntrack.Conns, p)
} }
func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
if ft.AnyProto.match(p, incoming, c, caPool) {
return true
}
switch p.Protocol {
case iputil.IPProtocolTCP:
if ft.TCP.match(p, incoming, c, caPool) {
return true
}
case iputil.IPProtocolUDP:
if ft.UDP.match(p, incoming, c, caPool) {
return true
}
case iputil.IPProtocolICMP, iputil.IPProtocolICMPv6:
if ft.ICMP.match(p, incoming, c, caPool) {
return true
}
}
return false
}
func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
if startPort > endPort {
return fmt.Errorf("start port was lower than end port")
}
for i := startPort; i <= endPort; i++ {
if _, ok := fp[i]; !ok {
fp[i] = &FirewallCA{
CANames: make(map[string]*FirewallRule),
CAShas: make(map[string]*FirewallRule),
}
}
if err := fp[i].addRule(f, groups, host, cidr, localCidr, caName, caSha); err != nil {
return err
}
}
return nil
}
func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
// We don't have any allowed ports, bail
if fp == nil {
return false
}
// this branch is here to catch traffic from FirewallTable.Any.match and FirewallTable.ICMP.match
if p.Protocol == iputil.IPProtocolICMP || p.Protocol == iputil.IPProtocolICMPv6 {
// port numbers are re-used for connection tracking of ICMP,
// but we don't want to actually filter on them.
return fp[firewall.PortAny].match(p, c, caPool)
}
var port int32
if p.Fragment {
port = firewall.PortFragment
} else if incoming {
port = int32(p.LocalPort)
} else {
port = int32(p.RemotePort)
}
if fp[port].match(p, c, caPool) {
return true
}
return fp[firewall.PortAny].match(p, c, caPool)
}
func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, localCidr, caName, caSha string) error {
fr := func() *FirewallRule {
return &FirewallRule{
Hosts: make(map[string]*firewallLocalCIDR),
Groups: make([]*firewallGroups, 0),
CIDR: new(bart.Table[*firewallLocalCIDR]),
}
}
if caSha == "" && caName == "" {
if fc.Any == nil {
fc.Any = fr()
}
return fc.Any.addRule(f, groups, host, cidr, localCidr)
}
if caSha != "" {
if _, ok := fc.CAShas[caSha]; !ok {
fc.CAShas[caSha] = fr()
}
err := fc.CAShas[caSha].addRule(f, groups, host, cidr, localCidr)
if err != nil {
return err
}
}
if caName != "" {
if _, ok := fc.CANames[caName]; !ok {
fc.CANames[caName] = fr()
}
err := fc.CANames[caName].addRule(f, groups, host, cidr, localCidr)
if err != nil {
return err
}
}
return nil
}
func (fc *FirewallCA) match(p firewall.Packet, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
if fc == nil {
return false
}
if fc.Any.match(p, c) {
return true
}
if t, ok := fc.CAShas[c.Certificate.Issuer()]; ok {
if t.match(p, c) {
return true
}
}
s, err := caPool.GetCAForCert(c.Certificate)
if err != nil {
return false
}
return fc.CANames[s.Certificate.Name()].match(p, c)
}
func (fr *FirewallRule) addRule(f *Firewall, groups []string, host, cidr, localCidr string) error {
flc := func() *firewallLocalCIDR {
return &firewallLocalCIDR{
LocalCIDR: new(bart.Lite),
}
}
if fr.isAny(groups, host, cidr) {
if fr.Any == nil {
fr.Any = flc()
}
return fr.Any.addRule(f, localCidr)
}
if len(groups) > 0 {
nlc := flc()
err := nlc.addRule(f, localCidr)
if err != nil {
return err
}
fr.Groups = append(fr.Groups, &firewallGroups{
Groups: groups,
LocalCIDR: nlc,
})
}
if host != "" {
nlc := fr.Hosts[host]
if nlc == nil {
nlc = flc()
}
err := nlc.addRule(f, localCidr)
if err != nil {
return err
}
fr.Hosts[host] = nlc
}
if cidr != "" {
c, err := netip.ParsePrefix(cidr)
if err != nil {
return err
}
nlc, _ := fr.CIDR.Get(c)
if nlc == nil {
nlc = flc()
}
err = nlc.addRule(f, localCidr)
if err != nil {
return err
}
fr.CIDR.Insert(c, nlc)
}
return nil
}
func (fr *FirewallRule) isAny(groups []string, host string, cidr string) bool {
if len(groups) == 0 && host == "" && cidr == "" {
return true
}
if slices.Contains(groups, "any") {
return true
}
if host == "any" {
return true
}
if cidr == "any" {
return true
}
return false
}
func (fr *FirewallRule) match(p firewall.Packet, c *cert.CachedCertificate) bool {
if fr == nil {
return false
}
// Shortcut path for if groups, hosts, or cidr contained an `any`
if fr.Any.match(p, c) {
return true
}
// Need any of group, host, or cidr to match
for _, sg := range fr.Groups {
found := false
for _, g := range sg.Groups {
if _, ok := c.InvertedGroups[g]; !ok {
found = false
break
}
found = true
}
if found && sg.LocalCIDR.match(p, c) {
return true
}
}
if fr.Hosts != nil {
if flc, ok := fr.Hosts[c.Certificate.Name()]; ok {
if flc.match(p, c) {
return true
}
}
}
for _, v := range fr.CIDR.Supernets(netip.PrefixFrom(p.RemoteAddr, p.RemoteAddr.BitLen())) {
if v.match(p, c) {
return true
}
}
return false
}
func (flc *firewallLocalCIDR) addRule(f *Firewall, localCidr string) error {
if localCidr == "any" {
flc.Any = true
return nil
}
if localCidr == "" {
if len(f.unsafeNetworks) == 0 || f.defaultLocalCIDRAny {
flc.Any = true
return nil
}
for _, network := range f.assignedNetworks {
flc.LocalCIDR.Insert(network)
}
return nil
}
c, err := netip.ParsePrefix(localCidr)
if err != nil {
return err
}
flc.LocalCIDR.Insert(c)
return nil
}
func (flc *firewallLocalCIDR) match(p firewall.Packet, c *cert.CachedCertificate) bool {
if flc == nil {
return false
}
if flc.Any {
return true
}
return flc.LocalCIDR.Contains(p.LocalAddr)
}
type rule struct {
Port string
Code string
Proto string
Host string
Groups []string
Cidr string
LocalCidr string
CAName string
CASha string
}
func convertRule(l *slog.Logger, p any, table string, i int) (rule, error) {
r := rule{}
m, ok := p.(map[string]any)
if !ok {
return r, errors.New("could not parse rule")
}
toString := func(k string, m map[string]any) string {
v, ok := m[k]
if !ok {
return ""
}
return fmt.Sprintf("%v", v)
}
r.Port = toString("port", m)
r.Code = toString("code", m)
r.Proto = toString("proto", m)
r.Host = toString("host", m)
r.Cidr = toString("cidr", m)
r.LocalCidr = toString("local_cidr", m)
r.CAName = toString("ca_name", m)
r.CASha = toString("ca_sha", m)
// Make sure group isn't an array
if v, ok := m["group"].([]any); ok {
if len(v) > 1 {
return r, errors.New("group should contain a single value, an array with more than one entry was provided")
}
l.Warn("group was an array with a single value, converting to simple value",
"table", table,
"rule", i,
)
m["group"] = v[0]
}
singleGroup := toString("group", m)
if rg, ok := m["groups"]; ok {
switch reflect.TypeOf(rg).Kind() {
case reflect.Slice:
v := reflect.ValueOf(rg)
r.Groups = make([]string, v.Len())
for i := 0; i < v.Len(); i++ {
r.Groups[i] = v.Index(i).Interface().(string)
}
case reflect.String:
r.Groups = []string{rg.(string)}
default:
r.Groups = []string{fmt.Sprintf("%v", rg)}
}
}
//flatten group vs groups
if singleGroup != "" {
// Check if we have both groups and group provided in the rule config
if len(r.Groups) > 0 {
return r, fmt.Errorf("only one of group or groups should be defined, both provided")
}
r.Groups = []string{singleGroup}
}
return r, nil
}
// sanity returns an error if the rule would be evaluated in a way that would short-circuit a configured check on a wildcard value
// rules are evaluated as "port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND local_cidr"
func (r *rule) sanity() error {
//port, proto, local_cidr are AND, no need to check here
//ca_sha and ca_name don't have a wildcard value, no need to check here
groupsEmpty := len(r.Groups) == 0
hostEmpty := r.Host == ""
cidrEmpty := r.Cidr == ""
if (groupsEmpty && hostEmpty && cidrEmpty) == true {
return nil //no content!
}
groupsHasAny := slices.Contains(r.Groups, "any")
if groupsHasAny && len(r.Groups) > 1 {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the other groups specified", r.Groups)
}
if r.Host == "any" {
if !groupsEmpty {
return fmt.Errorf("groups specified as %s, but host=any will match any host, regardless of groups", r.Groups)
}
if !cidrEmpty {
return fmt.Errorf("cidr specified as %s, but host=any will match any host, regardless of cidr", r.Cidr)
}
}
if groupsHasAny {
if !hostEmpty && r.Host != "any" {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified host %s", r.Groups, r.Host)
}
if !cidrEmpty {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified cidr %s", r.Groups, r.Cidr)
}
}
if r.Code != "" {
return fmt.Errorf("code specified as [%s]. Support for 'code' will be dropped in a future release, as it has never been functional", r.Code)
}
//todo alert on cidr-any
return nil
}
func parsePort(s string) (int32, int32, error) {
const notAPort int32 = -2
if s == "any" {
return firewall.PortAny, firewall.PortAny, nil
}
if s == "fragment" {
return firewall.PortFragment, firewall.PortFragment, nil
}
if !strings.Contains(s, `-`) {
rPort, err := parsePortValue("", s)
if err != nil {
return notAPort, notAPort, err
}
return rPort, rPort, nil
}
sPorts := strings.SplitN(s, `-`, 2)
for i := range sPorts {
sPorts[i] = strings.Trim(sPorts[i], " ")
}
if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
return notAPort, notAPort, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
}
startPort, err := parsePortValue("beginning range ", sPorts[0])
if err != nil {
return notAPort, notAPort, err
}
endPort, err := parsePortValue("ending range ", sPorts[1])
if err != nil {
return notAPort, notAPort, err
}
if startPort == firewall.PortAny {
endPort = firewall.PortAny
}
return startPort, endPort, nil
}
// parsePortValue accepts a base-10 decimal in [0, 65535] and returns it
// widened to int32. Using strconv.ParseUint with bitSize 16 rejects
// negative input, out-of-range input (>65535), and any non-decimal byte
// by construction, so the int32 widening that follows is provably safe
// and cannot collide with firewall.PortAny (0) or firewall.PortFragment
// (-1) via integer truncation.
//
// prefix is prepended to both error messages so callers can disambiguate
// the single-port path (prefix="") from the range bounds (prefix="beginning
// range " / "ending range "), preserving the historical error strings.
func parsePortValue(prefix, s string) (int32, error) {
n, err := strconv.ParseUint(s, 10, 16)
if err == nil {
return int32(n), nil
}
if errors.Is(err, strconv.ErrRange) {
return 0, fmt.Errorf("%sout of range [0,65535]; `%s`", prefix, s)
}
return 0, fmt.Errorf("%swas not a number; `%s`", prefix, s)
}
+310
View File
@@ -0,0 +1,310 @@
package firewall
import (
"errors"
"fmt"
"log/slog"
"net/netip"
"reflect"
"slices"
"strconv"
"strings"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/iputil"
)
// ruleAdder takes firewall rules, it lets tests see what addRulesFromConfig adds
type ruleAdder interface {
AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
}
// RulesFromConfig reads the inbound and outbound firewall rules from c and builds them, see RulesBuilder.Build
func RulesFromConfig(l *slog.Logger, c *config.C, vpnNetworks, unsafeNetworks []netip.Prefix) (Rules, error) {
b := NewRulesBuilder(l)
b.defaultLocalCIDRAny = c.GetBool("firewall.default_local_cidr_any", false)
if err := addRulesFromConfig(l, false, c, b); err != nil {
return Rules{}, err
}
if err := addRulesFromConfig(l, true, c, b); err != nil {
return Rules{}, err
}
return b.Build(vpnNetworks, unsafeNetworks), nil
}
func addRulesFromConfig(l *slog.Logger, inbound bool, c *config.C, ra ruleAdder) error {
var table string
if inbound {
table = "firewall.inbound"
} else {
table = "firewall.outbound"
}
r := c.Get(table)
if r == nil {
return nil
}
rs, ok := r.([]any)
if !ok {
return fmt.Errorf("%s failed to parse, should be an array of rules", table)
}
for i, t := range rs {
r, err := yamlToConfigRule(l, t, table, i)
if err != nil {
return fmt.Errorf("%s rule #%v; %s", table, i, err)
}
if r.Code != "" && r.Port != "" {
return fmt.Errorf("%s rule #%v; only one of port or code should be provided", table, i)
}
if r.Host == "" && len(r.Groups) == 0 && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
return fmt.Errorf("%s rule #%v; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided", table, i)
}
var sPort, errPort string
if r.Code != "" {
errPort = "code"
sPort = r.Code
} else {
errPort = "port"
sPort = r.Port
}
proto, ok := protoByName(r.Proto)
if !ok {
// Any other protocol by number. 0 is reserved for `any`.
n, perr := strconv.ParseUint(r.Proto, 10, 8)
if perr != nil || n == 0 {
return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
}
proto = uint8(n)
}
var startPort, endPort int32
if proto == iputil.IPProtocolICMP || proto == iputil.IPProtocolICMPv6 {
// RulesBuilder.AddRule coerces ICMP ports to `any` as well. This copy stays so that an ICMP rule's
// port is warned about rather than parsed, whatever it holds.
startPort = PortAny
endPort = PortAny
if sPort != "" {
l.Warn("ignoring port specification for ICMP firewall rule", "port", sPort)
}
} else {
startPort, endPort, err = parsePort(sPort)
}
if err != nil {
return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err)
}
if warning := r.sanity(); warning != nil {
l.Warn("firewall rule sanity check",
"table", table,
"rule", i,
"warning", warning,
)
}
err = ra.AddRule(inbound, proto, startPort, endPort, r.Groups, r.Host, r.Cidr, r.LocalCidr, r.CAName, r.CASha)
if err != nil {
return fmt.Errorf("%s rule #%v; %s", table, i, err)
}
}
return nil
}
type configRule struct {
Port string
Code string
Proto string
Host string
Groups []string
Cidr string
LocalCidr string
CAName string
CASha string
}
func yamlToConfigRule(l *slog.Logger, p any, table string, i int) (configRule, error) {
r := configRule{}
m, ok := p.(map[string]any)
if !ok {
return r, errors.New("could not parse rule")
}
toString := func(k string, m map[string]any) string {
v, ok := m[k]
if !ok {
return ""
}
return fmt.Sprintf("%v", v)
}
r.Port = toString("port", m)
r.Code = toString("code", m)
r.Proto = toString("proto", m)
r.Host = toString("host", m)
r.Cidr = toString("cidr", m)
r.LocalCidr = toString("local_cidr", m)
r.CAName = toString("ca_name", m)
r.CASha = toString("ca_sha", m)
// Make sure group isn't an array
if v, ok := m["group"].([]any); ok {
if len(v) > 1 {
return r, errors.New("group should contain a single value, an array with more than one entry was provided")
}
if len(v) == 0 {
return r, errors.New("group should contain a single value, an empty array was provided")
}
l.Warn("group was an array with a single value, converting to simple value",
"table", table,
"rule", i,
)
m["group"] = v[0]
}
singleGroup := toString("group", m)
// A null value, `groups:` with nothing after it, is the same as no groups
if rg, ok := m["groups"]; ok && rg != nil {
switch reflect.TypeOf(rg).Kind() {
case reflect.Slice:
// Every entry is read as a string, as the other fields are, so `groups: [1, 2]` is the groups "1" and "2"
v := reflect.ValueOf(rg)
r.Groups = make([]string, v.Len())
for i := 0; i < v.Len(); i++ {
r.Groups[i] = fmt.Sprintf("%v", v.Index(i).Interface())
}
case reflect.String:
r.Groups = []string{rg.(string)}
default:
r.Groups = []string{fmt.Sprintf("%v", rg)}
}
}
//flatten group vs groups
if singleGroup != "" {
// Check if we have both groups and group provided in the rule config
if len(r.Groups) > 0 {
return r, fmt.Errorf("only one of group or groups should be defined, both provided")
}
r.Groups = []string{singleGroup}
}
return r, nil
}
// sanity returns an error if the rule would be evaluated in a way that would short-circuit a configured check on a wildcard value
// rules are evaluated as "port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND local_cidr"
func (r *configRule) sanity() error {
//port, proto, local_cidr are AND, no need to check here
//ca_sha and ca_name don't have a wildcard value, no need to check here
groupsEmpty := len(r.Groups) == 0
hostEmpty := r.Host == ""
cidrEmpty := r.Cidr == ""
if (groupsEmpty && hostEmpty && cidrEmpty) == true {
return nil //no content!
}
groupsHasAny := slices.Contains(r.Groups, "any")
if groupsHasAny && len(r.Groups) > 1 {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the other groups specified", r.Groups)
}
if r.Host == "any" {
if !groupsEmpty {
return fmt.Errorf("groups specified as %s, but host=any will match any host, regardless of groups", r.Groups)
}
if !cidrEmpty {
return fmt.Errorf("cidr specified as %s, but host=any will match any host, regardless of cidr", r.Cidr)
}
}
if groupsHasAny {
if !hostEmpty && r.Host != "any" {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified host %s", r.Groups, r.Host)
}
if !cidrEmpty {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified cidr %s", r.Groups, r.Cidr)
}
}
if r.Code != "" {
return fmt.Errorf("code specified as [%s]. Support for 'code' will be dropped in a future release, as it has never been functional", r.Code)
}
//todo alert on cidr-any
return nil
}
func parsePort(s string) (int32, int32, error) {
const notAPort int32 = -2
if s == "any" {
return PortAny, PortAny, nil
}
if s == "fragment" {
return PortFragment, PortFragment, nil
}
if !strings.Contains(s, `-`) {
rPort, err := parsePortValue("", s)
if err != nil {
return notAPort, notAPort, err
}
return rPort, rPort, nil
}
sPorts := strings.SplitN(s, `-`, 2)
for i := range sPorts {
sPorts[i] = strings.Trim(sPorts[i], " ")
}
if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
return notAPort, notAPort, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
}
startPort, err := parsePortValue("beginning range ", sPorts[0])
if err != nil {
return notAPort, notAPort, err
}
endPort, err := parsePortValue("ending range ", sPorts[1])
if err != nil {
return notAPort, notAPort, err
}
if startPort == PortAny {
endPort = PortAny
}
return startPort, endPort, nil
}
// parsePortValue accepts a base-10 decimal in [0, 65535] and returns it
// widened to int32. Using strconv.ParseUint with bitSize 16 rejects
// negative input, out-of-range input (>65535), and any non-decimal byte
// by construction, so the int32 widening that follows is provably safe
// and cannot collide with PortAny (0) or PortFragment
// (-1) via integer truncation.
//
// prefix is prepended to both error messages so callers can disambiguate
// the single-port path (prefix="") from the range bounds (prefix="beginning
// range " / "ending range "), preserving the historical error strings.
func parsePortValue(prefix, s string) (int32, error) {
n, err := strconv.ParseUint(s, 10, 16)
if err == nil {
return int32(n), nil
}
if errors.Is(err, strconv.ErrRange) {
return 0, fmt.Errorf("%sout of range [0,65535]; `%s`", prefix, s)
}
return 0, fmt.Errorf("%swas not a number; `%s`", prefix, s)
}
+516
View File
@@ -0,0 +1,516 @@
package firewall
import (
"bytes"
"errors"
"net/netip"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_parsePort(t *testing.T) {
_, _, err := parsePort("")
require.EqualError(t, err, "was not a number; ``")
_, _, err = parsePort(" ")
require.EqualError(t, err, "was not a number; ` `")
_, _, err = parsePort("-")
require.EqualError(t, err, "appears to be a range but could not be parsed; `-`")
_, _, err = parsePort(" - ")
require.EqualError(t, err, "appears to be a range but could not be parsed; ` - `")
_, _, err = parsePort("a-b")
require.EqualError(t, err, "beginning range was not a number; `a`")
_, _, err = parsePort("1-b")
require.EqualError(t, err, "ending range was not a number; `b`")
s, e, err := parsePort(" 1 - 2 ")
assert.Equal(t, int32(1), s)
assert.Equal(t, int32(2), e)
require.NoError(t, err)
s, e, err = parsePort("0-1")
assert.Equal(t, int32(0), s)
assert.Equal(t, int32(0), e)
require.NoError(t, err)
s, e, err = parsePort("9919")
assert.Equal(t, int32(9919), s)
assert.Equal(t, int32(9919), e)
require.NoError(t, err)
s, e, err = parsePort("any")
assert.Equal(t, int32(0), s)
assert.Equal(t, int32(0), e)
require.NoError(t, err)
}
// Test_parsePort_invalid covers inputs that must error. The named bug is
// that int32(strconv.Atoi("4294967296")) truncates to 0 == PortAny,
// silently turning a typo into a match-all-ports rule; the rest are
// representative syntax/range probes.
func Test_parsePort_invalid(t *testing.T) {
tests := []struct {
name string
input string
wantErrContains string
}{
// Numeric overflow (the named bug + boundary).
{"named bug: 2^32 truncates to PortAny", "4294967296", "out of range"},
{"just above max real port", "65536", "out of range"},
// Negatives route through the range branch and hit the empty-half
// guard; included as defense in depth so a future refactor cannot
// accidentally reach the int32 cast.
{"negative", "-1", "could not be parsed"},
// Syntax probes.
{"NUL between digits", "4\x002", "was not a number"},
{"hex notation", "0x10", "was not a number"},
{"scientific notation", "1e3", "was not a number"},
{"leading whitespace", " 42", "was not a number"},
{"fullwidth digits", "42", "was not a number"},
// Range branch.
{"range upper out of range", "1-65536", "ending range out of range"},
{"range lower out of range", "65536-65537", "beginning range out of range"},
{"range with negative upper", "1--1", "ending range was not a number"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, _, err := parsePort(tc.input)
require.Error(t, err, "input %q must error", tc.input)
require.ErrorContains(t, err, tc.wantErrContains)
})
}
}
// Test_parsePort_valid_boundaries locks in success cases at 0, 1, and 65535
// so a future refactor cannot regress the boundaries.
func Test_parsePort_valid_boundaries(t *testing.T) {
tests := []struct {
name string
input string
wantStart int32
wantEnd int32
}{
{"zero is PortAny", "0", 0, 0},
{"min real port", "1", 1, 1},
{"max real port", "65535", 65535, 65535},
{"range zero to max forces end to zero", "0-65535", 0, 0},
{"range max to max", "65535-65535", 65535, 65535},
{"range one to max", "1-65535", 1, 65535},
{"range with whitespace inside", " 1 - 2 ", 1, 2},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
s, e, err := parsePort(tc.input)
require.NoError(t, err)
assert.Equal(t, tc.wantStart, s, "start port")
assert.Equal(t, tc.wantEnd, e, "end port")
})
}
}
func TestAddRulesFromConfig(t *testing.T) {
l := test.NewLogger()
// Test adding tcp rule
conf := config.NewC(test.NewLogger())
mf := &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "tcp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolTCP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding udp rule
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "udp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolUDP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding udplite, dccp, and sctp rules
for name, proto := range map[string]uint8{"udplite": iputil.IPProtocolUDPLite, "dccp": iputil.IPProtocolDCCP, "sctp": iputil.IPProtocolSCTP} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1-2", "proto": name, "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf), name)
assert.Equal(t, addRuleCall{incoming: false, proto: proto, startPort: 1, endPort: 2, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall, name)
}
// Test adding icmp rule
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "icmp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolICMP, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding icmp rule no port
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"proto": "icmp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolICMP, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding rules by protocol number, as yaml would give a string or an int
for _, proto := range []any{"47", 47} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "any", "proto": proto, "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf), proto)
assert.Equal(t, addRuleCall{incoming: false, proto: 47, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall, proto)
}
// Test ICMP and ICMPv6, by name and by number, still ignore ports
for _, tc := range []struct {
given any
want uint8
}{
{"icmp", iputil.IPProtocolICMP},
{"icmpv6", iputil.IPProtocolICMPv6},
{iputil.IPProtocolICMP, iputil.IPProtocolICMP},
{iputil.IPProtocolICMPv6, iputil.IPProtocolICMPv6},
} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": tc.given, "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf), tc.given)
assert.Equal(t, addRuleCall{incoming: false, proto: tc.want, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall, tc.given)
}
// Test protocols that aren't a known name or a number from 1 to 255
for _, proto := range []string{"0", "256", "-1", "gre"} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "any", "proto": proto, "host": "a"}}}
require.EqualError(t, addRulesFromConfig(l, false, conf, mf), "firewall.outbound rule #0; proto was not understood; `"+proto+"`")
}
// Test adding any rule
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding rule with cidr
cidr := netip.MustParsePrefix("10.0.0.0/8")
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr.String(), localIp: ""}, mf.lastCall)
// Test adding rule with local_cidr
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: cidr.String()}, mf.lastCall)
// Test adding rule with cidr ipv6
cidr6 := netip.MustParsePrefix("fd00::/8")
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr6.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr6.String(), localIp: ""}, mf.lastCall)
// Test adding rule with any cidr
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "any"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "any", localIp: ""}, mf.lastCall)
// Test adding rule with junk cidr, which RulesBuilder rejects
conf = config.NewC(test.NewLogger())
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "junk/junk"}}}
require.EqualError(t, addRulesFromConfig(l, true, conf, NewRulesBuilder(l)), "firewall.inbound rule #0; cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP")
// Test adding rule with local_cidr ipv6
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr6.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: cidr6.String()}, mf.lastCall)
// Test adding rule with any local_cidr
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "any"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, localIp: "any"}, mf.lastCall)
// Test adding rule with junk local_cidr, which RulesBuilder rejects
conf = config.NewC(test.NewLogger())
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "junk/junk"}}}
require.EqualError(t, addRulesFromConfig(l, true, conf, NewRulesBuilder(l)), "firewall.inbound rule #0; local_cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP")
// Test adding rule with ca_sha
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_sha": "12312313123"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caSha: "12312313123"}, mf.lastCall)
// Test adding rule with ca_name
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_name": "root01"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caName: "root01"}, mf.lastCall)
// Test single group
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "group": "a"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall)
// Test single groups
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": "a"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall)
// Test multiple AND groups
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": []string{"a", "b"}}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: []string{"a", "b"}, ip: "", localIp: ""}, mf.lastCall)
// Test group and groups values that used to crash: an empty group array is rejected, groups entries that
// aren't strings are read as strings, and a null groups is no groups
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "group": []any{}}}}
require.EqualError(t, addRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; group should contain a single value, an empty array was provided")
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": []any{1, 2}}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, []string{"1", "2"}, mf.lastCall.groups)
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": nil, "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Empty(t, mf.lastCall.groups)
assert.Equal(t, "a", mf.lastCall.host)
// Test Add error
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
mf.nextCallReturn = errors.New("test error")
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "host": "a"}}}
require.EqualError(t, addRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; test error")
}
// TestRulesFromConfig_defaultLocalCIDR ensures rules without a local_cidr allow only the vpn networks when there are
// unsafe networks, unless default_local_cidr_any is set, and that a rule's own local_cidr is kept either way
func TestRulesFromConfig_defaultLocalCIDR(t *testing.T) {
vpnNetworks := []netip.Prefix{netip.MustParsePrefix("10.1.0.5/16")}
unsafeNetworks := []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}
vpnAddr := netip.MustParseAddr("10.1.9.9")
unsafeAddr := netip.MustParseAddr("192.168.0.3")
// ruleOnPort returns the rule on proto whose port clause is exactly port, chosen from the rules that a packet
// on port is checked against.
ruleOnPort := func(t *testing.T, table *Table, proto uint8, port int32) *rule {
t.Helper()
for _, r := range table.rulesAt(proto, port) {
if r.startPort == port {
return r
}
}
require.Failf(t, "no rule found", "proto %d port %d", proto, port)
return nil
}
for _, tc := range []struct {
name string
unsafeNetworks []netip.Prefix
defaultLocalCIDRAny bool
wantUnsafe bool
}{
{"no unsafe networks", nil, false, true},
{"unsafe networks", unsafeNetworks, false, false},
{"unsafe networks and default_local_cidr_any", unsafeNetworks, true, true},
} {
t.Run(tc.name, func(t *testing.T) {
conf := config.NewC(test.NewLogger())
conf.Settings["firewall"] = map[string]any{
"default_local_cidr_any": tc.defaultLocalCIDRAny,
"inbound": []any{
map[string]any{"port": "any", "proto": "any", "host": "any"},
map[string]any{"port": "22", "proto": "tcp", "host": "any"},
map[string]any{"port": "80", "proto": "tcp", "host": "any", "local_cidr": "192.168.0.0/24"},
map[string]any{"port": "443", "proto": "tcp", "host": "any", "local_cidr": "any"},
},
}
rules, err := RulesFromConfig(test.NewLogger(), conf, vpnNetworks, tc.unsafeNetworks)
require.NoError(t, err)
// The proto `any` rule, as indexed by a protocol with rules of its own and as shared by one without.
for _, proto := range []uint8{iputil.IPProtocolTCP, 47} {
r := ruleOnPort(t, rules.In, proto, PortAny)
assert.True(t, r.matchLocal(vpnAddr), "proto %d", proto)
assert.Equal(t, tc.wantUnsafe, r.matchLocal(unsafeAddr), "proto %d", proto)
}
r := ruleOnPort(t, rules.In, iputil.IPProtocolTCP, 22)
assert.True(t, r.matchLocal(vpnAddr))
assert.Equal(t, tc.wantUnsafe, r.matchLocal(unsafeAddr))
r = ruleOnPort(t, rules.In, iputil.IPProtocolTCP, 80)
assert.False(t, r.matchLocal(vpnAddr))
assert.True(t, r.matchLocal(unsafeAddr))
r = ruleOnPort(t, rules.In, iputil.IPProtocolTCP, 443)
assert.True(t, r.matchLocal(vpnAddr))
assert.True(t, r.matchLocal(unsafeAddr))
})
}
}
func Test_convertRule(t *testing.T) {
ob := &bytes.Buffer{}
l := test.NewLoggerWithOutput(ob)
// Ensure group array of 1 is converted and a warning is printed
c := map[string]any{
"group": []any{"group1"},
}
r, err := yamlToConfigRule(l, c, "test", 1)
assert.Contains(t, ob.String(), "group was an array with a single value, converting to simple value")
assert.Contains(t, ob.String(), "table=test")
assert.Contains(t, ob.String(), "rule=1")
require.NoError(t, err)
assert.Equal(t, []string{"group1"}, r.Groups)
// Ensure group array of > 1 is errord
ob.Reset()
c = map[string]any{
"group": []any{"group1", "group2"},
}
r, err = yamlToConfigRule(l, c, "test", 1)
assert.Empty(t, ob.String())
require.Error(t, err, "group should contain a single value, an array with more than one entry was provided")
// Make sure a well formed group is alright
ob.Reset()
c = map[string]any{
"group": "group1",
}
r, err = yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
assert.Equal(t, []string{"group1"}, r.Groups)
}
func Test_convertRuleSanity(t *testing.T) {
ob := &bytes.Buffer{}
l := test.NewLoggerWithOutput(ob)
noWarningPlease := []map[string]any{
{"group": "group1"},
{"groups": []any{"group2"}},
{"host": "bob"},
{"cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "host": "bob"},
{"cidr": "1.1.1.1/1", "host": "bob"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
}
for _, c := range noWarningPlease {
r, err := yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
require.NoError(t, r.sanity(), "should not generate a sanity warning, %+v", c)
}
yesWarningPlease := []map[string]any{
{"group": "group1"},
{"groups": []any{"group2"}},
{"cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "host": "bob"},
{"cidr": "1.1.1.1/1", "host": "bob"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
}
for _, c := range yesWarningPlease {
c["host"] = "any"
r, err := yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
err = r.sanity()
require.Error(t, err, "I wanted a warning: %+v", c)
}
//reset the list
yesWarningPlease = []map[string]any{
{"group": "group1"},
{"groups": []any{"group2"}},
{"cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "host": "bob"},
{"cidr": "1.1.1.1/1", "host": "bob"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
}
for _, c := range yesWarningPlease {
r, err := yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
r.Groups = append(r.Groups, "any")
err = r.sanity()
require.Error(t, err, "I wanted a warning: %+v", c)
}
}
type addRuleCall struct {
incoming bool
proto uint8
startPort int32
endPort int32
groups []string
host string
ip string
localIp string
caName string
caSha string
}
type mockFirewall struct {
lastCall addRuleCall
nextCallReturn error
}
func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip, localIp, caName string, caSha string) error {
mf.lastCall = addRuleCall{
incoming: incoming,
proto: proto,
startPort: startPort,
endPort: endPort,
groups: groups,
host: host,
ip: ip,
localIp: localIp,
caName: caName,
caSha: caSha,
}
err := mf.nextCallReturn
mf.nextCallReturn = nil
return err
}
+40 -14
View File
@@ -40,29 +40,55 @@ func (fp *Packet) Copy() *Packet {
} }
func (fp Packet) MarshalJSON() ([]byte, error) { func (fp Packet) MarshalJSON() ([]byte, error) {
var proto string
switch fp.Protocol {
case iputil.IPProtocolTCP:
proto = "tcp"
case iputil.IPProtocolICMP:
proto = "icmp"
case iputil.IPProtocolICMPv6:
proto = "icmpv6"
case iputil.IPProtocolUDP:
proto = "udp"
default:
proto = fmt.Sprintf("unknown %v", fp.Protocol)
}
return json.Marshal(m{ return json.Marshal(m{
"LocalAddr": fp.LocalAddr.String(), "LocalAddr": fp.LocalAddr.String(),
"RemoteAddr": fp.RemoteAddr.String(), "RemoteAddr": fp.RemoteAddr.String(),
"LocalPort": fp.LocalPort, "LocalPort": fp.LocalPort,
"RemotePort": fp.RemotePort, "RemotePort": fp.RemotePort,
"Protocol": proto, "Protocol": protoName(fp.Protocol),
"Fragment": fp.Fragment, "Fragment": fp.Fragment,
}) })
} }
// protoNames are the protocol names that rules accept and packets report, and their numbers. Any other protocol
// is given by number.
var protoNames = []struct {
name string
proto uint8
}{
{"any", ProtoAny},
{"tcp", iputil.IPProtocolTCP},
{"udp", iputil.IPProtocolUDP},
{"udplite", iputil.IPProtocolUDPLite},
{"dccp", iputil.IPProtocolDCCP},
{"sctp", iputil.IPProtocolSCTP},
{"icmp", iputil.IPProtocolICMP},
{"icmpv6", iputil.IPProtocolICMPv6},
}
// protoByName returns the number of a protocol named in protoNames.
func protoByName(name string) (uint8, bool) {
for _, p := range protoNames {
if p.name == name {
return p.proto, true
}
}
return 0, false
}
// protoName returns the name in protoNames of a packet's protocol, or "unknown" and its number. `any` is only
// a rule's protocol, so a packet's protocol 0 is unknown.
func protoName(proto uint8) string {
if proto != ProtoAny {
for _, p := range protoNames {
if p.proto == proto {
return p.name
}
}
}
return fmt.Sprintf("unknown %v", proto)
}
// ParsedPacket is a Packet plus the parse byproducts the RX path reuses // ParsedPacket is a Packet plus the parse byproducts the RX path reuses
type ParsedPacket struct { type ParsedPacket struct {
Packet Packet
+258
View File
@@ -0,0 +1,258 @@
package firewall
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"hash/fnv"
"log/slog"
"math"
"net/netip"
"slices"
"strings"
"github.com/slackhq/nebula/iputil"
)
// RulesBuilder collects the inbound and outbound rules, Build turns them into a Table for each direction
type RulesBuilder struct {
// in and out are each direction's rules in the order they were added
in, out []rule
// defaultLocalCIDRAny makes rules without a local cidr allow any local address, even with unsafe networks
defaultLocalCIDRAny bool
// ruleText describes every rule as it is added, one line each. Build hashes it; see Rules.
ruleText strings.Builder
l *slog.Logger
}
func NewRulesBuilder(l *slog.Logger) *RulesBuilder {
return &RulesBuilder{l: l}
}
// AddRule adds a rule for incoming or outgoing traffic, on proto or on every protocol when proto is ProtoAny.
// The rule is parsed now, so a bad rule is reported when it's added rather than when the Tables are built.
func (b *RulesBuilder) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
// ICMPv6 rules are ICMP rules, and a packet of either protocol is checked against both
if proto == iputil.IPProtocolICMPv6 {
proto = iputil.IPProtocolICMP
}
if proto == iputil.IPProtocolICMP {
//ICMP traffic doesn't have ports, so we always coerce to "any", even if a value is provided
if startPort != PortAny {
b.l.Warn("ignoring port specification for ICMP firewall rule", "startPort", startPort)
}
startPort = PortAny
endPort = PortAny
}
r, err := parseRule(proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha)
if err != nil {
return err
}
if incoming {
b.in = append(b.in, r)
} else {
b.out = append(b.out, r)
}
// The rule hashes are of these lines, changing them changes every hash
fmt.Fprintf(&b.ruleText,
"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s\n",
incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha,
)
direction := "incoming"
if !incoming {
direction = "outgoing"
}
b.l.Info("Firewall rule added",
"firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha},
)
return nil
}
// Build turns the rules into a Table for each direction, and hashes them. Rules without a local cidr allow
// vpnNetworks when there are unsafe networks, otherwise any local address.
func (b *RulesBuilder) Build(vpnNetworks, unsafeNetworks []netip.Prefix) Rules {
var defaultLocal []netip.Prefix
if len(unsafeNetworks) > 0 && !b.defaultLocalCIDRAny {
// Never nil, even when there are no vpn networks, so that the rules allow nothing rather than everything.
defaultLocal = append([]netip.Prefix{}, vpnNetworks...)
}
text := []byte(b.ruleText.String())
sha := sha256.Sum256(text)
h := fnv.New32a()
h.Write(text)
return Rules{
In: buildTable(b.in, defaultLocal),
Out: buildTable(b.out, defaultLocal),
Hash: hex.EncodeToString(sha[:]),
HashFNV: h.Sum32(),
}
}
func parseRule(proto uint8, startPort, endPort int32, groups []string, host, cidr, localCidr, caName, caSha string) (rule, error) {
if startPort > endPort {
return rule{}, fmt.Errorf("start port was lower than end port")
}
if startPort < PortFragment || endPort > math.MaxUint16 {
return rule{}, fmt.Errorf("port range %d-%d is outside of %d-%d", startPort, endPort, PortFragment, math.MaxUint16)
}
// A range that includes port 0 is port `any`.
if startPort <= PortAny && PortAny <= endPort {
startPort, endPort = PortAny, PortAny
}
// A port rule on a protocol without ports could never match. ICMP rules were coerced to `any` by AddRule,
// and a proto `any` rule with a port applies to the protocols that have them.
if startPort > PortAny && proto != ProtoAny && !iputil.HasPorts(proto) {
return rule{}, fmt.Errorf("protocol %d has no ports, port must be any or fragment", proto)
}
r := rule{
proto: proto,
startPort: startPort,
endPort: endPort,
groups: groups,
host: host,
caName: caName,
caSha: caSha,
}
var anyCIDR bool
var err error
r.cidr, anyCIDR, err = parseCIDR(cidr)
if err != nil {
return rule{}, fmt.Errorf("cidr did not parse; %w", err)
}
// A rule with no group, host, or cidr allows any host, as does a rule with `any` for one of them.
r.remoteAny = anyCIDR || host == "any" || slices.Contains(groups, "any") ||
(len(groups) == 0 && host == "" && !r.cidr.IsValid())
var localCIDR netip.Prefix
localCIDR, r.localAny, err = parseCIDR(localCidr)
if err != nil {
return rule{}, fmt.Errorf("local_cidr did not parse; %w", err)
}
if localCIDR.IsValid() {
r.local = []netip.Prefix{localCIDR}
}
return r, nil
}
// buildTable turns one direction's rules into a Table. Rules without a local cidr allow defaultLocal,
// or any local address when defaultLocal is nil.
func buildTable(rules []rule, defaultLocal []netip.Prefix) *Table {
t := &Table{rules: make([]rule, len(rules))}
var hasOwn [256]bool
for id, r := range rules {
t.rules[id] = r.withDefaultLocal(defaultLocal)
if r.proto != ProtoAny {
hasOwn[r.proto] = true
}
}
// A protocol with rules of its own gets an index of those rules plus the proto `any` rules, so a packet
// checks a single index. The other protocols share an index of the proto `any` rules, one for protocols with
// ports and one for protocols without.
anyPorts := newProtoIndex(true, t.rules, ProtoAny)
anyNoPorts := newProtoIndex(false, t.rules, ProtoAny)
for proto := range t.protos {
switch {
case hasOwn[proto]:
t.protos[proto] = newProtoIndex(iputil.HasPorts(uint8(proto)), t.rules, uint8(proto))
case iputil.HasPorts(uint8(proto)):
t.protos[proto] = anyPorts
default:
t.protos[proto] = anyNoPorts
}
}
// ICMPv6 rules are ICMP rules, see AddRule, so ICMPv6 packets are checked against the ICMP index
t.protos[iputil.IPProtocolICMPv6] = t.protos[iputil.IPProtocolICMP]
return t
}
// newProtoIndex builds an index of the proto `any` rules and proto's own rules, for a protocol with or without
// ports, or returns nil when there are none. With ProtoAny, it indexes only the proto `any` rules.
func newProtoIndex(hasPorts bool, rules []rule, proto uint8) *protoIndex {
var pi *protoIndex
for id := range rules {
r := &rules[id]
if r.proto != ProtoAny && r.proto != proto {
continue
}
if pi == nil {
pi = &protoIndex{hasPorts: hasPorts, fragment: newRuleSet(len(rules))}
if hasPorts {
pi.byPort = newRuleSets(math.MaxUint16+1, len(rules))
} else {
pi.packet = newRuleSet(len(rules))
}
}
pi.add(r, id)
}
return pi
}
// add puts id in every set that r's port clause covers.
func (pi *protoIndex) add(r *rule, id int) {
switch r.startPort {
case PortAny:
pi.fragment.add(id)
if pi.hasPorts {
pi.addPorts(0, math.MaxUint16, id)
} else {
pi.packet.add(id)
}
case PortFragment:
pi.fragment.add(id)
default:
// A proto `any` rule with a port is indexed for the protocols that have ports, parseRule rejects a port on
// a protocol without them.
if pi.hasPorts {
pi.addPorts(int(r.startPort), int(r.endPort), id)
}
}
}
// addPorts puts id in the set of every port from lo through hi.
func (pi *protoIndex) addPorts(lo, hi, id int) {
for port := lo; port <= hi; port++ {
pi.byPort.at(port).add(id)
}
}
// parseCIDR parses a rule's cidr, which is empty, `any`, or a prefix
func parseCIDR(s string) (prefix netip.Prefix, isAny bool, err error) {
switch s {
case "":
return netip.Prefix{}, false, nil
case "any":
return netip.Prefix{}, true, nil
}
prefix, err = netip.ParsePrefix(s)
return prefix, false, err
}
// withDefaultLocal returns r allowing defaultLocal, or any local address when defaultLocal is nil,
// unless r has a local cidr of its own
func (r rule) withDefaultLocal(defaultLocal []netip.Prefix) rule {
if !r.localAny && r.local == nil {
r.localAny = defaultLocal == nil
r.local = defaultLocal
}
return r
}
+367
View File
@@ -0,0 +1,367 @@
package firewall
import (
"fmt"
"math"
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// rulesAt returns the rules that a packet on proto with the given port is checked against, in id order.
// port may also be PortAny or PortFragment.
func (t *Table) rulesAt(proto uint8, port int32) []*rule {
pi := t.protos[proto]
if pi == nil {
return nil
}
p := &Packet{Protocol: proto, Fragment: port == PortFragment}
if port > 0 {
p.LocalPort = uint16(port)
}
var rules []*rule
for id := range pi.whichMatch(p, true).all() {
rules = append(rules, &t.rules[id])
}
return rules
}
// oneRuleAt returns the single rule that rulesAt finds, and fails the test when there is not exactly one.
func oneRuleAt(t *testing.T, table *Table, proto uint8, key int32) *rule {
t.Helper()
rules := table.rulesAt(proto, key)
require.Len(t, rules, 1)
return rules[0]
}
func TestRulesBuilder_AddRule(t *testing.T) {
l := test.NewLogger()
ti := netip.MustParsePrefix("1.2.3.4/32")
ti6 := netip.MustParsePrefix("fd12::34/128")
rb := NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, []string{}, "", "", "", "", ""))
rules := rb.Build(nil, nil)
// An empty rule is any
r := oneRuleAt(t, rules.In, iputil.IPProtocolTCP, 1)
assert.True(t, r.remoteAny)
assert.Empty(t, r.groups)
assert.Empty(t, r.host)
// It applies only to its own port.
assert.Empty(t, rules.In.rulesAt(iputil.IPProtocolTCP, 2))
assert.Empty(t, rules.In.rulesAt(iputil.IPProtocolTCP, PortAny))
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolUDP, 1, 1, []string{"g1"}, "", "", "", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.In, iputil.IPProtocolUDP, 1)
assert.False(t, r.remoteAny)
assert.Equal(t, []string{"g1"}, r.groups)
assert.Empty(t, r.host)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolICMP, 1, 1, []string{}, "h1", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolICMPv6, 1, 1, []string{}, "h2", "", "", "", ""))
rules = rb.Build(nil, nil)
// An ICMP rule is port `any` regardless of the port given, and ICMP and ICMPv6 share one set of rules.
for _, proto := range []uint8{iputil.IPProtocolICMP, iputil.IPProtocolICMPv6} {
rs := rules.In.rulesAt(proto, PortAny)
require.Len(t, rs, 2, "proto %d", proto)
assert.False(t, rs[0].remoteAny)
assert.Equal(t, "h1", rs[0].host)
assert.Equal(t, "h2", rs[1].host)
}
assert.Same(t, rules.In.protos[iputil.IPProtocolICMP], rules.In.protos[iputil.IPProtocolICMPv6])
// Any other protocol number gets its own rules
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, 47, 0, 0, []string{"g1"}, "", "", "", "", "")) // GRE
rules = rb.Build(nil, nil)
assert.Equal(t, []string{"g1"}, oneRuleAt(t, rules.In, 47, PortAny).groups)
assert.Nil(t, rules.In.protos[iputil.IPProtocolTCP])
// A proto `any` rule with a port applies to every protocol with ports, and ProtoAny itself has none.
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", ti.String(), "", "", ""))
rules = rb.Build(nil, nil)
assert.Empty(t, rules.Out.rulesAt(ProtoAny, 1))
r = oneRuleAt(t, rules.Out, iputil.IPProtocolTCP, 1)
assert.False(t, r.remoteAny)
assert.Equal(t, ti, r.cidr)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", ti6.String(), "", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, iputil.IPProtocolTCP, 1)
assert.False(t, r.remoteAny)
assert.Equal(t, ti6, r.cidr)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", "", ti.String(), "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, iputil.IPProtocolTCP, 1)
assert.True(t, r.remoteAny)
assert.False(t, r.localAny)
assert.Equal(t, []netip.Prefix{ti}, r.local)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", "", ti6.String(), "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, iputil.IPProtocolTCP, 1)
assert.True(t, r.remoteAny)
assert.False(t, r.localAny)
assert.Equal(t, []netip.Prefix{ti6}, r.local)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolUDP, 1, 1, []string{"g1"}, "", "", "", "ca-name", ""))
rules = rb.Build(nil, nil)
assert.Equal(t, "ca-name", oneRuleAt(t, rules.In, iputil.IPProtocolUDP, 1).caName)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolUDP, 1, 1, []string{"g1"}, "", "", "", "", "ca-sha"))
rules = rb.Build(nil, nil)
assert.Equal(t, "ca-sha", oneRuleAt(t, rules.In, iputil.IPProtocolUDP, 1).caSha)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "any", "", "", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, oneRuleAt(t, rules.Out, ProtoAny, 0).remoteAny)
// 0.0.0.0/0 allows any IPv4 address, but no IPv6 address.
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "0.0.0.0/0", "", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, ProtoAny, 0)
assert.False(t, r.remoteAny)
assert.True(t, r.matchRemote(netip.MustParseAddr("1.1.1.1"), nil))
assert.False(t, r.matchRemote(netip.MustParseAddr("9::9"), nil))
// ::/0 allows any IPv6 address, but no IPv4 address.
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "::/0", "", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, ProtoAny, 0)
assert.False(t, r.remoteAny)
assert.True(t, r.matchRemote(netip.MustParseAddr("9::9"), nil))
assert.False(t, r.matchRemote(netip.MustParseAddr("1.1.1.1"), nil))
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "any", "", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, oneRuleAt(t, rules.Out, ProtoAny, 0).remoteAny)
// A local cidr is likewise limited to its own address family.
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "", "0.0.0.0/0", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, ProtoAny, 0)
assert.False(t, r.localAny)
assert.True(t, r.matchLocal(netip.MustParseAddr("1.1.1.1")))
assert.False(t, r.matchLocal(netip.MustParseAddr("9::9")))
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "", "::/0", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, ProtoAny, 0)
assert.False(t, r.localAny)
assert.True(t, r.matchLocal(netip.MustParseAddr("9::9")))
assert.False(t, r.matchLocal(netip.MustParseAddr("1.1.1.1")))
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "", "any", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, oneRuleAt(t, rules.Out, ProtoAny, 0).localAny)
// Every protocol number is accepted
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, math.MaxUint8, 0, 0, []string{}, "", "", "", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, oneRuleAt(t, rules.In, math.MaxUint8, PortAny).remoteAny)
// Test error conditions, a bad rule is reported when it's added
rb = NewRulesBuilder(l)
require.Error(t, rb.AddRule(true, ProtoAny, 10, 0, []string{}, "", "", "", "", ""))
require.Error(t, rb.AddRule(true, ProtoAny, PortFragment-1, 0, []string{}, "", "", "", "", ""))
require.Error(t, rb.AddRule(true, ProtoAny, 1, math.MaxUint16+1, []string{}, "", "", "", "", ""))
require.Error(t, rb.AddRule(true, ProtoAny, 0, 0, []string{}, "", "junk", "", "", ""))
require.Error(t, rb.AddRule(true, ProtoAny, 0, 0, []string{}, "", "", "junk", "", ""))
// ICMP and ICMPv6 share one copy of each proto `any` rule
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, ProtoAny, PortAny, PortAny, []string{"default-group"}, "", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolICMPv6, PortAny, PortAny, []string{"nope"}, "", "", "", "", ""))
rules = rb.Build(nil, nil)
assert.Len(t, rules.In.rules, 2)
assert.Len(t, rules.In.rulesAt(iputil.IPProtocolICMP, PortAny), 2)
assert.Len(t, rules.In.rulesAt(iputil.IPProtocolICMPv6, PortAny), 2)
// No rules allow nothing
rules = NewRulesBuilder(l).Build(nil, nil)
for proto := range rules.In.protos {
assert.Nil(t, rules.In.protos[proto], "proto %d", proto)
assert.Nil(t, rules.Out.protos[proto], "proto %d", proto)
}
}
// TestRulesBuilder_protoIndex checks which packets each kind of port clause covers, and which protocols
// share an index.
func TestRulesBuilder_protoIndex(t *testing.T) {
rb := NewRulesBuilder(test.NewLogger())
require.NoError(t, rb.AddRule(true, ProtoAny, PortAny, PortAny, nil, "any", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 22, 22, nil, "port", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1024, math.MaxUint16, nil, "range", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, PortFragment, PortFragment, nil, "fragment", "", "", "", ""))
// A range that includes port 0 is port `any`.
require.NoError(t, rb.AddRule(true, iputil.IPProtocolUDP, 0, 100, nil, "zero", "", "", "", ""))
// GRE has no ports, so a port rule on it is rejected rather than never applying, and `any` and `fragment` apply
require.Error(t, rb.AddRule(true, 47, 22, 22, nil, "gre", "", "", "", ""))
require.NoError(t, rb.AddRule(true, 47, PortFragment, PortFragment, nil, "gre", "", "", "", ""))
rules := rb.Build(nil, nil)
hosts := func(proto uint8, key int32) []string {
var hosts []string
for _, r := range rules.In.rulesAt(proto, key) {
hosts = append(hosts, r.host)
}
return hosts
}
assert.Equal(t, []string{"any", "fragment"}, hosts(iputil.IPProtocolTCP, PortFragment))
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolTCP, PortAny))
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolTCP, 21))
assert.Equal(t, []string{"any", "port"}, hosts(iputil.IPProtocolTCP, 22))
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolTCP, 23))
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolTCP, 1023))
assert.Equal(t, []string{"any", "range"}, hosts(iputil.IPProtocolTCP, 1024))
assert.Equal(t, []string{"any", "range"}, hosts(iputil.IPProtocolTCP, math.MaxUint16))
assert.Equal(t, []string{"any", "zero"}, hosts(iputil.IPProtocolUDP, PortFragment))
assert.Equal(t, []string{"any", "zero"}, hosts(iputil.IPProtocolUDP, PortAny))
assert.Equal(t, []string{"any", "zero"}, hosts(iputil.IPProtocolUDP, math.MaxUint16))
// A protocol without ports sees only the port `any` rules, and the port `fragment` rules for a fragment.
assert.False(t, rules.In.protos[47].hasPorts)
assert.Equal(t, []string{"any"}, hosts(47, PortAny))
assert.Equal(t, []string{"any"}, hosts(47, 22))
assert.Equal(t, []string{"any", "gre"}, hosts(47, PortFragment))
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolICMP, PortFragment))
// Protocols without rules of their own share an index of the proto `any` rules: one for protocols with
// ports and one for protocols without.
assert.True(t, rules.In.protos[iputil.IPProtocolSCTP].hasPorts)
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolSCTP, 5000))
assert.Same(t, rules.In.protos[iputil.IPProtocolSCTP], rules.In.protos[iputil.IPProtocolDCCP])
assert.NotSame(t, rules.In.protos[iputil.IPProtocolSCTP], rules.In.protos[iputil.IPProtocolTCP])
assert.Equal(t, []string{"any"}, hosts(50, PortAny))
assert.Same(t, rules.In.protos[ProtoAny], rules.In.protos[50])
assert.Same(t, rules.In.protos[ProtoAny], rules.In.protos[iputil.IPProtocolICMP])
assert.NotSame(t, rules.In.protos[ProtoAny], rules.In.protos[iputil.IPProtocolSCTP])
assert.Nil(t, rules.Out.protos[iputil.IPProtocolTCP])
}
// TestRulesBuilder_manyRules checks that rule ids beyond the first byte of a rule set are indexed correctly.
func TestRulesBuilder_manyRules(t *testing.T) {
rb := NewRulesBuilder(test.NewLogger())
const n = 130
for i := range n {
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, int32(i+1), int32(i+1), nil, fmt.Sprintf("h%d", i), "", "", "", ""))
}
require.NoError(t, rb.AddRule(true, ProtoAny, PortAny, PortAny, nil, "any", "", "", "", ""))
rules := rb.Build(nil, nil)
assert.Equal(t, ruleSetLen(n+1), rules.In.protos[iputil.IPProtocolTCP].byPort.setLen)
// Ids are the order the rules were added in, so the proto `any` rule comes last
for i := range n {
rs := rules.In.rulesAt(iputil.IPProtocolTCP, int32(i+1))
require.Len(t, rs, 2, "port %d", i+1)
assert.Equal(t, fmt.Sprintf("h%d", i), rs[0].host)
assert.Equal(t, "any", rs[1].host)
}
assert.Len(t, rules.In.rulesAt(iputil.IPProtocolTCP, n+1), 1)
}
// TestRulesBuilder_defaultLocalCIDR checks which local addresses rules allow when there are unsafe networks.
// A rule without a local cidr allows only the vpn networks, and a rule with one allows only that cidr.
func TestRulesBuilder_defaultLocalCIDR(t *testing.T) {
vpnNetworks := []netip.Prefix{
netip.MustParsePrefix("10.1.0.5/16"),
netip.MustParsePrefix("10.2.0.5/16"),
netip.MustParsePrefix("fd00:1::5/64"),
}
unsafeNetworks := []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}
vpn1 := netip.MustParseAddr("10.1.9.9")
vpn2 := netip.MustParseAddr("10.2.9.9")
vpn6 := netip.MustParseAddr("fd00:1::9")
h1Extra := netip.MustParseAddr("10.3.0.1")
h1Extra6 := netip.MustParseAddr("fd00:2::1")
h2Own := netip.MustParseAddr("10.4.0.1")
rb := NewRulesBuilder(test.NewLogger())
// h1 allows the vpn networks by default on ports 1 and 2, and 10.3.0.0/16 and fd00:2::/64 as well on port 1
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 2, nil, "h1", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, nil, "h1", "", "10.3.0.0/16", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, nil, "h1", "", "fd00:2::/64", "", ""))
// h2 allows 10.4.0.0/16 on ports 1 and 2, and the vpn networks by default as well on port 1
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 2, nil, "h2", "", "10.4.0.0/16", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, nil, "h2", "", "", "", ""))
// h3 only has the default
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 3, 3, nil, "h3", "", "", "", ""))
rules := rb.Build(vpnNetworks, unsafeNetworks)
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
caPool := cert.NewCAPool()
require.NoError(t, caPool.AddCA(ca))
certs := map[string]*cert.CachedCertificate{}
for _, name := range []string{"h1", "h2", "h3"} {
crt, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, name, time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.9/16")}, nil, nil)
c, err := caPool.VerifyCertificate(time.Now(), crt)
require.NoError(t, err)
certs[name] = c
}
for _, tc := range []struct {
port uint16
host string
addr netip.Addr
want bool
}{
{1, "h1", vpn1, true},
{1, "h1", vpn2, true},
{1, "h1", vpn6, true},
{1, "h1", h1Extra, true},
{1, "h1", h1Extra6, true},
{1, "h1", h2Own, false},
{2, "h1", vpn1, true},
{2, "h1", h1Extra, false},
{2, "h1", h1Extra6, false},
{1, "h2", h2Own, true},
{1, "h2", vpn1, true},
{1, "h2", vpn2, true},
{1, "h2", vpn6, true},
{1, "h2", h1Extra, false},
{2, "h2", h2Own, true},
{2, "h2", vpn1, false},
{2, "h2", vpn6, false},
{3, "h3", vpn1, true},
{3, "h3", vpn2, true},
{3, "h3", vpn6, true},
{3, "h3", h1Extra, false},
{3, "h3", h1Extra6, false},
{3, "h3", h2Own, false},
// Only h3 is allowed on port 3.
{3, "h1", vpn1, false},
} {
p := &Packet{Protocol: iputil.IPProtocolTCP, LocalPort: tc.port, LocalAddr: tc.addr, RemoteAddr: netip.MustParseAddr("10.1.0.9")}
assert.Equal(t, tc.want, rules.In.Match(p, true, certs[tc.host], caPool), "port %d, %s, %s", tc.port, tc.host, tc.addr)
}
}
+60
View File
@@ -0,0 +1,60 @@
package firewall
import (
"iter"
"math/bits"
)
// ruleSet is a set of rule ids, stored as a bitset. s[i] holds ids i*8 through i*8+7, one per bit.
// Bytes rather than words keep a set to the size the rule count needs, which matters when there is a set for
// every port number.
type ruleSet []uint8
// ruleSetLen returns the length of a ruleSet that can hold ids below n.
func ruleSetLen(n int) int {
return (n + 7) / 8
}
// newRuleSet returns an empty set that can hold ids below idLimit.
func newRuleSet(idLimit int) ruleSet {
return make(ruleSet, ruleSetLen(idLimit))
}
// add puts id in s.
func (s ruleSet) add(id int) {
s[id/8] |= 1 << (id % 8)
}
// ruleSets holds several ruleSet objects of the same length in a single allocation.
type ruleSets struct {
// setLen is the number of bytes needed to represent one set
setLen int
// bits holds the sets in order. Set i starts at i*setLen.
bits []uint8
}
// newRuleSets returns n empty sets, each of which can hold ids below idLimit.
func newRuleSets(n, idLimit int) ruleSets {
setLen := ruleSetLen(idLimit)
return ruleSets{setLen: setLen, bits: make([]uint8, n*setLen)}
}
// at returns set i.
func (s ruleSets) at(i int) ruleSet {
return s.bits[i*s.setLen : (i+1)*s.setLen]
}
// all returns an iterator over the ids in s, in ascending order.
func (s ruleSet) all() iter.Seq[int] {
return func(yield func(int) bool) {
for i, ids := range s {
for ids != 0 {
id := i*8 + bits.TrailingZeros8(ids)
ids &= ids - 1 // Clear the lowest set bit, which is id's.
if !yield(id) {
return
}
}
}
}
}
+189
View File
@@ -0,0 +1,189 @@
package firewall
import (
"net/netip"
"github.com/slackhq/nebula/cert"
)
// Rules are the Tables built for each direction, with hashes of the rules they were built from
type Rules struct {
In, Out *Table
// Hash is a sha256 of the rules, HashFNV is an FNV-1a of them for use as a metric value.
// Rule sets with the same rules have the same hashes.
Hash string
HashFNV uint32
}
// Table holds the rules for one direction. A packet is allowed when every clause of a single rule allows it:
// proto AND port AND (CA SHA or CA name) AND local CIDR AND (group OR groups OR name OR remote CIDR).
// A Table does not change once built; see RulesBuilder.
type Table struct {
// rules holds every rule for the direction, in the order they were added. A rule's index in this slice is
// its id in the ruleSets.
rules []rule
// protos holds the index for each IP protocol number, so a packet is checked against a single index.
// A protocol with rules of its own has an index that also includes every proto `any` rule. The remaining
// protocols share an index of only the proto `any` rules: one for protocols with ports and one for
// protocols without; see hasPorts. An entry is nil when the protocol has no rules at all.
// ICMP and ICMPv6 share one index.
protos [256]*protoIndex
}
// protoIndex holds the rules that can apply to one protocol, in a set for each kind of packet. Rule ids are
// indexes into the direction's rules, so every set is sized by the direction's rule count, not the protocol's:
// byPort costs 64 KB per 8 rules in the direction.
type protoIndex struct {
// hasPorts reports whether the protocol has ports; see iputil.HasPorts.
hasPorts bool
// byPort has a set for every port number of a protocol with ports. Port 0 is covered only by port `any`
// rules, since a range starts at 1.
byPort ruleSets
// packet holds the rules for a packet of a protocol without ports: the port `any` rules.
packet ruleSet
// fragment holds the rules for a fragment of any protocol: the port `any` and port `fragment` rules.
fragment ruleSet
}
// whichMatch returns the set of rules whose proto and port clauses allow p.
func (pi *protoIndex) whichMatch(p *Packet, incoming bool) ruleSet {
switch {
case p.Fragment:
// The ports of a fragmented packet are in its first fragment, and this is a later one.
return pi.fragment
case !pi.hasPorts:
// The port fields are zero, or for ICMP the identifier, which is only for connection tracking.
return pi.packet
case incoming:
return pi.byPort.at(int(p.LocalPort))
default:
return pi.byPort.at(int(p.RemotePort))
}
}
// rule is a parsed rule. A packet is allowed by the rule when its port, certificate, and address clauses all pass.
type rule struct {
// proto is the IP protocol number the rule applies to, or ProtoAny for every protocol. ICMPv6 rules are kept
// as ICMP. The protoIndex checks it, so match does not.
proto uint8
// startPort and endPort are the port clause. A startPort of PortAny covers every port, every packet of a
// protocol without ports, and every fragment. PortFragment covers only fragments. A range covers the ports
// in it, so it never applies to a protocol without ports.
startPort, endPort int32
// caSha and caName pass a certificate issued by either CA. When neither is set, any CA passes.
caSha, caName string
// remoteAny makes host, groups, and cidr irrelevant. Otherwise, the peer passes when any one of them matches.
remoteAny bool
host string
groups []string
cidr netip.Prefix // Invalid when the rule has no cidr.
// localAny allows any local address. Otherwise, local must contain the address. Neither is set for a rule
// without a local cidr until the Table is built and the rule gets the default; see withDefaultLocal.
localAny bool
local []netip.Prefix
}
// Match reports whether a rule for the direction p is going allows it
func (r *Rules) Match(p *Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
t := r.Out
if incoming {
t = r.In
}
return t.Match(p, incoming, c, caPool)
}
// Match reports whether a rule allows p
func (t *Table) Match(p *Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
proto := t.protos[p.Protocol]
if proto == nil {
// The protocol has no rules of its own, and there are no proto `any` rules.
return false
}
// The index yields every rule whose proto and port clauses allow p.
// The packet is allowed when the remaining clauses of any of those rules allow it as well.
for id := range proto.whichMatch(p, incoming).all() {
if t.rules[id].match(p, c, caPool) {
return true
}
}
return false
}
// match reports whether r allows p from the peer with certificate c. The port clause is not checked here; the
// protoIndex has already done so.
func (r *rule) match(p *Packet, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
return r.matchLocal(p.LocalAddr) && r.matchRemote(p.RemoteAddr, c) && r.matchCA(c, caPool)
}
// matchLocal reports whether r allows the local address addr.
func (r *rule) matchLocal(addr netip.Addr) bool {
if r.localAny {
return true
}
for _, prefix := range r.local {
if prefix.Contains(addr) {
return true
}
}
return false
}
// matchRemote reports whether r allows the peer with certificate c, sending from addr. The cidr is checked with
// Prefix.Contains, so 0.0.0.0/0 allows any IPv4 address and ::/0 allows any IPv6 address; neither allows the
// other family.
func (r *rule) matchRemote(addr netip.Addr, c *cert.CachedCertificate) bool {
if r.remoteAny {
return true
}
if r.cidr.IsValid() && r.cidr.Contains(addr) {
return true
}
if r.host != "" && r.host == c.Certificate.Name() {
return true
}
// Every group must be present.
if len(r.groups) == 0 {
return false
}
for _, g := range r.groups {
if _, ok := c.InvertedGroups[g]; !ok {
return false
}
}
return true
}
// matchCA reports whether c was issued by a CA that r allows.
func (r *rule) matchCA(c *cert.CachedCertificate, caPool *cert.CAPool) bool {
if r.caSha == "" && r.caName == "" {
return true
}
if r.caSha != "" && r.caSha == c.Certificate.Issuer() {
return true
}
if r.caName != "" {
s, err := caPool.GetCAForCert(c.Certificate)
if err != nil {
return false
}
return r.caName == s.Certificate.Name()
}
return false
}
+333 -549
View File
File diff suppressed because it is too large. Load diff
+3 -4
View File
@@ -1,6 +1,6 @@
module github.com/slackhq/nebula module github.com/slackhq/nebula
go 1.27.2 go 1.26.0
require ( require (
dario.cat/mergo v1.0.2 dario.cat/mergo v1.0.2
@@ -33,7 +33,7 @@ require (
golang.zx2c4.com/wireguard/windows v1.0.1 golang.zx2c4.com/wireguard/windows v1.0.1
google.golang.org/protobuf v1.36.12 google.golang.org/protobuf v1.36.12
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
) )
require ( require (
@@ -45,6 +45,5 @@ require (
github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect
github.com/vishvananda/netns v0.0.5 // indirect github.com/vishvananda/netns v0.0.5 // indirect
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect golang.org/x/time v0.5.0 // indirect
golang.org/x/time v0.15.0 // indirect
) )
+4 -6
View File
@@ -157,8 +157,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -205,8 +203,8 @@ golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -245,5 +243,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 h1:ribyrTYQ/QYeUIs8x8bVLYwGyepa7RnHLqJzegT5YYk= gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g=
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980/go.mod h1:pK/dapF5Bf3KGW7/kd5CI5IC4lYh9FMFY1qETAIGLXk= gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
+5 -18
View File
@@ -31,7 +31,6 @@ package nebula
import ( import (
"bufio" "bufio"
"context"
"fmt" "fmt"
"net" "net"
"strconv" "strconv"
@@ -52,29 +51,17 @@ type graphiteConfigExport struct {
Percentiles []float64 // Percentiles to export from timers and histograms Percentiles []float64 // Percentiles to export from timers and histograms
} }
// graphiteTimeout bounds one submission, the connect and the write together. An export is a few KB, a host that can't
// take that in this long isn't going to.
const graphiteTimeout = 30 * time.Second
// graphiteOnce performs a single submission to Graphite, returning a non-nil // graphiteOnce performs a single submission to Graphite, returning a non-nil
// error if it could not connect or write all of it within timeout. It gives up // error on failed connections.
// as soon as ctx is done. func graphiteOnce(c graphiteConfigExport) error {
func graphiteOnce(ctx context.Context, c graphiteConfigExport, timeout time.Duration) error {
now := time.Now().Unix() now := time.Now().Unix()
du := float64(c.DurationUnit) du := float64(c.DurationUnit)
flushSeconds := float64(c.FlushInterval) / float64(time.Second) flushSeconds := float64(c.FlushInterval) / float64(time.Second)
deadline := time.Now().Add(timeout) conn, err := net.DialTCP("tcp", nil, c.Addr)
d := net.Dialer{Deadline: deadline}
conn, err := d.DialContext(ctx, "tcp", c.Addr.String())
if err != nil { if err != nil {
return err return err
} }
defer conn.Close() defer conn.Close()
if err := conn.SetDeadline(deadline); err != nil {
return err
}
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
defer stop()
w := bufio.NewWriter(conn) w := bufio.NewWriter(conn)
c.Registry.Each(func(name string, i any) { c.Registry.Each(func(name string, i any) {
switch metric := i.(type) { switch metric := i.(type) {
@@ -124,7 +111,7 @@ func graphiteOnce(ctx context.Context, c graphiteConfigExport, timeout time.Dura
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now) fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now)
fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now) fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now)
} }
w.Flush()
}) })
// A failed write sticks, so this reports the first one return nil
return w.Flush()
} }
-315
View File
@@ -1,315 +0,0 @@
package nebula
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"strings"
"sync"
"testing"
"time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// graphiteHost accepts connections and hands each one to the test
type graphiteHost struct {
ln net.Listener
conns chan net.Conn
mu sync.Mutex
closed bool
open []net.Conn
}
// newGraphiteHost accepts connections for the test. A non-zero window shrinks each connection's receive buffer, so a
// sender that is not read from blocks after a little whatever the host's TCP tuning. Only for tests that need a stall,
// a small window makes reading very slow on Linux.
func newGraphiteHost(t *testing.T, window int) *graphiteHost {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
h := &graphiteHost{ln: ln, conns: make(chan net.Conn, 8)}
done := make(chan struct{})
t.Cleanup(func() {
_ = ln.Close()
h.mu.Lock()
h.closed = true
for _, c := range h.open {
_ = c.Close()
}
h.mu.Unlock()
close(done)
})
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
h.mu.Lock()
if h.closed {
h.mu.Unlock()
_ = c.Close()
return
}
h.open = append(h.open, c)
h.mu.Unlock()
if tc, ok := c.(*net.TCPConn); ok && window > 0 {
_ = tc.SetReadBuffer(window)
}
select {
case h.conns <- c:
case <-done:
return
}
}
}()
return h
}
func (h *graphiteHost) addr(t *testing.T) *net.TCPAddr {
a, err := net.ResolveTCPAddr("tcp", h.ln.Addr().String())
require.NoError(t, err)
return a
}
// readExport reads what one send wrote, bounded so a send that never finishes fails the test rather than hanging it
func readExport(t *testing.T, c net.Conn) []byte {
t.Helper()
require.NoError(t, c.SetReadDeadline(time.Now().Add(10*time.Second)))
b, err := io.ReadAll(c)
require.NoError(t, err)
return b
}
func (h *graphiteHost) next(t *testing.T, wait time.Duration) net.Conn {
t.Helper()
select {
case c := <-h.conns:
return c
case <-time.After(wait):
t.Fatal("no connection")
return nil
}
}
// lockedBuffer is a log destination a test can read while the code under test writes to it from another goroutine
type lockedBuffer struct {
mu sync.Mutex
buf bytes.Buffer
}
func (b *lockedBuffer) Write(p []byte) (int, error) {
b.mu.Lock()
defer b.mu.Unlock()
return b.buf.Write(p)
}
func (b *lockedBuffer) String() string {
b.mu.Lock()
defer b.mu.Unlock()
return b.buf.String()
}
func testGraphiteConfig(addr *net.TCPAddr, r metrics.Registry) graphiteConfigExport {
return graphiteConfigExport{Addr: addr, Registry: r, FlushInterval: time.Second, DurationUnit: time.Nanosecond, Prefix: "nebula"}
}
// Enough metrics that one export overflows the socket buffers on any platform, so a send to a host that stops
// reading blocks
func bigGraphiteRegistry(r metrics.Registry, t *testing.T) {
for i := range 200000 {
name := fmt.Sprintf("big.gauge.%d", i)
metrics.GetOrRegisterGauge(name, r).Update(int64(i))
t.Cleanup(func() { r.Unregister(name) })
}
}
func TestGraphiteOnce_sends(t *testing.T) {
r := metrics.NewRegistry()
metrics.GetOrRegisterGauge("g", r).Update(7)
metrics.GetOrRegisterCounter("c", r).Inc(3)
h := newGraphiteHost(t, 0)
require.NoError(t, graphiteOnce(t.Context(), testGraphiteConfig(h.addr(t), r), time.Minute))
b := readExport(t, h.next(t, 5*time.Second))
assert.Contains(t, string(b), "nebula.g.value 7 ")
assert.Contains(t, string(b), "nebula.c.count 3 ")
assert.Contains(t, string(b), "nebula.c.count_ps 3.00 ")
}
// A host that accepts and never reads is given up on, and the failed write is reported
func TestGraphiteOnce_stalledHostTimesOut(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 4096)
done := make(chan error, 1)
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(t.Context(), cfg, 300*time.Millisecond) }()
h.next(t, 5*time.Second) // accepted, never read
select {
case err := <-done:
require.Error(t, err)
case <-time.After(10 * time.Second):
t.Fatal("the send did not give up on a host that stopped reading")
}
}
// The timeout covers the whole send, a host that keeps reading but too slowly is still given up on
func TestGraphiteOnce_timeoutCoversTheWholeSend(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 0)
done := make(chan error, 1)
start := time.Now()
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(t.Context(), cfg, 500*time.Millisecond) }()
c := h.next(t, 5*time.Second)
stopReading := make(chan struct{})
defer close(stopReading)
// 32KB every 10ms keeps every write well inside the timeout, the whole export takes seconds
go func() {
buf := make([]byte, 32<<10)
for {
select {
case <-stopReading:
return
case <-time.After(10 * time.Millisecond):
}
if _, err := c.Read(buf); err != nil {
return
}
}
}()
select {
case err := <-done:
require.Error(t, err)
assert.Less(t, time.Since(start), 5*time.Second)
case <-time.After(10 * time.Second):
t.Fatal("a host that kept reading slowly held the send past its timeout")
}
}
func TestGraphiteOnce_cancelledCtxDoesNotConnect(t *testing.T) {
h := newGraphiteHost(t, 0)
ctx, cancel := context.WithCancel(t.Context())
cancel()
require.Error(t, graphiteOnce(ctx, testGraphiteConfig(h.addr(t), metrics.NewRegistry()), time.Minute))
select {
case <-h.conns:
t.Fatal("connected on a cancelled ctx")
case <-time.After(200 * time.Millisecond):
}
}
// A stop or reload abandons a stuck send rather than waiting out its timeout
func TestGraphiteOnce_cancelAbandonsSend(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 4096)
ctx, cancel := context.WithCancel(t.Context())
done := make(chan error, 1)
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(ctx, cfg, time.Minute) }()
h.next(t, 5*time.Second)
select {
case err := <-done:
t.Fatalf("the send should be stuck on a host that is not reading, it ended with %v", err)
case <-time.After(200 * time.Millisecond):
}
cancel()
select {
case err := <-done:
require.Error(t, err)
case <-time.After(5 * time.Second):
t.Fatal("the send is still stuck after its ctx was cancelled")
}
}
// buildRuntime hands graphite the configured prefix and interval
func TestStatsServer_graphiteExportUsesConfig(t *testing.T) {
const name = "graphite.wiring.test"
metrics.GetOrRegisterCounter(name, nil).Inc(8)
t.Cleanup(func() { metrics.DefaultRegistry.Unregister(name) })
h := newGraphiteHost(t, 0)
s, c := newTestStatsServer(t)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "4s", "prefix": "pfx", "host": h.ln.Addr().String()})
cfg, err := loadStatsConfig(c)
require.NoError(t, err)
s.runMu.Lock()
fns, _ := s.buildRuntime(t.Context(), cfg)
s.runMu.Unlock()
for _, fn := range fns {
fn()
}
b := readExport(t, h.next(t, 5*time.Second))
assert.Contains(t, string(b), "pfx."+name+".count 8 ")
assert.Contains(t, string(b), "pfx."+name+".count_ps 2.00 ")
}
func TestStatsServer_graphiteFailureIsLogged(t *testing.T) {
logs := &lockedBuffer{}
s, c := newTestStatsServer(t)
s.l = test.NewLoggerWithOutput(logs)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)})
cfg, err := loadStatsConfig(c)
require.NoError(t, err)
s.runMu.Lock()
fns, _ := s.buildRuntime(t.Context(), cfg)
s.runMu.Unlock()
for _, fn := range fns {
fn()
}
assert.Contains(t, logs.String(), "Graphite export failed")
}
// Stop abandons the running export, so it goes through Start rather than handing buildRuntime a ctx, and a stop is
// not logged as a failure
func TestStatsServer_stopAbandonsGraphiteSend(t *testing.T) {
bigGraphiteRegistry(metrics.DefaultRegistry, t)
h := newGraphiteHost(t, 4096)
logs := &lockedBuffer{}
s, c := newTestStatsServer(t)
s.l = test.NewLoggerWithOutput(logs)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "50ms", "host": h.ln.Addr().String()})
require.NoError(t, s.reload(c, true))
started := make(chan struct{})
go func() {
s.Start()
close(started)
}()
stuck := h.next(t, 30*time.Second)
s.Stop()
select {
case <-started:
case <-time.After(5 * time.Second):
t.Fatal("Start did not return after Stop")
}
// A closed sender resets the connection once it sees this, one still sending just buffers it. Either way the
// read below doesn't have to drain the export through the small window to find out
_, err := stuck.Write([]byte("x"))
require.NoError(t, err)
require.NoError(t, stuck.SetReadDeadline(time.Now().Add(10*time.Second)))
n, err := io.Copy(io.Discard, stuck)
var ne net.Error
if errors.As(err, &ne) && ne.Timeout() {
t.Fatalf("the send kept the connection after Stop, read %d bytes", n)
}
assert.Less(t, n, int64(1<<20), "the send kept going after Stop")
// The capture loop logs on its own goroutine after Start has returned, give it the chance
assert.Never(t, func() bool { return strings.Contains(logs.String(), "Graphite export failed") }, 500*time.Millisecond, 10*time.Millisecond, "a stop is not a failure")
}
+12 -9
View File
@@ -54,8 +54,7 @@ type HandshakeConfig struct {
} }
type HandshakeManager struct { type HandshakeManager struct {
// Mutex for interacting with the vpnIps and indexes maps. // Mutex for interacting with the vpnIps and indexes maps
// Lock order is hostmap, then this lock, then the lighthouse, then a RemoteList.
sync.RWMutex sync.RWMutex
vpnIps map[netip.Addr]*HandshakeHostInfo vpnIps map[netip.Addr]*HandshakeHostInfo
@@ -260,6 +259,14 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
"style": header.SubTypeName(header.Handshake, hh.machine.Subtype()), "style": header.SubTypeName(header.Handshake, hh.machine.Subtype()),
} }
// Get a remotes object if we don't already have one.
// This is mainly to protect us as this should never be the case
// NB ^ This comment doesn't jive. It's how the thing gets initialized.
// It's the common path. Should it update every time, in case a future LH query/queries give us more info?
if hostinfo.remotes == nil {
hostinfo.remotes = hm.lightHouse.QueryCache([]netip.Addr{vpnIp})
}
remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges()) remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges())
remotesHaveChanged := !slices.Equal(remotes, hh.lastRemotes) remotesHaveChanged := !slices.Equal(remotes, hh.lastRemotes)
@@ -364,9 +371,6 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
hostinfo := &HostInfo{ hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr}, vpnAddrs: []netip.Addr{vpnAddr},
HandshakePacket: make(map[uint8][]byte, 0), HandshakePacket: make(map[uint8][]byte, 0),
// Attached here so a pending hostinfo never exists without one: SetRemote dereferences
// it, and create-tunnel -address can reach SetRemote before the handshake loop runs.
remotes: hm.lightHouse.QueryCache([]netip.Addr{vpnAddr}),
relayState: RelayState{ relayState: RelayState{
relays: nil, relays: nil,
relayForByAddr: map[netip.Addr]*Relay{}, relayForByAddr: map[netip.Addr]*Relay{},
@@ -942,14 +946,13 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
hm.DeleteHostInfo(hostinfo) hm.DeleteHostInfo(hostinfo)
hm.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) { hm.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) {
newHH.hostinfo.remotes = hostinfo.remotes
newHH.hostinfo.remotes.BlockRemote(via) newHH.hostinfo.remotes.BlockRemote(via)
newHH.packetStore = hh.packetStore newHH.packetStore = hh.packetStore
hh.packetStore = []*cachedPacket{} hh.packetStore = []*cachedPacket{}
hostinfo.vpnAddrs = vpnAddrs
f.sendCloseTunnel(hostinfo)
}) })
// sendCloseTunnel may take the hostmap lock, so it must not run under the handshake
// manager lock that StartHandshake holds around the callback.
hostinfo.vpnAddrs = vpnAddrs
f.sendCloseTunnel(hostinfo)
return return
} }
+2 -68
View File
@@ -11,7 +11,6 @@ import (
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func Test_NewHandshakeManagerVpnIp(t *testing.T) { func Test_NewHandshakeManagerVpnIp(t *testing.T) {
@@ -43,6 +42,8 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
i2 := blah.StartHandshake(ip, nil) i2 := blah.StartHandshake(ip, nil)
assert.Same(t, i, i2) assert.Same(t, i, i2)
i.remotes = NewRemoteList([]netip.Addr{}, nil)
// Adding something to pending should not affect the main hostmap // Adding something to pending should not affect the main hostmap
assert.Empty(t, mainHM.Hosts) assert.Empty(t, mainHM.Hosts)
@@ -234,70 +235,3 @@ func TestHandleIncomingDispatch(t *testing.T) {
assert.Empty(t, hm.indexes, "orphan stage-2 must not create state") assert.Empty(t, hm.indexes, "orphan stage-2 must not create state")
}) })
} }
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
func Test_PendingHandshakeHasRemotes(t *testing.T) {
l := test.NewLogger()
mainHM := newHostMap(l)
preferredRanges := []netip.Prefix{}
mainHM.preferredRanges.Store(&preferredRanges)
hm := NewHandshakeManager(l, mainHM, newTestLighthouse(), &udp.NoopConn{}, defaultHandshakeConfig)
vpnIp := netip.MustParseAddr("172.1.1.2")
remote := netip.MustParseAddrPort("192.168.1.9:4242")
hi := hm.StartHandshake(vpnIp, func(hh *HandshakeHostInfo) {
require.NotNil(t, hh.hostinfo.remotes, "a pending handshake must never be without its RemoteList")
hh.hostinfo.SetRemote(remote)
})
assert.Contains(t, hi.remotes.CopyAddrs(preferredRanges), remote, "the handshake must be sent to the supplied remote")
assert.Equal(t, remote, hi.GetRemote())
// A RemoteList keeps one learned address per host, so a second remote replaces the
// first — the same behaviour change-remote has on an established tunnel.
other := netip.MustParseAddrPort("192.168.1.10:4242")
hi.SetRemote(other)
assert.Equal(t, []netip.AddrPort{other}, hi.remotes.CopyAddrs(preferredRanges))
assert.Equal(t, other, hi.GetRemote())
}
// newRemotesRaceFixture builds a handshake manager with an established tunnel to vpnAddr whose
// RemoteList is the lighthouse's cached list, the state every teardown-during-handshake race starts from.
func newRemotesRaceFixture(t *testing.T, vpnAddr netip.Addr) (*HandshakeManager, *Interface, *HostInfo) {
t.Helper()
l := test.NewLogger()
mainHM := newHostMap(l)
preferredRanges := []netip.Prefix{}
mainHM.preferredRanges.Store(&preferredRanges)
lh := newTestLighthouse()
hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig)
f := &Interface{hostMap: mainHM, lightHouse: lh, handshakeManager: hm, l: l}
hm.f = f
established := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
localIndexId: 1,
remoteIndexId: 101,
remotes: lh.QueryCache([]netip.Addr{vpnAddr}),
HandshakePacket: map[uint8][]byte{},
}
mainHM.Lock()
mainHM.unlockedAddHostInfo(established, f)
mainHM.Unlock()
return hm, f, established
}
// With no handshake pending, tearing down the last tunnel still evicts the list.
func Test_CloseTunnelEvictsWithoutPendingHandshake(t *testing.T) {
vpnAddr := netip.MustParseAddr("172.1.1.2")
_, f, established := newRemotesRaceFixture(t, vpnAddr)
f.closeTunnel(established)
assert.NotContains(t, f.hostMap.Hosts, vpnAddr)
assert.NotContains(t, f.lightHouse.addrMap, vpnAddr)
}
-5
View File
@@ -24,11 +24,6 @@ type m = map[string]any
const ( const (
Version uint8 = 1 Version uint8 = 1
Len = 16 Len = 16
// MaxOverhead is the most a payload grows on its way to the underlay.
// A relayed packet has two headers and two AEAD tags, one for the inner tunnel and another for the relay tunnel.
// Every supported cipher (today) has a 16 byte tag.
MaxOverhead = 2 * (Len + 16)
) )
type MessageType uint8 type MessageType uint8
+5 -7
View File
@@ -316,9 +316,9 @@ type cachedPacketMetrics struct {
func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap { func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap {
hm := newHostMap(l) hm := newHostMap(l)
hm.reload(c, true, nil) hm.reload(c, true)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
hm.reload(c, false, r) hm.reload(c, false)
}) })
l.Info("Main HostMap created", "preferredRanges", hm.GetPreferredRanges()) l.Info("Main HostMap created", "preferredRanges", hm.GetPreferredRanges())
@@ -337,8 +337,7 @@ func newHostMap(l *slog.Logger) *HostMap {
} }
} }
// r is nil on the initial load func (hm *HostMap) reload(c *config.C, initial bool) {
func (hm *HostMap) reload(c *config.C, initial bool, r *config.Reload) {
if initial || c.HasChanged("preferred_ranges") { if initial || c.HasChanged("preferred_ranges") {
var preferredRanges []netip.Prefix var preferredRanges []netip.Prefix
rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{}) rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{})
@@ -349,9 +348,8 @@ func (hm *HostMap) reload(c *config.C, initial bool, r *config.Reload) {
if err != nil { if err != nil {
hm.l.Warn("Failed to parse preferred ranges, ignoring", hm.l.Warn("Failed to parse preferred ranges, ignoring",
"error", err, "error", err,
"range", rawPreferredRange, "range", rawPreferredRanges,
) )
r.Failed()
continue continue
} }
-13
View File
@@ -447,16 +447,3 @@ func TestHostInfo_markOutOnly(t *testing.T) {
assert.True(t, h.markOut(6), "a relayed send must not consume the epoch edge") assert.True(t, h.markOut(6), "a relayed send must not consume the epoch edge")
assert.False(t, h.markOut(6)) assert.False(t, h.markOut(6))
} }
// A preferred range that doesn't parse is skipped and the rest are used, the reload says one was skipped
func TestHostMap_reloadBadPreferredRangeFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("preferred_ranges: [10.0.0.0/8]\n"))
hm := NewHostMapFromConfig(l, c)
res := c.ReloadConfigString("preferred_ranges: [10.0.0.0/8, nope]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, hm.GetPreferredRanges())
}
+18 -35
View File
@@ -10,7 +10,6 @@ import (
"github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/noiseutil" "github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
@@ -199,8 +198,8 @@ func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt tio.Packet, nb []b
} }
err = tio.SegmentSuperpacket(pkt, func(seg []byte) error { err = tio.SegmentSuperpacket(pkt, func(seg []byte) error {
//relay header + header + plaintext + AEAD tag + relay tag //relay header + header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305) + relay tag
scratch := sendBatch.Reserve(len(seg) + header.MaxOverhead) scratch := sendBatch.Reserve(header.Len + header.Len + len(seg) + 16 + 16)
innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb) innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb)
if innerPacket == nil { if innerPacket == nil {
@@ -518,9 +517,7 @@ func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, noc
err = f.writers[q].WriteTo(toSend, via.GetRemote()) err = f.writers[q].WriteTo(toSend, via.GetRemote())
if err != nil { if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpRelayTxErrLimiter); ok { via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err)
via.logger(l).Info("Failed to WriteTo in sendVia", "error", err)
}
} }
} }
@@ -529,26 +526,16 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
return return
} }
useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid() useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid()
// The encrypt appends to out in place and a relayed packet is then wrapped in place,
// so out has to fit the whole packet up front or the encrypt reallocates out from under SendVia.
// We cap the MTU of tun devices and routes we create ourselves, so only a host that set a bigger MTU than we allow gets here.
need := header.Len + len(p) + ci.eKey.Overhead()
if useRelay {
// The relay tunnel runs the same cipher, and adds its own header and tag
need += header.Len + ci.eKey.Overhead()
}
if cap(out) < need {
if l, ok := logging.RateLimited(f.l, f.txOversizeLimiter); ok {
hostinfo.logger(l).Error("Dropping outbound packet, out buffer not large enough", "outCap", cap(out), "payloadLen", len(p), "relayed", useRelay)
}
return
}
fullOut := out fullOut := out
if useRelay { if useRelay {
if len(out) < header.Len {
// out always has a capacity of mtu, but not always a length greater than the header.Len.
// Grow it to make sure the next operation works.
out = out[:header.Len]
}
// Save a header's worth of data at the front of the 'out' buffer. // Save a header's worth of data at the front of the 'out' buffer.
out = out[header.Len:header.Len] out = out[header.Len:]
} }
if noiseutil.EncryptLockNeeded { if noiseutil.EncryptLockNeeded {
@@ -595,22 +582,18 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
if remote.IsValid() { if remote.IsValid() {
err = f.writers[q].WriteTo(out, remote) err = f.writers[q].WriteTo(out, remote)
if err != nil { if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok { hostinfo.logger(f.l).Error("Failed to write outgoing packet",
hostinfo.logger(l).Error("Failed to write outgoing packet", "error", err,
"error", err, "udpAddr", remote,
"udpAddr", remote, )
)
}
} }
} else if hr := hostinfo.GetRemote(); hr.IsValid() { } else if hr := hostinfo.GetRemote(); hr.IsValid() {
err = f.writers[q].WriteTo(out, hr) err = f.writers[q].WriteTo(out, hr)
if err != nil { if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok { hostinfo.logger(f.l).Error("Failed to write outgoing packet",
hostinfo.logger(l).Error("Failed to write outgoing packet", "error", err,
"error", err, "udpAddr", hr,
"udpAddr", hr, )
)
}
} }
} else { } else {
// Try to send via a relay // Try to send via a relay
+23 -34
View File
@@ -24,7 +24,6 @@ import (
"github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/overlay"
"github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/ratelimit"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
) )
@@ -125,10 +124,6 @@ type Interface struct {
batchers []*batch.MultiCoalescer batchers []*batch.MultiCoalescer
wg sync.WaitGroup wg sync.WaitGroup
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
txOversizeLimiter *ratelimit.Limiter // oversized outbound packet drop log limiter
// fatalErr holds the first unexpected reader error that caused shutdown. // fatalErr holds the first unexpected reader error that caused shutdown.
// nil means "no fatal error" (yet) // nil means "no fatal error" (yet)
fatalErr atomic.Pointer[error] fatalErr atomic.Pointer[error]
@@ -203,19 +198,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
return nil, errors.New("no connection manager") return nil, errors.New("no connection manager")
} }
udpTxErrLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
udpRelayTxErrLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
txOversizeLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
if c.routines <= 1 { if c.routines <= 1 {
c.PinThreads = false //pinning is not useful unless there's more than one tun reader c.PinThreads = false //pinning is not useful unless there's more than one tun reader
} }
@@ -248,9 +230,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
conntrackCacheTimeout: c.ConntrackCacheTimeout, conntrackCacheTimeout: c.ConntrackCacheTimeout,
cpuAffinity: c.CpuAffinity, cpuAffinity: c.CpuAffinity,
pinThreads: c.PinThreads, pinThreads: c.PinThreads,
udpTxErrLimiter: udpTxErrLimiter,
udpRelayTxErrLimiter: udpRelayTxErrLimiter,
txOversizeLimiter: txOversizeLimiter,
metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)), metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
metricTxDropped: metrics.GetOrRegisterCounter("udp.tx.dropped", nil), metricTxDropped: metrics.GetOrRegisterCounter("udp.tx.dropped", nil),
@@ -384,8 +363,6 @@ type rxContext struct {
hostmapCache map[uint32]*HostInfo hostmapCache map[uint32]*HostInfo
lhh *LightHouseHandler lhh *LightHouseHandler
ctCache *firewall.ConntrackCacheTicker ctCache *firewall.ConntrackCacheTicker
relayBatch *batch.SendBatch
relayRemote netip.AddrPort
} }
func newRxContext(f *Interface, q int) *rxContext { func newRxContext(f *Interface, q int) *rxContext {
@@ -416,7 +393,6 @@ func (f *Interface) listenOut(i int) {
} }
flusher := func() { flusher := func() {
f.flushRelayBatch(rxc)
if err := f.batchers[i].Flush(); err != nil { if err := f.batchers[i].Flush(); err != nil {
f.l.Error("Failed to flush tun coalescer", "error", err) f.l.Error("Failed to flush tun coalescer", "error", err)
} }
@@ -512,11 +488,10 @@ func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) { func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
c.RegisterReloadCallback(f.reloadFirewall) c.RegisterReloadCallback(f.reloadFirewall)
for _, reload := range []func(*config.C){ c.RegisterReloadCallback(f.reloadSendRecvError)
f.reloadSendRecvError, f.reloadAcceptRecvError, f.reloadDisconnectInvalid, f.reloadMisc, c.RegisterReloadCallback(f.reloadAcceptRecvError)
} { c.RegisterReloadCallback(f.reloadDisconnectInvalid)
c.RegisterReloadCallback(func(c *config.C, _ *config.Reload) { reload(c) }) c.RegisterReloadCallback(f.reloadMisc)
}
for _, udpConn := range f.writers { for _, udpConn := range f.writers {
c.RegisterReloadCallback(udpConn.ReloadConfig) c.RegisterReloadCallback(udpConn.ReloadConfig)
@@ -533,7 +508,7 @@ func (f *Interface) reloadDisconnectInvalid(c *config.C) {
} }
} }
func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) { func (f *Interface) reloadFirewall(c *config.C) {
cs := f.pki.getCertState() cs := f.pki.getCertState()
curCert := cs.getCertificate(cert.Version2) curCert := cs.getCertificate(cert.Version2)
if curCert == nil { if curCert == nil {
@@ -552,7 +527,6 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
fw, err := NewFirewallFromConfig(f.l, cs, c) fw, err := NewFirewallFromConfig(f.l, cs, c)
if err != nil { if err != nil {
f.l.Error("Error while creating firewall during reload", "error", err) f.l.Error("Error while creating firewall during reload", "error", err)
r.Failed()
return return
} }
@@ -650,15 +624,17 @@ func (f *Interface) reloadMisc(c *config.C) {
} }
} }
// statsEmitter updates the interface's gauges, the stats server runs it on its capture loop. func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
func (f *Interface) statsEmitter() func() { ticker := time.NewTicker(i)
defer ticker.Stop()
udpStats := udp.NewUDPStatsEmitter(f.writers) udpStats := udp.NewUDPStatsEmitter(f.writers)
certExpirationGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil) certExpirationGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil)
certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil) certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil)
certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil) certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil)
return func() { emit := func() {
f.firewall.EmitStats() f.firewall.EmitStats()
f.handshakeManager.EmitStats() f.handshakeManager.EmitStats()
udpStats() udpStats()
@@ -675,6 +651,19 @@ func (f *Interface) statsEmitter() func() {
certMaxVersion.Update(int64(certState.v1Cert.Version())) certMaxVersion.Update(int64(certState.v1Cert.Version()))
} }
} }
// Prime gauges so a Prometheus scrape that lands before the first tick
// sees real values instead of the zero defaults (issue #907).
emit()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
emit()
}
}
} }
func (f *Interface) GetHostInfo(vpnIp netip.Addr) *HostInfo { func (f *Interface) GetHostInfo(vpnIp netip.Addr) *HostInfo {
+14 -5
View File
@@ -3,6 +3,7 @@
package nebula package nebula
import ( import (
"context"
"net/netip" "net/netip"
"testing" "testing"
"time" "time"
@@ -17,8 +18,11 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
// Priming before the first tick (issue #907) is Start's job, see TestStatsServer_Start_primes. // Test_emitStats_primesGauges covers issue #907: a Prometheus scrape that
func Test_statsEmitter_setsGauges(t *testing.T) { // landed before the first ticker fire used to read 0 for the cert gauges.
// emitStats now primes the gauges before entering the ticker loop. We assert
// the gauge is zero before the first call and non-zero after.
func Test_emitStats_primesGauges(t *testing.T) {
defer metrics.DefaultRegistry.UnregisterAll() defer metrics.DefaultRegistry.UnregisterAll()
l := test.NewLogger() l := test.NewLogger()
@@ -52,12 +56,17 @@ func Test_statsEmitter_setsGauges(t *testing.T) {
ifce.pki.cs.Store(cs) ifce.pki.cs.Store(cs)
ttlGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil) ttlGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil)
require.Zero(t, ttlGauge.Value(), "gauge should be zero before the emitter runs") require.Zero(t, ttlGauge.Value(), "gauge should be zero before emitStats runs")
ifce.statsEmitter()() // Pre-cancel the context so emitStats returns after priming the gauges
// without ever reading from ticker.C. The one hour interval is just a
// belt-and-suspenders, the test does not expect the ticker to fire.
ctx, cancel := context.WithCancel(context.Background())
cancel()
ifce.emitStats(ctx, time.Hour)
ttl := ttlGauge.Value() ttl := ttlGauge.Value()
assert.Positive(t, ttl, "ttl gauge should be set by one run of the emitter") assert.Positive(t, ttl, "ttl gauge should be primed by emitStats before its first tick")
assert.LessOrEqual(t, ttl, int64(3600)) assert.LessOrEqual(t, ttl, int64(3600))
assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.initiating_version", nil).Value()) assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.initiating_version", nil).Value())
assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.max_version", nil).Value()) assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.max_version", nil).Value())
+4 -26
View File
@@ -65,10 +65,10 @@ func TestReloadFirewall_CertUnsafeNetworksChanged(t *testing.T) {
pki.cs.Store(&CertState{v2Cert: c2, initiatingVersion: cert.Version2}) pki.cs.Store(&CertState{v2Cert: c2, initiatingVersion: cert.Version2})
// Reload with the same YAML so HasChanged("firewall") reports false. // Reload with the same YAML so HasChanged("firewall") reports false.
require.NoError(t, cfg.ReloadConfigString(rawYAML).Err) require.NoError(t, cfg.ReloadConfigString(rawYAML))
require.False(t, cfg.HasChanged("firewall")) require.False(t, cfg.HasChanged("firewall"))
f.reloadFirewall(cfg, nil) f.reloadFirewall(cfg)
assert.NotSame(t, fw, f.firewall, "firewall pointer should have been replaced") assert.NotSame(t, fw, f.firewall, "firewall pointer should have been replaced")
assert.Equal(t, newUnsafe, f.firewall.unsafeNetworks) assert.Equal(t, newUnsafe, f.firewall.unsafeNetworks)
@@ -113,30 +113,8 @@ func TestReloadFirewall_NoChange(t *testing.T) {
l: l, l: l,
} }
require.NoError(t, cfg.ReloadConfigString(rawYAML).Err) require.NoError(t, cfg.ReloadConfigString(rawYAML))
f.reloadFirewall(cfg, nil) f.reloadFirewall(cfg)
assert.Same(t, fw, f.firewall, "firewall should not have been replaced") assert.Same(t, fw, f.firewall, "firewall should not have been replaced")
} }
// Rules that don't parse keep the old firewall, and the reload says it could not apply them
func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
l := test.NewLogger()
pki := &PKI{}
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}}, initiatingVersion: cert.Version2})
rules := func(proto string) string {
return "firewall:\n outbound:\n - port: any\n proto: any\n host: any\n inbound:\n - port: any\n proto: " + proto + "\n host: any\n"
}
cfg := config.NewC(l)
require.NoError(t, cfg.LoadString(rules("any")))
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
require.NoError(t, err)
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
+13
View File
@@ -32,10 +32,23 @@ const (
IPProtocolICMPv6 = 58 IPProtocolICMPv6 = 58
IPProtocolTCP = 6 IPProtocolTCP = 6
IPProtocolUDP = 17 IPProtocolUDP = 17
IPProtocolUDPLite = 136
IPProtocolDCCP = 33
IPProtocolSCTP = 132
ICMPv6TypeEchoRequest = 128 ICMPv6TypeEchoRequest = 128
ICMPv6TypeEchoReply = 129 ICMPv6TypeEchoReply = 129
) )
// HasPorts reports whether packets of proto carry port numbers that the packet parsers read: TCP, UDP, UDPLite,
// DCCP, and SCTP. Firewall rules with a port or a port range only apply to these protocols.
func HasPorts(proto uint8) bool {
switch proto {
case IPProtocolTCP, IPProtocolUDP, IPProtocolUDPLite, IPProtocolDCCP, IPProtocolSCTP:
return true
}
return false
}
func CreateRejectPacket(packet []byte, out []byte) []byte { func CreateRejectPacket(packet []byte, out []byte) []byte {
if len(packet) < 1 { if len(packet) < 1 {
return nil return nil
+9 -38
View File
@@ -15,7 +15,6 @@ import (
"time" "time"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
@@ -82,8 +81,6 @@ type LightHouse struct {
updateTrigger chan struct{} updateTrigger chan struct{}
queryChan chan netip.Addr queryChan chan netip.Addr
// queryDropped counts queries QueryServer dropped because queryChan was full.
queryDropped metrics.Counter
calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote
@@ -119,8 +116,7 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
nebulaPort: nebulaPort, nebulaPort: nebulaPort,
punchy: p, punchy: p,
updateTrigger: make(chan struct{}, 1), updateTrigger: make(chan struct{}, 1),
queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 1024)), queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
queryDropped: metrics.GetOrRegisterCounter("lighthouse.query.dropped", nil),
l: l, l: l,
} }
h.localAddrsFn = func(al *LocalAllowList) []netip.Addr { h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
@@ -138,21 +134,18 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
h.metrics = newLighthouseMetrics() h.metrics = newLighthouseMetrics()
} }
err := h.reload(c, true, nil) err := h.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("lighthouse.am_lighthouse", "listen.port", "handshakes.query_buffer", "stats.lighthouse_metrics") err := h.reload(c, false)
err := h.reload(c, false, r)
switch v := err.(type) { switch v := err.(type) {
case *util.ContextualError: case *util.ContextualError:
v.Log(l) v.Log(l)
r.Failed()
case error: case error:
l.Error("failed to reload lighthouse", "error", err) l.Error("failed to reload lighthouse", "error", err)
r.Failed()
} }
}) })
@@ -193,8 +186,7 @@ func (lh *LightHouse) GetUpdateInterval() int64 {
return lh.interval.Load() return lh.interval.Load()
} }
// r is nil on the initial load func (lh *LightHouse) reload(c *config.C, initial bool) error {
func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error {
if initial || c.HasChanged("lighthouse.advertise_addrs") { if initial || c.HasChanged("lighthouse.advertise_addrs") {
rawAdvAddrs := c.GetStringSlice("lighthouse.advertise_addrs", []string{}) rawAdvAddrs := c.GetStringSlice("lighthouse.advertise_addrs", []string{})
advAddrs := make([]netip.AddrPort, 0) advAddrs := make([]netip.AddrPort, 0)
@@ -229,7 +221,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error
"addr", rawAddr, "addr", rawAddr,
"entry", i+1, "entry", i+1,
) )
r.Failed()
continue continue
} }
@@ -364,13 +355,7 @@ func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error
} }
} }
// am_relay decides whether relay.relays applies to us at all if initial || c.HasChanged("relay.relays") {
if initial || c.HasChanged("relay.relays") || c.HasChanged("relay.am_relay") {
var before []netip.Addr
if !initial {
before = lh.GetRelaysForMe()
}
switch c.GetBool("relay.am_relay", false) { switch c.GetBool("relay.am_relay", false) {
case true: case true:
// Relays aren't allowed to specify other relays // Relays aren't allowed to specify other relays
@@ -388,7 +373,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error
"relay", v, "relay", v,
"error", err, "error", err,
) )
r.Failed()
} else { } else {
lh.l.Info("Read relay from config", "relay", v) lh.l.Info("Read relay from config", "relay", v)
relaysForMe = append(relaysForMe, configRIP) relaysForMe = append(relaysForMe, configRIP)
@@ -396,11 +380,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error
} }
lh.relaysForMe.Store(&relaysForMe) lh.relaysForMe.Store(&relaysForMe)
} }
// Tell the lighthouses now rather than at the next interval, but only when what we advertise changed
if !initial && !slices.Equal(before, lh.GetRelaysForMe()) {
lh.TriggerUpdate()
}
} }
return nil return nil
@@ -533,19 +512,14 @@ func (lh *LightHouse) Query(vpnAddr netip.Addr) *RemoteList {
return nil return nil
} }
// QueryServer is asynchronous and non-blocking, no reply should be expected. // QueryServer is asynchronous so no reply should be expected
// If the query channel is full, the query is dropped.
func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) { func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) {
// Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses // Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses
if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) { if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) {
return return
} }
select { lh.queryChan <- vpnAddr
case lh.queryChan <- vpnAddr:
default:
lh.queryDropped.Inc(1)
}
} }
func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList { func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList {
@@ -1261,13 +1235,10 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []neti
// sendHostPunchNotification signals the other side to punch some zero byte udp packets // sendHostPunchNotification signals the other side to punch some zero byte udp packets
func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, punchNotifDest netip.Addr, w EncWriter) { func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, punchNotifDest netip.Addr, w EncWriter) {
whereToPunch := fromVpnAddrs[0] whereToPunch := fromVpnAddrs[0]
// Resolve the target before taking the querier's RemoteList lock below. GetHostInfo takes the
// hostmap lock, and the hostmap is taken before the lighthouse, which is taken before a
// RemoteList, so looking it up inside the callback would deadlock.
targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest)
found, ln, err := lhh.lh.queryAndPrepMessage(whereToPunch, func(c *cache) (int, error) { found, ln, err := lhh.lh.queryAndPrepMessage(whereToPunch, func(c *cache) (int, error) {
n = lhh.resetMeta() n = lhh.resetMeta()
n.Type = NebulaMeta_HostPunchNotification n.Type = NebulaMeta_HostPunchNotification
targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest)
var useVersion cert.Version var useVersion cert.Version
if targetHI == nil { if targetHI == nil {
useVersion = lhh.lh.ifce.GetCertState().initiatingVersion useVersion = lhh.lh.ifce.GetCertState().initiatingVersion
+10 -257
View File
@@ -6,16 +6,13 @@ import (
"errors" "errors"
"fmt" "fmt"
"net/netip" "net/netip"
"sync"
"testing" "testing"
"time"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.yaml.in/yaml/v3" "go.yaml.in/yaml/v3"
@@ -87,15 +84,15 @@ func TestReloadLighthouseInterval(t *testing.T) {
lh.ifce = &mockEncWriter{} lh.ifce = &mockEncWriter{}
// The first one routine is kicked off by main.go currently, lets make sure that one dies // The first one routine is kicked off by main.go currently, lets make sure that one dies
require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 5").Err) require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 5"))
assert.Equal(t, int64(5), lh.interval.Load()) assert.Equal(t, int64(5), lh.interval.Load())
// Subsequent calls are killed off by the LightHouse.Reload function // Subsequent calls are killed off by the LightHouse.Reload function
require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 10").Err) require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 10"))
assert.Equal(t, int64(10), lh.interval.Load()) assert.Equal(t, int64(10), lh.interval.Load())
// If this completes then nothing is stealing our reload routine // If this completes then nothing is stealing our reload routine
require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 11").Err) require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 11"))
assert.Equal(t, int64(11), lh.interval.Load()) assert.Equal(t, int64(11), lh.interval.Load())
} }
@@ -296,7 +293,7 @@ func TestLighthouse_reload(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
c.ReloadConfigString(string(rc)) c.ReloadConfigString(string(rc))
err = lh.reload(c, false, nil) err = lh.reload(c, false)
require.NoError(t, err) require.NoError(t, err)
} }
@@ -334,7 +331,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err := yaml.Marshal(nc) rc, err := yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
rl := lh.Query(staticHost) rl := lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -350,7 +347,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
rl = lh.Query(staticHost) rl = lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -358,7 +355,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, rl.CopyAddrs([]netip.Prefix{})) assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, rl.CopyAddrs([]netip.Prefix{}))
// Reload with the same config. An unchanged entry must not duplicate. // Reload with the same config. An unchanged entry must not duplicate.
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
rl = lh.Query(staticHost) rl = lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -373,7 +370,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
// Add a second host alongside the first. Both should be present, neither duplicated. // Add a second host alongside the first. Both should be present, neither duplicated.
nc = map[string]any{ nc = map[string]any{
@@ -384,7 +381,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
rl = lh.Query(staticHost) rl = lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -406,7 +403,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
_, isStatic := lh.GetStaticHostList()[staticHost] _, isStatic := lh.GetStaticHostList()[staticHost]
assert.False(t, isStatic) assert.False(t, isStatic)
@@ -798,247 +795,3 @@ func TestLightHouse_logLocalAddrsErr(t *testing.T) {
lh.logLocalAddrsErr(errors.New("something else")) lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN") assert.Contains(t, out.String(), "level=WARN")
} }
// heldEncWriter holds the query worker in its first GetHostInfo until release closes.
type heldEncWriter struct {
mockEncWriter
once sync.Once
held chan struct{}
release chan struct{}
}
func (w *heldEncWriter) GetHostInfo(netip.Addr) *HostInfo {
w.once.Do(func() { close(w.held) })
<-w.release
return nil
}
// QueryServer never waits on the query worker, since the packet readers call it and nothing drains the channel at
// shutdown; a query that finds the channel full is dropped and counted. The worker is held on its first query so the
// channel fills.
func TestLighthouse_QueryServerDoesNotBlock(t *testing.T) {
l := test.NewLogger()
cs := testCertState(netip.MustParsePrefix("10.128.0.1/16"))
lh1 := "10.128.0.2"
c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}}
c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
c.Settings["handshakes"] = map[string]any{"query_buffer": 1}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err)
w := &heldEncWriter{held: make(chan struct{}), release: make(chan struct{})}
lh.ifce = w
t.Cleanup(func() { close(w.release) })
lh.QueryServer(netip.MustParseAddr("10.128.0.10"))
select {
case <-w.held:
case <-time.After(5 * time.Second):
t.Fatal("the query worker never took the first query")
}
dropped := lh.queryDropped.Count()
done := make(chan struct{})
go func() {
lh.QueryServer(netip.MustParseAddr("10.128.0.11"))
lh.QueryServer(netip.MustParseAddr("10.128.0.12"))
close(done)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("QueryServer blocked on a full query channel")
}
assert.EqualValues(t, 1, lh.queryDropped.Count()-dropped)
}
// hostmapEncWriter answers GetHostInfo from a real hostmap, the way Interface does.
type hostmapEncWriter struct {
mockEncWriter
hm *HostMap
}
func (w *hostmapEncWriter) GetHostInfo(a netip.Addr) *HostInfo { return w.hm.QueryVpnAddr(a) }
func (w *hostmapEncWriter) GetCertState() *CertState {
return &CertState{initiatingVersion: cert.Version1}
}
// A lighthouse runs four paths that each hold one lock while waiting for the next: CheckAndComplete
// holds the hostmap and takes the handshake manager, StartHandshake holds the handshake manager and
// takes the lighthouse, a host update holds the lighthouse and takes the sender's RemoteList, and a
// punch notification holds the querier's RemoteList while it resolves the target. If that last step
// reaches the hostmap under the RemoteList lock, the four deadlock. Drive them all at once.
func TestLighthouse_PunchNotificationDoesNotCloseLockCycle(t *testing.T) {
l := test.NewLogger()
myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true}
c.Settings["listen"] = map[string]any{"port": 4242}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(myVpnNet), nil, nil)
require.NoError(t, err)
mainHM := newHostMap(l)
pr := []netip.Prefix{}
mainHM.preferredRanges.Store(&pr)
lh.ifce = &hostmapEncWriter{hm: mainHM}
hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig)
f := &Interface{hostMap: mainHM, lightHouse: lh, handshakeManager: hm, l: l}
hm.f = f
peer := netip.MustParseAddr("10.128.0.2")
peerUdp := netip.MustParseAddrPort("10.0.0.2:4242")
target := netip.MustParseAddr("10.128.0.3")
targetUdp := netip.MustParseAddrPort("10.0.0.3:4242")
outbound := netip.MustParseAddr("10.128.0.4")
inbound := netip.MustParseAddr("10.128.0.5")
seed := lh.NewRequestHandler()
newLHHostUpdate(peerUdp, peer, []netip.AddrPort{peerUdp}, seed)
newLHHostUpdate(targetUdp, target, []netip.AddrPort{targetUdp}, seed)
stop := make(chan struct{})
var wg sync.WaitGroup
loop := func(body func()) {
defer wg.Done()
for {
select {
case <-stop:
return
default:
body()
}
}
}
wg.Add(4)
// Outbound handshakes keep starting and being dropped.
go loop(func() {
hm.DeleteHostInfo(hm.StartHandshake(outbound, nil))
})
// Inbound handshakes keep completing and being torn down.
var idx uint32
go loop(func() {
idx++
hi := &HostInfo{
vpnAddrs: []netip.Addr{inbound},
localIndexId: idx,
remoteIndexId: idx,
remotes: lh.QueryCache([]netip.Addr{inbound}),
HandshakePacket: map[uint8][]byte{handshakePacketStage0: {byte(idx)}},
}
if _, err := hm.CheckAndComplete(hi, handshakePacketStage0, f); err == nil {
mainHM.DeleteHostInfo(hi)
}
})
// The peer keeps sending host updates.
updates := lh.NewRequestHandler()
go loop(func() {
newLHHostUpdate(peerUdp, peer, []netip.AddrPort{peerUdp}, updates)
})
// The same peer keeps querying for the target, which ends in a punch notification.
queries := lh.NewRequestHandler()
go loop(func() {
sendLHHostRequest(peerUdp, peer, target, queries, nil)
})
time.Sleep(time.Second)
close(stop)
done := make(chan struct{})
go func() { wg.Wait(); close(done) }()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("deadlock: the lighthouse paths never finished after stop was signalled")
}
}
func TestLighthouse_reloadRelaysTriggersUpdate(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
base := "lighthouse:\n am_lighthouse: true\nlisten:\n port: 4242\n"
require.NoError(t, c.LoadString(base+"relay:\n relays: [10.128.0.5]\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
assert.Empty(t, lh.updateTrigger)
require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6]\n").Err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
<-lh.updateTrigger
// The config changed but what we advertise did not, a bad entry is skipped
require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6, nope]\n").Err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe())
assert.Empty(t, lh.updateTrigger)
}
func TestLighthouse_reloadAmRelayAlone(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
base := "lighthouse:\n am_lighthouse: true\nlisten:\n port: 4242\nrelay:\n relays: [10.128.0.5]\n"
require.NoError(t, c.LoadString(base+" am_relay: false\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe())
// Relays aren't allowed to have relays, flipping am_relay alone must drop them and tell the lighthouses
require.NoError(t, c.ReloadConfigString(base+" am_relay: true\n").Err)
assert.Empty(t, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
<-lh.updateTrigger
require.NoError(t, c.ReloadConfigString(base+" am_relay: false\n").Err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
}
func TestLighthouse_reloadRestartRequired(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
base := "lighthouse:\n am_lighthouse: true\n"
require.NoError(t, c.LoadString(base+"listen:\n port: 4242\n"))
_, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
res := c.ReloadConfigString(base + "listen:\n port: 4242\n")
require.NoError(t, res.Err)
assert.False(t, res.RestartRequired)
res = c.ReloadConfigString(base + "listen:\n port: 4243\n")
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
}
// A relay that doesn't parse is skipped and the rest are used, the reload says one was skipped
func TestLighthouse_reloadBadRelayFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("listen:\n port: 4242\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
res := c.ReloadConfigString("listen:\n port: 4242\nrelay:\n relays: [10.128.0.2, nope]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.2")}, lh.GetRelaysForMe())
}
// An advertise_addrs entry inside our own network is skipped and the rest are used, the reload says one was skipped
func TestLighthouse_reloadAdvertiseAddrInNetworkFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("listen:\n port: 4242\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
res := c.ReloadConfigString("listen:\n port: 4242\nlighthouse:\n advertise_addrs: [\"1.2.3.4:4242\", \"10.128.0.5:4242\"]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.2.3.4:4242")}, lh.GetAdvertiseAddrs())
}
-31
View File
@@ -1,31 +0,0 @@
package logging
import "log/slog"
// RateLimiter is what RateLimited needs from a rate limiter, such as a *ratelimit.Limiter.
// Allow reports whether this call may proceed, and how many calls were denied since the last one that did.
type RateLimiter interface {
Allow() (denied uint64, ok bool)
}
// RateLimited checks lim before anything is built.
// When allowed it returns l, with a "suppressed" count attached if lim denied calls since its last allowed one.
// A nil lim allows everything.
//
// if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok {
// hostinfo.logger(l).Error("Failed to write outgoing packet", "error", err)
// }
func RateLimited(l *slog.Logger, lim RateLimiter) (*slog.Logger, bool) {
if lim == nil {
return l, true
}
denied, ok := lim.Allow()
if !ok {
return nil, false
}
if denied > 0 {
return l.With("suppressed", denied), true
}
return l, true
}
-106
View File
@@ -1,106 +0,0 @@
package logging
import (
"bytes"
"errors"
"io"
"log/slog"
"net/netip"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/ratelimit"
)
type fakeLimiter struct {
denied uint64
ok bool
}
func (f fakeLimiter) Allow() (uint64, bool) { return f.denied, f.ok }
func TestRateLimited(t *testing.T) {
var buf bytes.Buffer
base := NewLogger(&buf)
if l, ok := RateLimited(base, fakeLimiter{ok: false}); ok || l != nil {
t.Fatalf("denied call: ok=%v logger=%v", ok, l)
}
if l, ok := RateLimited(base, fakeLimiter{ok: true}); !ok || l != base {
t.Fatal("nothing denied should hand back the same logger")
}
if l, ok := RateLimited(base, nil); !ok || l != base {
t.Fatal("a nil limiter should allow with the same logger")
}
l, ok := RateLimited(base, fakeLimiter{denied: 7, ok: true})
if !ok {
t.Fatal("allowed call denied")
}
l.Info("hello", "k", "v")
if !strings.Contains(buf.String(), "suppressed=7") {
t.Fatalf("want suppressed=7, got %q", buf.String())
}
}
// The benchmarks compare a log line through slog directly against the same line behind RateLimited: logged with
// nothing denied, denied, and logged after denials.
var (
benchErr = errors.New("sendto: no buffer space available")
benchAddr = netip.MustParseAddrPort("192.0.2.1:4242")
)
func benchLogger() *slog.Logger {
return NewLogger(io.Discard).With("localIndex", 1234, "remoteIndex", 5678)
}
func BenchmarkRateLimitedBaseline(b *testing.B) {
lg := benchLogger()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
lg.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
func BenchmarkRateLimitedAllowed(b *testing.B) {
lg, lim := benchLogger(), mustLimiter(b, 1, 0)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func BenchmarkRateLimitedDenied(b *testing.B) {
lg, lim := benchLogger(), mustLimiter(b, 1, time.Hour)
lim.Allow()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func BenchmarkRateLimitedAfterDenied(b *testing.B) {
lg, lim := benchLogger(), fakeLimiter{denied: 5, ok: true}
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func mustLimiter(b *testing.B, burst int, every time.Duration) *ratelimit.Limiter {
l, err := ratelimit.New(burst, every)
if err != nil {
b.Fatal(err)
}
return l
}
+5 -17
View File
@@ -78,7 +78,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
wireSSHReload(l, ssh, c) wireSSHReload(l, ssh, c)
var sshStart func() var sshStart func()
if c.GetBool("sshd.enabled", false) { if c.GetBool("sshd.enabled", false) {
sshStart, err = configSSH(l, ssh, c, nil) sshStart, err = configSSH(l, ssh, c)
if err != nil { if err != nil {
l.Warn("Failed to configure sshd, ssh debugging will not be available", "error", err) l.Warn("Failed to configure sshd, ssh debugging will not be available", "error", err)
sshStart = nil sshStart = nil
@@ -197,7 +197,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
if err != nil { if err != nil {
return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err) return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err)
} }
udpServer.ReloadConfig(c, nil) udpServer.ReloadConfig(c)
udpConns[i] = udpServer udpConns[i] = udpServer
// If port is dynamic, discover it before the next pass through the for loop // If port is dynamic, discover it before the next pass through the for loop
@@ -300,7 +300,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
} }
var ifce *Interface var ifce *Interface
var emitStats func()
if !configTest { if !configTest {
ifce, err = NewInterface(ctx, ifConfig) ifce, err = NewInterface(ctx, ifConfig)
if err != nil { if err != nil {
@@ -311,16 +310,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
lightHouse.ifce = ifce lightHouse.ifce = ifce
ifce.RegisterConfigChangeCallbacks(c) ifce.RegisterConfigChangeCallbacks(c)
// These are only read at startup
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged(
"routines", "tun.routines", "listen.routines", "firewall.conntrack.routine_cache_timeout",
"listen.port", "listen.host", "stats.message_metrics",
"handshakes.try_interval", "handshakes.retries", "handshakes.trigger_buffer",
"tun.drop_local_broadcast", "tun.drop_multicast",
)
r.RestartRequiredIfChanged(osStartupOnlyKeys...)
})
ifce.reloadDisconnectInvalid(c) ifce.reloadDisconnectInvalid(c)
ifce.reloadSendRecvError(c) ifce.reloadSendRecvError(c)
ifce.reloadAcceptRecvError(c) ifce.reloadAcceptRecvError(c)
@@ -329,12 +318,9 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
go handshakeManager.Run(ctx) go handshakeManager.Run(ctx)
punchy.Start(ctx, ifce, hostMap, lightHouse) punchy.Start(ctx, ifce, hostMap, lightHouse)
// On the stats server's loop, so the gauges follow the interval it accepted
emitStats = ifce.statsEmitter()
} }
stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest, emitStats) stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest)
if err != nil { if err != nil {
return nil, util.ContextualizeIfNeeded("Failed to start stats emitter", err) return nil, util.ContextualizeIfNeeded("Failed to start stats emitter", err)
} }
@@ -343,6 +329,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
return nil, nil return nil, nil
} }
go ifce.emitStats(ctx, c.GetDuration("stats.interval", time.Second*10))
attachCommands(l, c, ssh, ifce) attachCommands(l, c, ssh, ifce)
networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c) networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
-6
View File
@@ -1,6 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
// osStartupOnlyKeys are startup only keys that only matter on linux: udp batch size, udp offloads and thread pinning
var osStartupOnlyKeys = []string{"listen.batch", "listen.udp_offloads", "tun.pin_threads", "tun.pin_threads_key", "tun.cpu_affinity"}
-6
View File
@@ -1,6 +0,0 @@
//go:build !linux || android || e2e_testing
package nebula
// osStartupOnlyKeys is empty, none of the OS specific startup keys matter here
var osStartupOnlyKeys []string
+39 -72
View File
@@ -25,7 +25,7 @@ var ErrOutOfWindow = errors.New("out of window packet")
// readOutsidePackets processes one received underlay packet. // readOutsidePackets processes one received underlay packet.
// Message payloads are decrypted IN PLACE, so packet must stay untouched // Message payloads are decrypted IN PLACE, so packet must stay untouched
// by the caller until the TUN and relay batches for queue q have been flushed. // by the caller until the batcher for queue q has been flushed
func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) { func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) {
h := rxc.h h := rxc.h
err := h.Parse(packet) err := h.Parse(packet)
@@ -77,12 +77,6 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
f.messageMetrics.Rx(h.Type, h.Subtype, 1) f.messageMetrics.Rx(h.Type, h.Subtype, 1)
} }
// Earlier forwards must leave before this packet can generate a response.
isMessageRelay := h.Type == header.Message && h.Subtype == header.MessageRelay
if !isMessageRelay {
f.flushRelayBatch(rxc)
}
// Unencrypted packets // Unencrypted packets
switch h.Type { switch h.Type {
case header.Handshake: case header.Handshake:
@@ -94,6 +88,9 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
return return
} }
// Relay packets are special
isMessageRelay := (h.Type == header.Message && h.Subtype == header.MessageRelay)
var hostinfo *HostInfo var hostinfo *HostInfo
if isMessageRelay { if isMessageRelay {
hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex) hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex)
@@ -105,8 +102,6 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
// recvError if necessary // recvError if necessary
if hostinfo == nil || hostinfo.ConnectionState == nil { if hostinfo == nil || hostinfo.ConnectionState == nil {
if !via.IsRelayed { if !via.IsRelayed {
// An unknown relay index can also produce a response.
f.flushRelayBatch(rxc)
f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex) f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex)
} }
return return
@@ -165,7 +160,9 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
case header.TestReply: case header.TestReply:
// No-op, useful for the Roaming and connectionManager side-effects above // No-op, useful for the Roaming and connectionManager side-effects above
case header.TestRequest: case header.TestRequest:
if header.MaxOverhead+len(out) > len(rxc.scratch) { const maxCipherOverhead = 16 //todo we use this too often, needs a real importable const
const maxOverhead = header.Len + header.Len + maxCipherOverhead + maxCipherOverhead
if maxOverhead+len(out) > len(rxc.scratch) {
// A reply that cannot fit in scratch is dropped no matter the log level. // A reply that cannot fit in scratch is dropped no matter the log level.
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("dropping oversized test request", "payloadLen", len(out), "from", via) hostinfo.logger(f.l).Debug("dropping oversized test request", "payloadLen", len(out), "from", via)
@@ -238,10 +235,8 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
// Forward this packet through the relay tunnel, rebuilding it in place. // Forward this packet through the relay tunnel, rebuilding it in place.
// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was // Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
fwdBuf := packet[:0] fwdBuf := packet[:0]
out, err := f.prepareSendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true) //todo it would potentially be nice to batch these
if err == nil { f.SendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true, rxc.q)
f.queueRelayPacket(out, targetHI.GetRemote(), rxc)
}
case TerminalType: case TerminalType:
hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal") hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
return return
@@ -266,39 +261,13 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
} }
} }
// queueRelayPacket borrows a receive buffer until the batch is flushed. Destination changes and the send batch
// limit end a run, preserving arrival order while allowing WriteBatch to use UDP segmentation offload.
func (f *Interface) queueRelayPacket(packet []byte, remote netip.AddrPort, rxc *rxContext) {
if rxc.relayBatch == nil {
rxc.relayBatch = batch.NewSendBatch(f.writers[rxc.q], batch.SendBatchCap, 0)
} else if rxc.relayRemote != remote {
f.flushRelayBatch(rxc)
}
rxc.relayRemote = remote
rxc.relayBatch.Commit(packet, remote)
if rxc.relayBatch.Len() >= batch.SendBatchCap {
f.flushRelayBatch(rxc)
}
}
func (f *Interface) flushRelayBatch(rxc *rxContext) {
if rxc.relayBatch != nil {
f.flushSendBatch(rxc.relayBatch, rxc.q)
}
}
// closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
func (f *Interface) closeTunnel(hostInfo *HostInfo) { func (f *Interface) closeTunnel(hostInfo *HostInfo) {
// Hold the hostmap lock across both the delete and the lighthouse eviction, final := f.hostMap.DeleteHostInfo(hostInfo)
// to avoid a race with handshake completion. if final {
f.hostMap.Lock() // We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage
defer f.hostMap.Unlock() f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs)
if !f.hostMap.unlockedDeleteHostInfo(hostInfo) {
return
} }
// We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage
f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs)
} }
// sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote // sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote
@@ -392,31 +361,29 @@ func parseV6(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
fp.Fragment = isFragment fp.Fragment = isFragment
fp.FragAny = anyFragment fp.FragAny = anyFragment
fp.IPHdrLen = offset fp.IPHdrLen = offset
// fp is reused, so the ports are cleared for the packets that have none: fragments, protocols Nebula
// doesn't inspect, and ICMPv6 other than echo. Only the cases below set them.
fp.RemotePort, fp.LocalPort = 0, 0
if isFragment { if isFragment {
// Non-first fragments carry no transport header, so we have no ports to read // Non-first fragments carry no transport header, so we have no ports to read
fp.RemotePort = 0
fp.LocalPort = 0
return nil return nil
} }
switch proto { switch {
case iputil.IPProtocolICMPv6: case proto == iputil.IPProtocolICMPv6:
// An ICMPv6 message is at least type, code and checksum, 4 bytes. Only echo carries more than we read. // An ICMPv6 message is at least type, code and checksum, 4 bytes. Only echo carries more than we read.
if dataLen < offset+4 { if dataLen < offset+4 {
return ErrIPv6PacketTooShort return ErrIPv6PacketTooShort
} }
fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6
switch data[offset] { //icmp type switch data[offset] { //icmp type
case iputil.ICMPv6TypeEchoRequest, iputil.ICMPv6TypeEchoReply: case iputil.ICMPv6TypeEchoRequest, iputil.ICMPv6TypeEchoReply:
if dataLen < offset+6 { if dataLen < offset+6 {
return ErrIPv6PacketTooShort return ErrIPv6PacketTooShort
} }
fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier, orientation doesn't matter
default:
fp.RemotePort = 0
} }
case iputil.IPProtocolTCP, iputil.IPProtocolUDP: case iputil.HasPorts(proto):
if dataLen < offset+4 { if dataLen < offset+4 {
return ErrIPv6PacketTooShort return ErrIPv6PacketTooShort
} }
@@ -427,11 +394,6 @@ func parseV6(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2]) fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4]) fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
} }
default:
// don't set ports for protocols Nebula doesn't inspect
fp.RemotePort = 0
fp.LocalPort = 0
} }
return nil return nil
@@ -484,18 +446,25 @@ func parseV4(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
fp.RemoteAddr, _ = netip.AddrFromSlice(data[16:20]) fp.RemoteAddr, _ = netip.AddrFromSlice(data[16:20])
} }
// fp is reused, so the ports are cleared for the packets that have none: fragments, and protocols Nebula
// doesn't inspect, whose first 4 bytes (GRE, IPIP, AH, etc.) are not ports. Only the cases below set them.
fp.RemotePort, fp.LocalPort = 0, 0
if fp.Fragment { if fp.Fragment {
fp.RemotePort = 0 return nil
fp.LocalPort = 0 }
} else if fp.Protocol == iputil.IPProtocolICMP { //note that orientation doesn't matter on ICMP
fp.RemotePort = binary.BigEndian.Uint16(data[ihl+4 : ihl+6]) //identifier switch {
fp.LocalPort = 0 //code would be uint16(data[ihl+1]) case fp.Protocol == iputil.IPProtocolICMP: //note that orientation doesn't matter on ICMP
} else if incoming { fp.RemotePort = binary.BigEndian.Uint16(data[ihl+4 : ihl+6]) //identifier, LocalPort stays 0 but could be the code, data[ihl+1]
fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2]) //src port
fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port case iputil.HasPorts(fp.Protocol):
} else { if incoming {
fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2]) //src port fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2]) //src port
fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
} else {
fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2]) //src port
fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
}
} }
return nil return nil
@@ -504,9 +473,7 @@ func parseV4(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounter uint64, out []byte, rxc *rxContext) { func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounter uint64, out []byte, rxc *rxContext) {
err := newPacket(out, true, rxc.fwPacket) err := newPacket(out, true, rxc.fwPacket)
if err != nil { if err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) { hostinfo.logger(f.l).Warn("Error while validating inbound packet", "error", err, "packet", out)
hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out)
}
return return
} }
-291
View File
@@ -1,291 +0,0 @@
package nebula
import (
"bytes"
"net/netip"
"sync"
"testing"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/require"
)
type relayBatchWrite struct {
packets [][]byte
remotes []netip.AddrPort
batched bool
}
type relayBatchConn struct {
udp.NoopConn
read func(udp.EncReader, func())
writes []relayBatchWrite
}
func (c *relayBatchConn) ListenOut(r udp.EncReader, flush func()) error {
c.read(r, flush)
return nil
}
func (c *relayBatchConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
w := relayBatchWrite{remotes: append([]netip.AddrPort(nil), addrs...), batched: true}
for _, b := range bufs {
w.packets = append(w.packets, bytes.Clone(b))
}
c.writes = append(c.writes, w)
return len(bufs), nil
}
func (c *relayBatchConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.writes = append(c.writes, relayBatchWrite{packets: [][]byte{bytes.Clone(b)}, remotes: []netip.AddrPort{addr}})
return nil
}
func (c *relayBatchConn) packets() [][]byte {
var packets [][]byte
for _, w := range c.writes {
packets = append(packets, w.packets...)
}
return packets
}
type relayBatchPath struct {
f *Interface
conn *relayBatchConn
source, target *HostInfo
incoming, outgoing *Relay
sourceKey noiseutil.CipherState
cipher noise.CipherFunc
counter uint64
}
// Separate cipher instances let the reference SendVia path use the same keys and
// counters without rewinding an AEAD instance (including FIPS nonce checks).
func relayBatchKey(cipher noise.CipherFunc, id byte) noiseutil.CipherState {
suite := noise.NewCipherSuite(noise.DH25519, cipher, noise.HashSHA256)
return noiseutil.NewCipherState(noise.UnsafeNewCipherState(suite, [32]byte{id}, 0), cipher)
}
func newRelayBatchPath(t *testing.T, cipher noise.CipherFunc) *relayBatchPath {
t.Helper()
l := test.NewLogger()
conn := &relayBatchConn{}
f := &Interface{
ctx: t.Context(), l: l, hostMap: newHostMap(l), outside: conn,
writers: []udp.Conn{conn}, lightHouse: &LightHouse{l: l},
myVpnNetworksTable: &bart.Lite{}, metricTxDropped: metrics.NewCounter(),
batchers: []*batch.MultiCoalescer{batch.NewMultiCoalescer(&capturingTun{}, l)},
}
f.connectionManager = &connectionManager{
intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{},
}
newHost := func(addr string, remote string, id byte) *HostInfo {
h := &HostInfo{
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
ConnectionState: &ConnectionState{
eKey: relayBatchKey(cipher, id), dKey: relayBatchKey(cipher, id), window: NewBits(ReplayWindow),
},
relayState: RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}},
}
u := netip.MustParseAddrPort(remote)
h.remote.Store(&u)
f.hostMap.Hosts[h.vpnAddrs[0]] = h
return h
}
source := newHost("192.0.2.1", "198.51.100.1:4242", 1)
target := newHost("192.0.2.2", "198.51.100.2:4242", 2)
incoming := &Relay{Type: ForwardingType, State: Established, LocalIndex: 301, RemoteIndex: 302, PeerAddr: target.vpnAddrs[0]}
outgoing := &Relay{Type: ForwardingType, State: Established, LocalIndex: 303, RemoteIndex: 304, PeerAddr: source.vpnAddrs[0]}
source.relayState.InsertRelay(incoming.PeerAddr, incoming.LocalIndex, incoming)
target.relayState.InsertRelay(outgoing.PeerAddr, outgoing.LocalIndex, outgoing)
f.hostMap.Relays[incoming.LocalIndex] = source
f.hostMap.Indexes[100] = source
return &relayBatchPath{f: f, conn: conn, source: source, target: target, incoming: incoming, outgoing: outgoing, sourceKey: relayBatchKey(cipher, 1), cipher: cipher}
}
func (p *relayBatchPath) wrap(t *testing.T, payload []byte) []byte {
t.Helper()
p.counter++
b := header.Encode(make([]byte, header.Len), header.Version, header.Message, header.MessageRelay, p.incoming.LocalIndex, p.counter)
b = append(b, payload...)
b, err := p.sourceKey.EncryptDanger(b, b, nil, p.counter, make([]byte, 12))
require.NoError(t, err)
return b
}
func (p *relayBatchPath) reference(payloads [][]byte) [][]byte {
conn := &relayBatchConn{}
f := &Interface{writers: []udp.Conn{conn}, connectionManager: p.f.connectionManager, l: p.f.l}
h := &HostInfo{ConnectionState: &ConnectionState{eKey: relayBatchKey(p.cipher, 2)}}
u := p.target.GetRemote()
h.remote.Store(&u)
for _, payload := range payloads {
f.SendVia(h, p.outgoing, payload, make([]byte, 12), make([]byte, 0, len(payload)+header.MaxOverhead), false, 0)
}
return conn.packets()
}
func TestRelayReceiveBatch(t *testing.T) {
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
t.Run(cipher.CipherName(), func(t *testing.T) {
p := newRelayBatchPath(t, cipher)
const count = 8
var payloads, wire [][]byte
for i := range count * 2 {
payload := bytes.Repeat([]byte{byte(i)}, 64)
payloads = append(payloads, payload)
wire = append(wire, p.wrap(t, payload))
}
want := p.reference(payloads)
// Model a listener reusing the same slots for the next receive batch.
buffers := make([][]byte, count)
for i := range buffers {
buffers[i] = make([]byte, len(wire[i]))
}
p.conn.read = func(r udp.EncReader, flush func()) {
for b := range 2 {
for i := range count {
copy(buffers[i], wire[b*count+i])
r(p.source.GetRemote(), buffers[i])
require.Len(t, p.conn.writes, b, "forwarding waits for the batch boundary")
}
flush()
require.Len(t, p.conn.writes, b+1, "flush must send before receive slots are reused")
for _, buffer := range buffers {
clear(buffer)
}
}
}
p.f.listenOut(0)
require.Equal(t, want, p.conn.packets())
for _, w := range p.conn.writes {
require.True(t, w.batched)
require.Len(t, w.packets, count)
for _, remote := range w.remotes {
require.Equal(t, p.target.GetRemote(), remote)
}
}
})
}
}
func TestRelayBatchDestinationChange(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, second := p.target.GetRemote(), netip.MustParseAddrPort("198.51.100.3:4242")
remotes := []netip.AddrPort{first, first, second, second, first}
var payloads, wire [][]byte
for i := range remotes {
payload := []byte{byte(i)}
payloads = append(payloads, payload)
wire = append(wire, p.wrap(t, payload))
}
want := p.reference(payloads)
p.conn.read = func(r udp.EncReader, flush func()) {
for i, remote := range remotes {
p.target.remote.Store(&remote)
r(p.source.GetRemote(), wire[i])
}
flush()
}
p.f.listenOut(0)
require.Equal(t, want, p.conn.packets())
require.Len(t, p.conn.writes, 3)
for i, wantRemotes := range [][]netip.AddrPort{{first, first}, {second, second}, {first}} {
require.Equal(t, wantRemotes, p.conn.writes[i].remotes)
}
}
func TestRelayBatchCapacity(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
p.conn.read = func(r udp.EncReader, flush func()) {
for i := range batch.SendBatchCap + 1 {
r(p.source.GetRemote(), p.wrap(t, []byte{byte(i)}))
}
require.Len(t, p.conn.writes, 1)
require.Len(t, p.conn.writes[0].packets, batch.SendBatchCap)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 2)
require.Len(t, p.conn.writes[1].packets, 1)
}
func TestRelayBatchBeforeResponse(t *testing.T) {
for _, unknownRelay := range []bool{false, true} {
name := "test request"
if unknownRelay {
name = "unknown relay"
}
t.Run(name, func(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, last := p.wrap(t, []byte("first")), p.wrap(t, []byte("last"))
p.counter++
request := header.Encode(make([]byte, header.Len), header.Version, header.Test, header.TestRequest, 100, p.counter)
request, err := p.sourceKey.EncryptDanger(request, request, []byte("ping"), p.counter, make([]byte, 12))
require.NoError(t, err)
if unknownRelay {
request = header.Encode(make([]byte, header.Len), header.Version, header.Message, header.MessageRelay, 999, p.counter)
}
p.conn.read = func(r udp.EncReader, flush func()) {
r(p.source.GetRemote(), first)
require.Empty(t, p.conn.writes)
r(p.source.GetRemote(), request)
require.Len(t, p.conn.writes, 2)
r(p.source.GetRemote(), last)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 3)
require.True(t, p.conn.writes[0].batched)
require.False(t, p.conn.writes[1].batched)
require.True(t, p.conn.writes[2].batched)
require.Equal(t, p.source.GetRemote(), p.conn.writes[1].remotes[0])
h := &header.H{}
require.NoError(t, h.Parse(p.conn.writes[1].packets[0]))
if unknownRelay {
require.Equal(t, header.RecvError, h.Type)
require.EqualValues(t, 999, h.RemoteIndex)
} else {
require.Equal(t, header.Test, h.Type)
require.Equal(t, header.TestReply, h.Subtype)
reply := p.conn.writes[1].packets[0]
out, err := p.sourceKey.DecryptDanger(nil, reply[:header.Len], reply[header.Len:], h.MessageCounter, make([]byte, 12))
require.NoError(t, err)
require.Equal(t, []byte("ping"), out)
}
})
}
}
func TestRelayBatchRemovedTarget(t *testing.T) {
for _, remove := range []string{"host", "relay", "disestablished"} {
t.Run(remove, func(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, removed := p.wrap(t, []byte("first")), p.wrap(t, []byte("removed"))
want := p.reference([][]byte{[]byte("first")})
p.conn.read = func(r udp.EncReader, flush func()) {
r(p.source.GetRemote(), first)
switch remove {
case "host":
delete(p.f.hostMap.Hosts, p.target.vpnAddrs[0])
case "relay":
delete(p.target.relayState.relayForByAddr, p.source.vpnAddrs[0])
case "disestablished":
p.target.relayState.UpdateRelayForByIpState(p.source.vpnAddrs[0], Disestablished)
}
r(p.source.GetRemote(), removed)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 1)
require.Equal(t, want, p.conn.packets(), "only the packet accepted before removal is sent")
})
}
}
+86 -21
View File
@@ -77,7 +77,7 @@ func Test_newPacket(t *testing.T) {
// account for variable ip header length - outgoing // account for variable ip header length - outgoing
h = ipv4.Header{ h = ipv4.Header{
Version: 1, Version: 1,
Protocol: 2, Protocol: iputil.IPProtocolUDP,
Len: 100, Len: 100,
Src: net.IPv4(10, 0, 0, 1), Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2), Dst: net.IPv4(10, 0, 0, 2),
@@ -89,7 +89,7 @@ func Test_newPacket(t *testing.T) {
err = newPacket(b, false, p) err = newPacket(b, false, p)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, uint8(2), p.Protocol) assert.Equal(t, uint8(iputil.IPProtocolUDP), p.Protocol)
assert.Equal(t, netip.MustParseAddr("10.0.0.1"), p.LocalAddr) assert.Equal(t, netip.MustParseAddr("10.0.0.1"), p.LocalAddr)
assert.Equal(t, netip.MustParseAddr("10.0.0.2"), p.RemoteAddr) assert.Equal(t, netip.MustParseAddr("10.0.0.2"), p.RemoteAddr)
assert.Equal(t, uint16(6), p.RemotePort) assert.Equal(t, uint16(6), p.RemotePort)
@@ -97,6 +97,71 @@ func Test_newPacket(t *testing.T) {
assert.False(t, p.Fragment) assert.False(t, p.Fragment)
} }
// Test_newPacket_v4NoPorts ensures only protocols that lead with src/dst ports report them. The first 4 bytes of other protocols are not ports,
// and are often chosen by the sender: a GRE header's protocol type, an IPIP inner total length, a 6in4 flow label.
// Reading them as ports would let a `proto: any` rule with a port match tunneled traffic.
func Test_newPacket_v4NoPorts(t *testing.T) {
p := &firewall.ParsedPacket{}
for _, proto := range []uint8{4, 41, 47, 50, 51} { // IPIP, 6in4, GRE, ESP, AH
h := ipv4.Header{
Version: 4,
Len: ipv4.HeaderLen,
Protocol: int(proto),
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
}
b, err := h.Marshal()
require.NoError(t, err)
b = append(b, []byte{0x00, 0x00, 0x86, 0xdd}...) // a GRE header carrying IPv6 would read as port 34525
for _, incoming := range []bool{true, false} {
require.NoError(t, newPacket(b, incoming, p))
assert.Equal(t, proto, p.Protocol)
assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment)
}
}
}
// Test_newPacket_portedProtocols ensures UDP-Lite, DCCP, and SCTP report ports from the same leading
// src/dst port words as TCP and UDP, oriented the same way.
func Test_newPacket_portedProtocols(t *testing.T) {
p := &firewall.ParsedPacket{}
ports := []byte{0x13, 0x88, 0x00, 0x50} // src 5000, dst 80
for _, proto := range []uint8{iputil.IPProtocolUDPLite, iputil.IPProtocolDCCP, iputil.IPProtocolSCTP} {
h := ipv4.Header{
Version: 4,
Len: ipv4.HeaderLen,
Protocol: int(proto),
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
}
v4, err := h.Marshal()
require.NoError(t, err)
v4 = append(v4, ports...)
v6 := make([]byte, 40, 44)
v6[0] = 0x60
v6[6] = proto
v6[7] = 64
v6 = append(v6, ports...)
for _, b := range [][]byte{v4, v6} {
require.NoError(t, newPacket(b, true, p))
assert.Equal(t, proto, p.Protocol)
assert.Equal(t, uint16(5000), p.RemotePort)
assert.Equal(t, uint16(80), p.LocalPort)
require.NoError(t, newPacket(b, false, p))
assert.Equal(t, uint16(5000), p.LocalPort)
assert.Equal(t, uint16(80), p.RemotePort)
}
}
}
func Test_newPacket_v6(t *testing.T) { func Test_newPacket_v6(t *testing.T) {
p := &firewall.ParsedPacket{} p := &firewall.ParsedPacket{}
@@ -712,46 +777,46 @@ func Test_newPacket_v6ExtHeaderPastBuffer(t *testing.T) {
// Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized // Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized
// Next Header as if it were an ipv6 extension header. A real upper layer protocol Nebula doesn't // Next Header as if it were an ipv6 extension header. A real upper layer protocol Nebula doesn't
// dissect (SCTP here) is not walkable, so applying the (len+1)*8 formula marched into the SCTP // dissect (GRE here) is not walkable, so applying the (len+1)*8 formula marched into the GRE
// payload and landed on a byte that looked like UDP, forging a protocol/port pair the firewall // payload and landed on a byte that looked like UDP, forging a protocol/port pair the firewall
// would trust while the host delivered the real SCTP datagram. The fix fails closed: the packet // would trust while the host delivered the real GRE packet. The fix fails closed: the packet
// is classified as its true protocol with no ports, so it only matches an `any` rule. // is classified as its true protocol with no ports, so it only matches an `any` rule.
func Test_newPacket_v6ExtHeaderConfusion(t *testing.T) { func Test_newPacket_v6ExtHeaderConfusion(t *testing.T) {
p := &firewall.ParsedPacket{} p := &firewall.ParsedPacket{}
pkt := make([]byte, 52) pkt := make([]byte, 52)
pkt[0] = 0x60 // version 6 pkt[0] = 0x60 // version 6
pkt[6] = byte(layers.IPProtocolSCTP) // NextHeader = SCTP, a real protocol, not an extension header pkt[6] = byte(layers.IPProtocolGRE) // NextHeader = GRE, a real protocol, not an extension header
pkt[7] = 64 // hop limit pkt[7] = 64 // hop limit
// Real SCTP header at offset 40. Pre-fix parseV6 walked SCTP as an extension header: byte 41 (0x00, the // Real GRE header at offset 40. Pre-fix parseV6 walked GRE as an extension header: byte 41 (0x00, the
// low byte of the src port below) was read as the header length, giving next=(0+1)*8=8, which landed the // low byte of the flags/version word below) was read as the header length, giving next=(0+1)*8=8, which
// walk on byte 40 (0x11), misread as NextHeader=UDP, then bytes 48-51 as ports. // landed the walk on byte 40 (0x11), misread as NextHeader=UDP, then bytes 48-51 as ports.
binary.BigEndian.PutUint16(pkt[40:42], 0x1100) // SCTP src port; byte 40=0x11, byte 41=0x00 binary.BigEndian.PutUint16(pkt[40:42], 0x1100) // GRE flags/version; byte 40=0x11, byte 41=0x00
binary.BigEndian.PutUint16(pkt[42:44], 445) // SCTP dst port, never read by parseV6 binary.BigEndian.PutUint16(pkt[42:44], 0x86dd) // GRE protocol type, never read by parseV6
binary.BigEndian.PutUint16(pkt[48:50], 53) // SCTP checksum bytes, pre-fix forged RemotePort binary.BigEndian.PutUint16(pkt[48:50], 53) // GRE payload, pre-fix forged RemotePort
binary.BigEndian.PutUint16(pkt[50:52], 53) // pre-fix forged LocalPort binary.BigEndian.PutUint16(pkt[50:52], 53) // pre-fix forged LocalPort
require.NoError(t, newPacket(pkt, true, p)) require.NoError(t, newPacket(pkt, true, p))
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must classify as the true protocol, not the forged UDP") assert.Equal(t, uint8(layers.IPProtocolGRE), p.Protocol, "must classify as the true protocol, not the forged UDP")
assert.Equal(t, uint16(0), p.RemotePort) assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort) assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment) assert.False(t, p.Fragment)
// Same confusion, but the unknown protocol sits after a real extension header. The HopByHop is walked // Same confusion, but the unknown protocol sits after a real extension header. The HopByHop is walked
// correctly, then SCTP must still fail closed instead of being walked into its own payload. Protocol is // correctly, then GRE must still fail closed instead of being walked into its own payload. Protocol is
// the only assertion that discriminates the fix here, a regression that walked SCTP would misclassify it. // the only assertion that discriminates the fix here, a regression that walked GRE would misclassify it.
chained := make([]byte, 60) chained := make([]byte, 60)
chained[0] = 0x60 // version 6 chained[0] = 0x60 // version 6
chained[6] = byte(layers.IPProtocolIPv6HopByHop) // NextHeader = HopByHop extension chained[6] = byte(layers.IPProtocolIPv6HopByHop) // NextHeader = HopByHop extension
chained[7] = 64 // hop limit chained[7] = 64 // hop limit
chained[40] = byte(layers.IPProtocolSCTP) // HopByHop NextHeader = SCTP chained[40] = byte(layers.IPProtocolGRE) // HopByHop NextHeader = GRE
chained[41] = 0 // HopByHop length 0 -> 8 bytes, SCTP begins at offset 48 chained[41] = 0 // HopByHop length 0 -> 8 bytes, GRE begins at offset 48
binary.BigEndian.PutUint16(chained[48:50], 0x1100) // SCTP src port, pre-fix forged NextHeader/length bait binary.BigEndian.PutUint16(chained[48:50], 0x1100) // GRE flags/version, pre-fix forged NextHeader/length bait
binary.BigEndian.PutUint16(chained[50:52], 445) // SCTP dst port, never read by parseV6 binary.BigEndian.PutUint16(chained[50:52], 0x86dd) // GRE protocol type, never read by parseV6
require.NoError(t, newPacket(chained, true, p)) require.NoError(t, newPacket(chained, true, p))
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must fail closed on the unknown protocol after the extension header") assert.Equal(t, uint8(layers.IPProtocolGRE), p.Protocol, "must fail closed on the unknown protocol after the extension header")
assert.Equal(t, uint16(0), p.RemotePort) assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort) assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment) assert.False(t, p.Fragment)
+2 -2
View File
@@ -10,8 +10,8 @@ type batchWriter interface {
} }
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch. // SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
// Each reader goroutine owns its own SendBatch; no locking is needed. // One SendBatch is owned by each listenIn goroutine; no locking is needed.
// Slots can borrow receive buffers or be backed by an Arena (see its docs). // Slots are backed by an Arena (see its docs)
type SendBatch struct { type SendBatch struct {
out batchWriter out batchWriter
bufs [][]byte bufs [][]byte
-57
View File
@@ -1,57 +0,0 @@
//go:build !e2e_testing
package overlay
import (
"fmt"
"log/slog"
"golang.org/x/sys/windows"
"golang.org/x/sys/windows/registry"
)
var tcpipInterfaceKeys = []string{
`SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces`,
`SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces`,
}
// disableDNSRegistration prevents publishing overlay addresses in the host's
// DNS records, where clients outside the overlay cannot reach them.
// Failures are non-fatal so registry access cannot prevent tunnel creation.
func disableDNSRegistration(l *slog.Logger, guid windows.GUID) {
id := guid.String()
for _, base := range tcpipInterfaceKeys {
path := base + `\` + id
if err := setDNSRegistrationValues(path); err != nil {
l.Warn("Failed to disable DNS registration on the tun adapter", "error", err, "key", path)
continue
}
l.Debug("Disabled DNS registration on the tun adapter", "key", path)
}
}
func setDNSRegistrationValues(path string) error {
// The stack may not have created the adapter's interface key yet.
k, _, err := registry.CreateKey(registry.LOCAL_MACHINE, path, registry.SET_VALUE)
if err != nil {
return fmt.Errorf("open HKLM\\%s: %w", path, err)
}
defer k.Close()
// Neither RegistrationEnabled nor DisableDynamicUpdate alone reliably
// prevents registration, so all three settings are required.
values := []struct {
name string
data uint32
}{
{"RegistrationEnabled", 0},
{"DisableDynamicUpdate", 1},
{"MaxNumberOfAddressesToRegister", 0},
}
for _, v := range values {
if err := k.SetDWordValue(v.name, v.data); err != nil {
return fmt.Errorf("set %s: %w", v.name, err)
}
}
return nil
}
+4 -28
View File
@@ -50,13 +50,7 @@ func (r Route) String() string {
func makeRouteTree(l *slog.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) { func makeRouteTree(l *slog.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) {
routeTree := new(bart.Table[routing.Gateways]) routeTree := new(bart.Table[routing.Gateways])
seen := make(map[netip.Prefix]bool, len(routes))
for _, r := range routes { for _, r := range routes {
if seen[r.Cidr] {
l.Warn("route is listed more than once, only the last entry's via is used", "route", r)
}
seen[r.Cidr] = true
if !allowMTU && r.MTU > 0 { if !allowMTU && r.MTU > 0 {
l.Warn("route MTU is not supported on this platform", l.Warn("route MTU is not supported on this platform",
"goos", runtime.GOOS, "goos", runtime.GOOS,
@@ -104,11 +98,7 @@ func parseRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
mtu, ok := rMtu.(int) mtu, ok := rMtu.(int)
if !ok { if !ok {
s, ok := rMtu.(string) mtu, err = strconv.Atoi(rMtu.(string))
if !ok {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, rMtu)
}
mtu, err = strconv.Atoi(s)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, err) return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, err)
} }
@@ -184,11 +174,7 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
if rMtu, ok := m["mtu"]; ok { if rMtu, ok := m["mtu"]; ok {
mtu, ok = rMtu.(int) mtu, ok = rMtu.(int)
if !ok { if !ok {
s, ok := rMtu.(string) mtu, err = strconv.Atoi(rMtu.(string))
if !ok {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, rMtu)
}
mtu, err = strconv.Atoi(s)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err) return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err)
} }
@@ -206,15 +192,10 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
metric, ok := rMetric.(int) metric, ok := rMetric.(int)
if !ok { if !ok {
s, ok := rMetric.(string) _, err = strconv.ParseInt(rMetric.(string), 10, 32)
if !ok {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, rMetric)
}
n, err := strconv.ParseInt(s, 10, 32)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, err) return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, err)
} }
metric = int(n)
} }
if metric < 0 || metric > math.MaxInt32 { if metric < 0 || metric > math.MaxInt32 {
@@ -267,15 +248,10 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
gatewayWeight, ok := rGatewayWeight.(int) gatewayWeight, ok := rGatewayWeight.(int)
if !ok { if !ok {
s, ok := rGatewayWeight.(string) _, err = strconv.ParseInt(rGatewayWeight.(string), 10, 32)
if !ok {
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
}
w, err := strconv.ParseInt(s, 10, 32)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1) return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
} }
gatewayWeight = int(w)
} }
if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 { if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 {
-116
View File
@@ -1,17 +1,13 @@
package overlay package overlay
import ( import (
"bytes"
"fmt" "fmt"
"net/netip" "net/netip"
"strings"
"testing" "testing"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -406,115 +402,3 @@ func Test_makeMultipathUnsafeRouteTree(t *testing.T) {
routing.CalculateBucketsForGateways(expectedGateways) routing.CalculateBucketsForGateways(expectedGateways)
assert.ElementsMatch(t, expectedGateways, r) assert.ElementsMatch(t, expectedGateways, r)
} }
func Test_getMTU(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
// unset
assert.Equal(t, DefaultMTU, getMTU(l, c))
// the largest a relayed packet can carry in one underlay buffer
require.Equal(t, udp.MTU, MaxMTU+header.MaxOverhead)
c.Settings["tun"] = map[string]any{"mtu": MaxMTU}
assert.Equal(t, MaxMTU, getMTU(l, c))
// too big is capped
c.Settings["tun"] = map[string]any{"mtu": 9000}
assert.Equal(t, MaxMTU, getMTU(l, c))
}
func Test_getAllRoutesFromConfigCapsMTU(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
c.Settings["tun"] = map[string]any{
"routes": []any{
map[string]any{"mtu": "9000", "route": "10.0.0.0/29"},
map[string]any{"mtu": "1300", "route": "10.0.0.8/29"},
},
"unsafe_routes": []any{
map[string]any{"via": "127.0.0.1", "mtu": "9000", "route": "1.0.0.0/29"},
},
}
changed, routes, err := getAllRoutesFromConfig(l, c, []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")}, true)
require.NoError(t, err)
assert.True(t, changed)
mtus := map[string]int{}
for _, r := range routes {
mtus[r.Cidr.String()] = r.MTU
}
assert.Equal(t, map[string]int{"10.0.0.0/29": MaxMTU, "10.0.0.8/29": 1300, "1.0.0.0/29": MaxMTU}, mtus)
}
// The route tree keeps one entry per CIDR, so a CIDR listed twice only routes by the last one. Say so instead of
// letting the earlier via, or on Windows the earlier metric, quietly lose
func Test_makeRouteTree_WarnsOnDuplicate(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewLoggerWithOutput(buf)
via := func(s string) routing.Gateways {
return routing.Gateways{routing.NewGateway(netip.MustParseAddr(s), 1)}
}
cidr := netip.MustParsePrefix("10.251.1.0/24")
_, err := makeRouteTree(l, []Route{
{Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100},
{Cidr: netip.MustParsePrefix("10.251.2.0/24"), Via: via("10.250.0.2"), Install: true},
}, false)
require.NoError(t, err)
assert.Empty(t, buf.String())
tree, err := makeRouteTree(l, []Route{
{Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100},
{Cidr: cidr, Via: via("10.250.0.3"), Install: true, Metric: 200},
}, false)
require.NoError(t, err)
assert.Equal(t, 1, strings.Count(buf.String(), "route is listed more than once, only the last entry's via is used"))
gw, ok := tree.Lookup(netip.MustParseAddr("10.251.1.1"))
require.True(t, ok)
assert.Equal(t, netip.MustParseAddr("10.250.0.3"), gw[0].Addr(), "the last entry wins")
}
// A quoted metric or weight is a number written as a string, it means that number. Anything else is an error, not a
// panic
func Test_parseUnsafeRoutes_QuotedNumbers(t *testing.T) {
c := config.NewC(test.NewLogger())
n := netip.MustParsePrefix("10.0.0.1/24")
route := func(metric, weight any) {
c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{
"route": "1.0.0.0/8",
"metric": metric,
"via": []any{map[string]any{"gateway": "10.0.0.2", "weight": weight}},
}}}
}
route("100", "5")
routes, err := parseUnsafeRoutes(c, []netip.Prefix{n})
require.NoError(t, err)
require.Len(t, routes, 1)
assert.Equal(t, 100, routes[0].Metric)
assert.Equal(t, "{addr: 10.0.0.2, weight: 5}", routes[0].Via[0].String())
route(1.5, 1)
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.metric in tun.unsafe_routes is not an integer: 1.5")
route(1, true)
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry .weight in tun.unsafe_routes[1].via[1] is not an integer")
}
// An mtu that is neither a number nor a quoted one is an error, not a panic
func Test_parseRoutes_MTUNotANumber(t *testing.T) {
c := config.NewC(test.NewLogger())
n := netip.MustParsePrefix("10.0.0.1/24")
c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"route": "1.0.0.0/8", "mtu": 1500.5}}}
_, err := parseRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.mtu in tun.routes is not an integer: 1500.5")
c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"route": "1.0.0.0/8", "via": "10.0.0.2", "mtu": 1500.5}}}
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is not an integer: 1500.5")
}
+2 -31
View File
@@ -7,27 +7,11 @@ import (
"net/netip" "net/netip"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
) )
const DefaultMTU = 1300 const DefaultMTU = 1300
// MaxMTU is the largest tun.mtu, or route mtu, the underlay can carry.
// Every underlay buffer is udp.MTU bytes, and a relayed packet adds header.MaxOverhead to its plaintext.
const MaxMTU = udp.MTU - header.MaxOverhead
// getMTU reads tun.mtu, capping a value too big for the underlay to carry.
func getMTU(l *slog.Logger, c *config.C) int {
mtu := c.GetInt("tun.mtu", DefaultMTU)
if mtu > MaxMTU {
l.Warn("tun.mtu is too big for the underlay to carry, capping it", "mtu", mtu, "maxMTU", MaxMTU)
return MaxMTU
}
return mtu
}
type NameError struct { type NameError struct {
Name string Name string
Underlying error Underlying error
@@ -41,16 +25,9 @@ func (e *NameError) Error() string {
type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error)
func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) { func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.disabled")
})
switch { switch {
case c.GetBool("tun.disabled", false): case c.GetBool("tun.disabled", false):
tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 5000), c.GetBool("stats.message_metrics", false), l) tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.tx_queue", "stats.message_metrics")
})
return tun, nil return tun, nil
default: default:
@@ -64,7 +41,7 @@ func NewFdDeviceFromConfig(fd *int) DeviceFactory {
} }
} }
func getAllRoutesFromConfig(l *slog.Logger, c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) { func getAllRoutesFromConfig(c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) {
if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") { if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") {
return false, nil, nil return false, nil, nil
} }
@@ -80,12 +57,6 @@ func getAllRoutesFromConfig(l *slog.Logger, c *config.C, vpnNetworks []netip.Pre
} }
routes = append(routes, unsafeRoutes...) routes = append(routes, unsafeRoutes...)
for i, r := range routes {
if r.MTU > MaxMTU {
l.Warn("route mtu is too big for the underlay to carry, capping it", "route", r.Cidr, "mtu", r.MTU, "maxMTU", MaxMTU)
routes[i].MTU = MaxMTU
}
}
return true, routes, nil return true, routes, nil
} }
+5 -6
View File
@@ -38,17 +38,16 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
l: l, l: l,
} }
err := t.reload(c, true, nil) err := t.reload(c, true)
if err != nil { if err != nil {
_ = file.Close() _ = file.Close()
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
err := t.reload(c, false, r) err := t.reload(c, false)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -68,8 +67,8 @@ func (t *tun) Activate() error {
return nil return nil
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
-45
View File
@@ -1,45 +0,0 @@
//go:build (openbsd || netbsd) && !e2e_testing
package overlay
import (
"errors"
"fmt"
"log/slog"
"os"
"regexp"
"runtime"
"golang.org/x/sys/unix"
)
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
// tunDevNeedsRestart is true when tun.dev asks for a particular tun that isn't the running one. Anything else takes the
// first free one, and the one we have will do
func tunDevNeedsRestart(name, running string) bool {
return deviceNameRE.MatchString(name) && name != running
}
// openTun opens /dev/name, or the first /dev/tunN nothing else has open when name is empty or not tunN
func openTun(l *slog.Logger, name string) (int, string, error) {
if name != "" && !deviceNameRE.MatchString(name) {
// The name doesn't mean anything here, so like darwin a bad one isn't worth refusing to start over
l.Warn("interface name must be tun[0-9]+ on "+runtime.GOOS+", ignoring", "name", name)
name = ""
}
if name != "" {
fd, err := unix.Open("/dev/"+name, os.O_RDWR, 0)
return fd, name, err
}
for i := 0; ; i++ {
name = fmt.Sprintf("tun%d", i)
fd, err := unix.Open("/dev/"+name, os.O_RDWR, 0)
if err == nil {
return fd, name, nil
}
if !errors.Is(err, unix.EBUSY) {
return -1, "", fmt.Errorf("no free tun device, %s: %w", name, err)
}
}
}
-80
View File
@@ -1,80 +0,0 @@
//go:build (openbsd || netbsd) && !e2e_testing
package overlay
import (
"net/netip"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// A tun.dev that isn't tunN is ignored like darwin does, and the first free tun is used. Only naming a different tun
// asks for a restart
func TestTunReloadDevRestart(t *testing.T) {
newWithDev := func(dev string) (*config.C, *tun, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n dev: "+dev+"\n"))
tn, err := newTun(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, false)
if err == nil {
t.Cleanup(func() { _ = tn.Close() })
}
return c, tn, err
}
if _, _, err := newWithDev(""); err != nil {
t.Skipf("can't create a tun device here: %v", err)
}
c, tn, err := newWithDev("nope")
require.NoError(t, err, "a bad name should fall back to a free tun")
require.Regexp(t, `^tun[0-9]+$`, tn.Device)
for _, dev := range []string{"nope", "", tn.Device, "nope"} {
res := c.ReloadConfigString("tun:\n dev: " + dev + "\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed, dev)
assert.False(t, res.RestartRequired, dev)
}
other := "tun0"
if tn.Device == other {
other = "tun1"
}
res := c.ReloadConfigString("tun:\n dev: " + other + "\n")
require.NoError(t, res.Err)
assert.Equal(t, []string{"tun.dev"}, res.RestartKeys)
}
// An empty tun.dev takes the first tun nothing else has open. Reloading it empty, or naming the one we have, needs
// no restart
func TestTunEmptyDevPicksAFreeOne(t *testing.T) {
newEmpty := func() (*config.C, *tun, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun: {}\n"))
tn, err := newTun(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, false)
if err == nil {
t.Cleanup(func() { _ = tn.Close() })
}
return c, tn, err
}
c, first, err := newEmpty()
if err != nil {
t.Skipf("can't create a tun device here: %v", err)
}
_, second, err := newEmpty()
require.NoError(t, err, "the second tun should skip the one the first has open")
assert.NotEqual(t, first.Device, second.Device)
for _, cfg := range []string{
"tun: {}\npunchy:\n punch: true\n",
"tun:\n dev: " + first.Device + "\n",
"tun: {}\n",
} {
res := c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired, cfg)
}
}
-3
View File
@@ -10,9 +10,6 @@ import (
"github.com/slackhq/nebula/wfp" "github.com/slackhq/nebula/wfp"
) )
// interfaceBypassKeys are only read when the interface bypass is installed
var interfaceBypassKeys = []string{"tun.windows_bypass_wdf"}
// installInterfaceBypass installs a WFP PERMIT filter scoped to the wintun interface LUID so inbound traffic on the // installInterfaceBypass installs a WFP PERMIT filter scoped to the wintun interface LUID so inbound traffic on the
// nebula adapter bypasses Windows Defender Firewall. // nebula adapter bypasses Windows Defender Firewall.
func installInterfaceBypass(l *slog.Logger, luid uint64) closer { func installInterfaceBypass(l *slog.Logger, luid uint64) closer {
-3
View File
@@ -4,9 +4,6 @@ package overlay
import "log/slog" import "log/slog"
// interfaceBypassKeys is empty, windows-386 has no bypass
var interfaceBypassKeys []string
// installInterfaceBypass is a no-op on windows-386 because we don't currently build for it. // installInterfaceBypass is a no-op on windows-386 because we don't currently build for it.
func installInterfaceBypass(_ *slog.Logger, _ uint64) closer { func installInterfaceBypass(_ *slog.Logger, _ uint64) closer {
return nil return nil
+41 -152
View File
@@ -6,10 +6,8 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"math/bits"
"net/netip" "net/netip"
"os" "os"
"sync"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -30,7 +28,7 @@ type tun struct {
DefaultMTU int DefaultMTU int
Routes atomic.Pointer[[]Route] Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]] routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr atomic.Pointer[netroute.LinkAddr] linkAddr *netroute.LinkAddr
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu // hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
// and routes for it. NEPacketTunnelProvider on darwin does this. // and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool hostOwned bool
@@ -80,31 +78,17 @@ type ifreqAlias6 struct {
Lifetime addrLifetime Lifetime addrLifetime
} }
// utunIndex is the utun tun.dev asks for, or -1 for whichever is free. ok is false for a name that isn't utunN, which
// also gets whichever is free
func utunIndex(name string) (index int, ok bool) {
if name == "" || name == "utun" {
return -1, true
}
if _, err := fmt.Sscanf(name, "utun%d", &index); err != nil || index < 0 {
return -1, false
}
return index, true
}
// tunDevNeedsRestart is true when tun.dev asks for a particular utun that isn't the running one. Anything else takes
// whichever is free, and the one we have will do
func tunDevNeedsRestart(name, running string) bool {
index, _ := utunIndex(name)
return index >= 0 && fmt.Sprintf("utun%d", index) != running
}
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
ifIndex, ok := utunIndex(c.GetString("tun.dev", "")) name := c.GetString("tun.dev", "")
if !ok { ifIndex := -1
// NOTE: we don't make this error so we don't break existing if name != "" && name != "utun" {
// configs that set a name before it was used. _, err := fmt.Sscanf(name, "utun%d", &ifIndex)
l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring") if err != nil || ifIndex < 0 {
// NOTE: we don't make this error so we don't break existing
// configs that set a name before it was used.
l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring")
ifIndex = -1
}
} }
fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL) fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL)
@@ -128,7 +112,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
return nil, fmt.Errorf("SYS_CONNECT: %v", err) return nil, fmt.Errorf("SYS_CONNECT: %v", err)
} }
name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME) name, err = unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to retrieve tun name: %w", err) return nil, fmt.Errorf("failed to retrieve tun name: %w", err)
} }
@@ -142,24 +126,19 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
f: os.NewFile(uintptr(fd), ""), f: os.NewFile(uintptr(fd), ""),
Device: name, Device: name,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c), DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
l: l, l: l,
} }
err = t.reload(c, true, nil) err = t.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("tun.mtu") err := t.reload(c, false)
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -188,20 +167,19 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
f: file, f: file,
Device: utunNameFromFd(deviceFd), Device: utunNameFromFd(deviceFd),
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c), DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
hostOwned: true, hostOwned: true,
l: l, l: l,
} }
if err := t.reload(c, true, nil); err != nil { if err := t.reload(c, true); err != nil {
_ = file.Close() _ = file.Close()
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
if err := t.reload(c, false, r); err != nil { if err := t.reload(c, false); err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -265,7 +243,7 @@ func (t *tun) Activate() error {
if linkAddr == nil { if linkAddr == nil {
return fmt.Errorf("unable to discover link_addr for tun interface") return fmt.Errorf("unable to discover link_addr for tun interface")
} }
t.linkAddr.Store(linkAddr) t.linkAddr = linkAddr
for _, network := range t.vpnNetworks { for _, network := range t.vpnNetworks {
if network.Addr().Is4() { if network.Addr().Is4() {
@@ -288,10 +266,9 @@ func (t *tun) Activate() error {
} }
// Unsafe path routes // Unsafe path routes
return t.addRoutes(false, nil) return t.addRoutes(false)
} }
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
func (t *tun) activate4(network netip.Prefix) error { func (t *tun) activate4(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET, unix.AF_INET,
@@ -318,32 +295,22 @@ func (t *tun) activate4(network netip.Prefix) error {
MaskAddr: unix.RawSockaddrInet4{ MaskAddr: unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4, Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET, Family: unix.AF_INET,
Addr: [4]byte{255, 255, 255, 255}, //always make the address a /32 Addr: prefixToMask(network).As4(),
}, },
} }
// On Darwin, VPNs don't edit the routing table directly.
// A network extension hands its routes to the system, and configd installs them.
// configd replaces any route already holding the same prefix, including ours.
// As an example, Tailscale with an exit node treats any interface prefix wider than one address as a LAN.
// With LAN access off, it routes each LAN into Tailscale's utun.
// With LAN access on, it excludes each LAN from its tunnel.
// configd routes an excluded prefix like its own routes would, ignoring ours, so it goes out the default route.
// Either way our route is replaced, and traffic for the vpn network no longer reaches nebula.
// A single address isn't a LAN to Tailscale, so our route is left alone.
// We route the vpn network to the tun explicitly instead.
if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun v4 address: %s", err) return fmt.Errorf("failed to set tun v4 address: %s", err)
} }
if err := addRoute(network, t.linkAddr.Load()); err != nil { err = addRoute(network, t.linkAddr)
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err) if err != nil {
return err
} }
return nil return nil
} }
// activate6 is activate4 for IPv6
func (t *tun) activate6(network netip.Prefix) error { func (t *tun) activate6(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET6, unix.AF_INET6,
@@ -365,29 +332,25 @@ func (t *tun) activate6(network netip.Prefix) error {
PrefixMask: unix.RawSockaddrInet6{ PrefixMask: unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6, Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6, Family: unix.AF_INET6,
// Always set the netmask to be a /128. See activate4 for why we do this. Addr: prefixToMask(network).As16(),
Addr: [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
}, },
Lifetime: addrLifetime{ Lifetime: addrLifetime{
// never expires // never expires
Vltime: 0xffffffff, Vltime: 0xffffffff,
Pltime: 0xffffffff, Pltime: 0xffffffff,
}, },
Flags: _IN6_IFF_NODAD, // No DAD, a utun has no neighbors. Flags: _IN6_IFF_NODAD,
} }
if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun address: %s", err) return fmt.Errorf("failed to set tun address: %s", err)
} }
if err := addRoute(network, t.linkAddr.Load()); err != nil {
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
}
return nil return nil
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -407,18 +370,16 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -433,56 +394,6 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
return routing.Gateways{} return routing.Gateways{}
} }
func fetchRoutes() ([]netroute.Message, error) {
rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_DUMP, 0)
if err != nil {
return nil, err
}
return netroute.ParseRIB(unix.NET_RT_DUMP, rib)
}
// routeInterfaceIn returns the index of the interface prefix is routed through, or 0 if msgs has no route for exactly
// that prefix. Routes scoped to an interface are skipped, they don't stop us adding ours.
func routeInterfaceIn(msgs []netroute.Message, prefix netip.Prefix) int {
for _, msg := range msgs {
m, ok := msg.(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK || m.Flags&unix.RTF_IFSCOPE != 0 {
continue
}
if p, ok := routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK]); ok && p == prefix.Masked() {
return m.Index
}
}
return 0
}
// routePrefix returns the prefix a route covers. Host routes have no netmask.
func routePrefix(dst, mask netroute.Addr) (netip.Prefix, bool) {
switch d := dst.(type) {
case *netroute.Inet4Addr:
bits := 32
if m, ok := mask.(*netroute.Inet4Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom4(d.IP), bits), true
case *netroute.Inet6Addr:
bits := 128
if m, ok := mask.(*netroute.Inet6Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom16(d.IP), bits), true
}
return netip.Prefix{}, false
}
func maskBits(mask []byte) int {
n := 0
for _, b := range mask {
n += bits.OnesCount8(b)
}
return n
}
// Get the LinkAddr for the interface of the given name // Get the LinkAddr for the interface of the given name
// Is there an easier way to fetch this when we create the interface? // Is there an easier way to fetch this when we create the interface?
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers. // Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
@@ -511,20 +422,13 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
return nil, nil return nil, nil
} }
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { func (t *tun) addRoutes(logErrors bool) error {
// The route tree is still ours, the system routing table is not // The route tree is still ours, the system routing table is not
if t.hostOwned { if t.hostOwned {
return nil return nil
} }
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load() routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
@@ -532,54 +436,39 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
continue continue
} }
err := addRoute(r.Cidr, linkAddr) err := addRoute(r.Cidr, t.linkAddr)
if err != nil { if err != nil {
if errors.Is(err, unix.EEXIST) { if errors.Is(err, unix.EEXIST) {
// Every route we added already exists on a reload, only warn about someone else's t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr)
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else { } else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
} }
} else { } else {
t.l.Info("Added route", "route", r) t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
} }
} }
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { func (t *tun) removeRoutes(routes []Route) error {
linkAddr := t.linkAddr.Load() if t.hostOwned {
if t.hostOwned || linkAddr == nil {
return nil return nil
} }
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, linkAddr) err := delRoute(r.Cidr, t.linkAddr)
if errors.Is(err, unix.ESRCH) { if err != nil {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
-118
View File
@@ -1,118 +0,0 @@
//go:build darwin && !ios && !e2e_testing
package overlay
import (
"net"
"net/netip"
"os"
"os/exec"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Every mac routes loopback through lo0
func TestRouteInterfaceIn(t *testing.T) {
msgs, err := fetchRoutes()
require.NoError(t, err)
routeInterface := func(s string) int { return routeInterfaceIn(msgs, netip.MustParsePrefix(s)) }
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterface("127.0.0.0/8"))
assert.Equal(t, lo0.Index, routeInterface("127.0.0.1/32"), "a host route has no netmask")
assert.Equal(t, lo0.Index, routeInterface("::1/128"))
assert.Equal(t, 0, routeInterface("127.0.0.0/16"), "only an exact prefix counts")
// Most macs also list the default route scoped to their other interfaces
out, err := exec.Command("route", "-n", "get", "default").Output()
require.NoError(t, err)
var name string
for _, line := range strings.Split(string(out), "\n") {
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "interface: "); ok {
name = v
}
}
if name == "" {
t.Skip("no default route")
}
def, err := net.InterfaceByName(name)
require.NoError(t, err)
assert.Equal(t, def.Index, routeInterface("0.0.0.0/0"))
}
func TestRouteInterfaceIn_SkipsScopedCopies(t *testing.T) {
route := func(index, flags int) netroute.Message {
addrs := make([]netroute.Addr, unix.RTAX_MAX)
addrs[unix.RTAX_DST] = &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}}
addrs[unix.RTAX_NETMASK] = &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}}
return &netroute.RouteMessage{Index: index, Flags: flags, Addrs: addrs}
}
p := netip.MustParsePrefix("192.168.1.0/24")
assert.Equal(t, 7, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE), route(7, unix.RTF_UP)}, p))
assert.Equal(t, 0, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE)}, p))
}
// A reload can happen before Activate
func TestTunRoutesBeforeActivate(t *testing.T) {
tn := &tun{l: test.NewLogger()}
routes := []Route{{
Cidr: netip.MustParsePrefix("10.253.1.0/24"),
Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)},
Install: true,
}}
tn.Routes.Store(&routes)
assert.NoError(t, tn.addRoutes(true, nil))
assert.NoError(t, tn.removeRoutes(routes, nil))
}
// A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that
// belongs to someone else
func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating a utun and adding routes needs root")
}
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
cidr := "10.250.0.128/25"
require.NoError(t, exec.Command("route", "-q", "-n", "add", "-net", cidr, "-interface", "lo0").Run())
t.Cleanup(func() { _ = exec.Command("route", "-q", "-n", "delete", "-net", cidr).Run() })
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("tun:\n routes:\n - route: "+cidr+"\n mtu: 1300\n"))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
res := c.ReloadConfigString("tun: {}\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed, "removing a route we never installed failed the reload")
msgs, err := fetchRoutes()
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
func TestTunDevNeedsRestart(t *testing.T) {
for name, want := range map[string]bool{
"": false,
"utun": false,
"nope": false,
"utun7": false,
"utun3": true,
} {
assert.Equal(t, want, tunDevNeedsRestart(name, "utun7"), name)
}
}
-121
View File
@@ -1,121 +0,0 @@
//go:build !ios && !e2e_testing
package overlay
import (
"bytes"
"os"
"syscall"
"testing"
"github.com/stretchr/testify/require"
)
func TestTun_WriteFraming(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: w}
for _, tc := range []struct {
name string
pkt []byte
af byte
}{
// Literal families rather than syscall.AF_*, so a wrong constant in Write can't also fix the expectation.
{"ipv4", []byte{0x45, 1, 2, 3, 4, 5}, 2},
{"ipv6", []byte{0x60, 1, 2, 3, 4, 5}, 30},
} {
t.Run(tc.name, func(t *testing.T) {
n, err := tn.Write(tc.pkt)
require.NoError(t, err)
require.Equal(t, len(tc.pkt), n)
got := make([]byte, 64)
n, err = r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, tc.af}, tc.pkt...), got[:n])
})
}
_, err = tn.Write([]byte{0x10, 1, 2, 3})
require.Error(t, err)
_, err = tn.Write([]byte{})
require.ErrorIs(t, err, syscall.EIO)
// The rejected writes must not have put anything on the fd, so the next read sees only this packet.
pkt := []byte{0x45, 9, 9}
_, err = tn.Write(pkt)
require.NoError(t, err)
got := make([]byte, 64)
n, err := r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, 2}, pkt...), got[:n])
}
func TestTun_ReadStripsHeader(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
pkt := []byte{0x45, 1, 2, 3, 4, 5}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err := tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
func TestTun_ReadShort(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
// Fewer bytes than the protocol header is not a packet.
_, err = w.Write([]byte{0, 0})
require.NoError(t, err)
n, err := tn.Read(make([]byte, 64))
require.NoError(t, err)
require.Zero(t, n)
// The next packet reads cleanly only if the short read consumed exactly what was written.
pkt := []byte{0x45, 1, 2, 3}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err = tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
// A heap allocation per packet rarely shows in a short throughput run, only across GC pauses, so
// this pins Read and Write to zero allocations directly.
func TestTun_ReadWriteDoNotAllocate(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tw := &tun{f: w}
tr := &tun{f: r}
pkts := [][]byte{bytes.Repeat([]byte{0x45}, 1300), bytes.Repeat([]byte{0x60}, 1300)}
buf := make([]byte, 1500)
i := 0
allocs := testing.AllocsPerRun(1000, func() {
i++
if _, err := tw.Write(pkts[i%len(pkts)]); err != nil {
t.Fatal(err)
}
if _, err := tr.Read(buf); err != nil {
t.Fatal(err)
}
})
require.Zero(t, allocs)
}
+13 -23
View File
@@ -370,7 +370,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
t := &tun{ t := &tun{
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
l: l, l: l,
fd: fd, fd: fd,
shutdownR: shutdownR, shutdownR: shutdownR,
@@ -385,18 +385,16 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
}, },
} }
err = t.reload(c, true, nil) err = t.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
closeOnErr = false closeOnErr = false
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("tun.dev", "tun.mtu") err := t.reload(c, false)
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -495,7 +493,7 @@ func (t *tun) Activate() error {
} }
} }
return t.addRoutes(false, nil) return t.addRoutes(false)
} }
func (t *tun) setMTU() error { func (t *tun) setMTU() error {
@@ -511,8 +509,8 @@ func (t *tun) setMTU() error {
return err return err
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -532,18 +530,16 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -567,7 +563,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
} }
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { func (t *tun) addRoutes(logErrors bool) error {
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
@@ -580,7 +576,6 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -592,20 +587,15 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.linkAddr) err := delRoute(r.Cidr, t.linkAddr)
if errors.Is(err, unix.ESRCH) { if err != nil {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
+5 -6
View File
@@ -47,17 +47,16 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
l: l, l: l,
} }
err := t.reload(c, true, nil) err := t.reload(c, true)
if err != nil { if err != nil {
_ = file.Close() _ = file.Close()
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
err := t.reload(c, false, r) err := t.reload(c, false)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -68,8 +67,8 @@ func (t *tun) Activate() error {
return nil return nil
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
+18 -41
View File
@@ -185,10 +185,6 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
} }
t.Device = name t.Device = name
// Only read when we create the device, an fd handed to us already has its name and offloads
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.dev", "tun.use_offloads")
})
return t, nil return t, nil
} }
@@ -225,32 +221,30 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
vnetHdr: vnetHdr, vnetHdr: vnetHdr,
offloadFlags: offloadFlags, offloadFlags: offloadFlags,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
TXQueueLen: c.GetInt("tun.tx_queue", 5000), TXQueueLen: c.GetInt("tun.tx_queue", 500),
useSystemRoutes: c.GetBool("tun.use_system_route_table", false), useSystemRoutes: c.GetBool("tun.use_system_route_table", false),
useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0), useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
routesFromSystem: map[netip.Prefix]routing.Gateways{}, routesFromSystem: map[netip.Prefix]routing.Gateways{},
l: l, l: l,
} }
if err = t.reload(c, true, nil); err != nil { if err = t.reload(c, true); err != nil {
_ = t.Close() _ = t.Close()
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("tun.tx_queue", "tun.use_system_route_table", "tun.use_system_route_table_buffer_size") err := t.reload(c, false)
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
return t, nil return t, nil
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
routeChange, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -259,15 +253,6 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
return nil return nil
} }
// Only the mtu changed. The routes are the same, parse them again so the ones without their own mtu pick up the
// new default. Without this an mtu change would remove them all
if !routeChange {
_, routes, err = getAllRoutesFromConfig(t.l, c, t.vpnNetworks, true)
if err != nil {
return err
}
}
routeTree, err := makeRouteTree(t.l, routes, true) routeTree, err := makeRouteTree(t.l, routes, true)
if err != nil { if err != nil {
return err return err
@@ -282,14 +267,14 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
oldDefaultMTU := t.DefaultMTU oldDefaultMTU := t.DefaultMTU
oldMaxMTU := t.MaxMTU oldMaxMTU := t.MaxMTU
newDefaultMTU := getMTU(t.l, c) newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU)
newMaxMTU := newDefaultMTU newMaxMTU := newDefaultMTU
for i, r := range routes { for i, r := range routes {
if r.MTU == 0 { if r.MTU == 0 {
routes[i].MTU = newDefaultMTU routes[i].MTU = newDefaultMTU
} }
if r.MTU > newMaxMTU { if r.MTU > t.MaxMTU {
newMaxMTU = r.MTU newMaxMTU = r.MTU
} }
} }
@@ -303,7 +288,7 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
if oldMaxMTU != newMaxMTU { if oldMaxMTU != newMaxMTU {
t.setMTU(r) t.setMTU()
t.l.Info("Set max MTU", "mtu", t.MaxMTU, "oldMTU", oldMaxMTU) t.l.Info("Set max MTU", "mtu", t.MaxMTU, "oldMTU", oldMaxMTU)
} }
@@ -311,8 +296,7 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
for i := range t.vpnNetworks { for i := range t.vpnNetworks {
err := t.setDefaultRoute(t.vpnNetworks[i]) err := t.setDefaultRoute(t.vpnNetworks[i])
if err != nil { if err != nil {
t.l.Error(err.Error()) t.l.Warn(err.Error())
r.Failed()
} else { } else {
t.l.Info("Set default MTU", "mtu", t.DefaultMTU, "oldMTU", oldDefaultMTU) t.l.Info("Set default MTU", "mtu", t.DefaultMTU, "oldMTU", oldDefaultMTU)
} }
@@ -320,14 +304,13 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
} }
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// This should never be called since addRoutes should log its own errors in a reload condition // This should never be called since addRoutes should log its own errors in a reload condition
util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l) util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l)
r.Failed()
} }
} }
@@ -482,7 +465,7 @@ func (t *tun) Activate() error {
t.deviceIndex = link.Attrs().Index t.deviceIndex = link.Attrs().Index
// Setup our default MTU // Setup our default MTU
t.setMTU(nil) t.setMTU()
// Set the transmit queue length // Set the transmit queue length
ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)} ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)}
@@ -514,7 +497,7 @@ func (t *tun) Activate() error {
} }
// Set the routes // Set the routes
if err = t.addRoutes(false, nil); err != nil { if err = t.addRoutes(false); err != nil {
return err return err
} }
@@ -527,13 +510,12 @@ func (t *tun) Activate() error {
return nil return nil
} }
func (t *tun) setMTU(r *config.Reload) { func (t *tun) setMTU() {
// Set the MTU on the device // Set the MTU on the device
ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)} ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)}
if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil { if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
// This is currently a non fatal condition because the route table must have the MTU set appropriately as well // This is currently a non fatal condition because the route table must have the MTU set appropriately as well
t.l.Error("Failed to set tun mtu", "error", err) t.l.Error("Failed to set tun mtu", "error", err)
r.Failed()
} }
} }
@@ -586,7 +568,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
return nil return nil
} }
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { func (t *tun) addRoutes(logErrors bool) error {
// Path routes // Path routes
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -616,7 +598,6 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -628,7 +609,7 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) { func (t *tun) removeRoutes(routes []Route) {
for _, r := range routes { for _, r := range routes {
if !r.Install { if !r.Install {
continue continue
@@ -652,12 +633,8 @@ func (t *tun) removeRoutes(routes []Route, rl *config.Reload) {
} }
err := netlink.RouteDel(&nr) err := netlink.RouteDel(&nr)
if errors.Is(err, unix.ESRCH) { if err != nil {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
+27 -27
View File
@@ -8,6 +8,7 @@ import (
"log/slog" "log/slog"
"net/netip" "net/netip"
"os" "os"
"regexp"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -66,12 +67,24 @@ type tun struct {
fd int fd int
} }
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
return nil, fmt.Errorf("newTunFromFd not supported in NetBSD") return nil, fmt.Errorf("newTunFromFd not supported in NetBSD")
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
fd, deviceName, err := openTun(l, c.GetString("tun.dev", "")) // Try to open tun device
var err error
deviceName := c.GetString("tun.dev", "")
if deviceName == "" {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
if !deviceNameRE.MatchString(deviceName) {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -86,24 +99,19 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
fd: fd, fd: fd,
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
l: l, l: l,
} }
err = t.reload(c, true, nil) err = t.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("tun.mtu") err := t.reload(c, false)
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -316,7 +324,7 @@ func (t *tun) Activate() error {
} }
} }
return t.addRoutes(false, nil) return t.addRoutes(false)
} }
func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
@@ -331,8 +339,8 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
return err return err
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -352,18 +360,16 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -387,7 +393,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
} }
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { func (t *tun) addRoutes(logErrors bool) error {
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -401,7 +407,6 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -413,20 +418,15 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.vpnNetworks) err := delRoute(r.Cidr, t.vpnNetworks)
if errors.Is(err, unix.ESRCH) { if err != nil {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
+27 -27
View File
@@ -8,6 +8,7 @@ import (
"log/slog" "log/slog"
"net/netip" "net/netip"
"os" "os"
"regexp"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -57,12 +58,24 @@ type tun struct {
fd int fd int
} }
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
return nil, fmt.Errorf("newTunFromFd not supported in openbsd") return nil, fmt.Errorf("newTunFromFd not supported in openbsd")
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
fd, deviceName, err := openTun(l, c.GetString("tun.dev", "")) // Try to open tun device
var err error
deviceName := c.GetString("tun.dev", "")
if deviceName == "" {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
if !deviceNameRE.MatchString(deviceName) {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -77,24 +90,19 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
fd: fd, fd: fd,
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
l: l, l: l,
} }
err = t.reload(c, true, nil) err = t.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("tun.mtu") err := t.reload(c, false)
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -295,7 +303,7 @@ func (t *tun) Activate() error {
} }
} }
return t.addRoutes(false, nil) return t.addRoutes(false)
} }
func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
@@ -310,8 +318,8 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
return err return err
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -331,18 +339,16 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -366,7 +372,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
} }
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { func (t *tun) addRoutes(logErrors bool) error {
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -380,7 +386,6 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -392,20 +397,15 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.vpnNetworks) err := delRoute(r.Cidr, t.vpnNetworks)
if errors.Is(err, unix.ESRCH) { if err != nil {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
-107
View File
@@ -1,107 +0,0 @@
//go:build !e2e_testing
package overlay
import (
"net/netip"
"os"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Only tun.mtu changes. The system calls fail without a real device, what matters is the routes nebula keeps
func TestReloadMTUOnlyKeepsRoutes(t *testing.T) {
routes := " unsafe_routes:\n - route: 192.168.100.0/24\n via: 10.1.0.1\n"
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n mtu: 1300\n"+routes))
tn := &tun{
l: test.NewLogger(),
vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")},
ioctlFd: ^uintptr(0),
routesFromSystem: map[netip.Prefix]routing.Gateways{},
}
require.NoError(t, tn.reload(c, true, nil))
require.NoError(t, c.ReloadConfigString("tun:\n mtu: 1400\n"+routes).Err)
_ = tn.reload(c, false, nil)
got := *tn.Routes.Load()
require.Len(t, got, 1)
assert.Equal(t, netip.MustParsePrefix("192.168.100.0/24"), got[0].Cidr)
assert.Equal(t, 1400, got[0].MTU)
assert.Equal(t, 1400, tn.DefaultMTU)
}
// The device mtu has to fit the largest route mtu, wherever it sits in the list
func TestReloadMaxMTUIsTheLargestRoute(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n mtu: 1300\n unsafe_routes:\n"+
" - route: 192.168.100.0/24\n via: 10.1.0.1\n mtu: 1500\n"+
" - route: 192.168.101.0/24\n via: 10.1.0.1\n mtu: 1400\n"))
tn := &tun{
l: test.NewLogger(),
vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")},
ioctlFd: ^uintptr(0),
routesFromSystem: map[netip.Prefix]routing.Gateways{},
}
require.NoError(t, tn.reload(c, true, nil))
assert.Equal(t, 1500, tn.MaxMTU)
assert.Equal(t, 1300, tn.DefaultMTU)
}
// There is no device behind this tun, so adding a route always fails. Removing one is refused without root, as root the
// kernel answers that the route isn't there, which is what we wanted
func TestReloadRouteFailures(t *testing.T) {
route := "tun:\n mtu: 1300\n unsafe_routes:\n - route: 192.168.100.0/24\n via: 10.1.0.1\n"
cases := []struct {
name string
from, to string
wantFails bool
}{
{"added", "tun:\n mtu: 1300\n", route, true},
{"removed", route, "tun:\n mtu: 1300\n", os.Geteuid() != 0},
{"unchanged", route, route + "punchy:\n punch: true\n", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(tc.from))
tn := &tun{
l: test.NewLogger(),
vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")},
ioctlFd: ^uintptr(0),
routesFromSystem: map[netip.Prefix]routing.Gateways{},
}
require.NoError(t, tn.reload(c, true, nil))
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
require.NoError(t, tn.reload(c, false, r))
})
res := c.ReloadConfigString(tc.to)
require.NoError(t, res.Err)
assert.Equal(t, tc.wantFails, res.Failed)
})
}
}
// Keys only read when the device is created need a restart to apply
func TestTunReloadRestartKeys(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n dev: nebrk0\n"))
tn, err := NewDeviceFromConfig(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, 1)
if err != nil {
t.Skipf("can't create a tun device here: %v", err)
}
t.Cleanup(func() { _ = tn.Close() })
res := c.ReloadConfigString("tun:\n disabled: true\n dev: nebrk1\n tx_queue: 100\n")
require.NoError(t, res.Err)
assert.ElementsMatch(t, []string{"tun.disabled", "tun.dev", "tun.tx_queue"}, res.RestartKeys)
}
-24
View File
@@ -1,24 +0,0 @@
package overlay
import (
"net/netip"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// tun.disabled and the disabled tun's settings are only read when the device is made
func TestDisabledTunRestartKeys(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n disabled: true\n"))
d, err := NewDeviceFromConfig(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, 1)
require.NoError(t, err)
t.Cleanup(func() { _ = d.Close() })
res := c.ReloadConfigString("tun:\n disabled: false\n tx_queue: 100\n")
require.NoError(t, res.Err)
assert.ElementsMatch(t, []string{"tun.disabled", "tun.tx_queue"}, res.RestartKeys)
}
+1 -1
View File
@@ -31,7 +31,7 @@ type TestTun struct {
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) {
_, routes, err := getAllRoutesFromConfig(l, c, vpnNetworks, true) _, routes, err := getAllRoutesFromConfig(c, vpnNetworks, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
+15 -107
View File
@@ -4,7 +4,6 @@ package overlay
import ( import (
"crypto" "crypto"
"errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net/netip" "net/netip"
@@ -76,7 +75,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
t := &winTun{ t := &winTun{
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
guid: *guid, guid: *guid,
networkCategory: cat, networkCategory: cat,
setCategory: setCat, setCategory: setCat,
@@ -84,7 +83,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
l: l, l: l,
} }
err = t.reload(c, true, nil) err = t.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -105,32 +104,18 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
} }
t.tun = tunDevice.(*wintun.NativeTun) t.tun = tunDevice.(*wintun.NativeTun)
// Disable registration before Activate assigns addresses and triggers it. c.RegisterReloadCallback(func(c *config.C) {
disableDNSRegistration(l, *guid) err := t.reload(c, false)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.dev", "tun.mtu")
// Compare with the running category, not the last config. A restart with one we can't parse would fail, so say
// that now rather than ask for one, and going back to the running one after a bad reload needs no restart
if cat, setCat, err := parseNetworkCategory(c.GetString("tun.network_category", "private")); err != nil {
t.l.Error("Invalid tun.network_category, a restart with it would fail", "error", err)
r.Failed()
} else if cat != t.networkCategory || setCat != t.setCategory {
r.RestartRequiredIfChanged("tun.network_category")
}
r.RestartRequiredIfChanged(interfaceBypassKeys...)
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
return t, nil return t, nil
} }
func (t *winTun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *winTun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -150,18 +135,16 @@ func (t *winTun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes), r) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -176,7 +159,7 @@ func (t *winTun) Activate() error {
return fmt.Errorf("failed to set address: %w", err) return fmt.Errorf("failed to set address: %w", err)
} }
err = t.addRoutes(false, nil) err = t.addRoutes(false)
if err != nil { if err != nil {
return err return err
} }
@@ -195,49 +178,33 @@ func (t *winTun) Activate() error {
return nil return nil
} }
func (t *winTun) addRoutes(logErrors bool, rl *config.Reload) error { func (t *winTun) addRoutes(logErrors bool) error {
luid := winipcfg.LUID(t.tun.LUID()) luid := winipcfg.LUID(t.tun.LUID())
routes := *t.Routes.Load() routes := *t.Routes.Load()
foundDefault4 := false foundDefault4 := false
carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() }) carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() })
// A CIDR listed twice routes by its last entry, install only that one or each reload would flip between them. The for _, r := range routes {
// same rule as systemRoutes
last := make(map[netip.Prefix]int, len(routes))
for i, r := range routes {
last[r.Cidr] = i
}
for i, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via // We don't allow route MTUs so only install routes with a via
continue continue
} }
if last[r.Cidr] != i {
continue
}
// A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter. // A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter.
carriesV6 = carriesV6 || r.Cidr.Addr().Is6() carriesV6 = carriesV6 || r.Cidr.Addr().Is6()
// Add our unsafe route as an on-link route to the nebula tun device. // Add our unsafe route as an on-link route to the nebula tun device.
result, err := addRoute(luid, r) err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric))
if err != nil { if err != nil {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
continue continue
} else { } else {
return retErr return retErr
} }
} else { } else {
switch result { t.l.Info("Added route", "route", r)
case routeAdded:
t.l.Info("Added route", "route", r)
case routeMetricUpdated:
t.l.Info("Updated route metric", "route", r)
}
} }
if !foundDefault4 { if !foundDefault4 {
@@ -288,7 +255,7 @@ func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error
return nil return nil
} }
func (t *winTun) removeRoutes(routes []Route, rl *config.Reload) error { func (t *winTun) removeRoutes(routes []Route) error {
luid := winipcfg.LUID(t.tun.LUID()) luid := winipcfg.LUID(t.tun.LUID())
for _, r := range routes { for _, r := range routes {
@@ -298,12 +265,8 @@ func (t *winTun) removeRoutes(routes []Route, rl *config.Reload) error {
// See comment on luid.AddRoute // See comment on luid.AddRoute
err := luid.DeleteRoute(r.Cidr, unspecifiedNextHop(r.Cidr)) err := luid.DeleteRoute(r.Cidr, unspecifiedNextHop(r.Cidr))
if errors.Is(err, windows.ERROR_NOT_FOUND) { if err != nil {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
@@ -353,61 +316,6 @@ func (t *winTun) Close() error {
return t.tun.Close() return t.tun.Close()
} }
// systemRoutes maps each CIDR to the route that puts it in the system table. A CIDR's last entry decides, including
// one that says not to install it
func systemRoutes(routes []Route) map[netip.Prefix]Route {
out := make(map[netip.Prefix]Route, len(routes))
for _, r := range routes {
if len(r.Via) > 0 && r.Install {
out[r.Cidr] = r
} else {
delete(out, r.Cidr)
}
}
return out
}
// findRemovedSystemRoutes is the routes to delete from the system table. Windows deletes by destination and next hop,
// so only a CIDR nothing installs anymore goes, a changed metric is set on the route in place by addRoute
func findRemovedSystemRoutes(newRoutes, oldRoutes []Route) []Route {
keep := systemRoutes(newRoutes)
var removed []Route
for _, r := range systemRoutes(oldRoutes) {
if _, ok := keep[r.Cidr]; !ok {
removed = append(removed, r)
}
}
return removed
}
type routeResult int
const (
routeAdded routeResult = iota
routeMetricUpdated
routeAlreadyInstalled
)
// addRoute installs r on the interface. A reload re-adds every route to catch any the system dropped, and Windows
// refuses one it has with the same destination and next hop whatever its metric. One that is there with another
// metric, ours from before a metric change or someone else's, is brought to ours in place
func addRoute(luid winipcfg.LUID, r Route) (routeResult, error) {
nextHop := unspecifiedNextHop(r.Cidr)
err := luid.AddRoute(r.Cidr, nextHop, uint32(r.Metric))
if !errors.Is(err, windows.ERROR_OBJECT_ALREADY_EXISTS) {
return routeAdded, err
}
row, err := luid.Route(r.Cidr, nextHop)
if err != nil {
return routeAlreadyInstalled, err
}
if row.Metric != uint32(r.Metric) {
row.Metric = uint32(r.Metric)
return routeMetricUpdated, row.Set()
}
return routeAlreadyInstalled, nil
}
func unspecifiedNextHop(p netip.Prefix) netip.Addr { func unspecifiedNextHop(p netip.Prefix) netip.Addr {
if p.Addr().Is4() { if p.Addr().Is4() {
return netip.IPv4Unspecified() return netip.IPv4Unspecified()
-209
View File
@@ -1,209 +0,0 @@
//go:build windows && !e2e_testing
package overlay
import (
"bytes"
"fmt"
"net/netip"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/windows"
"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
)
type routeTest struct {
c *config.C
tun *winTun
luid winipcfg.LUID
log *bytes.Buffer
}
// newRouteTest makes a wintun device of its own, before installing tun.unsafe_routes from cfg
func newRouteTest(t *testing.T, cfg string) *routeTest {
t.Helper()
if !windows.GetCurrentProcessToken().IsElevated() {
t.Skip("creating a wintun device needs an elevated process")
}
if err := checkWinTunExists(); err != nil {
t.Skipf("wintun.dll is not where nebula looks for it: %v", err)
}
rt := &routeTest{log: &bytes.Buffer{}}
l := test.NewLoggerWithOutput(rt.log)
rt.c = config.NewC(l)
require.NoError(t, rt.c.LoadString(cfg))
var err error
rt.tun, err = newTun(rt.c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = rt.tun.Close() })
rt.luid = winipcfg.LUID(rt.tun.tun.LUID())
return rt
}
func (rt *routeTest) metric(t *testing.T, cidr string) uint32 {
t.Helper()
p := netip.MustParsePrefix(cidr)
row, err := rt.luid.Route(p, unspecifiedNextHop(p))
require.NoError(t, err)
return row.Metric
}
// reload applies cfg and expects nothing in it to fail
func (rt *routeTest) reload(t *testing.T, cfg string) {
t.Helper()
res := rt.c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed, "the reload reported a failure")
}
func unsafeRoutes(routes ...string) string {
var b strings.Builder
b.WriteString("tun:\n dev: nebroutetest\n unsafe_routes:\n")
// cidr[@metric][!], the ! for install: false
for _, r := range routes {
r, noInstall := strings.CutSuffix(r, "!")
cidr, metric, _ := strings.Cut(r, "@")
fmt.Fprintf(&b, " - route: %s\n via: 10.250.0.2\n", cidr)
if metric != "" {
fmt.Fprintf(&b, " metric: %s\n", metric)
}
if noInstall {
b.WriteString(" install: false\n")
}
}
return b.String()
}
// A reload that adds a route re-adds the ones already installed, Windows refusing those as already there isn't a failure
func TestWinTun_ReloadKeepsInstalledRoutes(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a))
require.NoError(t, rt.tun.Activate())
rt.reload(t, unsafeRoutes(a, b))
assert.NotContains(t, rt.log.String(), "Failed to add route")
rt.metric(t, a)
rt.metric(t, b)
})
}
// Windows calls a route the same by destination and next hop, so a changed metric is set on the installed route in
// place rather than deleted and added, which would leave a moment with no route
func TestWinTun_ReloadMetricChangeApplies(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
require.NoError(t, rt.tun.Activate())
require.Equal(t, uint32(100), rt.metric(t, a))
rt.reload(t, unsafeRoutes(a+"@200"))
assert.NotContains(t, rt.log.String(), "Failed to add route")
assert.NotContains(t, rt.log.String(), "Removed route")
assert.Equal(t, uint32(200), rt.metric(t, a))
})
}
// A route someone else put on the device with another metric is brought to ours, at startup too
func TestWinTun_ForeignRouteGetsOurMetric(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
p := netip.MustParsePrefix(a)
require.NoError(t, rt.luid.AddRoute(p, unspecifiedNextHop(p), 50))
require.NoError(t, rt.tun.Activate())
assert.Equal(t, uint32(100), rt.metric(t, a))
assert.Contains(t, rt.log.String(), "level=INFO msg=\"Updated route metric\"")
})
}
// A CIDR listed twice is installed once, by its last entry, and reloads leave it there instead of flipping between the
// two metrics
func TestWinTun_DuplicateCidrInstallsTheLastEntry(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200"))
require.NoError(t, rt.tun.Activate())
assert.Equal(t, uint32(200), rt.metric(t, a))
rt.reload(t, unsafeRoutes(a+"@100", a+"@200", b))
assert.Equal(t, uint32(200), rt.metric(t, a))
assert.NotContains(t, rt.log.String(), "Updated route metric")
assert.NotContains(t, rt.log.String(), "Failed to add route")
})
}
// Dropping the duplicate that wasn't installed leaves the installed route alone. Windows deletes by destination and
// next hop, deleting the dropped one would take the live route with it
func TestWinTun_DroppingAnUninstalledDuplicateKeepsTheRoute(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200"))
require.NoError(t, rt.tun.Activate())
rt.reload(t, unsafeRoutes(a+"@200"))
assert.NotContains(t, rt.log.String(), "Removed route")
assert.Equal(t, uint32(200), rt.metric(t, a))
})
}
// A reload ends where a fresh start with the same config would, a CIDR whose last entry says not to install it is
// removed
func TestWinTun_ReloadMatchesAFreshStart(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
cidr := netip.MustParsePrefix(a)
routes := unsafeRoutes(a+"@100", a+"@100!")
fresh := newRouteTest(t, routes)
require.NoError(t, fresh.tun.Activate())
_, err := fresh.luid.Route(cidr, unspecifiedNextHop(cidr))
require.Error(t, err, "a fresh start installed it")
require.NoError(t, fresh.tun.Close())
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
require.NoError(t, rt.tun.Activate())
rt.reload(t, routes)
_, err = rt.luid.Route(cidr, unspecifiedNextHop(cidr))
assert.Error(t, err, "the reload left it installed")
})
}
// Removing a route that is already gone is what we wanted, not a failure
func TestWinTun_ReloadRouteAlreadyGone(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a, b))
require.NoError(t, rt.tun.Activate())
p := netip.MustParsePrefix(b)
require.NoError(t, rt.luid.DeleteRoute(p, unspecifiedNextHop(p)))
rt.reload(t, unsafeRoutes(a))
})
}
// A network category we can't parse would fail the restart, so it fails the reload instead of asking for one
func TestWinTun_ReloadInvalidNetworkCategoryFails(t *testing.T) {
rt := newRouteTest(t, unsafeRoutes())
for range 2 {
res := rt.c.ReloadConfigString(unsafeRoutes() + " network_category: nope\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed, "a bad category that didn't change since the last reload is still bad")
assert.False(t, res.RestartRequired)
}
// Fixing it back to the running category needs nothing
res := rt.c.ReloadConfigString(unsafeRoutes() + " network_category: private\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired)
res = rt.c.ReloadConfigString(unsafeRoutes() + " network_category: public\n")
require.NoError(t, res.Err)
assert.Equal(t, []string{"tun.network_category"}, res.RestartKeys)
}
// forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4
func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) {
t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") })
t.Run("ipv6", func(t *testing.T) { f(t, "fd00:251:1::/64", "fd00:251:2::/64") })
}
+18 -70
View File
@@ -2,9 +2,7 @@ package nebula
import ( import (
"crypto/fips140" "crypto/fips140"
"crypto/sha256"
"encoding/binary" "encoding/binary"
"encoding/hex"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@@ -52,30 +50,15 @@ type CertState struct {
myVpnBroadcastAddrsTable *bart.Lite myVpnBroadcastAddrsTable *bart.Lite
} }
// errCertNeedsRestart marks a new cert that is refused on reload because only a restart can apply it
var errCertNeedsRestart = errors.New("a restart is required to use the new cert")
func NewPKIFromConfig(l *slog.Logger, c *config.C) (*PKI, error) { func NewPKIFromConfig(l *slog.Logger, c *config.C) (*PKI, error) {
pki := &PKI{l: l} pki := &PKI{l: l}
err := pki.reload(c, true, nil) err := pki.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
// Compare with the running cipher, not the last config. A restart with one we can't run would fail, so say that rErr := pki.reload(c, false)
// now rather than ask for one, and going back to the running one after a bad reload needs no restart
if cipher := c.GetString("cipher", "aes"); !validCipher(cipher) {
l.Error("Unknown cipher, a restart with it would fail", "cipher", cipher)
r.Failed()
} else if cipher == "chachapoly" && fipsOnly() {
// newCipherSuite refuses it at startup
l.Error("ChaChaPoly is not allowed in FIPS 140-only mode, a restart with it would fail", "cipher", cipher)
r.Failed()
} else if cipher != pki.getCertState().cipher {
r.RestartRequiredIfChanged("cipher")
}
rErr := pki.reload(c, false, r)
if rErr != nil { if rErr != nil {
util.LogWithContextIfNeeded("Failed to reload PKI from config", rErr, l) util.LogWithContextIfNeeded("Failed to reload PKI from config", rErr, l)
} }
@@ -92,29 +75,13 @@ func (p *PKI) getCertState() *CertState {
return p.cs.Load() return p.cs.Load()
} }
// r is nil on the initial load func (p *PKI) reload(c *config.C, initial bool) error {
func (p *PKI) reload(c *config.C, initial bool, r *config.Reload) error {
before := p.cs.Load()
err := p.reloadCerts(c, initial) err := p.reloadCerts(c, initial)
if err != nil { if err != nil {
if initial { if initial {
return err return err
} }
err.Log(p.l) err.Log(p.l)
if errors.Is(err, errCertNeedsRestart) {
r.RestartRequired()
} else {
r.Failed()
}
} else if !initial {
// A cert for a version we weren't running is accepted without comparing its networks to ours. The tun and the
// lighthouse keep the networks they started with
after := p.cs.Load()
if !slices.Equal(before.myVpnNetworks, after.myVpnNetworks) {
p.l.Warn("The new cert changes our networks, a restart is required to use them",
"old_networks", before.myVpnNetworks, "new_networks", after.myVpnNetworks)
r.RestartRequired()
}
} }
err = p.reloadCAPool(c) err = p.reloadCAPool(c)
@@ -123,27 +90,21 @@ func (p *PKI) reload(c *config.C, initial bool, r *config.Reload) error {
return err return err
} }
err.Log(p.l) err.Log(p.l)
r.Failed()
} }
return nil return nil
} }
// fipsOnly is fips140.Enforced, a variable so a test can run the FIPS 140-only reload check without GODEBUG
var fipsOnly = fips140.Enforced
// validCipher is true for the ciphers nebula can run. Each post-handshake CipherState in noiseutil hardcodes its own
// nonce endianness, so there's nothing to set up for either.
func validCipher(cipher string) bool {
return cipher == "aes" || cipher == "chachapoly"
}
func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError { func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
var cipher string var cipher string
var currentState *CertState var currentState *CertState
if initial { if initial {
cipher = c.GetString("cipher", "aes") cipher = c.GetString("cipher", "aes")
if !validCipher(cipher) { switch cipher {
case "aes", "chachapoly":
// Each post-handshake CipherState in noiseutil hardcodes its own
// nonce endianness now, so there's nothing to set up here.
default:
return util.NewContextualError( return util.NewContextualError(
"unknown cipher", "unknown cipher",
m{"cipher": cipher}, m{"cipher": cipher},
@@ -171,7 +132,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Networks in new cert was different from old", "Networks in new cert was different from old",
m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks(), "cert_version": cert.Version1}, m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks(), "cert_version": cert.Version1},
errCertNeedsRestart, nil,
) )
} }
@@ -179,7 +140,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Curve in new v1 cert was different from old", "Curve in new v1 cert was different from old",
m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve(), "cert_version": cert.Version1}, m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve(), "cert_version": cert.Version1},
errCertNeedsRestart, nil,
) )
} }
} }
@@ -194,7 +155,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Networks in new cert was different from old", "Networks in new cert was different from old",
m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks(), "cert_version": cert.Version2}, m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks(), "cert_version": cert.Version2},
errCertNeedsRestart, nil,
) )
} }
@@ -202,7 +163,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Curve in new cert was different from old", "Curve in new cert was different from old",
m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve(), "cert_version": cert.Version2}, m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve(), "cert_version": cert.Version2},
errCertNeedsRestart, nil,
) )
} }
} }
@@ -217,7 +178,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Removing a V2 cert is not permitted unless it has identical networks to the new V1 cert", "Removing a V2 cert is not permitted unless it has identical networks to the new V1 cert",
m{"new_v1_networks": newState.v1Cert.Networks(), "old_v2_networks": currentState.v2Cert.Networks()}, m{"new_v1_networks": newState.v1Cert.Networks(), "old_v2_networks": currentState.v2Cert.Networks()},
errCertNeedsRestart, nil,
) )
} }
} }
@@ -235,14 +196,12 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError { func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError {
caPool, err := loadCAPoolFromConfig(p.l, c) caPool, err := loadCAPoolFromConfig(p.l, c)
if caPool != nil {
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
}
if err != nil { if err != nil {
return util.NewContextualError("Failed to load ca from config", nil, err) return util.NewContextualError("Failed to load ca from config", nil, err)
} }
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
return nil return nil
} }
@@ -617,22 +576,11 @@ func loadCAPoolFromConfig(l *slog.Logger, c *config.C) (*cert.CAPool, error) {
bl := c.GetStringSlice("pki.blocklist", []string{}) bl := c.GetStringSlice("pki.blocklist", []string{})
if len(bl) > 0 { if len(bl) > 0 {
var bad []string for _, fp := range bl {
for i, raw := range bl {
// Fingerprints are compared as lowercase hex strings, a malformed entry would silently block nothing
fp := strings.ToLower(strings.TrimSpace(raw))
if b, err := hex.DecodeString(fp); err != nil || len(b) != sha256.Size {
bad = append(bad, fmt.Sprintf("entry %d %q", i+1, raw))
continue
}
caPool.BlocklistFingerprint(fp) caPool.BlocklistFingerprint(fp)
} }
l.Info("Blocklisted certificates", "fingerprintCount", len(bl)-len(bad)) l.Info("Blocklisted certificates", "fingerprintCount", len(bl))
if len(bad) > 0 {
// The valid entries are still returned so a failed reload blocks them, startup refuses the error
return caPool, fmt.Errorf("pki.blocklist entries are not hex certificate fingerprints: %s", strings.Join(bad, ", "))
}
} }
return caPool, nil return caPool, nil
-168
View File
@@ -1,17 +1,10 @@
package nebula package nebula
import ( import (
"crypto/fips140"
"fmt"
"net/netip"
"strings" "strings"
"testing" "testing"
"time"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -99,164 +92,3 @@ func TestNewCipherSuiteUnsupportedCurve(t *testing.T) {
assert.True(t, strings.HasPrefix(err.Error(), "unsupported curve:"), "got: %v", err) assert.True(t, strings.HasPrefix(err.Error(), "unsupported curve:"), "got: %v", err)
assert.Nil(t, cs) assert.Nil(t, cs)
} }
func TestPKIReloadRestartRequired(t *testing.T) {
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
pkiConfig := func(cipher, network string) string {
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
return fmt.Sprintf("cipher: %s\npki:\n ca: %q\n cert: %q\n key: %q\n", cipher, caPEM, certPEM, keyPEM)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig("aes", "10.1.0.2/24")))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
// A cert that changes our networks is refused and needs a restart, it is not a failure
res := c.ReloadConfigString(pkiConfig("aes", "10.1.0.3/24"))
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
assert.False(t, res.Failed)
assert.Equal(t, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, p.getCertState().myVpnNetworks)
// The cipher only applies at startup
c2 := config.NewC(test.NewLogger())
require.NoError(t, c2.LoadString(pkiConfig("aes", "10.1.0.2/24")))
p, err = NewPKIFromConfig(test.NewLogger(), c2)
require.NoError(t, err)
res = c2.ReloadConfigString(pkiConfig("chachapoly", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
assert.Equal(t, "aes", p.getCertState().cipher)
// A cipher we can't run would fail the restart, so it fails the reload instead of asking for one
c4 := config.NewC(test.NewLogger())
require.NoError(t, c4.LoadString(pkiConfig("aes", "10.1.0.2/24")))
_, err = NewPKIFromConfig(test.NewLogger(), c4)
require.NoError(t, err)
for range 2 {
res = c4.ReloadConfigString(pkiConfig("nope", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.True(t, res.Failed, "a bad cipher that didn't change since the last reload is still bad")
assert.False(t, res.RestartRequired)
}
// Fixing it back to the running cipher needs nothing
res = c4.ReloadConfigString(pkiConfig("aes", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired)
// A cert that keeps our networks applies without a restart
c3 := config.NewC(test.NewLogger())
require.NoError(t, c3.LoadString(pkiConfig("aes", "10.1.0.2/24")))
_, err = NewPKIFromConfig(test.NewLogger(), c3)
require.NoError(t, err)
res = c3.ReloadConfigString(pkiConfig("aes", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.False(t, res.RestartRequired)
// A cert that can't be read is a failure, and a restart wouldn't help
res = c3.ReloadConfigString(strings.Replace(pkiConfig("aes", "10.1.0.2/24"), "cert: \"", "cert: \"garbage", 1))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.False(t, res.RestartRequired)
}
// A cert for a version we weren't running is accepted without comparing networks, so switching to one with different
// networks needs a restart. One with the same networks doesn't
func TestPKIReloadNewVersionWithOtherNetworks(t *testing.T) {
ca1, _, ca1Key, ca1PEM := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
ca2, _, ca2Key, ca2PEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
caPEM := string(ca1PEM) + string(ca2PEM)
pkiConfig := func(v cert.Version, network string) string {
ca, caKey := ca1, ca1Key
if v == cert.Version2 {
ca, caKey = ca2, ca2Key
}
_, _, keyPEM, certPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
return fmt.Sprintf("pki:\n ca: %q\n cert: %q\n key: %q\n", caPEM, certPEM, keyPEM)
}
for _, tc := range []struct {
network string
restart bool
}{
{"10.9.0.7/24", true},
{"10.1.0.2/24", false},
} {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig(cert.Version1, "10.1.0.2/24")))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig(cert.Version2, tc.network))
require.NoError(t, res.Err)
assert.False(t, res.Failed, tc.network)
assert.Equal(t, tc.restart, res.RestartRequired, tc.network)
assert.Equal(t, []netip.Prefix{netip.MustParsePrefix(tc.network)}, p.getCertState().myVpnNetworks, "accepted either way, as before")
}
}
// FIPS 140-only mode refuses chachapoly at startup, so reloading to it fails rather than asks for a restart
func TestPKIReloadChachapolyInFIPSOnlyFails(t *testing.T) {
fipsOnly = func() bool { return true }
t.Cleanup(func() { fipsOnly = fips140.Enforced })
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_P256, time.Time{}, time.Time{}, nil, nil, nil)
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_P256, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil)
pkiConfig := func(cipher string) string {
return fmt.Sprintf("cipher: %s\npki:\n ca: %q\n cert: %q\n key: %q\n", cipher, caPEM, certPEM, keyPEM)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig("aes")))
_, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig("chachapoly"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.False(t, res.RestartRequired)
}
func TestLoadCAPoolBlocklist(t *testing.T) {
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
crt, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil)
fp, err := crt.Fingerprint()
require.NoError(t, err)
load := func(entry string) (*cert.CAPool, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(fmt.Sprintf("pki:\n ca: %q\n blocklist:\n - %q\n", caPEM, entry)))
return loadCAPoolFromConfig(test.NewLogger(), c)
}
for _, entry := range []string{fp, strings.ToUpper(fp), " " + fp + "\t"} {
pool, err := load(entry)
require.NoError(t, err, entry)
assert.True(t, pool.IsBlocklisted(fp), entry)
}
for _, entry := range []string{fp[1:], fp + "00", "sha256:" + fp, "zz" + fp[2:], ""} {
_, err := load(entry)
require.Error(t, err, entry)
}
// A bad entry on reload fails it, the valid entries are still blocked
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "me", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.1/24")}, nil, nil)
pkiConfig := func(entry string) string {
return fmt.Sprintf("pki:\n ca: %q\n cert: %q\n key: %q\n blocklist:\n - %q\n", caPEM, certPEM, keyPEM, entry)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig(fp)))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
other, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "other", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.3/24")}, nil, nil)
otherFp, err := other.Fingerprint()
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig(fp) + fmt.Sprintf(" - %q\n - %q\n", otherFp, fp[1:]))
assert.True(t, res.Failed)
assert.True(t, p.GetCAPool().IsBlocklisted(fp))
assert.True(t, p.GetCAPool().IsBlocklisted(otherFp))
}
+1 -2
View File
@@ -67,8 +67,7 @@ func NewPunchyFromConfig(l *slog.Logger, c *config.C, punchConn udp.Conn) *Punch
} }
p.reload(c, true) p.reload(c, true)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("stats.lighthouse_metrics")
p.reload(c, false) p.reload(c, false)
}) })
+6 -16
View File
@@ -76,7 +76,7 @@ punchy:
punch: true punch: true
delay: 10m delay: 10m
respond: true respond: true
`).Err) `))
p.reload(c, false) p.reload(c, false)
assert.True(t, p.punch.Load()) assert.True(t, p.punch.Load())
assert.Equal(t, newDelay, time.Duration(p.delay.Load())) assert.Equal(t, newDelay, time.Duration(p.delay.Load()))
@@ -170,7 +170,7 @@ func TestPunchy_LogFormat_ReloadPunch(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {punch: true}`).Err) require.NoError(t, c.ReloadConfigString(`punchy: {punch: true}`))
entry := findEntry(t, hook.entries, "punchy.punch changed") entry := findEntry(t, hook.entries, "punchy.punch changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -184,7 +184,7 @@ func TestPunchy_LogFormat_ReloadRespond(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {respond: true}`).Err) require.NoError(t, c.ReloadConfigString(`punchy: {respond: true}`))
entry := findEntry(t, hook.entries, "punchy.respond changed") entry := findEntry(t, hook.entries, "punchy.respond changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -198,7 +198,7 @@ func TestPunchy_LogFormat_ReloadDelay(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {delay: 10s}`).Err) require.NoError(t, c.ReloadConfigString(`punchy: {delay: 10s}`))
entry := findEntry(t, hook.entries, "punchy.delay changed") entry := findEntry(t, hook.entries, "punchy.delay changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -212,7 +212,7 @@ func TestPunchy_LogFormat_ReloadTargetAllRemotes(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {target_all_remotes: true}`).Err) require.NoError(t, c.ReloadConfigString(`punchy: {target_all_remotes: true}`))
entry := findEntry(t, hook.entries, "punchy.target_all_remotes changed") entry := findEntry(t, hook.entries, "punchy.target_all_remotes changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -226,19 +226,9 @@ func TestPunchy_LogFormat_ReloadRespondDelay(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {respond_delay: 15s}`).Err) require.NoError(t, c.ReloadConfigString(`punchy: {respond_delay: 15s}`))
entry := findEntry(t, hook.entries, "punchy.respond_delay changed") entry := findEntry(t, hook.entries, "punchy.respond_delay changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
assert.Equal(t, map[string]any{"respond_delay": 15 * time.Second}, entry.Attrs) assert.Equal(t, map[string]any{"respond_delay": 15 * time.Second}, entry.Attrs)
} }
func TestPunchyReloadRestartRequired(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("stats:\n lighthouse_metrics: false\n"))
NewPunchyFromConfig(test.NewLogger(), c, nil)
res := c.ReloadConfigString("stats:\n lighthouse_metrics: true\n")
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
}
-91
View File
@@ -1,91 +0,0 @@
package ratelimit
import (
"fmt"
"math"
"sync/atomic"
"time"
)
// Limiter is a token bucket: `burst` calls are allowed at once, then one more per `every`.
// It counts the calls it denies and hands that count to the next allowed call.
//
// The budget belongs to the Limiter instance.
// Every Allow that returns true spends from it, whatever the caller then does.
//
// Time follows Go's monotonic clock.
//
// Create one Limiter for each thing you want to limit,
// sharing one between unrelated events lets the noisy one starve the quiet one.
//
// Check Allow before doing the work being limited, so a denied call stays cheap:
//
// if denied, ok := probeLimiter.Allow(); ok {
// sendProbe()
// probesDenied.Inc(int64(denied))
// }
//
// For log lines use logging.RateLimited, which attaches the denied count to the logger as "suppressed".
type Limiter struct {
every int64
slack int64
tat atomic.Int64
denied atomic.Uint64
}
// maxSpan bounds `burst` * `every` to half of int64, leaving the other half for process uptime so the bucket
// math can't overflow.
const maxSpan = math.MaxInt64 / 2
// New returns a Limiter allowing `burst` calls at once and one more per `every`.
// A `burst` under 1 is 1, and an `every` of 0 or less never limits.
// It returns an error if `burst` * `every` is over about 146 years.
func New(burst int, every time.Duration) (*Limiter, error) {
l := &Limiter{}
if every > 0 {
burst = max(burst, 1)
if int64(burst) > maxSpan/int64(every) {
return nil, fmt.Errorf("ratelimit: burst %d * every %v is too long", burst, every)
}
l.every = int64(every)
l.slack = int64(burst-1) * l.every
}
return l, nil
}
var monoStart = time.Now()
func monoNow() int64 { return int64(time.Since(monoStart)) }
// Allow reports whether this call is allowed.
// If a token is available it spends it and returns the calls denied since the last allowed one, and true.
// Otherwise it returns 0 and false, and counts the denial.
// A nil Limiter allows everything.
func (l *Limiter) Allow() (uint64, bool) {
if l == nil {
return 0, true
}
return l.allow(monoNow())
}
// allow is Allow at a given monotonic time.
func (l *Limiter) allow(now int64) (uint64, bool) {
// Concurrent Allow calls read the clock before racing for the bucket, so one can see a time another already spent
// past and be denied. An unlimited Limiter must never deny, so it skips the bucket.
if l.every == 0 {
return 0, true
}
for {
tat := l.tat.Load()
next := max(tat, now)
if next-now > l.slack {
l.denied.Add(1)
return 0, false
}
if l.tat.CompareAndSwap(tat, next+l.every) {
return l.denied.Swap(0), true
}
}
}
-264
View File
@@ -1,264 +0,0 @@
package ratelimit
import (
"math"
"sync"
"sync/atomic"
"testing"
"time"
)
func mustLimiter(t testing.TB, burst int, every time.Duration) *Limiter {
t.Helper()
l, err := New(burst, every)
if err != nil {
t.Fatal(err)
}
return l
}
func TestLimiterBurstAndRefill(t *testing.T) {
l, now := mustLimiter(t, 3, time.Second), int64(0)
for i := 0; i < 3; i++ {
if n, ok := l.allow(now); !ok || n != 0 {
t.Fatalf("burst line %d: ok=%v denied=%d", i, ok, n)
}
}
for i := 0; i < 5; i++ {
if _, ok := l.allow(now); ok {
t.Fatalf("line %d past the burst logged", i)
}
}
now += int64(time.Second)
if n, ok := l.allow(now); !ok || n != 5 {
t.Fatalf("after one refill: ok=%v denied=%d, want true 5", ok, n)
}
if _, ok := l.allow(now); ok {
t.Fatal("one refill earned more than one line")
}
// A long quiet spell refills to burst, not past it, and the count reported above does not carry over.
now += int64(time.Hour)
for i, want := range []uint64{1, 0, 0} {
if n, ok := l.allow(now); !ok || n != want {
t.Fatalf("refilled line %d: ok=%v denied=%d, want true %d", i, ok, n, want)
}
}
if _, ok := l.allow(now); ok {
t.Fatal("bucket refilled past burst")
}
}
func TestLimiterBounds(t *testing.T) {
for _, every := range []time.Duration{0, -time.Second} {
l := mustLimiter(t, 1, every)
for i := 0; i < 100; i++ {
if _, ok := l.allow(0); !ok {
t.Fatalf("every=%v limited line %d", every, i)
}
}
}
l := mustLimiter(t, 0, time.Second)
if _, ok := l.allow(0); !ok {
t.Fatal("burst 0 should allow one line")
}
if _, ok := l.allow(0); ok {
t.Fatal("burst 0 should allow only one line")
}
}
func TestLimiterOverflow(t *testing.T) {
for _, c := range []struct {
burst int
every time.Duration
}{
{math.MaxInt, time.Second},
{1, math.MaxInt64},
{2, maxSpan/2 + 1},
} {
if _, err := New(c.burst, c.every); err == nil {
t.Fatalf("burst %d every %d should not fit", c.burst, c.every)
}
}
l := mustLimiter(t, 2, maxSpan/2)
for i := 0; i < 2; i++ {
if _, ok := l.allow(0); !ok {
t.Fatalf("the largest span should allow its burst, denied call %d", i)
}
}
if _, ok := l.allow(0); ok {
t.Fatal("the largest span allowed past its burst")
}
}
// Concurrent callers read the clock at slightly different times, which must not limit a Limiter that never limits.
func TestLimiterUnlimitedConcurrent(t *testing.T) {
l := mustLimiter(t, 1, 0)
var denied atomic.Int64
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 10000; j++ {
if _, ok := l.Allow(); !ok {
denied.Add(1)
}
}
}()
}
wg.Wait()
if got := denied.Load(); got != 0 {
t.Fatalf("denied %d lines", got)
}
}
func TestLimiterNil(t *testing.T) {
var l *Limiter
for i := 0; i < 10; i++ {
if _, ok := l.Allow(); !ok {
t.Fatal("nil limiter limited")
}
}
}
func TestLimiterConcurrent(t *testing.T) {
l := mustLimiter(t, 10, time.Hour)
var allowed atomic.Int64
var reported atomic.Uint64
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 1000; j++ {
if n, ok := l.allow(0); ok {
allowed.Add(1)
reported.Add(n)
}
}
}()
}
wg.Wait()
if got := allowed.Load(); got != 10 {
t.Fatalf("allowed %d lines, want the burst of 10", got)
}
// An allowed call that lands after some denials reports them, so the rest are still pending.
if got := reported.Load() + l.denied.Load(); got != 16*1000-10 {
t.Fatalf("denied %d, want %d", got, 16*1000-10)
}
}
func BenchmarkAllowAllowed(b *testing.B) {
l := mustLimiter(b, 1, time.Nanosecond)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
l.tat.Store(0)
l.Allow()
}
}
func BenchmarkAllowDenied(b *testing.B) {
l := mustLimiter(b, 1, time.Hour)
l.Allow()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
l.Allow()
}
}
func BenchmarkAllowDeniedParallel(b *testing.B) {
l := mustLimiter(b, 1, time.Hour)
l.Allow()
b.ReportAllocs()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
l.Allow()
}
})
}
// FuzzLimiter checks the bucket against its exact definition over arbitrary call timings.
// A call is denied if and only if some allowed call a would leave too many calls for the time since a:
// (allowed calls since a, plus this one) * every > burst * every + (now - a).
// Every denied call must also be reported exactly once, by a later allowed call or still pending.
func FuzzLimiter(f *testing.F) {
f.Add(uint8(3), uint16(1000), []byte{0, 0, 0, 0, 16, 0, 0, 200, 1, 1, 1, 15, 17, 0})
f.Add(uint8(1), uint16(1), []byte{0, 16, 15, 16, 0, 255, 0, 0})
f.Fuzz(func(t *testing.T, burst uint8, everyUs uint16, steps []byte) {
if len(steps) > 512 {
steps = steps[:512]
}
b := int64(burst%16) + 1
every := int64(everyUs%1000+1) * int64(time.Microsecond)
l := mustLimiter(t, int(b), time.Duration(every))
var now int64
var allowedAt []int64
var denied, reported uint64
for _, s := range steps {
// Each step advances by a sixteenth of every, so timings land on and around the refill.
now += int64(s) * every / 16
over := false
for i, a := range allowedAt {
if int64(len(allowedAt)-i+1)*every > b*every+now-a {
over = true
break
}
}
n, ok := l.allow(now)
if ok == over {
t.Fatalf("at %d with %d allowed before: ok=%v, the definition says %v", now, len(allowedAt), ok, !over)
}
if !ok {
denied++
continue
}
reported += n
allowedAt = append(allowedAt, now)
}
if got := reported + l.denied.Load(); got != denied {
t.Fatalf("reported %d denied calls, made %d", got, denied)
}
})
}
// Concurrent callers on the real clock must account for every denied call exactly once.
func TestLimiterDeniedConservedConcurrent(t *testing.T) {
l := mustLimiter(t, 4, time.Microsecond)
var denied, reported atomic.Uint64
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 20000; j++ {
if n, ok := l.Allow(); ok {
reported.Add(n)
} else {
denied.Add(1)
}
}
}()
}
wg.Wait()
if denied.Load() == 0 {
t.Fatal("nothing was denied, the test proves nothing")
}
if got := reported.Load() + l.denied.Load(); got != denied.Load() {
t.Fatalf("reported %d denied calls, made %d", got, denied.Load())
}
}
+1 -2
View File
@@ -28,11 +28,10 @@ func NewRelayManager(ctx context.Context, l *slog.Logger, hostmap *HostMap, c *c
hostmap: hostmap, hostmap: hostmap,
} }
rm.reload(c, true) rm.reload(c, true)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
err := rm.reload(c, false) err := rm.reload(c, false)
if err != nil { if err != nil {
rm.l.Error("Failed to reload relay_manager", "error", err) rm.l.Error("Failed to reload relay_manager", "error", err)
r.Failed()
} }
}) })
return rm return rm
+10 -27
View File
@@ -1,8 +1,8 @@
package service package service
import ( import (
"io"
"net" "net"
"sync"
) )
type tcpListener struct { type tcpListener struct {
@@ -10,40 +10,23 @@ type tcpListener struct {
s *Service s *Service
addr *net.TCPAddr addr *net.TCPAddr
accept chan net.Conn accept chan net.Conn
// closed is closed instead of accept so a handler blocked handing off a
// connection can't send on a closed channel.
closed chan struct{}
closeOnce sync.Once
} }
func (l *tcpListener) Accept() (net.Conn, error) { func (l *tcpListener) Accept() (net.Conn, error) {
// select picks randomly among ready cases, so without this a handler still conn, ok := <-l.accept
// waiting to hand off could win over closed after Close has returned. if !ok {
select { return nil, io.EOF
case <-l.closed:
return nil, net.ErrClosed
default:
}
select {
case conn := <-l.accept:
return conn, nil
case <-l.closed:
return nil, net.ErrClosed
} }
return conn, nil
} }
func (l *tcpListener) Close() error { func (l *tcpListener) Close() error {
l.closeOnce.Do(func() { l.s.mu.Lock()
// Close before leaving the map so a concurrent closeListeners that defer l.s.mu.Unlock()
// no longer sees this listener knows Accept is already released. delete(l.s.mu.listeners, uint16(l.addr.Port))
close(l.closed)
close(l.accept)
l.s.mu.Lock()
delete(l.s.mu.listeners, l.port)
l.s.mu.Unlock()
})
return nil return nil
} }
+10 -79
View File
@@ -32,18 +32,13 @@ import (
const nicID = 1 const nicID = 1
type Service struct { type Service struct {
eg *errgroup.Group
control *nebula.Control control *nebula.Control
ipstack *stack.Stack ipstack *stack.Stack
// done is closed once the forwarding goroutines have exited and the
// netstack is torn down, at which point err holds the first fatal error.
done chan struct{}
err error
mu struct { mu struct {
sync.Mutex sync.Mutex
closed bool
listeners map[uint16]*tcpListener listeners map[uint16]*tcpListener
} }
} }
@@ -70,6 +65,7 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
ctx := control.Context() ctx := control.Context()
eg, ctx := errgroup.WithContext(ctx) eg, ctx := errgroup.WithContext(ctx)
s := Service{ s := Service{
eg: eg,
control: control, control: control,
} }
s.mu.listeners = map[uint16]*tcpListener{} s.mu.listeners = map[uint16]*tcpListener{}
@@ -78,12 +74,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol}, NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6}, TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6},
}) })
// The teardown goroutine only owns the stack once New succeeds
defer func() {
if reterr != nil {
s.ipstack.Destroy()
}
}()
sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default
tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt) tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt)
if tcpipErr != nil { if tcpipErr != nil {
@@ -133,12 +123,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
// this will read exactly one packet // this will read exactly one packet
n, err := reader.Read(buf) n, err := reader.Read(buf)
if err != nil { if err != nil {
// An error after shutdown began is noise from the device pipes
// closing, anything else is fatal and must stop nebula too
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err return err
} }
packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{ packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{
@@ -146,8 +130,8 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
}) })
linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf) linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf)
if ctx.Err() != nil { if err := ctx.Err(); err != nil {
return nil return err
} }
} }
}) })
@@ -155,18 +139,13 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
for { for {
packet := linkEP.ReadContext(ctx) packet := linkEP.ReadContext(ctx)
if packet == nil { if packet == nil {
if ctx.Err() != nil { if err := ctx.Err(); err != nil {
return nil return err
} }
continue continue
} }
bufView := packet.ToView() bufView := packet.ToView()
if _, err := bufView.WriteTo(writer); err != nil { if _, err := bufView.WriteTo(writer); err != nil {
// Same shutdown noise handling as the inbound reader
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err return err
} }
bufView.Release() bufView.Release()
@@ -179,22 +158,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
return control.Wait() return control.Wait()
}) })
// Tear down here rather than in Close so it also happens when nebula stops
// on its own. The forwarding goroutines must exit before the stack they feed
// is destroyed.
s.done = make(chan struct{})
go func() {
s.err = eg.Wait()
s.closeListeners()
// Destroy aborts the endpoints that exist when it starts, then waits on
// every endpoint, so a Dial registering in between would hold Wait until
// its connect timed out. Without the NIC new dials fail route lookup.
// RemoveNIC also closes linkEP.
_ = s.ipstack.RemoveNIC(nicID)
s.ipstack.Destroy()
close(s.done)
}()
return &s, nil return &s, nil
} }
@@ -268,15 +231,11 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
s: s, s: s,
addr: addr, addr: addr,
accept: make(chan net.Conn), accept: make(chan net.Conn),
closed: make(chan struct{}),
} }
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
if s.mu.closed {
return nil, net.ErrClosed
}
if _, ok := s.mu.listeners[port]; ok { if _, ok := s.mu.listeners[port]; ok {
return nil, fmt.Errorf("already listening on port %d", port) return nil, fmt.Errorf("already listening on port %d", port)
} }
@@ -285,44 +244,22 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
return l, nil return l, nil
} }
// Wait blocks until the service has fully stopped, either via Close or a fatal
// error, and returns the first fatal error if there was one.
func (s *Service) Wait() error { func (s *Service) Wait() error {
<-s.done return s.eg.Wait()
return s.err
} }
// Close closes all listeners and stops nebula. Use Wait to block until
// shutdown has completed.
func (s *Service) Close() error { func (s *Service) Close() error {
s.closeListeners()
s.control.Stop() s.control.Stop()
return nil return nil
} }
// closeListeners unblocks pending Accept calls and rejects new listeners.
func (s *Service) closeListeners() {
// Every caller closes every listener rather than returning early once
// closed is set, so none returns while another is still closing them.
s.mu.Lock()
s.mu.closed = true
listeners := make([]*tcpListener, 0, len(s.mu.listeners))
for _, l := range s.mu.listeners {
listeners = append(listeners, l)
}
s.mu.Unlock()
for _, l := range listeners {
_ = l.Close()
}
}
func (s *Service) tcpHandler(r *tcp.ForwarderRequest) { func (s *Service) tcpHandler(r *tcp.ForwarderRequest) {
endpointID := r.ID() endpointID := r.ID()
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock()
l, ok := s.mu.listeners[endpointID.LocalPort] l, ok := s.mu.listeners[endpointID.LocalPort]
s.mu.Unlock()
if !ok { if !ok {
r.Complete(true) r.Complete(true)
return return
@@ -338,12 +275,6 @@ func (s *Service) tcpHandler(r *tcp.ForwarderRequest) {
r.Complete(false) r.Complete(false)
ep.SocketOptions().SetKeepAlive(true) ep.SocketOptions().SetKeepAlive(true)
// The handoff blocks until Accept is called, so it must not hold s.mu or a
// listener nobody accepts from stalls every other port.
conn := gonet.NewTCPConn(&wq, ep) conn := gonet.NewTCPConn(&wq, ep)
select { l.accept <- conn
case l.accept <- conn:
case <-l.closed:
_ = conn.Close()
}
} }
-298
View File
@@ -188,301 +188,3 @@ func TestService(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
} }
// A listener that is never accepted from must not stall connections to other
// ports, and closing it must not block.
func TestServiceUnacceptedListener(t *testing.T) {
lighthousePort := ephemeralUDPPort(t)
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"lighthouse": m{
"am_lighthouse": true,
},
"listen": m{
"host": "0.0.0.0",
"port": lighthousePort,
},
})
t.Cleanup(func() { _ = a.Close() })
b := newSimpleService(ca, caKey, "b", netip.MustParseAddr("10.0.0.2"), m{
"static_host_map": m{
"10.0.0.1": []string{fmt.Sprintf("127.0.0.1:%d", lighthousePort)},
},
"lighthouse": m{
"hosts": []string{"10.0.0.1"},
"interval": 1,
},
})
t.Cleanup(func() { _ = b.Close() })
idle, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
active, err := a.Listen("tcp", ":1235")
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
idleConn, err := b.DialContext(ctx, "tcp", "10.0.0.1:1234")
if err != nil {
t.Fatal(err)
}
defer idleConn.Close()
accepted := make(chan error, 1)
go func() {
conn, err := active.Accept()
if err == nil {
err = conn.Close()
}
accepted <- err
}()
ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
activeConn, err := b.DialContext(ctx, "tcp", "10.0.0.1:1235")
if err != nil {
t.Fatalf("dial to accepted port stalled behind idle listener: %v", err)
}
defer activeConn.Close()
select {
case err := <-accepted:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("accept stalled behind idle listener")
}
closed := make(chan error, 1)
go func() { closed <- idle.Close() }()
select {
case err := <-closed:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("closing idle listener blocked")
}
if _, err := idle.Accept(); !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from closed listener, got %v", err)
}
if err := idle.Close(); err != nil {
t.Fatalf("second close: %v", err)
}
}
// Accept must report net.ErrClosed after Close even while a handler is still
// trying to hand off a connection.
func TestListenerAcceptAfterClose(t *testing.T) {
s := &Service{}
s.mu.listeners = map[uint16]*tcpListener{}
l := &tcpListener{
port: 1234,
s: s,
accept: make(chan net.Conn),
closed: make(chan struct{}),
}
s.mu.listeners[l.port] = l
if err := l.Close(); err != nil {
t.Fatal(err)
}
// Keep a send pending so every Accept has a connection ready alongside closed.
stop := make(chan struct{})
done := make(chan struct{})
go func() {
defer close(done)
for {
server, client := net.Pipe()
_ = client.Close()
select {
case l.accept <- server:
case <-stop:
_ = server.Close()
return
}
}
}()
defer func() {
close(stop)
<-done
}()
for range 100 {
time.Sleep(time.Millisecond)
conn, err := l.Accept()
if conn != nil {
_ = conn.Close()
}
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("Accept after Close: got err %v, want net.ErrClosed", err)
}
}
}
// Close must release blocked Accept calls and let Wait report a clean shutdown.
func TestServiceClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
if err := a.Close(); err != nil {
t.Fatal(err)
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after Close")
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after Close")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
if err := a.Close(); err != nil {
t.Fatalf("second close: %v", err)
}
if _, err := a.Listen("tcp", ":1235"); !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Listen after Close, got %v", err)
}
}
// If nebula stops without Close being called, the service must still release
// Accept and tear down the netstack.
func TestServiceNebulaStopped(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
a.control.Stop()
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after nebula stopped")
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after nebula stopped")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
}
// Dials racing Close must not hold Wait until their connect times out.
func TestServiceDialDuringClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
stop := make(chan struct{})
var eg errgroup.Group
for range 8 {
eg.Go(func() error {
for {
select {
case <-stop:
return nil
default:
}
// 10.0.0.2 has no tunnel, so each dial sits in SYN-SENT until aborted
if c, err := a.Dial("tcp", "10.0.0.2:80"); err == nil {
_ = c.Close()
}
}
})
}
time.Sleep(50 * time.Millisecond)
if err := a.Close(); err != nil {
t.Fatal(err)
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait blocked behind a dial that raced Close")
}
close(stop)
dialersDone := make(chan error, 1)
go func() { dialersDone <- eg.Wait() }()
select {
case <-dialersDone:
case <-time.After(5 * time.Second):
t.Fatal("Dial still blocked after Wait")
}
}
+196 -68
View File
@@ -11,8 +11,11 @@ import (
"net" "net"
"net/netip" "net/netip"
"os" "os"
"slices" "path/filepath"
"runtime"
"runtime/pprof"
"sort" "sort"
"strconv"
"strings" "strings"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
@@ -55,28 +58,12 @@ type sshDeviceInfoFlags struct {
} }
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
if c.GetBool("sshd.enabled", false) { if c.GetBool("sshd.enabled", false) {
// Restarting the server drops every open session, the one that asked for this reload included. Only restart sshRun, err := configSSH(l, ssh, c)
// when it isn't running or its config changed. The host key file can change under the same config, new
// handshakes use the new key without a restart
if ssh.Running() && !slices.ContainsFunc(sshdServerKeys, c.HasChanged) {
key, err := loadSSHHostKey(c)
if err == nil {
err = ssh.SetHostKey(key)
}
if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err)
ssh.Stop()
r.Failed()
}
return
}
sshRun, err := configSSH(l, ssh, c, r)
if err != nil { if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err) l.Error("Failed to reconfigure the sshd", "error", err)
ssh.Stop() ssh.Stop()
r.Failed()
} }
if sshRun != nil { if sshRun != nil {
go sshRun() go sshRun()
@@ -87,31 +74,11 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
}) })
} }
// sshdServerKeys are what configSSH reads
var sshdServerKeys = []string{"sshd.enabled", "sshd.listen", "sshd.host_key", "sshd.trusted_cas", "sshd.authorized_users"}
// loadSSHHostKey returns sshd.host_key, read from its file when it names one.
func loadSSHHostKey(c *config.C) ([]byte, error) {
hostKeyPathOrKey := c.GetString("sshd.host_key", "")
if hostKeyPathOrKey == "" {
return nil, fmt.Errorf("sshd.host_key must be provided")
}
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
return []byte(hostKeyPathOrKey), nil
}
b, err := os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
return b, nil
}
// configSSH reads the ssh info out of the passed-in Config and // configSSH reads the ssh info out of the passed-in Config and
// updates the passed-in SSHServer. On success, it returns a function // updates the passed-in SSHServer. On success, it returns a function
// that callers may invoke to run the configured ssh server. On // that callers may invoke to run the configured ssh server. On
// failure, it returns nil, error. // failure, it returns nil, error.
// r is nil at startup, entries that are skipped with a warning fail it on a reload func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) {
func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reload) (func(), error) {
listen := c.GetString("sshd.listen", "") listen := c.GetString("sshd.listen", "")
if listen == "" { if listen == "" {
return nil, fmt.Errorf("sshd.listen must be provided") return nil, fmt.Errorf("sshd.listen must be provided")
@@ -125,9 +92,19 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
return nil, fmt.Errorf("sshd.listen can not use port 22") return nil, fmt.Errorf("sshd.listen can not use port 22")
} }
hostKeyBytes, err := loadSSHHostKey(c) hostKeyPathOrKey := c.GetString("sshd.host_key", "")
if err != nil { if hostKeyPathOrKey == "" {
return nil, err return nil, fmt.Errorf("sshd.host_key must be provided")
}
var hostKeyBytes []byte
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
hostKeyBytes = []byte(hostKeyPathOrKey)
} else {
hostKeyBytes, err = os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
} }
err = ssh.SetHostKey(hostKeyBytes) err = ssh.SetHostKey(hostKeyBytes)
@@ -144,7 +121,6 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
err := ssh.AddTrustedCA(caAuthorizedKey) err := ssh.AddTrustedCA(caAuthorizedKey)
if err != nil { if err != nil {
l.Warn("SSH CA had an error, ignoring", "error", err, "sshCA", caAuthorizedKey) l.Warn("SSH CA had an error, ignoring", "error", err, "sshCA", caAuthorizedKey)
r.Failed()
continue continue
} }
} }
@@ -156,14 +132,12 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
kDef, ok := rk.(map[string]any) kDef, ok := rk.(map[string]any)
if !ok { if !ok {
l.Warn("Authorized user had an error, ignoring", "sshKeyConfig", rk) l.Warn("Authorized user had an error, ignoring", "sshKeyConfig", rk)
r.Failed()
continue continue
} }
user, ok := kDef["user"].(string) user, ok := kDef["user"].(string)
if !ok { if !ok {
l.Warn("Authorized user is missing the user field", "sshKeyConfig", rk) l.Warn("Authorized user is missing the user field", "sshKeyConfig", rk)
r.Failed()
continue continue
} }
@@ -177,7 +151,6 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
"sshKeyConfig", rk, "sshKeyConfig", rk,
"sshKey", v, "sshKey", v,
) )
r.Failed()
continue continue
} }
@@ -189,7 +162,6 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
"sshKeyConfig", rk, "sshKeyConfig", rk,
"sshKey", subK, "sshKey", subK,
) )
r.Failed()
continue continue
} }
@@ -199,14 +171,12 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
"error", err, "error", err,
"sshKeyConfig", sk, "sshKeyConfig", sk,
) )
r.Failed()
continue continue
} }
} }
default: default:
l.Warn("Authorized user is missing the keys field or was not understood", "sshKeyConfig", rk) l.Warn("Authorized user is missing the keys field or was not understood", "sshKeyConfig", rk)
r.Failed()
} }
} }
} else { } else {
@@ -229,6 +199,12 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
} }
func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) { func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
// sandboxDir defaults to a dir in temp. The intention is that end user will
// create this dir as needed. Overriding this config value to "" allows
// writing to anywhere in the system.
defaultDir := filepath.Join(os.TempDir(), "nebula-debug")
sandboxDir := c.GetString("sshd.sandbox_dir", defaultDir)
ssh.RegisterCommand(&sshd.Command{ ssh.RegisterCommand(&sshd.Command{
Name: "list-hostmap", Name: "list-hostmap",
ShortDescription: "List all known previously connected hosts", ShortDescription: "List all known previously connected hosts",
@@ -280,7 +256,46 @@ func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interfa
Name: "reload", Name: "reload",
ShortDescription: "Reloads configuration from disk, same as sending HUP to the process", ShortDescription: "Reloads configuration from disk, same as sending HUP to the process",
Callback: func(fs any, a []string, w sshd.StringWriter) error { Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshReload(l, c, w) return sshReload(c, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "start-cpu-profile",
ShortDescription: "Starts a cpu profile and write output to the provided file, ex: `cpu-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshStartCpuProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "stop-cpu-profile",
ShortDescription: "Stops a cpu profile and writes output to the previously provided file",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
pprof.StopCPUProfile()
return w.WriteLine("If a CPU profile was running it is now stopped")
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "save-heap-profile",
ShortDescription: "Saves a heap profile to the provided path, ex: `heap-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetHeapProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "mutex-profile-fraction",
ShortDescription: "Gets or sets runtime.SetMutexProfileFraction",
Callback: sshMutexProfileFraction,
})
ssh.RegisterCommand(&sshd.Command{
Name: "save-mutex-profile",
ShortDescription: "Saves a mutex profile to the provided path, ex: `mutex-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetMutexProfile(sandboxDir, fs, a, w)
}, },
}) })
@@ -514,6 +529,58 @@ func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) er
return nil return nil
} }
// sshSanitizeFilePath validates that the given file path is within the sandbox directory.
// If sandboxDir is empty, the path is returned as-is for backwards compatibility.
func sshSanitizeFilePath(sandboxDir, filePath string) (string, error) {
if sandboxDir == "" {
return filePath, nil
}
// Clean and resolve the path relative to the sandbox directory
if !filepath.IsAbs(filePath) {
filePath = filepath.Join(sandboxDir, filePath)
}
cleaned := filepath.Clean(filePath)
// Ensure the resolved path is within the sandbox directory
cleanedSandbox := filepath.Clean(sandboxDir)
if cleaned == cleanedSandbox {
return "", fmt.Errorf("path %q resolves to the sandbox directory itself %q", filePath, sandboxDir)
}
if !strings.HasPrefix(cleaned, cleanedSandbox+string(filepath.Separator)) {
return "", fmt.Errorf("path %q is outside the sandbox directory %q", filePath, sandboxDir)
}
return cleaned, nil
}
func sshStartCpuProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
err := w.WriteLine("No path to write profile provided")
return err
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
}
err = pprof.StartCPUProfile(file)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to start cpu profile: %s", err))
return err
}
err = w.WriteLine(fmt.Sprintf("Started cpu profile, issue stop-cpu-profile to write the output to %s", a))
return err
}
func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
return w.WriteLine(fmt.Sprintf("%s", ifce.version)) return w.WriteLine(fmt.Sprintf("%s", ifce.version))
} }
@@ -617,14 +684,10 @@ func sshCreateTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) e
} }
} }
// Set the remote from the callback, which StartHandshake runs under the handshake manager lock hostInfo = ifce.handshakeManager.StartHandshake(vpnAddr, nil)
// before the hostinfo reaches the handshake loop or the static-host trigger, so the address is if addr.IsValid() {
// in the RemoteList before the first handshake is sent. hostInfo.SetRemote(addr)
ifce.handshakeManager.StartHandshake(vpnAddr, func(hh *HandshakeHostInfo) { }
if addr.IsValid() {
hh.hostinfo.SetRemote(addr)
}
})
return w.WriteLine("Created") return w.WriteLine("Created")
} }
@@ -666,6 +729,76 @@ func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) e
return w.WriteLine("Changed") return w.WriteLine("Changed")
} }
func sshGetHeapProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
}
err = pprof.WriteHeapProfile(file)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
return err
}
err = w.WriteLine(fmt.Sprintf("Mem profile created at %s", a))
return err
}
func sshMutexProfileFraction(fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
rate := runtime.SetMutexProfileFraction(-1)
return w.WriteLine(fmt.Sprintf("Current value: %d", rate))
}
newRate, err := strconv.Atoi(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("Invalid argument: %s", a[0]))
}
oldRate := runtime.SetMutexProfileFraction(newRate)
return w.WriteLine(fmt.Sprintf("New value: %d. Old value: %d", newRate, oldRate))
}
func sshGetMutexProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
return w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
}
defer file.Close()
mutexProfile := pprof.Lookup("mutex")
if mutexProfile == nil {
return w.WriteLine("Unable to get pprof.Lookup(\"mutex\")")
}
err = mutexProfile.WriteTo(file, 0)
if err != nil {
return w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
}
return w.WriteLine(fmt.Sprintf("Mutex profile created at %s", a))
}
func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error { func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
ctrl, ok := l.Handler().(interface { ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level GetLevel() slog.Level
@@ -918,13 +1051,8 @@ func sshDeviceInfo(ifce *Interface, fs any, w sshd.StringWriter) error {
} }
} }
// sshReload reloads the config and writes the result to the client. It reloads even if the client has gone away func sshReload(c *config.C, w sshd.StringWriter) error {
func sshReload(l *slog.Logger, c *config.C, w sshd.StringWriter) error { err := w.WriteLine("Reloading config")
writeErr := w.WriteLine("Reloading config") c.ReloadConfig()
res := c.ReloadConfig() return err
res.Log(l)
if err := w.WriteLine(res.String()); writeErr == nil {
writeErr = err
}
return writeErr
} }
-83
View File
@@ -1,83 +0,0 @@
package nebula
import (
"errors"
"io"
"os"
"path/filepath"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type lineWriter struct {
lines []string
err error
}
func (w *lineWriter) WriteLine(s string) error {
w.lines = append(w.lines, s)
return w.err
}
func (w *lineWriter) Write(s string) error { return w.WriteLine(s) }
func (w *lineWriter) WriteBytes(b []byte) error { return w.WriteLine(string(b)) }
func (w *lineWriter) GetWriter() io.Writer { return io.Discard }
func TestSSHReloadReportsTheResult(t *testing.T) {
dir := t.TempDir()
write := func(body string) {
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(body), 0o600))
}
write("a: 1\nb: 1")
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.Load(dir))
fail := false
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("b")
if fail {
r.Failed()
}
})
reload := func() []string {
w := &lineWriter{}
require.NoError(t, sshReload(l, c, w))
return w.lines
}
write("a: 2\nb: 1")
assert.Equal(t, []string{"Reloading config", "Config reload complete"}, reload())
fail = true
write("a: 3\nb: 1")
assert.Equal(t, []string{"Reloading config", "Config reload completed with errors, review the logs"}, reload())
fail = false
write("a: 3\nb: 2")
assert.Equal(t, []string{"Reloading config", "Config reload was completed but requires a restart to fully apply, keys [b]"}, reload())
write("a: [")
lines := reload()
require.Len(t, lines, 2)
assert.Contains(t, lines[1], "Config reload failed, the new config did not load")
assert.Contains(t, lines[1], dir, "names the config that did not load")
}
// A client that went away still gets the reload it asked for
func TestSSHReloadReloadsWhenTheClientIsGone(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte("a: 1"), 0o600))
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.Load(dir))
reloaded := false
c.RegisterReloadCallback(func(*config.C, *config.Reload) { reloaded = true })
gone := errors.New("client is gone")
require.ErrorIs(t, sshReload(l, c, &lineWriter{err: gone}), gone)
assert.True(t, reloaded)
}
+13 -48
View File
@@ -8,18 +8,14 @@ import (
"log/slog" "log/slog"
"net" "net"
"sync" "sync"
"sync/atomic"
"github.com/armon/go-radix" "github.com/armon/go-radix"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
) )
type SSHServer struct { type SSHServer struct {
// config is replaced whole when the host key changes, ssh.ServerConfig.AddHostKey only replaces a key of the same config *ssh.ServerConfig
// algorithm and appends any other l *slog.Logger
config atomic.Pointer[ssh.ServerConfig]
authenticate func(ssh.ConnMetadata, ssh.PublicKey) (*ssh.Permissions, error)
l *slog.Logger
certChecker *ssh.CertChecker certChecker *ssh.CertChecker
@@ -32,12 +28,7 @@ type SSHServer struct {
// List of available commands // List of available commands
helpCommand *Command helpCommand *Command
commands *radix.Tree commands *radix.Tree
listener atomic.Pointer[net.Listener] listener net.Listener
// runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned
runs atomic.Int32
// addr is where the newest Run listens, nil when none is
addr atomic.Pointer[net.Addr]
// ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even // ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even
// across reloads, since each Run derives a fresh child rather than reusing this one directly. // across reloads, since each Run derives a fresh child rather than reusing this one directly.
@@ -94,8 +85,10 @@ func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
}, },
} }
s.authenticate = cc.Authenticate s.config = &ssh.ServerConfig{
s.config.Store(s.newConfig()) PublicKeyCallback: cc.Authenticate,
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
s.RegisterCommand(&Command{ s.RegisterCommand(&Command{
Name: "help", Name: "help",
@@ -114,32 +107,10 @@ func (s *SSHServer) SetHostKey(hostPrivateKey []byte) error {
return fmt.Errorf("failed to parse private key: %s", err) return fmt.Errorf("failed to parse private key: %s", err)
} }
config := s.newConfig() s.config.AddHostKey(private)
config.AddHostKey(private)
s.config.Store(config)
return nil return nil
} }
func (s *SSHServer) newConfig() *ssh.ServerConfig {
return &ssh.ServerConfig{
PublicKeyCallback: s.authenticate,
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
}
// Addr is the address the newest Run listens on, nil when none is.
func (s *SSHServer) Addr() net.Addr {
if a := s.addr.Load(); a != nil {
return *a
}
return nil
}
// Running is true while a Run is listening.
func (s *SSHServer) Running() bool {
return s.runs.Load() > 0
}
func (s *SSHServer) ClearTrustedCAs() { func (s *SSHServer) ClearTrustedCAs() {
s.authLock.Lock() s.authLock.Lock()
s.trustedCAs = []ssh.PublicKey{} s.trustedCAs = []ssh.PublicKey{}
@@ -210,16 +181,10 @@ func (s *SSHServer) Run(addr string) error {
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's // this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
// listener before this run's watcher fires, so each run must close its own listener via the local // listener before this run's watcher fires, so each run must close its own listener via the local
// reference. // reference.
s.listener.Store(&listener) s.listener = listener
s.runs.Add(1)
bound := listener.Addr()
s.addr.Store(&bound)
runCtx, cancel := context.WithCancel(s.ctx) runCtx, cancel := context.WithCancel(s.ctx)
defer cancel() defer cancel()
// Deferred after cancel so they run first, a run on its way out no longer counts as listening
defer s.runs.Add(-1)
defer s.addr.CompareAndSwap(&bound, nil)
// Close the listener when this run's context is cancelled. That can come from the parent // Close the listener when this run's context is cancelled. That can come from the parent
// (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling // (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling
@@ -232,7 +197,7 @@ func (s *SSHServer) Run(addr string) error {
} }
}() }()
s.l.Info("SSH server is listening", "sshListener", bound) s.l.Info("SSH server is listening", "sshListener", addr)
// Run loops until there is an error // Run loops until there is an error
s.run(runCtx, listener) s.run(runCtx, listener)
@@ -261,7 +226,7 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
<-sessionContext.Done() <-sessionContext.Done()
c.Close() c.Close()
}() }()
conn, chans, reqs, err := ssh.NewServerConn(c, s.config.Load()) conn, chans, reqs, err := ssh.NewServerConn(c, s.config)
fp := "" fp := ""
if conn != nil { if conn != nil {
fp = conn.Permissions.Extensions["fp"] fp = conn.Permissions.Extensions["fp"]
@@ -299,8 +264,8 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
} }
func (s *SSHServer) Stop() { func (s *SSHServer) Stop() {
if l := s.listener.Load(); l != nil { if s.listener != nil {
if err := (*l).Close(); err != nil && !errors.Is(err, net.ErrClosed) { if err := s.listener.Close(); err != nil {
s.l.Warn("Failed to close the sshd listener", "error", err) s.l.Warn("Failed to close the sshd listener", "error", err)
} }
} }
-240
View File
@@ -1,240 +0,0 @@
package nebula
import (
"crypto"
"crypto/ed25519"
"crypto/rand"
"crypto/rsa"
"encoding/pem"
"fmt"
"net"
"os"
"path/filepath"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
type sshdReloadHarness struct {
c *config.C
server *sshd.SSHServer
keyFile string
signer ssh.Signer
listen string
users []string
extra string
}
func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
t.Helper()
l := test.NewLogger()
h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key"), listen: "127.0.0.1:0", users: []string{"test"}}
h.writeHostKey(t, newEd25519Key(t))
_, userKey, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
h.signer, err = ssh.NewSignerFromKey(userKey)
require.NoError(t, err)
h.c = config.NewC(l)
require.NoError(t, h.c.LoadString(h.config()))
h.server, err = sshd.NewSSHServer(t.Context(), l)
require.NoError(t, err)
wireSSHReload(l, h.server, h.c)
run, err := configSSH(l, h.server, h.c, nil)
require.NoError(t, err)
go run()
t.Cleanup(h.server.Stop)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
return h
}
func newEd25519Key(t *testing.T) crypto.Signer {
_, key, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
return key
}
// writeHostKey replaces the host key file and returns the public key a client should now see
func (h *sshdReloadHarness) writeHostKey(t *testing.T, key crypto.Signer) ssh.PublicKey {
t.Helper()
block, err := ssh.MarshalPrivateKey(key, "")
require.NoError(t, err)
require.NoError(t, os.WriteFile(h.keyFile, pem.EncodeToMemory(block), 0o600))
pub, err := ssh.NewPublicKey(key.Public())
require.NoError(t, err)
return pub
}
func (h *sshdReloadHarness) config() string {
users := ""
for _, u := range h.users {
users += fmt.Sprintf(" - user: %s\n keys: [%q]\n", u, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())))
}
return fmt.Sprintf("sshd:\n enabled: true\n listen: %s\n host_key: %s\n authorized_users:\n%s%s",
h.listen, h.keyFile, users, h.extra)
}
// dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys
// the client accepts, none means any
func (h *sshdReloadHarness) dial(hostKeyAlgorithms ...string) (*ssh.Client, ssh.PublicKey, error) {
addr := h.server.Addr()
if addr == nil {
return nil, nil, fmt.Errorf("not listening")
}
var hostKey ssh.PublicKey
c, err := ssh.Dial("tcp", addr.String(), &ssh.ClientConfig{
User: "test",
Auth: []ssh.AuthMethod{ssh.PublicKeys(h.signer)},
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
hostKey = key
return nil
},
HostKeyAlgorithms: hostKeyAlgorithms,
Timeout: time.Second,
})
return c, hostKey, err
}
func (h *sshdReloadHarness) connect(t *testing.T) (*ssh.Client, ssh.PublicKey) {
t.Helper()
var client *ssh.Client
var hostKey ssh.PublicKey
require.Eventually(t, func() bool {
var err error
client, hostKey, err = h.dial()
return err == nil
}, 5*time.Second, 20*time.Millisecond)
t.Cleanup(func() { _ = client.Close() })
return client, hostKey
}
// dropped is true when the server closed the client's connection within wait
func dropped(client *ssh.Client, wait time.Duration) bool {
done := make(chan struct{})
go func() {
_ = client.Wait()
close(done)
}()
select {
case <-done:
return true
case <-time.After(wait):
return false
}
}
// Restarting sshd drops every open session, the one that asked for the reload included. A reload that touches nothing
// sshd reads leaves it alone
func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
require.NoError(t, h.c.ReloadConfigString(h.config()+"punchy:\n punch: true\n").Err)
assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted")
}
// Every key the server reads restarts it when it changes
func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
cases := map[string]func(t *testing.T, h *sshdReloadHarness){
"listen": func(t *testing.T, h *sshdReloadHarness) { h.listen = "localhost:0" },
"authorized_users": func(t *testing.T, h *sshdReloadHarness) { h.users = append(h.users, "other") },
"trusted_cas": func(t *testing.T, h *sshdReloadHarness) { h.extra = " trusted_cas: []\n" },
"host_key": func(t *testing.T, h *sshdReloadHarness) {
h.keyFile = filepath.Join(t.TempDir(), "other_host_key")
h.writeHostKey(t, newEd25519Key(t))
},
}
for name, change := range cases {
t.Run(name, func(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
change(t, h)
require.NoError(t, h.c.ReloadConfigString(h.config()).Err)
assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied")
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
})
}
}
// The config can stay the same while the host key file it names changes. New connections get the new key, open
// sessions are left alone
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
want := h.writeHostKey(t, newEd25519Key(t))
require.NoError(t, h.c.ReloadConfigString(h.config()).Err)
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal(), "a new host key was not applied")
assert.False(t, dropped(client, 500*time.Millisecond), "a new host key restarted sshd")
}
// A host key of another algorithm replaces the old one rather than being offered next to it
func TestSSHDReload_HostKeyAlgorithmChange(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
want := h.writeHostKey(t, rsaKey)
require.NoError(t, h.c.ReloadConfigString(h.config()).Err)
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal())
_, _, err = h.dial(ssh.KeyAlgoED25519)
assert.Error(t, err, "the old ed25519 host key is still offered")
}
// A server that isn't running, say its port was busy, is tried again on every reload as before
func TestSSHDReload_NotRunningRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
h.server.Stop()
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
require.NoError(t, h.c.ReloadConfigString(h.config()).Err)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// A host key that no longer loads stops sshd, the same as a config that doesn't
func TestSSHDReload_BadHostKeyFileStops(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
require.NoError(t, os.WriteFile(h.keyFile, []byte("not a key"), 0o600))
assert.True(t, h.c.ReloadConfigString(h.config()).Failed)
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
}
// A trusted CA or authorized user that is skipped with a warning fails the reload, the valid ones are still used
func TestSSHDReload_SkippedEntriesFail(t *testing.T) {
cases := map[string]string{
"bad trusted ca": " trusted_cas: [nope]\n",
"user entry not a map": " - nope\n",
"user missing": " - keys: [nope]\n",
"bad key": " - user: other\n keys: nope\n",
"key in a list not a string": " - user: other\n keys: [5]\n",
"bad key in a list": " - user: other\n keys: [nope]\n",
"keys missing": " - user: other\n",
}
for name, extra := range cases {
t.Run(name, func(t *testing.T) {
h := newSSHDReloadHarness(t)
h.extra = extra
res := h.c.ReloadConfigString(h.config())
require.NoError(t, res.Err)
assert.True(t, res.Failed)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
})
}
}
+22 -47
View File
@@ -38,11 +38,6 @@ type statsServer struct {
runMu sync.Mutex runMu sync.Mutex
runCfg *statsConfig runCfg *statsConfig
run *statsRuntime // non-nil while a runtime is live run *statsRuntime // non-nil while a runtime is live
// Closed when the last runtime's capture loop has returned. Stop waits on it so nothing captures once it returns,
// and a reload's new runtime never overlaps the old one, go-metrics keeps its GC capture state in unguarded globals
loopDone chan struct{}
// See newStatsServerFromConfig, never changes after it
emitStats func()
} }
// statsRuntime is the live state owned by a single Start invocation. Start // statsRuntime is the live state owned by a single Start invocation. Start
@@ -87,22 +82,17 @@ type promConfig struct {
// //
// Start is safe to call unconditionally: it no-ops when stats are disabled. // Start is safe to call unconditionally: it no-ops when stats are disabled.
// The returned pointer is always non-nil, even on error. // The returned pointer is always non-nil, even on error.
// func newStatsServerFromConfig(ctx context.Context, l *slog.Logger, c *config.C, buildVersion string, configTest bool) (*statsServer, error) {
// emitStats, if set, updates metrics on every capture, ahead of the exporters so each export sees what it just set. It
// is fixed here, before the reload callback exists, so every runtime has it.
func newStatsServerFromConfig(ctx context.Context, l *slog.Logger, c *config.C, buildVersion string, configTest bool, emitStats func()) (*statsServer, error) {
s := &statsServer{ s := &statsServer{
l: l, l: l,
ctx: ctx, ctx: ctx,
buildVersion: buildVersion, buildVersion: buildVersion,
configTest: configTest, configTest: configTest,
emitStats: emitStats,
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
if err := s.reload(c, false); err != nil { if err := s.reload(c, false); err != nil {
s.l.Error("Failed to reload stats from config", "error", err) s.l.Error("Failed to reload stats from config", "error", err)
r.Failed()
} }
}) })
@@ -130,6 +120,7 @@ func (s *statsServer) reload(c *config.C, initial bool) error {
s.runMu.Lock() s.runMu.Lock()
sameCfg := s.runCfg != nil && *s.runCfg == newCfg sameCfg := s.runCfg != nil && *s.runCfg == newCfg
s.runCfg = &newCfg s.runCfg = &newCfg
running := s.run != nil
s.runMu.Unlock() s.runMu.Unlock()
s.enabled.Store(enabled) s.enabled.Store(enabled)
@@ -138,8 +129,9 @@ func (s *statsServer) reload(c *config.C, initial bool) error {
return nil return nil
} }
// Even with nothing running, a runtime whose listener failed can still be finishing its last pass if running {
s.Stop() s.Stop()
}
if enabled && !s.configTest { if enabled && !s.configTest {
go s.Start() go s.Start()
} }
@@ -161,19 +153,13 @@ func (s *statsServer) Start() {
return return
} }
cfg := *s.runCfg cfg := *s.runCfg
captureFns, listener := s.buildRuntime(cfg)
runCtx, cancel := context.WithCancel(s.ctx) runCtx, cancel := context.WithCancel(s.ctx)
captureFns, listener := s.buildRuntime(runCtx, cfg)
rt := &statsRuntime{cancel: cancel, listener: listener} rt := &statsRuntime{cancel: cancel, listener: listener}
s.run = rt s.run = rt
done := make(chan struct{})
s.loopDone = done
s.runMu.Unlock() s.runMu.Unlock()
// Prime before serving, so a scrape that lands before the first tick sees real values instead of zeros (issue #907) go captureStatsLoop(runCtx, cfg.interval, captureFns)
for _, fn := range captureFns {
fn()
}
go captureStatsLoop(runCtx, cfg.interval, captureFns, done)
cleanExit := true cleanExit := true
if listener == nil { if listener == nil {
@@ -232,32 +218,26 @@ func (s *statsServer) serveListener(listener *http.Server) bool {
// Stop tears down the active runtime, if any. Idempotent. // Stop tears down the active runtime, if any. Idempotent.
func (s *statsServer) Stop() { func (s *statsServer) Stop() {
// Held until the last pass is done, a Start that slipped in after s.run cleared would capture alongside it
s.runMu.Lock() s.runMu.Lock()
defer s.runMu.Unlock()
rt := s.run rt := s.run
s.run = nil s.run = nil
if rt != nil { s.runMu.Unlock()
rt.cancel() if rt == nil {
if rt.listener != nil { return
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
if err := rt.listener.Shutdown(shutdownCtx); err != nil {
s.l.Warn("Failed to shut down prometheus stats listener", "error", err)
}
cancel()
}
} }
// Nothing captures once Stop returns. The last runtime is cancelled by now, whoever cleared it, so this is at most rt.cancel()
// the pass it was in. Nothing in a pass takes runMu if rt.listener != nil {
if s.loopDone != nil { shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
<-s.loopDone if err := rt.listener.Shutdown(shutdownCtx); err != nil {
s.l.Warn("Failed to shut down prometheus stats listener", "error", err)
}
cancel()
} }
} }
// buildRuntime produces the capture functions and, for prometheus, an un-served // buildRuntime produces the capture functions and, for prometheus, an un-served
// http.Server from cfg. cfg has already been validated by loadStatsConfig. // http.Server from cfg. cfg has already been validated by loadStatsConfig.
// A graphite send in progress is abandoned when ctx is done. func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) {
func (s *statsServer) buildRuntime(ctx context.Context, cfg statsConfig) ([]func(), *http.Server) {
// rcrowley/go-metrics guards these registrations with a private sync.Once, // rcrowley/go-metrics guards these registrations with a private sync.Once,
// so subsequent reloads are no-ops. // so subsequent reloads are no-ops.
metrics.RegisterDebugGCStats(metrics.DefaultRegistry) metrics.RegisterDebugGCStats(metrics.DefaultRegistry)
@@ -267,9 +247,6 @@ func (s *statsServer) buildRuntime(ctx context.Context, cfg statsConfig) ([]func
func() { metrics.CaptureDebugGCStatsOnce(metrics.DefaultRegistry) }, func() { metrics.CaptureDebugGCStatsOnce(metrics.DefaultRegistry) },
func() { metrics.CaptureRuntimeMemStatsOnce(metrics.DefaultRegistry) }, func() { metrics.CaptureRuntimeMemStatsOnce(metrics.DefaultRegistry) },
} }
if s.emitStats != nil {
captureFns = append(captureFns, s.emitStats)
}
switch cfg.typ { switch cfg.typ {
case "graphite": case "graphite":
@@ -285,7 +262,7 @@ func (s *statsServer) buildRuntime(ctx context.Context, cfg statsConfig) ([]func
Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999}, Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999},
} }
captureFns = append(captureFns, func() { captureFns = append(captureFns, func() {
if err := graphiteOnce(ctx, gcfg, graphiteTimeout); err != nil && ctx.Err() == nil { if err := graphiteOnce(gcfg); err != nil {
s.l.Error("Graphite export failed", "error", err) s.l.Error("Graphite export failed", "error", err)
} }
}) })
@@ -332,10 +309,8 @@ func (s *statsServer) buildRuntime(ctx context.Context, cfg statsConfig) ([]func
return captureFns, nil return captureFns, nil
} }
// captureStatsLoop runs each fn on every tick of d until ctx is cancelled, then // captureStatsLoop runs each fn on every tick of d until ctx is cancelled.
// closes done. func captureStatsLoop(ctx context.Context, d time.Duration, fns []func()) {
func captureStatsLoop(ctx context.Context, d time.Duration, fns []func(), done chan struct{}) {
defer close(done)
t := time.NewTicker(d) t := time.NewTicker(d)
defer t.Stop() defer t.Stop()
for { for {
+30 -295
View File
@@ -2,18 +2,13 @@ package nebula
import ( import (
"context" "context"
"fmt"
"io" "io"
"log/slog" "log/slog"
"net" "net"
"net/http"
"net/http/httptest"
"strconv" "strconv"
"sync/atomic"
"testing" "testing"
"time" "time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -115,14 +110,15 @@ func TestStatsServer_reload_initial_prometheus(t *testing.T) {
} }
func TestStatsServer_Start_graphite_blocksUntilStop(t *testing.T) { func TestStatsServer_Start_graphite_blocksUntilStop(t *testing.T) {
h := newGraphiteHost(t, 0) sink := newGraphiteSink(t)
defer sink.Close()
s, c := newTestStatsServer(t) s, c := newTestStatsServer(t)
setStatsConfig(c, map[string]any{ setStatsConfig(c, map[string]any{
"type": "graphite", "type": "graphite",
"interval": "1s", "interval": "1s",
"protocol": "tcp", "protocol": "tcp",
"host": h.ln.Addr().String(), "host": sink.Addr(),
"prefix": "test", "prefix": "test",
}) })
require.NoError(t, s.reload(c, true)) require.NoError(t, s.reload(c, true))
@@ -375,19 +371,35 @@ func waitForListening(t *testing.T, addr string) {
}) })
} }
// stopAtCleanup returns the ctx to build *s with, and stops it when the test ends. Cancelling first turns away a Start // graphiteSink is a minimal TCP accept-and-discard server so graphiteOnce
// a reload spawned that hasn't run yet, then Stop waits out the last pass, so nothing captures into the next test // calls in tests don't spam error logs or wedge on connection refused.
func stopAtCleanup(t *testing.T, s **statsServer) context.Context { type graphiteSink struct {
ctx, cancel := context.WithCancel(context.Background()) ln net.Listener
t.Cleanup(func() {
cancel()
if *s != nil {
(*s).Stop()
}
})
return ctx
} }
func newGraphiteSink(t *testing.T) *graphiteSink {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
g := &graphiteSink{ln: ln}
go func() {
for {
conn, err := ln.Accept()
if err != nil {
return
}
go func(c net.Conn) {
_, _ = io.Copy(io.Discard, c)
_ = c.Close()
}(conn)
}
}()
return g
}
func (g *graphiteSink) Addr() string { return g.ln.Addr().String() }
func (g *graphiteSink) Close() { _ = g.ln.Close() }
func freeTCPPort(t *testing.T) string { func freeTCPPort(t *testing.T) string {
t.Helper() t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0") ln, err := net.Listen("tcp", "127.0.0.1:0")
@@ -396,280 +408,3 @@ func freeTCPPort(t *testing.T) string {
require.NoError(t, ln.Close()) require.NoError(t, ln.Close())
return strconv.Itoa(port) return strconv.Itoa(port)
} }
// Each export has to see what emitStats set in the same pass
func TestStatsServer_emitStatsRunsBeforeExport(t *testing.T) {
defer metrics.DefaultRegistry.Unregister("emitter.order.test")
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{
"type": "prometheus",
"interval": "1s",
"listen": "127.0.0.1:0",
"path": "/metrics",
})
gauge := metrics.GetOrRegisterGauge("emitter.order.test", nil)
s, err := newStatsServerFromConfig(t.Context(), l, c, "", false, func() { gauge.Update(42) })
require.NoError(t, err)
s.runMu.Lock()
fns, srv := s.buildRuntime(t.Context(), *s.runCfg)
s.runMu.Unlock()
require.NotNil(t, srv)
for _, fn := range fns {
fn()
}
rec := httptest.NewRecorder()
srv.Handler.ServeHTTP(rec, httptest.NewRequest("GET", "/metrics", nil))
assert.Contains(t, rec.Body.String(), "emitter_order_test 42")
}
// A scrape that lands the moment the listener is up already sees what emitStats set (issue #907)
func TestStatsServer_Start_primesBeforeServing(t *testing.T) {
defer metrics.DefaultRegistry.Unregister("prime.before.serving")
port := freeTCPPort(t)
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{
"type": "prometheus",
"interval": "1h",
"listen": "127.0.0.1:" + port,
"path": "/metrics",
})
gauge := metrics.GetOrRegisterGauge("prime.before.serving", nil)
// The pause would let a listener that came up before the prime serve the zero value
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() {
time.Sleep(200 * time.Millisecond)
gauge.Update(42)
})
require.NoError(t, err)
go s.Start()
waitForListening(t, "127.0.0.1:"+port)
resp, err := http.Get("http://127.0.0.1:" + port + "/metrics")
require.NoError(t, err)
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
assert.Contains(t, string(body), "prime_before_serving 42")
}
// Start captures once before the first tick, the interval here is an hour
func TestStatsServer_Start_primes(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer ln.Close()
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": ln.Addr().String()})
primed := make(chan struct{}, 1)
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err = newStatsServerFromConfig(ctx, l, c, "", false, func() {
select {
case primed <- struct{}{}:
default:
}
})
require.NoError(t, err)
go s.Start()
select {
case <-primed:
case <-time.After(5 * time.Second):
t.Fatal("nothing was captured before the first tick")
}
}
// Graphite has to ship what emitStats set in the same pass, not the pass before
func TestStatsServer_emitStatsRunsBeforeGraphiteExport(t *testing.T) {
defer metrics.DefaultRegistry.Unregister("emitter.graphite.test")
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer ln.Close()
received := make(chan string, 1)
go func() {
conn, err := ln.Accept()
if err != nil {
return
}
defer conn.Close()
b, _ := io.ReadAll(conn)
received <- string(b)
}()
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": ln.Addr().String()})
gauge := metrics.GetOrRegisterGauge("emitter.graphite.test", nil)
s, err := newStatsServerFromConfig(t.Context(), l, c, "", false, func() { gauge.Update(42) })
require.NoError(t, err)
s.runMu.Lock()
fns, _ := s.buildRuntime(t.Context(), *s.runCfg)
s.runMu.Unlock()
for _, fn := range fns {
fn()
}
select {
case body := <-received:
assert.Contains(t, body, ".emitter.graphite.test.value 42 ")
case <-time.After(5 * time.Second):
t.Fatal("graphite sent nothing")
}
}
// stats.interval reloads, and the runtime a reload starts still has emitStats
func TestStatsServer_reloadMovesTheInterval(t *testing.T) {
var passes atomic.Int64
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
// Nothing listens here, each export fails at once
host := "127.0.0.1:" + freeTCPPort(t)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": host})
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() { passes.Add(1) })
require.NoError(t, err)
go s.Start()
waitFor(t, func() bool { return passes.Load() == 1 })
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "20ms", "host": host})
require.NoError(t, s.reload(c, false))
waitFor(t, func() bool { return passes.Load() >= 6 })
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": host})
require.NoError(t, s.reload(c, false))
// The reload's Stop waited out the 20ms loop. The new runtime's prime may land either side of this read, it is the
// only pass left to come
stopped := passes.Load()
time.Sleep(300 * time.Millisecond)
assert.LessOrEqual(t, passes.Load()-stopped, int64(1), "the 20ms loop kept running after the reload")
}
// A reload's new runtime never captures while the old one is still in a pass, go-metrics keeps its GC capture state
// in unguarded globals
func TestStatsServer_reloadWaitsForTheLastPass(t *testing.T) {
var active, most atomic.Int64
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
host := "127.0.0.1:" + freeTCPPort(t)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "10ms", "host": host})
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() {
n := active.Add(1)
for {
m := most.Load()
if n <= m || most.CompareAndSwap(m, n) {
break
}
}
// Long enough that a reload almost always lands mid pass
time.Sleep(50 * time.Millisecond)
active.Add(-1)
})
require.NoError(t, err)
go s.Start()
for i := range 6 {
setStatsConfig(c, map[string]any{"type": "graphite", "interval": fmt.Sprintf("%dms", 11-i%2), "host": host})
require.NoError(t, s.reload(c, false))
time.Sleep(30 * time.Millisecond)
}
assert.Equal(t, int64(1), most.Load())
}
// Nothing captures once Stop returns, a pass in progress is waited out
func TestStatsServer_Stop_waitsForThePass(t *testing.T) {
var inPass atomic.Bool
entered := make(chan struct{}, 1)
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)})
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() {
inPass.Store(true)
select {
case entered <- struct{}{}:
default:
}
time.Sleep(100 * time.Millisecond)
inPass.Store(false)
})
require.NoError(t, err)
go s.Start()
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("never captured")
}
s.Stop()
assert.False(t, inPass.Load(), "Stop returned with a pass still running")
}
// A Start that arrives while Stop waits out the last pass doesn't capture alongside it
func TestStatsServer_Start_duringStopWaitsForThePass(t *testing.T) {
var active, most atomic.Int64
entered := make(chan struct{}, 1)
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)})
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() {
n := active.Add(1)
for {
m := most.Load()
if n <= m || most.CompareAndSwap(m, n) {
break
}
}
select {
case entered <- struct{}{}:
default:
}
time.Sleep(200 * time.Millisecond)
active.Add(-1)
})
require.NoError(t, err)
go s.Start()
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("never captured")
}
stopped := make(chan struct{})
go func() {
s.Stop()
close(stopped)
}()
// Stop holds runMu for its whole wait, the next Start has to arrive after it took it
waitFor(t, func() bool {
if s.runMu.TryLock() {
s.runMu.Unlock()
return false
}
return true
})
go s.Start()
select {
case <-stopped:
case <-time.After(5 * time.Second):
t.Fatal("Stop did not return")
}
waitFor(t, func() bool { return len(entered) == 1 })
assert.Equal(t, int64(1), most.Load(), "a Start captured alongside the pass Stop was waiting out")
}
+3 -4
View File
@@ -41,10 +41,9 @@ type Conn interface {
// //
// Returns the number of packets successfully written. A destination the kernel rejects costs only // Returns the number of packets successfully written. A destination the kernel rejects costs only
// its own packet, so a short count means some peers were undeliverable, not that the batch failed. // its own packet, so a short count means some peers were undeliverable, not that the batch failed.
// Safe for concurrent use on the same Conn. // Not safe for concurrent use on the same Conn.
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
// ReloadConfig applies the listen settings that can change, r is nil at startup ReloadConfig(c *config.C)
ReloadConfig(c *config.C, r *config.Reload)
SupportsMultipleReaders() bool SupportsMultipleReaders() bool
Close() error Close() error
} }
@@ -69,7 +68,7 @@ func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) { func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
return len(bufs), nil return len(bufs), nil
} }
func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) { func (NoopConn) ReloadConfig(_ *config.C) {
return return
} }
func (NoopConn) Close() error { func (NoopConn) Close() error {
-6
View File
@@ -24,12 +24,6 @@ type NetworkChangeMonitor struct {
// NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start // NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start
// is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it. // is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it.
func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor { func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor {
if watchesNetworkChanges {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("listen.rebind_on_network_change")
})
}
return &NetworkChangeMonitor{ return &NetworkChangeMonitor{
l: l, l: l,
ctx: ctx, ctx: ctx,
-3
View File
@@ -13,9 +13,6 @@ import (
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
// watchesNetworkChanges is true where Start does anything
const watchesNetworkChanges = true
const ( const (
// netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A // netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A
// single network change is never a single message, it is a burst: the link drops, addresses go away, new ones // single network change is never a single message, it is a burst: the link drops, addresses go away, new ones
-3
View File
@@ -7,9 +7,6 @@ import (
"log/slog" "log/slog"
) )
// watchesNetworkChanges is true where Start does anything
const watchesNetworkChanges = false
// watchNetworkChanges is a no-op outside of darwin. // watchNetworkChanges is a no-op outside of darwin.
// //
// Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket // Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket
-16
View File
@@ -37,19 +37,3 @@ func TestNetworkChangeMonitorNilRebindIsANoOp(t *testing.T) {
m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: true\n") m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: true\n")
m.Start(nil) m.Start(nil)
} }
// The monitor only reads listen.rebind_on_network_change when it is made
func TestNetworkChangeMonitorRestartKey(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("listen:\n rebind_on_network_change: true\n"))
NewNetworkChangeMonitor(context.Background(), l, c)
res := c.ReloadConfigString("listen:\n rebind_on_network_change: false\n")
require.NoError(t, res.Err)
if watchesNetworkChanges {
assert.Equal(t, []string{"listen.rebind_on_network_change"}, res.RestartKeys)
} else {
assert.Empty(t, res.RestartKeys, "the setting does nothing here")
}
}
+2 -4
View File
@@ -26,9 +26,7 @@ type bypassConn struct {
session *wfp.Session session *wfp.Session
} }
func (b *bypassConn) ReloadConfig(c *config.C, r *config.Reload) { func (b *bypassConn) ReloadConfig(c *config.C) {
// Only read on the first call, at startup
r.RestartRequiredIfChanged("listen.windows_bypass_wdf")
b.installOnce.Do(func() { b.installOnce.Do(func() {
if !c.GetBool("listen.windows_bypass_wdf", true) { if !c.GetBool("listen.windows_bypass_wdf", true) {
return return
@@ -47,7 +45,7 @@ func (b *bypassConn) ReloadConfig(c *config.C, r *config.Reload) {
"port", addr.Port()) "port", addr.Port())
b.session = s b.session = s
}) })
b.Conn.ReloadConfig(c, r) b.Conn.ReloadConfig(c)
} }
func (b *bypassConn) Close() error { func (b *bypassConn) Close() error {
Loaded 100 of 108 files, more files were not shown because too many files have changed in this diff. Show more