Compare commits

...
Author SHA1 Message Date
Ryan Huber d3e21a8fc8 relay: Batch forwarded packets on the relay host 2026-10-17 18:53:36 +00:00
Wade Simmons cda3710d5a fix error return in handleOutsideMessagePacket (#1970)
smoke-extra / freebsd-amd64 (push) Failing after 17s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 16s
smoke-extra / netbsd-amd64 (push) Failing after 16s
smoke-extra / openbsd-amd64 (push) Failing after 16s
smoke-extra / linux-386 (push) Failing after 17s
smoke / Run multi node smoke test (push) Failing after 1m39s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m18s
Build and test / Test linux-pkcs11 (push) Failing after 2m5s
Build and test / Test linux-boringcrypto (push) Failing after 3m1s
Build and test / Test linux-fips140 (push) Failing after 2m55s
Build and test / Cross-build linux-arm (push) Successful in 3m0s
Build and test / Cross-build linux-mips (push) Successful in 3m42s
Build and test / Cross-build linux-other (push) Successful in 3m5s
Build and test / Cross-build windows (push) Successful in 1m2s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m31s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m21s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
Introduced by #1961, we should still return early for invalid packets
2026-10-17 11:09:25 -04:00
John Maguire c26871fe08 service: Close listeners and tear down the netstack on shutdown (#1954) 2026-10-17 11:08:29 -04:00
Caleb Jasik 7daacc6a48 overlay/darwin: test tun framing and zero-alloc Read/Write (#1906)
smoke-extra / freebsd-amd64 (push) Failing after 16s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 16s
smoke-extra / netbsd-amd64 (push) Failing after 17s
smoke-extra / openbsd-amd64 (push) Failing after 17s
smoke-extra / linux-386 (push) Failing after 16s
smoke / Run multi node smoke test (push) Failing after 1m41s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m47s
Build and test / Test linux-pkcs11 (push) Failing after 2m6s
Build and test / Test linux-boringcrypto (push) Failing after 2m57s
Build and test / Test linux-fips140 (push) Failing after 2m53s
Build and test / Cross-build linux-arm (push) Successful in 3m7s
Build and test / Cross-build linux-mips (push) Successful in 3m55s
Build and test / Cross-build linux-other (push) Successful in 3m13s
Build and test / Cross-build windows (push) Successful in 1m4s
Build and test / Cross-build freebsd (push) Successful in 1m35s
Build and test / Cross-build netbsd (push) Successful in 1m35s
Build and test / Cross-build openbsd (push) Successful in 1m34s
Build and test / Cross-build mobile (push) Successful in 3m26s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-16 18:12:58 -05:00
John Maguire 428dad94b3 Reject v2 cert names and groups at signing the parser rejects (#1967)
smoke-extra / freebsd-amd64 (push) Failing after 18s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 18s
smoke-extra / netbsd-amd64 (push) Failing after 16s
smoke-extra / openbsd-amd64 (push) Failing after 17s
smoke-extra / linux-386 (push) Failing after 17s
smoke / Run multi node smoke test (push) Failing after 1m41s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m19s
Build and test / Test linux-pkcs11 (push) Failing after 2m5s
Build and test / Test linux-boringcrypto (push) Failing after 2m56s
Build and test / Test linux-fips140 (push) Failing after 2m51s
Build and test / Cross-build linux-arm (push) Successful in 3m0s
Build and test / Cross-build linux-mips (push) Successful in 3m43s
Build and test / Cross-build linux-other (push) Successful in 3m7s
Build and test / Cross-build windows (push) Successful in 1m2s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m33s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m23s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-16 18:05:51 -04:00
John Maguire 5bf666dd90 Treat uppercase blocked fingerprints the same as lowercase (#1964)
- Error for invalid on startup 
- Still block the valid blocklist entries when a reload has a bad one
2026-10-16 16:28:37 -04:00
John Maguire 49207da077 Remove the ssh debug profiling commands (#1965)
We plan to move this functionality into the debug socket.
2026-10-16 15:47:56 -04:00
Jack Doan bbc49669b7 fix noisy log line (#1961) 2026-10-16 12:08:55 -05:00
Nate Brown 915495d5f4 Fail the reload on a startup only value that would fail a restart, and stop asking for tun.dev restarts that change nothing (#1953)
smoke-extra / freebsd-amd64 (push) Failing after 25s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 13s
smoke-extra / netbsd-amd64 (push) Failing after 11s
smoke-extra / openbsd-amd64 (push) Failing after 12s
smoke-extra / linux-386 (push) Failing after 14s
smoke / Run multi node smoke test (push) Failing after 1m43s
Build and test / Static checks (push) Successful in 2m25s
Build and test / Test linux (push) Failing after 1m28s
Build and test / Test linux-pkcs11 (push) Failing after 2m5s
Build and test / Test linux-boringcrypto (push) Failing after 2m56s
Build and test / Test linux-fips140 (push) Failing after 3m3s
Build and test / Cross-build linux-arm (push) Successful in 3m14s
Build and test / Cross-build linux-mips (push) Successful in 3m48s
Build and test / Cross-build linux-other (push) Successful in 3m13s
Build and test / Cross-build windows (push) Successful in 1m3s
Build and test / Cross-build freebsd (push) Successful in 1m34s
Build and test / Cross-build netbsd (push) Successful in 1m36s
Build and test / Cross-build openbsd (push) Successful in 1m37s
Build and test / Cross-build mobile (push) Successful in 3m33s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
smoke-extra / Run windows smoke test (push) Canceled after 0s
2026-10-15 23:34:09 -05:00
Nate Brown a203d044bb Fail the reload when a config entry is skipped with a warning (#1952) 2026-10-15 23:22:47 -05:00
Nate Brown 316bf87780 Don't restart sshd when only sshd.sandbox_dir changes (#1948) 2026-10-15 22:12:12 -05:00
Nate Brown 1a0ac25db0 Report how a config reload went: failures, restarts required, and the keys that need one (#1935) 2026-10-15 15:36:39 -05:00
John Maguire 42597dda7d service: Don't hold listener lock while handing off a connection (#1939) 2026-10-15 15:14:39 -04:00
John Maguire afbdc87ed9 Update gvisor and bump to go1.27 (#1940)
gvisor's arm64 race-build compare-and-swap (pkg/sync/race_arm64.s)
loaded the new value into the register meant for the old one (R1
instead of R2), so it never succeeded. Sleepers could not commit to
parking and spun: under -race on arm64 every idle service netstack
busied about 14 cores, and TestService's static host lookup of
localhost missed its 250ms timeout and stalled for the 30s static_map
cadence. Fixed in gvisor 1b2946c4fa.

Go 1.27.2 is now the minimum because gvisor's go branch requires it.
golangci-lint is bumped to v2.14, the first release that can read
go1.27.2 export data.
2026-10-14 21:30:59 -04:00
Nate Brown 94372f4f51 Don't delete routes we never installed on darwin and the BSDs (#1933) 2026-10-13 16:34:38 -05:00
John Maguire 42280f04ce darwin: Assign host address to the utun and route vpn explicitly (#1931)
Tailscale with an exit node treats every interface address prefix as an
attached LAN and either claims it or excludes it, and configd then rewrites
the matching route away from nebula's utun. Interfaces holding a single
address are skipped, so assign /32 and /128 and add the network route
ourselves for both families.
2026-10-13 13:50:37 -04:00
Nate Brown 11c1917387 Warn when an unsafe_route is already routed through another interface on darwin (#1925) 2026-10-12 16:57:14 -05:00
Nate Brown 5ba7aff4b3 ssh: Only restart sshd on reload when its config or host key changed (#1923) 2026-10-12 16:56:51 -05:00
Ryan Huber 8a2de9b1b5 examples: set socket buffers and raise tun tx_queue (#1928)
* examples: set socket buffers and raise tun tx_queue

* examples: use the default tcp max window for socket buffers

* examples: note buffers are linux only, clarify tx_queue default

* tun: default tx_queue to 5000
2026-10-11 00:30:07 -04:00
79 changed files with 2868 additions and 520 deletions

No files matched your search

+3 -3
View File
@@ -14,7 +14,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -40,7 +40,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -83,7 +83,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
# GitHub holds ARNs, not credentials, and ARNs outlive a rotation # GitHub holds ARNs, not credentials, and ARNs outlive a rotation
+3 -3
View File
@@ -34,7 +34,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -66,7 +66,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -92,7 +92,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own # WSL2 + Ubuntu so the smoke can run a real linux peer with its own
+2 -2
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Smoke Docker - name: Smoke Docker
@@ -51,7 +51,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: build - name: build
+4 -4
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Install goimports - name: Install goimports
@@ -42,7 +42,7 @@ jobs:
- name: golangci-lint - name: golangci-lint
uses: golangci/golangci-lint-action@v9 uses: golangci/golangci-lint-action@v9
with: with:
version: v2.12 version: v2.14
test: test:
name: Test ${{ matrix.name }} name: Test ${{ matrix.name }}
@@ -87,7 +87,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -132,7 +132,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Build ${{ matrix.name }} - name: Build ${{ matrix.name }}
+7
View File
@@ -7,8 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
### Removed
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
`mutex-profile-fraction` commands, and `sshd.sandbox_dir`, which is now ignored if set.
### Fixed ### Fixed
- Signing a v2 certificate with an empty name, a name longer than 253 bytes, or an empty group now fails.
These certificates were signed successfully but could not be loaded.
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula - `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
+10 -1
View File
@@ -389,7 +389,16 @@ func (c *certificateV2) fromTBSCertificate(t *TBSCertificate) error {
} }
func (c *certificateV2) validate() error { func (c *certificateV2) validate() error {
// Empty names are allowed // Mirror the name and group rules in unmarshalDetails so signing fails instead of producing an unparseable certificate
if c.details.name == "" || len(c.details.name) > MaxNameLength {
return NewErrInvalidCertificateProperties("name must be between 1 and %d bytes", MaxNameLength)
}
for _, group := range c.details.groups {
if group == "" {
return NewErrInvalidCertificateProperties("groups may not be empty")
}
}
if len(c.publicKey) == 0 { if len(c.publicKey) == 0 {
return ErrInvalidPublicKey return ErrInvalidPublicKey
+45
View File
@@ -6,6 +6,7 @@ import (
"crypto/elliptic" "crypto/elliptic"
"crypto/rand" "crypto/rand"
"net/netip" "net/netip"
"strings"
"testing" "testing"
"time" "time"
@@ -135,3 +136,47 @@ func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
assert.True(t, normie) assert.True(t, normie)
} }
} }
func TestCertificateV2_SignRejectsUnparseable(t *testing.T) {
_, priv, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
var invalid *ErrInvalidCertificateProperties
newTBS := func() TBSCertificate {
return TBSCertificate{
Version: Version2,
Name: "testing",
Networks: []netip.Prefix{mustParsePrefixUnmapped("10.1.1.1/24")},
Groups: []string{"test-group1"},
NotBefore: time.Now().Add(-time.Minute),
NotAfter: time.Now().Add(time.Minute),
PublicKey: []byte("1234567890abcedfghij1234567890ab"),
Curve: Curve_CURVE25519,
IsCA: true,
}
}
tbs := newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength)
c, err := tbs.Sign(nil, Curve_CURVE25519, priv)
require.NoError(t, err)
b, err := c.Marshal()
require.NoError(t, err)
_, err = unmarshalCertificateV2(b, nil, Curve_CURVE25519)
require.NoError(t, err)
tbs = newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength+1)
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Name = ""
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Groups = []string{"test-group1", ""}
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
}
+2 -1
View File
@@ -85,9 +85,10 @@ func main() {
fmt.Printf("failed to apply logging config: %s", err) fmt.Printf("failed to apply logging config: %s", err)
os.Exit(1) os.Exit(1)
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
l.Error("Failed to reconfigure logger on reload", "error", err) l.Error("Failed to reconfigure logger on reload", "error", err)
r.Failed()
} }
}) })
+2 -1
View File
@@ -34,9 +34,10 @@ func (p *program) Start(s service.Service) error {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
return fmt.Errorf("failed to apply logging config: %s", err) return fmt.Errorf("failed to apply logging config: %s", err)
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
l.Error("Failed to reconfigure logger on reload", "error", err) l.Error("Failed to reconfigure logger on reload", "error", err)
r.Failed()
} }
}) })
+2 -1
View File
@@ -71,9 +71,10 @@ func main() {
fmt.Printf("failed to apply logging config: %s", err) fmt.Printf("failed to apply logging config: %s", err)
os.Exit(1) os.Exit(1)
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
l.Error("Failed to reconfigure logger on reload", "error", err) l.Error("Failed to reconfigure logger on reload", "error", err)
r.Failed()
} }
}) })
+31 -22
View File
@@ -26,9 +26,12 @@ type C struct {
files []string files []string
Settings map[string]any Settings map[string]any
oldSettings map[string]any oldSettings map[string]any
callbacks []func(*C) callbacks []func(*C, *Reload)
l *slog.Logger l *slog.Logger
reloadLock sync.Mutex reloadLock sync.Mutex
// Set by the first reload that needs a restart and never cleared. Guarded by reloadLock
needsRestart bool
} }
func NewC(l *slog.Logger) *C { func NewC(l *slog.Logger) *C {
@@ -72,8 +75,9 @@ func (c *C) LoadString(raw string) error {
// RegisterReloadCallback stores a function to be called when a config reload is triggered. The functions registered // RegisterReloadCallback stores a function to be called when a config reload is triggered. The functions registered
// here should decide if they need to make a change to the current process before making the change. HasChanged can be // here should decide if they need to make a change to the current process before making the change. HasChanged can be
// used to help decide if a change is necessary. // used to help decide if a change is necessary.
// These functions should return quickly or spawn their own go routine if they will take a while // These functions should return quickly or spawn their own go routine if they will take a while.
func (c *C) RegisterReloadCallback(f func(*C)) { // Use r to report anything that failed or needs a restart to apply.
func (c *C) RegisterReloadCallback(f func(c *C, r *Reload)) {
c.callbacks = append(c.callbacks, f) c.callbacks = append(c.callbacks, f)
} }
@@ -144,50 +148,55 @@ func (c *C) CatchHUP(ctx context.Context) {
return return
case <-ch: case <-ch:
c.l.Info("Caught HUP, reloading config") c.l.Info("Caught HUP, reloading config")
c.ReloadConfig() c.ReloadConfig().Log(c.l)
} }
} }
}() }()
} }
func (c *C) ReloadConfig() { // ReloadConfig reloads the config from the path provided to Load and runs every registered callback. A config that
// fails to load is logged here, like any other failure, and the result carries the error for the summary.
func (c *C) ReloadConfig() ReloadResult {
c.reloadLock.Lock() c.reloadLock.Lock()
defer c.reloadLock.Unlock() defer c.reloadLock.Unlock()
c.oldSettings = make(map[string]any) c.oldSettings = make(map[string]any)
maps.Copy(c.oldSettings, c.Settings) maps.Copy(c.oldSettings, c.Settings)
err := c.Load(c.path) if err := c.Load(c.path); err != nil {
if err != nil { c.l.Error("Error occurred while reloading config", "config_path", c.path, "error", err)
c.l.Error("Error occurred while reloading config", return ReloadResult{Err: err, RestartRequired: c.needsRestart}
"config_path", c.path,
"error", err,
)
return
} }
for _, v := range c.callbacks { return c.runCallbacks()
v(c)
}
} }
func (c *C) ReloadConfigString(raw string) error { // ReloadConfigString is ReloadConfig for a config held in memory rather than on disk.
func (c *C) ReloadConfigString(raw string) ReloadResult {
c.reloadLock.Lock() c.reloadLock.Lock()
defer c.reloadLock.Unlock() defer c.reloadLock.Unlock()
c.oldSettings = make(map[string]any) c.oldSettings = make(map[string]any)
maps.Copy(c.oldSettings, c.Settings) maps.Copy(c.oldSettings, c.Settings)
err := c.LoadString(raw) if err := c.LoadString(raw); err != nil {
if err != nil { c.l.Error("Error occurred while reloading config", "error", err)
return err return ReloadResult{Err: err, RestartRequired: c.needsRestart}
} }
return c.runCallbacks()
}
func (c *C) runCallbacks() ReloadResult {
r := &Reload{c: c}
for _, v := range c.callbacks { for _, v := range c.callbacks {
v(c) v(c, r)
} }
return nil if r.restart {
c.needsRestart = true
}
return ReloadResult{Failed: r.failed, RestartRequired: c.needsRestart, RestartKeys: r.keys}
} }
// GetString will get the string for k or return the default d if not found or invalid // GetString will get the string for k or return the default d if not found or invalid
@@ -391,7 +400,7 @@ func (c *C) parse() error {
var nm map[string]any var nm map[string]any
err = yaml.Unmarshal(b, &nm) err = yaml.Unmarshal(b, &nm)
if err != nil { if err != nil {
return err return fmt.Errorf("%s: %w", path, err)
} }
// We need to use WithAppendSlice so that firewall rules in separate // We need to use WithAppendSlice so that firewall rules in separate
+2 -2
View File
@@ -19,7 +19,7 @@ func TestConfig_Load(t *testing.T) {
// invalid yaml // invalid yaml
c := NewC(l) c := NewC(l)
os.WriteFile(filepath.Join(dir, "01.yaml"), []byte(" invalid yaml"), 0644) os.WriteFile(filepath.Join(dir, "01.yaml"), []byte(" invalid yaml"), 0644)
require.EqualError(t, c.Load(dir), "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into map[string]interface {}") require.EqualError(t, c.Load(dir), filepath.Join(dir, "01.yaml")+": yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into map[string]interface {}")
// simple multi config merge // simple multi config merge
c = NewC(l) c = NewC(l)
@@ -129,7 +129,7 @@ func TestConfig_ReloadConfig(t *testing.T) {
os.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n inner: ho"), 0644) os.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n inner: ho"), 0644)
c.RegisterReloadCallback(func(c *C) { c.RegisterReloadCallback(func(c *C, _ *Reload) {
done <- true done <- true
}) })
+98
View File
@@ -0,0 +1,98 @@
package config
import (
"context"
"fmt"
"log/slog"
"slices"
)
// ReloadResult describes how a reload went. Problems are logged where they happen, Log writes a one line summary.
type ReloadResult struct {
// Err is set when the new config could not be read or parsed, no callbacks ran
Err error
// Failed is true when a callback could not apply part of the new config
Failed bool
// RestartRequired is true once any reload has needed a restart, and stays true until the process restarts
RestartRequired bool
// RestartKeys are the keys this reload changed that only apply on a restart
RestartKeys []string
}
// Summary returns a one line summary of the reload and the level to log it at.
func (r ReloadResult) Summary() (slog.Level, string) {
switch {
case r.Err != nil && r.RestartRequired:
return slog.LevelError, "Config reload failed, the new config did not load and a restart is still required"
case r.Err != nil:
return slog.LevelError, "Config reload failed, the new config did not load"
case r.Failed && r.RestartRequired:
return slog.LevelError, "Config reload completed with errors and requires a restart to fully apply, review the logs"
case r.Failed:
return slog.LevelError, "Config reload completed with errors, review the logs"
case r.RestartRequired:
return slog.LevelWarn, "Config reload was completed but requires a restart to fully apply"
default:
return slog.LevelInfo, "Config reload complete"
}
}
// String is the summary with Err and RestartKeys included, for output other than the log.
func (r ReloadResult) String() string {
_, msg := r.Summary()
if r.Err != nil {
msg = fmt.Sprintf("%s: %v", msg, r.Err)
}
if len(r.RestartKeys) > 0 {
msg = fmt.Sprintf("%s, keys %v", msg, r.RestartKeys)
}
return msg
}
// Log writes Summary to l.
func (r ReloadResult) Log(l *slog.Logger) {
level, msg := r.Summary()
if len(r.RestartKeys) > 0 {
l.Log(context.Background(), level, msg, "keys", r.RestartKeys)
return
}
l.Log(context.Background(), level, msg)
}
// Reload is passed to each reload callback to report how applying the new config went.
type Reload struct {
c *C
failed bool
restart bool
// Changed keys that only apply on a restart, from every callback
keys []string
}
// Failed marks the reload as failed, the caller logs why. Safe to call on a nil Reload, which is what the initial load
// passes.
func (r *Reload) Failed() {
if r != nil {
r.failed = true
}
}
// RestartRequired marks the reload as needing a restart, the caller logs why. Safe to call on a nil Reload.
func (r *Reload) RestartRequired() {
if r != nil {
r.restart = true
}
}
// RestartRequiredIfChanged marks the reload as needing a restart if any of keys changed, and adds them to RestartKeys.
// Use it for keys that are only read at startup. Safe to call on a nil Reload.
func (r *Reload) RestartRequiredIfChanged(keys ...string) {
if r == nil {
return
}
for _, k := range keys {
if r.c.HasChanged(k) && !slices.Contains(r.keys, k) {
r.keys = append(r.keys, k)
r.restart = true
}
}
}
+108
View File
@@ -0,0 +1,108 @@
package config
import (
"errors"
"log/slog"
"testing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReloadConfigString(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1"))
fail := false
ran := 0
c.RegisterReloadCallback(func(c *C, r *Reload) {
ran++
if fail {
r.Failed()
}
})
assert.Equal(t, ReloadResult{}, c.ReloadConfigString("a: 2"))
fail = true
assert.Equal(t, ReloadResult{Failed: true}, c.ReloadConfigString("a: 3"))
assert.Equal(t, 2, ran)
// A config that fails to load runs no callbacks and returns the error
res := c.ReloadConfigString("a: [")
require.Error(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, 2, ran)
assert.Equal(t, 3, c.GetInt("a", 0))
}
func TestReloadRestartRequiredLatches(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1\nb: 1"))
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a", "b") })
// Nothing that needs a restart changed
assert.Equal(t, ReloadResult{}, c.ReloadConfigString("a: 1\nb: 1\nc: 1"))
assert.Equal(t, ReloadResult{RestartRequired: true, RestartKeys: []string{"a"}}, c.ReloadConfigString("a: 2\nb: 1\nc: 1"))
// The restart sticks through a reload with no changes, reverting the key and a config that fails to load. Only the
// reload that changed the key lists it
assert.Equal(t, ReloadResult{RestartRequired: true}, c.ReloadConfigString("a: 2\nb: 1\nc: 1"))
assert.Equal(t, ReloadResult{RestartRequired: true, RestartKeys: []string{"a"}}, c.ReloadConfigString("a: 1\nb: 1\nc: 1"))
res := c.ReloadConfigString("a: [")
require.Error(t, res.Err)
assert.True(t, res.RestartRequired)
}
// A key reported by more than one callback is listed once
func TestReloadRestartKeysDeduped(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1\nb: 1"))
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a") })
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a", "b") })
assert.Equal(t, []string{"a", "b"}, c.ReloadConfigString("a: 2\nb: 2").RestartKeys)
}
// A callback can require a restart without a key changing, a file the config points at may have
func TestReloadRestartRequiredWithoutAChange(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1"))
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequired() })
assert.Equal(t, ReloadResult{RestartRequired: true}, c.ReloadConfigString("a: 1"))
}
func TestReloadResultSummary(t *testing.T) {
cases := []struct {
res ReloadResult
level slog.Level
msg string
}{
{ReloadResult{}, slog.LevelInfo, "Config reload complete"},
{ReloadResult{RestartRequired: true}, slog.LevelWarn, "Config reload was completed but requires a restart to fully apply"},
{ReloadResult{Failed: true}, slog.LevelError, "Config reload completed with errors, review the logs"},
{ReloadResult{Failed: true, RestartRequired: true}, slog.LevelError, "Config reload completed with errors and requires a restart to fully apply, review the logs"},
{ReloadResult{Err: errors.New("bad yaml")}, slog.LevelError, "Config reload failed, the new config did not load"},
{ReloadResult{Err: errors.New("bad yaml"), RestartRequired: true}, slog.LevelError, "Config reload failed, the new config did not load and a restart is still required"},
}
for _, tc := range cases {
level, msg := tc.res.Summary()
assert.Equal(t, tc.level, level, tc.msg)
assert.Equal(t, tc.msg, msg)
}
res := ReloadResult{RestartRequired: true, RestartKeys: []string{"a", "b"}}
assert.Equal(t, "Config reload was completed but requires a restart to fully apply, keys [a b]", res.String())
// String includes the error
assert.Equal(t, "Config reload failed, the new config did not load: bad yaml", ReloadResult{Err: errors.New("bad yaml")}.String())
}
// The initial load passes a nil Reload
func TestReloadNilIsSafe(t *testing.T) {
var r *Reload
r.Failed()
r.RestartRequired()
r.RestartRequiredIfChanged("a")
}
+2 -1
View File
@@ -57,7 +57,8 @@ func newConnectionManagerFromConfig(l *slog.Logger, c *config.C, hm *HostMap, p
} }
cm.reload(c, true) cm.reload(c, true)
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("timers.connection_alive_interval", "timers.pending_deletion_interval")
cm.reload(c, false) cm.reload(c, false)
}) })
+3 -3
View File
@@ -151,9 +151,9 @@ func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) { func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
return len(bufs), nil return len(bufs), nil
} }
func (c *fakeConn) ReloadConfig(_ *config.C) {} func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {}
func (c *fakeConn) SupportsMultipleReaders() bool { return true } func (c *fakeConn) SupportsMultipleReaders() bool { return true }
func (c *fakeConn) Close() error { c.closed = true; return nil } func (c *fakeConn) Close() error { c.closed = true; return nil }
type multiqueueDevice struct { type multiqueueDevice struct {
*fakeDevice *fakeDevice
+2 -1
View File
@@ -69,9 +69,10 @@ func newDnsServerFromConfig(ctx context.Context, l *slog.Logger, pki *PKI, hostM
ds.mux = dns.NewServeMux() ds.mux = dns.NewServeMux()
ds.mux.HandleFunc(".", ds.handleDnsRequest) ds.mux.HandleFunc(".", ds.handleDnsRequest)
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
if err := ds.reload(c, false); err != nil { if err := ds.reload(c, false); err != nil {
ds.l.Error("Failed to reload DNS responder from config", "error", err) ds.l.Error("Failed to reload DNS responder from config", "error", err)
r.Failed()
} }
}) })
+1 -1
View File
@@ -1513,7 +1513,7 @@ func TestLighthouseUpdateOnReload(t *testing.T) {
// Reload the config. The lighthouse.hosts change triggers TriggerUpdate, // Reload the config. The lighthouse.hosts change triggers TriggerUpdate,
// which wakes the update worker. It calls SendUpdate, initiating a // which wakes the update worker. It calls SendUpdate, initiating a
// handshake to the new lighthouse and caching the HostUpdateNotification. // handshake to the new lighthouse and caching the HostUpdateNotification.
require.NoError(t, myConfig.ReloadConfigString(string(newCfg))) require.NoError(t, myConfig.ReloadConfigString(string(newCfg)).Err)
// Route until the lighthouse receives the HostUpdateNotification. // Route until the lighthouse receives the HostUpdateNotification.
// This covers: handshake stage 1, stage 2, then the cached update. // This covers: handshake stage 1, stage 2, then the cached update.
+2 -2
View File
@@ -129,7 +129,7 @@ func TestCertUpgrade(t *testing.T) {
} }
r.Logf("reload new v2-only config") r.Logf("reload new v2-only config")
err = myC.ReloadConfigString(string(cb)) err = myC.ReloadConfigString(string(cb)).Err
assert.NoError(t, err) assert.NoError(t, err)
r.Log("yay, spin until their sees it") r.Log("yay, spin until their sees it")
waitStart := time.Now() waitStart := time.Now()
@@ -222,7 +222,7 @@ func TestCertDowngrade(t *testing.T) {
} }
r.Logf("reload new v1-only config") r.Logf("reload new v1-only config")
err = myC.ReloadConfigString(string(cb)) err = myC.ReloadConfigString(string(cb)).Err
assert.NoError(t, err) assert.NoError(t, err)
r.Log("yay, spin until their sees it") r.Log("yay, spin until their sees it")
waitStart := time.Now() waitStart := time.Now()
+23 -17
View File
@@ -8,6 +8,7 @@ pki:
cert: /etc/nebula/host.crt cert: /etc/nebula/host.crt
key: /etc/nebula/host.key key: /etc/nebula/host.key
# blocklist is a list of certificate fingerprints that we will refuse to talk to # blocklist is a list of certificate fingerprints that we will refuse to talk to
# Entries are the hex fingerprints shown by `nebula-cert print`, an invalid entry fails startup and reload.
#blocklist: #blocklist:
# - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72 # - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72
# disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid. # disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid.
@@ -43,7 +44,7 @@ static_host_map:
lighthouse: lighthouse:
# am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes # am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
# you have configured to be lighthouses in your network # you have configured to be lighthouses in your network. Not reloadable.
am_lighthouse: false am_lighthouse: false
# serve_dns optionally starts a dns listener that responds to various queries and can even be # serve_dns optionally starts a dns listener that responds to various queries and can even be
# delegated to for resolution # delegated to for resolution
@@ -126,8 +127,9 @@ lighthouse:
# Port Nebula will be listening on. The default here is 4242. For a lighthouse node, the port should be defined, # Port Nebula will be listening on. The default here is 4242. For a lighthouse node, the port should be defined,
# however using port 0 will dynamically assign a port and is recommended for roaming nodes. # however using port 0 will dynamically assign a port and is recommended for roaming nodes.
listen: listen:
# To listen on only ipv4, use "0.0.0.0" # To listen on only ipv4, use "0.0.0.0". Not reloadable.
host: "::" host: "::"
# Not reloadable.
port: 4242 port: 4242
# Sets the max number of packets to pull from the kernel for each syscall (under systems that support recvmmsg) # Sets the max number of packets to pull from the kernel for each syscall (under systems that support recvmmsg)
# default is 64, does not support reload # default is 64, does not support reload
@@ -135,12 +137,13 @@ listen:
# superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the # superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the
# default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts. # default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts.
#batch: 64 #batch: 64
# Linux only. Reloadable, but removing a value leaves the socket at the old size until Nebula restarts.
# Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel # Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
# Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default) # Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
# Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide # Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide
# max, net.core.rmem_max and net.core.wmem_max # max, net.core.rmem_max and net.core.wmem_max
#read_buffer: 10485760 read_buffer: 3412512
#write_buffer: 10485760 write_buffer: 3412512
# On Windows only # On Windows only
# When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port. # When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port.
@@ -179,7 +182,7 @@ listen:
# Currently, this defaults to 1 which means we have 1 tun queue reader and 1 # Currently, this defaults to 1 which means we have 1 tun queue reader and 1
# UDP queue reader. Setting this above one will set IFF_MULTI_QUEUE on the tun # UDP queue reader. Setting this above one will set IFF_MULTI_QUEUE on the tun
# device and SO_REUSEPORT on the UDP socket to allow multiple queues. # device and SO_REUSEPORT on the UDP socket to allow multiple queues.
# This option is only supported on Linux. # This option is only supported on Linux. Not reloadable.
#routines: 1 #routines: 1
punchy: punchy:
@@ -203,6 +206,7 @@ punchy:
# Cipher allows you to choose between the available ciphers for your network. Options are chachapoly or aes # Cipher allows you to choose between the available ciphers for your network. Options are chachapoly or aes
# IMPORTANT: this value must be identical on ALL NODES/LIGHTHOUSES. We do not/will not support use of different ciphers simultaneously! # IMPORTANT: this value must be identical on ALL NODES/LIGHTHOUSES. We do not/will not support use of different ciphers simultaneously!
# Not reloadable.
#cipher: aes #cipher: aes
# Preferred ranges is used to define a hint about the local network ranges, which speeds up discovering the fastest # Preferred ranges is used to define a hint about the local network ranges, which speeds up discovering the fastest
@@ -229,12 +233,6 @@ punchy:
# Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access. # Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
#trusted_cas: #trusted_cas:
#- "ssh public key string" #- "ssh public key string"
# sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
# save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
# and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
# The directory is NOT automatically created.
# Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
#sandbox_dir: /var/tmp/nebula-debug
# EXPERIMENTAL: relay support for networks that can't establish direct connections. # EXPERIMENTAL: relay support for networks that can't establish direct connections.
relay: relay:
@@ -253,19 +251,25 @@ relay:
# Configure the private interface. Note: addr is baked into the nebula certificate # Configure the private interface. Note: addr is baked into the nebula certificate
tun: tun:
# When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root) # When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root)
# Not reloadable.
disabled: false disabled: false
# Name of the device. If not set, a default will be chosen by the OS. # Name of the device. If not set, a default will be chosen by the OS.
# For macOS: if set, must be in the form `utun[0-9]+`. # For macOS: if set, must be in the form `utun[0-9]+`.
# For NetBSD: Required to be set, must be in the form `tun[0-9]+` # For OpenBSD and NetBSD: if set, must be in the form `tun[0-9]+`, otherwise the first free one is used.
# Not reloadable.
dev: nebula1 dev: nebula1
# Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert # Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert
# Not reloadable.
drop_local_broadcast: false drop_local_broadcast: false
# Toggles forwarding of multicast packets # Toggles forwarding of multicast packets. Not reloadable.
drop_multicast: false drop_multicast: false
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500 # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 5000
tx_queue: 500 # Not reloadable.
tx_queue: 5000
# Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
# Anything above 8937, here or on a route below, is capped to 8937 with a warning # Anything above 8937, here or on a route below, is capped to 8937 with a warning
# Reloadable on Linux only. Where the tun is handed to nebula by a host app (iOS, Android, macOS network extensions)
# the host app owns the MTU.
mtu: 1300 mtu: 1300
# the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable. # the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable.
@@ -358,7 +362,7 @@ tun:
#use_system_route_table: false #use_system_route_table: false
# Buffer size for reading routes updates. 0 means default system buffer size. (/proc/sys/net/core/rmem_default). # Buffer size for reading routes updates. 0 means default system buffer size. (/proc/sys/net/core/rmem_default).
# If using massive routes updates, for example BGP, you may need to increase this value to avoid packet loss. # If using massive routes updates, for example BGP, you may need to increase this value to avoid packet loss.
# SO_RCVBUFFORCE is used to avoid having to raise the system wide max # SO_RCVBUFFORCE is used to avoid having to raise the system wide max. Not reloadable.
#use_system_route_table_buffer_size: 0 #use_system_route_table_buffer_size: 0
# Configure logging level # Configure logging level
@@ -408,14 +412,16 @@ logging:
# Handshakes are sent to all known addresses at each interval with a linear backoff, # Handshakes are sent to all known addresses at each interval with a linear backoff,
# Wait try_interval after the 1st attempt, 2 * try_interval after the 2nd, etc, until the handshake is older than timeout # Wait try_interval after the 1st attempt, 2 * try_interval after the 2nd, etc, until the handshake is older than timeout
# A 100ms interval with the default 10 retries will give a handshake 5.5 seconds to resolve before timing out # A 100ms interval with the default 10 retries will give a handshake 5.5 seconds to resolve before timing out
# Not reloadable.
#try_interval: 100ms #try_interval: 100ms
#retries: 20 #retries: 20
# query_buffer is the size of the buffer channel for querying lighthouses, drops are recorded in the lighthouse.query.dropped metric. # query_buffer is the size of the buffer channel for querying lighthouses, drops are recorded in the lighthouse.query.dropped metric.
# Not reloadable.
#query_buffer: 1024 #query_buffer: 1024
# trigger_buffer is the size of the buffer channel for quickly sending handshakes # trigger_buffer is the size of the buffer channel for quickly sending handshakes
# after receiving the response for lighthouse queries # after receiving the response for lighthouse queries. Not reloadable.
#trigger_buffer: 64 #trigger_buffer: 64
# Tunnel manager settings # Tunnel manager settings
+4 -3
View File
@@ -1,6 +1,6 @@
module github.com/slackhq/nebula module github.com/slackhq/nebula
go 1.26.0 go 1.27.2
require ( require (
dario.cat/mergo v1.0.2 dario.cat/mergo v1.0.2
@@ -33,7 +33,7 @@ require (
golang.zx2c4.com/wireguard/windows v1.0.1 golang.zx2c4.com/wireguard/windows v1.0.1
google.golang.org/protobuf v1.36.12 google.golang.org/protobuf v1.36.12
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980
) )
require ( require (
@@ -45,5 +45,6 @@ require (
github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect
github.com/vishvananda/netns v0.0.5 // indirect github.com/vishvananda/netns v0.0.5 // indirect
golang.org/x/time v0.5.0 // indirect golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect
golang.org/x/time v0.15.0 // indirect
) )
+6 -4
View File
@@ -157,6 +157,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -203,8 +205,8 @@ golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -243,5 +245,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g= gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 h1:ribyrTYQ/QYeUIs8x8bVLYwGyepa7RnHLqJzegT5YYk=
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU= gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980/go.mod h1:pK/dapF5Bf3KGW7/kd5CI5IC4lYh9FMFY1qETAIGLXk=
+7 -5
View File
@@ -316,9 +316,9 @@ type cachedPacketMetrics struct {
func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap { func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap {
hm := newHostMap(l) hm := newHostMap(l)
hm.reload(c, true) hm.reload(c, true, nil)
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
hm.reload(c, false) hm.reload(c, false, r)
}) })
l.Info("Main HostMap created", "preferredRanges", hm.GetPreferredRanges()) l.Info("Main HostMap created", "preferredRanges", hm.GetPreferredRanges())
@@ -337,7 +337,8 @@ func newHostMap(l *slog.Logger) *HostMap {
} }
} }
func (hm *HostMap) reload(c *config.C, initial bool) { // r is nil on the initial load
func (hm *HostMap) reload(c *config.C, initial bool, r *config.Reload) {
if initial || c.HasChanged("preferred_ranges") { if initial || c.HasChanged("preferred_ranges") {
var preferredRanges []netip.Prefix var preferredRanges []netip.Prefix
rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{}) rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{})
@@ -348,8 +349,9 @@ func (hm *HostMap) reload(c *config.C, initial bool) {
if err != nil { if err != nil {
hm.l.Warn("Failed to parse preferred ranges, ignoring", hm.l.Warn("Failed to parse preferred ranges, ignoring",
"error", err, "error", err,
"range", rawPreferredRanges, "range", rawPreferredRange,
) )
r.Failed()
continue continue
} }
+13
View File
@@ -447,3 +447,16 @@ func TestHostInfo_markOutOnly(t *testing.T) {
assert.True(t, h.markOut(6), "a relayed send must not consume the epoch edge") assert.True(t, h.markOut(6), "a relayed send must not consume the epoch edge")
assert.False(t, h.markOut(6)) assert.False(t, h.markOut(6))
} }
// A preferred range that doesn't parse is skipped and the rest are used, the reload says one was skipped
func TestHostMap_reloadBadPreferredRangeFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("preferred_ranges: [10.0.0.0/8]\n"))
hm := NewHostMapFromConfig(l, c)
res := c.ReloadConfigString("preferred_ranges: [10.0.0.0/8, nope]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, hm.GetPreferredRanges())
}
+10 -5
View File
@@ -384,6 +384,8 @@ type rxContext struct {
hostmapCache map[uint32]*HostInfo hostmapCache map[uint32]*HostInfo
lhh *LightHouseHandler lhh *LightHouseHandler
ctCache *firewall.ConntrackCacheTicker ctCache *firewall.ConntrackCacheTicker
relayBatch *batch.SendBatch
relayRemote netip.AddrPort
} }
func newRxContext(f *Interface, q int) *rxContext { func newRxContext(f *Interface, q int) *rxContext {
@@ -414,6 +416,7 @@ func (f *Interface) listenOut(i int) {
} }
flusher := func() { flusher := func() {
f.flushRelayBatch(rxc)
if err := f.batchers[i].Flush(); err != nil { if err := f.batchers[i].Flush(); err != nil {
f.l.Error("Failed to flush tun coalescer", "error", err) f.l.Error("Failed to flush tun coalescer", "error", err)
} }
@@ -509,10 +512,11 @@ func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) { func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
c.RegisterReloadCallback(f.reloadFirewall) c.RegisterReloadCallback(f.reloadFirewall)
c.RegisterReloadCallback(f.reloadSendRecvError) for _, reload := range []func(*config.C){
c.RegisterReloadCallback(f.reloadAcceptRecvError) f.reloadSendRecvError, f.reloadAcceptRecvError, f.reloadDisconnectInvalid, f.reloadMisc,
c.RegisterReloadCallback(f.reloadDisconnectInvalid) } {
c.RegisterReloadCallback(f.reloadMisc) c.RegisterReloadCallback(func(c *config.C, _ *config.Reload) { reload(c) })
}
for _, udpConn := range f.writers { for _, udpConn := range f.writers {
c.RegisterReloadCallback(udpConn.ReloadConfig) c.RegisterReloadCallback(udpConn.ReloadConfig)
@@ -529,7 +533,7 @@ func (f *Interface) reloadDisconnectInvalid(c *config.C) {
} }
} }
func (f *Interface) reloadFirewall(c *config.C) { func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
cs := f.pki.getCertState() cs := f.pki.getCertState()
curCert := cs.getCertificate(cert.Version2) curCert := cs.getCertificate(cert.Version2)
if curCert == nil { if curCert == nil {
@@ -548,6 +552,7 @@ func (f *Interface) reloadFirewall(c *config.C) {
fw, err := NewFirewallFromConfig(f.l, cs, c) fw, err := NewFirewallFromConfig(f.l, cs, c)
if err != nil { if err != nil {
f.l.Error("Error while creating firewall during reload", "error", err) f.l.Error("Error while creating firewall during reload", "error", err)
r.Failed()
return return
} }
+26 -4
View File
@@ -65,10 +65,10 @@ func TestReloadFirewall_CertUnsafeNetworksChanged(t *testing.T) {
pki.cs.Store(&CertState{v2Cert: c2, initiatingVersion: cert.Version2}) pki.cs.Store(&CertState{v2Cert: c2, initiatingVersion: cert.Version2})
// Reload with the same YAML so HasChanged("firewall") reports false. // Reload with the same YAML so HasChanged("firewall") reports false.
require.NoError(t, cfg.ReloadConfigString(rawYAML)) require.NoError(t, cfg.ReloadConfigString(rawYAML).Err)
require.False(t, cfg.HasChanged("firewall")) require.False(t, cfg.HasChanged("firewall"))
f.reloadFirewall(cfg) f.reloadFirewall(cfg, nil)
assert.NotSame(t, fw, f.firewall, "firewall pointer should have been replaced") assert.NotSame(t, fw, f.firewall, "firewall pointer should have been replaced")
assert.Equal(t, newUnsafe, f.firewall.unsafeNetworks) assert.Equal(t, newUnsafe, f.firewall.unsafeNetworks)
@@ -113,8 +113,30 @@ func TestReloadFirewall_NoChange(t *testing.T) {
l: l, l: l,
} }
require.NoError(t, cfg.ReloadConfigString(rawYAML)) require.NoError(t, cfg.ReloadConfigString(rawYAML).Err)
f.reloadFirewall(cfg) f.reloadFirewall(cfg, nil)
assert.Same(t, fw, f.firewall, "firewall should not have been replaced") assert.Same(t, fw, f.firewall, "firewall should not have been replaced")
} }
// Rules that don't parse keep the old firewall, and the reload says it could not apply them
func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
l := test.NewLogger()
pki := &PKI{}
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}}, initiatingVersion: cert.Version2})
rules := func(proto string) string {
return "firewall:\n outbound:\n - port: any\n proto: any\n host: any\n inbound:\n - port: any\n proto: " + proto + "\n host: any\n"
}
cfg := config.NewC(l)
require.NoError(t, cfg.LoadString(rules("any")))
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
require.NoError(t, err)
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
+10 -4
View File
@@ -138,18 +138,21 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
h.metrics = newLighthouseMetrics() h.metrics = newLighthouseMetrics()
} }
err := h.reload(c, true) err := h.reload(c, true, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
err := h.reload(c, false) r.RestartRequiredIfChanged("lighthouse.am_lighthouse", "listen.port", "handshakes.query_buffer", "stats.lighthouse_metrics")
err := h.reload(c, false, r)
switch v := err.(type) { switch v := err.(type) {
case *util.ContextualError: case *util.ContextualError:
v.Log(l) v.Log(l)
r.Failed()
case error: case error:
l.Error("failed to reload lighthouse", "error", err) l.Error("failed to reload lighthouse", "error", err)
r.Failed()
} }
}) })
@@ -190,7 +193,8 @@ func (lh *LightHouse) GetUpdateInterval() int64 {
return lh.interval.Load() return lh.interval.Load()
} }
func (lh *LightHouse) reload(c *config.C, initial bool) error { // r is nil on the initial load
func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error {
if initial || c.HasChanged("lighthouse.advertise_addrs") { if initial || c.HasChanged("lighthouse.advertise_addrs") {
rawAdvAddrs := c.GetStringSlice("lighthouse.advertise_addrs", []string{}) rawAdvAddrs := c.GetStringSlice("lighthouse.advertise_addrs", []string{})
advAddrs := make([]netip.AddrPort, 0) advAddrs := make([]netip.AddrPort, 0)
@@ -225,6 +229,7 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
"addr", rawAddr, "addr", rawAddr,
"entry", i+1, "entry", i+1,
) )
r.Failed()
continue continue
} }
@@ -383,6 +388,7 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
"relay", v, "relay", v,
"error", err, "error", err,
) )
r.Failed()
} else { } else {
lh.l.Info("Read relay from config", "relay", v) lh.l.Info("Read relay from config", "relay", v)
relaysForMe = append(relaysForMe, configRIP) relaysForMe = append(relaysForMe, configRIP)
+59 -14
View File
@@ -87,15 +87,15 @@ func TestReloadLighthouseInterval(t *testing.T) {
lh.ifce = &mockEncWriter{} lh.ifce = &mockEncWriter{}
// The first one routine is kicked off by main.go currently, lets make sure that one dies // The first one routine is kicked off by main.go currently, lets make sure that one dies
require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 5")) require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 5").Err)
assert.Equal(t, int64(5), lh.interval.Load()) assert.Equal(t, int64(5), lh.interval.Load())
// Subsequent calls are killed off by the LightHouse.Reload function // Subsequent calls are killed off by the LightHouse.Reload function
require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 10")) require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 10").Err)
assert.Equal(t, int64(10), lh.interval.Load()) assert.Equal(t, int64(10), lh.interval.Load())
// If this completes then nothing is stealing our reload routine // If this completes then nothing is stealing our reload routine
require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 11")) require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 11").Err)
assert.Equal(t, int64(11), lh.interval.Load()) assert.Equal(t, int64(11), lh.interval.Load())
} }
@@ -296,7 +296,7 @@ func TestLighthouse_reload(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
c.ReloadConfigString(string(rc)) c.ReloadConfigString(string(rc))
err = lh.reload(c, false) err = lh.reload(c, false, nil)
require.NoError(t, err) require.NoError(t, err)
} }
@@ -334,7 +334,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err := yaml.Marshal(nc) rc, err := yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc))) require.NoError(t, c.ReloadConfigString(string(rc)).Err)
rl := lh.Query(staticHost) rl := lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -350,7 +350,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc))) require.NoError(t, c.ReloadConfigString(string(rc)).Err)
rl = lh.Query(staticHost) rl = lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -358,7 +358,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, rl.CopyAddrs([]netip.Prefix{})) assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, rl.CopyAddrs([]netip.Prefix{}))
// Reload with the same config. An unchanged entry must not duplicate. // Reload with the same config. An unchanged entry must not duplicate.
require.NoError(t, c.ReloadConfigString(string(rc))) require.NoError(t, c.ReloadConfigString(string(rc)).Err)
rl = lh.Query(staticHost) rl = lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -373,7 +373,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc))) require.NoError(t, c.ReloadConfigString(string(rc)).Err)
// Add a second host alongside the first. Both should be present, neither duplicated. // Add a second host alongside the first. Both should be present, neither duplicated.
nc = map[string]any{ nc = map[string]any{
@@ -384,7 +384,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc))) require.NoError(t, c.ReloadConfigString(string(rc)).Err)
rl = lh.Query(staticHost) rl = lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -406,7 +406,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc))) require.NoError(t, c.ReloadConfigString(string(rc)).Err)
_, isStatic := lh.GetStaticHostList()[staticHost] _, isStatic := lh.GetStaticHostList()[staticHost]
assert.False(t, isStatic) assert.False(t, isStatic)
@@ -966,13 +966,13 @@ func TestLighthouse_reloadRelaysTriggersUpdate(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
assert.Empty(t, lh.updateTrigger) assert.Empty(t, lh.updateTrigger)
require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6]\n")) require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6]\n").Err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe()) assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1) require.Len(t, lh.updateTrigger, 1)
<-lh.updateTrigger <-lh.updateTrigger
// The config changed but what we advertise did not, a bad entry is skipped // The config changed but what we advertise did not, a bad entry is skipped
require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6, nope]\n")) require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6, nope]\n").Err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe()) assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe())
assert.Empty(t, lh.updateTrigger) assert.Empty(t, lh.updateTrigger)
} }
@@ -988,12 +988,57 @@ func TestLighthouse_reloadAmRelayAlone(t *testing.T) {
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe()) assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe())
// Relays aren't allowed to have relays, flipping am_relay alone must drop them and tell the lighthouses // Relays aren't allowed to have relays, flipping am_relay alone must drop them and tell the lighthouses
require.NoError(t, c.ReloadConfigString(base+" am_relay: true\n")) require.NoError(t, c.ReloadConfigString(base+" am_relay: true\n").Err)
assert.Empty(t, lh.GetRelaysForMe()) assert.Empty(t, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1) require.Len(t, lh.updateTrigger, 1)
<-lh.updateTrigger <-lh.updateTrigger
require.NoError(t, c.ReloadConfigString(base+" am_relay: false\n")) require.NoError(t, c.ReloadConfigString(base+" am_relay: false\n").Err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe()) assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1) require.Len(t, lh.updateTrigger, 1)
} }
func TestLighthouse_reloadRestartRequired(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
base := "lighthouse:\n am_lighthouse: true\n"
require.NoError(t, c.LoadString(base+"listen:\n port: 4242\n"))
_, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
res := c.ReloadConfigString(base + "listen:\n port: 4242\n")
require.NoError(t, res.Err)
assert.False(t, res.RestartRequired)
res = c.ReloadConfigString(base + "listen:\n port: 4243\n")
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
}
// A relay that doesn't parse is skipped and the rest are used, the reload says one was skipped
func TestLighthouse_reloadBadRelayFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("listen:\n port: 4242\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
res := c.ReloadConfigString("listen:\n port: 4242\nrelay:\n relays: [10.128.0.2, nope]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.2")}, lh.GetRelaysForMe())
}
// An advertise_addrs entry inside our own network is skipped and the rest are used, the reload says one was skipped
func TestLighthouse_reloadAdvertiseAddrInNetworkFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("listen:\n port: 4242\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
res := c.ReloadConfigString("listen:\n port: 4242\nlighthouse:\n advertise_addrs: [\"1.2.3.4:4242\", \"10.128.0.5:4242\"]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.2.3.4:4242")}, lh.GetAdvertiseAddrs())
}
+12 -2
View File
@@ -78,7 +78,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
wireSSHReload(l, ssh, c) wireSSHReload(l, ssh, c)
var sshStart func() var sshStart func()
if c.GetBool("sshd.enabled", false) { if c.GetBool("sshd.enabled", false) {
sshStart, err = configSSH(l, ssh, c) sshStart, err = configSSH(l, ssh, c, nil)
if err != nil { if err != nil {
l.Warn("Failed to configure sshd, ssh debugging will not be available", "error", err) l.Warn("Failed to configure sshd, ssh debugging will not be available", "error", err)
sshStart = nil sshStart = nil
@@ -197,7 +197,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
if err != nil { if err != nil {
return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err) return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err)
} }
udpServer.ReloadConfig(c) udpServer.ReloadConfig(c, nil)
udpConns[i] = udpServer udpConns[i] = udpServer
// If port is dynamic, discover it before the next pass through the for loop // If port is dynamic, discover it before the next pass through the for loop
@@ -311,6 +311,16 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
lightHouse.ifce = ifce lightHouse.ifce = ifce
ifce.RegisterConfigChangeCallbacks(c) ifce.RegisterConfigChangeCallbacks(c)
// These are only read at startup
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged(
"routines", "tun.routines", "listen.routines", "firewall.conntrack.routine_cache_timeout",
"listen.port", "listen.host", "stats.message_metrics",
"handshakes.try_interval", "handshakes.retries", "handshakes.trigger_buffer",
"tun.drop_local_broadcast", "tun.drop_multicast",
)
r.RestartRequiredIfChanged(osStartupOnlyKeys...)
})
ifce.reloadDisconnectInvalid(c) ifce.reloadDisconnectInvalid(c)
ifce.reloadSendRecvError(c) ifce.reloadSendRecvError(c)
ifce.reloadAcceptRecvError(c) ifce.reloadAcceptRecvError(c)
+6
View File
@@ -0,0 +1,6 @@
//go:build linux && !android && !e2e_testing
package nebula
// osStartupOnlyKeys are startup only keys that only matter on linux: udp batch size, udp offloads and thread pinning
var osStartupOnlyKeys = []string{"listen.batch", "listen.udp_offloads", "tun.pin_threads", "tun.pin_threads_key", "tun.cpu_affinity"}
+6
View File
@@ -0,0 +1,6 @@
//go:build !linux || android || e2e_testing
package nebula
// osStartupOnlyKeys is empty, none of the OS specific startup keys matter here
var osStartupOnlyKeys []string
+37 -7
View File
@@ -25,7 +25,7 @@ var ErrOutOfWindow = errors.New("out of window packet")
// readOutsidePackets processes one received underlay packet. // readOutsidePackets processes one received underlay packet.
// Message payloads are decrypted IN PLACE, so packet must stay untouched // Message payloads are decrypted IN PLACE, so packet must stay untouched
// by the caller until the batcher for queue q has been flushed // by the caller until the TUN and relay batches for queue q have been flushed.
func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) { func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) {
h := rxc.h h := rxc.h
err := h.Parse(packet) err := h.Parse(packet)
@@ -77,6 +77,12 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
f.messageMetrics.Rx(h.Type, h.Subtype, 1) f.messageMetrics.Rx(h.Type, h.Subtype, 1)
} }
// Earlier forwards must leave before this packet can generate a response.
isMessageRelay := h.Type == header.Message && h.Subtype == header.MessageRelay
if !isMessageRelay {
f.flushRelayBatch(rxc)
}
// Unencrypted packets // Unencrypted packets
switch h.Type { switch h.Type {
case header.Handshake: case header.Handshake:
@@ -88,9 +94,6 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
return return
} }
// Relay packets are special
isMessageRelay := (h.Type == header.Message && h.Subtype == header.MessageRelay)
var hostinfo *HostInfo var hostinfo *HostInfo
if isMessageRelay { if isMessageRelay {
hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex) hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex)
@@ -102,6 +105,8 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
// recvError if necessary // recvError if necessary
if hostinfo == nil || hostinfo.ConnectionState == nil { if hostinfo == nil || hostinfo.ConnectionState == nil {
if !via.IsRelayed { if !via.IsRelayed {
// An unknown relay index can also produce a response.
f.flushRelayBatch(rxc)
f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex) f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex)
} }
return return
@@ -233,8 +238,10 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
// Forward this packet through the relay tunnel, rebuilding it in place. // Forward this packet through the relay tunnel, rebuilding it in place.
// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was // Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
fwdBuf := packet[:0] fwdBuf := packet[:0]
//todo it would potentially be nice to batch these out, err := f.prepareSendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true)
f.SendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true, rxc.q) if err == nil {
f.queueRelayPacket(out, targetHI.GetRemote(), rxc)
}
case TerminalType: case TerminalType:
hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal") hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
return return
@@ -259,6 +266,27 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
} }
} }
// queueRelayPacket borrows a receive buffer until the batch is flushed. Destination changes and the send batch
// limit end a run, preserving arrival order while allowing WriteBatch to use UDP segmentation offload.
func (f *Interface) queueRelayPacket(packet []byte, remote netip.AddrPort, rxc *rxContext) {
if rxc.relayBatch == nil {
rxc.relayBatch = batch.NewSendBatch(f.writers[rxc.q], batch.SendBatchCap, 0)
} else if rxc.relayRemote != remote {
f.flushRelayBatch(rxc)
}
rxc.relayRemote = remote
rxc.relayBatch.Commit(packet, remote)
if rxc.relayBatch.Len() >= batch.SendBatchCap {
f.flushRelayBatch(rxc)
}
}
func (f *Interface) flushRelayBatch(rxc *rxContext) {
if rxc.relayBatch != nil {
f.flushSendBatch(rxc.relayBatch, rxc.q)
}
}
// closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
func (f *Interface) closeTunnel(hostInfo *HostInfo) { func (f *Interface) closeTunnel(hostInfo *HostInfo) {
// Hold the hostmap lock across both the delete and the lighthouse eviction, // Hold the hostmap lock across both the delete and the lighthouse eviction,
@@ -476,7 +504,9 @@ func parseV4(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounter uint64, out []byte, rxc *rxContext) { func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounter uint64, out []byte, rxc *rxContext) {
err := newPacket(out, true, rxc.fwPacket) err := newPacket(out, true, rxc.fwPacket)
if err != nil { if err != nil {
hostinfo.logger(f.l).Warn("Error while validating inbound packet", "error", err, "packet", out) if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out)
}
return return
} }
+291
View File
@@ -0,0 +1,291 @@
package nebula
import (
"bytes"
"net/netip"
"sync"
"testing"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/require"
)
type relayBatchWrite struct {
packets [][]byte
remotes []netip.AddrPort
batched bool
}
type relayBatchConn struct {
udp.NoopConn
read func(udp.EncReader, func())
writes []relayBatchWrite
}
func (c *relayBatchConn) ListenOut(r udp.EncReader, flush func()) error {
c.read(r, flush)
return nil
}
func (c *relayBatchConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
w := relayBatchWrite{remotes: append([]netip.AddrPort(nil), addrs...), batched: true}
for _, b := range bufs {
w.packets = append(w.packets, bytes.Clone(b))
}
c.writes = append(c.writes, w)
return len(bufs), nil
}
func (c *relayBatchConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.writes = append(c.writes, relayBatchWrite{packets: [][]byte{bytes.Clone(b)}, remotes: []netip.AddrPort{addr}})
return nil
}
func (c *relayBatchConn) packets() [][]byte {
var packets [][]byte
for _, w := range c.writes {
packets = append(packets, w.packets...)
}
return packets
}
type relayBatchPath struct {
f *Interface
conn *relayBatchConn
source, target *HostInfo
incoming, outgoing *Relay
sourceKey noiseutil.CipherState
cipher noise.CipherFunc
counter uint64
}
// Separate cipher instances let the reference SendVia path use the same keys and
// counters without rewinding an AEAD instance (including FIPS nonce checks).
func relayBatchKey(cipher noise.CipherFunc, id byte) noiseutil.CipherState {
suite := noise.NewCipherSuite(noise.DH25519, cipher, noise.HashSHA256)
return noiseutil.NewCipherState(noise.UnsafeNewCipherState(suite, [32]byte{id}, 0), cipher)
}
func newRelayBatchPath(t *testing.T, cipher noise.CipherFunc) *relayBatchPath {
t.Helper()
l := test.NewLogger()
conn := &relayBatchConn{}
f := &Interface{
ctx: t.Context(), l: l, hostMap: newHostMap(l), outside: conn,
writers: []udp.Conn{conn}, lightHouse: &LightHouse{l: l},
myVpnNetworksTable: &bart.Lite{}, metricTxDropped: metrics.NewCounter(),
batchers: []*batch.MultiCoalescer{batch.NewMultiCoalescer(&capturingTun{}, l)},
}
f.connectionManager = &connectionManager{
intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{},
}
newHost := func(addr string, remote string, id byte) *HostInfo {
h := &HostInfo{
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
ConnectionState: &ConnectionState{
eKey: relayBatchKey(cipher, id), dKey: relayBatchKey(cipher, id), window: NewBits(ReplayWindow),
},
relayState: RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}},
}
u := netip.MustParseAddrPort(remote)
h.remote.Store(&u)
f.hostMap.Hosts[h.vpnAddrs[0]] = h
return h
}
source := newHost("192.0.2.1", "198.51.100.1:4242", 1)
target := newHost("192.0.2.2", "198.51.100.2:4242", 2)
incoming := &Relay{Type: ForwardingType, State: Established, LocalIndex: 301, RemoteIndex: 302, PeerAddr: target.vpnAddrs[0]}
outgoing := &Relay{Type: ForwardingType, State: Established, LocalIndex: 303, RemoteIndex: 304, PeerAddr: source.vpnAddrs[0]}
source.relayState.InsertRelay(incoming.PeerAddr, incoming.LocalIndex, incoming)
target.relayState.InsertRelay(outgoing.PeerAddr, outgoing.LocalIndex, outgoing)
f.hostMap.Relays[incoming.LocalIndex] = source
f.hostMap.Indexes[100] = source
return &relayBatchPath{f: f, conn: conn, source: source, target: target, incoming: incoming, outgoing: outgoing, sourceKey: relayBatchKey(cipher, 1), cipher: cipher}
}
func (p *relayBatchPath) wrap(t *testing.T, payload []byte) []byte {
t.Helper()
p.counter++
b := header.Encode(make([]byte, header.Len), header.Version, header.Message, header.MessageRelay, p.incoming.LocalIndex, p.counter)
b = append(b, payload...)
b, err := p.sourceKey.EncryptDanger(b, b, nil, p.counter, make([]byte, 12))
require.NoError(t, err)
return b
}
func (p *relayBatchPath) reference(payloads [][]byte) [][]byte {
conn := &relayBatchConn{}
f := &Interface{writers: []udp.Conn{conn}, connectionManager: p.f.connectionManager, l: p.f.l}
h := &HostInfo{ConnectionState: &ConnectionState{eKey: relayBatchKey(p.cipher, 2)}}
u := p.target.GetRemote()
h.remote.Store(&u)
for _, payload := range payloads {
f.SendVia(h, p.outgoing, payload, make([]byte, 12), make([]byte, 0, len(payload)+header.MaxOverhead), false, 0)
}
return conn.packets()
}
func TestRelayReceiveBatch(t *testing.T) {
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
t.Run(cipher.CipherName(), func(t *testing.T) {
p := newRelayBatchPath(t, cipher)
const count = 8
var payloads, wire [][]byte
for i := range count * 2 {
payload := bytes.Repeat([]byte{byte(i)}, 64)
payloads = append(payloads, payload)
wire = append(wire, p.wrap(t, payload))
}
want := p.reference(payloads)
// Model a listener reusing the same slots for the next receive batch.
buffers := make([][]byte, count)
for i := range buffers {
buffers[i] = make([]byte, len(wire[i]))
}
p.conn.read = func(r udp.EncReader, flush func()) {
for b := range 2 {
for i := range count {
copy(buffers[i], wire[b*count+i])
r(p.source.GetRemote(), buffers[i])
require.Len(t, p.conn.writes, b, "forwarding waits for the batch boundary")
}
flush()
require.Len(t, p.conn.writes, b+1, "flush must send before receive slots are reused")
for _, buffer := range buffers {
clear(buffer)
}
}
}
p.f.listenOut(0)
require.Equal(t, want, p.conn.packets())
for _, w := range p.conn.writes {
require.True(t, w.batched)
require.Len(t, w.packets, count)
for _, remote := range w.remotes {
require.Equal(t, p.target.GetRemote(), remote)
}
}
})
}
}
func TestRelayBatchDestinationChange(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, second := p.target.GetRemote(), netip.MustParseAddrPort("198.51.100.3:4242")
remotes := []netip.AddrPort{first, first, second, second, first}
var payloads, wire [][]byte
for i := range remotes {
payload := []byte{byte(i)}
payloads = append(payloads, payload)
wire = append(wire, p.wrap(t, payload))
}
want := p.reference(payloads)
p.conn.read = func(r udp.EncReader, flush func()) {
for i, remote := range remotes {
p.target.remote.Store(&remote)
r(p.source.GetRemote(), wire[i])
}
flush()
}
p.f.listenOut(0)
require.Equal(t, want, p.conn.packets())
require.Len(t, p.conn.writes, 3)
for i, wantRemotes := range [][]netip.AddrPort{{first, first}, {second, second}, {first}} {
require.Equal(t, wantRemotes, p.conn.writes[i].remotes)
}
}
func TestRelayBatchCapacity(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
p.conn.read = func(r udp.EncReader, flush func()) {
for i := range batch.SendBatchCap + 1 {
r(p.source.GetRemote(), p.wrap(t, []byte{byte(i)}))
}
require.Len(t, p.conn.writes, 1)
require.Len(t, p.conn.writes[0].packets, batch.SendBatchCap)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 2)
require.Len(t, p.conn.writes[1].packets, 1)
}
func TestRelayBatchBeforeResponse(t *testing.T) {
for _, unknownRelay := range []bool{false, true} {
name := "test request"
if unknownRelay {
name = "unknown relay"
}
t.Run(name, func(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, last := p.wrap(t, []byte("first")), p.wrap(t, []byte("last"))
p.counter++
request := header.Encode(make([]byte, header.Len), header.Version, header.Test, header.TestRequest, 100, p.counter)
request, err := p.sourceKey.EncryptDanger(request, request, []byte("ping"), p.counter, make([]byte, 12))
require.NoError(t, err)
if unknownRelay {
request = header.Encode(make([]byte, header.Len), header.Version, header.Message, header.MessageRelay, 999, p.counter)
}
p.conn.read = func(r udp.EncReader, flush func()) {
r(p.source.GetRemote(), first)
require.Empty(t, p.conn.writes)
r(p.source.GetRemote(), request)
require.Len(t, p.conn.writes, 2)
r(p.source.GetRemote(), last)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 3)
require.True(t, p.conn.writes[0].batched)
require.False(t, p.conn.writes[1].batched)
require.True(t, p.conn.writes[2].batched)
require.Equal(t, p.source.GetRemote(), p.conn.writes[1].remotes[0])
h := &header.H{}
require.NoError(t, h.Parse(p.conn.writes[1].packets[0]))
if unknownRelay {
require.Equal(t, header.RecvError, h.Type)
require.EqualValues(t, 999, h.RemoteIndex)
} else {
require.Equal(t, header.Test, h.Type)
require.Equal(t, header.TestReply, h.Subtype)
reply := p.conn.writes[1].packets[0]
out, err := p.sourceKey.DecryptDanger(nil, reply[:header.Len], reply[header.Len:], h.MessageCounter, make([]byte, 12))
require.NoError(t, err)
require.Equal(t, []byte("ping"), out)
}
})
}
}
func TestRelayBatchRemovedTarget(t *testing.T) {
for _, remove := range []string{"host", "relay", "disestablished"} {
t.Run(remove, func(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, removed := p.wrap(t, []byte("first")), p.wrap(t, []byte("removed"))
want := p.reference([][]byte{[]byte("first")})
p.conn.read = func(r udp.EncReader, flush func()) {
r(p.source.GetRemote(), first)
switch remove {
case "host":
delete(p.f.hostMap.Hosts, p.target.vpnAddrs[0])
case "relay":
delete(p.target.relayState.relayForByAddr, p.source.vpnAddrs[0])
case "disestablished":
p.target.relayState.UpdateRelayForByIpState(p.source.vpnAddrs[0], Disestablished)
}
r(p.source.GetRemote(), removed)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 1)
require.Equal(t, want, p.conn.packets(), "only the packet accepted before removal is sent")
})
}
}
+2 -2
View File
@@ -10,8 +10,8 @@ type batchWriter interface {
} }
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch. // SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
// One SendBatch is owned by each listenIn goroutine; no locking is needed. // Each reader goroutine owns its own SendBatch; no locking is needed.
// Slots are backed by an Arena (see its docs) // Slots can borrow receive buffers or be backed by an Arena (see its docs).
type SendBatch struct { type SendBatch struct {
out batchWriter out batchWriter
bufs [][]byte bufs [][]byte
+8 -1
View File
@@ -41,9 +41,16 @@ func (e *NameError) Error() string {
type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error)
func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) { func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.disabled")
})
switch { switch {
case c.GetBool("tun.disabled", false): case c.GetBool("tun.disabled", false):
tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l) tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 5000), c.GetBool("stats.message_metrics", false), l)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.tx_queue", "stats.message_metrics")
})
return tun, nil return tun, nil
default: default:
+5 -4
View File
@@ -38,16 +38,17 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
l: l, l: l,
} }
err := t.reload(c, true) err := t.reload(c, true, nil)
if err != nil { if err != nil {
_ = file.Close() _ = file.Close()
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
err := t.reload(c, false) err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -67,7 +68,7 @@ func (t *tun) Activate() error {
return nil return nil
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
+45
View File
@@ -0,0 +1,45 @@
//go:build (openbsd || netbsd) && !e2e_testing
package overlay
import (
"errors"
"fmt"
"log/slog"
"os"
"regexp"
"runtime"
"golang.org/x/sys/unix"
)
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
// tunDevNeedsRestart is true when tun.dev asks for a particular tun that isn't the running one. Anything else takes the
// first free one, and the one we have will do
func tunDevNeedsRestart(name, running string) bool {
return deviceNameRE.MatchString(name) && name != running
}
// openTun opens /dev/name, or the first /dev/tunN nothing else has open when name is empty or not tunN
func openTun(l *slog.Logger, name string) (int, string, error) {
if name != "" && !deviceNameRE.MatchString(name) {
// The name doesn't mean anything here, so like darwin a bad one isn't worth refusing to start over
l.Warn("interface name must be tun[0-9]+ on "+runtime.GOOS+", ignoring", "name", name)
name = ""
}
if name != "" {
fd, err := unix.Open("/dev/"+name, os.O_RDWR, 0)
return fd, name, err
}
for i := 0; ; i++ {
name = fmt.Sprintf("tun%d", i)
fd, err := unix.Open("/dev/"+name, os.O_RDWR, 0)
if err == nil {
return fd, name, nil
}
if !errors.Is(err, unix.EBUSY) {
return -1, "", fmt.Errorf("no free tun device, %s: %w", name, err)
}
}
}
+80
View File
@@ -0,0 +1,80 @@
//go:build (openbsd || netbsd) && !e2e_testing
package overlay
import (
"net/netip"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// A tun.dev that isn't tunN is ignored like darwin does, and the first free tun is used. Only naming a different tun
// asks for a restart
func TestTunReloadDevRestart(t *testing.T) {
newWithDev := func(dev string) (*config.C, *tun, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n dev: "+dev+"\n"))
tn, err := newTun(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, false)
if err == nil {
t.Cleanup(func() { _ = tn.Close() })
}
return c, tn, err
}
if _, _, err := newWithDev(""); err != nil {
t.Skipf("can't create a tun device here: %v", err)
}
c, tn, err := newWithDev("nope")
require.NoError(t, err, "a bad name should fall back to a free tun")
require.Regexp(t, `^tun[0-9]+$`, tn.Device)
for _, dev := range []string{"nope", "", tn.Device, "nope"} {
res := c.ReloadConfigString("tun:\n dev: " + dev + "\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed, dev)
assert.False(t, res.RestartRequired, dev)
}
other := "tun0"
if tn.Device == other {
other = "tun1"
}
res := c.ReloadConfigString("tun:\n dev: " + other + "\n")
require.NoError(t, res.Err)
assert.Equal(t, []string{"tun.dev"}, res.RestartKeys)
}
// An empty tun.dev takes the first tun nothing else has open. Reloading it empty, or naming the one we have, needs
// no restart
func TestTunEmptyDevPicksAFreeOne(t *testing.T) {
newEmpty := func() (*config.C, *tun, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun: {}\n"))
tn, err := newTun(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, false)
if err == nil {
t.Cleanup(func() { _ = tn.Close() })
}
return c, tn, err
}
c, first, err := newEmpty()
if err != nil {
t.Skipf("can't create a tun device here: %v", err)
}
_, second, err := newEmpty()
require.NoError(t, err, "the second tun should skip the one the first has open")
assert.NotEqual(t, first.Device, second.Device)
for _, cfg := range []string{
"tun: {}\npunchy:\n punch: true\n",
"tun:\n dev: " + first.Device + "\n",
"tun: {}\n",
} {
res := c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired, cfg)
}
}
+3
View File
@@ -10,6 +10,9 @@ import (
"github.com/slackhq/nebula/wfp" "github.com/slackhq/nebula/wfp"
) )
// interfaceBypassKeys are only read when the interface bypass is installed
var interfaceBypassKeys = []string{"tun.windows_bypass_wdf"}
// installInterfaceBypass installs a WFP PERMIT filter scoped to the wintun interface LUID so inbound traffic on the // installInterfaceBypass installs a WFP PERMIT filter scoped to the wintun interface LUID so inbound traffic on the
// nebula adapter bypasses Windows Defender Firewall. // nebula adapter bypasses Windows Defender Firewall.
func installInterfaceBypass(l *slog.Logger, luid uint64) closer { func installInterfaceBypass(l *slog.Logger, luid uint64) closer {
+3
View File
@@ -4,6 +4,9 @@ package overlay
import "log/slog" import "log/slog"
// interfaceBypassKeys is empty, windows-386 has no bypass
var interfaceBypassKeys []string
// installInterfaceBypass is a no-op on windows-386 because we don't currently build for it. // installInterfaceBypass is a no-op on windows-386 because we don't currently build for it.
func installInterfaceBypass(_ *slog.Logger, _ uint64) closer { func installInterfaceBypass(_ *slog.Logger, _ uint64) closer {
return nil return nil
+149 -38
View File
@@ -6,8 +6,10 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"math/bits"
"net/netip" "net/netip"
"os" "os"
"sync"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -28,7 +30,7 @@ type tun struct {
DefaultMTU int DefaultMTU int
Routes atomic.Pointer[[]Route] Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]] routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr *netroute.LinkAddr linkAddr atomic.Pointer[netroute.LinkAddr]
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu // hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
// and routes for it. NEPacketTunnelProvider on darwin does this. // and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool hostOwned bool
@@ -78,17 +80,31 @@ type ifreqAlias6 struct {
Lifetime addrLifetime Lifetime addrLifetime
} }
// utunIndex is the utun tun.dev asks for, or -1 for whichever is free. ok is false for a name that isn't utunN, which
// also gets whichever is free
func utunIndex(name string) (index int, ok bool) {
if name == "" || name == "utun" {
return -1, true
}
if _, err := fmt.Sscanf(name, "utun%d", &index); err != nil || index < 0 {
return -1, false
}
return index, true
}
// tunDevNeedsRestart is true when tun.dev asks for a particular utun that isn't the running one. Anything else takes
// whichever is free, and the one we have will do
func tunDevNeedsRestart(name, running string) bool {
index, _ := utunIndex(name)
return index >= 0 && fmt.Sprintf("utun%d", index) != running
}
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
name := c.GetString("tun.dev", "") ifIndex, ok := utunIndex(c.GetString("tun.dev", ""))
ifIndex := -1 if !ok {
if name != "" && name != "utun" { // NOTE: we don't make this error so we don't break existing
_, err := fmt.Sscanf(name, "utun%d", &ifIndex) // configs that set a name before it was used.
if err != nil || ifIndex < 0 { l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring")
// NOTE: we don't make this error so we don't break existing
// configs that set a name before it was used.
l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring")
ifIndex = -1
}
} }
fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL) fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL)
@@ -112,7 +128,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
return nil, fmt.Errorf("SYS_CONNECT: %v", err) return nil, fmt.Errorf("SYS_CONNECT: %v", err)
} }
name, err = unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME) name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to retrieve tun name: %w", err) return nil, fmt.Errorf("failed to retrieve tun name: %w", err)
} }
@@ -130,15 +146,20 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
l: l, l: l,
} }
err = t.reload(c, true) err = t.reload(c, true, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
err := t.reload(c, false) r.RestartRequiredIfChanged("tun.mtu")
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -172,14 +193,15 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
l: l, l: l,
} }
if err := t.reload(c, true); err != nil { if err := t.reload(c, true, nil); err != nil {
_ = file.Close() _ = file.Close()
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
if err := t.reload(c, false); err != nil { if err := t.reload(c, false, r); err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -243,7 +265,7 @@ func (t *tun) Activate() error {
if linkAddr == nil { if linkAddr == nil {
return fmt.Errorf("unable to discover link_addr for tun interface") return fmt.Errorf("unable to discover link_addr for tun interface")
} }
t.linkAddr = linkAddr t.linkAddr.Store(linkAddr)
for _, network := range t.vpnNetworks { for _, network := range t.vpnNetworks {
if network.Addr().Is4() { if network.Addr().Is4() {
@@ -266,9 +288,10 @@ func (t *tun) Activate() error {
} }
// Unsafe path routes // Unsafe path routes
return t.addRoutes(false) return t.addRoutes(false, nil)
} }
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
func (t *tun) activate4(network netip.Prefix) error { func (t *tun) activate4(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET, unix.AF_INET,
@@ -295,22 +318,32 @@ func (t *tun) activate4(network netip.Prefix) error {
MaskAddr: unix.RawSockaddrInet4{ MaskAddr: unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4, Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET, Family: unix.AF_INET,
Addr: prefixToMask(network).As4(), Addr: [4]byte{255, 255, 255, 255}, //always make the address a /32
}, },
} }
// On Darwin, VPNs don't edit the routing table directly.
// A network extension hands its routes to the system, and configd installs them.
// configd replaces any route already holding the same prefix, including ours.
// As an example, Tailscale with an exit node treats any interface prefix wider than one address as a LAN.
// With LAN access off, it routes each LAN into Tailscale's utun.
// With LAN access on, it excludes each LAN from its tunnel.
// configd routes an excluded prefix like its own routes would, ignoring ours, so it goes out the default route.
// Either way our route is replaced, and traffic for the vpn network no longer reaches nebula.
// A single address isn't a LAN to Tailscale, so our route is left alone.
// We route the vpn network to the tun explicitly instead.
if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun v4 address: %s", err) return fmt.Errorf("failed to set tun v4 address: %s", err)
} }
err = addRoute(network, t.linkAddr) if err := addRoute(network, t.linkAddr.Load()); err != nil {
if err != nil { return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
return err
} }
return nil return nil
} }
// activate6 is activate4 for IPv6
func (t *tun) activate6(network netip.Prefix) error { func (t *tun) activate6(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET6, unix.AF_INET6,
@@ -332,24 +365,28 @@ func (t *tun) activate6(network netip.Prefix) error {
PrefixMask: unix.RawSockaddrInet6{ PrefixMask: unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6, Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6, Family: unix.AF_INET6,
Addr: prefixToMask(network).As16(), // Always set the netmask to be a /128. See activate4 for why we do this.
Addr: [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
}, },
Lifetime: addrLifetime{ Lifetime: addrLifetime{
// never expires // never expires
Vltime: 0xffffffff, Vltime: 0xffffffff,
Pltime: 0xffffffff, Pltime: 0xffffffff,
}, },
Flags: _IN6_IFF_NODAD, Flags: _IN6_IFF_NODAD, // No DAD, a utun has no neighbors.
} }
if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun address: %s", err) return fmt.Errorf("failed to set tun address: %s", err)
} }
if err := addRoute(network, t.linkAddr.Load()); err != nil {
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
}
return nil return nil
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
@@ -370,16 +407,18 @@ func (t *tun) reload(c *config.C, initial bool) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true) err = t.addRoutes(true, r)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -394,6 +433,56 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
return routing.Gateways{} return routing.Gateways{}
} }
func fetchRoutes() ([]netroute.Message, error) {
rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_DUMP, 0)
if err != nil {
return nil, err
}
return netroute.ParseRIB(unix.NET_RT_DUMP, rib)
}
// routeInterfaceIn returns the index of the interface prefix is routed through, or 0 if msgs has no route for exactly
// that prefix. Routes scoped to an interface are skipped, they don't stop us adding ours.
func routeInterfaceIn(msgs []netroute.Message, prefix netip.Prefix) int {
for _, msg := range msgs {
m, ok := msg.(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK || m.Flags&unix.RTF_IFSCOPE != 0 {
continue
}
if p, ok := routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK]); ok && p == prefix.Masked() {
return m.Index
}
}
return 0
}
// routePrefix returns the prefix a route covers. Host routes have no netmask.
func routePrefix(dst, mask netroute.Addr) (netip.Prefix, bool) {
switch d := dst.(type) {
case *netroute.Inet4Addr:
bits := 32
if m, ok := mask.(*netroute.Inet4Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom4(d.IP), bits), true
case *netroute.Inet6Addr:
bits := 128
if m, ok := mask.(*netroute.Inet6Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom16(d.IP), bits), true
}
return netip.Prefix{}, false
}
func maskBits(mask []byte) int {
n := 0
for _, b := range mask {
n += bits.OnesCount8(b)
}
return n
}
// Get the LinkAddr for the interface of the given name // Get the LinkAddr for the interface of the given name
// Is there an easier way to fetch this when we create the interface? // Is there an easier way to fetch this when we create the interface?
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers. // Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
@@ -422,13 +511,20 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
return nil, nil return nil, nil
} }
func (t *tun) addRoutes(logErrors bool) error { func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not // The route tree is still ours, the system routing table is not
if t.hostOwned { if t.hostOwned {
return nil return nil
} }
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load() routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
@@ -436,39 +532,54 @@ func (t *tun) addRoutes(logErrors bool) error {
continue continue
} }
err := addRoute(r.Cidr, t.linkAddr) err := addRoute(r.Cidr, linkAddr)
if err != nil { if err != nil {
if errors.Is(err, unix.EEXIST) { if errors.Is(err, unix.EEXIST) {
t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr) // Every route we added already exists on a reload, only warn about someone else's
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else { } else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
} }
} else { } else {
t.l.Info("Added route", "route", r) t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
} }
} }
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
if t.hostOwned { linkAddr := t.linkAddr.Load()
if t.hostOwned || linkAddr == nil {
return nil return nil
} }
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.linkAddr) err := delRoute(r.Cidr, linkAddr)
if err != nil { if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
+118
View File
@@ -0,0 +1,118 @@
//go:build darwin && !ios && !e2e_testing
package overlay
import (
"net"
"net/netip"
"os"
"os/exec"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Every mac routes loopback through lo0
func TestRouteInterfaceIn(t *testing.T) {
msgs, err := fetchRoutes()
require.NoError(t, err)
routeInterface := func(s string) int { return routeInterfaceIn(msgs, netip.MustParsePrefix(s)) }
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterface("127.0.0.0/8"))
assert.Equal(t, lo0.Index, routeInterface("127.0.0.1/32"), "a host route has no netmask")
assert.Equal(t, lo0.Index, routeInterface("::1/128"))
assert.Equal(t, 0, routeInterface("127.0.0.0/16"), "only an exact prefix counts")
// Most macs also list the default route scoped to their other interfaces
out, err := exec.Command("route", "-n", "get", "default").Output()
require.NoError(t, err)
var name string
for _, line := range strings.Split(string(out), "\n") {
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "interface: "); ok {
name = v
}
}
if name == "" {
t.Skip("no default route")
}
def, err := net.InterfaceByName(name)
require.NoError(t, err)
assert.Equal(t, def.Index, routeInterface("0.0.0.0/0"))
}
func TestRouteInterfaceIn_SkipsScopedCopies(t *testing.T) {
route := func(index, flags int) netroute.Message {
addrs := make([]netroute.Addr, unix.RTAX_MAX)
addrs[unix.RTAX_DST] = &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}}
addrs[unix.RTAX_NETMASK] = &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}}
return &netroute.RouteMessage{Index: index, Flags: flags, Addrs: addrs}
}
p := netip.MustParsePrefix("192.168.1.0/24")
assert.Equal(t, 7, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE), route(7, unix.RTF_UP)}, p))
assert.Equal(t, 0, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE)}, p))
}
// A reload can happen before Activate
func TestTunRoutesBeforeActivate(t *testing.T) {
tn := &tun{l: test.NewLogger()}
routes := []Route{{
Cidr: netip.MustParsePrefix("10.253.1.0/24"),
Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)},
Install: true,
}}
tn.Routes.Store(&routes)
assert.NoError(t, tn.addRoutes(true, nil))
assert.NoError(t, tn.removeRoutes(routes, nil))
}
// A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that
// belongs to someone else
func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating a utun and adding routes needs root")
}
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
cidr := "10.250.0.128/25"
require.NoError(t, exec.Command("route", "-q", "-n", "add", "-net", cidr, "-interface", "lo0").Run())
t.Cleanup(func() { _ = exec.Command("route", "-q", "-n", "delete", "-net", cidr).Run() })
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("tun:\n routes:\n - route: "+cidr+"\n mtu: 1300\n"))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
res := c.ReloadConfigString("tun: {}\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed, "removing a route we never installed failed the reload")
msgs, err := fetchRoutes()
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
func TestTunDevNeedsRestart(t *testing.T) {
for name, want := range map[string]bool{
"": false,
"utun": false,
"nope": false,
"utun7": false,
"utun3": true,
} {
assert.Equal(t, want, tunDevNeedsRestart(name, "utun7"), name)
}
}
+121
View File
@@ -0,0 +1,121 @@
//go:build !ios && !e2e_testing
package overlay
import (
"bytes"
"os"
"syscall"
"testing"
"github.com/stretchr/testify/require"
)
func TestTun_WriteFraming(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: w}
for _, tc := range []struct {
name string
pkt []byte
af byte
}{
// Literal families rather than syscall.AF_*, so a wrong constant in Write can't also fix the expectation.
{"ipv4", []byte{0x45, 1, 2, 3, 4, 5}, 2},
{"ipv6", []byte{0x60, 1, 2, 3, 4, 5}, 30},
} {
t.Run(tc.name, func(t *testing.T) {
n, err := tn.Write(tc.pkt)
require.NoError(t, err)
require.Equal(t, len(tc.pkt), n)
got := make([]byte, 64)
n, err = r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, tc.af}, tc.pkt...), got[:n])
})
}
_, err = tn.Write([]byte{0x10, 1, 2, 3})
require.Error(t, err)
_, err = tn.Write([]byte{})
require.ErrorIs(t, err, syscall.EIO)
// The rejected writes must not have put anything on the fd, so the next read sees only this packet.
pkt := []byte{0x45, 9, 9}
_, err = tn.Write(pkt)
require.NoError(t, err)
got := make([]byte, 64)
n, err := r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, 2}, pkt...), got[:n])
}
func TestTun_ReadStripsHeader(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
pkt := []byte{0x45, 1, 2, 3, 4, 5}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err := tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
func TestTun_ReadShort(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
// Fewer bytes than the protocol header is not a packet.
_, err = w.Write([]byte{0, 0})
require.NoError(t, err)
n, err := tn.Read(make([]byte, 64))
require.NoError(t, err)
require.Zero(t, n)
// The next packet reads cleanly only if the short read consumed exactly what was written.
pkt := []byte{0x45, 1, 2, 3}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err = tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
// A heap allocation per packet rarely shows in a short throughput run, only across GC pauses, so
// this pins Read and Write to zero allocations directly.
func TestTun_ReadWriteDoNotAllocate(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tw := &tun{f: w}
tr := &tun{f: r}
pkts := [][]byte{bytes.Repeat([]byte{0x45}, 1300), bytes.Repeat([]byte{0x60}, 1300)}
buf := make([]byte, 1500)
i := 0
allocs := testing.AllocsPerRun(1000, func() {
i++
if _, err := tw.Write(pkts[i%len(pkts)]); err != nil {
t.Fatal(err)
}
if _, err := tr.Read(buf); err != nil {
t.Fatal(err)
}
})
require.Zero(t, allocs)
}
+21 -11
View File
@@ -385,16 +385,18 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
}, },
} }
err = t.reload(c, true) err = t.reload(c, true, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
closeOnErr = false closeOnErr = false
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
err := t.reload(c, false) r.RestartRequiredIfChanged("tun.dev", "tun.mtu")
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -493,7 +495,7 @@ func (t *tun) Activate() error {
} }
} }
return t.addRoutes(false) return t.addRoutes(false, nil)
} }
func (t *tun) setMTU() error { func (t *tun) setMTU() error {
@@ -509,7 +511,7 @@ func (t *tun) setMTU() error {
return err return err
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
@@ -530,16 +532,18 @@ func (t *tun) reload(c *config.C, initial bool) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true) err = t.addRoutes(true, r)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -563,7 +567,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
} }
func (t *tun) addRoutes(logErrors bool) error { func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
@@ -576,6 +580,7 @@ func (t *tun) addRoutes(logErrors bool) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -587,15 +592,20 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.linkAddr) err := delRoute(r.Cidr, t.linkAddr)
if err != nil { if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
+5 -4
View File
@@ -47,16 +47,17 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
l: l, l: l,
} }
err := t.reload(c, true) err := t.reload(c, true, nil)
if err != nil { if err != nil {
_ = file.Close() _ = file.Close()
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
err := t.reload(c, false) err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -67,7 +68,7 @@ func (t *tun) Activate() error {
return nil return nil
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
+29 -15
View File
@@ -185,6 +185,10 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
} }
t.Device = name t.Device = name
// Only read when we create the device, an fd handed to us already has its name and offloads
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.dev", "tun.use_offloads")
})
return t, nil return t, nil
} }
@@ -221,29 +225,31 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
vnetHdr: vnetHdr, vnetHdr: vnetHdr,
offloadFlags: offloadFlags, offloadFlags: offloadFlags,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
TXQueueLen: c.GetInt("tun.tx_queue", 500), TXQueueLen: c.GetInt("tun.tx_queue", 5000),
useSystemRoutes: c.GetBool("tun.use_system_route_table", false), useSystemRoutes: c.GetBool("tun.use_system_route_table", false),
useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0), useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
routesFromSystem: map[netip.Prefix]routing.Gateways{}, routesFromSystem: map[netip.Prefix]routing.Gateways{},
l: l, l: l,
} }
if err = t.reload(c, true); err != nil { if err = t.reload(c, true, nil); err != nil {
_ = t.Close() _ = t.Close()
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
err := t.reload(c, false) r.RestartRequiredIfChanged("tun.tx_queue", "tun.use_system_route_table", "tun.use_system_route_table_buffer_size")
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
return t, nil return t, nil
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
routeChange, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) routeChange, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
@@ -297,7 +303,7 @@ func (t *tun) reload(c *config.C, initial bool) error {
if !initial { if !initial {
if oldMaxMTU != newMaxMTU { if oldMaxMTU != newMaxMTU {
t.setMTU() t.setMTU(r)
t.l.Info("Set max MTU", "mtu", t.MaxMTU, "oldMTU", oldMaxMTU) t.l.Info("Set max MTU", "mtu", t.MaxMTU, "oldMTU", oldMaxMTU)
} }
@@ -305,7 +311,8 @@ func (t *tun) reload(c *config.C, initial bool) error {
for i := range t.vpnNetworks { for i := range t.vpnNetworks {
err := t.setDefaultRoute(t.vpnNetworks[i]) err := t.setDefaultRoute(t.vpnNetworks[i])
if err != nil { if err != nil {
t.l.Warn(err.Error()) t.l.Error(err.Error())
r.Failed()
} else { } else {
t.l.Info("Set default MTU", "mtu", t.DefaultMTU, "oldMTU", oldDefaultMTU) t.l.Info("Set default MTU", "mtu", t.DefaultMTU, "oldMTU", oldDefaultMTU)
} }
@@ -313,13 +320,14 @@ func (t *tun) reload(c *config.C, initial bool) error {
} }
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r)
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true) err = t.addRoutes(true, r)
if err != nil { if err != nil {
// This should never be called since addRoutes should log its own errors in a reload condition // This should never be called since addRoutes should log its own errors in a reload condition
util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l) util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l)
r.Failed()
} }
} }
@@ -474,7 +482,7 @@ func (t *tun) Activate() error {
t.deviceIndex = link.Attrs().Index t.deviceIndex = link.Attrs().Index
// Setup our default MTU // Setup our default MTU
t.setMTU() t.setMTU(nil)
// Set the transmit queue length // Set the transmit queue length
ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)} ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)}
@@ -506,7 +514,7 @@ func (t *tun) Activate() error {
} }
// Set the routes // Set the routes
if err = t.addRoutes(false); err != nil { if err = t.addRoutes(false, nil); err != nil {
return err return err
} }
@@ -519,12 +527,13 @@ func (t *tun) Activate() error {
return nil return nil
} }
func (t *tun) setMTU() { func (t *tun) setMTU(r *config.Reload) {
// Set the MTU on the device // Set the MTU on the device
ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)} ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)}
if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil { if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
// This is currently a non fatal condition because the route table must have the MTU set appropriately as well // This is currently a non fatal condition because the route table must have the MTU set appropriately as well
t.l.Error("Failed to set tun mtu", "error", err) t.l.Error("Failed to set tun mtu", "error", err)
r.Failed()
} }
} }
@@ -577,7 +586,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
return nil return nil
} }
func (t *tun) addRoutes(logErrors bool) error { func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// Path routes // Path routes
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -607,6 +616,7 @@ func (t *tun) addRoutes(logErrors bool) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -618,7 +628,7 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route) { func (t *tun) removeRoutes(routes []Route, rl *config.Reload) {
for _, r := range routes { for _, r := range routes {
if !r.Install { if !r.Install {
continue continue
@@ -642,8 +652,12 @@ func (t *tun) removeRoutes(routes []Route) {
} }
err := netlink.RouteDel(&nr) err := netlink.RouteDel(&nr)
if err != nil { if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
+25 -25
View File
@@ -8,7 +8,6 @@ import (
"log/slog" "log/slog"
"net/netip" "net/netip"
"os" "os"
"regexp"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -67,24 +66,12 @@ type tun struct {
fd int fd int
} }
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
return nil, fmt.Errorf("newTunFromFd not supported in NetBSD") return nil, fmt.Errorf("newTunFromFd not supported in NetBSD")
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
// Try to open tun device fd, deviceName, err := openTun(l, c.GetString("tun.dev", ""))
var err error
deviceName := c.GetString("tun.dev", "")
if deviceName == "" {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
if !deviceNameRE.MatchString(deviceName) {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -103,15 +90,20 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
l: l, l: l,
} }
err = t.reload(c, true) err = t.reload(c, true, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
err := t.reload(c, false) r.RestartRequiredIfChanged("tun.mtu")
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -324,7 +316,7 @@ func (t *tun) Activate() error {
} }
} }
return t.addRoutes(false) return t.addRoutes(false, nil)
} }
func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
@@ -339,7 +331,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
return err return err
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
@@ -360,16 +352,18 @@ func (t *tun) reload(c *config.C, initial bool) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true) err = t.addRoutes(true, r)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -393,7 +387,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
} }
func (t *tun) addRoutes(logErrors bool) error { func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -407,6 +401,7 @@ func (t *tun) addRoutes(logErrors bool) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -418,15 +413,20 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.vpnNetworks) err := delRoute(r.Cidr, t.vpnNetworks)
if err != nil { if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
+25 -25
View File
@@ -8,7 +8,6 @@ import (
"log/slog" "log/slog"
"net/netip" "net/netip"
"os" "os"
"regexp"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -58,24 +57,12 @@ type tun struct {
fd int fd int
} }
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
return nil, fmt.Errorf("newTunFromFd not supported in openbsd") return nil, fmt.Errorf("newTunFromFd not supported in openbsd")
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
// Try to open tun device fd, deviceName, err := openTun(l, c.GetString("tun.dev", ""))
var err error
deviceName := c.GetString("tun.dev", "")
if deviceName == "" {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
if !deviceNameRE.MatchString(deviceName) {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -94,15 +81,20 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
l: l, l: l,
} }
err = t.reload(c, true) err = t.reload(c, true, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
err := t.reload(c, false) r.RestartRequiredIfChanged("tun.mtu")
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -303,7 +295,7 @@ func (t *tun) Activate() error {
} }
} }
return t.addRoutes(false) return t.addRoutes(false, nil)
} }
func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
@@ -318,7 +310,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
return err return err
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
@@ -339,16 +331,18 @@ func (t *tun) reload(c *config.C, initial bool) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true) err = t.addRoutes(true, r)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -372,7 +366,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
} }
func (t *tun) addRoutes(logErrors bool) error { func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -386,6 +380,7 @@ func (t *tun) addRoutes(logErrors bool) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -397,15 +392,20 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.vpnNetworks) err := delRoute(r.Cidr, t.vpnNetworks)
if err != nil { if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
+55 -4
View File
@@ -4,6 +4,7 @@ package overlay
import ( import (
"net/netip" "net/netip"
"os"
"testing" "testing"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
@@ -25,10 +26,10 @@ func TestReloadMTUOnlyKeepsRoutes(t *testing.T) {
ioctlFd: ^uintptr(0), ioctlFd: ^uintptr(0),
routesFromSystem: map[netip.Prefix]routing.Gateways{}, routesFromSystem: map[netip.Prefix]routing.Gateways{},
} }
require.NoError(t, tn.reload(c, true)) require.NoError(t, tn.reload(c, true, nil))
require.NoError(t, c.ReloadConfigString("tun:\n mtu: 1400\n"+routes)) require.NoError(t, c.ReloadConfigString("tun:\n mtu: 1400\n"+routes).Err)
_ = tn.reload(c, false) _ = tn.reload(c, false, nil)
got := *tn.Routes.Load() got := *tn.Routes.Load()
require.Len(t, got, 1) require.Len(t, got, 1)
@@ -50,7 +51,57 @@ func TestReloadMaxMTUIsTheLargestRoute(t *testing.T) {
ioctlFd: ^uintptr(0), ioctlFd: ^uintptr(0),
routesFromSystem: map[netip.Prefix]routing.Gateways{}, routesFromSystem: map[netip.Prefix]routing.Gateways{},
} }
require.NoError(t, tn.reload(c, true)) require.NoError(t, tn.reload(c, true, nil))
assert.Equal(t, 1500, tn.MaxMTU) assert.Equal(t, 1500, tn.MaxMTU)
assert.Equal(t, 1300, tn.DefaultMTU) assert.Equal(t, 1300, tn.DefaultMTU)
} }
// There is no device behind this tun, so adding a route always fails. Removing one is refused without root, as root the
// kernel answers that the route isn't there, which is what we wanted
func TestReloadRouteFailures(t *testing.T) {
route := "tun:\n mtu: 1300\n unsafe_routes:\n - route: 192.168.100.0/24\n via: 10.1.0.1\n"
cases := []struct {
name string
from, to string
wantFails bool
}{
{"added", "tun:\n mtu: 1300\n", route, true},
{"removed", route, "tun:\n mtu: 1300\n", os.Geteuid() != 0},
{"unchanged", route, route + "punchy:\n punch: true\n", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(tc.from))
tn := &tun{
l: test.NewLogger(),
vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")},
ioctlFd: ^uintptr(0),
routesFromSystem: map[netip.Prefix]routing.Gateways{},
}
require.NoError(t, tn.reload(c, true, nil))
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
require.NoError(t, tn.reload(c, false, r))
})
res := c.ReloadConfigString(tc.to)
require.NoError(t, res.Err)
assert.Equal(t, tc.wantFails, res.Failed)
})
}
}
// Keys only read when the device is created need a restart to apply
func TestTunReloadRestartKeys(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n dev: nebrk0\n"))
tn, err := NewDeviceFromConfig(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, 1)
if err != nil {
t.Skipf("can't create a tun device here: %v", err)
}
t.Cleanup(func() { _ = tn.Close() })
res := c.ReloadConfigString("tun:\n disabled: true\n dev: nebrk1\n tx_queue: 100\n")
require.NoError(t, res.Err)
assert.ElementsMatch(t, []string{"tun.disabled", "tun.dev", "tun.tx_queue"}, res.RestartKeys)
}
+24
View File
@@ -0,0 +1,24 @@
package overlay
import (
"net/netip"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// tun.disabled and the disabled tun's settings are only read when the device is made
func TestDisabledTunRestartKeys(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n disabled: true\n"))
d, err := NewDeviceFromConfig(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, 1)
require.NoError(t, err)
t.Cleanup(func() { _ = d.Close() })
res := c.ReloadConfigString("tun:\n disabled: false\n tx_queue: 100\n")
require.NoError(t, res.Err)
assert.ElementsMatch(t, []string{"tun.disabled", "tun.tx_queue"}, res.RestartKeys)
}
+28 -10
View File
@@ -84,7 +84,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
l: l, l: l,
} }
err = t.reload(c, true) err = t.reload(c, true, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -108,17 +108,28 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
// Disable registration before Activate assigns addresses and triggers it. // Disable registration before Activate assigns addresses and triggers it.
disableDNSRegistration(l, *guid) disableDNSRegistration(l, *guid)
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
err := t.reload(c, false) r.RestartRequiredIfChanged("tun.dev", "tun.mtu")
// Compare with the running category, not the last config. A restart with one we can't parse would fail, so say
// that now rather than ask for one, and going back to the running one after a bad reload needs no restart
if cat, setCat, err := parseNetworkCategory(c.GetString("tun.network_category", "private")); err != nil {
t.l.Error("Invalid tun.network_category, a restart with it would fail", "error", err)
r.Failed()
} else if cat != t.networkCategory || setCat != t.setCategory {
r.RestartRequiredIfChanged("tun.network_category")
}
r.RestartRequiredIfChanged(interfaceBypassKeys...)
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
return t, nil return t, nil
} }
func (t *winTun) reload(c *config.C, initial bool) error { func (t *winTun) reload(c *config.C, initial bool, r *config.Reload) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
@@ -139,16 +150,18 @@ func (t *winTun) reload(c *config.C, initial bool) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes)) err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes), r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true) err = t.addRoutes(true, r)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -163,7 +176,7 @@ func (t *winTun) Activate() error {
return fmt.Errorf("failed to set address: %w", err) return fmt.Errorf("failed to set address: %w", err)
} }
err = t.addRoutes(false) err = t.addRoutes(false, nil)
if err != nil { if err != nil {
return err return err
} }
@@ -182,7 +195,7 @@ func (t *winTun) Activate() error {
return nil return nil
} }
func (t *winTun) addRoutes(logErrors bool) error { func (t *winTun) addRoutes(logErrors bool, rl *config.Reload) error {
luid := winipcfg.LUID(t.tun.LUID()) luid := winipcfg.LUID(t.tun.LUID())
routes := *t.Routes.Load() routes := *t.Routes.Load()
foundDefault4 := false foundDefault4 := false
@@ -213,6 +226,7 @@ func (t *winTun) addRoutes(logErrors bool) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
continue continue
} else { } else {
return retErr return retErr
@@ -274,7 +288,7 @@ func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error
return nil return nil
} }
func (t *winTun) removeRoutes(routes []Route) error { func (t *winTun) removeRoutes(routes []Route, rl *config.Reload) error {
luid := winipcfg.LUID(t.tun.LUID()) luid := winipcfg.LUID(t.tun.LUID())
for _, r := range routes { for _, r := range routes {
@@ -284,8 +298,12 @@ func (t *winTun) removeRoutes(routes []Route) error {
// See comment on luid.AddRoute // See comment on luid.AddRoute
err := luid.DeleteRoute(r.Cidr, unspecifiedNextHop(r.Cidr)) err := luid.DeleteRoute(r.Cidr, unspecifiedNextHop(r.Cidr))
if err != nil { if errors.Is(err, windows.ERROR_NOT_FOUND) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
+46 -5
View File
@@ -53,6 +53,14 @@ func (rt *routeTest) metric(t *testing.T, cidr string) uint32 {
return row.Metric return row.Metric
} }
// reload applies cfg and expects nothing in it to fail
func (rt *routeTest) reload(t *testing.T, cfg string) {
t.Helper()
res := rt.c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed, "the reload reported a failure")
}
func unsafeRoutes(routes ...string) string { func unsafeRoutes(routes ...string) string {
var b strings.Builder var b strings.Builder
b.WriteString("tun:\n dev: nebroutetest\n unsafe_routes:\n") b.WriteString("tun:\n dev: nebroutetest\n unsafe_routes:\n")
@@ -77,7 +85,7 @@ func TestWinTun_ReloadKeepsInstalledRoutes(t *testing.T) {
rt := newRouteTest(t, unsafeRoutes(a)) rt := newRouteTest(t, unsafeRoutes(a))
require.NoError(t, rt.tun.Activate()) require.NoError(t, rt.tun.Activate())
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a, b))) rt.reload(t, unsafeRoutes(a, b))
assert.NotContains(t, rt.log.String(), "Failed to add route") assert.NotContains(t, rt.log.String(), "Failed to add route")
rt.metric(t, a) rt.metric(t, a)
rt.metric(t, b) rt.metric(t, b)
@@ -92,7 +100,7 @@ func TestWinTun_ReloadMetricChangeApplies(t *testing.T) {
require.NoError(t, rt.tun.Activate()) require.NoError(t, rt.tun.Activate())
require.Equal(t, uint32(100), rt.metric(t, a)) require.Equal(t, uint32(100), rt.metric(t, a))
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200"))) rt.reload(t, unsafeRoutes(a+"@200"))
assert.NotContains(t, rt.log.String(), "Failed to add route") assert.NotContains(t, rt.log.String(), "Failed to add route")
assert.NotContains(t, rt.log.String(), "Removed route") assert.NotContains(t, rt.log.String(), "Removed route")
assert.Equal(t, uint32(200), rt.metric(t, a)) assert.Equal(t, uint32(200), rt.metric(t, a))
@@ -120,7 +128,7 @@ func TestWinTun_DuplicateCidrInstallsTheLastEntry(t *testing.T) {
require.NoError(t, rt.tun.Activate()) require.NoError(t, rt.tun.Activate())
assert.Equal(t, uint32(200), rt.metric(t, a)) assert.Equal(t, uint32(200), rt.metric(t, a))
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@100", a+"@200", b))) rt.reload(t, unsafeRoutes(a+"@100", a+"@200", b))
assert.Equal(t, uint32(200), rt.metric(t, a)) assert.Equal(t, uint32(200), rt.metric(t, a))
assert.NotContains(t, rt.log.String(), "Updated route metric") assert.NotContains(t, rt.log.String(), "Updated route metric")
assert.NotContains(t, rt.log.String(), "Failed to add route") assert.NotContains(t, rt.log.String(), "Failed to add route")
@@ -134,7 +142,7 @@ func TestWinTun_DroppingAnUninstalledDuplicateKeepsTheRoute(t *testing.T) {
rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200")) rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200"))
require.NoError(t, rt.tun.Activate()) require.NoError(t, rt.tun.Activate())
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200"))) rt.reload(t, unsafeRoutes(a+"@200"))
assert.NotContains(t, rt.log.String(), "Removed route") assert.NotContains(t, rt.log.String(), "Removed route")
assert.Equal(t, uint32(200), rt.metric(t, a)) assert.Equal(t, uint32(200), rt.metric(t, a))
}) })
@@ -155,12 +163,45 @@ func TestWinTun_ReloadMatchesAFreshStart(t *testing.T) {
rt := newRouteTest(t, unsafeRoutes(a+"@100")) rt := newRouteTest(t, unsafeRoutes(a+"@100"))
require.NoError(t, rt.tun.Activate()) require.NoError(t, rt.tun.Activate())
require.NoError(t, rt.c.ReloadConfigString(routes)) rt.reload(t, routes)
_, err = rt.luid.Route(cidr, unspecifiedNextHop(cidr)) _, err = rt.luid.Route(cidr, unspecifiedNextHop(cidr))
assert.Error(t, err, "the reload left it installed") assert.Error(t, err, "the reload left it installed")
}) })
} }
// Removing a route that is already gone is what we wanted, not a failure
func TestWinTun_ReloadRouteAlreadyGone(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a, b))
require.NoError(t, rt.tun.Activate())
p := netip.MustParsePrefix(b)
require.NoError(t, rt.luid.DeleteRoute(p, unspecifiedNextHop(p)))
rt.reload(t, unsafeRoutes(a))
})
}
// A network category we can't parse would fail the restart, so it fails the reload instead of asking for one
func TestWinTun_ReloadInvalidNetworkCategoryFails(t *testing.T) {
rt := newRouteTest(t, unsafeRoutes())
for range 2 {
res := rt.c.ReloadConfigString(unsafeRoutes() + " network_category: nope\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed, "a bad category that didn't change since the last reload is still bad")
assert.False(t, res.RestartRequired)
}
// Fixing it back to the running category needs nothing
res := rt.c.ReloadConfigString(unsafeRoutes() + " network_category: private\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired)
res = rt.c.ReloadConfigString(unsafeRoutes() + " network_category: public\n")
require.NoError(t, res.Err)
assert.Equal(t, []string{"tun.network_category"}, res.RestartKeys)
}
// forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4 // forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4
func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) { func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) {
t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") }) t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") })
+70 -18
View File
@@ -2,7 +2,9 @@ package nebula
import ( import (
"crypto/fips140" "crypto/fips140"
"crypto/sha256"
"encoding/binary" "encoding/binary"
"encoding/hex"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@@ -50,15 +52,30 @@ type CertState struct {
myVpnBroadcastAddrsTable *bart.Lite myVpnBroadcastAddrsTable *bart.Lite
} }
// errCertNeedsRestart marks a new cert that is refused on reload because only a restart can apply it
var errCertNeedsRestart = errors.New("a restart is required to use the new cert")
func NewPKIFromConfig(l *slog.Logger, c *config.C) (*PKI, error) { func NewPKIFromConfig(l *slog.Logger, c *config.C) (*PKI, error) {
pki := &PKI{l: l} pki := &PKI{l: l}
err := pki.reload(c, true) err := pki.reload(c, true, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
rErr := pki.reload(c, false) // Compare with the running cipher, not the last config. A restart with one we can't run would fail, so say that
// now rather than ask for one, and going back to the running one after a bad reload needs no restart
if cipher := c.GetString("cipher", "aes"); !validCipher(cipher) {
l.Error("Unknown cipher, a restart with it would fail", "cipher", cipher)
r.Failed()
} else if cipher == "chachapoly" && fipsOnly() {
// newCipherSuite refuses it at startup
l.Error("ChaChaPoly is not allowed in FIPS 140-only mode, a restart with it would fail", "cipher", cipher)
r.Failed()
} else if cipher != pki.getCertState().cipher {
r.RestartRequiredIfChanged("cipher")
}
rErr := pki.reload(c, false, r)
if rErr != nil { if rErr != nil {
util.LogWithContextIfNeeded("Failed to reload PKI from config", rErr, l) util.LogWithContextIfNeeded("Failed to reload PKI from config", rErr, l)
} }
@@ -75,13 +92,29 @@ func (p *PKI) getCertState() *CertState {
return p.cs.Load() return p.cs.Load()
} }
func (p *PKI) reload(c *config.C, initial bool) error { // r is nil on the initial load
func (p *PKI) reload(c *config.C, initial bool, r *config.Reload) error {
before := p.cs.Load()
err := p.reloadCerts(c, initial) err := p.reloadCerts(c, initial)
if err != nil { if err != nil {
if initial { if initial {
return err return err
} }
err.Log(p.l) err.Log(p.l)
if errors.Is(err, errCertNeedsRestart) {
r.RestartRequired()
} else {
r.Failed()
}
} else if !initial {
// A cert for a version we weren't running is accepted without comparing its networks to ours. The tun and the
// lighthouse keep the networks they started with
after := p.cs.Load()
if !slices.Equal(before.myVpnNetworks, after.myVpnNetworks) {
p.l.Warn("The new cert changes our networks, a restart is required to use them",
"old_networks", before.myVpnNetworks, "new_networks", after.myVpnNetworks)
r.RestartRequired()
}
} }
err = p.reloadCAPool(c) err = p.reloadCAPool(c)
@@ -90,21 +123,27 @@ func (p *PKI) reload(c *config.C, initial bool) error {
return err return err
} }
err.Log(p.l) err.Log(p.l)
r.Failed()
} }
return nil return nil
} }
// fipsOnly is fips140.Enforced, a variable so a test can run the FIPS 140-only reload check without GODEBUG
var fipsOnly = fips140.Enforced
// validCipher is true for the ciphers nebula can run. Each post-handshake CipherState in noiseutil hardcodes its own
// nonce endianness, so there's nothing to set up for either.
func validCipher(cipher string) bool {
return cipher == "aes" || cipher == "chachapoly"
}
func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError { func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
var cipher string var cipher string
var currentState *CertState var currentState *CertState
if initial { if initial {
cipher = c.GetString("cipher", "aes") cipher = c.GetString("cipher", "aes")
switch cipher { if !validCipher(cipher) {
case "aes", "chachapoly":
// Each post-handshake CipherState in noiseutil hardcodes its own
// nonce endianness now, so there's nothing to set up here.
default:
return util.NewContextualError( return util.NewContextualError(
"unknown cipher", "unknown cipher",
m{"cipher": cipher}, m{"cipher": cipher},
@@ -132,7 +171,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Networks in new cert was different from old", "Networks in new cert was different from old",
m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks(), "cert_version": cert.Version1}, m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks(), "cert_version": cert.Version1},
nil, errCertNeedsRestart,
) )
} }
@@ -140,7 +179,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Curve in new v1 cert was different from old", "Curve in new v1 cert was different from old",
m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve(), "cert_version": cert.Version1}, m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve(), "cert_version": cert.Version1},
nil, errCertNeedsRestart,
) )
} }
} }
@@ -155,7 +194,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Networks in new cert was different from old", "Networks in new cert was different from old",
m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks(), "cert_version": cert.Version2}, m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks(), "cert_version": cert.Version2},
nil, errCertNeedsRestart,
) )
} }
@@ -163,7 +202,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Curve in new cert was different from old", "Curve in new cert was different from old",
m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve(), "cert_version": cert.Version2}, m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve(), "cert_version": cert.Version2},
nil, errCertNeedsRestart,
) )
} }
} }
@@ -178,7 +217,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Removing a V2 cert is not permitted unless it has identical networks to the new V1 cert", "Removing a V2 cert is not permitted unless it has identical networks to the new V1 cert",
m{"new_v1_networks": newState.v1Cert.Networks(), "old_v2_networks": currentState.v2Cert.Networks()}, m{"new_v1_networks": newState.v1Cert.Networks(), "old_v2_networks": currentState.v2Cert.Networks()},
nil, errCertNeedsRestart,
) )
} }
} }
@@ -196,12 +235,14 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError { func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError {
caPool, err := loadCAPoolFromConfig(p.l, c) caPool, err := loadCAPoolFromConfig(p.l, c)
if caPool != nil {
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
}
if err != nil { if err != nil {
return util.NewContextualError("Failed to load ca from config", nil, err) return util.NewContextualError("Failed to load ca from config", nil, err)
} }
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
return nil return nil
} }
@@ -576,11 +617,22 @@ func loadCAPoolFromConfig(l *slog.Logger, c *config.C) (*cert.CAPool, error) {
bl := c.GetStringSlice("pki.blocklist", []string{}) bl := c.GetStringSlice("pki.blocklist", []string{})
if len(bl) > 0 { if len(bl) > 0 {
for _, fp := range bl { var bad []string
for i, raw := range bl {
// Fingerprints are compared as lowercase hex strings, a malformed entry would silently block nothing
fp := strings.ToLower(strings.TrimSpace(raw))
if b, err := hex.DecodeString(fp); err != nil || len(b) != sha256.Size {
bad = append(bad, fmt.Sprintf("entry %d %q", i+1, raw))
continue
}
caPool.BlocklistFingerprint(fp) caPool.BlocklistFingerprint(fp)
} }
l.Info("Blocklisted certificates", "fingerprintCount", len(bl)) l.Info("Blocklisted certificates", "fingerprintCount", len(bl)-len(bad))
if len(bad) > 0 {
// The valid entries are still returned so a failed reload blocks them, startup refuses the error
return caPool, fmt.Errorf("pki.blocklist entries are not hex certificate fingerprints: %s", strings.Join(bad, ", "))
}
} }
return caPool, nil return caPool, nil
+168
View File
@@ -1,10 +1,17 @@
package nebula package nebula
import ( import (
"crypto/fips140"
"fmt"
"net/netip"
"strings" "strings"
"testing" "testing"
"time"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -92,3 +99,164 @@ func TestNewCipherSuiteUnsupportedCurve(t *testing.T) {
assert.True(t, strings.HasPrefix(err.Error(), "unsupported curve:"), "got: %v", err) assert.True(t, strings.HasPrefix(err.Error(), "unsupported curve:"), "got: %v", err)
assert.Nil(t, cs) assert.Nil(t, cs)
} }
func TestPKIReloadRestartRequired(t *testing.T) {
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
pkiConfig := func(cipher, network string) string {
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
return fmt.Sprintf("cipher: %s\npki:\n ca: %q\n cert: %q\n key: %q\n", cipher, caPEM, certPEM, keyPEM)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig("aes", "10.1.0.2/24")))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
// A cert that changes our networks is refused and needs a restart, it is not a failure
res := c.ReloadConfigString(pkiConfig("aes", "10.1.0.3/24"))
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
assert.False(t, res.Failed)
assert.Equal(t, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, p.getCertState().myVpnNetworks)
// The cipher only applies at startup
c2 := config.NewC(test.NewLogger())
require.NoError(t, c2.LoadString(pkiConfig("aes", "10.1.0.2/24")))
p, err = NewPKIFromConfig(test.NewLogger(), c2)
require.NoError(t, err)
res = c2.ReloadConfigString(pkiConfig("chachapoly", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
assert.Equal(t, "aes", p.getCertState().cipher)
// A cipher we can't run would fail the restart, so it fails the reload instead of asking for one
c4 := config.NewC(test.NewLogger())
require.NoError(t, c4.LoadString(pkiConfig("aes", "10.1.0.2/24")))
_, err = NewPKIFromConfig(test.NewLogger(), c4)
require.NoError(t, err)
for range 2 {
res = c4.ReloadConfigString(pkiConfig("nope", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.True(t, res.Failed, "a bad cipher that didn't change since the last reload is still bad")
assert.False(t, res.RestartRequired)
}
// Fixing it back to the running cipher needs nothing
res = c4.ReloadConfigString(pkiConfig("aes", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired)
// A cert that keeps our networks applies without a restart
c3 := config.NewC(test.NewLogger())
require.NoError(t, c3.LoadString(pkiConfig("aes", "10.1.0.2/24")))
_, err = NewPKIFromConfig(test.NewLogger(), c3)
require.NoError(t, err)
res = c3.ReloadConfigString(pkiConfig("aes", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.False(t, res.RestartRequired)
// A cert that can't be read is a failure, and a restart wouldn't help
res = c3.ReloadConfigString(strings.Replace(pkiConfig("aes", "10.1.0.2/24"), "cert: \"", "cert: \"garbage", 1))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.False(t, res.RestartRequired)
}
// A cert for a version we weren't running is accepted without comparing networks, so switching to one with different
// networks needs a restart. One with the same networks doesn't
func TestPKIReloadNewVersionWithOtherNetworks(t *testing.T) {
ca1, _, ca1Key, ca1PEM := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
ca2, _, ca2Key, ca2PEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
caPEM := string(ca1PEM) + string(ca2PEM)
pkiConfig := func(v cert.Version, network string) string {
ca, caKey := ca1, ca1Key
if v == cert.Version2 {
ca, caKey = ca2, ca2Key
}
_, _, keyPEM, certPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
return fmt.Sprintf("pki:\n ca: %q\n cert: %q\n key: %q\n", caPEM, certPEM, keyPEM)
}
for _, tc := range []struct {
network string
restart bool
}{
{"10.9.0.7/24", true},
{"10.1.0.2/24", false},
} {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig(cert.Version1, "10.1.0.2/24")))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig(cert.Version2, tc.network))
require.NoError(t, res.Err)
assert.False(t, res.Failed, tc.network)
assert.Equal(t, tc.restart, res.RestartRequired, tc.network)
assert.Equal(t, []netip.Prefix{netip.MustParsePrefix(tc.network)}, p.getCertState().myVpnNetworks, "accepted either way, as before")
}
}
// FIPS 140-only mode refuses chachapoly at startup, so reloading to it fails rather than asks for a restart
func TestPKIReloadChachapolyInFIPSOnlyFails(t *testing.T) {
fipsOnly = func() bool { return true }
t.Cleanup(func() { fipsOnly = fips140.Enforced })
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_P256, time.Time{}, time.Time{}, nil, nil, nil)
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_P256, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil)
pkiConfig := func(cipher string) string {
return fmt.Sprintf("cipher: %s\npki:\n ca: %q\n cert: %q\n key: %q\n", cipher, caPEM, certPEM, keyPEM)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig("aes")))
_, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig("chachapoly"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.False(t, res.RestartRequired)
}
func TestLoadCAPoolBlocklist(t *testing.T) {
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
crt, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil)
fp, err := crt.Fingerprint()
require.NoError(t, err)
load := func(entry string) (*cert.CAPool, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(fmt.Sprintf("pki:\n ca: %q\n blocklist:\n - %q\n", caPEM, entry)))
return loadCAPoolFromConfig(test.NewLogger(), c)
}
for _, entry := range []string{fp, strings.ToUpper(fp), " " + fp + "\t"} {
pool, err := load(entry)
require.NoError(t, err, entry)
assert.True(t, pool.IsBlocklisted(fp), entry)
}
for _, entry := range []string{fp[1:], fp + "00", "sha256:" + fp, "zz" + fp[2:], ""} {
_, err := load(entry)
require.Error(t, err, entry)
}
// A bad entry on reload fails it, the valid entries are still blocked
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "me", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.1/24")}, nil, nil)
pkiConfig := func(entry string) string {
return fmt.Sprintf("pki:\n ca: %q\n cert: %q\n key: %q\n blocklist:\n - %q\n", caPEM, certPEM, keyPEM, entry)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig(fp)))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
other, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "other", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.3/24")}, nil, nil)
otherFp, err := other.Fingerprint()
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig(fp) + fmt.Sprintf(" - %q\n - %q\n", otherFp, fp[1:]))
assert.True(t, res.Failed)
assert.True(t, p.GetCAPool().IsBlocklisted(fp))
assert.True(t, p.GetCAPool().IsBlocklisted(otherFp))
}
+2 -1
View File
@@ -67,7 +67,8 @@ func NewPunchyFromConfig(l *slog.Logger, c *config.C, punchConn udp.Conn) *Punch
} }
p.reload(c, true) p.reload(c, true)
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("stats.lighthouse_metrics")
p.reload(c, false) p.reload(c, false)
}) })
+16 -6
View File
@@ -76,7 +76,7 @@ punchy:
punch: true punch: true
delay: 10m delay: 10m
respond: true respond: true
`)) `).Err)
p.reload(c, false) p.reload(c, false)
assert.True(t, p.punch.Load()) assert.True(t, p.punch.Load())
assert.Equal(t, newDelay, time.Duration(p.delay.Load())) assert.Equal(t, newDelay, time.Duration(p.delay.Load()))
@@ -170,7 +170,7 @@ func TestPunchy_LogFormat_ReloadPunch(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {punch: true}`)) require.NoError(t, c.ReloadConfigString(`punchy: {punch: true}`).Err)
entry := findEntry(t, hook.entries, "punchy.punch changed") entry := findEntry(t, hook.entries, "punchy.punch changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -184,7 +184,7 @@ func TestPunchy_LogFormat_ReloadRespond(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {respond: true}`)) require.NoError(t, c.ReloadConfigString(`punchy: {respond: true}`).Err)
entry := findEntry(t, hook.entries, "punchy.respond changed") entry := findEntry(t, hook.entries, "punchy.respond changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -198,7 +198,7 @@ func TestPunchy_LogFormat_ReloadDelay(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {delay: 10s}`)) require.NoError(t, c.ReloadConfigString(`punchy: {delay: 10s}`).Err)
entry := findEntry(t, hook.entries, "punchy.delay changed") entry := findEntry(t, hook.entries, "punchy.delay changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -212,7 +212,7 @@ func TestPunchy_LogFormat_ReloadTargetAllRemotes(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {target_all_remotes: true}`)) require.NoError(t, c.ReloadConfigString(`punchy: {target_all_remotes: true}`).Err)
entry := findEntry(t, hook.entries, "punchy.target_all_remotes changed") entry := findEntry(t, hook.entries, "punchy.target_all_remotes changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -226,9 +226,19 @@ func TestPunchy_LogFormat_ReloadRespondDelay(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {respond_delay: 15s}`)) require.NoError(t, c.ReloadConfigString(`punchy: {respond_delay: 15s}`).Err)
entry := findEntry(t, hook.entries, "punchy.respond_delay changed") entry := findEntry(t, hook.entries, "punchy.respond_delay changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
assert.Equal(t, map[string]any{"respond_delay": 15 * time.Second}, entry.Attrs) assert.Equal(t, map[string]any{"respond_delay": 15 * time.Second}, entry.Attrs)
} }
func TestPunchyReloadRestartRequired(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("stats:\n lighthouse_metrics: false\n"))
NewPunchyFromConfig(test.NewLogger(), c, nil)
res := c.ReloadConfigString("stats:\n lighthouse_metrics: true\n")
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
}
+2 -1
View File
@@ -28,10 +28,11 @@ func NewRelayManager(ctx context.Context, l *slog.Logger, hostmap *HostMap, c *c
hostmap: hostmap, hostmap: hostmap,
} }
rm.reload(c, true) rm.reload(c, true)
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
err := rm.reload(c, false) err := rm.reload(c, false)
if err != nil { if err != nil {
rm.l.Error("Failed to reload relay_manager", "error", err) rm.l.Error("Failed to reload relay_manager", "error", err)
r.Failed()
} }
}) })
return rm return rm
+27 -10
View File
@@ -1,8 +1,8 @@
package service package service
import ( import (
"io"
"net" "net"
"sync"
) )
type tcpListener struct { type tcpListener struct {
@@ -10,23 +10,40 @@ type tcpListener struct {
s *Service s *Service
addr *net.TCPAddr addr *net.TCPAddr
accept chan net.Conn accept chan net.Conn
// closed is closed instead of accept so a handler blocked handing off a
// connection can't send on a closed channel.
closed chan struct{}
closeOnce sync.Once
} }
func (l *tcpListener) Accept() (net.Conn, error) { func (l *tcpListener) Accept() (net.Conn, error) {
conn, ok := <-l.accept // select picks randomly among ready cases, so without this a handler still
if !ok { // waiting to hand off could win over closed after Close has returned.
return nil, io.EOF select {
case <-l.closed:
return nil, net.ErrClosed
default:
}
select {
case conn := <-l.accept:
return conn, nil
case <-l.closed:
return nil, net.ErrClosed
} }
return conn, nil
} }
func (l *tcpListener) Close() error { func (l *tcpListener) Close() error {
l.s.mu.Lock() l.closeOnce.Do(func() {
defer l.s.mu.Unlock() // Close before leaving the map so a concurrent closeListeners that
delete(l.s.mu.listeners, uint16(l.addr.Port)) // no longer sees this listener knows Accept is already released.
close(l.closed)
close(l.accept)
l.s.mu.Lock()
delete(l.s.mu.listeners, l.port)
l.s.mu.Unlock()
})
return nil return nil
} }
+79 -10
View File
@@ -32,13 +32,18 @@ import (
const nicID = 1 const nicID = 1
type Service struct { type Service struct {
eg *errgroup.Group
control *nebula.Control control *nebula.Control
ipstack *stack.Stack ipstack *stack.Stack
// done is closed once the forwarding goroutines have exited and the
// netstack is torn down, at which point err holds the first fatal error.
done chan struct{}
err error
mu struct { mu struct {
sync.Mutex sync.Mutex
closed bool
listeners map[uint16]*tcpListener listeners map[uint16]*tcpListener
} }
} }
@@ -65,7 +70,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
ctx := control.Context() ctx := control.Context()
eg, ctx := errgroup.WithContext(ctx) eg, ctx := errgroup.WithContext(ctx)
s := Service{ s := Service{
eg: eg,
control: control, control: control,
} }
s.mu.listeners = map[uint16]*tcpListener{} s.mu.listeners = map[uint16]*tcpListener{}
@@ -74,6 +78,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol}, NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6}, TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6},
}) })
// The teardown goroutine only owns the stack once New succeeds
defer func() {
if reterr != nil {
s.ipstack.Destroy()
}
}()
sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default
tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt) tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt)
if tcpipErr != nil { if tcpipErr != nil {
@@ -123,6 +133,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
// this will read exactly one packet // this will read exactly one packet
n, err := reader.Read(buf) n, err := reader.Read(buf)
if err != nil { if err != nil {
// An error after shutdown began is noise from the device pipes
// closing, anything else is fatal and must stop nebula too
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err return err
} }
packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{ packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{
@@ -130,8 +146,8 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
}) })
linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf) linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf)
if err := ctx.Err(); err != nil { if ctx.Err() != nil {
return err return nil
} }
} }
}) })
@@ -139,13 +155,18 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
for { for {
packet := linkEP.ReadContext(ctx) packet := linkEP.ReadContext(ctx)
if packet == nil { if packet == nil {
if err := ctx.Err(); err != nil { if ctx.Err() != nil {
return err return nil
} }
continue continue
} }
bufView := packet.ToView() bufView := packet.ToView()
if _, err := bufView.WriteTo(writer); err != nil { if _, err := bufView.WriteTo(writer); err != nil {
// Same shutdown noise handling as the inbound reader
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err return err
} }
bufView.Release() bufView.Release()
@@ -158,6 +179,22 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
return control.Wait() return control.Wait()
}) })
// Tear down here rather than in Close so it also happens when nebula stops
// on its own. The forwarding goroutines must exit before the stack they feed
// is destroyed.
s.done = make(chan struct{})
go func() {
s.err = eg.Wait()
s.closeListeners()
// Destroy aborts the endpoints that exist when it starts, then waits on
// every endpoint, so a Dial registering in between would hold Wait until
// its connect timed out. Without the NIC new dials fail route lookup.
// RemoveNIC also closes linkEP.
_ = s.ipstack.RemoveNIC(nicID)
s.ipstack.Destroy()
close(s.done)
}()
return &s, nil return &s, nil
} }
@@ -231,11 +268,15 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
s: s, s: s,
addr: addr, addr: addr,
accept: make(chan net.Conn), accept: make(chan net.Conn),
closed: make(chan struct{}),
} }
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
if s.mu.closed {
return nil, net.ErrClosed
}
if _, ok := s.mu.listeners[port]; ok { if _, ok := s.mu.listeners[port]; ok {
return nil, fmt.Errorf("already listening on port %d", port) return nil, fmt.Errorf("already listening on port %d", port)
} }
@@ -244,22 +285,44 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
return l, nil return l, nil
} }
// Wait blocks until the service has fully stopped, either via Close or a fatal
// error, and returns the first fatal error if there was one.
func (s *Service) Wait() error { func (s *Service) Wait() error {
return s.eg.Wait() <-s.done
return s.err
} }
// Close closes all listeners and stops nebula. Use Wait to block until
// shutdown has completed.
func (s *Service) Close() error { func (s *Service) Close() error {
s.closeListeners()
s.control.Stop() s.control.Stop()
return nil return nil
} }
// closeListeners unblocks pending Accept calls and rejects new listeners.
func (s *Service) closeListeners() {
// Every caller closes every listener rather than returning early once
// closed is set, so none returns while another is still closing them.
s.mu.Lock()
s.mu.closed = true
listeners := make([]*tcpListener, 0, len(s.mu.listeners))
for _, l := range s.mu.listeners {
listeners = append(listeners, l)
}
s.mu.Unlock()
for _, l := range listeners {
_ = l.Close()
}
}
func (s *Service) tcpHandler(r *tcp.ForwarderRequest) { func (s *Service) tcpHandler(r *tcp.ForwarderRequest) {
endpointID := r.ID() endpointID := r.ID()
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock()
l, ok := s.mu.listeners[endpointID.LocalPort] l, ok := s.mu.listeners[endpointID.LocalPort]
s.mu.Unlock()
if !ok { if !ok {
r.Complete(true) r.Complete(true)
return return
@@ -275,6 +338,12 @@ func (s *Service) tcpHandler(r *tcp.ForwarderRequest) {
r.Complete(false) r.Complete(false)
ep.SocketOptions().SetKeepAlive(true) ep.SocketOptions().SetKeepAlive(true)
// The handoff blocks until Accept is called, so it must not hold s.mu or a
// listener nobody accepts from stalls every other port.
conn := gonet.NewTCPConn(&wq, ep) conn := gonet.NewTCPConn(&wq, ep)
l.accept <- conn select {
case l.accept <- conn:
case <-l.closed:
_ = conn.Close()
}
} }
+298
View File
@@ -188,3 +188,301 @@ func TestService(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
} }
// A listener that is never accepted from must not stall connections to other
// ports, and closing it must not block.
func TestServiceUnacceptedListener(t *testing.T) {
lighthousePort := ephemeralUDPPort(t)
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"lighthouse": m{
"am_lighthouse": true,
},
"listen": m{
"host": "0.0.0.0",
"port": lighthousePort,
},
})
t.Cleanup(func() { _ = a.Close() })
b := newSimpleService(ca, caKey, "b", netip.MustParseAddr("10.0.0.2"), m{
"static_host_map": m{
"10.0.0.1": []string{fmt.Sprintf("127.0.0.1:%d", lighthousePort)},
},
"lighthouse": m{
"hosts": []string{"10.0.0.1"},
"interval": 1,
},
})
t.Cleanup(func() { _ = b.Close() })
idle, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
active, err := a.Listen("tcp", ":1235")
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
idleConn, err := b.DialContext(ctx, "tcp", "10.0.0.1:1234")
if err != nil {
t.Fatal(err)
}
defer idleConn.Close()
accepted := make(chan error, 1)
go func() {
conn, err := active.Accept()
if err == nil {
err = conn.Close()
}
accepted <- err
}()
ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
activeConn, err := b.DialContext(ctx, "tcp", "10.0.0.1:1235")
if err != nil {
t.Fatalf("dial to accepted port stalled behind idle listener: %v", err)
}
defer activeConn.Close()
select {
case err := <-accepted:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("accept stalled behind idle listener")
}
closed := make(chan error, 1)
go func() { closed <- idle.Close() }()
select {
case err := <-closed:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("closing idle listener blocked")
}
if _, err := idle.Accept(); !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from closed listener, got %v", err)
}
if err := idle.Close(); err != nil {
t.Fatalf("second close: %v", err)
}
}
// Accept must report net.ErrClosed after Close even while a handler is still
// trying to hand off a connection.
func TestListenerAcceptAfterClose(t *testing.T) {
s := &Service{}
s.mu.listeners = map[uint16]*tcpListener{}
l := &tcpListener{
port: 1234,
s: s,
accept: make(chan net.Conn),
closed: make(chan struct{}),
}
s.mu.listeners[l.port] = l
if err := l.Close(); err != nil {
t.Fatal(err)
}
// Keep a send pending so every Accept has a connection ready alongside closed.
stop := make(chan struct{})
done := make(chan struct{})
go func() {
defer close(done)
for {
server, client := net.Pipe()
_ = client.Close()
select {
case l.accept <- server:
case <-stop:
_ = server.Close()
return
}
}
}()
defer func() {
close(stop)
<-done
}()
for range 100 {
time.Sleep(time.Millisecond)
conn, err := l.Accept()
if conn != nil {
_ = conn.Close()
}
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("Accept after Close: got err %v, want net.ErrClosed", err)
}
}
}
// Close must release blocked Accept calls and let Wait report a clean shutdown.
func TestServiceClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
if err := a.Close(); err != nil {
t.Fatal(err)
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after Close")
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after Close")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
if err := a.Close(); err != nil {
t.Fatalf("second close: %v", err)
}
if _, err := a.Listen("tcp", ":1235"); !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Listen after Close, got %v", err)
}
}
// If nebula stops without Close being called, the service must still release
// Accept and tear down the netstack.
func TestServiceNebulaStopped(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
a.control.Stop()
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after nebula stopped")
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after nebula stopped")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
}
// Dials racing Close must not hold Wait until their connect times out.
func TestServiceDialDuringClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
stop := make(chan struct{})
var eg errgroup.Group
for range 8 {
eg.Go(func() error {
for {
select {
case <-stop:
return nil
default:
}
// 10.0.0.2 has no tunnel, so each dial sits in SYN-SENT until aborted
if c, err := a.Dial("tcp", "10.0.0.2:80"); err == nil {
_ = c.Close()
}
}
})
}
time.Sleep(50 * time.Millisecond)
if err := a.Close(); err != nil {
t.Fatal(err)
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait blocked behind a dial that raced Close")
}
close(stop)
dialersDone := make(chan error, 1)
go func() { dialersDone <- eg.Wait() }()
select {
case <-dialersDone:
case <-time.After(5 * time.Second):
t.Fatal("Dial still blocked after Wait")
}
}
+60 -192
View File
@@ -11,11 +11,8 @@ import (
"net" "net"
"net/netip" "net/netip"
"os" "os"
"path/filepath" "slices"
"runtime"
"runtime/pprof"
"sort" "sort"
"strconv"
"strings" "strings"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
@@ -58,12 +55,28 @@ type sshDeviceInfoFlags struct {
} }
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
if c.GetBool("sshd.enabled", false) { if c.GetBool("sshd.enabled", false) {
sshRun, err := configSSH(l, ssh, c) // Restarting the server drops every open session, the one that asked for this reload included. Only restart
// when it isn't running or its config changed. The host key file can change under the same config, new
// handshakes use the new key without a restart
if ssh.Running() && !slices.ContainsFunc(sshdServerKeys, c.HasChanged) {
key, err := loadSSHHostKey(c)
if err == nil {
err = ssh.SetHostKey(key)
}
if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err)
ssh.Stop()
r.Failed()
}
return
}
sshRun, err := configSSH(l, ssh, c, r)
if err != nil { if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err) l.Error("Failed to reconfigure the sshd", "error", err)
ssh.Stop() ssh.Stop()
r.Failed()
} }
if sshRun != nil { if sshRun != nil {
go sshRun() go sshRun()
@@ -74,11 +87,31 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
}) })
} }
// sshdServerKeys are what configSSH reads
var sshdServerKeys = []string{"sshd.enabled", "sshd.listen", "sshd.host_key", "sshd.trusted_cas", "sshd.authorized_users"}
// loadSSHHostKey returns sshd.host_key, read from its file when it names one.
func loadSSHHostKey(c *config.C) ([]byte, error) {
hostKeyPathOrKey := c.GetString("sshd.host_key", "")
if hostKeyPathOrKey == "" {
return nil, fmt.Errorf("sshd.host_key must be provided")
}
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
return []byte(hostKeyPathOrKey), nil
}
b, err := os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
return b, nil
}
// configSSH reads the ssh info out of the passed-in Config and // configSSH reads the ssh info out of the passed-in Config and
// updates the passed-in SSHServer. On success, it returns a function // updates the passed-in SSHServer. On success, it returns a function
// that callers may invoke to run the configured ssh server. On // that callers may invoke to run the configured ssh server. On
// failure, it returns nil, error. // failure, it returns nil, error.
func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) { // r is nil at startup, entries that are skipped with a warning fail it on a reload
func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reload) (func(), error) {
listen := c.GetString("sshd.listen", "") listen := c.GetString("sshd.listen", "")
if listen == "" { if listen == "" {
return nil, fmt.Errorf("sshd.listen must be provided") return nil, fmt.Errorf("sshd.listen must be provided")
@@ -92,19 +125,9 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
return nil, fmt.Errorf("sshd.listen can not use port 22") return nil, fmt.Errorf("sshd.listen can not use port 22")
} }
hostKeyPathOrKey := c.GetString("sshd.host_key", "") hostKeyBytes, err := loadSSHHostKey(c)
if hostKeyPathOrKey == "" { if err != nil {
return nil, fmt.Errorf("sshd.host_key must be provided") return nil, err
}
var hostKeyBytes []byte
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
hostKeyBytes = []byte(hostKeyPathOrKey)
} else {
hostKeyBytes, err = os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
} }
err = ssh.SetHostKey(hostKeyBytes) err = ssh.SetHostKey(hostKeyBytes)
@@ -121,6 +144,7 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
err := ssh.AddTrustedCA(caAuthorizedKey) err := ssh.AddTrustedCA(caAuthorizedKey)
if err != nil { if err != nil {
l.Warn("SSH CA had an error, ignoring", "error", err, "sshCA", caAuthorizedKey) l.Warn("SSH CA had an error, ignoring", "error", err, "sshCA", caAuthorizedKey)
r.Failed()
continue continue
} }
} }
@@ -132,12 +156,14 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
kDef, ok := rk.(map[string]any) kDef, ok := rk.(map[string]any)
if !ok { if !ok {
l.Warn("Authorized user had an error, ignoring", "sshKeyConfig", rk) l.Warn("Authorized user had an error, ignoring", "sshKeyConfig", rk)
r.Failed()
continue continue
} }
user, ok := kDef["user"].(string) user, ok := kDef["user"].(string)
if !ok { if !ok {
l.Warn("Authorized user is missing the user field", "sshKeyConfig", rk) l.Warn("Authorized user is missing the user field", "sshKeyConfig", rk)
r.Failed()
continue continue
} }
@@ -151,6 +177,7 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
"sshKeyConfig", rk, "sshKeyConfig", rk,
"sshKey", v, "sshKey", v,
) )
r.Failed()
continue continue
} }
@@ -162,6 +189,7 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
"sshKeyConfig", rk, "sshKeyConfig", rk,
"sshKey", subK, "sshKey", subK,
) )
r.Failed()
continue continue
} }
@@ -171,12 +199,14 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
"error", err, "error", err,
"sshKeyConfig", sk, "sshKeyConfig", sk,
) )
r.Failed()
continue continue
} }
} }
default: default:
l.Warn("Authorized user is missing the keys field or was not understood", "sshKeyConfig", rk) l.Warn("Authorized user is missing the keys field or was not understood", "sshKeyConfig", rk)
r.Failed()
} }
} }
} else { } else {
@@ -199,12 +229,6 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
} }
func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) { func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
// sandboxDir defaults to a dir in temp. The intention is that end user will
// create this dir as needed. Overriding this config value to "" allows
// writing to anywhere in the system.
defaultDir := filepath.Join(os.TempDir(), "nebula-debug")
sandboxDir := c.GetString("sshd.sandbox_dir", defaultDir)
ssh.RegisterCommand(&sshd.Command{ ssh.RegisterCommand(&sshd.Command{
Name: "list-hostmap", Name: "list-hostmap",
ShortDescription: "List all known previously connected hosts", ShortDescription: "List all known previously connected hosts",
@@ -256,46 +280,7 @@ func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interfa
Name: "reload", Name: "reload",
ShortDescription: "Reloads configuration from disk, same as sending HUP to the process", ShortDescription: "Reloads configuration from disk, same as sending HUP to the process",
Callback: func(fs any, a []string, w sshd.StringWriter) error { Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshReload(c, w) return sshReload(l, c, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "start-cpu-profile",
ShortDescription: "Starts a cpu profile and write output to the provided file, ex: `cpu-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshStartCpuProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "stop-cpu-profile",
ShortDescription: "Stops a cpu profile and writes output to the previously provided file",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
pprof.StopCPUProfile()
return w.WriteLine("If a CPU profile was running it is now stopped")
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "save-heap-profile",
ShortDescription: "Saves a heap profile to the provided path, ex: `heap-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetHeapProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "mutex-profile-fraction",
ShortDescription: "Gets or sets runtime.SetMutexProfileFraction",
Callback: sshMutexProfileFraction,
})
ssh.RegisterCommand(&sshd.Command{
Name: "save-mutex-profile",
ShortDescription: "Saves a mutex profile to the provided path, ex: `mutex-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetMutexProfile(sandboxDir, fs, a, w)
}, },
}) })
@@ -529,58 +514,6 @@ func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) er
return nil return nil
} }
// sshSanitizeFilePath validates that the given file path is within the sandbox directory.
// If sandboxDir is empty, the path is returned as-is for backwards compatibility.
func sshSanitizeFilePath(sandboxDir, filePath string) (string, error) {
if sandboxDir == "" {
return filePath, nil
}
// Clean and resolve the path relative to the sandbox directory
if !filepath.IsAbs(filePath) {
filePath = filepath.Join(sandboxDir, filePath)
}
cleaned := filepath.Clean(filePath)
// Ensure the resolved path is within the sandbox directory
cleanedSandbox := filepath.Clean(sandboxDir)
if cleaned == cleanedSandbox {
return "", fmt.Errorf("path %q resolves to the sandbox directory itself %q", filePath, sandboxDir)
}
if !strings.HasPrefix(cleaned, cleanedSandbox+string(filepath.Separator)) {
return "", fmt.Errorf("path %q is outside the sandbox directory %q", filePath, sandboxDir)
}
return cleaned, nil
}
func sshStartCpuProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
err := w.WriteLine("No path to write profile provided")
return err
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
}
err = pprof.StartCPUProfile(file)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to start cpu profile: %s", err))
return err
}
err = w.WriteLine(fmt.Sprintf("Started cpu profile, issue stop-cpu-profile to write the output to %s", a))
return err
}
func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
return w.WriteLine(fmt.Sprintf("%s", ifce.version)) return w.WriteLine(fmt.Sprintf("%s", ifce.version))
} }
@@ -733,76 +666,6 @@ func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) e
return w.WriteLine("Changed") return w.WriteLine("Changed")
} }
func sshGetHeapProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
}
err = pprof.WriteHeapProfile(file)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
return err
}
err = w.WriteLine(fmt.Sprintf("Mem profile created at %s", a))
return err
}
func sshMutexProfileFraction(fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
rate := runtime.SetMutexProfileFraction(-1)
return w.WriteLine(fmt.Sprintf("Current value: %d", rate))
}
newRate, err := strconv.Atoi(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("Invalid argument: %s", a[0]))
}
oldRate := runtime.SetMutexProfileFraction(newRate)
return w.WriteLine(fmt.Sprintf("New value: %d. Old value: %d", newRate, oldRate))
}
func sshGetMutexProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
return w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
}
defer file.Close()
mutexProfile := pprof.Lookup("mutex")
if mutexProfile == nil {
return w.WriteLine("Unable to get pprof.Lookup(\"mutex\")")
}
err = mutexProfile.WriteTo(file, 0)
if err != nil {
return w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
}
return w.WriteLine(fmt.Sprintf("Mutex profile created at %s", a))
}
func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error { func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
ctrl, ok := l.Handler().(interface { ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level GetLevel() slog.Level
@@ -1055,8 +918,13 @@ func sshDeviceInfo(ifce *Interface, fs any, w sshd.StringWriter) error {
} }
} }
func sshReload(c *config.C, w sshd.StringWriter) error { // sshReload reloads the config and writes the result to the client. It reloads even if the client has gone away
err := w.WriteLine("Reloading config") func sshReload(l *slog.Logger, c *config.C, w sshd.StringWriter) error {
c.ReloadConfig() writeErr := w.WriteLine("Reloading config")
return err res := c.ReloadConfig()
res.Log(l)
if err := w.WriteLine(res.String()); writeErr == nil {
writeErr = err
}
return writeErr
} }
+83
View File
@@ -0,0 +1,83 @@
package nebula
import (
"errors"
"io"
"os"
"path/filepath"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type lineWriter struct {
lines []string
err error
}
func (w *lineWriter) WriteLine(s string) error {
w.lines = append(w.lines, s)
return w.err
}
func (w *lineWriter) Write(s string) error { return w.WriteLine(s) }
func (w *lineWriter) WriteBytes(b []byte) error { return w.WriteLine(string(b)) }
func (w *lineWriter) GetWriter() io.Writer { return io.Discard }
func TestSSHReloadReportsTheResult(t *testing.T) {
dir := t.TempDir()
write := func(body string) {
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(body), 0o600))
}
write("a: 1\nb: 1")
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.Load(dir))
fail := false
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("b")
if fail {
r.Failed()
}
})
reload := func() []string {
w := &lineWriter{}
require.NoError(t, sshReload(l, c, w))
return w.lines
}
write("a: 2\nb: 1")
assert.Equal(t, []string{"Reloading config", "Config reload complete"}, reload())
fail = true
write("a: 3\nb: 1")
assert.Equal(t, []string{"Reloading config", "Config reload completed with errors, review the logs"}, reload())
fail = false
write("a: 3\nb: 2")
assert.Equal(t, []string{"Reloading config", "Config reload was completed but requires a restart to fully apply, keys [b]"}, reload())
write("a: [")
lines := reload()
require.Len(t, lines, 2)
assert.Contains(t, lines[1], "Config reload failed, the new config did not load")
assert.Contains(t, lines[1], dir, "names the config that did not load")
}
// A client that went away still gets the reload it asked for
func TestSSHReloadReloadsWhenTheClientIsGone(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte("a: 1"), 0o600))
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.Load(dir))
reloaded := false
c.RegisterReloadCallback(func(*config.C, *config.Reload) { reloaded = true })
gone := errors.New("client is gone")
require.ErrorIs(t, sshReload(l, c, &lineWriter{err: gone}), gone)
assert.True(t, reloaded)
}
+48 -13
View File
@@ -8,14 +8,18 @@ import (
"log/slog" "log/slog"
"net" "net"
"sync" "sync"
"sync/atomic"
"github.com/armon/go-radix" "github.com/armon/go-radix"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
) )
type SSHServer struct { type SSHServer struct {
config *ssh.ServerConfig // config is replaced whole when the host key changes, ssh.ServerConfig.AddHostKey only replaces a key of the same
l *slog.Logger // algorithm and appends any other
config atomic.Pointer[ssh.ServerConfig]
authenticate func(ssh.ConnMetadata, ssh.PublicKey) (*ssh.Permissions, error)
l *slog.Logger
certChecker *ssh.CertChecker certChecker *ssh.CertChecker
@@ -28,7 +32,12 @@ type SSHServer struct {
// List of available commands // List of available commands
helpCommand *Command helpCommand *Command
commands *radix.Tree commands *radix.Tree
listener net.Listener listener atomic.Pointer[net.Listener]
// runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned
runs atomic.Int32
// addr is where the newest Run listens, nil when none is
addr atomic.Pointer[net.Addr]
// ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even // ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even
// across reloads, since each Run derives a fresh child rather than reusing this one directly. // across reloads, since each Run derives a fresh child rather than reusing this one directly.
@@ -85,10 +94,8 @@ func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
}, },
} }
s.config = &ssh.ServerConfig{ s.authenticate = cc.Authenticate
PublicKeyCallback: cc.Authenticate, s.config.Store(s.newConfig())
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
s.RegisterCommand(&Command{ s.RegisterCommand(&Command{
Name: "help", Name: "help",
@@ -107,10 +114,32 @@ func (s *SSHServer) SetHostKey(hostPrivateKey []byte) error {
return fmt.Errorf("failed to parse private key: %s", err) return fmt.Errorf("failed to parse private key: %s", err)
} }
s.config.AddHostKey(private) config := s.newConfig()
config.AddHostKey(private)
s.config.Store(config)
return nil return nil
} }
func (s *SSHServer) newConfig() *ssh.ServerConfig {
return &ssh.ServerConfig{
PublicKeyCallback: s.authenticate,
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
}
// Addr is the address the newest Run listens on, nil when none is.
func (s *SSHServer) Addr() net.Addr {
if a := s.addr.Load(); a != nil {
return *a
}
return nil
}
// Running is true while a Run is listening.
func (s *SSHServer) Running() bool {
return s.runs.Load() > 0
}
func (s *SSHServer) ClearTrustedCAs() { func (s *SSHServer) ClearTrustedCAs() {
s.authLock.Lock() s.authLock.Lock()
s.trustedCAs = []ssh.PublicKey{} s.trustedCAs = []ssh.PublicKey{}
@@ -181,10 +210,16 @@ func (s *SSHServer) Run(addr string) error {
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's // this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
// listener before this run's watcher fires, so each run must close its own listener via the local // listener before this run's watcher fires, so each run must close its own listener via the local
// reference. // reference.
s.listener = listener s.listener.Store(&listener)
s.runs.Add(1)
bound := listener.Addr()
s.addr.Store(&bound)
runCtx, cancel := context.WithCancel(s.ctx) runCtx, cancel := context.WithCancel(s.ctx)
defer cancel() defer cancel()
// Deferred after cancel so they run first, a run on its way out no longer counts as listening
defer s.runs.Add(-1)
defer s.addr.CompareAndSwap(&bound, nil)
// Close the listener when this run's context is cancelled. That can come from the parent // Close the listener when this run's context is cancelled. That can come from the parent
// (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling // (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling
@@ -197,7 +232,7 @@ func (s *SSHServer) Run(addr string) error {
} }
}() }()
s.l.Info("SSH server is listening", "sshListener", addr) s.l.Info("SSH server is listening", "sshListener", bound)
// Run loops until there is an error // Run loops until there is an error
s.run(runCtx, listener) s.run(runCtx, listener)
@@ -226,7 +261,7 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
<-sessionContext.Done() <-sessionContext.Done()
c.Close() c.Close()
}() }()
conn, chans, reqs, err := ssh.NewServerConn(c, s.config) conn, chans, reqs, err := ssh.NewServerConn(c, s.config.Load())
fp := "" fp := ""
if conn != nil { if conn != nil {
fp = conn.Permissions.Extensions["fp"] fp = conn.Permissions.Extensions["fp"]
@@ -264,8 +299,8 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
} }
func (s *SSHServer) Stop() { func (s *SSHServer) Stop() {
if s.listener != nil { if l := s.listener.Load(); l != nil {
if err := s.listener.Close(); err != nil { if err := (*l).Close(); err != nil && !errors.Is(err, net.ErrClosed) {
s.l.Warn("Failed to close the sshd listener", "error", err) s.l.Warn("Failed to close the sshd listener", "error", err)
} }
} }
+240
View File
@@ -0,0 +1,240 @@
package nebula
import (
"crypto"
"crypto/ed25519"
"crypto/rand"
"crypto/rsa"
"encoding/pem"
"fmt"
"net"
"os"
"path/filepath"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
type sshdReloadHarness struct {
c *config.C
server *sshd.SSHServer
keyFile string
signer ssh.Signer
listen string
users []string
extra string
}
func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
t.Helper()
l := test.NewLogger()
h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key"), listen: "127.0.0.1:0", users: []string{"test"}}
h.writeHostKey(t, newEd25519Key(t))
_, userKey, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
h.signer, err = ssh.NewSignerFromKey(userKey)
require.NoError(t, err)
h.c = config.NewC(l)
require.NoError(t, h.c.LoadString(h.config()))
h.server, err = sshd.NewSSHServer(t.Context(), l)
require.NoError(t, err)
wireSSHReload(l, h.server, h.c)
run, err := configSSH(l, h.server, h.c, nil)
require.NoError(t, err)
go run()
t.Cleanup(h.server.Stop)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
return h
}
func newEd25519Key(t *testing.T) crypto.Signer {
_, key, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
return key
}
// writeHostKey replaces the host key file and returns the public key a client should now see
func (h *sshdReloadHarness) writeHostKey(t *testing.T, key crypto.Signer) ssh.PublicKey {
t.Helper()
block, err := ssh.MarshalPrivateKey(key, "")
require.NoError(t, err)
require.NoError(t, os.WriteFile(h.keyFile, pem.EncodeToMemory(block), 0o600))
pub, err := ssh.NewPublicKey(key.Public())
require.NoError(t, err)
return pub
}
func (h *sshdReloadHarness) config() string {
users := ""
for _, u := range h.users {
users += fmt.Sprintf(" - user: %s\n keys: [%q]\n", u, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())))
}
return fmt.Sprintf("sshd:\n enabled: true\n listen: %s\n host_key: %s\n authorized_users:\n%s%s",
h.listen, h.keyFile, users, h.extra)
}
// dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys
// the client accepts, none means any
func (h *sshdReloadHarness) dial(hostKeyAlgorithms ...string) (*ssh.Client, ssh.PublicKey, error) {
addr := h.server.Addr()
if addr == nil {
return nil, nil, fmt.Errorf("not listening")
}
var hostKey ssh.PublicKey
c, err := ssh.Dial("tcp", addr.String(), &ssh.ClientConfig{
User: "test",
Auth: []ssh.AuthMethod{ssh.PublicKeys(h.signer)},
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
hostKey = key
return nil
},
HostKeyAlgorithms: hostKeyAlgorithms,
Timeout: time.Second,
})
return c, hostKey, err
}
func (h *sshdReloadHarness) connect(t *testing.T) (*ssh.Client, ssh.PublicKey) {
t.Helper()
var client *ssh.Client
var hostKey ssh.PublicKey
require.Eventually(t, func() bool {
var err error
client, hostKey, err = h.dial()
return err == nil
}, 5*time.Second, 20*time.Millisecond)
t.Cleanup(func() { _ = client.Close() })
return client, hostKey
}
// dropped is true when the server closed the client's connection within wait
func dropped(client *ssh.Client, wait time.Duration) bool {
done := make(chan struct{})
go func() {
_ = client.Wait()
close(done)
}()
select {
case <-done:
return true
case <-time.After(wait):
return false
}
}
// Restarting sshd drops every open session, the one that asked for the reload included. A reload that touches nothing
// sshd reads leaves it alone
func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
require.NoError(t, h.c.ReloadConfigString(h.config()+"punchy:\n punch: true\n").Err)
assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted")
}
// Every key the server reads restarts it when it changes
func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
cases := map[string]func(t *testing.T, h *sshdReloadHarness){
"listen": func(t *testing.T, h *sshdReloadHarness) { h.listen = "localhost:0" },
"authorized_users": func(t *testing.T, h *sshdReloadHarness) { h.users = append(h.users, "other") },
"trusted_cas": func(t *testing.T, h *sshdReloadHarness) { h.extra = " trusted_cas: []\n" },
"host_key": func(t *testing.T, h *sshdReloadHarness) {
h.keyFile = filepath.Join(t.TempDir(), "other_host_key")
h.writeHostKey(t, newEd25519Key(t))
},
}
for name, change := range cases {
t.Run(name, func(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
change(t, h)
require.NoError(t, h.c.ReloadConfigString(h.config()).Err)
assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied")
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
})
}
}
// The config can stay the same while the host key file it names changes. New connections get the new key, open
// sessions are left alone
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
want := h.writeHostKey(t, newEd25519Key(t))
require.NoError(t, h.c.ReloadConfigString(h.config()).Err)
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal(), "a new host key was not applied")
assert.False(t, dropped(client, 500*time.Millisecond), "a new host key restarted sshd")
}
// A host key of another algorithm replaces the old one rather than being offered next to it
func TestSSHDReload_HostKeyAlgorithmChange(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
want := h.writeHostKey(t, rsaKey)
require.NoError(t, h.c.ReloadConfigString(h.config()).Err)
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal())
_, _, err = h.dial(ssh.KeyAlgoED25519)
assert.Error(t, err, "the old ed25519 host key is still offered")
}
// A server that isn't running, say its port was busy, is tried again on every reload as before
func TestSSHDReload_NotRunningRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
h.server.Stop()
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
require.NoError(t, h.c.ReloadConfigString(h.config()).Err)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// A host key that no longer loads stops sshd, the same as a config that doesn't
func TestSSHDReload_BadHostKeyFileStops(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
require.NoError(t, os.WriteFile(h.keyFile, []byte("not a key"), 0o600))
assert.True(t, h.c.ReloadConfigString(h.config()).Failed)
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
}
// A trusted CA or authorized user that is skipped with a warning fails the reload, the valid ones are still used
func TestSSHDReload_SkippedEntriesFail(t *testing.T) {
cases := map[string]string{
"bad trusted ca": " trusted_cas: [nope]\n",
"user entry not a map": " - nope\n",
"user missing": " - keys: [nope]\n",
"bad key": " - user: other\n keys: nope\n",
"key in a list not a string": " - user: other\n keys: [5]\n",
"bad key in a list": " - user: other\n keys: [nope]\n",
"keys missing": " - user: other\n",
}
for name, extra := range cases {
t.Run(name, func(t *testing.T) {
h := newSSHDReloadHarness(t)
h.extra = extra
res := h.c.ReloadConfigString(h.config())
require.NoError(t, res.Err)
assert.True(t, res.Failed)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
})
}
}
+2 -1
View File
@@ -99,9 +99,10 @@ func newStatsServerFromConfig(ctx context.Context, l *slog.Logger, c *config.C,
emitStats: emitStats, emitStats: emitStats,
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
if err := s.reload(c, false); err != nil { if err := s.reload(c, false); err != nil {
s.l.Error("Failed to reload stats from config", "error", err) s.l.Error("Failed to reload stats from config", "error", err)
r.Failed()
} }
}) })
+4 -3
View File
@@ -41,9 +41,10 @@ type Conn interface {
// //
// Returns the number of packets successfully written. A destination the kernel rejects costs only // Returns the number of packets successfully written. A destination the kernel rejects costs only
// its own packet, so a short count means some peers were undeliverable, not that the batch failed. // its own packet, so a short count means some peers were undeliverable, not that the batch failed.
// Not safe for concurrent use on the same Conn. // Safe for concurrent use on the same Conn.
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
ReloadConfig(c *config.C) // ReloadConfig applies the listen settings that can change, r is nil at startup
ReloadConfig(c *config.C, r *config.Reload)
SupportsMultipleReaders() bool SupportsMultipleReaders() bool
Close() error Close() error
} }
@@ -68,7 +69,7 @@ func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) { func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
return len(bufs), nil return len(bufs), nil
} }
func (NoopConn) ReloadConfig(_ *config.C) { func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) {
return return
} }
func (NoopConn) Close() error { func (NoopConn) Close() error {
+6
View File
@@ -24,6 +24,12 @@ type NetworkChangeMonitor struct {
// NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start // NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start
// is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it. // is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it.
func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor { func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor {
if watchesNetworkChanges {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("listen.rebind_on_network_change")
})
}
return &NetworkChangeMonitor{ return &NetworkChangeMonitor{
l: l, l: l,
ctx: ctx, ctx: ctx,
+3
View File
@@ -13,6 +13,9 @@ import (
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
// watchesNetworkChanges is true where Start does anything
const watchesNetworkChanges = true
const ( const (
// netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A // netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A
// single network change is never a single message, it is a burst: the link drops, addresses go away, new ones // single network change is never a single message, it is a burst: the link drops, addresses go away, new ones
+3
View File
@@ -7,6 +7,9 @@ import (
"log/slog" "log/slog"
) )
// watchesNetworkChanges is true where Start does anything
const watchesNetworkChanges = false
// watchNetworkChanges is a no-op outside of darwin. // watchNetworkChanges is a no-op outside of darwin.
// //
// Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket // Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket
+16
View File
@@ -37,3 +37,19 @@ func TestNetworkChangeMonitorNilRebindIsANoOp(t *testing.T) {
m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: true\n") m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: true\n")
m.Start(nil) m.Start(nil)
} }
// The monitor only reads listen.rebind_on_network_change when it is made
func TestNetworkChangeMonitorRestartKey(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("listen:\n rebind_on_network_change: true\n"))
NewNetworkChangeMonitor(context.Background(), l, c)
res := c.ReloadConfigString("listen:\n rebind_on_network_change: false\n")
require.NoError(t, res.Err)
if watchesNetworkChanges {
assert.Equal(t, []string{"listen.rebind_on_network_change"}, res.RestartKeys)
} else {
assert.Empty(t, res.RestartKeys, "the setting does nothing here")
}
}
+4 -2
View File
@@ -26,7 +26,9 @@ type bypassConn struct {
session *wfp.Session session *wfp.Session
} }
func (b *bypassConn) ReloadConfig(c *config.C) { func (b *bypassConn) ReloadConfig(c *config.C, r *config.Reload) {
// Only read on the first call, at startup
r.RestartRequiredIfChanged("listen.windows_bypass_wdf")
b.installOnce.Do(func() { b.installOnce.Do(func() {
if !c.GetBool("listen.windows_bypass_wdf", true) { if !c.GetBool("listen.windows_bypass_wdf", true) {
return return
@@ -45,7 +47,7 @@ func (b *bypassConn) ReloadConfig(c *config.C) {
"port", addr.Port()) "port", addr.Port())
b.session = s b.session = s
}) })
b.Conn.ReloadConfig(c) b.Conn.ReloadConfig(c, r)
} }
func (b *bypassConn) Close() error { func (b *bypassConn) Close() error {
+1 -1
View File
@@ -171,7 +171,7 @@ func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
} }
} }
func (u *StdConn) ReloadConfig(c *config.C) { func (u *StdConn) ReloadConfig(c *config.C, r *config.Reload) {
// TODO // TODO
} }
+1 -1
View File
@@ -73,7 +73,7 @@ func (u *GenericConn) LocalAddr() (netip.AddrPort, error) {
} }
} }
func (u *GenericConn) ReloadConfig(c *config.C) { func (u *GenericConn) ReloadConfig(c *config.C, r *config.Reload) {
} }
+4 -1
View File
@@ -392,7 +392,7 @@ func writeSockaddr(buf []byte, addr netip.AddrPort, isV4 bool) (int, error) {
return unix.SizeofSockaddrInet6, nil return unix.SizeofSockaddrInet6, nil
} }
func (u *StdConn) ReloadConfig(c *config.C) { func (u *StdConn) ReloadConfig(c *config.C, r *config.Reload) {
b := c.GetInt("listen.read_buffer", 0) b := c.GetInt("listen.read_buffer", 0)
if b > 0 { if b > 0 {
if err := u.SetRecvBuffer(b); err == nil { if err := u.SetRecvBuffer(b); err == nil {
@@ -403,6 +403,7 @@ func (u *StdConn) ReloadConfig(c *config.C) {
} }
} else { } else {
u.l.Error("Failed to set listen.read_buffer", "error", err) u.l.Error("Failed to set listen.read_buffer", "error", err)
r.Failed()
} }
} }
@@ -416,6 +417,7 @@ func (u *StdConn) ReloadConfig(c *config.C) {
} }
} else { } else {
u.l.Error("Failed to set listen.write_buffer", "error", err) u.l.Error("Failed to set listen.write_buffer", "error", err)
r.Failed()
} }
} }
@@ -430,6 +432,7 @@ func (u *StdConn) ReloadConfig(c *config.C) {
} }
} else { } else {
u.l.Error("Failed to set listen.so_mark", "error", err) u.l.Error("Failed to set listen.so_mark", "error", err)
r.Failed()
} }
} }
} }
+31
View File
@@ -0,0 +1,31 @@
//go:build linux && !android && !e2e_testing
package udp
import (
"net/netip"
"os"
"testing"
"github.com/slackhq/nebula/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// A setting the kernel refuses fails the reload
func TestReloadConfig_RefusedSettingFailsTheReload(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("root may set listen.so_mark, run as an unprivileged user")
}
conn, err := NewListener(testLogger(), Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Batch: 1})
require.NoError(t, err)
t.Cleanup(func() { _ = conn.Close() })
c := config.NewC(testLogger())
require.NoError(t, c.LoadString("listen:\n port: 0\n"))
c.RegisterReloadCallback(conn.ReloadConfig)
res := c.ReloadConfigString("listen:\n port: 0\n so_mark: 42\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
}
+6 -1
View File
@@ -12,6 +12,7 @@ import (
"net/netip" "net/netip"
"strconv" "strconv"
"strings" "strings"
"sync"
"unsafe" "unsafe"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
@@ -20,7 +21,7 @@ import (
// batchWriter owns the sendmmsg(2)/UDP-GSO transmit path for a StdConn: the // batchWriter owns the sendmmsg(2)/UDP-GSO transmit path for a StdConn: the
// scratch WriteBatch packs mmsghdr entries into, plus the GSO capability // scratch WriteBatch packs mmsghdr entries into, plus the GSO capability
// state probed at socket creation. Each queue has its own StdConn and // state probed at socket creation. Each queue has its own StdConn and
// batchWriter, so no locking is needed. // batchWriter; mu serializes batches from the TUN reader and relay forwarding.
// //
// Terminology, smallest to largest: // Terminology, smallest to largest:
// //
@@ -39,6 +40,7 @@ import (
// chunk the entries packed for one sendmmsg call, at most MaxWriteBatch. // chunk the entries packed for one sendmmsg call, at most MaxWriteBatch.
// batch the caller's whole bufs/addrs pair, processed as one or more chunks. // batch the caller's whole bufs/addrs pair, processed as one or more chunks.
type batchWriter struct { type batchWriter struct {
mu sync.Mutex
fd int fd int
isV4 bool isV4 bool
@@ -192,6 +194,9 @@ func parseRelease(r string) (major, minor int) {
// Returns the number of packets sent. An error means the call itself failed. // Returns the number of packets sent. An error means the call itself failed.
// A short count means some destinations were undeliverable. // A short count means some destinations were undeliverable.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()
if len(bufs) != len(addrs) { if len(bufs) != len(addrs) {
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs)) return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
} }
@@ -0,0 +1,61 @@
//go:build linux && !android && !e2e_testing
package udp
import (
"net/netip"
"runtime"
"sync"
"testing"
"github.com/stretchr/testify/require"
)
// The TUN reader and relay forwarding share a socket. Exercise their concurrent
// batches against the actual GSO planner and scratch, without relying on a NIC.
func TestWriteBatchConcurrent(t *testing.T) {
const batches = 64
w := newRewindTestWriter()
sent := make(chan [][]byte, 2*batches)
w.sendFn = func(start, n int) (int, error) {
runtime.Gosched()
sent <- capturePrepared(w, start, n)
return n, nil
}
conn := &StdConn{bw: w}
remote := netip.MustParseAddrPort("192.0.2.1:4242")
var wg sync.WaitGroup
start := make(chan struct{})
for producer := range 2 {
wg.Go(func() {
<-start
for i := range batches {
var bufs [][]byte
var addrs []netip.AddrPort
for j := range 4 {
bufs = append(bufs, []byte{byte(producer), byte(i), byte(j), 0})
addrs = append(addrs, remote)
}
n, err := conn.WriteBatch(bufs, addrs)
if err != nil || n != len(bufs) {
t.Errorf("WriteBatch = %d, %v; want %d, nil", n, err, len(bufs))
return
}
}
})
}
close(start)
wg.Wait()
close(sent)
var counts [2]int
for packets := range sent {
require.Len(t, packets, 4)
producer := packets[0][0]
require.Less(t, int(producer), len(counts))
for j, packet := range packets {
require.Equal(t, []byte{producer, byte(counts[producer]), byte(j), 0}, packet)
}
counts[producer]++
}
require.Equal(t, [2]int{batches, batches}, counts)
}
+1 -1
View File
@@ -351,7 +351,7 @@ func (u *RIOConn) Rebind() error {
return nil return nil
} }
func (u *RIOConn) ReloadConfig(*config.C) {} func (u *RIOConn) ReloadConfig(*config.C, *config.Reload) {}
func (u *RIOConn) Close() error { func (u *RIOConn) Close() error {
if !u.isOpen.CompareAndSwap(true, false) { if !u.isOpen.CompareAndSwap(true, false) {
+1 -1
View File
@@ -197,7 +197,7 @@ func (u *TesterConn) ListenOut(r EncReader, flush func()) error {
} }
} }
func (u *TesterConn) ReloadConfig(*config.C) {} func (u *TesterConn) ReloadConfig(*config.C, *config.Reload) {}
func NewUDPStatsEmitter(_ []Conn) func() { func NewUDPStatsEmitter(_ []Conn) func() {
// No UDP stats for non-linux // No UDP stats for non-linux