Compare commits

..
Author SHA1 Message Date
Wade Simmons a9e53a4088 Apply suggestion from @wadey 2026-09-21 14:31:18 -04:00
Wade Simmons f24fb308a0 Update security report submission guidelines
Clarified submission process for security reports.
2026-09-21 09:44:32 -04:00
133 changed files with 1816 additions and 9190 deletions

No files matched your search

-2
View File
@@ -1,2 +0,0 @@
# Goldens are compared byte for byte, so they must come out of git the way they went in.
*.golden -text
+3 -3
View File
@@ -14,7 +14,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -40,7 +40,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -83,7 +83,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
# GitHub holds ARNs, not credentials, and ARNs outlive a rotation # GitHub holds ARNs, not credentials, and ARNs outlive a rotation
+3 -3
View File
@@ -34,7 +34,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -66,7 +66,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -92,7 +92,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own # WSL2 + Ubuntu so the smoke can run a real linux peer with its own
+2 -2
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Smoke Docker - name: Smoke Docker
@@ -51,7 +51,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: build - name: build
-14
View File
@@ -263,20 +263,6 @@ try {
} }
Write-Host "OK: WSL peer -> windows lighthouse over v6" Write-Host "OK: WSL peer -> windows lighthouse over v6"
# The NlMtu check above proves the value was written, not that windows honors it. A payload this size only
# crosses the tunnel as fragments cut at NlMtu on the way out of the adapter. Left at the adapter default it
# reaches nebula whole, overflows the udp.MTU (9001) write buffer and is dropped, so the ping never answers.
# Anything smaller would ride out as one oversized datagram and survive on IP fragmentation of the underlay.
$BigPayload = 9000
foreach ($target in @(@{ Family = 'v4'; Ip = $Ip2 }, @{ Family = 'v6'; Ip = $Ip6_2 })) {
Wait-Until -TimeoutSec 30 -What "$($target.Family) ping with a $BigPayload byte payload from windows lighthouse to WSL peer ($($target.Ip))" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the large $($target.Family) ping succeeded" }
$null = & ping.exe -n 1 -w 1000 -l $BigPayload $target.Ip
$LASTEXITCODE -eq 0
}
Write-Host "OK: windows lighthouse -> WSL peer, $BigPayload byte $($target.Family) payload"
}
Write-Host '' Write-Host ''
Write-Host 'All smoke checks passed.' Write-Host 'All smoke checks passed.'
} }
+4 -4
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Install goimports - name: Install goimports
@@ -42,7 +42,7 @@ jobs:
- name: golangci-lint - name: golangci-lint
uses: golangci/golangci-lint-action@v9 uses: golangci/golangci-lint-action@v9
with: with:
version: v2.14 version: v2.12
test: test:
name: Test ${{ matrix.name }} name: Test ${{ matrix.name }}
@@ -87,7 +87,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -132,7 +132,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build ${{ matrix.name }} - name: Build ${{ matrix.name }}
-14
View File
@@ -7,20 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
### Removed
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
`mutex-profile-fraction` commands, and `sshd.sandbox_dir`, which is now ignored if set.
### Fixed
- Signing a v2 certificate with an empty name, a name longer than 253 bytes, or an empty group now fails.
These certificates were signed successfully but could not be loaded.
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
the handshake starts, so a pending hostinfo never exists without one.
## [1.11.1] - 2026-08-21 ## [1.11.1] - 2026-08-21
See the [v1.11.1](https://github.com/slackhq/nebula/milestone/30?closed=1) milestone for a complete list of changes. See the [v1.11.1](https://github.com/slackhq/nebula/milestone/30?closed=1) milestone for a complete list of changes.
-3
View File
@@ -242,14 +242,11 @@ test-cov-html:
go test -coverprofile=coverage.out go test -coverprofile=coverage.out
go tool cover -html=coverage.out go tool cover -html=coverage.out
# The package builds only compile. The final line links an android binary so a linker-only failure
# cannot pass CI.
build-test-mobile: build-test-mobile:
GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build -o /dev/null ${NEBULA_CMD_PATH}
bench: bench:
go test -bench=. go test -bench=.
+3 -2
View File
@@ -8,5 +8,6 @@ If you believe you have found a security vulnerability with Nebula, please let
us know right away. We will investigate all reports and do our best to quickly us know right away. We will investigate all reports and do our best to quickly
fix valid issues. fix valid issues.
You can submit your report on [HackerOne](https://hackerone.com/slack) and our For critical reports, if you are interested in a bug bounty: You can submit your report on [HackerOne](https://hackerone.com/slack).
security team will respond as soon as possible.
For other reports: You can [submit a report via GitHub](https://github.com/slackhq/nebula/security/advisories/new).
+1 -10
View File
@@ -389,16 +389,7 @@ func (c *certificateV2) fromTBSCertificate(t *TBSCertificate) error {
} }
func (c *certificateV2) validate() error { func (c *certificateV2) validate() error {
// Mirror the name and group rules in unmarshalDetails so signing fails instead of producing an unparseable certificate // Empty names are allowed
if c.details.name == "" || len(c.details.name) > MaxNameLength {
return NewErrInvalidCertificateProperties("name must be between 1 and %d bytes", MaxNameLength)
}
for _, group := range c.details.groups {
if group == "" {
return NewErrInvalidCertificateProperties("groups may not be empty")
}
}
if len(c.publicKey) == 0 { if len(c.publicKey) == 0 {
return ErrInvalidPublicKey return ErrInvalidPublicKey
-45
View File
@@ -6,7 +6,6 @@ import (
"crypto/elliptic" "crypto/elliptic"
"crypto/rand" "crypto/rand"
"net/netip" "net/netip"
"strings"
"testing" "testing"
"time" "time"
@@ -136,47 +135,3 @@ func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
assert.True(t, normie) assert.True(t, normie)
} }
} }
func TestCertificateV2_SignRejectsUnparseable(t *testing.T) {
_, priv, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
var invalid *ErrInvalidCertificateProperties
newTBS := func() TBSCertificate {
return TBSCertificate{
Version: Version2,
Name: "testing",
Networks: []netip.Prefix{mustParsePrefixUnmapped("10.1.1.1/24")},
Groups: []string{"test-group1"},
NotBefore: time.Now().Add(-time.Minute),
NotAfter: time.Now().Add(time.Minute),
PublicKey: []byte("1234567890abcedfghij1234567890ab"),
Curve: Curve_CURVE25519,
IsCA: true,
}
}
tbs := newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength)
c, err := tbs.Sign(nil, Curve_CURVE25519, priv)
require.NoError(t, err)
b, err := c.Marshal()
require.NoError(t, err)
_, err = unmarshalCertificateV2(b, nil, Curve_CURVE25519)
require.NoError(t, err)
tbs = newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength+1)
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Name = ""
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Groups = []string{"test-group1", ""}
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
}
+1 -2
View File
@@ -85,10 +85,9 @@ func main() {
fmt.Printf("failed to apply logging config: %s", err) fmt.Printf("failed to apply logging config: %s", err)
os.Exit(1) os.Exit(1)
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
l.Error("Failed to reconfigure logger on reload", "error", err) l.Error("Failed to reconfigure logger on reload", "error", err)
r.Failed()
} }
}) })
+1 -2
View File
@@ -34,10 +34,9 @@ func (p *program) Start(s service.Service) error {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
return fmt.Errorf("failed to apply logging config: %s", err) return fmt.Errorf("failed to apply logging config: %s", err)
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
l.Error("Failed to reconfigure logger on reload", "error", err) l.Error("Failed to reconfigure logger on reload", "error", err)
r.Failed()
} }
}) })
+1 -2
View File
@@ -71,10 +71,9 @@ func main() {
fmt.Printf("failed to apply logging config: %s", err) fmt.Printf("failed to apply logging config: %s", err)
os.Exit(1) os.Exit(1)
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
if err := logging.ApplyConfig(l, c); err != nil { if err := logging.ApplyConfig(l, c); err != nil {
l.Error("Failed to reconfigure logger on reload", "error", err) l.Error("Failed to reconfigure logger on reload", "error", err)
r.Failed()
} }
}) })
-710
View File
@@ -1,710 +0,0 @@
package nebula
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"log/slog"
"maps"
"net/netip"
"sort"
"strings"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/logging"
)
// The commands nebula serves. attachCommands builds the one registry every surface
// reads. See the commands package for what a Command is.
type listHostMapFlags struct {
Json bool
Pretty bool
ByIndex bool
}
// listHostMapFlagSet is the flag set list-hostmap and list-pending-hostmap share.
func listHostMapFlagSet() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := listHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table")
return fl, &s
}
type printCertFlags struct {
Json bool
Pretty bool
Raw bool
}
type printTunnelFlags struct {
Pretty bool
}
type changeRemoteFlags struct {
Address string
}
type closeTunnelFlags struct {
LocalOnly bool
}
type createTunnelFlags struct {
Address string
}
type deviceInfoFlags struct {
Json bool
Pretty bool
}
func attachCommands(l *slog.Logger, c *config.C, reg *commands.Registry, f *Interface) {
reg.Register(&commands.Command{
Name: commands.ListHostmap,
ShortDescription: "List all known previously connected hosts",
Flags: listHostMapFlagSet,
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listHostMapCommand(f.hostMap, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.ListPendingHostmap,
ShortDescription: "List all handshaking hosts",
Flags: listHostMapFlagSet,
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listHostMapCommand(f.handshakeManager, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.ListLighthouseAddrmap,
ShortDescription: "List all lighthouse map entries",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := listHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listLighthouseAddrmapCommand(f.lightHouse, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.Reload,
ShortDescription: "Reloads configuration from disk, same as sending HUP to the process",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return reloadCommand(l, c, w)
},
})
reg.Register(&commands.Command{
Name: commands.LogLevel,
ShortDescription: "Gets or sets the current log level",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return logLevelCommand(l, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.LogFormat,
ShortDescription: "Gets or sets the current log format",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return logFormatCommand(l, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.Version,
ShortDescription: "Prints the currently running version of nebula",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return versionCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.DeviceInfo,
ShortDescription: "Prints information about the network device.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := deviceInfoFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return deviceInfoCommand(f, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintCert,
ShortDescription: "Prints the current certificate being used or the certificate for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printCertFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.Raw, "raw", false, "raw prints the PEM encoded certificate, not compatible with -json or -pretty")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printCertCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintTunnel,
ShortDescription: "Prints json details about a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintRelays,
ShortDescription: "Prints json details about all relay info",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printRelaysCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.ChangeRemote,
ShortDescription: "Changes the remote address used in the tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := changeRemoteFlags{}
fl.StringVar(&s.Address, "address", "", "The new remote address, ip:port")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return changeRemoteCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.CloseTunnel,
ShortDescription: "Closes a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := closeTunnelFlags{}
fl.BoolVar(&s.LocalOnly, "local-only", false, "Disables notifying the remote that the tunnel is shutting down")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return closeTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.CreateTunnel,
ShortDescription: "Creates a tunnel for the provided vpn address",
Help: "The lighthouses will be queried for real addresses but you can provide one as well.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := createTunnelFlags{}
fl.StringVar(&s.Address, "address", "", "Optionally provide a real remote address, ip:port ")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return createTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.QueryLighthouse,
ShortDescription: "Query the lighthouses for the provided vpn address",
Help: "This command is asynchronous. Only currently known udp addresses will be printed.",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return queryLighthouseCommand(f, fs, a, w)
},
})
}
func listHostMapCommand(hl controlHostLister, a any, w commands.StringWriter) error {
fs, ok := a.(*listHostMapFlags)
if !ok {
return fmt.Errorf("internal error: expected *listHostMapFlags but was %T", a)
}
var hm []ControlHostInfo
if fs.ByIndex {
hm = listHostMapIndexes(hl)
} else {
hm = listHostMapHosts(hl)
}
sort.Slice(hm, func(i, j int) bool {
return hm[i].VpnAddrs[0].Compare(hm[j].VpnAddrs[0]) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
if err := js.Encode(hm); err != nil {
return err
}
} else {
for _, v := range hm {
err := w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddrs, v.RemoteAddrs))
if err != nil {
return err
}
}
}
return nil
}
func listLighthouseAddrmapCommand(lightHouse *LightHouse, a any, w commands.StringWriter) error {
fs, ok := a.(*listHostMapFlags)
if !ok {
return fmt.Errorf("internal error: expected *listHostMapFlags but was %T", a)
}
type lighthouseInfo struct {
VpnAddr string `json:"vpnAddr"`
Addrs *CacheMap `json:"addrs"`
}
lightHouse.RLock()
addrMap := make([]lighthouseInfo, len(lightHouse.addrMap))
x := 0
for k, v := range lightHouse.addrMap {
addrMap[x] = lighthouseInfo{
VpnAddr: k.String(),
Addrs: v.CopyCache(),
}
x++
}
lightHouse.RUnlock()
sort.Slice(addrMap, func(i, j int) bool {
return strings.Compare(addrMap[i].VpnAddr, addrMap[j].VpnAddr) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
if err := js.Encode(addrMap); err != nil {
return err
}
} else {
for _, v := range addrMap {
b, err := json.Marshal(v.Addrs)
if err != nil {
return err
}
err = w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddr, string(b)))
if err != nil {
return err
}
}
}
return nil
}
// parseVpnAddr parses the vpn address a command takes as its first operand, failing to w
// when there is none or it does not parse. noun is how the command's messages name it,
// "vpn address" or "vpn addr", which the output golden pins per command.
func parseVpnAddr(a []string, noun string, w commands.StringWriter) (netip.Addr, error) {
if len(a) == 0 {
return netip.Addr{}, commands.Fail(w, "No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return netip.Addr{}, commands.Fail(w, fmt.Sprintf("The provided %s could not be parsed: %s", noun, a[0]))
}
return vpnAddr, nil
}
func versionCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
return w.WriteLine(ifce.version)
}
func queryLighthouseCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
var cm *CacheMap
rl := ifce.lightHouse.Query(vpnAddr)
if rl != nil {
cm = rl.CopyCache()
}
return json.NewEncoder(w.GetWriter()).Encode(cm)
}
func closeTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*closeTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *closeTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
if !flags.LocalOnly {
ifce.sendCloseTunnel(hostInfo)
}
ifce.closeTunnel(hostInfo)
return w.WriteLine("Closed")
}
func createTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*createTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *createTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return commands.Fail(w, "Tunnel already exists")
}
hostInfo = ifce.handshakeManager.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return commands.Fail(w, "Tunnel already handshaking")
}
var addr netip.AddrPort
if flags.Address != "" {
addr, err = netip.ParseAddrPort(flags.Address)
if err != nil {
return commands.Fail(w, "Address could not be parsed")
}
}
// Set the remote from the callback, which StartHandshake runs under the handshake manager lock
// before the hostinfo reaches the handshake loop or the static-host trigger, so the address is
// in the RemoteList before the first handshake is sent.
ifce.handshakeManager.StartHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
if addr.IsValid() {
hh.hostinfo.SetRemote(addr)
}
})
return w.WriteLine("Created")
}
func changeRemoteCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*changeRemoteFlags)
if !ok {
return fmt.Errorf("internal error: expected *changeRemoteFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
if flags.Address == "" {
return commands.Fail(w, "No address was provided")
}
addr, err := netip.ParseAddrPort(flags.Address)
if err != nil {
return commands.Fail(w, "Address could not be parsed")
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
hostInfo.SetRemote(addr)
return w.WriteLine("Changed")
}
func logLevelCommand(l *slog.Logger, fs any, a []string, w commands.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level
SetLevel(slog.Level)
})
if !ok {
return commands.Fail(w, "Log level is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
level, err := logging.ParseLevel(strings.ToLower(a[0]))
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unknown log level %s. Possible log levels: trace, debug, info, warn, error", a))
}
ctrl.SetLevel(level)
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
func logFormatCommand(l *slog.Logger, fs any, a []string, w commands.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetFormat() string
SetFormat(string) error
})
if !ok {
return commands.Fail(w, "Log format is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
if err := ctrl.SetFormat(strings.ToLower(a[0])); err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to set log format: %s", err))
}
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
func printCertCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printCertFlags)
if !ok {
return fmt.Errorf("internal error: expected *printCertFlags but was %T", fs)
}
cert := ifce.pki.getCertState().GetDefaultCertificate()
if len(a) > 0 {
vpnAddr, err := parseVpnAddr(a, "vpn addr", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
cert = hostInfo.GetCert().Certificate
}
if args.Json || args.Pretty {
b, err := cert.MarshalJSON()
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
if args.Pretty {
buf := new(bytes.Buffer)
if err := json.Indent(buf, b, "", " "); err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
b = buf.Bytes()
}
return w.WriteBytes(b)
}
if args.Raw {
b, err := cert.MarshalPEM()
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
return w.WriteBytes(b)
}
return w.WriteLine(cert.String())
}
func printRelaysCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *printTunnelFlags but was %T", fs)
}
ifce.hostMap.RLock()
relays := maps.Clone(ifce.hostMap.Relays)
ifce.hostMap.RUnlock()
type RelayFor struct {
Error string `json:",omitempty"`
Type string
State string
PeerAddr netip.Addr
LocalIndex uint32
RemoteIndex uint32
RelayedThrough []netip.Addr
}
type RelayOutput struct {
NebulaAddr netip.Addr
RelayForAddrs []RelayFor
}
type CmdOutput struct {
Relays []*RelayOutput
}
co := CmdOutput{}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
for k, v := range relays {
ro := RelayOutput{NebulaAddr: v.vpnAddrs[0]}
co.Relays = append(co.Relays, &ro)
relayHI := ifce.hostMap.QueryVpnAddr(v.vpnAddrs[0])
if relayHI == nil {
ro.RelayForAddrs = append(ro.RelayForAddrs, RelayFor{Error: "could not find hostinfo"})
continue
}
for _, vpnAddr := range relayHI.relayState.CopyRelayForIps() {
rf := RelayFor{}
r, ok := relayHI.relayState.GetRelayForByAddr(vpnAddr)
if ok {
t := ""
switch r.Type {
case ForwardingType:
t = "forwarding"
case TerminalType:
t = "terminal"
default:
t = "unknown"
}
s := ""
switch r.State {
case Requested:
s = "requested"
case Established:
s = "established"
default:
s = "unknown"
}
rf.LocalIndex = r.LocalIndex
rf.RemoteIndex = r.RemoteIndex
rf.PeerAddr = r.PeerAddr
rf.Type = t
rf.State = s
if rf.LocalIndex != k {
rf.Error = fmt.Sprintf("hostmap LocalIndex '%v' does not match RelayState LocalIndex", k)
}
}
relayedHI := ifce.hostMap.QueryVpnAddr(vpnAddr)
if relayedHI != nil {
rf.RelayedThrough = append(rf.RelayedThrough, relayedHI.relayState.CopyRelayIps()...)
}
ro.RelayForAddrs = append(ro.RelayForAddrs, rf)
}
}
return enc.Encode(co)
}
func printTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *printTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn addr", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.GetPreferredRanges()))
}
func deviceInfoCommand(ifce *Interface, fs any, w commands.StringWriter) error {
flags, ok := fs.(*deviceInfoFlags)
if !ok {
return fmt.Errorf("internal error: expected flags to be deviceInfoFlags but was %+v", fs)
}
data := struct {
Name string `json:"name"`
Cidr []netip.Prefix `json:"cidr"`
}{
Name: ifce.inside.Name(),
Cidr: make([]netip.Prefix, len(ifce.inside.Networks())),
}
copy(data.Cidr, ifce.inside.Networks())
if flags.Json || flags.Pretty {
js := json.NewEncoder(w.GetWriter())
if flags.Pretty {
js.SetIndent("", " ")
}
return js.Encode(data)
} else {
return w.WriteLine(fmt.Sprintf("name=%v cidr=%v", data.Name, data.Cidr))
}
}
// reloadCommand reloads the config and writes the result to the client. It reloads even if the client has gone away
func reloadCommand(l *slog.Logger, c *config.C, w commands.StringWriter) error {
writeErr := w.WriteLine("Reloading config")
res := c.ReloadConfig()
res.Log(l)
if err := w.WriteLine(res.String()); writeErr == nil {
writeErr = err
}
if writeErr != nil {
return writeErr
}
if res.Err != nil || res.Failed {
return &commands.Failure{Msg: res.String()}
}
return nil
}
-83
View File
@@ -1,83 +0,0 @@
package commands
import (
"flag"
)
// CommandFlags is called before help or command execution to parse command line flags.
// It should return a flag.FlagSet instance and a pointer to the struct that will contain parsed flags
type CommandFlags func() (*flag.FlagSet, any)
// CommandCallback is called when your command should execute.
// fs will be a pointer to the struct provided by Command.Flags callback, if there was one. -h and -help are reserved
// and handled automatically for you.
// a will be any unconsumed arguments, if no Command.Flags was available this will be all the flags passed in.
// w is the writer to use when sending messages back to the client.
// An error returned by the callback is handed back to whoever dispatched the command, which reports a failure: exit
// status 1 over ssh. The error is not shown to the user. For a failure the user caused or can act on, write the
// message and return it as a *Failure, which Fail does in one step; the surface then reports the failure without
// logging what the user was already told. Any other error is logged by the surface as well.
type CommandCallback func(fs any, a []string, w StringWriter) error
type Command struct {
Name Name
ShortDescription string
Help string
Flags CommandFlags
Callback CommandCallback
}
// Failure is the error a command returns for a failure it has already explained to the user:
// the message it wrote is the whole story. A surface reports it as a failure and does not
// log it.
type Failure struct {
Msg string
}
func (f *Failure) Error() string {
return f.Msg
}
// Fail writes msg to w and returns it as a *Failure, so a command reports a failure to the
// user and to its surface in one line. If the write itself fails, that error is returned
// instead: the user was not told anything.
func Fail(w StringWriter, msg string) error {
if err := w.WriteLine(msg); err != nil {
return err
}
return &Failure{Msg: msg}
}
func execCommand(c *Command, args []string, w StringWriter) error {
var (
fl *flag.FlagSet
fs any
)
if c.Flags != nil {
fl, fs = c.Flags()
if fl != nil {
// SetOutput() here in case fl.Parse dumps usage.
fl.SetOutput(w.GetWriter())
err := fl.Parse(args)
if err != nil {
// fl.Parse has dumped error information to the user via the w writer.
return err
}
args = fl.Args()
}
}
return c.Callback(fs, args, w)
}
func checkHelpArgs(args []string) bool {
for _, a := range args {
if a == "-h" || a == "-help" {
return true
}
}
return false
}
-28
View File
@@ -1,28 +0,0 @@
package commands
// Name identifies a command. Every command nebula serves is a constant here, so a
// surface, a client and a test spell it the same way and are told at compile time when
// they do not. It is a string rather than an integer so a test can register a throwaway
// command without adding it here, and so the wire carries the name as the string it has
// always been.
type Name string
const (
ChangeRemote Name = "change-remote"
CloseTunnel Name = "close-tunnel"
CreateTunnel Name = "create-tunnel"
DeviceInfo Name = "device-info"
Help Name = "help"
ListHostmap Name = "list-hostmap"
ListLighthouseAddrmap Name = "list-lighthouse-addrmap"
ListPendingHostmap Name = "list-pending-hostmap"
LogFormat Name = "log-format"
LogLevel Name = "log-level"
Logout Name = "logout"
PrintCert Name = "print-cert"
PrintRelays Name = "print-relays"
PrintTunnel Name = "print-tunnel"
QueryLighthouse Name = "query-lighthouse"
Reload Name = "reload"
Version Name = "version"
)
-181
View File
@@ -1,181 +0,0 @@
package commands
import (
"cmp"
"fmt"
"maps"
"slices"
"strings"
"sync"
"github.com/anmitsu/go-shlex"
)
// Registry holds a set of commands. New is the only way to make one.
//
// It is safe for concurrent use. Commands may be registered while a surface is serving
// from the registry: the ssh server clones it per session, and a clone may overlap a
// Register.
type Registry struct {
mu sync.RWMutex
commands map[Name]*Command
}
// New returns an empty Registry with only `help` available.
func New() *Registry {
r := &Registry{commands: map[Name]*Command{}}
r.registerHelp()
return r
}
func (r *Registry) registerHelp() {
r.Register(&Command{
Name: Help,
ShortDescription: "prints available commands or help <command> for specific usage info",
Callback: func(a any, args []string, w StringWriter) error {
return r.help(args, w)
},
})
}
// Register adds a command that can be run by a user. A command registered under a name
// already held replaces it; see Clone for what that means for Help.
func (r *Registry) Register(c *Command) {
r.mu.Lock()
defer r.mu.Unlock()
r.commands[c.Name] = c
}
// Get returns the command registered under name, or nil if there is no such command.
func (r *Registry) Get(name Name) *Command {
r.mu.RLock()
defer r.mu.RUnlock()
return r.commands[name]
}
// Match returns the names of every command starting with prefix, sorted, as strings for
// a prompt to complete and print.
func (r *Registry) Match(prefix string) []string {
cmds := make([]string, 0)
r.mu.RLock()
for n := range r.commands {
if strings.HasPrefix(string(n), prefix) {
cmds = append(cmds, string(n))
}
}
r.mu.RUnlock()
slices.Sort(cmds)
return cmds
}
// All returns every registered command sorted by name, which is the order help lists them.
func (r *Registry) All() []*Command {
r.mu.RLock()
defer r.mu.RUnlock()
cmds := slices.Collect(maps.Values(r.commands))
slices.SortFunc(cmds, func(a, b *Command) int { return cmp.Compare(a.Name, b.Name) })
return cmds
}
// Clone returns a copy holding the same commands, with help bound to the copy: a
// command registered on the clone alone is listed and described there, and one the
// clone withholds is not. That rebinding replaces whatever was registered under Help on
// the source, built-in or not.
func (r *Registry) Clone() *Registry {
r.mu.RLock()
c := &Registry{commands: maps.Clone(r.commands)}
r.mu.RUnlock()
c.registerHelp()
return c
}
// Dispatch parses a command line and runs it, reporting problems to w. The error is for
// the caller to log, not to show; see CommandCallback.
func (r *Registry) Dispatch(line string, w StringWriter) error {
// A line shlex cannot split, an unbalanced quote.
args, err := shlex.Split(line, true)
if err != nil {
return Fail(w, "could not parse the command line: "+err.Error())
}
return r.DispatchArgs(args, w)
}
// DispatchArgs runs an already tokenized command line. The error is as for Dispatch.
func (r *Registry) DispatchArgs(args []string, w StringWriter) error {
if len(args) == 0 {
r.dumpCommands(w)
return nil
}
c := r.Get(Name(args[0]))
if c == nil {
// Only the first word was looked up, so only it is named.
msg := fmt.Sprintf("did not understand: %s", args[0])
if err := w.WriteLine(msg); err != nil {
return err
}
r.dumpCommands(w)
return &Failure{Msg: msg}
}
if checkHelpArgs(args) {
// Through the registered help command, so -h and `help <name>` are one path.
return execCommand(r.Get(Help), []string{string(c.Name)}, w)
}
return execCommand(c, args[1:], w)
}
// dumpCommands writes the list of available commands to w.
func (r *Registry) dumpCommands(w StringWriter) {
err := w.WriteLine("Available commands:")
if err != nil {
return
}
// In the order All gives, which is the display order.
cmds := make([]string, 0)
for _, l := range r.All() {
cmds = append(cmds, fmt.Sprintf("%s - %s", l.Name, l.ShortDescription))
}
_ = w.Write(strings.Join(cmds, "\n") + "\n\n")
}
// help writes the command list, or usage for the single command named in a.
func (r *Registry) help(a []string, w StringWriter) error {
// Just typed help
if len(a) == 0 {
r.dumpCommands(w)
return nil
}
// We are printing a specific commands help text
cmd := r.Get(Name(a[0]))
if cmd == nil {
return Fail(w, "Command not available "+a[0])
}
if err := w.WriteLine(fmt.Sprintf("%s - %s", cmd.Name, cmd.ShortDescription)); err != nil {
return err
}
if cmd.Help != "" {
if err := w.WriteLine(fmt.Sprintf(" %s", cmd.Help)); err != nil {
return err
}
}
if cmd.Flags != nil {
fs, _ := cmd.Flags()
if fs != nil {
fs.SetOutput(w.GetWriter())
fs.PrintDefaults()
}
}
return nil
}
-176
View File
@@ -1,176 +0,0 @@
package commands
import (
"bytes"
"errors"
"flag"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testRegistry() *Registry {
r := New()
r.Register(&Command{
Name: "served",
ShortDescription: "a command this surface serves",
Callback: func(fs any, a []string, w StringWriter) error {
return w.WriteLine("read " + strings.Join(a, ","))
},
})
r.Register(&Command{
Name: "withheld",
ShortDescription: "a command this surface withholds",
Callback: func(fs any, a []string, w StringWriter) error {
return w.WriteLine("wrote")
},
})
return r
}
// TestFlagUsageGoesToTheWriter pins that a parse failure's usage lands on the command's
// writer.
func TestFlagUsageGoesToTheWriter(t *testing.T) {
r := New()
r.Register(&Command{
Name: "emit",
ShortDescription: "emits",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
fl.Bool("json", false, "emit json")
return fl, nil
},
Callback: func(fs any, a []string, w StringWriter) error { return nil },
})
merged := &bytes.Buffer{}
require.Error(t, r.DispatchArgs([]string{"emit", "-nope"}, NewStringWriter(merged)))
assert.Contains(t, merged.String(), "flag provided but not defined")
}
// TestDispatchReportsUnparseableLine: a line shlex rejects, an unbalanced quote, is a
// failure the user is told about.
func TestDispatchReportsUnparseableLine(t *testing.T) {
out := &bytes.Buffer{}
err := testRegistry().Dispatch(`served "unterminated`, NewStringWriter(out))
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, f.Msg+"\n", out.String())
assert.Contains(t, out.String(), "could not parse the command line: ")
}
// TestFailTellsTheUserAndTheSurface: Fail writes the message and returns it as a Failure;
// when the write fails, the user heard nothing, so the write error is what comes back.
func TestFailTellsTheUserAndTheSurface(t *testing.T) {
out := &bytes.Buffer{}
err := Fail(NewStringWriter(out), "no such tunnel")
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, "no such tunnel", f.Msg)
assert.Equal(t, "no such tunnel\n", out.String())
err = Fail(NewStringWriter(failingWriter{}), "no such tunnel")
require.EqualError(t, err, "gone")
assert.NotErrorAs(t, err, &f)
}
type failingWriter struct{}
func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("gone") }
// TestAllSortsByName: registration order does not matter.
func TestAllSortsByName(t *testing.T) {
r := New()
for _, n := range []Name{"zz", Version, "aa", ChangeRemote} {
r.Register(&Command{Name: n})
}
var got []Name
for _, c := range r.All() {
got = append(got, c.Name)
}
assert.Equal(t, []Name{"aa", ChangeRemote, Help, Version, "zz"}, got)
}
// TestHelpFlagAgreesWithHelpCommand pins that -h answers through the registered help
// command, and that help on a clone describes the clone: a command registered only there
// is known to both -h and `help <name>`, and one withheld from it is known to neither.
func TestHelpFlagAgreesWithHelpCommand(t *testing.T) {
r := testRegistry()
c := r.Clone()
c.Register(&Command{
Name: Logout,
ShortDescription: "ends the session",
Callback: func(fs any, a []string, w StringWriter) error { return nil },
})
flag := &bytes.Buffer{}
c.Dispatch("logout -h", NewStringWriter(flag))
cmd := &bytes.Buffer{}
c.Dispatch("help logout", NewStringWriter(cmd))
assert.Equal(t, cmd.String(), flag.String())
assert.Equal(t, "logout - ends the session\n", flag.String())
// The source never learns of it.
src := &bytes.Buffer{}
r.Dispatch("help logout", NewStringWriter(src))
assert.Equal(t, "Command not available logout\n", src.String())
// And a command both hold is described either way.
flag.Reset()
c.Dispatch("served -h", NewStringWriter(flag))
assert.Equal(t, "served - a command this surface serves\n", flag.String())
}
// TestCloneHelpListsTheClone: the command list help prints is the clone's, so a clone
// that withholds a command does not name it, and the source still does.
func TestCloneHelpListsTheClone(t *testing.T) {
r := testRegistry()
c := New()
c.Register(r.Get("served"))
c = c.Clone()
out := &bytes.Buffer{}
c.Dispatch("help", NewStringWriter(out))
assert.Equal(t, "Available commands:\n"+
"help - prints available commands or help <command> for specific usage info\n"+
"served - a command this surface serves\n\n", out.String())
out.Reset()
c.Dispatch("help withheld", NewStringWriter(out))
assert.Equal(t, "Command not available withheld\n", out.String())
out.Reset()
r.Dispatch("help withheld", NewStringWriter(out))
assert.Equal(t, "withheld - a command this surface withholds\n", out.String())
}
// TestDispatchReturnsTheCommandsError: what the callback returns reaches the caller, so
// a surface can log it.
func TestDispatchReturnsTheCommandsError(t *testing.T) {
r := New()
r.Register(&Command{
Name: "fails",
Callback: func(fs any, a []string, w StringWriter) error {
return errors.New("boom")
},
})
out := &bytes.Buffer{}
require.EqualError(t, r.Dispatch("fails", NewStringWriter(out)), "boom")
assert.Empty(t, out.String(), "a command's own error is not shown; the command speaks for itself")
}
// TestUnknownCommandNamesTheWord: only the first word is looked up, so only it is
// reported, followed by the command list, and the line is a failure to the surface.
func TestUnknownCommandNamesTheWord(t *testing.T) {
out := &bytes.Buffer{}
err := testRegistry().Dispatch(`nope "with args"`, NewStringWriter(out))
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, "did not understand: nope", f.Msg)
assert.True(t, strings.HasPrefix(out.String(), "did not understand: nope\nAvailable commands:\n"), out.String())
}
-67
View File
@@ -1,67 +0,0 @@
package nebula
import (
"bytes"
"io"
"net/netip"
"testing"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestLogFormatRefusalIsShown: a format the logger does not know is reported to the user
// as a failure, rather than returned bare, which over ssh was silence.
func TestLogFormatRefusalIsShown(t *testing.T) {
l := logging.NewLogger(io.Discard)
out := &bytes.Buffer{}
w := commands.NewStringWriter(out)
var f *commands.Failure
require.ErrorAs(t, logFormatCommand(l, nil, []string{"bogus"}, w), &f)
assert.Equal(t, "Unable to set log format: unknown log format `bogus`. possible formats: [text json]\n", out.String())
out.Reset()
require.NoError(t, logFormatCommand(l, nil, []string{"JSON"}, w))
assert.Equal(t, "Log format is: json\n", out.String())
}
// TestPrintRelaysReportsErrorsAsText: the Error field was an error value, which encodes
// as {} and lost the message. It is the message now, and absent when there is none.
func TestPrintRelaysReportsErrorsAsText(t *testing.T) {
l := test.NewLogger()
hm := newHostMap(l)
relay := netip.MustParseAddr("10.0.0.1")
peer := netip.MustParseAddr("10.0.0.2")
// A relay host that is in Relays but not in Hosts, so its hostinfo cannot be found.
hm.Relays[1] = &HostInfo{vpnAddrs: []netip.Addr{relay}}
ifce := &Interface{hostMap: hm}
out := &bytes.Buffer{}
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.JSONEq(t, `{"Relays":[{"NebulaAddr":"10.0.0.1","RelayForAddrs":[{"Error":"could not find hostinfo","Type":"","State":"","PeerAddr":"","LocalIndex":0,"RemoteIndex":0,"RelayedThrough":null}]}]}`, out.String())
// The same relay host, found, relaying for a peer whose index matches: no Error key.
hi := hm.Relays[1]
r := &Relay{Type: ForwardingType, State: Established, PeerAddr: peer, LocalIndex: 1, RemoteIndex: 2}
hi.relayState = RelayState{
relayForByAddr: map[netip.Addr]*Relay{peer: r},
relayForByIdx: map[uint32]*Relay{r.LocalIndex: r},
}
hm.Hosts[relay] = hi
out.Reset()
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.JSONEq(t, `{"Relays":[{"NebulaAddr":"10.0.0.1","RelayForAddrs":[{"Type":"forwarding","State":"established","PeerAddr":"10.0.0.2","LocalIndex":1,"RemoteIndex":2,"RelayedThrough":null}]}]}`, out.String())
// And an index mismatch names it.
hm.Relays[7] = hi
delete(hm.Relays, 1)
out.Reset()
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.Contains(t, out.String(), `"Error":"hostmap LocalIndex '7' does not match RelayState LocalIndex"`)
}
+31 -55
View File
@@ -26,12 +26,9 @@ type C struct {
files []string files []string
Settings map[string]any Settings map[string]any
oldSettings map[string]any oldSettings map[string]any
callbacks []func(*C, *Reload) callbacks []func(*C)
l *slog.Logger l *slog.Logger
reloadLock sync.Mutex reloadLock sync.Mutex
// Set by the first reload that needs a restart and never cleared. Guarded by reloadLock
needsRestart bool
} }
func NewC(l *slog.Logger) *C { func NewC(l *slog.Logger) *C {
@@ -75,9 +72,8 @@ func (c *C) LoadString(raw string) error {
// RegisterReloadCallback stores a function to be called when a config reload is triggered. The functions registered // RegisterReloadCallback stores a function to be called when a config reload is triggered. The functions registered
// here should decide if they need to make a change to the current process before making the change. HasChanged can be // here should decide if they need to make a change to the current process before making the change. HasChanged can be
// used to help decide if a change is necessary. // used to help decide if a change is necessary.
// These functions should return quickly or spawn their own go routine if they will take a while. // These functions should return quickly or spawn their own go routine if they will take a while
// Use r to report anything that failed or needs a restart to apply. func (c *C) RegisterReloadCallback(f func(*C)) {
func (c *C) RegisterReloadCallback(f func(c *C, r *Reload)) {
c.callbacks = append(c.callbacks, f) c.callbacks = append(c.callbacks, f)
} }
@@ -86,21 +82,6 @@ func (c *C) InitialLoad() bool {
return c.oldSettings == nil return c.oldSettings == nil
} }
// Render returns keys and their values as yaml, for a subsystem to remember what it applied. Empty means they couldn't
// be rendered and should never match.
func (c *C) Render(keys ...string) string {
m := make(map[string]any, len(keys))
for _, k := range keys {
m[k] = c.Get(k)
}
b, err := yaml.Marshal(m)
if err != nil {
c.l.Debug("Failed to render config", "keys", keys, "error", err)
return ""
}
return string(b)
}
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of // HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
// k in both the old and new settings will be serialized, the result of the string comparison is returned. // k in both the old and new settings will be serialized, the result of the string comparison is returned.
// If k is an empty string the entire config is tested. // If k is an empty string the entire config is tested.
@@ -163,55 +144,50 @@ func (c *C) CatchHUP(ctx context.Context) {
return return
case <-ch: case <-ch:
c.l.Info("Caught HUP, reloading config") c.l.Info("Caught HUP, reloading config")
c.ReloadConfig().Log(c.l) c.ReloadConfig()
} }
} }
}() }()
} }
// ReloadConfig reloads the config from the path provided to Load and runs every registered callback. A config that func (c *C) ReloadConfig() {
// fails to load is logged here, like any other failure, and the result carries the error for the summary.
func (c *C) ReloadConfig() ReloadResult {
c.reloadLock.Lock() c.reloadLock.Lock()
defer c.reloadLock.Unlock() defer c.reloadLock.Unlock()
c.oldSettings = make(map[string]any) c.oldSettings = make(map[string]any)
maps.Copy(c.oldSettings, c.Settings) maps.Copy(c.oldSettings, c.Settings)
if err := c.Load(c.path); err != nil { err := c.Load(c.path)
c.l.Error("Error occurred while reloading config", "config_path", c.path, "error", err) if err != nil {
return ReloadResult{Err: err, RestartRequired: c.needsRestart} c.l.Error("Error occurred while reloading config",
"config_path", c.path,
"error", err,
)
return
} }
return c.runCallbacks()
}
// ReloadConfigString is ReloadConfig for a config held in memory rather than on disk.
func (c *C) ReloadConfigString(raw string) ReloadResult {
c.reloadLock.Lock()
defer c.reloadLock.Unlock()
c.oldSettings = make(map[string]any)
maps.Copy(c.oldSettings, c.Settings)
if err := c.LoadString(raw); err != nil {
c.l.Error("Error occurred while reloading config", "error", err)
return ReloadResult{Err: err, RestartRequired: c.needsRestart}
}
return c.runCallbacks()
}
func (c *C) runCallbacks() ReloadResult {
r := &Reload{c: c}
for _, v := range c.callbacks { for _, v := range c.callbacks {
v(c, r) v(c)
}
} }
if r.restart { func (c *C) ReloadConfigString(raw string) error {
c.needsRestart = true c.reloadLock.Lock()
defer c.reloadLock.Unlock()
c.oldSettings = make(map[string]any)
maps.Copy(c.oldSettings, c.Settings)
err := c.LoadString(raw)
if err != nil {
return err
} }
return ReloadResult{Failed: r.failed, RestartRequired: c.needsRestart, RestartKeys: r.keys}
for _, v := range c.callbacks {
v(c)
}
return nil
} }
// GetString will get the string for k or return the default d if not found or invalid // GetString will get the string for k or return the default d if not found or invalid
@@ -415,7 +391,7 @@ func (c *C) parse() error {
var nm map[string]any var nm map[string]any
err = yaml.Unmarshal(b, &nm) err = yaml.Unmarshal(b, &nm)
if err != nil { if err != nil {
return fmt.Errorf("%s: %w", path, err) return err
} }
// We need to use WithAppendSlice so that firewall rules in separate // We need to use WithAppendSlice so that firewall rules in separate
+2 -2
View File
@@ -19,7 +19,7 @@ func TestConfig_Load(t *testing.T) {
// invalid yaml // invalid yaml
c := NewC(l) c := NewC(l)
os.WriteFile(filepath.Join(dir, "01.yaml"), []byte(" invalid yaml"), 0644) os.WriteFile(filepath.Join(dir, "01.yaml"), []byte(" invalid yaml"), 0644)
require.EqualError(t, c.Load(dir), filepath.Join(dir, "01.yaml")+": yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into map[string]interface {}") require.EqualError(t, c.Load(dir), "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into map[string]interface {}")
// simple multi config merge // simple multi config merge
c = NewC(l) c = NewC(l)
@@ -129,7 +129,7 @@ func TestConfig_ReloadConfig(t *testing.T) {
os.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n inner: ho"), 0644) os.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n inner: ho"), 0644)
c.RegisterReloadCallback(func(c *C, _ *Reload) { c.RegisterReloadCallback(func(c *C) {
done <- true done <- true
}) })
-98
View File
@@ -1,98 +0,0 @@
package config
import (
"context"
"fmt"
"log/slog"
"slices"
)
// ReloadResult describes how a reload went. Problems are logged where they happen, Log writes a one line summary.
type ReloadResult struct {
// Err is set when the new config could not be read or parsed, no callbacks ran
Err error
// Failed is true when a callback could not apply part of the new config
Failed bool
// RestartRequired is true once any reload has needed a restart, and stays true until the process restarts
RestartRequired bool
// RestartKeys are the keys this reload changed that only apply on a restart
RestartKeys []string
}
// Summary returns a one line summary of the reload and the level to log it at.
func (r ReloadResult) Summary() (slog.Level, string) {
switch {
case r.Err != nil && r.RestartRequired:
return slog.LevelError, "Config reload failed, the new config did not load and a restart is still required"
case r.Err != nil:
return slog.LevelError, "Config reload failed, the new config did not load"
case r.Failed && r.RestartRequired:
return slog.LevelError, "Config reload completed with errors and requires a restart to fully apply, review the logs"
case r.Failed:
return slog.LevelError, "Config reload completed with errors, review the logs"
case r.RestartRequired:
return slog.LevelWarn, "Config reload was completed but requires a restart to fully apply"
default:
return slog.LevelInfo, "Config reload complete"
}
}
// String is the summary with Err and RestartKeys included, for output other than the log.
func (r ReloadResult) String() string {
_, msg := r.Summary()
if r.Err != nil {
msg = fmt.Sprintf("%s: %v", msg, r.Err)
}
if len(r.RestartKeys) > 0 {
msg = fmt.Sprintf("%s, keys %v", msg, r.RestartKeys)
}
return msg
}
// Log writes Summary to l.
func (r ReloadResult) Log(l *slog.Logger) {
level, msg := r.Summary()
if len(r.RestartKeys) > 0 {
l.Log(context.Background(), level, msg, "keys", r.RestartKeys)
return
}
l.Log(context.Background(), level, msg)
}
// Reload is passed to each reload callback to report how applying the new config went.
type Reload struct {
c *C
failed bool
restart bool
// Changed keys that only apply on a restart, from every callback
keys []string
}
// Failed marks the reload as failed, the caller logs why. Safe to call on a nil Reload, which is what the initial load
// passes.
func (r *Reload) Failed() {
if r != nil {
r.failed = true
}
}
// RestartRequired marks the reload as needing a restart, the caller logs why. Safe to call on a nil Reload.
func (r *Reload) RestartRequired() {
if r != nil {
r.restart = true
}
}
// RestartRequiredIfChanged marks the reload as needing a restart if any of keys changed, and adds them to RestartKeys.
// Use it for keys that are only read at startup. Safe to call on a nil Reload.
func (r *Reload) RestartRequiredIfChanged(keys ...string) {
if r == nil {
return
}
for _, k := range keys {
if r.c.HasChanged(k) && !slices.Contains(r.keys, k) {
r.keys = append(r.keys, k)
r.restart = true
}
}
}
-108
View File
@@ -1,108 +0,0 @@
package config
import (
"errors"
"log/slog"
"testing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReloadConfigString(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1"))
fail := false
ran := 0
c.RegisterReloadCallback(func(c *C, r *Reload) {
ran++
if fail {
r.Failed()
}
})
assert.Equal(t, ReloadResult{}, c.ReloadConfigString("a: 2"))
fail = true
assert.Equal(t, ReloadResult{Failed: true}, c.ReloadConfigString("a: 3"))
assert.Equal(t, 2, ran)
// A config that fails to load runs no callbacks and returns the error
res := c.ReloadConfigString("a: [")
require.Error(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, 2, ran)
assert.Equal(t, 3, c.GetInt("a", 0))
}
func TestReloadRestartRequiredLatches(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1\nb: 1"))
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a", "b") })
// Nothing that needs a restart changed
assert.Equal(t, ReloadResult{}, c.ReloadConfigString("a: 1\nb: 1\nc: 1"))
assert.Equal(t, ReloadResult{RestartRequired: true, RestartKeys: []string{"a"}}, c.ReloadConfigString("a: 2\nb: 1\nc: 1"))
// The restart sticks through a reload with no changes, reverting the key and a config that fails to load. Only the
// reload that changed the key lists it
assert.Equal(t, ReloadResult{RestartRequired: true}, c.ReloadConfigString("a: 2\nb: 1\nc: 1"))
assert.Equal(t, ReloadResult{RestartRequired: true, RestartKeys: []string{"a"}}, c.ReloadConfigString("a: 1\nb: 1\nc: 1"))
res := c.ReloadConfigString("a: [")
require.Error(t, res.Err)
assert.True(t, res.RestartRequired)
}
// A key reported by more than one callback is listed once
func TestReloadRestartKeysDeduped(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1\nb: 1"))
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a") })
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a", "b") })
assert.Equal(t, []string{"a", "b"}, c.ReloadConfigString("a: 2\nb: 2").RestartKeys)
}
// A callback can require a restart without a key changing, a file the config points at may have
func TestReloadRestartRequiredWithoutAChange(t *testing.T) {
c := NewC(test.NewLogger())
require.NoError(t, c.LoadString("a: 1"))
c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequired() })
assert.Equal(t, ReloadResult{RestartRequired: true}, c.ReloadConfigString("a: 1"))
}
func TestReloadResultSummary(t *testing.T) {
cases := []struct {
res ReloadResult
level slog.Level
msg string
}{
{ReloadResult{}, slog.LevelInfo, "Config reload complete"},
{ReloadResult{RestartRequired: true}, slog.LevelWarn, "Config reload was completed but requires a restart to fully apply"},
{ReloadResult{Failed: true}, slog.LevelError, "Config reload completed with errors, review the logs"},
{ReloadResult{Failed: true, RestartRequired: true}, slog.LevelError, "Config reload completed with errors and requires a restart to fully apply, review the logs"},
{ReloadResult{Err: errors.New("bad yaml")}, slog.LevelError, "Config reload failed, the new config did not load"},
{ReloadResult{Err: errors.New("bad yaml"), RestartRequired: true}, slog.LevelError, "Config reload failed, the new config did not load and a restart is still required"},
}
for _, tc := range cases {
level, msg := tc.res.Summary()
assert.Equal(t, tc.level, level, tc.msg)
assert.Equal(t, tc.msg, msg)
}
res := ReloadResult{RestartRequired: true, RestartKeys: []string{"a", "b"}}
assert.Equal(t, "Config reload was completed but requires a restart to fully apply, keys [a b]", res.String())
// String includes the error
assert.Equal(t, "Config reload failed, the new config did not load: bad yaml", ReloadResult{Err: errors.New("bad yaml")}.String())
}
// The initial load passes a nil Reload
func TestReloadNilIsSafe(t *testing.T) {
var r *Reload
r.Failed()
r.RestartRequired()
r.RestartRequiredIfChanged("a")
}
+5 -4
View File
@@ -57,8 +57,7 @@ func newConnectionManagerFromConfig(l *slog.Logger, c *config.C, hm *HostMap, p
} }
cm.reload(c, true) cm.reload(c, true)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("timers.connection_alive_interval", "timers.pending_deletion_interval")
cm.reload(c, false) cm.reload(c, false)
}) })
@@ -175,8 +174,10 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
switch decision { switch decision {
case deleteTunnel: case deleteTunnel:
// Local delete only, clearing the lighthouse cache if this was the last hostinfo for the vpn addr if cm.hostMap.DeleteHostInfo(hostinfo) {
cm.intf.closeTunnel(hostinfo) // Only clearing the lighthouse cache if this is the last hostinfo for this vpn ip in the hostmap
cm.intf.lightHouse.DeleteVpnAddrs(hostinfo.vpnAddrs)
}
case closeTunnel: case closeTunnel:
cm.intf.sendCloseTunnel(hostinfo) cm.intf.sendCloseTunnel(hostinfo)
-3
View File
@@ -7,7 +7,6 @@ import (
"testing" "testing"
"time" "time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/overlay/overlaytest" "github.com/slackhq/nebula/overlay/overlaytest"
@@ -22,8 +21,6 @@ func newTestLighthouse() *LightHouse {
l: test.NewLogger(), l: test.NewLogger(),
addrMap: map[netip.Addr]*RemoteList{}, addrMap: map[netip.Addr]*RemoteList{},
queryChan: make(chan netip.Addr, 10), queryChan: make(chan netip.Addr, 10),
// Nothing drains queryChan here, so an 11th query is dropped.
queryDropped: metrics.NilCounter{},
} }
lighthouses := []netip.Addr{} lighthouses := []netip.Addr{}
staticList := map[netip.Addr]struct{}{} staticList := map[netip.Addr]struct{}{}
-119
View File
@@ -2,8 +2,6 @@ package nebula
import ( import (
"net/netip" "net/netip"
"strconv"
"sync"
"testing" "testing"
"time" "time"
@@ -13,7 +11,6 @@ import (
ct "github.com/slackhq/nebula/cert_test" ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake" "github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -150,122 +147,6 @@ func TestSendNoMetricsCloseTunnelKeepsRebindEpoch(t *testing.T) {
assert.True(t, hostinfo.markOut(1), "a CloseTunnel send must not consume the rebind epoch") assert.True(t, hostinfo.markOut(1), "a CloseTunnel send must not consume the rebind epoch")
} }
// capturingConn is a udp.Conn that records every WriteTo.
type capturingConn struct {
udp.NoopConn
writes [][]byte
addrs []netip.AddrPort
}
func (c *capturingConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.writes = append(c.writes, append([]byte(nil), b...))
c.addrs = append(c.addrs, addr)
return nil
}
// TestSendNoMetricsViaRelay sends through a relay from the mtu sized buffer the cached packet flush uses. Payloads up
// to overlay.MaxMTU go out intact, anything bigger is dropped rather than outgrowing the buffer, which once panicked.
func TestSendNoMetricsViaRelay(t *testing.T) {
tests := []struct {
n int
sent bool
}{
{0, true},
{1300, true},
{overlay.MaxMTU, true},
{overlay.MaxMTU + 1, false}, // too big for the relay's tag
{mtu - 47, false}, // too big for the inner tag, this panicked
{9000, false}, // a full packet at tun.mtu 9000
}
for _, tt := range tests {
t.Run(strconv.Itoa(tt.n), func(t *testing.T) {
// One tunnel to the target, carried end to end, and one to the relay that wraps it
targetInit, targetResp := runTestHandshake(t)
relayInit, relayResp := runTestHandshake(t)
ci, err := newConnectionStateFromResult(targetInit)
require.NoError(t, err)
relayCI, err := newConnectionStateFromResult(relayInit)
require.NoError(t, err)
targetAddr := netip.MustParseAddr("10.0.0.2")
relayAddr := netip.MustParseAddr("10.0.0.3")
relayRemote := netip.MustParseAddrPort("192.0.2.1:4242")
relay := &Relay{Type: TerminalType, State: Established, LocalIndex: 5, RemoteIndex: 6, PeerAddr: targetAddr}
relayHI := &HostInfo{
vpnAddrs: []netip.Addr{relayAddr},
ConnectionState: relayCI,
relayState: RelayState{
relayForByAddr: map[netip.Addr]*Relay{targetAddr: relay},
relayForByIdx: map[uint32]*Relay{relay.LocalIndex: relay},
},
}
relayHI.remote.Store(&relayRemote)
// No remote of its own, so the send has to go via the relay
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{targetAddr},
ConnectionState: ci,
remoteIndexId: 7,
relayState: RelayState{
relays: []netip.Addr{relayAddr},
relayForByAddr: map[netip.Addr]*Relay{},
relayForByIdx: map[uint32]*Relay{},
},
}
l := test.NewLogger()
hm := newHostMap(l)
hm.Hosts[relayAddr] = relayHI
conn := &capturingConn{}
f := &Interface{
l: l,
hostMap: hm,
messageMetrics: &MessageMetrics{txExhausted: metrics.NewCounter()},
writers: []udp.Conn{conn},
}
f.connectionManager = &connectionManager{intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{}}
payload := make([]byte, tt.n)
for i := range payload {
payload[i] = byte(i)
}
counter := ci.messageCounter.Load()
f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, payload, make([]byte, 12), make([]byte, mtu), 0)
if !tt.sent {
assert.Empty(t, conn.writes)
assert.Equal(t, counter, ci.messageCounter.Load(), "a dropped packet must not spend a counter")
return
}
require.Len(t, conn.writes, 1)
assert.Equal(t, relayRemote, conn.addrs[0])
pkt := conn.writes[0]
// The relay authenticates the whole packet under its own header
outer := &header.H{}
require.NoError(t, outer.Parse(pkt))
assert.Equal(t, header.MessageRelay, outer.Subtype)
assert.Equal(t, relay.RemoteIndex, outer.RemoteIndex)
relayPeer, err := newConnectionStateFromResult(relayResp)
require.NoError(t, err)
require.NoError(t, relayPeer.VerifyRelay(l, outer.MessageCounter, pkt, make([]byte, 12)))
// Inside it, the target decrypts the end to end packet back to the payload
inner := pkt[header.Len : len(pkt)-relayPeer.dKey.Overhead()]
h := &header.H{}
require.NoError(t, h.Parse(inner))
assert.Equal(t, header.Message, h.Type)
assert.Equal(t, hostinfo.remoteIndexId, h.RemoteIndex)
targetPeer, err := newConnectionStateFromResult(targetResp)
require.NoError(t, err)
got, err := targetPeer.Decrypt(l, h.MessageCounter, inner, make([]byte, 12))
require.NoError(t, err)
assert.Equal(t, payload, got)
})
}
}
func TestNewConnectionStateFromResult(t *testing.T) { func TestNewConnectionStateFromResult(t *testing.T) {
initR, respR := runTestHandshake(t) initR, respR := runTestHandshake(t)
-9
View File
@@ -11,7 +11,6 @@ import (
"syscall" "syscall"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/overlay"
) )
@@ -48,7 +47,6 @@ type Control struct {
f *Interface f *Interface
l *slog.Logger l *slog.Logger
commands *commands.Registry
ctx context.Context ctx context.Context
cancel context.CancelFunc cancel context.CancelFunc
sshStart func() sshStart func()
@@ -374,13 +372,6 @@ func (c *Control) CloseAllTunnels(excludeLighthouses bool) (closed int) {
return return
} }
// Commands returns the registry the daemon's surfaces serve from. It is the live
// registry, not a copy: a command registered on it is served to every ssh session that
// starts afterwards. Clone it to run commands without that reach.
func (c *Control) Commands() *commands.Registry {
return c.commands
}
func (c *Control) Device() overlay.Device { func (c *Control) Device() overlay.Device {
return c.f.inside return c.f.inside
} }
+1 -1
View File
@@ -151,7 +151,7 @@ func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) { func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
return len(bufs), nil return len(bufs), nil
} }
func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {} func (c *fakeConn) ReloadConfig(_ *config.C) {}
func (c *fakeConn) SupportsMultipleReaders() bool { return true } func (c *fakeConn) SupportsMultipleReaders() bool { return true }
func (c *fakeConn) Close() error { c.closed = true; return nil } func (c *fakeConn) Close() error { c.closed = true; return nil }
+1 -2
View File
@@ -69,10 +69,9 @@ func newDnsServerFromConfig(ctx context.Context, l *slog.Logger, pki *PKI, hostM
ds.mux = dns.NewServeMux() ds.mux = dns.NewServeMux()
ds.mux.HandleFunc(".", ds.handleDnsRequest) ds.mux.HandleFunc(".", ds.handleDnsRequest)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
if err := ds.reload(c, false); err != nil { if err := ds.reload(c, false); err != nil {
ds.l.Error("Failed to reload DNS responder from config", "error", err) ds.l.Error("Failed to reload DNS responder from config", "error", err)
r.Failed()
} }
}) })
-164
View File
@@ -1,164 +0,0 @@
//go:build e2e_testing
package e2e
import (
"bytes"
"fmt"
"path/filepath"
"regexp"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/require"
)
// volatile lists everything in command output that legitimately differs between runs.
// Each run mints fresh certificates and indexes, so these are replaced with a marker
// before comparison. Keep the list short: anything added here is something the golden
// stops guarding.
var volatile = []struct {
re *regexp.Regexp
with string
}{
{regexp.MustCompile(`[0-9a-f]{40,}`), "<hex>"}, // fingerprints, keys, signatures
{regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})`), "<time>"}, // RFC 3339
{regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)? [+-]\d{4} \w+`), "<time>"}, // time.Time.String()
{regexp.MustCompile(`(?s)(-----BEGIN [A-Z ]+-----\n).*?(\n-----END [A-Z ]+-----)`), "${1}<pem>${2}"}, // print-cert -raw
{regexp.MustCompile(`(-----BEGIN [A-Z ]+-----\\n).*?(\\n-----END [A-Z ]+-----)`), "${1}<pem>${2}"}, // the same, inside a JSON string
{regexp.MustCompile(`"(localIndex|remoteIndex|messageCounter)":\d+`), `"$1":<n>`},
{regexp.MustCompile(`(?m)^(\s*"?(?:LocalIndex|RemoteIndex|localIndex|remoteIndex|messageCounter)"?:?\s*)\d+`), "${1}<n>"}, // and the -pretty form
}
func normalise(s string) string {
for _, v := range volatile {
s = v.re.ReplaceAllString(s, v.with)
}
return s
}
// outputCase is one command line to run through the ssh dispatch path.
type outputCase struct {
name string
args []string
}
// TestCommandOutput pins the bytes every command produces, in every format, through the
// ssh dispatch path. It is the safety net for moving rendering out of the commands: a
// change here is a change someone at an ssh prompt would see. Regenerate with:
//
// go test -tags e2e_testing ./e2e -run TestCommandOutput -update
func TestCommandOutput(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(
cert.Version1, cert.Curve_CURVE25519,
time.Now(), time.Now().Add(10*time.Minute),
nil, nil, []string{},
)
control, _, _, _ := newSimpleServer(cert.Version1, ca, caKey, "output-golden", "10.222.0.1/24", nil)
control.Start()
t.Cleanup(func() { control.Stop() })
// As an ssh session does: a clone of the daemon's registry with logout added, which
// help on the clone lists and describes. There is no session to end here.
reg := control.Commands().Clone()
reg.Register(sshd.LogoutCommand(func() {}))
// run dispatches as the ssh exec path does and returns the output and the error, which
// over ssh is exit status 1.
run := func(args ...string) (string, error) {
buf := &bytes.Buffer{}
err := reg.DispatchArgs(args, commands.NewStringWriter(buf))
return buf.String(), err
}
// Seed a pending tunnel so the listing and per-address commands have something to show.
seed, err := run("create-tunnel", "10.222.0.9")
require.NoError(t, err)
require.Equal(t, "Created\n", seed, "seeding a tunnel should succeed before anything else is recorded")
cases := []outputCase{
{"version", nil},
{"device-info", nil},
{"list-hostmap", nil},
{"list-hostmap", []string{"-by-index"}},
{"list-pending-hostmap", nil},
{"list-pending-hostmap", []string{"-by-index"}},
{"list-lighthouse-addrmap", nil},
{"print-cert", nil},
{"print-cert", []string{"-raw"}},
{"print-cert", []string{"10.222.0.9"}},
{"print-cert", []string{"10.222.0.250"}},
{"print-cert", []string{"not-an-address"}},
{"print-tunnel", nil},
{"print-tunnel", []string{"10.222.0.9"}},
{"print-tunnel", []string{"10.222.0.250"}},
{"print-relays", nil},
{"query-lighthouse", nil},
{"query-lighthouse", []string{"10.222.0.9"}},
{"query-lighthouse", []string{"not-an-address"}},
// The test logger cannot change level or format, so these cases record the
// refusal each command prints on it, not a successful change.
{"log-level", nil},
{"log-level", []string{"debug"}},
{"log-level", []string{"bogus"}},
{"log-level", []string{"info"}},
{"log-format", nil},
{"log-format", []string{"json"}},
{"log-format", []string{"text"}},
{"change-remote", nil},
{"change-remote", []string{"10.222.0.9"}},
{"change-remote", []string{"-address", "10.94.0.10:4242", "10.222.0.9"}},
{"change-remote", []string{"-address", "10.94.0.10:4242", "10.222.0.250"}},
{"create-tunnel", nil},
{"create-tunnel", []string{"10.222.0.9"}},
// A fresh address, so this exercises the handshake start with a remote attached
// rather than the "already handshaking" path above.
{"create-tunnel", []string{"-address", "10.94.0.10:4242", "10.222.0.10"}},
{"create-tunnel", []string{"not-an-address"}},
{"close-tunnel", nil},
{"close-tunnel", []string{"10.222.0.250"}},
{"close-tunnel", []string{"not-an-address"}},
{"help", nil},
{"help", []string{"print-cert"}},
{"help", []string{"logout"}},
{"logout", []string{"-h"}},
{"not-a-command", []string{"10.222.0.9"}},
// reload is last so that whatever a reload does to the daemon cannot leak into an
// earlier case. In this harness it fails: the config was loaded from a string and
// has no path to re-read, which is what the recorded result says.
{"reload", nil},
}
got := &strings.Builder{}
for _, c := range cases {
for _, fmtFlags := range [][]string{nil, {"-json"}, {"-pretty"}} {
args := append([]string{c.name}, fmtFlags...)
args = append(args, c.args...)
fmt.Fprintf(got, "$ %s\n", strings.Join(args, " "))
out, err := run(args...)
out = normalise(out)
got.WriteString(out)
// Output that does not end in a newline is recorded as such, the way diff marks
// it, so the golden sees that byte too and the next prompt still starts a line.
if out != "" && !strings.HasSuffix(out, "\n") {
got.WriteString("\\ No newline at end of output\n")
}
// A failed command is recorded as the exit status an ssh caller sees, so the
// golden pins which outcomes count as failures.
if err != nil {
got.WriteString("exit status 1\n")
}
got.WriteString("\n")
}
}
test.Golden(t, filepath.Join("testdata", "commands_output.golden"), got.String())
}
+1 -1
View File
@@ -1513,7 +1513,7 @@ func TestLighthouseUpdateOnReload(t *testing.T) {
// Reload the config. The lighthouse.hosts change triggers TriggerUpdate, // Reload the config. The lighthouse.hosts change triggers TriggerUpdate,
// which wakes the update worker. It calls SendUpdate, initiating a // which wakes the update worker. It calls SendUpdate, initiating a
// handshake to the new lighthouse and caching the HostUpdateNotification. // handshake to the new lighthouse and caching the HostUpdateNotification.
require.NoError(t, myConfig.ReloadConfigString(string(newCfg)).Err) require.NoError(t, myConfig.ReloadConfigString(string(newCfg)))
// Route until the lighthouse receives the HostUpdateNotification. // Route until the lighthouse receives the HostUpdateNotification.
// This covers: handshake stage 1, stage 2, then the cached update. // This covers: handshake stage 1, stage 2, then the cached update.
-720
View File
@@ -1,720 +0,0 @@
$ version
e2e-test
$ version -json
e2e-test
$ version -pretty
e2e-test
$ device-info
name= cidr=[10.222.0.1/24]
$ device-info -json
{"name":"","cidr":["10.222.0.1/24"]}
$ device-info -pretty
{
"name": "",
"cidr": [
"10.222.0.1/24"
]
}
$ list-hostmap
$ list-hostmap -json
[]
$ list-hostmap -pretty
[]
$ list-hostmap -by-index
$ list-hostmap -json -by-index
[]
$ list-hostmap -pretty -by-index
[]
$ list-pending-hostmap
[10.222.0.9]: []
$ list-pending-hostmap -json
[{"vpnAddrs":["10.222.0.9"],"localIndex":<n>,"remoteIndex":<n>,"remoteAddrs":[],"cert":null,"messageCounter":<n>,"currentRemote":"","currentRelaysToMe":[],"currentRelaysThroughMe":[]}]
$ list-pending-hostmap -pretty
[
{
"vpnAddrs": [
"10.222.0.9"
],
"localIndex": <n>,
"remoteIndex": <n>,
"remoteAddrs": [],
"cert": null,
"messageCounter": <n>,
"currentRemote": "",
"currentRelaysToMe": [],
"currentRelaysThroughMe": []
}
]
$ list-pending-hostmap -by-index
$ list-pending-hostmap -json -by-index
[]
$ list-pending-hostmap -pretty -by-index
[]
$ list-lighthouse-addrmap
10.222.0.9: {}
$ list-lighthouse-addrmap -json
[{"vpnAddr":"10.222.0.9","addrs":{}}]
$ list-lighthouse-addrmap -pretty
[
{
"vpnAddr": "10.222.0.9",
"addrs": {}
}
]
$ print-cert
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}
$ print-cert -json
{"details":{"curve":"CURVE25519","groups":[],"isCa":false,"issuer":"<hex>","name":"output-golden","networks":["10.222.0.1/24"],"notAfter":"<time>","notBefore":"<time>","publicKey":"<hex>","unsafeNetworks":[]},"fingerprint":"<hex>","signature":"<hex>","version":1}\ No newline at end of output
$ print-cert -pretty
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}\ No newline at end of output
$ print-cert -raw
-----BEGIN NEBULA CERTIFICATE-----
<pem>
-----END NEBULA CERTIFICATE-----
$ print-cert -json -raw
{"details":{"curve":"CURVE25519","groups":[],"isCa":false,"issuer":"<hex>","name":"output-golden","networks":["10.222.0.1/24"],"notAfter":"<time>","notBefore":"<time>","publicKey":"<hex>","unsafeNetworks":[]},"fingerprint":"<hex>","signature":"<hex>","version":1}\ No newline at end of output
$ print-cert -pretty -raw
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}\ No newline at end of output
$ print-cert 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert -json 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert -pretty 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert -json 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert -pretty 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-cert -json not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-cert -pretty not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-tunnel
No vpn address was provided
exit status 1
$ print-tunnel -json
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty
No vpn address was provided
exit status 1
$ print-tunnel 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-tunnel -json 10.222.0.9
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-tunnel 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-tunnel -json 10.222.0.250
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-relays
{"Relays":null}
$ print-relays -json
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-relays -pretty
{
"Relays": null
}
$ query-lighthouse
No vpn address was provided
exit status 1
$ query-lighthouse -json
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty
The provided vpn address could not be parsed: -pretty
exit status 1
$ query-lighthouse 10.222.0.9
{}
$ query-lighthouse -json 10.222.0.9
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty 10.222.0.9
The provided vpn address could not be parsed: -pretty
exit status 1
$ query-lighthouse not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ query-lighthouse -json not-an-address
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty not-an-address
The provided vpn address could not be parsed: -pretty
exit status 1
$ log-level
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty
Log level is not reconfigurable on this logger
exit status 1
$ log-level debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level info
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json info
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty info
Log level is not reconfigurable on this logger
exit status 1
$ log-format
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty
Log format is not reconfigurable on this logger
exit status 1
$ log-format json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty json
Log format is not reconfigurable on this logger
exit status 1
$ log-format text
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json text
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty text
Log format is not reconfigurable on this logger
exit status 1
$ change-remote
No vpn address was provided
exit status 1
$ change-remote -json
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote 10.222.0.9
No address was provided
exit status 1
$ change-remote -json 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -address 10.94.0.10:4242 10.222.0.9
Could not find tunnel for vpn address: 10.222.0.9
exit status 1
$ change-remote -json -address 10.94.0.10:4242 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty -address 10.94.0.10:4242 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -address 10.94.0.10:4242 10.222.0.250
Could not find tunnel for vpn address: 10.222.0.250
exit status 1
$ change-remote -json -address 10.94.0.10:4242 10.222.0.250
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty -address 10.94.0.10:4242 10.222.0.250
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ create-tunnel
No vpn address was provided
exit status 1
$ create-tunnel -json
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel 10.222.0.9
Tunnel already handshaking
exit status 1
$ create-tunnel -json 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -address 10.94.0.10:4242 10.222.0.10
Created
$ create-tunnel -json -address 10.94.0.10:4242 10.222.0.10
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty -address 10.94.0.10:4242 10.222.0.10
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ create-tunnel -json not-an-address
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty not-an-address
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ close-tunnel
No vpn address was provided
exit status 1
$ close-tunnel -json
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel 10.222.0.250
Could not find tunnel for vpn address: 10.222.0.250
exit status 1
$ close-tunnel -json 10.222.0.250
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty 10.222.0.250
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ close-tunnel -json not-an-address
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty not-an-address
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ help
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
$ help -json
Command not available -json
exit status 1
$ help -pretty
Command not available -pretty
exit status 1
$ help print-cert
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
-json
outputs as json
-pretty
pretty prints json, assumes -json
-raw
raw prints the PEM encoded certificate, not compatible with -json or -pretty
$ help -json print-cert
Command not available -json
exit status 1
$ help -pretty print-cert
Command not available -pretty
exit status 1
$ help logout
logout - Ends the current session
$ help -json logout
Command not available -json
exit status 1
$ help -pretty logout
Command not available -pretty
exit status 1
$ logout -h
logout - Ends the current session
$ logout -json -h
logout - Ends the current session
$ logout -pretty -h
logout - Ends the current session
$ not-a-command 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ not-a-command -json 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ not-a-command -pretty 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ reload
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
$ reload -json
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
$ reload -pretty
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
+2 -2
View File
@@ -129,7 +129,7 @@ func TestCertUpgrade(t *testing.T) {
} }
r.Logf("reload new v2-only config") r.Logf("reload new v2-only config")
err = myC.ReloadConfigString(string(cb)).Err err = myC.ReloadConfigString(string(cb))
assert.NoError(t, err) assert.NoError(t, err)
r.Log("yay, spin until their sees it") r.Log("yay, spin until their sees it")
waitStart := time.Now() waitStart := time.Now()
@@ -222,7 +222,7 @@ func TestCertDowngrade(t *testing.T) {
} }
r.Logf("reload new v1-only config") r.Logf("reload new v1-only config")
err = myC.ReloadConfigString(string(cb)).Err err = myC.ReloadConfigString(string(cb))
assert.NoError(t, err) assert.NoError(t, err)
r.Log("yay, spin until their sees it") r.Log("yay, spin until their sees it")
waitStart := time.Now() waitStart := time.Now()
+21 -32
View File
@@ -8,7 +8,6 @@ pki:
cert: /etc/nebula/host.crt cert: /etc/nebula/host.crt
key: /etc/nebula/host.key key: /etc/nebula/host.key
# blocklist is a list of certificate fingerprints that we will refuse to talk to # blocklist is a list of certificate fingerprints that we will refuse to talk to
# Entries are the hex fingerprints shown by `nebula-cert print`, an invalid entry fails startup and reload.
#blocklist: #blocklist:
# - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72 # - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72
# disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid. # disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid.
@@ -39,13 +38,12 @@ static_host_map:
# public address. Other valid options are "ip6" and "ip" (returns both.) # public address. Other valid options are "ip6" and "ip" (returns both.)
#network: ip4 #network: ip4
# lookup_timeout is the DNS query timeout. A failed lookup keeps the last known good addresses, only a not found # lookup_timeout is the DNS query timeout.
# answer removes them. #lookup_timeout: 250ms
#lookup_timeout: 5s
lighthouse: lighthouse:
# am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes # am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
# you have configured to be lighthouses in your network. Not reloadable. # you have configured to be lighthouses in your network
am_lighthouse: false am_lighthouse: false
# serve_dns optionally starts a dns listener that responds to various queries and can even be # serve_dns optionally starts a dns listener that responds to various queries and can even be
# delegated to for resolution # delegated to for resolution
@@ -128,9 +126,8 @@ lighthouse:
# Port Nebula will be listening on. The default here is 4242. For a lighthouse node, the port should be defined, # Port Nebula will be listening on. The default here is 4242. For a lighthouse node, the port should be defined,
# however using port 0 will dynamically assign a port and is recommended for roaming nodes. # however using port 0 will dynamically assign a port and is recommended for roaming nodes.
listen: listen:
# To listen on only ipv4, use "0.0.0.0". Not reloadable. # To listen on only ipv4, use "0.0.0.0"
host: "::" host: "::"
# Not reloadable.
port: 4242 port: 4242
# Sets the max number of packets to pull from the kernel for each syscall (under systems that support recvmmsg) # Sets the max number of packets to pull from the kernel for each syscall (under systems that support recvmmsg)
# default is 64, does not support reload # default is 64, does not support reload
@@ -138,13 +135,12 @@ listen:
# superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the # superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the
# default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts. # default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts.
#batch: 64 #batch: 64
# Linux only. Reloadable, but removing a value leaves the socket at the old size until Nebula restarts.
# Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel # Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
# Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default) # Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
# Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide # Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide
# max, net.core.rmem_max and net.core.wmem_max # max, net.core.rmem_max and net.core.wmem_max
read_buffer: 3412512 #read_buffer: 10485760
write_buffer: 3412512 #write_buffer: 10485760
# On Windows only # On Windows only
# When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port. # When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port.
@@ -183,7 +179,7 @@ listen:
# Currently, this defaults to 1 which means we have 1 tun queue reader and 1 # Currently, this defaults to 1 which means we have 1 tun queue reader and 1
# UDP queue reader. Setting this above one will set IFF_MULTI_QUEUE on the tun # UDP queue reader. Setting this above one will set IFF_MULTI_QUEUE on the tun
# device and SO_REUSEPORT on the UDP socket to allow multiple queues. # device and SO_REUSEPORT on the UDP socket to allow multiple queues.
# This option is only supported on Linux. Not reloadable. # This option is only supported on Linux.
#routines: 1 #routines: 1
punchy: punchy:
@@ -207,7 +203,6 @@ punchy:
# Cipher allows you to choose between the available ciphers for your network. Options are chachapoly or aes # Cipher allows you to choose between the available ciphers for your network. Options are chachapoly or aes
# IMPORTANT: this value must be identical on ALL NODES/LIGHTHOUSES. We do not/will not support use of different ciphers simultaneously! # IMPORTANT: this value must be identical on ALL NODES/LIGHTHOUSES. We do not/will not support use of different ciphers simultaneously!
# Not reloadable.
#cipher: aes #cipher: aes
# Preferred ranges is used to define a hint about the local network ranges, which speeds up discovering the fastest # Preferred ranges is used to define a hint about the local network ranges, which speeds up discovering the fastest
@@ -234,6 +229,12 @@ punchy:
# Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access. # Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
#trusted_cas: #trusted_cas:
#- "ssh public key string" #- "ssh public key string"
# sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
# save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
# and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
# The directory is NOT automatically created.
# Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
#sandbox_dir: /var/tmp/nebula-debug
# EXPERIMENTAL: relay support for networks that can't establish direct connections. # EXPERIMENTAL: relay support for networks that can't establish direct connections.
relay: relay:
@@ -252,25 +253,18 @@ relay:
# Configure the private interface. Note: addr is baked into the nebula certificate # Configure the private interface. Note: addr is baked into the nebula certificate
tun: tun:
# When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root) # When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root)
# Not reloadable.
disabled: false disabled: false
# Name of the device. If not set, a default will be chosen by the OS. # Name of the device. If not set, a default will be chosen by the OS.
# For macOS: if set, must be in the form `utun[0-9]+`. # For macOS: if set, must be in the form `utun[0-9]+`.
# For OpenBSD and NetBSD: if set, must be in the form `tun[0-9]+`, otherwise the first free one is used. # For NetBSD: Required to be set, must be in the form `tun[0-9]+`
# Not reloadable.
dev: nebula1 dev: nebula1
# Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert # Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert
# Not reloadable.
drop_local_broadcast: false drop_local_broadcast: false
# Toggles forwarding of multicast packets. Not reloadable. # Toggles forwarding of multicast packets
drop_multicast: false drop_multicast: false
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 5000 # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500
# Not reloadable. tx_queue: 500
tx_queue: 5000
# Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
# Anything above 8937, here or on a route below, is capped to 8937 with a warning
# Reloadable on Linux only. Where the tun is handed to nebula by a host app (iOS, Android, macOS network extensions)
# the host app owns the MTU.
mtu: 1300 mtu: 1300
# the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable. # the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable.
@@ -332,9 +326,6 @@ tun:
# `mtu`: will default to tun mtu if this option is not specified # `mtu`: will default to tun mtu if this option is not specified
# `metric`: will default to 0 if this option is not specified # `metric`: will default to 0 if this option is not specified
# `install`: will default to true, controls whether this route is installed in the systems routing table. # `install`: will default to true, controls whether this route is installed in the systems routing table.
# On macOS, a route whose destination another interface already routes, like the local network, is installed once
# that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another
# interface routes its destination and lets it go.
# This setting is reloadable. # This setting is reloadable.
unsafe_routes: unsafe_routes:
#- route: 172.16.1.0/24 #- route: 172.16.1.0/24
@@ -366,7 +357,7 @@ tun:
#use_system_route_table: false #use_system_route_table: false
# Buffer size for reading routes updates. 0 means default system buffer size. (/proc/sys/net/core/rmem_default). # Buffer size for reading routes updates. 0 means default system buffer size. (/proc/sys/net/core/rmem_default).
# If using massive routes updates, for example BGP, you may need to increase this value to avoid packet loss. # If using massive routes updates, for example BGP, you may need to increase this value to avoid packet loss.
# SO_RCVBUFFORCE is used to avoid having to raise the system wide max. Not reloadable. # SO_RCVBUFFORCE is used to avoid having to raise the system wide max
#use_system_route_table_buffer_size: 0 #use_system_route_table_buffer_size: 0
# Configure logging level # Configure logging level
@@ -416,16 +407,14 @@ logging:
# Handshakes are sent to all known addresses at each interval with a linear backoff, # Handshakes are sent to all known addresses at each interval with a linear backoff,
# Wait try_interval after the 1st attempt, 2 * try_interval after the 2nd, etc, until the handshake is older than timeout # Wait try_interval after the 1st attempt, 2 * try_interval after the 2nd, etc, until the handshake is older than timeout
# A 100ms interval with the default 10 retries will give a handshake 5.5 seconds to resolve before timing out # A 100ms interval with the default 10 retries will give a handshake 5.5 seconds to resolve before timing out
# Not reloadable.
#try_interval: 100ms #try_interval: 100ms
#retries: 20 #retries: 20
# query_buffer is the size of the buffer channel for querying lighthouses, drops are recorded in the lighthouse.query.dropped metric. # query_buffer is the size of the buffer channel for querying lighthouses
# Not reloadable. #query_buffer: 64
#query_buffer: 1024
# trigger_buffer is the size of the buffer channel for quickly sending handshakes # trigger_buffer is the size of the buffer channel for quickly sending handshakes
# after receiving the response for lighthouse queries. Not reloadable. # after receiving the response for lighthouse queries
#trigger_buffer: 64 #trigger_buffer: 64
# Tunnel manager settings # Tunnel manager settings
-4
View File
@@ -66,9 +66,6 @@ type Firewall struct {
rules string rules string
rulesVersion uint16 rulesVersion uint16
// source is the rendered firewall config this firewall was built from, the cert networks are kept above
source string
defaultLocalCIDRAny bool defaultLocalCIDRAny bool
incomingMetrics firewallMetrics incomingMetrics firewallMetrics
outgoingMetrics firewallMetrics outgoingMetrics firewallMetrics
@@ -249,7 +246,6 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
return nil, err return nil, err
} }
fw.source = c.Render("firewall")
return fw, nil return fw, nil
} }
+5 -5
View File
@@ -1,12 +1,12 @@
module github.com/slackhq/nebula module github.com/slackhq/nebula
go 1.27.2 go 1.26.0
require ( require (
dario.cat/mergo v1.0.2 dario.cat/mergo v1.0.2
filippo.io/bigmod v0.1.0 filippo.io/bigmod v0.1.0
github.com/DefinedNet/netlink v1.3.2-dn.1
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
github.com/armon/go-radix v1.0.0
github.com/flynn/noise v1.1.0 github.com/flynn/noise v1.1.0
github.com/gaissmai/bart v0.29.0 github.com/gaissmai/bart v0.29.0
github.com/gogo/protobuf v1.3.2 github.com/gogo/protobuf v1.3.2
@@ -20,6 +20,7 @@ require (
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
github.com/stretchr/testify v1.12.1 github.com/stretchr/testify v1.12.1
github.com/vishvananda/netlink v1.3.1
go.uber.org/goleak v1.3.0 go.uber.org/goleak v1.3.0
go.yaml.in/yaml/v3 v3.0.5 go.yaml.in/yaml/v3 v3.0.5
golang.org/x/crypto v0.57.0 golang.org/x/crypto v0.57.0
@@ -32,7 +33,7 @@ require (
golang.zx2c4.com/wireguard/windows v1.0.1 golang.zx2c4.com/wireguard/windows v1.0.1
google.golang.org/protobuf v1.36.12 google.golang.org/protobuf v1.36.12
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
) )
require ( require (
@@ -44,6 +45,5 @@ require (
github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect
github.com/vishvananda/netns v0.0.5 // indirect github.com/vishvananda/netns v0.0.5 // indirect
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect golang.org/x/time v0.5.0 // indirect
golang.org/x/time v0.15.0 // indirect
) )
+10 -8
View File
@@ -3,8 +3,6 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8= filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8=
filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI= filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI=
github.com/DefinedNet/netlink v1.3.2-dn.1 h1:Md2PsUE1Xuy1Zxs/Bm6356n1kt5bq2HspXlPJ6pEspg=
github.com/DefinedNet/netlink v1.3.2-dn.1/go.mod h1:o4YSn8Jw7zoDcYXDmV77TRd4eS5sSLuE+vu2zAN67C8=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -12,6 +10,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -138,6 +138,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -155,8 +157,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -195,6 +195,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -203,8 +205,8 @@ golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -243,5 +245,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 h1:ribyrTYQ/QYeUIs8x8bVLYwGyepa7RnHLqJzegT5YYk= gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g=
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980/go.mod h1:pK/dapF5Bf3KGW7/kd5CI5IC4lYh9FMFY1qETAIGLXk= gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
+5 -18
View File
@@ -31,7 +31,6 @@ package nebula
import ( import (
"bufio" "bufio"
"context"
"fmt" "fmt"
"net" "net"
"strconv" "strconv"
@@ -52,29 +51,17 @@ type graphiteConfigExport struct {
Percentiles []float64 // Percentiles to export from timers and histograms Percentiles []float64 // Percentiles to export from timers and histograms
} }
// graphiteTimeout bounds one submission, the connect and the write together. An export is a few KB, a host that can't
// take that in this long isn't going to.
const graphiteTimeout = 30 * time.Second
// graphiteOnce performs a single submission to Graphite, returning a non-nil // graphiteOnce performs a single submission to Graphite, returning a non-nil
// error if it could not connect or write all of it within timeout. It gives up // error on failed connections.
// as soon as ctx is done. func graphiteOnce(c graphiteConfigExport) error {
func graphiteOnce(ctx context.Context, c graphiteConfigExport, timeout time.Duration) error {
now := time.Now().Unix() now := time.Now().Unix()
du := float64(c.DurationUnit) du := float64(c.DurationUnit)
flushSeconds := float64(c.FlushInterval) / float64(time.Second) flushSeconds := float64(c.FlushInterval) / float64(time.Second)
deadline := time.Now().Add(timeout) conn, err := net.DialTCP("tcp", nil, c.Addr)
d := net.Dialer{Deadline: deadline}
conn, err := d.DialContext(ctx, "tcp", c.Addr.String())
if err != nil { if err != nil {
return err return err
} }
defer conn.Close() defer conn.Close()
if err := conn.SetDeadline(deadline); err != nil {
return err
}
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
defer stop()
w := bufio.NewWriter(conn) w := bufio.NewWriter(conn)
c.Registry.Each(func(name string, i any) { c.Registry.Each(func(name string, i any) {
switch metric := i.(type) { switch metric := i.(type) {
@@ -124,7 +111,7 @@ func graphiteOnce(ctx context.Context, c graphiteConfigExport, timeout time.Dura
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now) fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now)
fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now) fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now)
} }
w.Flush()
}) })
// A failed write sticks, so this reports the first one return nil
return w.Flush()
} }
-315
View File
@@ -1,315 +0,0 @@
package nebula
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"strings"
"sync"
"testing"
"time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// graphiteHost accepts connections and hands each one to the test
type graphiteHost struct {
ln net.Listener
conns chan net.Conn
mu sync.Mutex
closed bool
open []net.Conn
}
// newGraphiteHost accepts connections for the test. A non-zero window shrinks each connection's receive buffer, so a
// sender that is not read from blocks after a little whatever the host's TCP tuning. Only for tests that need a stall,
// a small window makes reading very slow on Linux.
func newGraphiteHost(t *testing.T, window int) *graphiteHost {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
h := &graphiteHost{ln: ln, conns: make(chan net.Conn, 8)}
done := make(chan struct{})
t.Cleanup(func() {
_ = ln.Close()
h.mu.Lock()
h.closed = true
for _, c := range h.open {
_ = c.Close()
}
h.mu.Unlock()
close(done)
})
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
h.mu.Lock()
if h.closed {
h.mu.Unlock()
_ = c.Close()
return
}
h.open = append(h.open, c)
h.mu.Unlock()
if tc, ok := c.(*net.TCPConn); ok && window > 0 {
_ = tc.SetReadBuffer(window)
}
select {
case h.conns <- c:
case <-done:
return
}
}
}()
return h
}
func (h *graphiteHost) addr(t *testing.T) *net.TCPAddr {
a, err := net.ResolveTCPAddr("tcp", h.ln.Addr().String())
require.NoError(t, err)
return a
}
// readExport reads what one send wrote, bounded so a send that never finishes fails the test rather than hanging it
func readExport(t *testing.T, c net.Conn) []byte {
t.Helper()
require.NoError(t, c.SetReadDeadline(time.Now().Add(10*time.Second)))
b, err := io.ReadAll(c)
require.NoError(t, err)
return b
}
func (h *graphiteHost) next(t *testing.T, wait time.Duration) net.Conn {
t.Helper()
select {
case c := <-h.conns:
return c
case <-time.After(wait):
t.Fatal("no connection")
return nil
}
}
// lockedBuffer is a log destination a test can read while the code under test writes to it from another goroutine
type lockedBuffer struct {
mu sync.Mutex
buf bytes.Buffer
}
func (b *lockedBuffer) Write(p []byte) (int, error) {
b.mu.Lock()
defer b.mu.Unlock()
return b.buf.Write(p)
}
func (b *lockedBuffer) String() string {
b.mu.Lock()
defer b.mu.Unlock()
return b.buf.String()
}
func testGraphiteConfig(addr *net.TCPAddr, r metrics.Registry) graphiteConfigExport {
return graphiteConfigExport{Addr: addr, Registry: r, FlushInterval: time.Second, DurationUnit: time.Nanosecond, Prefix: "nebula"}
}
// Enough metrics that one export overflows the socket buffers on any platform, so a send to a host that stops
// reading blocks
func bigGraphiteRegistry(r metrics.Registry, t *testing.T) {
for i := range 200000 {
name := fmt.Sprintf("big.gauge.%d", i)
metrics.GetOrRegisterGauge(name, r).Update(int64(i))
t.Cleanup(func() { r.Unregister(name) })
}
}
func TestGraphiteOnce_sends(t *testing.T) {
r := metrics.NewRegistry()
metrics.GetOrRegisterGauge("g", r).Update(7)
metrics.GetOrRegisterCounter("c", r).Inc(3)
h := newGraphiteHost(t, 0)
require.NoError(t, graphiteOnce(t.Context(), testGraphiteConfig(h.addr(t), r), time.Minute))
b := readExport(t, h.next(t, 5*time.Second))
assert.Contains(t, string(b), "nebula.g.value 7 ")
assert.Contains(t, string(b), "nebula.c.count 3 ")
assert.Contains(t, string(b), "nebula.c.count_ps 3.00 ")
}
// A host that accepts and never reads is given up on, and the failed write is reported
func TestGraphiteOnce_stalledHostTimesOut(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 4096)
done := make(chan error, 1)
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(t.Context(), cfg, 300*time.Millisecond) }()
h.next(t, 5*time.Second) // accepted, never read
select {
case err := <-done:
require.Error(t, err)
case <-time.After(10 * time.Second):
t.Fatal("the send did not give up on a host that stopped reading")
}
}
// The timeout covers the whole send, a host that keeps reading but too slowly is still given up on
func TestGraphiteOnce_timeoutCoversTheWholeSend(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 0)
done := make(chan error, 1)
start := time.Now()
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(t.Context(), cfg, 500*time.Millisecond) }()
c := h.next(t, 5*time.Second)
stopReading := make(chan struct{})
defer close(stopReading)
// 32KB every 10ms keeps every write well inside the timeout, the whole export takes seconds
go func() {
buf := make([]byte, 32<<10)
for {
select {
case <-stopReading:
return
case <-time.After(10 * time.Millisecond):
}
if _, err := c.Read(buf); err != nil {
return
}
}
}()
select {
case err := <-done:
require.Error(t, err)
assert.Less(t, time.Since(start), 5*time.Second)
case <-time.After(10 * time.Second):
t.Fatal("a host that kept reading slowly held the send past its timeout")
}
}
func TestGraphiteOnce_cancelledCtxDoesNotConnect(t *testing.T) {
h := newGraphiteHost(t, 0)
ctx, cancel := context.WithCancel(t.Context())
cancel()
require.Error(t, graphiteOnce(ctx, testGraphiteConfig(h.addr(t), metrics.NewRegistry()), time.Minute))
select {
case <-h.conns:
t.Fatal("connected on a cancelled ctx")
case <-time.After(200 * time.Millisecond):
}
}
// A stop or reload abandons a stuck send rather than waiting out its timeout
func TestGraphiteOnce_cancelAbandonsSend(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 4096)
ctx, cancel := context.WithCancel(t.Context())
done := make(chan error, 1)
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(ctx, cfg, time.Minute) }()
h.next(t, 5*time.Second)
select {
case err := <-done:
t.Fatalf("the send should be stuck on a host that is not reading, it ended with %v", err)
case <-time.After(200 * time.Millisecond):
}
cancel()
select {
case err := <-done:
require.Error(t, err)
case <-time.After(5 * time.Second):
t.Fatal("the send is still stuck after its ctx was cancelled")
}
}
// buildRuntime hands graphite the configured prefix and interval
func TestStatsServer_graphiteExportUsesConfig(t *testing.T) {
const name = "graphite.wiring.test"
metrics.GetOrRegisterCounter(name, nil).Inc(8)
t.Cleanup(func() { metrics.DefaultRegistry.Unregister(name) })
h := newGraphiteHost(t, 0)
s, c := newTestStatsServer(t)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "4s", "prefix": "pfx", "host": h.ln.Addr().String()})
cfg, err := loadStatsConfig(c)
require.NoError(t, err)
s.runMu.Lock()
fns, _ := s.buildRuntime(t.Context(), cfg)
s.runMu.Unlock()
for _, fn := range fns {
fn()
}
b := readExport(t, h.next(t, 5*time.Second))
assert.Contains(t, string(b), "pfx."+name+".count 8 ")
assert.Contains(t, string(b), "pfx."+name+".count_ps 2.00 ")
}
func TestStatsServer_graphiteFailureIsLogged(t *testing.T) {
logs := &lockedBuffer{}
s, c := newTestStatsServer(t)
s.l = test.NewLoggerWithOutput(logs)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)})
cfg, err := loadStatsConfig(c)
require.NoError(t, err)
s.runMu.Lock()
fns, _ := s.buildRuntime(t.Context(), cfg)
s.runMu.Unlock()
for _, fn := range fns {
fn()
}
assert.Contains(t, logs.String(), "Graphite export failed")
}
// Stop abandons the running export, so it goes through Start rather than handing buildRuntime a ctx, and a stop is
// not logged as a failure
func TestStatsServer_stopAbandonsGraphiteSend(t *testing.T) {
bigGraphiteRegistry(metrics.DefaultRegistry, t)
h := newGraphiteHost(t, 4096)
logs := &lockedBuffer{}
s, c := newTestStatsServer(t)
s.l = test.NewLoggerWithOutput(logs)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "50ms", "host": h.ln.Addr().String()})
require.NoError(t, s.reload(c, true))
started := make(chan struct{})
go func() {
s.Start()
close(started)
}()
stuck := h.next(t, 30*time.Second)
s.Stop()
select {
case <-started:
case <-time.After(5 * time.Second):
t.Fatal("Start did not return after Stop")
}
// A closed sender resets the connection once it sees this, one still sending just buffers it. Either way the
// read below doesn't have to drain the export through the small window to find out
_, err := stuck.Write([]byte("x"))
require.NoError(t, err)
require.NoError(t, stuck.SetReadDeadline(time.Now().Add(10*time.Second)))
n, err := io.Copy(io.Discard, stuck)
var ne net.Error
if errors.As(err, &ne) && ne.Timeout() {
t.Fatalf("the send kept the connection after Stop, read %d bytes", n)
}
assert.Less(t, n, int64(1<<20), "the send kept going after Stop")
// The capture loop logs on its own goroutine after Start has returned, give it the chance
assert.Never(t, func() bool { return strings.Contains(logs.String(), "Graphite export failed") }, 500*time.Millisecond, 10*time.Millisecond, "a stop is not a failure")
}
+12 -21
View File
@@ -54,8 +54,7 @@ type HandshakeConfig struct {
} }
type HandshakeManager struct { type HandshakeManager struct {
// Mutex for interacting with the vpnIps and indexes maps. // Mutex for interacting with the vpnIps and indexes maps
// Lock order is hostmap, then this lock, then the lighthouse, then a RemoteList.
sync.RWMutex sync.RWMutex
vpnIps map[netip.Addr]*HandshakeHostInfo vpnIps map[netip.Addr]*HandshakeHostInfo
@@ -69,7 +68,6 @@ type HandshakeManager struct {
messageMetrics *MessageMetrics messageMetrics *MessageMetrics
metricInitiated metrics.Counter metricInitiated metrics.Counter
metricTimedOut metrics.Counter metricTimedOut metrics.Counter
metricRejectedBlocklisted metrics.Counter
f *Interface f *Interface
l *slog.Logger l *slog.Logger
@@ -130,7 +128,6 @@ func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *Light
messageMetrics: config.messageMetrics, messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil), metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil), metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
metricRejectedBlocklisted: metrics.GetOrRegisterCounter("handshake_manager.rejected.blocklisted", nil),
l: l, l: l,
} }
} }
@@ -262,6 +259,14 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
"style": header.SubTypeName(header.Handshake, hh.machine.Subtype()), "style": header.SubTypeName(header.Handshake, hh.machine.Subtype()),
} }
// Get a remotes object if we don't already have one.
// This is mainly to protect us as this should never be the case
// NB ^ This comment doesn't jive. It's how the thing gets initialized.
// It's the common path. Should it update every time, in case a future LH query/queries give us more info?
if hostinfo.remotes == nil {
hostinfo.remotes = hm.lightHouse.QueryCache([]netip.Addr{vpnIp})
}
remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges()) remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges())
remotesHaveChanged := !slices.Equal(remotes, hh.lastRemotes) remotesHaveChanged := !slices.Equal(remotes, hh.lastRemotes)
@@ -366,9 +371,6 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
hostinfo := &HostInfo{ hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr}, vpnAddrs: []netip.Addr{vpnAddr},
HandshakePacket: make(map[uint8][]byte, 0), HandshakePacket: make(map[uint8][]byte, 0),
// Attached here so a pending hostinfo never exists without one: SetRemote dereferences
// it, and create-tunnel -address can reach SetRemote before the handshake loop runs.
remotes: hm.lightHouse.QueryCache([]netip.Addr{vpnAddr}),
relayState: RelayState{ relayState: RelayState{
relays: nil, relays: nil,
relayForByAddr: map[netip.Addr]*Relay{}, relayForByAddr: map[netip.Addr]*Relay{},
@@ -405,8 +407,6 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
hm.Unlock() hm.Unlock()
hm.lightHouse.QueryServer(vpnAddr) hm.lightHouse.QueryServer(vpnAddr)
// If this host came from a static_map hostname its addresses may have moved, retries will pick up anything new
hostinfo.remotes.ResolveHostnames()
return hostinfo return hostinfo
} }
@@ -721,12 +721,7 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
response, result, err := machine.ProcessPacket(nil, packet) response, result, err := machine.ProcessPacket(nil, packet)
if err != nil { if err != nil {
level := slog.LevelError f.l.Error("Failed to process handshake packet", "from", via, "error", err)
if errors.Is(err, cert.ErrBlockListed) {
level = slog.LevelDebug
hm.metricRejectedBlocklisted.Inc(1)
}
f.l.Log(context.Background(), level, "Failed to process handshake packet", "from", via, "error", err)
return return
} }
@@ -858,9 +853,6 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
response, result, err := machine.ProcessPacket(nil, packet) response, result, err := machine.ProcessPacket(nil, packet)
if err != nil { if err != nil {
if errors.Is(err, cert.ErrBlockListed) {
hm.metricRejectedBlocklisted.Inc(1)
}
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn. // Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
if machine.Failed() { if machine.Failed() {
f.l.Warn("Failed to process handshake packet, abandoning", f.l.Warn("Failed to process handshake packet, abandoning",
@@ -954,14 +946,13 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
hm.DeleteHostInfo(hostinfo) hm.DeleteHostInfo(hostinfo)
hm.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) { hm.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) {
newHH.hostinfo.remotes = hostinfo.remotes
newHH.hostinfo.remotes.BlockRemote(via) newHH.hostinfo.remotes.BlockRemote(via)
newHH.packetStore = hh.packetStore newHH.packetStore = hh.packetStore
hh.packetStore = []*cachedPacket{} hh.packetStore = []*cachedPacket{}
})
// sendCloseTunnel may take the hostmap lock, so it must not run under the handshake
// manager lock that StartHandshake holds around the callback.
hostinfo.vpnAddrs = vpnAddrs hostinfo.vpnAddrs = vpnAddrs
f.sendCloseTunnel(hostinfo) f.sendCloseTunnel(hostinfo)
})
return return
} }
+2 -156
View File
@@ -1,22 +1,16 @@
package nebula package nebula
import ( import (
"log/slog"
"net/netip" "net/netip"
"testing" "testing"
"time" "time"
"github.com/flynn/noise"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func Test_NewHandshakeManagerVpnIp(t *testing.T) { func Test_NewHandshakeManagerVpnIp(t *testing.T) {
@@ -48,6 +42,8 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
i2 := blah.StartHandshake(ip, nil) i2 := blah.StartHandshake(ip, nil)
assert.Same(t, i, i2) assert.Same(t, i, i2)
i.remotes = NewRemoteList([]netip.Addr{}, nil)
// Adding something to pending should not affect the main hostmap // Adding something to pending should not affect the main hostmap
assert.Empty(t, mainHM.Hosts) assert.Empty(t, mainHM.Hosts)
@@ -239,153 +235,3 @@ func TestHandleIncomingDispatch(t *testing.T) {
assert.Empty(t, hm.indexes, "orphan stage-2 must not create state") assert.Empty(t, hm.indexes, "orphan stage-2 must not create state")
}) })
} }
func TestBlocklistedHandshakeLogLevelAndCounter(t *testing.T) {
ca, _, caKey, _ := ct.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
pool := ct.NewTestCAPool(ca)
newCredential := func(name, network string) (cert.Certificate, *handshake.Credential) {
c, _, rawKey, _ := ct.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey,
name, ca.NotBefore(), ca.NotAfter(), []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
privateKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey)
require.NoError(t, err)
certBytes, err := c.MarshalForHandshakes()
require.NoError(t, err)
cipherSuite := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
return c, handshake.NewCredential(c, certBytes, privateKey, cipherSuite)
}
initiatorCert, initiatorCredential := newCredential("initiator", "10.0.0.1/24")
responderCert, responderCredential := newCredential("responder", "10.0.0.2/24")
fingerprint, err := initiatorCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(fingerprint)
initiator, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return initiatorCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return pool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 1, nil }, true, header.HandshakeIXPSK0)
require.NoError(t, err)
packet, err := initiator.Initiate(nil)
require.NoError(t, err)
handler := &capturingHandler{}
logger := slog.New(handler)
pki := &PKI{}
pki.cs.Store(&CertState{initiatingVersion: cert.Version2, v2Credential: responderCredential})
pki.caPool.Store(pool)
hm := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
hm.metricRejectedBlocklisted = metrics.NewCounter()
hm.f = &Interface{pki: pki, l: logger}
via := ViaSender{UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242")}
hm.beginHandshake(via, packet, nil)
entry := findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelDebug, entry.Level)
require.ErrorIs(t, entry.Attrs["error"].(error), cert.ErrBlockListed)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
handler.entries = nil
hm.beginHandshake(via, nil, nil)
entry = findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelError, entry.Level)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
// The initiator sees the same rejection in continueHandshake, after a
// responder has accepted its first packet and sent a valid reply.
cleanPool := ct.NewTestCAPool(ca)
responder, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return responderCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return cleanPool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 2, nil }, false, header.HandshakeIXPSK0)
require.NoError(t, err)
response, _, err := responder.ProcessPacket(nil, packet)
require.NoError(t, err)
responderFingerprint, err := responderCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(responderFingerprint)
initiatingHM := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
initiatingHM.metricRejectedBlocklisted = metrics.NewCounter()
initiatingHM.f = &Interface{l: logger}
hh := &HandshakeHostInfo{
machine: initiator,
hostinfo: &HostInfo{localIndexId: 1, vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}},
}
initiatingHM.indexes[1] = hh
via.IsRelayed = true // skip the remote allow list before certificate validation
initiatingHM.continueHandshake(via, hh, response)
assert.Equal(t, int64(1), initiatingHM.metricRejectedBlocklisted.Count())
}
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
func Test_PendingHandshakeHasRemotes(t *testing.T) {
l := test.NewLogger()
mainHM := newHostMap(l)
preferredRanges := []netip.Prefix{}
mainHM.preferredRanges.Store(&preferredRanges)
hm := NewHandshakeManager(l, mainHM, newTestLighthouse(), &udp.NoopConn{}, defaultHandshakeConfig)
vpnIp := netip.MustParseAddr("172.1.1.2")
remote := netip.MustParseAddrPort("192.168.1.9:4242")
hi := hm.StartHandshake(vpnIp, func(hh *HandshakeHostInfo) {
require.NotNil(t, hh.hostinfo.remotes, "a pending handshake must never be without its RemoteList")
hh.hostinfo.SetRemote(remote)
})
assert.Contains(t, hi.remotes.CopyAddrs(preferredRanges), remote, "the handshake must be sent to the supplied remote")
assert.Equal(t, remote, hi.GetRemote())
// A RemoteList keeps one learned address per host, so a second remote replaces the
// first — the same behaviour change-remote has on an established tunnel.
other := netip.MustParseAddrPort("192.168.1.10:4242")
hi.SetRemote(other)
assert.Equal(t, []netip.AddrPort{other}, hi.remotes.CopyAddrs(preferredRanges))
assert.Equal(t, other, hi.GetRemote())
}
// newRemotesRaceFixture builds a handshake manager with an established tunnel to vpnAddr whose
// RemoteList is the lighthouse's cached list, the state every teardown-during-handshake race starts from.
func newRemotesRaceFixture(t *testing.T, vpnAddr netip.Addr) (*HandshakeManager, *Interface, *HostInfo) {
t.Helper()
l := test.NewLogger()
mainHM := newHostMap(l)
preferredRanges := []netip.Prefix{}
mainHM.preferredRanges.Store(&preferredRanges)
lh := newTestLighthouse()
hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig)
f := &Interface{hostMap: mainHM, lightHouse: lh, handshakeManager: hm, l: l}
hm.f = f
established := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
localIndexId: 1,
remoteIndexId: 101,
remotes: lh.QueryCache([]netip.Addr{vpnAddr}),
HandshakePacket: map[uint8][]byte{},
}
mainHM.Lock()
mainHM.unlockedAddHostInfo(established, f)
mainHM.Unlock()
return hm, f, established
}
// With no handshake pending, tearing down the last tunnel still evicts the list.
func Test_CloseTunnelEvictsWithoutPendingHandshake(t *testing.T) {
vpnAddr := netip.MustParseAddr("172.1.1.2")
_, f, established := newRemotesRaceFixture(t, vpnAddr)
f.closeTunnel(established)
assert.NotContains(t, f.hostMap.Hosts, vpnAddr)
assert.NotContains(t, f.lightHouse.addrMap, vpnAddr)
}
-5
View File
@@ -24,11 +24,6 @@ type m = map[string]any
const ( const (
Version uint8 = 1 Version uint8 = 1
Len = 16 Len = 16
// MaxOverhead is the most a payload grows on its way to the underlay.
// A relayed packet has two headers and two AEAD tags, one for the inner tunnel and another for the relay tunnel.
// Every supported cipher (today) has a 16 byte tag.
MaxOverhead = 2 * (Len + 16)
) )
type MessageType uint8 type MessageType uint8
+63 -7
View File
@@ -6,6 +6,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net"
"net/netip" "net/netip"
"slices" "slices"
"sync" "sync"
@@ -17,6 +18,7 @@ import (
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging"
) )
const defaultPromoteEvery = 1000 // Count of packets sent before we try moving a tunnel to a preferred underlay ip address const defaultPromoteEvery = 1000 // Count of packets sent before we try moving a tunnel to a preferred underlay ip address
@@ -316,9 +318,9 @@ type cachedPacketMetrics struct {
func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap { func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap {
hm := newHostMap(l) hm := newHostMap(l)
hm.reload(c, true, nil) hm.reload(c, true)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
hm.reload(c, false, r) hm.reload(c, false)
}) })
l.Info("Main HostMap created", "preferredRanges", hm.GetPreferredRanges()) l.Info("Main HostMap created", "preferredRanges", hm.GetPreferredRanges())
@@ -337,8 +339,7 @@ func newHostMap(l *slog.Logger) *HostMap {
} }
} }
// r is nil on the initial load func (hm *HostMap) reload(c *config.C, initial bool) {
func (hm *HostMap) reload(c *config.C, initial bool, r *config.Reload) {
if initial || c.HasChanged("preferred_ranges") { if initial || c.HasChanged("preferred_ranges") {
var preferredRanges []netip.Prefix var preferredRanges []netip.Prefix
rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{}) rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{})
@@ -349,9 +350,8 @@ func (hm *HostMap) reload(c *config.C, initial bool, r *config.Reload) {
if err != nil { if err != nil {
hm.l.Warn("Failed to parse preferred ranges, ignoring", hm.l.Warn("Failed to parse preferred ranges, ignoring",
"error", err, "error", err,
"range", rawPreferredRange, "range", rawPreferredRanges,
) )
r.Failed()
continue continue
} }
@@ -930,3 +930,59 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
return li return li
} }
// Utility functions
func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
//FIXME: This function is pretty garbage
var finalAddrs []netip.Addr
ifaces, _ := net.Interfaces()
for _, i := range ifaces {
allow := allowList.AllowName(i.Name)
if l.Enabled(context.Background(), logging.LevelTrace) {
l.Log(context.Background(), logging.LevelTrace, "localAllowList.AllowName",
"interfaceName", i.Name,
"allow", allow,
)
}
if !allow {
continue
}
addrs, _ := i.Addrs()
for _, rawAddr := range addrs {
var addr netip.Addr
switch v := rawAddr.(type) {
case *net.IPNet:
//continue
addr, _ = netip.AddrFromSlice(v.IP)
case *net.IPAddr:
addr, _ = netip.AddrFromSlice(v.IP)
}
if !addr.IsValid() {
if l.Enabled(context.Background(), slog.LevelDebug) {
l.Debug("addr was invalid", "localAddr", rawAddr)
}
continue
}
addr = addr.Unmap()
if addr.IsLoopback() == false && addr.IsLinkLocalUnicast() == false {
isAllowed := allowList.Allow(addr)
if l.Enabled(context.Background(), logging.LevelTrace) {
l.Log(context.Background(), logging.LevelTrace, "localAllowList.Allow",
"localAddr", addr,
"allowed", isAllowed,
)
}
if !isAllowed {
continue
}
finalAddrs = append(finalAddrs, addr)
}
}
}
return finalAddrs
}
-13
View File
@@ -447,16 +447,3 @@ func TestHostInfo_markOutOnly(t *testing.T) {
assert.True(t, h.markOut(6), "a relayed send must not consume the epoch edge") assert.True(t, h.markOut(6), "a relayed send must not consume the epoch edge")
assert.False(t, h.markOut(6)) assert.False(t, h.markOut(6))
} }
// A preferred range that doesn't parse is skipped and the rest are used, the reload says one was skipped
func TestHostMap_reloadBadPreferredRangeFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("preferred_ranges: [10.0.0.0/8]\n"))
hm := NewHostMapFromConfig(l, c)
res := c.ReloadConfigString("preferred_ranges: [10.0.0.0/8, nope]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, hm.GetPreferredRanges())
}
+12 -29
View File
@@ -10,7 +10,6 @@ import (
"github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/noiseutil" "github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
@@ -199,8 +198,8 @@ func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt tio.Packet, nb []b
} }
err = tio.SegmentSuperpacket(pkt, func(seg []byte) error { err = tio.SegmentSuperpacket(pkt, func(seg []byte) error {
//relay header + header + plaintext + AEAD tag + relay tag //relay header + header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305) + relay tag
scratch := sendBatch.Reserve(len(seg) + header.MaxOverhead) scratch := sendBatch.Reserve(header.Len + header.Len + len(seg) + 16 + 16)
innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb) innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb)
if innerPacket == nil { if innerPacket == nil {
@@ -518,9 +517,7 @@ func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, noc
err = f.writers[q].WriteTo(toSend, via.GetRemote()) err = f.writers[q].WriteTo(toSend, via.GetRemote())
if err != nil { if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpRelayTxErrLimiter); ok { via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err)
via.logger(l).Info("Failed to WriteTo in sendVia", "error", err)
}
} }
} }
@@ -529,26 +526,16 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
return return
} }
useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid() useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid()
// The encrypt appends to out in place and a relayed packet is then wrapped in place,
// so out has to fit the whole packet up front or the encrypt reallocates out from under SendVia.
// We cap the MTU of tun devices and routes we create ourselves, so only a host that set a bigger MTU than we allow gets here.
need := header.Len + len(p) + ci.eKey.Overhead()
if useRelay {
// The relay tunnel runs the same cipher, and adds its own header and tag
need += header.Len + ci.eKey.Overhead()
}
if cap(out) < need {
if l, ok := logging.RateLimited(f.l, f.txOversizeLimiter); ok {
hostinfo.logger(l).Error("Dropping outbound packet, out buffer not large enough", "outCap", cap(out), "payloadLen", len(p), "relayed", useRelay)
}
return
}
fullOut := out fullOut := out
if useRelay { if useRelay {
if len(out) < header.Len {
// out always has a capacity of mtu, but not always a length greater than the header.Len.
// Grow it to make sure the next operation works.
out = out[:header.Len]
}
// Save a header's worth of data at the front of the 'out' buffer. // Save a header's worth of data at the front of the 'out' buffer.
out = out[header.Len:header.Len] out = out[header.Len:]
} }
if noiseutil.EncryptLockNeeded { if noiseutil.EncryptLockNeeded {
@@ -595,23 +582,19 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
if remote.IsValid() { if remote.IsValid() {
err = f.writers[q].WriteTo(out, remote) err = f.writers[q].WriteTo(out, remote)
if err != nil { if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok { hostinfo.logger(f.l).Error("Failed to write outgoing packet",
hostinfo.logger(l).Error("Failed to write outgoing packet",
"error", err, "error", err,
"udpAddr", remote, "udpAddr", remote,
) )
} }
}
} else if hr := hostinfo.GetRemote(); hr.IsValid() { } else if hr := hostinfo.GetRemote(); hr.IsValid() {
err = f.writers[q].WriteTo(out, hr) err = f.writers[q].WriteTo(out, hr)
if err != nil { if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok { hostinfo.logger(f.l).Error("Failed to write outgoing packet",
hostinfo.logger(l).Error("Failed to write outgoing packet",
"error", err, "error", err,
"udpAddr", hr, "udpAddr", hr,
) )
} }
}
} else { } else {
// Try to send via a relay // Try to send via a relay
for _, relayIP := range hostinfo.relayState.CopyRelayIps() { for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
+37 -95
View File
@@ -24,7 +24,6 @@ import (
"github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/overlay"
"github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/ratelimit"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
) )
@@ -124,12 +123,6 @@ type Interface struct {
// reused (every udp.Conn ListenOut guarantees that ordering). // reused (every udp.Conn ListenOut guarantees that ordering).
batchers []*batch.MultiCoalescer batchers []*batch.MultiCoalescer
wg sync.WaitGroup wg sync.WaitGroup
// readers covers just the listenIn and listenOut goroutines, wg also holds the construction token Close releases
readers sync.WaitGroup
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
txOversizeLimiter *ratelimit.Limiter // oversized outbound packet drop log limiter
// fatalErr holds the first unexpected reader error that caused shutdown. // fatalErr holds the first unexpected reader error that caused shutdown.
// nil means "no fatal error" (yet) // nil means "no fatal error" (yet)
@@ -205,19 +198,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
return nil, errors.New("no connection manager") return nil, errors.New("no connection manager")
} }
udpTxErrLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
udpRelayTxErrLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
txOversizeLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
if c.routines <= 1 { if c.routines <= 1 {
c.PinThreads = false //pinning is not useful unless there's more than one tun reader c.PinThreads = false //pinning is not useful unless there's more than one tun reader
} }
@@ -250,9 +230,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
conntrackCacheTimeout: c.ConntrackCacheTimeout, conntrackCacheTimeout: c.ConntrackCacheTimeout,
cpuAffinity: c.CpuAffinity, cpuAffinity: c.CpuAffinity,
pinThreads: c.PinThreads, pinThreads: c.PinThreads,
udpTxErrLimiter: udpTxErrLimiter,
udpRelayTxErrLimiter: udpRelayTxErrLimiter,
txOversizeLimiter: txOversizeLimiter,
metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)), metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
metricTxDropped: metrics.GetOrRegisterCounter("udp.tx.dropped", nil), metricTxDropped: metrics.GetOrRegisterCounter("udp.tx.dropped", nil),
@@ -343,18 +320,14 @@ func (f *Interface) activate() error {
func (f *Interface) run() { func (f *Interface) run() {
// Launch n queues to read packets from udp // Launch n queues to read packets from udp
for i := 0; i < f.routines; i++ { for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() { f.wg.Go(func() {
defer f.readers.Done()
f.listenOut(i) f.listenOut(i)
}) })
} }
// Launch n queues to read packets from tun dev // Launch n queues to read packets from tun dev
for i := 0; i < f.routines; i++ { for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() { f.wg.Go(func() {
defer f.readers.Done()
f.listenIn(f.queues[i], i) f.listenIn(f.queues[i], i)
}) })
} }
@@ -515,11 +488,10 @@ func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) { func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
c.RegisterReloadCallback(f.reloadFirewall) c.RegisterReloadCallback(f.reloadFirewall)
for _, reload := range []func(*config.C){ c.RegisterReloadCallback(f.reloadSendRecvError)
f.reloadSendRecvError, f.reloadAcceptRecvError, f.reloadDisconnectInvalid, f.reloadMisc, c.RegisterReloadCallback(f.reloadAcceptRecvError)
} { c.RegisterReloadCallback(f.reloadDisconnectInvalid)
c.RegisterReloadCallback(func(c *config.C, _ *config.Reload) { reload(c) }) c.RegisterReloadCallback(f.reloadMisc)
}
for _, udpConn := range f.writers { for _, udpConn := range f.writers {
c.RegisterReloadCallback(udpConn.ReloadConfig) c.RegisterReloadCallback(udpConn.ReloadConfig)
@@ -536,19 +508,18 @@ func (f *Interface) reloadDisconnectInvalid(c *config.C) {
} }
} }
func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) { func (f *Interface) reloadFirewall(c *config.C) {
cs := f.pki.getCertState() cs := f.pki.getCertState()
curCert := cs.getCertificate(cert.Version2) curCert := cs.getCertificate(cert.Version2)
if curCert == nil { if curCert == nil {
curCert = cs.getCertificate(cert.Version1) curCert = cs.getCertificate(cert.Version1)
} }
// Compare with what the running firewall was built from, not the last config, so a firewall that failed to build // The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction.
// is tried again. The cert is the one this reload brought, PKI registers its reload callback before this one // Check to see if that set has changed, and if so, rebuild the firewall.
src := c.Render("firewall") certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks)
if src != "" && src == f.firewall.source && curCert != nil &&
slices.Equal(curCert.Networks(), f.firewall.assignedNetworks) && if !c.HasChanged("firewall") && !certUnsafeChanged {
slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) {
f.l.Debug("No firewall config change detected") f.l.Debug("No firewall config change detected")
return return
} }
@@ -556,7 +527,6 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
fw, err := NewFirewallFromConfig(f.l, cs, c) fw, err := NewFirewallFromConfig(f.l, cs, c)
if err != nil { if err != nil {
f.l.Error("Error while creating firewall during reload", "error", err) f.l.Error("Error while creating firewall during reload", "error", err)
r.Failed()
return return
} }
@@ -654,15 +624,17 @@ func (f *Interface) reloadMisc(c *config.C) {
} }
} }
// statsEmitter updates the interface's gauges, the stats server runs it on its capture loop. func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
func (f *Interface) statsEmitter() func() { ticker := time.NewTicker(i)
defer ticker.Stop()
udpStats := udp.NewUDPStatsEmitter(f.writers) udpStats := udp.NewUDPStatsEmitter(f.writers)
certExpirationGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil) certExpirationGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil)
certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil) certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil)
certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil) certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil)
return func() { emit := func() {
f.firewall.EmitStats() f.firewall.EmitStats()
f.handshakeManager.EmitStats() f.handshakeManager.EmitStats()
udpStats() udpStats()
@@ -679,6 +651,19 @@ func (f *Interface) statsEmitter() func() {
certMaxVersion.Update(int64(certState.v1Cert.Version())) certMaxVersion.Update(int64(certState.v1Cert.Version()))
} }
} }
// Prime gauges so a Prometheus scrape that lands before the first tick
// sees real values instead of the zero defaults (issue #907).
emit()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
emit()
}
}
} }
func (f *Interface) GetHostInfo(vpnIp netip.Addr) *HostInfo { func (f *Interface) GetHostInfo(vpnIp netip.Addr) *HostInfo {
@@ -689,14 +674,6 @@ func (f *Interface) GetCertState() *CertState {
return f.pki.getCertState() return f.pki.getCertState()
} }
// shutdowner is a udp.Conn or overlay.Device whose readers can be woken without releasing their fds
type shutdowner interface {
Shutdown() error
}
// readerShutdownTimeout bounds how long Close waits on woken readers before releasing their fds anyway
const readerShutdownTimeout = 5 * time.Second
// Close releases the interface's resources: the udp sockets and the tun device. // Close releases the interface's resources: the udp sockets and the tun device.
// It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated, // It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated,
// calls after the first return nil without doing anything. // calls after the first return nil without doing anything.
@@ -706,55 +683,20 @@ func (f *Interface) Close() error {
} }
var errs []error var errs []error
closeUdp := func(i int, u udp.Conn) {
if err := u.Close(); err != nil { // Release the udp readers
for i, u := range f.writers {
err := u.Close()
if err != nil {
f.l.Error("Error while closing udp socket", "error", err, "writer", i) f.l.Error("Error while closing udp socket", "error", err, "writer", i)
errs = append(errs, err) errs = append(errs, err)
} }
} }
closeTun := func() {
if err := f.inside.Close(); err != nil {
errs = append(errs, err)
}
}
// Wake every reader before releasing any fd, one that loops back into a syscall on a reused fd number never wakes. // Release the tun device (closing the tun also closes all readers)
// Anything without Shutdown is closed right away, as before. closeErr := f.inside.Close()
var deferred []func() if closeErr != nil {
for i, u := range f.writers { errs = append(errs, closeErr)
if s, ok := u.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
f.l.Error("Error while shutting down udp socket", "error", err, "writer", i)
errs = append(errs, err)
}
deferred = append(deferred, func() { closeUdp(i, u) })
} else {
closeUdp(i, u)
}
}
if s, ok := f.inside.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
errs = append(errs, err)
}
deferred = append(deferred, closeTun)
} else {
closeTun()
}
if len(deferred) > 0 {
readersDone := make(chan struct{})
go func() {
f.readers.Wait()
close(readersDone)
}()
select {
case <-readersDone:
case <-time.After(readerShutdownTimeout):
f.l.Warn("Readers did not stop, closing their fds anyway", "timeout", readerShutdownTimeout)
}
for _, closeFn := range deferred {
closeFn()
}
} }
// Release the construction token so waiters know the resources are gone // Release the construction token so waiters know the resources are gone
+14 -5
View File
@@ -3,6 +3,7 @@
package nebula package nebula
import ( import (
"context"
"net/netip" "net/netip"
"testing" "testing"
"time" "time"
@@ -17,8 +18,11 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
// Priming before the first tick (issue #907) is Start's job, see TestStatsServer_Start_primes. // Test_emitStats_primesGauges covers issue #907: a Prometheus scrape that
func Test_statsEmitter_setsGauges(t *testing.T) { // landed before the first ticker fire used to read 0 for the cert gauges.
// emitStats now primes the gauges before entering the ticker loop. We assert
// the gauge is zero before the first call and non-zero after.
func Test_emitStats_primesGauges(t *testing.T) {
defer metrics.DefaultRegistry.UnregisterAll() defer metrics.DefaultRegistry.UnregisterAll()
l := test.NewLogger() l := test.NewLogger()
@@ -52,12 +56,17 @@ func Test_statsEmitter_setsGauges(t *testing.T) {
ifce.pki.cs.Store(cs) ifce.pki.cs.Store(cs)
ttlGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil) ttlGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil)
require.Zero(t, ttlGauge.Value(), "gauge should be zero before the emitter runs") require.Zero(t, ttlGauge.Value(), "gauge should be zero before emitStats runs")
ifce.statsEmitter()() // Pre-cancel the context so emitStats returns after priming the gauges
// without ever reading from ticker.C. The one hour interval is just a
// belt-and-suspenders, the test does not expect the ticker to fire.
ctx, cancel := context.WithCancel(context.Background())
cancel()
ifce.emitStats(ctx, time.Hour)
ttl := ttlGauge.Value() ttl := ttlGauge.Value()
assert.Positive(t, ttl, "ttl gauge should be set by one run of the emitter") assert.Positive(t, ttl, "ttl gauge should be primed by emitStats before its first tick")
assert.LessOrEqual(t, ttl, int64(3600)) assert.LessOrEqual(t, ttl, int64(3600))
assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.initiating_version", nil).Value()) assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.initiating_version", nil).Value())
assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.max_version", nil).Value()) assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.max_version", nil).Value())
+4 -81
View File
@@ -65,10 +65,10 @@ func TestReloadFirewall_CertUnsafeNetworksChanged(t *testing.T) {
pki.cs.Store(&CertState{v2Cert: c2, initiatingVersion: cert.Version2}) pki.cs.Store(&CertState{v2Cert: c2, initiatingVersion: cert.Version2})
// Reload with the same YAML so HasChanged("firewall") reports false. // Reload with the same YAML so HasChanged("firewall") reports false.
require.NoError(t, cfg.ReloadConfigString(rawYAML).Err) require.NoError(t, cfg.ReloadConfigString(rawYAML))
require.False(t, cfg.HasChanged("firewall")) require.False(t, cfg.HasChanged("firewall"))
f.reloadFirewall(cfg, nil) f.reloadFirewall(cfg)
assert.NotSame(t, fw, f.firewall, "firewall pointer should have been replaced") assert.NotSame(t, fw, f.firewall, "firewall pointer should have been replaced")
assert.Equal(t, newUnsafe, f.firewall.unsafeNetworks) assert.Equal(t, newUnsafe, f.firewall.unsafeNetworks)
@@ -113,85 +113,8 @@ func TestReloadFirewall_NoChange(t *testing.T) {
l: l, l: l,
} }
require.NoError(t, cfg.ReloadConfigString(rawYAML).Err) require.NoError(t, cfg.ReloadConfigString(rawYAML))
f.reloadFirewall(cfg, nil) f.reloadFirewall(cfg)
assert.Same(t, fw, f.firewall, "firewall should not have been replaced") assert.Same(t, fw, f.firewall, "firewall should not have been replaced")
} }
// Rules that don't parse keep the old firewall, and the reload says it could not apply them
func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
l := test.NewLogger()
pki := &PKI{}
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}}, initiatingVersion: cert.Version2})
rules := func(proto string) string {
return "firewall:\n outbound:\n - port: any\n proto: any\n host: any\n inbound:\n - port: any\n proto: " + proto + "\n host: any\n"
}
cfg := config.NewC(l)
require.NoError(t, cfg.LoadString(rules("any")))
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
require.NoError(t, err)
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
// The running firewall still isn't built from what the config asks for, so the same reload fails again
for range 2 {
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
res := cfg.ReloadConfigString(rules("tcp"))
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.NotSame(t, fw, f.firewall)
}
// Every input the firewall is built from rebuilds it when it changes, and nothing else does
func TestReloadFirewall_RebuildsOnAnyInput(t *testing.T) {
l := test.NewLogger()
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
base := "firewall:\n inbound:\n - port: any\n proto: any\n host: any\n"
setup := func() (*config.C, *Interface, *PKI) {
pki := &PKI{}
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks}, initiatingVersion: cert.Version2})
cfg := config.NewC(l)
require.NoError(t, cfg.LoadString(base))
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
require.NoError(t, err)
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
return cfg, f, pki
}
setCert := func(pki *PKI, networks, unsafe []netip.Prefix) {
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks, unsafeNetworks: unsafe}, initiatingVersion: cert.Version2})
}
cases := map[string]struct {
cfg string
cert func(*PKI)
want bool
}{
"nothing": {base + "punchy:\n punch: true\n", nil, false},
"rules": {base + " outbound:\n - port: any\n proto: any\n host: any\n", nil, true},
"inbound_action": {base + " inbound_action: reject\n", nil, true},
"conntrack": {base + " conntrack:\n tcp_timeout: 1m\n", nil, true},
"networks": {base, func(p *PKI) { setCert(p, []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil) }, true},
"unsafe_networks": {base, func(p *PKI) { setCert(p, networks, []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}) }, true},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
cfg, f, pki := setup()
old := f.firewall
if tc.cert != nil {
tc.cert(pki)
}
res := cfg.ReloadConfigString(tc.cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, tc.want, f.firewall != old)
})
}
}
+13 -3
View File
@@ -45,7 +45,8 @@ func setTransportChecksum4(packet []byte) {
if ihl < ipv4.HeaderLen || end < ihl || end > len(packet) { if ihl < ipv4.HeaderLen || end < ihl || end > len(packet) {
return return
} }
// do not attempt to set checksums for fragments // The checksum covers the whole datagram, which a fragment (MF set or a
// non-zero offset) does not carry.
if binary.BigEndian.Uint16(packet[6:8])&0x3fff != 0 { if binary.BigEndian.Uint16(packet[6:8])&0x3fff != 0 {
return return
} }
@@ -67,7 +68,10 @@ func setTransportChecksum6(packet []byte) {
return return
} }
// we must find the transport header in order to write a checksum // The checksum covers the whole datagram, which a fragment does not carry.
// An unknown extension header hides where the transport header starts. A
// chain longer than the walk's budget ends it early, at an offset that was
// never checked against the packet.
proto, offset, _, anyFragment, err := IPv6FindUpperProtocol(packet[:end]) proto, offset, _, anyFragment, err := IPv6FindUpperProtocol(packet[:end])
if err != nil || anyFragment || offset >= end { if err != nil || anyFragment || offset >= end {
return return
@@ -105,7 +109,8 @@ func transportExtent(transport []byte, proto uint8) ([]byte, bool) {
// writeTransportChecksum stores the checksum of transport, taken over the // writeTransportChecksum stores the checksum of transport, taken over the
// pseudo-header sum csum, in the header's checksum field. A UDP checksum that // pseudo-header sum csum, in the header's checksum field. A UDP checksum that
// computes to zero goes on the wire as 0xffff: zero means no checksum was // computes to zero goes on the wire as 0xffff: zero means no checksum was
// computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200 section 8.1). // computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200
// section 8.1).
func writeTransportChecksum(transport []byte, proto uint8, csum uint32) { func writeTransportChecksum(transport []byte, proto uint8, csum uint32) {
var at, minLen int var at, minLen int
switch proto { switch proto {
@@ -128,6 +133,11 @@ func writeTransportChecksum(transport []byte, proto uint8, csum uint32) {
binary.BigEndian.PutUint16(transport[at:], sum) binary.BigEndian.PutUint16(transport[at:], sum)
} }
// fold reduces a pseudo-header sum to the 16 bit seed Checksum takes. Carrying
// the high half back into the low half is what keeps the reduction lossless, so
// the seed sums exactly as the wider value would; 0xffff is its fixed point.
// Every term of that sum comes from a 16 bit field, so it stays far below the
// width at which the accumulator would wrap.
func fold(csum uint32) uint16 { func fold(csum uint32) uint16 {
for csum > 0xffff { for csum > 0xffff {
csum = (csum >> 16) + (csum & 0xffff) csum = (csum >> 16) + (csum & 0xffff)
+16 -13
View File
@@ -345,11 +345,14 @@ func ipv6CreateRejectTCPPacket(packet []byte, out []byte, offset int) []byte {
return out return out
} }
// IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the offset it begins at, and whether the packet is a non-first fragment. // IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the
// Only the RFC 8200 and IANA extension headers below are walked. // offset it begins at, and whether the packet is a non-first fragment. Only the RFC 8200 and IANA extension
// Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental 253/254, and real upper layer protocols like SCTP or GRE, is terminal. // headers below are walked. Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental
// For a non-first fragment the returned protocol is the fragmented protocol and offset points at the fragment header, there is no transport header to locate. // 253/254, and real upper layer protocols like SCTP or GRE, is terminal. Walking those as extension headers
// Returns ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header, or the chain is truncated, or if more than 8 extension headers are present. // is a firewall bypass, so they fail closed. For a non-first fragment the returned protocol is the fragmented
// protocol and offset points at the fragment header, there is no transport header to locate. Returns
// ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header or the chain is truncated before a
// terminal protocol is reached.
func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool, anyFragment bool, err error) { func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool, anyFragment bool, err error) {
const maxIPv6ExtHeaders = 8 const maxIPv6ExtHeaders = 8
if len(packet) < ipv6.HeaderLen { if len(packet) < ipv6.HeaderLen {
@@ -362,14 +365,14 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm
switch nextHeader { switch nextHeader {
case 0, 43, 60: // Hop-by-Hop, Routing, Destination case 0, 43, 60: // Hop-by-Hop, Routing, Destination
if len(packet) < offset+2 { if len(packet) < offset+2 {
return 0, 0, false, false, ErrIPv6CouldNotFindPayload return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
} }
nextHeader = packet[offset] nextHeader = packet[offset]
offset += (int(packet[offset+1]) + 1) << 3 offset += (int(packet[offset+1]) + 1) << 3
case 44: // Fragment case 44: // Fragment
if len(packet) < offset+8 { if len(packet) < offset+8 {
return 0, 0, false, false, ErrIPv6CouldNotFindPayload return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
} }
anyFragment = true anyFragment = true
// Non-first fragments carry no transport header, report the fragmented protocol and stop // Non-first fragments carry no transport header, report the fragmented protocol and stop
@@ -381,21 +384,21 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm
case 51: // AH case 51: // AH
if len(packet) < offset+2 { if len(packet) < offset+2 {
return 0, 0, false, false, ErrIPv6CouldNotFindPayload return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
} }
nextHeader = packet[offset] nextHeader = packet[offset]
offset += (int(packet[offset+1]) + 2) << 2 offset += (int(packet[offset+1]) + 2) << 2
default: default:
// A prior extension header can declare a length that advances offset past the packet. The terminal
// protocol's header isn't actually here, so treat the chain as truncated rather than classifying it.
if offset > len(packet) { if offset > len(packet) {
// A prior extension header can declare a length that advances offset past the packet, reject this. return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
} }
return nextHeader, offset, isFragment, anyFragment, nil //success! return nextHeader, offset, isFragment, anyFragment, nil
} }
} }
// Fail closed return nextHeader, offset, isFragment, anyFragment, nil
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
} }
func CreateICMPEchoResponse(packet, out []byte) []byte { func CreateICMPEchoResponse(packet, out []byte) []byte {
+5 -17
View File
@@ -529,17 +529,6 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
firstFragToUDP := []byte{17, 0, 0, 1, 0, 0, 0, 1} // frag offset 0, M=1, next = UDP firstFragToUDP := []byte{17, 0, 0, 1, 0, 0, 0, 1} // frag offset 0, M=1, next = UDP
nonFirstFrag := []byte{17, 0, 0, 9, 0, 0, 0, 1} // frag offset non-zero, next = UDP nonFirstFrag := []byte{17, 0, 0, 9, 0, 0, 0, 1} // frag offset non-zero, next = UDP
transport := []byte{0, 80, 1, 187, 0, 0, 0, 0} // stand-in bytes, IPv6FindUpperProtocol never reads ports transport := []byte{0, 80, 1, 187, 0, 0, 0, 0} // stand-in bytes, IPv6FindUpperProtocol never reads ports
extToHbH := []byte{0, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = Hop-by-Hop
// 8 walked headers spend the budget before the 9th slot's terminal is ever classified. This is the
// report 4040546 shape: without failing closed the walk hands back offset 104, and a caller reads ports
// from the attacker's bytes there.
var budgetChain []byte
for range 7 {
budgetChain = append(budgetChain, extToHbH...)
}
budgetChain = append(budgetChain, extToTCP...)
budgetChain = append(budgetChain, transport...)
tests := []struct { tests := []struct {
name string name string
@@ -560,10 +549,9 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
{"first fragment walks to transport", 44, append(firstFragToUDP, transport...), 17, ipv6.HeaderLen + 8, false, true, nil}, {"first fragment walks to transport", 44, append(firstFragToUDP, transport...), 17, ipv6.HeaderLen + 8, false, true, nil},
{"non-first fragment stops", 44, append(nonFirstFrag, transport...), 17, ipv6.HeaderLen, true, true, nil}, {"non-first fragment stops", 44, append(nonFirstFrag, transport...), 17, ipv6.HeaderLen, true, true, nil},
{"unknown protocol is terminal", 132, transport, 132, ipv6.HeaderLen, false, false, nil}, // SCTP {"unknown protocol is terminal", 132, transport, 132, ipv6.HeaderLen, false, false, nil}, // SCTP
{"truncated extension header", 0, nil, 0, 0, false, false, ErrIPv6CouldNotFindPayload}, {"truncated extension header", 0, nil, 0, ipv6.HeaderLen, false, false, ErrIPv6CouldNotFindPayload},
// Destination Options with a declared length (255+1)*8 = 2048 that runs past the 48 byte buffer, next = SCTP // Destination Options with a declared length (255+1)*8 = 2048 that runs past the 48 byte buffer, next = SCTP
{"extension length past buffer", 0, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 0, 0, false, false, ErrIPv6CouldNotFindPayload}, {"extension length past buffer", 60, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 132, ipv6.HeaderLen + 2048, false, false, ErrIPv6CouldNotFindPayload},
{"budget exhausted fails closed", 0, budgetChain, 0, 0, false, false, ErrIPv6CouldNotFindPayload},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -571,10 +559,10 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
packet := makeIPv6Packet(src, dst, tt.nextHeader, tt.payload) packet := makeIPv6Packet(src, dst, tt.nextHeader, tt.payload)
proto, offset, isFragment, anyFragment, err := IPv6FindUpperProtocol(packet) proto, offset, isFragment, anyFragment, err := IPv6FindUpperProtocol(packet)
if tt.wantErr != nil { if tt.wantErr != nil {
require.ErrorIs(t, err, tt.wantErr) assert.ErrorIs(t, err, tt.wantErr)
} else { return
require.NoError(t, err)
} }
require.NoError(t, err)
assert.Equal(t, tt.wantProto, proto) assert.Equal(t, tt.wantProto, proto)
assert.Equal(t, tt.wantOffset, offset) assert.Equal(t, tt.wantOffset, offset)
assert.Equal(t, tt.wantFragment, isFragment) assert.Equal(t, tt.wantFragment, isFragment)
+13 -71
View File
@@ -15,12 +15,10 @@ import (
"time" "time"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging" "github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/nicfinder"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
) )
@@ -44,10 +42,6 @@ type LightHouse struct {
// addresses rather than whatever this machine's NICs happen to be. Set it before Start. // addresses rather than whatever this machine's NICs happen to be. Set it before Start.
localAddrsFn func(*LocalAllowList) []netip.Addr localAddrsFn func(*LocalAllowList) []netip.Addr
// lastLocalAddrsErr is the previous localAddrsFn failure. Enumeration runs on every update, so an
// unchanged failure is demoted to Debug rather than warning every lighthouse.interval forever.
lastLocalAddrsErr atomic.Pointer[string]
// Local cache of answers from light houses // Local cache of answers from light houses
// map of vpn addr to answers // map of vpn addr to answers
addrMap map[netip.Addr]*RemoteList addrMap map[netip.Addr]*RemoteList
@@ -82,8 +76,6 @@ type LightHouse struct {
updateTrigger chan struct{} updateTrigger chan struct{}
queryChan chan netip.Addr queryChan chan netip.Addr
// queryDropped counts queries QueryServer dropped because queryChan was full.
queryDropped metrics.Counter
calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote
@@ -119,14 +111,11 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
nebulaPort: nebulaPort, nebulaPort: nebulaPort,
punchy: p, punchy: p,
updateTrigger: make(chan struct{}, 1), updateTrigger: make(chan struct{}, 1),
queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 1024)), queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
queryDropped: metrics.GetOrRegisterCounter("lighthouse.query.dropped", nil),
l: l, l: l,
} }
h.localAddrsFn = func(al *LocalAllowList) []netip.Addr { h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
addrs, err := nicfinder.Find(ctx, h.l, al) return localAddrs(h.l, al)
h.logLocalAddrsErr(err)
return addrs
} }
lighthouses := make([]netip.Addr, 0) lighthouses := make([]netip.Addr, 0)
@@ -138,21 +127,18 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
h.metrics = newLighthouseMetrics() h.metrics = newLighthouseMetrics()
} }
err := h.reload(c, true, nil) err := h.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("lighthouse.am_lighthouse", "listen.port", "handshakes.query_buffer", "stats.lighthouse_metrics") err := h.reload(c, false)
err := h.reload(c, false, r)
switch v := err.(type) { switch v := err.(type) {
case *util.ContextualError: case *util.ContextualError:
v.Log(l) v.Log(l)
r.Failed()
case error: case error:
l.Error("failed to reload lighthouse", "error", err) l.Error("failed to reload lighthouse", "error", err)
r.Failed()
} }
}) })
@@ -193,8 +179,7 @@ func (lh *LightHouse) GetUpdateInterval() int64 {
return lh.interval.Load() return lh.interval.Load()
} }
// r is nil on the initial load func (lh *LightHouse) reload(c *config.C, initial bool) error {
func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error {
if initial || c.HasChanged("lighthouse.advertise_addrs") { if initial || c.HasChanged("lighthouse.advertise_addrs") {
rawAdvAddrs := c.GetStringSlice("lighthouse.advertise_addrs", []string{}) rawAdvAddrs := c.GetStringSlice("lighthouse.advertise_addrs", []string{})
advAddrs := make([]netip.AddrPort, 0) advAddrs := make([]netip.AddrPort, 0)
@@ -229,7 +214,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error
"addr", rawAddr, "addr", rawAddr,
"entry", i+1, "entry", i+1,
) )
r.Failed()
continue continue
} }
@@ -364,13 +348,7 @@ func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error
} }
} }
// am_relay decides whether relay.relays applies to us at all if initial || c.HasChanged("relay.relays") {
if initial || c.HasChanged("relay.relays") || c.HasChanged("relay.am_relay") {
var before []netip.Addr
if !initial {
before = lh.GetRelaysForMe()
}
switch c.GetBool("relay.am_relay", false) { switch c.GetBool("relay.am_relay", false) {
case true: case true:
// Relays aren't allowed to specify other relays // Relays aren't allowed to specify other relays
@@ -388,7 +366,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error
"relay", v, "relay", v,
"error", err, "error", err,
) )
r.Failed()
} else { } else {
lh.l.Info("Read relay from config", "relay", v) lh.l.Info("Read relay from config", "relay", v)
relaysForMe = append(relaysForMe, configRIP) relaysForMe = append(relaysForMe, configRIP)
@@ -396,11 +373,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error
} }
lh.relaysForMe.Store(&relaysForMe) lh.relaysForMe.Store(&relaysForMe)
} }
// Tell the lighthouses now rather than at the next interval, but only when what we advertise changed
if !initial && !slices.Equal(before, lh.GetRelaysForMe()) {
lh.TriggerUpdate()
}
} }
return nil return nil
@@ -452,7 +424,7 @@ func getStaticMapCadence(c *config.C) (time.Duration, error) {
} }
func getStaticMapLookupTimeout(c *config.C) (time.Duration, error) { func getStaticMapLookupTimeout(c *config.C) (time.Duration, error) {
lookupTimeout := c.GetString("static_map.lookup_timeout", "5s") lookupTimeout := c.GetString("static_map.lookup_timeout", "250ms")
d, err := time.ParseDuration(lookupTimeout) d, err := time.ParseDuration(lookupTimeout)
if err != nil { if err != nil {
return 0, err return 0, err
@@ -533,19 +505,14 @@ func (lh *LightHouse) Query(vpnAddr netip.Addr) *RemoteList {
return nil return nil
} }
// QueryServer is asynchronous and non-blocking, no reply should be expected. // QueryServer is asynchronous so no reply should be expected
// If the query channel is full, the query is dropped.
func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) { func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) {
// Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses // Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses
if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) { if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) {
return return
} }
select { lh.queryChan <- vpnAddr
case lh.queryChan <- vpnAddr:
default:
lh.queryDropped.Inc(1)
}
} }
func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList { func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList {
@@ -630,8 +597,7 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
ctx := lh.ctx ctx := lh.ctx
lh.Unlock() lh.Unlock()
// On reload am.hr is the previous resolver, its last known good addresses carry over in case DNS is down right now hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, func() {
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, am.hr, func() {
// This callback runs whenever the DNS hostname resolver finds a different set of addr's // This callback runs whenever the DNS hostname resolver finds a different set of addr's
// in its resolution for hostnames. // in its resolution for hostnames.
am.Lock() am.Lock()
@@ -643,8 +609,7 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
} }
am.unlockedSetHostnamesResults(hr) am.unlockedSetHostnamesResults(hr)
// Only literals belong in the cache, anything from DNS has to stay in hr where a not found answer can remove it for _, addrPort := range hr.GetAddrs() {
for _, addrPort := range hr.literalAddrs() {
if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) { if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) {
continue continue
} }
@@ -951,26 +916,6 @@ func (lh *LightHouse) TriggerUpdate() {
} }
} }
// logLocalAddrsErr reports a localAddrs failure at Warn the first time it is seen and at Debug while
// it persists unchanged, so a permanent failure does not warn on every update forever.
func (lh *LightHouse) logLocalAddrsErr(err error) {
if err == nil {
lh.lastLocalAddrsErr.Store(nil)
return
}
msg := err.Error()
prev := lh.lastLocalAddrsErr.Swap(&msg)
if prev != nil && *prev == msg {
if lh.l.Enabled(context.Background(), slog.LevelDebug) {
lh.l.Debug("Failed to collect local addresses to advertise", "error", err)
}
return
}
lh.l.Warn("Failed to collect local addresses to advertise", "error", err)
}
func (lh *LightHouse) SendUpdate() { func (lh *LightHouse) SendUpdate() {
var v4 []*V4AddrPort var v4 []*V4AddrPort
var v6 []*V6AddrPort var v6 []*V6AddrPort
@@ -1263,13 +1208,10 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []neti
// sendHostPunchNotification signals the other side to punch some zero byte udp packets // sendHostPunchNotification signals the other side to punch some zero byte udp packets
func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, punchNotifDest netip.Addr, w EncWriter) { func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, punchNotifDest netip.Addr, w EncWriter) {
whereToPunch := fromVpnAddrs[0] whereToPunch := fromVpnAddrs[0]
// Resolve the target before taking the querier's RemoteList lock below. GetHostInfo takes the
// hostmap lock, and the hostmap is taken before the lighthouse, which is taken before a
// RemoteList, so looking it up inside the callback would deadlock.
targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest)
found, ln, err := lhh.lh.queryAndPrepMessage(whereToPunch, func(c *cache) (int, error) { found, ln, err := lhh.lh.queryAndPrepMessage(whereToPunch, func(c *cache) (int, error) {
n = lhh.resetMeta() n = lhh.resetMeta()
n.Type = NebulaMeta_HostPunchNotification n.Type = NebulaMeta_HostPunchNotification
targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest)
var useVersion cert.Version var useVersion cert.Version
if targetHI == nil { if targetHI == nil {
useVersion = lhh.lh.ifce.GetCertState().initiatingVersion useVersion = lhh.lh.ifce.GetCertState().initiatingVersion
+10 -288
View File
@@ -1,21 +1,16 @@
package nebula package nebula
import ( import (
"bytes"
"encoding/binary" "encoding/binary"
"errors"
"fmt" "fmt"
"net/netip" "net/netip"
"sync"
"testing" "testing"
"time"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.yaml.in/yaml/v3" "go.yaml.in/yaml/v3"
@@ -87,15 +82,15 @@ func TestReloadLighthouseInterval(t *testing.T) {
lh.ifce = &mockEncWriter{} lh.ifce = &mockEncWriter{}
// The first one routine is kicked off by main.go currently, lets make sure that one dies // The first one routine is kicked off by main.go currently, lets make sure that one dies
require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 5").Err) require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 5"))
assert.Equal(t, int64(5), lh.interval.Load()) assert.Equal(t, int64(5), lh.interval.Load())
// Subsequent calls are killed off by the LightHouse.Reload function // Subsequent calls are killed off by the LightHouse.Reload function
require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 10").Err) require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 10"))
assert.Equal(t, int64(10), lh.interval.Load()) assert.Equal(t, int64(10), lh.interval.Load())
// If this completes then nothing is stealing our reload routine // If this completes then nothing is stealing our reload routine
require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 11").Err) require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 11"))
assert.Equal(t, int64(11), lh.interval.Load()) assert.Equal(t, int64(11), lh.interval.Load())
} }
@@ -296,7 +291,7 @@ func TestLighthouse_reload(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
c.ReloadConfigString(string(rc)) c.ReloadConfigString(string(rc))
err = lh.reload(c, false, nil) err = lh.reload(c, false)
require.NoError(t, err) require.NoError(t, err)
} }
@@ -334,7 +329,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err := yaml.Marshal(nc) rc, err := yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
rl := lh.Query(staticHost) rl := lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -350,7 +345,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
rl = lh.Query(staticHost) rl = lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -358,7 +353,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, rl.CopyAddrs([]netip.Prefix{})) assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, rl.CopyAddrs([]netip.Prefix{}))
// Reload with the same config. An unchanged entry must not duplicate. // Reload with the same config. An unchanged entry must not duplicate.
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
rl = lh.Query(staticHost) rl = lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -373,7 +368,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
// Add a second host alongside the first. Both should be present, neither duplicated. // Add a second host alongside the first. Both should be present, neither duplicated.
nc = map[string]any{ nc = map[string]any{
@@ -384,7 +379,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
rl = lh.Query(staticHost) rl = lh.Query(staticHost)
require.NotNil(t, rl) require.NotNil(t, rl)
@@ -406,7 +401,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
} }
rc, err = yaml.Marshal(nc) rc, err = yaml.Marshal(nc)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, c.ReloadConfigString(string(rc)).Err) require.NoError(t, c.ReloadConfigString(string(rc)))
_, isStatic := lh.GetStaticHostList()[staticHost] _, isStatic := lh.GetStaticHostList()[staticHost]
assert.False(t, isStatic) assert.False(t, isStatic)
@@ -769,276 +764,3 @@ func TestLighthouse_DeletesWork(t *testing.T) {
out = lh.Query(testHost) out = lh.Query(testHost)
assert.Nil(t, out) assert.Nil(t, out)
} }
func TestLightHouse_logLocalAddrsErr(t *testing.T) {
out := &bytes.Buffer{}
lh := &LightHouse{l: test.NewLoggerWithOutput(out)}
// The first sighting of a failure warns.
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "permission denied")
// Repeating unchanged does not warn again, which is what keeps a permanent failure from warning
// on every lighthouse.interval for the life of the process.
out.Reset()
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.NotContains(t, out.String(), "level=WARN")
// A different failure is a new event and warns.
out.Reset()
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "something else")
// Recovering resets, so the same failure returning later warns again.
out.Reset()
lh.logLocalAddrsErr(nil)
assert.Empty(t, out.String())
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
}
// heldEncWriter holds the query worker in its first GetHostInfo until release closes.
type heldEncWriter struct {
mockEncWriter
once sync.Once
held chan struct{}
release chan struct{}
}
func (w *heldEncWriter) GetHostInfo(netip.Addr) *HostInfo {
w.once.Do(func() { close(w.held) })
<-w.release
return nil
}
// QueryServer never waits on the query worker, since the packet readers call it and nothing drains the channel at
// shutdown; a query that finds the channel full is dropped and counted. The worker is held on its first query so the
// channel fills.
func TestLighthouse_QueryServerDoesNotBlock(t *testing.T) {
l := test.NewLogger()
cs := testCertState(netip.MustParsePrefix("10.128.0.1/16"))
lh1 := "10.128.0.2"
c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}}
c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
c.Settings["handshakes"] = map[string]any{"query_buffer": 1}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err)
w := &heldEncWriter{held: make(chan struct{}), release: make(chan struct{})}
lh.ifce = w
t.Cleanup(func() { close(w.release) })
lh.QueryServer(netip.MustParseAddr("10.128.0.10"))
select {
case <-w.held:
case <-time.After(5 * time.Second):
t.Fatal("the query worker never took the first query")
}
dropped := lh.queryDropped.Count()
done := make(chan struct{})
go func() {
lh.QueryServer(netip.MustParseAddr("10.128.0.11"))
lh.QueryServer(netip.MustParseAddr("10.128.0.12"))
close(done)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("QueryServer blocked on a full query channel")
}
assert.EqualValues(t, 1, lh.queryDropped.Count()-dropped)
}
// hostmapEncWriter answers GetHostInfo from a real hostmap, the way Interface does.
type hostmapEncWriter struct {
mockEncWriter
hm *HostMap
}
func (w *hostmapEncWriter) GetHostInfo(a netip.Addr) *HostInfo { return w.hm.QueryVpnAddr(a) }
func (w *hostmapEncWriter) GetCertState() *CertState {
return &CertState{initiatingVersion: cert.Version1}
}
// A lighthouse runs four paths that each hold one lock while waiting for the next: CheckAndComplete
// holds the hostmap and takes the handshake manager, StartHandshake holds the handshake manager and
// takes the lighthouse, a host update holds the lighthouse and takes the sender's RemoteList, and a
// punch notification holds the querier's RemoteList while it resolves the target. If that last step
// reaches the hostmap under the RemoteList lock, the four deadlock. Drive them all at once.
func TestLighthouse_PunchNotificationDoesNotCloseLockCycle(t *testing.T) {
l := test.NewLogger()
myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true}
c.Settings["listen"] = map[string]any{"port": 4242}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(myVpnNet), nil, nil)
require.NoError(t, err)
mainHM := newHostMap(l)
pr := []netip.Prefix{}
mainHM.preferredRanges.Store(&pr)
lh.ifce = &hostmapEncWriter{hm: mainHM}
hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig)
f := &Interface{hostMap: mainHM, lightHouse: lh, handshakeManager: hm, l: l}
hm.f = f
peer := netip.MustParseAddr("10.128.0.2")
peerUdp := netip.MustParseAddrPort("10.0.0.2:4242")
target := netip.MustParseAddr("10.128.0.3")
targetUdp := netip.MustParseAddrPort("10.0.0.3:4242")
outbound := netip.MustParseAddr("10.128.0.4")
inbound := netip.MustParseAddr("10.128.0.5")
seed := lh.NewRequestHandler()
newLHHostUpdate(peerUdp, peer, []netip.AddrPort{peerUdp}, seed)
newLHHostUpdate(targetUdp, target, []netip.AddrPort{targetUdp}, seed)
stop := make(chan struct{})
var wg sync.WaitGroup
loop := func(body func()) {
defer wg.Done()
for {
select {
case <-stop:
return
default:
body()
}
}
}
wg.Add(4)
// Outbound handshakes keep starting and being dropped.
go loop(func() {
hm.DeleteHostInfo(hm.StartHandshake(outbound, nil))
})
// Inbound handshakes keep completing and being torn down.
var idx uint32
go loop(func() {
idx++
hi := &HostInfo{
vpnAddrs: []netip.Addr{inbound},
localIndexId: idx,
remoteIndexId: idx,
remotes: lh.QueryCache([]netip.Addr{inbound}),
HandshakePacket: map[uint8][]byte{handshakePacketStage0: {byte(idx)}},
}
if _, err := hm.CheckAndComplete(hi, handshakePacketStage0, f); err == nil {
mainHM.DeleteHostInfo(hi)
}
})
// The peer keeps sending host updates.
updates := lh.NewRequestHandler()
go loop(func() {
newLHHostUpdate(peerUdp, peer, []netip.AddrPort{peerUdp}, updates)
})
// The same peer keeps querying for the target, which ends in a punch notification.
queries := lh.NewRequestHandler()
go loop(func() {
sendLHHostRequest(peerUdp, peer, target, queries, nil)
})
time.Sleep(time.Second)
close(stop)
done := make(chan struct{})
go func() { wg.Wait(); close(done) }()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("deadlock: the lighthouse paths never finished after stop was signalled")
}
}
func TestLighthouse_reloadRelaysTriggersUpdate(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
base := "lighthouse:\n am_lighthouse: true\nlisten:\n port: 4242\n"
require.NoError(t, c.LoadString(base+"relay:\n relays: [10.128.0.5]\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
assert.Empty(t, lh.updateTrigger)
require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6]\n").Err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
<-lh.updateTrigger
// The config changed but what we advertise did not, a bad entry is skipped
require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6, nope]\n").Err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe())
assert.Empty(t, lh.updateTrigger)
}
func TestLighthouse_reloadAmRelayAlone(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
base := "lighthouse:\n am_lighthouse: true\nlisten:\n port: 4242\nrelay:\n relays: [10.128.0.5]\n"
require.NoError(t, c.LoadString(base+" am_relay: false\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe())
// Relays aren't allowed to have relays, flipping am_relay alone must drop them and tell the lighthouses
require.NoError(t, c.ReloadConfigString(base+" am_relay: true\n").Err)
assert.Empty(t, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
<-lh.updateTrigger
require.NoError(t, c.ReloadConfigString(base+" am_relay: false\n").Err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
}
func TestLighthouse_reloadRestartRequired(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
base := "lighthouse:\n am_lighthouse: true\n"
require.NoError(t, c.LoadString(base+"listen:\n port: 4242\n"))
_, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
res := c.ReloadConfigString(base + "listen:\n port: 4242\n")
require.NoError(t, res.Err)
assert.False(t, res.RestartRequired)
res = c.ReloadConfigString(base + "listen:\n port: 4243\n")
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
}
// A relay that doesn't parse is skipped and the rest are used, the reload says one was skipped
func TestLighthouse_reloadBadRelayFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("listen:\n port: 4242\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
res := c.ReloadConfigString("listen:\n port: 4242\nrelay:\n relays: [10.128.0.2, nope]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.2")}, lh.GetRelaysForMe())
}
// An advertise_addrs entry inside our own network is skipped and the rest are used, the reload says one was skipped
func TestLighthouse_reloadAdvertiseAddrInNetworkFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("listen:\n port: 4242\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
res := c.ReloadConfigString("listen:\n port: 4242\nlighthouse:\n advertise_addrs: [\"1.2.3.4:4242\", \"10.128.0.5:4242\"]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.2.3.4:4242")}, lh.GetAdvertiseAddrs())
}
-31
View File
@@ -1,31 +0,0 @@
package logging
import "log/slog"
// RateLimiter is what RateLimited needs from a rate limiter, such as a *ratelimit.Limiter.
// Allow reports whether this call may proceed, and how many calls were denied since the last one that did.
type RateLimiter interface {
Allow() (denied uint64, ok bool)
}
// RateLimited checks lim before anything is built.
// When allowed it returns l, with a "suppressed" count attached if lim denied calls since its last allowed one.
// A nil lim allows everything.
//
// if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok {
// hostinfo.logger(l).Error("Failed to write outgoing packet", "error", err)
// }
func RateLimited(l *slog.Logger, lim RateLimiter) (*slog.Logger, bool) {
if lim == nil {
return l, true
}
denied, ok := lim.Allow()
if !ok {
return nil, false
}
if denied > 0 {
return l.With("suppressed", denied), true
}
return l, true
}
-106
View File
@@ -1,106 +0,0 @@
package logging
import (
"bytes"
"errors"
"io"
"log/slog"
"net/netip"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/ratelimit"
)
type fakeLimiter struct {
denied uint64
ok bool
}
func (f fakeLimiter) Allow() (uint64, bool) { return f.denied, f.ok }
func TestRateLimited(t *testing.T) {
var buf bytes.Buffer
base := NewLogger(&buf)
if l, ok := RateLimited(base, fakeLimiter{ok: false}); ok || l != nil {
t.Fatalf("denied call: ok=%v logger=%v", ok, l)
}
if l, ok := RateLimited(base, fakeLimiter{ok: true}); !ok || l != base {
t.Fatal("nothing denied should hand back the same logger")
}
if l, ok := RateLimited(base, nil); !ok || l != base {
t.Fatal("a nil limiter should allow with the same logger")
}
l, ok := RateLimited(base, fakeLimiter{denied: 7, ok: true})
if !ok {
t.Fatal("allowed call denied")
}
l.Info("hello", "k", "v")
if !strings.Contains(buf.String(), "suppressed=7") {
t.Fatalf("want suppressed=7, got %q", buf.String())
}
}
// The benchmarks compare a log line through slog directly against the same line behind RateLimited: logged with
// nothing denied, denied, and logged after denials.
var (
benchErr = errors.New("sendto: no buffer space available")
benchAddr = netip.MustParseAddrPort("192.0.2.1:4242")
)
func benchLogger() *slog.Logger {
return NewLogger(io.Discard).With("localIndex", 1234, "remoteIndex", 5678)
}
func BenchmarkRateLimitedBaseline(b *testing.B) {
lg := benchLogger()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
lg.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
func BenchmarkRateLimitedAllowed(b *testing.B) {
lg, lim := benchLogger(), mustLimiter(b, 1, 0)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func BenchmarkRateLimitedDenied(b *testing.B) {
lg, lim := benchLogger(), mustLimiter(b, 1, time.Hour)
lim.Allow()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func BenchmarkRateLimitedAfterDenied(b *testing.B) {
lg, lim := benchLogger(), fakeLimiter{denied: 5, ok: true}
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func mustLimiter(b *testing.B, burst int, every time.Duration) *ratelimit.Limiter {
l, err := ratelimit.New(burst, every)
if err != nil {
b.Fatal(err)
}
return l
}
+9 -30
View File
@@ -12,7 +12,6 @@ import (
"strings" "strings"
"time" "time"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/cpupick" "github.com/slackhq/nebula/cpupick"
"github.com/slackhq/nebula/noiseutil" "github.com/slackhq/nebula/noiseutil"
@@ -33,12 +32,9 @@ const _ = noiseutil.RejectHeadroom - 4*maxRoutines
func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) { func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
// The goroutines started below stop only when this context does, and only a caller holding the // Automatically cancel the context if Main returns an error, to signal all created goroutines to quit.
// Control can arrange that. Cancel whenever we are not handing one back, which covers an error
// and a config test alike: a config test used to leave the lighthouse query worker, and a
// hostname resolver per dns named static host, running for the life of the process.
defer func() { defer func() {
if retcon == nil { if reterr != nil {
cancel() cancel()
} }
}() }()
@@ -72,18 +68,14 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
} }
l.Info("Firewall started", "firewallHashes", fw.GetRuleHashes()) l.Info("Firewall started", "firewallHashes", fw.GetRuleHashes())
// The command registry is shared by every surface that exposes these commands, so a ssh, err := sshd.NewSSHServer(ctx, l.With("subsystem", "sshd"))
// command cannot reach one of them without reaching the others.
cmdRegistry := commands.New()
ssh, err := sshd.NewSSHServer(ctx, l.With("subsystem", "sshd"), cmdRegistry)
if err != nil { if err != nil {
return nil, util.ContextualizeIfNeeded("Error while creating SSH server", err) return nil, util.ContextualizeIfNeeded("Error while creating SSH server", err)
} }
wireSSHReload(l, ssh, c) wireSSHReload(l, ssh, c)
var sshStart func() var sshStart func()
if c.GetBool("sshd.enabled", false) { if c.GetBool("sshd.enabled", false) {
sshStart, err = configSSH(l, ssh, c, nil) sshStart, err = configSSH(l, ssh, c)
if err != nil { if err != nil {
l.Warn("Failed to configure sshd, ssh debugging will not be available", "error", err) l.Warn("Failed to configure sshd, ssh debugging will not be available", "error", err)
sshStart = nil sshStart = nil
@@ -202,7 +194,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
if err != nil { if err != nil {
return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err) return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err)
} }
udpServer.ReloadConfig(c, nil) udpServer.ReloadConfig(c)
udpConns[i] = udpServer udpConns[i] = udpServer
// If port is dynamic, discover it before the next pass through the for loop // If port is dynamic, discover it before the next pass through the for loop
@@ -305,7 +297,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
} }
var ifce *Interface var ifce *Interface
var emitStats func()
if !configTest { if !configTest {
ifce, err = NewInterface(ctx, ifConfig) ifce, err = NewInterface(ctx, ifConfig)
if err != nil { if err != nil {
@@ -316,16 +307,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
lightHouse.ifce = ifce lightHouse.ifce = ifce
ifce.RegisterConfigChangeCallbacks(c) ifce.RegisterConfigChangeCallbacks(c)
// These are only read at startup
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged(
"routines", "tun.routines", "listen.routines", "firewall.conntrack.routine_cache_timeout",
"listen.port", "listen.host", "stats.message_metrics",
"handshakes.try_interval", "handshakes.retries", "handshakes.trigger_buffer",
"tun.drop_local_broadcast", "tun.drop_multicast",
)
r.RestartRequiredIfChanged(osStartupOnlyKeys...)
})
ifce.reloadDisconnectInvalid(c) ifce.reloadDisconnectInvalid(c)
ifce.reloadSendRecvError(c) ifce.reloadSendRecvError(c)
ifce.reloadAcceptRecvError(c) ifce.reloadAcceptRecvError(c)
@@ -334,12 +315,9 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
go handshakeManager.Run(ctx) go handshakeManager.Run(ctx)
punchy.Start(ctx, ifce, hostMap, lightHouse) punchy.Start(ctx, ifce, hostMap, lightHouse)
// On the stats server's loop, so the gauges follow the interval it accepted
emitStats = ifce.statsEmitter()
} }
stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest, emitStats) stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest)
if err != nil { if err != nil {
return nil, util.ContextualizeIfNeeded("Failed to start stats emitter", err) return nil, util.ContextualizeIfNeeded("Failed to start stats emitter", err)
} }
@@ -348,14 +326,15 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
return nil, nil return nil, nil
} }
attachCommands(l, c, cmdRegistry, ifce) go ifce.emitStats(ctx, c.GetDuration("stats.interval", time.Second*10))
attachCommands(l, c, ssh, ifce)
networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c) networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
return &Control{ return &Control{
state: StateReady, state: StateReady,
f: ifce, f: ifce,
commands: cmdRegistry,
l: l, l: l,
ctx: ctx, ctx: ctx,
cancel: cancel, cancel: cancel,
-6
View File
@@ -1,6 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
// osStartupOnlyKeys are startup only keys that only matter on linux: udp batch size, udp offloads and thread pinning
var osStartupOnlyKeys = []string{"listen.batch", "listen.udp_offloads", "tun.pin_threads", "tun.pin_threads_key", "tun.cpu_affinity"}
-6
View File
@@ -1,6 +0,0 @@
//go:build !linux || android || e2e_testing
package nebula
// osStartupOnlyKeys is empty, none of the OS specific startup keys matter here
var osStartupOnlyKeys []string
-74
View File
@@ -1,21 +1,12 @@
package nebula package nebula
import ( import (
"fmt"
"net/netip"
"os"
"path/filepath"
"testing" "testing"
"time"
"github.com/slackhq/nebula/cert"
cert_test "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
) )
func TestParseCpuAffinity(t *testing.T) { func TestParseCpuAffinity(t *testing.T) {
@@ -58,68 +49,3 @@ func TestParseCpuAffinity(t *testing.T) {
assert.Nil(t, parseCpuAffinity(newConfig([]any{1 << 20}), l, 1)) assert.Nil(t, parseCpuAffinity(newConfig([]any{1 << 20}), l, 1))
} }
} }
// TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running
// when it hands back a Control to stop them with.
//
// A config test gets no Control, so anything it started had nothing to stop it: the lighthouse
// query worker, and a hostname resolver per dns named static host, ran for the life of the
// process. That matters to every embedder that validates a config in process rather than by
// exec'ing, dnclient and the apple clients included, because they do it on each config load and
// the leak accumulates.
func TestMain_ConfigTestReleasesItsGoroutines(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
l := test.NewLogger()
dir := t.TempDir()
before := time.Now().Add(-time.Hour)
after := time.Now().Add(time.Hour)
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
_, _, keyPEM, certPEM := cert_test.NewTestCert(
cert.Version2, cert.Curve_CURVE25519, ca, caKey, "config-test", before, after, networks, nil, nil)
caPath := filepath.Join(dir, "ca.pem")
certPath := filepath.Join(dir, "cert.pem")
keyPath := filepath.Join(dir, "key.pem")
require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
// A static host by address, not by name: the query worker is the goroutine under test and a
// hostname would drag a real dns lookup into a unit test.
configBody := fmt.Sprintf(`
pki:
ca: %s
cert: %s
key: %s
static_host_map:
"10.0.0.2": ["192.0.2.1:4242"]
lighthouse:
hosts:
- "10.0.0.2"
listen:
host: 127.0.0.1
port: 0
tun:
disabled: true
firewall:
outbound:
- port: any
proto: any
host: any
inbound:
- port: any
proto: any
host: any
`, caPath, certPath, keyPath)
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
c := config.NewC(l)
require.NoError(t, c.Load(dir))
ctrl, err := Main(c, true, "config-test", l, nil)
require.NoError(t, err)
require.Nil(t, ctrl, "a config test hands back nothing to stop, so it must stop itself")
}
-39
View File
@@ -1,39 +0,0 @@
//go:build !android
package nicfinder
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/netip"
)
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
ifaces, err := net.Interfaces()
if err != nil {
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
}
var out []netip.Addr
var errs []error
for _, i := range ifaces {
if !allowName(ctx, l, filter, i.Name) {
continue
}
addrs, err := i.Addrs()
if err != nil {
errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err))
continue
}
for _, rawAddr := range addrs {
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
out = append(out, addr)
}
}
}
return out, errors.Join(errs...)
}
-104
View File
@@ -1,104 +0,0 @@
//go:build android
package nicfinder
import (
"context"
"errors"
"fmt"
"log/slog"
"net/netip"
"os"
"github.com/DefinedNet/netlink"
"github.com/DefinedNet/netlink/nl"
"golang.org/x/sys/unix"
)
// Android denies bind() on netlink route sockets and RTM_GETLINK, so net.Interfaces fails with EACCES.
// Enumerate with an RTM_GETADDR dump on an unbound socket instead.
// localInterfaces returns every interface that has an address, with its addresses. It lists
// addresses over a netlink socket that is never bound and names each index with SIOCGIFNAME, the
// same operations Bionic's getifaddrs uses for app UIDs, which the sandbox permits.
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
var errs []error
addrs, err := netlinkAddrList()
if err != nil {
if !errors.Is(err, netlink.ErrDumpInterrupted) {
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
}
// The address table kept changing under the dump and netlink gave up retrying. What it
// returned is still worth advertising.
errs = append(errs, fmt.Errorf("local addresses may be incomplete: %w", err))
}
// Any socket will do for SIOCGIFNAME.
sfd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
defer unix.Close(sfd)
var out []netip.Addr
nameAllowed := make(map[int]bool) // by interface index, once named
for _, a := range addrs {
allow, named := nameAllowed[a.LinkIndex]
if (named && !allow) || a.IPNet == nil {
continue
}
addr, ok := allowedAddr(ctx, l, filter, a.IPNet)
if !ok {
continue
}
if !named {
name, err := interfaceNameByIndex(sfd, a.LinkIndex)
// ENODEV means the interface went away after the dump. Any other failure is reported
// and the interface skipped, so the rest are still returned.
if err != nil && !errors.Is(err, unix.ENODEV) {
errs = append(errs, fmt.Errorf("failed to resolve name of interface %d: %w", a.LinkIndex, err))
}
allow = err == nil && allowName(ctx, l, filter, name)
nameAllowed[a.LinkIndex] = allow
}
if allow {
out = append(out, addr)
}
}
return out, errors.Join(errs...)
}
// netlinkAddrList is netlink.AddrList over a socket that is never bound. The kernel autobinds it
// on the first send, which the sandbox permits.
func netlinkAddrList() ([]netlink.Addr, error) {
fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
s, err := nl.NewNetlinkSocketFromFd(fd)
if err != nil {
unix.Close(fd)
return nil, err
}
h := netlink.NewHandleFromSockets(
map[int]*nl.SocketHandle{unix.NETLINK_ROUTE: {Socket: s}},
netlink.HandleOptions{RetryInterrupted: true},
)
defer h.Close()
if err := h.SetSocketTimeout(netlink.GetSocketTimeout()); err != nil {
return nil, err
}
return h.AddrList(nil, netlink.FAMILY_ALL)
}
// interfaceNameByIndex resolves an interface index with SIOCGIFNAME.
func interfaceNameByIndex(fd int, index int) (string, error) {
var ifr unix.Ifreq
ifr.SetUint32(uint32(index)) // ifr_ifindex is in the ifr_ifru union
if err := unix.IoctlIfreq(fd, unix.SIOCGIFNAME, &ifr); err != nil {
return "", err
}
return ifr.Name(), nil
}
-59
View File
@@ -1,59 +0,0 @@
package nicfinder
import (
"context"
"log/slog"
"net"
"net/netip"
"github.com/slackhq/nebula/logging"
)
// Filter decides which interfaces, by name, and which of their addresses Find returns.
type Filter interface {
AllowName(string) bool
Allow(netip.Addr) bool
}
// Find returns the addresses of this host's network interfaces that allowList accepts, with
// loopback and link-local addresses left out. When only some interfaces could be read it returns
// their addresses alongside an error for the rest. allowList must not be nil; it is called as
// given, so a nil pointer of a type whose methods accept a nil receiver is fine.
func Find(ctx context.Context, l *slog.Logger, allowList Filter) ([]netip.Addr, error) {
return localAddrs(ctx, l, allowList)
}
// allowName asks filter about an interface name and traces the answer.
func allowName(ctx context.Context, l *slog.Logger, filter Filter, name string) bool {
allow := filter.AllowName(name)
if l.Enabled(ctx, logging.LevelTrace) {
l.Log(ctx, logging.LevelTrace, "localAllowList.AllowName", "interfaceName", name, "allow", allow)
}
return allow
}
// allowedAddr converts one address a platform reports and says whether to advertise it: it must
// be a *net.IPNet or *net.IPAddr carrying an IP, not be loopback or link-local, and be accepted by
// filter, whose answer is traced. IPv4 comes back unmapped.
func allowedAddr(ctx context.Context, l *slog.Logger, filter Filter, rawAddr net.Addr) (netip.Addr, bool) {
var ip net.IP
switch v := rawAddr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
addr, ok := netip.AddrFromSlice(ip)
if !ok {
return netip.Addr{}, false
}
addr = addr.Unmap()
if addr.IsLoopback() || addr.IsLinkLocalUnicast() {
return netip.Addr{}, false
}
isAllowed := filter.Allow(addr)
if l.Enabled(ctx, logging.LevelTrace) {
l.Log(ctx, logging.LevelTrace, "localAllowList.Allow", "localAddr", addr, "allowed", isAllowed)
}
return addr, isAllowed
}
-117
View File
@@ -1,117 +0,0 @@
package nicfinder
import (
"context"
"net"
"net/netip"
"regexp"
"testing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
)
type allowAll struct{}
func (allowAll) Allow(netip.Addr) bool { return true }
func (allowAll) AllowName(string) bool { return true }
type testAllowList struct {
bannedName *regexp.Regexp
bannedAddr netip.Addr
}
func (l *testAllowList) Allow(addr netip.Addr) bool {
return addr != l.bannedAddr
}
func (l *testAllowList) AllowName(n string) bool {
return !l.bannedName.MatchString(n)
}
func newTestAllowList(bannedName string) *testAllowList {
return &testAllowList{
bannedName: regexp.MustCompile(bannedName),
}
}
func TestAllowedAddr(t *testing.T) {
ctx, l := context.Background(), test.NewLogger()
allowed := func(filter Filter, addrs []net.Addr) []netip.Addr {
var out []netip.Addr
for _, rawAddr := range addrs {
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
out = append(out, addr)
}
}
return out
}
addrs := []net.Addr{
&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)},
&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
&net.IPNet{IP: net.ParseIP("10.0.0.5"), Mask: net.CIDRMask(24, 32)},
&net.IPNet{IP: net.ParseIP("169.254.10.10"), Mask: net.CIDRMask(16, 32)},
&net.IPNet{IP: net.ParseIP("fe80::1"), Mask: net.CIDRMask(64, 128)},
&net.IPAddr{IP: net.ParseIP("fd00::5")},
&net.IPNet{},
otherAddr{},
nil,
}
// Loopback, link local and anything without an IP are dropped, and IPv4 comes back unmapped
// even though net.ParseIP hands over a 16 byte slice.
assert.Equal(t, []netip.Addr{
netip.MustParseAddr("10.0.0.5"),
netip.MustParseAddr("fd00::5"),
}, allowed(allowAll{}, addrs))
// The filter sees the unmapped form and can drop a single address.
al := newTestAllowList("none")
al.bannedAddr = netip.MustParseAddr("10.0.0.5")
assert.Equal(t, []netip.Addr{netip.MustParseAddr("fd00::5")}, allowed(al, addrs))
}
// otherAddr is a net.Addr that is neither *net.IPNet nor *net.IPAddr. netlink.Addr is one such
// type: it satisfies net.Addr through an embedded *net.IPNet, and passing it directly once
// silently produced no addresses.
type otherAddr struct{}
func (otherAddr) Network() string { return "other" }
func (otherAddr) String() string { return "other" }
// hostFilter counts AllowName calls and can reject every name or every address.
type hostFilter struct {
rejectNames bool
rejectAddrs bool
asked map[string]int
}
func (f *hostFilter) Allow(netip.Addr) bool { return !f.rejectAddrs }
func (f *hostFilter) AllowName(name string) bool {
f.asked[name]++
return !f.rejectNames
}
// Runs this platform's localAddrs against the host and checks it honors the whole filter.
func TestLocalAddrsAppliesFilter(t *testing.T) {
ctx, l := context.Background(), test.NewLogger()
f := &hostFilter{asked: map[string]int{}}
all, err := localAddrs(ctx, l, f)
if len(f.asked) == 0 {
t.Skipf("no interfaces to enumerate here: %v", err)
}
for name, n := range f.asked {
assert.Equal(t, 1, n, "AllowName asked %d times for %s", n, name)
}
// Loopback and link-local never come back, whatever the filter says.
for _, addr := range all {
assert.False(t, addr.IsLoopback() || addr.IsLinkLocalUnicast(), "%s", addr)
}
none, _ := localAddrs(ctx, l, &hostFilter{rejectNames: true, asked: map[string]int{}})
assert.Empty(t, none, "addresses got past a filter that rejects every interface name")
none, _ = localAddrs(ctx, l, &hostFilter{rejectAddrs: true, asked: map[string]int{}})
assert.Empty(t, none, "addresses got past a filter that rejects every address")
}
+8 -13
View File
@@ -160,7 +160,9 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
case header.TestReply: case header.TestReply:
// No-op, useful for the Roaming and connectionManager side-effects above // No-op, useful for the Roaming and connectionManager side-effects above
case header.TestRequest: case header.TestRequest:
if header.MaxOverhead+len(out) > len(rxc.scratch) { const maxCipherOverhead = 16 //todo we use this too often, needs a real importable const
const maxOverhead = header.Len + header.Len + maxCipherOverhead + maxCipherOverhead
if maxOverhead+len(out) > len(rxc.scratch) {
// A reply that cannot fit in scratch is dropped no matter the log level. // A reply that cannot fit in scratch is dropped no matter the log level.
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("dropping oversized test request", "payloadLen", len(out), "from", via) hostinfo.logger(f.l).Debug("dropping oversized test request", "payloadLen", len(out), "from", via)
@@ -261,17 +263,12 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
// closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
func (f *Interface) closeTunnel(hostInfo *HostInfo) { func (f *Interface) closeTunnel(hostInfo *HostInfo) {
// Hold the hostmap lock across both the delete and the lighthouse eviction, final := f.hostMap.DeleteHostInfo(hostInfo)
// to avoid a race with handshake completion. if final {
f.hostMap.Lock()
defer f.hostMap.Unlock()
if !f.hostMap.unlockedDeleteHostInfo(hostInfo) {
return
}
// We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage // We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage
f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs) f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs)
} }
}
// sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote // sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote
func (f *Interface) sendCloseTunnel(h *HostInfo) { func (f *Interface) sendCloseTunnel(h *HostInfo) {
@@ -357,7 +354,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
// and cannot drift into misreading an unknown protocol (SCTP, GRE, etc.) as a forged transport. // and cannot drift into misreading an unknown protocol (SCTP, GRE, etc.) as a forged transport.
proto, offset, isFragment, anyFragment, err := iputil.IPv6FindUpperProtocol(data) proto, offset, isFragment, anyFragment, err := iputil.IPv6FindUpperProtocol(data)
if err != nil { if err != nil {
return err return ErrIPv6PacketTooShort
} }
fp.Protocol = proto fp.Protocol = proto
@@ -476,9 +473,7 @@ func parseV4(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounter uint64, out []byte, rxc *rxContext) { func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounter uint64, out []byte, rxc *rxContext) {
err := newPacket(out, true, rxc.fwPacket) err := newPacket(out, true, rxc.fwPacket)
if err != nil { if err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) { hostinfo.logger(f.l).Warn("Error while validating inbound packet", "error", err, "packet", out)
hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out)
}
return return
} }
+7 -7
View File
@@ -117,7 +117,7 @@ func Test_newPacket_v6(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
err = newPacket(buffer.Bytes(), true, p) err = newPacket(buffer.Bytes(), true, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) require.ErrorIs(t, err, ErrIPv6PacketTooShort)
// A v6 packet with a hop-by-hop extension // A v6 packet with a hop-by-hop extension
// ICMPv6 Payload (Echo Request) // ICMPv6 Payload (Echo Request)
@@ -151,12 +151,12 @@ func Test_newPacket_v6(t *testing.T) {
// A full IPv6 header and 1 byte in the first extension, but missing // A full IPv6 header and 1 byte in the first extension, but missing
// the length byte. // the length byte.
err = newPacket(buffer.Bytes()[:41], true, p) err = newPacket(buffer.Bytes()[:41], true, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) require.ErrorIs(t, err, ErrIPv6PacketTooShort)
// A full IPv6 header plus 1 full extension, but only 1 byte of the // A full IPv6 header plus 1 full extension, but only 1 byte of the
// next layer, missing length byte // next layer, missing length byte
err = newPacket(buffer.Bytes()[:49], true, p) err = newPacket(buffer.Bytes()[:49], true, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) require.ErrorIs(t, err, ErrIPv6PacketTooShort)
err = nil err = nil
// A good ICMP packet // A good ICMP packet
@@ -354,12 +354,12 @@ func Test_newPacket_v6(t *testing.T) {
// Ensure buffer bounds checking during processing, a truncated AH header can't reach the payload // Ensure buffer bounds checking during processing, a truncated AH header can't reach the payload
err = newPacket(b[:41], true, p) err = newPacket(b[:41], true, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) require.ErrorIs(t, err, ErrIPv6PacketTooShort)
// Invalid AH header // Invalid AH header
b = buffer.Bytes() b = buffer.Bytes()
err = newPacket(b, true, p) err = newPacket(b, true, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) require.ErrorIs(t, err, ErrIPv6PacketTooShort)
} }
func Test_newPacket_ipv6Fragment(t *testing.T) { func Test_newPacket_ipv6Fragment(t *testing.T) {
@@ -466,7 +466,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
// Too short of a fragment packet // Too short of a fragment packet
err = newPacket(secondFrag[:len(secondFrag)-10], false, p) err = newPacket(secondFrag[:len(secondFrag)-10], false, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) require.ErrorIs(t, err, ErrIPv6PacketTooShort)
} }
func BenchmarkParseV6(b *testing.B) { func BenchmarkParseV6(b *testing.B) {
@@ -707,7 +707,7 @@ func Test_newPacket_v6ExtHeaderPastBuffer(t *testing.T) {
pkt[40] = byte(layers.IPProtocolSCTP) // Dest Options next header = SCTP pkt[40] = byte(layers.IPProtocolSCTP) // Dest Options next header = SCTP
pkt[41] = 255 // declared length (255+1)*8 = 2048, past the 48 byte buffer pkt[41] = 255 // declared length (255+1)*8 = 2048, past the 48 byte buffer
require.ErrorIs(t, newPacket(pkt, true, p), iputil.ErrIPv6CouldNotFindPayload) require.ErrorIs(t, newPacket(pkt, true, p), ErrIPv6PacketTooShort)
} }
// Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized // Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized
-57
View File
@@ -1,57 +0,0 @@
//go:build !e2e_testing
package overlay
import (
"fmt"
"log/slog"
"golang.org/x/sys/windows"
"golang.org/x/sys/windows/registry"
)
var tcpipInterfaceKeys = []string{
`SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces`,
`SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces`,
}
// disableDNSRegistration prevents publishing overlay addresses in the host's
// DNS records, where clients outside the overlay cannot reach them.
// Failures are non-fatal so registry access cannot prevent tunnel creation.
func disableDNSRegistration(l *slog.Logger, guid windows.GUID) {
id := guid.String()
for _, base := range tcpipInterfaceKeys {
path := base + `\` + id
if err := setDNSRegistrationValues(path); err != nil {
l.Warn("Failed to disable DNS registration on the tun adapter", "error", err, "key", path)
continue
}
l.Debug("Disabled DNS registration on the tun adapter", "key", path)
}
}
func setDNSRegistrationValues(path string) error {
// The stack may not have created the adapter's interface key yet.
k, _, err := registry.CreateKey(registry.LOCAL_MACHINE, path, registry.SET_VALUE)
if err != nil {
return fmt.Errorf("open HKLM\\%s: %w", path, err)
}
defer k.Close()
// Neither RegistrationEnabled nor DisableDynamicUpdate alone reliably
// prevents registration, so all three settings are required.
values := []struct {
name string
data uint32
}{
{"RegistrationEnabled", 0},
{"DisableDynamicUpdate", 1},
{"MaxNumberOfAddressesToRegister", 0},
}
for _, v := range values {
if err := k.SetDWordValue(v.name, v.data); err != nil {
return fmt.Errorf("set %s: %w", v.name, err)
}
}
return nil
}
+4 -28
View File
@@ -50,13 +50,7 @@ func (r Route) String() string {
func makeRouteTree(l *slog.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) { func makeRouteTree(l *slog.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) {
routeTree := new(bart.Table[routing.Gateways]) routeTree := new(bart.Table[routing.Gateways])
seen := make(map[netip.Prefix]bool, len(routes))
for _, r := range routes { for _, r := range routes {
if seen[r.Cidr] {
l.Warn("route is listed more than once, only the last entry's via is used", "route", r)
}
seen[r.Cidr] = true
if !allowMTU && r.MTU > 0 { if !allowMTU && r.MTU > 0 {
l.Warn("route MTU is not supported on this platform", l.Warn("route MTU is not supported on this platform",
"goos", runtime.GOOS, "goos", runtime.GOOS,
@@ -104,11 +98,7 @@ func parseRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
mtu, ok := rMtu.(int) mtu, ok := rMtu.(int)
if !ok { if !ok {
s, ok := rMtu.(string) mtu, err = strconv.Atoi(rMtu.(string))
if !ok {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, rMtu)
}
mtu, err = strconv.Atoi(s)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, err) return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, err)
} }
@@ -184,11 +174,7 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
if rMtu, ok := m["mtu"]; ok { if rMtu, ok := m["mtu"]; ok {
mtu, ok = rMtu.(int) mtu, ok = rMtu.(int)
if !ok { if !ok {
s, ok := rMtu.(string) mtu, err = strconv.Atoi(rMtu.(string))
if !ok {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, rMtu)
}
mtu, err = strconv.Atoi(s)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err) return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err)
} }
@@ -206,15 +192,10 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
metric, ok := rMetric.(int) metric, ok := rMetric.(int)
if !ok { if !ok {
s, ok := rMetric.(string) _, err = strconv.ParseInt(rMetric.(string), 10, 32)
if !ok {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, rMetric)
}
n, err := strconv.ParseInt(s, 10, 32)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, err) return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, err)
} }
metric = int(n)
} }
if metric < 0 || metric > math.MaxInt32 { if metric < 0 || metric > math.MaxInt32 {
@@ -267,15 +248,10 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
gatewayWeight, ok := rGatewayWeight.(int) gatewayWeight, ok := rGatewayWeight.(int)
if !ok { if !ok {
s, ok := rGatewayWeight.(string) _, err = strconv.ParseInt(rGatewayWeight.(string), 10, 32)
if !ok {
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
}
w, err := strconv.ParseInt(s, 10, 32)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1) return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
} }
gatewayWeight = int(w)
} }
if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 { if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 {
-116
View File
@@ -1,17 +1,13 @@
package overlay package overlay
import ( import (
"bytes"
"fmt" "fmt"
"net/netip" "net/netip"
"strings"
"testing" "testing"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -406,115 +402,3 @@ func Test_makeMultipathUnsafeRouteTree(t *testing.T) {
routing.CalculateBucketsForGateways(expectedGateways) routing.CalculateBucketsForGateways(expectedGateways)
assert.ElementsMatch(t, expectedGateways, r) assert.ElementsMatch(t, expectedGateways, r)
} }
func Test_getMTU(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
// unset
assert.Equal(t, DefaultMTU, getMTU(l, c))
// the largest a relayed packet can carry in one underlay buffer
require.Equal(t, udp.MTU, MaxMTU+header.MaxOverhead)
c.Settings["tun"] = map[string]any{"mtu": MaxMTU}
assert.Equal(t, MaxMTU, getMTU(l, c))
// too big is capped
c.Settings["tun"] = map[string]any{"mtu": 9000}
assert.Equal(t, MaxMTU, getMTU(l, c))
}
func Test_getAllRoutesFromConfigCapsMTU(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
c.Settings["tun"] = map[string]any{
"routes": []any{
map[string]any{"mtu": "9000", "route": "10.0.0.0/29"},
map[string]any{"mtu": "1300", "route": "10.0.0.8/29"},
},
"unsafe_routes": []any{
map[string]any{"via": "127.0.0.1", "mtu": "9000", "route": "1.0.0.0/29"},
},
}
changed, routes, err := getAllRoutesFromConfig(l, c, []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")}, true)
require.NoError(t, err)
assert.True(t, changed)
mtus := map[string]int{}
for _, r := range routes {
mtus[r.Cidr.String()] = r.MTU
}
assert.Equal(t, map[string]int{"10.0.0.0/29": MaxMTU, "10.0.0.8/29": 1300, "1.0.0.0/29": MaxMTU}, mtus)
}
// The route tree keeps one entry per CIDR, so a CIDR listed twice only routes by the last one. Say so instead of
// letting the earlier via, or on Windows the earlier metric, quietly lose
func Test_makeRouteTree_WarnsOnDuplicate(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewLoggerWithOutput(buf)
via := func(s string) routing.Gateways {
return routing.Gateways{routing.NewGateway(netip.MustParseAddr(s), 1)}
}
cidr := netip.MustParsePrefix("10.251.1.0/24")
_, err := makeRouteTree(l, []Route{
{Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100},
{Cidr: netip.MustParsePrefix("10.251.2.0/24"), Via: via("10.250.0.2"), Install: true},
}, false)
require.NoError(t, err)
assert.Empty(t, buf.String())
tree, err := makeRouteTree(l, []Route{
{Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100},
{Cidr: cidr, Via: via("10.250.0.3"), Install: true, Metric: 200},
}, false)
require.NoError(t, err)
assert.Equal(t, 1, strings.Count(buf.String(), "route is listed more than once, only the last entry's via is used"))
gw, ok := tree.Lookup(netip.MustParseAddr("10.251.1.1"))
require.True(t, ok)
assert.Equal(t, netip.MustParseAddr("10.250.0.3"), gw[0].Addr(), "the last entry wins")
}
// A quoted metric or weight is a number written as a string, it means that number. Anything else is an error, not a
// panic
func Test_parseUnsafeRoutes_QuotedNumbers(t *testing.T) {
c := config.NewC(test.NewLogger())
n := netip.MustParsePrefix("10.0.0.1/24")
route := func(metric, weight any) {
c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{
"route": "1.0.0.0/8",
"metric": metric,
"via": []any{map[string]any{"gateway": "10.0.0.2", "weight": weight}},
}}}
}
route("100", "5")
routes, err := parseUnsafeRoutes(c, []netip.Prefix{n})
require.NoError(t, err)
require.Len(t, routes, 1)
assert.Equal(t, 100, routes[0].Metric)
assert.Equal(t, "{addr: 10.0.0.2, weight: 5}", routes[0].Via[0].String())
route(1.5, 1)
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.metric in tun.unsafe_routes is not an integer: 1.5")
route(1, true)
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry .weight in tun.unsafe_routes[1].via[1] is not an integer")
}
// An mtu that is neither a number nor a quoted one is an error, not a panic
func Test_parseRoutes_MTUNotANumber(t *testing.T) {
c := config.NewC(test.NewLogger())
n := netip.MustParsePrefix("10.0.0.1/24")
c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"route": "1.0.0.0/8", "mtu": 1500.5}}}
_, err := parseRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.mtu in tun.routes is not an integer: 1500.5")
c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"route": "1.0.0.0/8", "via": "10.0.0.2", "mtu": 1500.5}}}
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is not an integer: 1500.5")
}
+3 -13
View File
@@ -20,7 +20,6 @@ type offloadQueueSet struct {
// usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel. // usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel.
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission. // Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
usoEnabled bool usoEnabled bool
shutdown atomic.Bool
closed atomic.Bool closed atomic.Bool
// l is handed to each queue for its bad-vnet-header drop logging. // l is handed to each queue for its bad-vnet-header drop logging.
l *slog.Logger l *slog.Logger
@@ -70,17 +69,6 @@ func (c *offloadQueueSet) wakeForShutdown() error {
return err return err
} }
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *offloadQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *offloadQueueSet) Close() error { func (c *offloadQueueSet) Close() error {
if c.closed.Swap(true) { if c.closed.Swap(true) {
return nil return nil
@@ -88,7 +76,9 @@ func (c *offloadQueueSet) Close() error {
errs := []error{} errs := []error{}
if err := c.Shutdown(); err != nil { // Signal all readers blocked in poll to wake up and exit.
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
errs = append(errs, err) errs = append(errs, err)
} }
+3 -13
View File
@@ -16,7 +16,6 @@ type pollQueueSet struct {
// pqi is exactly the same as pq, but stored as the interface type // pqi is exactly the same as pq, but stored as the interface type
pqi []Queue pqi []Queue
shutdownFd int shutdownFd int
shutdown atomic.Bool
closed atomic.Bool closed atomic.Bool
} }
@@ -60,17 +59,6 @@ func (c *pollQueueSet) wakeForShutdown() error {
return err return err
} }
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *pollQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *pollQueueSet) Close() error { func (c *pollQueueSet) Close() error {
if c.closed.Swap(true) { if c.closed.Swap(true) {
return nil return nil
@@ -78,7 +66,9 @@ func (c *pollQueueSet) Close() error {
errs := []error{} errs := []error{}
if err := c.Shutdown(); err != nil { // Signal all readers blocked in poll to wake up and exit.
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
errs = append(errs, err) errs = append(errs, err)
} }
-2
View File
@@ -7,8 +7,6 @@ import (
// QueueSet holds one or many Queue objects and helps close them in an orderly way. // QueueSet holds one or many Queue objects and helps close them in an orderly way.
type QueueSet interface { type QueueSet interface {
io.Closer io.Closer
// Shutdown wakes every reader without releasing any fds, Close only once they have exited
Shutdown() error
Queues() []Queue Queues() []Queue
// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue. // Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
-5
View File
@@ -66,8 +66,6 @@ type Offload struct {
// usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD, // usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD,
// so writers can decide whether emitting GSO_UDP_L4 superpackets is safe. // so writers can decide whether emitting GSO_UDP_L4 superpackets is safe.
usoEnabled bool usoEnabled bool
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool closed atomic.Bool
// gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted // gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted
@@ -178,9 +176,6 @@ func (r *Offload) readPacket(block bool) (int, error) {
// This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs). // This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs).
// Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue. // Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue.
func (r *Offload) Read() ([]Packet, error) { func (r *Offload) Read() ([]Packet, error) {
if r.shutdown.Load() {
return nil, os.ErrClosed
}
r.pending = r.pending[:0] r.pending = r.pending[:0]
r.rxOff = 0 r.rxOff = 0
-5
View File
@@ -13,8 +13,6 @@ import (
type Poll struct { type Poll struct {
fd int fd int
shutdownFd int shutdownFd int
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool closed atomic.Bool
readBuf []byte readBuf []byte
@@ -54,9 +52,6 @@ func (t *Poll) blockOnWrite() error {
// config currently lose that batching. blockOn and the EAGAIN plumbing are // config currently lose that batching. blockOn and the EAGAIN plumbing are
// already shared; kept one-packet-per-Read for now to preserve behavior. // already shared; kept one-packet-per-Read for now to preserve behavior.
func (t *Poll) Read() ([]Packet, error) { func (t *Poll) Read() ([]Packet, error) {
if t.shutdown.Load() {
return nil, os.ErrClosed
}
n, err := t.readOne(t.readBuf) n, err := t.readOne(t.readBuf)
if err != nil { if err != nil {
return nil, err return nil, err
-27
View File
@@ -223,30 +223,3 @@ func TestOffloadQueueSet_Close_ClosesShutdownFd(t *testing.T) {
// Second Close must not touch fds (shutdownFd is now -1) and must return nil. // Second Close must not touch fds (shutdownFd is now -1) and must return nil.
require.NoError(t, qs.Close()) require.NoError(t, qs.Close())
} }
// A busy reader never polls the shutdown eventfd, so Read has to stop on its own with data still waiting
func TestQueueSet_ShutdownStopsReaderWithDataWaiting(t *testing.T) {
sets := map[string]func() (QueueSet, error){
"poll": NewPollQueueSet,
"offload": func() (QueueSet, error) { return NewOffloadQueueSet(false, slog.New(slog.DiscardHandler)) },
}
for name, newSet := range sets {
t.Run(name, func(t *testing.T) {
var fds [2]int
require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
t.Cleanup(func() { _ = unix.Close(fds[1]) })
qs, err := newSet()
require.NoError(t, err)
require.NoError(t, qs.Add(fds[0]))
t.Cleanup(func() { _ = qs.Close() })
_, err = unix.Write(fds[1], make([]byte, 64))
require.NoError(t, err)
require.NoError(t, qs.Shutdown())
_, err = qs.Queues()[0].Read()
require.ErrorIs(t, err, os.ErrClosed)
})
}
}
+2 -31
View File
@@ -7,27 +7,11 @@ import (
"net/netip" "net/netip"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
) )
const DefaultMTU = 1300 const DefaultMTU = 1300
// MaxMTU is the largest tun.mtu, or route mtu, the underlay can carry.
// Every underlay buffer is udp.MTU bytes, and a relayed packet adds header.MaxOverhead to its plaintext.
const MaxMTU = udp.MTU - header.MaxOverhead
// getMTU reads tun.mtu, capping a value too big for the underlay to carry.
func getMTU(l *slog.Logger, c *config.C) int {
mtu := c.GetInt("tun.mtu", DefaultMTU)
if mtu > MaxMTU {
l.Warn("tun.mtu is too big for the underlay to carry, capping it", "mtu", mtu, "maxMTU", MaxMTU)
return MaxMTU
}
return mtu
}
type NameError struct { type NameError struct {
Name string Name string
Underlying error Underlying error
@@ -41,16 +25,9 @@ func (e *NameError) Error() string {
type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error)
func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) { func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.disabled")
})
switch { switch {
case c.GetBool("tun.disabled", false): case c.GetBool("tun.disabled", false):
tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 5000), c.GetBool("stats.message_metrics", false), l) tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.tx_queue", "stats.message_metrics")
})
return tun, nil return tun, nil
default: default:
@@ -64,7 +41,7 @@ func NewFdDeviceFromConfig(fd *int) DeviceFactory {
} }
} }
func getAllRoutesFromConfig(l *slog.Logger, c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) { func getAllRoutesFromConfig(c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) {
if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") { if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") {
return false, nil, nil return false, nil, nil
} }
@@ -80,12 +57,6 @@ func getAllRoutesFromConfig(l *slog.Logger, c *config.C, vpnNetworks []netip.Pre
} }
routes = append(routes, unsafeRoutes...) routes = append(routes, unsafeRoutes...)
for i, r := range routes {
if r.MTU > MaxMTU {
l.Warn("route mtu is too big for the underlay to carry, capping it", "route", r.Cidr, "mtu", r.MTU, "maxMTU", MaxMTU)
routes[i].MTU = MaxMTU
}
}
return true, routes, nil return true, routes, nil
} }
+5 -6
View File
@@ -38,17 +38,16 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
l: l, l: l,
} }
err := t.reload(c, true, nil) err := t.reload(c, true)
if err != nil { if err != nil {
_ = file.Close() _ = file.Close()
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
err := t.reload(c, false, r) err := t.reload(c, false)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -68,8 +67,8 @@ func (t *tun) Activate() error {
return nil return nil
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
-45
View File
@@ -1,45 +0,0 @@
//go:build (openbsd || netbsd) && !e2e_testing
package overlay
import (
"errors"
"fmt"
"log/slog"
"os"
"regexp"
"runtime"
"golang.org/x/sys/unix"
)
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
// tunDevNeedsRestart is true when tun.dev asks for a particular tun that isn't the running one. Anything else takes the
// first free one, and the one we have will do
func tunDevNeedsRestart(name, running string) bool {
return deviceNameRE.MatchString(name) && name != running
}
// openTun opens /dev/name, or the first /dev/tunN nothing else has open when name is empty or not tunN
func openTun(l *slog.Logger, name string) (int, string, error) {
if name != "" && !deviceNameRE.MatchString(name) {
// The name doesn't mean anything here, so like darwin a bad one isn't worth refusing to start over
l.Warn("interface name must be tun[0-9]+ on "+runtime.GOOS+", ignoring", "name", name)
name = ""
}
if name != "" {
fd, err := unix.Open("/dev/"+name, os.O_RDWR, 0)
return fd, name, err
}
for i := 0; ; i++ {
name = fmt.Sprintf("tun%d", i)
fd, err := unix.Open("/dev/"+name, os.O_RDWR, 0)
if err == nil {
return fd, name, nil
}
if !errors.Is(err, unix.EBUSY) {
return -1, "", fmt.Errorf("no free tun device, %s: %w", name, err)
}
}
}
-80
View File
@@ -1,80 +0,0 @@
//go:build (openbsd || netbsd) && !e2e_testing
package overlay
import (
"net/netip"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// A tun.dev that isn't tunN is ignored like darwin does, and the first free tun is used. Only naming a different tun
// asks for a restart
func TestTunReloadDevRestart(t *testing.T) {
newWithDev := func(dev string) (*config.C, *tun, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n dev: "+dev+"\n"))
tn, err := newTun(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, false)
if err == nil {
t.Cleanup(func() { _ = tn.Close() })
}
return c, tn, err
}
if _, _, err := newWithDev(""); err != nil {
t.Skipf("can't create a tun device here: %v", err)
}
c, tn, err := newWithDev("nope")
require.NoError(t, err, "a bad name should fall back to a free tun")
require.Regexp(t, `^tun[0-9]+$`, tn.Device)
for _, dev := range []string{"nope", "", tn.Device, "nope"} {
res := c.ReloadConfigString("tun:\n dev: " + dev + "\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed, dev)
assert.False(t, res.RestartRequired, dev)
}
other := "tun0"
if tn.Device == other {
other = "tun1"
}
res := c.ReloadConfigString("tun:\n dev: " + other + "\n")
require.NoError(t, res.Err)
assert.Equal(t, []string{"tun.dev"}, res.RestartKeys)
}
// An empty tun.dev takes the first tun nothing else has open. Reloading it empty, or naming the one we have, needs
// no restart
func TestTunEmptyDevPicksAFreeOne(t *testing.T) {
newEmpty := func() (*config.C, *tun, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun: {}\n"))
tn, err := newTun(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, false)
if err == nil {
t.Cleanup(func() { _ = tn.Close() })
}
return c, tn, err
}
c, first, err := newEmpty()
if err != nil {
t.Skipf("can't create a tun device here: %v", err)
}
_, second, err := newEmpty()
require.NoError(t, err, "the second tun should skip the one the first has open")
assert.NotEqual(t, first.Device, second.Device)
for _, cfg := range []string{
"tun: {}\npunchy:\n punch: true\n",
"tun:\n dev: " + first.Device + "\n",
"tun: {}\n",
} {
res := c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired, cfg)
}
}
-3
View File
@@ -10,9 +10,6 @@ import (
"github.com/slackhq/nebula/wfp" "github.com/slackhq/nebula/wfp"
) )
// interfaceBypassKeys are only read when the interface bypass is installed
var interfaceBypassKeys = []string{"tun.windows_bypass_wdf"}
// installInterfaceBypass installs a WFP PERMIT filter scoped to the wintun interface LUID so inbound traffic on the // installInterfaceBypass installs a WFP PERMIT filter scoped to the wintun interface LUID so inbound traffic on the
// nebula adapter bypasses Windows Defender Firewall. // nebula adapter bypasses Windows Defender Firewall.
func installInterfaceBypass(l *slog.Logger, luid uint64) closer { func installInterfaceBypass(l *slog.Logger, luid uint64) closer {
-3
View File
@@ -4,9 +4,6 @@ package overlay
import "log/slog" import "log/slog"
// interfaceBypassKeys is empty, windows-386 has no bypass
var interfaceBypassKeys []string
// installInterfaceBypass is a no-op on windows-386 because we don't currently build for it. // installInterfaceBypass is a no-op on windows-386 because we don't currently build for it.
func installInterfaceBypass(_ *slog.Logger, _ uint64) closer { func installInterfaceBypass(_ *slog.Logger, _ uint64) closer {
return nil return nil
+75 -214
View File
@@ -3,15 +3,13 @@
package overlay package overlay
import ( import (
"errors"
"fmt" "fmt"
"log/slog" "log/slog"
"math/bits"
"net/netip" "net/netip"
"os" "os"
"sync"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"time"
"unsafe" "unsafe"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
@@ -30,31 +28,8 @@ type tun struct {
DefaultMTU int DefaultMTU int
Routes atomic.Pointer[[]Route] Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]] routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr atomic.Pointer[netroute.LinkAddr] linkAddr *netroute.LinkAddr
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
// and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool
l *slog.Logger l *slog.Logger
// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
// fields below it
routesMu sync.Mutex
// routeStates is who held each installable unsafe_route prefix when we last looked
routeStates map[netip.Prefix]routeState
// routeWatch is the routing socket watchRoutes reads, Close closes it
routeWatch *os.File
// routeSweep reconciles the unsafe_routes every routeSweepEvery
routeSweep *time.Timer
routeSweepEvery time.Duration
closed bool
// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
// lock so the route watcher never waits on a reconcile
routeMoversMu sync.Mutex
routeMovers map[netip.Prefix]routeMover
// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
lastRouteEvent atomic.Int64
} }
type ifReq struct { type ifReq struct {
@@ -100,31 +75,17 @@ type ifreqAlias6 struct {
Lifetime addrLifetime Lifetime addrLifetime
} }
// utunIndex is the utun tun.dev asks for, or -1 for whichever is free. ok is false for a name that isn't utunN, which
// also gets whichever is free
func utunIndex(name string) (index int, ok bool) {
if name == "" || name == "utun" {
return -1, true
}
if _, err := fmt.Sscanf(name, "utun%d", &index); err != nil || index < 0 {
return -1, false
}
return index, true
}
// tunDevNeedsRestart is true when tun.dev asks for a particular utun that isn't the running one. Anything else takes
// whichever is free, and the one we have will do
func tunDevNeedsRestart(name, running string) bool {
index, _ := utunIndex(name)
return index >= 0 && fmt.Sprintf("utun%d", index) != running
}
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
ifIndex, ok := utunIndex(c.GetString("tun.dev", "")) name := c.GetString("tun.dev", "")
if !ok { ifIndex := -1
if name != "" && name != "utun" {
_, err := fmt.Sscanf(name, "utun%d", &ifIndex)
if err != nil || ifIndex < 0 {
// NOTE: we don't make this error so we don't break existing // NOTE: we don't make this error so we don't break existing
// configs that set a name before it was used. // configs that set a name before it was used.
l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring") l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring")
ifIndex = -1
}
} }
fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL) fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL)
@@ -148,7 +109,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
return nil, fmt.Errorf("SYS_CONNECT: %v", err) return nil, fmt.Errorf("SYS_CONNECT: %v", err)
} }
name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME) name, err = unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to retrieve tun name: %w", err) return nil, fmt.Errorf("failed to retrieve tun name: %w", err)
} }
@@ -162,25 +123,19 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
f: os.NewFile(uintptr(fd), ""), f: os.NewFile(uintptr(fd), ""),
Device: name, Device: name,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c), DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
routeSweepEvery: routeSweepInterval,
l: l, l: l,
} }
err = t.reload(c, true, nil) err = t.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("tun.mtu") err := t.reload(c, false)
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -194,62 +149,11 @@ func (t *tun) deviceBytes() (o [16]byte) {
return return
} }
// newTunFromFd adopts a utun the host already created and configured, which is how a darwin func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
// network extension is handed its device. Everything about moving packets is shared with newTun, return nil, fmt.Errorf("newTunFromFd not supported in Darwin")
// only the setup differs: the host owns addressing and routing here.
func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
if err := unix.SetNonblock(deviceFd, true); err != nil {
// We own the fd from the moment it is handed to us
_ = unix.Close(deviceFd)
return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err)
}
file := os.NewFile(uintptr(deviceFd), "/dev/tun")
t := &tun{
f: file,
Device: utunNameFromFd(deviceFd),
vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c),
hostOwned: true,
l: l,
}
if err := t.reload(c, true, nil); err != nil {
_ = file.Close()
return nil, err
}
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
if err := t.reload(c, false, r); err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
}
})
return t, nil
}
// utunNameFromFd asks the socket what interface it is, for logs. A blank name is not worth
// failing a tunnel over, so an error just leaves it empty.
func utunNameFromFd(fd int) string {
name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME)
if err != nil {
return ""
}
return name
} }
func (t *tun) Close() error { func (t *tun) Close() error {
t.routesMu.Lock()
t.closed = true
if t.routeWatch != nil {
_ = t.routeWatch.Close()
}
if t.routeSweep != nil {
t.routeSweep.Stop()
}
t.routesMu.Unlock()
if t.f != nil { if t.f != nil {
return t.f.Close() return t.f.Close()
} }
@@ -257,12 +161,6 @@ func (t *tun) Close() error {
} }
func (t *tun) Activate() error { func (t *tun) Activate() error {
// The host handed us a configured device. Its addresses, mtu and routes come from the network
// settings it applied, and a sandboxed extension cannot change them anyway.
if t.hostOwned {
return nil
}
devName := t.deviceBytes() devName := t.deviceBytes()
s, err := unix.Socket( s, err := unix.Socket(
@@ -296,7 +194,7 @@ func (t *tun) Activate() error {
if linkAddr == nil { if linkAddr == nil {
return fmt.Errorf("unable to discover link_addr for tun interface") return fmt.Errorf("unable to discover link_addr for tun interface")
} }
t.linkAddr.Store(linkAddr) t.linkAddr = linkAddr
for _, network := range t.vpnNetworks { for _, network := range t.vpnNetworks {
if network.Addr().Is4() { if network.Addr().Is4() {
@@ -318,25 +216,10 @@ func (t *tun) Activate() error {
return fmt.Errorf("failed to run tun device: %s", err) return fmt.Errorf("failed to run tun device: %s", err)
} }
// Watch before adding so a route removed in between is still noticed
watch, err := openRouteSocket()
if err != nil {
t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
// Unsafe path routes // Unsafe path routes
if err = t.addRoutes(false, nil); err != nil { return t.addRoutes(false)
if watch != nil {
_ = watch.Close()
}
return err
} }
t.startRouteWatch(watch)
return nil
}
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
func (t *tun) activate4(network netip.Prefix) error { func (t *tun) activate4(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET, unix.AF_INET,
@@ -363,32 +246,22 @@ func (t *tun) activate4(network netip.Prefix) error {
MaskAddr: unix.RawSockaddrInet4{ MaskAddr: unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4, Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET, Family: unix.AF_INET,
Addr: [4]byte{255, 255, 255, 255}, //always make the address a /32 Addr: prefixToMask(network).As4(),
}, },
} }
// On Darwin, VPNs don't edit the routing table directly.
// A network extension hands its routes to the system, and configd installs them.
// configd replaces any route already holding the same prefix, including ours.
// As an example, Tailscale with an exit node treats any interface prefix wider than one address as a LAN.
// With LAN access off, it routes each LAN into Tailscale's utun.
// With LAN access on, it excludes each LAN from its tunnel.
// configd routes an excluded prefix like its own routes would, ignoring ours, so it goes out the default route.
// Either way our route is replaced, and traffic for the vpn network no longer reaches nebula.
// A single address isn't a LAN to Tailscale, so our route is left alone.
// We route the vpn network to the tun explicitly instead.
if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun v4 address: %s", err) return fmt.Errorf("failed to set tun v4 address: %s", err)
} }
if err := addRoute(network, t.linkAddr.Load()); err != nil { err = addRoute(network, t.linkAddr)
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err) if err != nil {
return err
} }
return nil return nil
} }
// activate6 is activate4 for IPv6
func (t *tun) activate6(network netip.Prefix) error { func (t *tun) activate6(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET6, unix.AF_INET6,
@@ -410,36 +283,31 @@ func (t *tun) activate6(network netip.Prefix) error {
PrefixMask: unix.RawSockaddrInet6{ PrefixMask: unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6, Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6, Family: unix.AF_INET6,
// Always set the netmask to be a /128. See activate4 for why we do this. Addr: prefixToMask(network).As16(),
Addr: [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
}, },
Lifetime: addrLifetime{ Lifetime: addrLifetime{
// never expires // never expires
Vltime: 0xffffffff, Vltime: 0xffffffff,
Pltime: 0xffffffff, Pltime: 0xffffffff,
}, },
Flags: _IN6_IFF_NODAD, // No DAD, a utun has no neighbors. Flags: _IN6_IFF_NODAD,
} }
if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun address: %s", err) return fmt.Errorf("failed to set tun address: %s", err)
} }
if err := addRoute(network, t.linkAddr.Load()); err != nil {
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
}
return nil return nil
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
if !initial && !change { if !initial && !change {
// Any reload puts back the unsafe_routes someone deleted return nil
return t.addRoutes(true, r)
} }
routeTree, err := makeRouteTree(t.l, routes, false) routeTree, err := makeRouteTree(t.l, routes, false)
@@ -453,18 +321,16 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -479,56 +345,6 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
return routing.Gateways{} return routing.Gateways{}
} }
func fetchRoutes() ([]netroute.Message, error) {
rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_DUMP, 0)
if err != nil {
return nil, err
}
return netroute.ParseRIB(unix.NET_RT_DUMP, rib)
}
// routeInterfaceIn returns the index of the interface prefix is routed through, or 0 if msgs has no route for exactly
// that prefix. Routes scoped to an interface are skipped, they don't stop us adding ours.
func routeInterfaceIn(msgs []netroute.Message, prefix netip.Prefix) int {
for _, msg := range msgs {
m, ok := msg.(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK || m.Flags&unix.RTF_IFSCOPE != 0 {
continue
}
if p, ok := routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK]); ok && p == prefix.Masked() {
return m.Index
}
}
return 0
}
// routePrefix returns the prefix a route covers. Host routes have no netmask.
func routePrefix(dst, mask netroute.Addr) (netip.Prefix, bool) {
switch d := dst.(type) {
case *netroute.Inet4Addr:
bits := 32
if m, ok := mask.(*netroute.Inet4Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom4(d.IP), bits), true
case *netroute.Inet6Addr:
bits := 128
if m, ok := mask.(*netroute.Inet6Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom16(d.IP), bits), true
}
return netip.Prefix{}, false
}
func maskBits(mask []byte) int {
n := 0
for _, b := range mask {
n += bits.OnesCount8(b)
}
return n
}
// Get the LinkAddr for the interface of the given name // Get the LinkAddr for the interface of the given name
// Is there an easier way to fetch this when we create the interface? // Is there an easier way to fetch this when we create the interface?
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers. // Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
@@ -557,6 +373,51 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
return nil, nil return nil, nil
} }
func (t *tun) addRoutes(logErrors bool) error {
routes := *t.Routes.Load()
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
err := addRoute(r.Cidr, t.linkAddr)
if err != nil {
if errors.Is(err, unix.EEXIST) {
t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr)
} else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
retErr.Log(t.l)
} else {
return retErr
}
}
} else {
t.l.Info("Added route", "route", r)
}
}
return nil
}
func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes {
if !r.Install {
continue
}
err := delRoute(r.Cidr, t.linkAddr)
if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error { func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil { if err != nil {
-484
View File
@@ -1,484 +0,0 @@
//go:build darwin && !ios && !e2e_testing
package overlay
import (
"encoding/binary"
"encoding/hex"
"fmt"
"net"
"net/netip"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Every mac routes loopback through lo0
func TestRouteInterfaceIn(t *testing.T) {
msgs, err := fetchRoutes()
require.NoError(t, err)
routeInterface := func(s string) int { return routeInterfaceIn(msgs, netip.MustParsePrefix(s)) }
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterface("127.0.0.0/8"))
assert.Equal(t, lo0.Index, routeInterface("127.0.0.1/32"), "a host route has no netmask")
assert.Equal(t, lo0.Index, routeInterface("::1/128"))
assert.Equal(t, 0, routeInterface("127.0.0.0/16"), "only an exact prefix counts")
// Most macs also list the default route scoped to their other interfaces
out, err := exec.Command("route", "-n", "get", "default").Output()
require.NoError(t, err)
var name string
for _, line := range strings.Split(string(out), "\n") {
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "interface: "); ok {
name = v
}
}
if name == "" {
t.Skip("no default route")
}
def, err := net.InterfaceByName(name)
require.NoError(t, err)
assert.Equal(t, def.Index, routeInterface("0.0.0.0/0"))
}
func TestRouteInterfaceIn_SkipsScopedCopies(t *testing.T) {
route := func(index, flags int) netroute.Message {
addrs := make([]netroute.Addr, unix.RTAX_MAX)
addrs[unix.RTAX_DST] = &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}}
addrs[unix.RTAX_NETMASK] = &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}}
return &netroute.RouteMessage{Index: index, Flags: flags, Addrs: addrs}
}
p := netip.MustParsePrefix("192.168.1.0/24")
assert.Equal(t, 7, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE), route(7, unix.RTF_UP)}, p))
assert.Equal(t, 0, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE)}, p))
}
// A reload can happen before Activate
func TestTunRoutesBeforeActivate(t *testing.T) {
tn := &tun{l: test.NewLogger()}
routes := []Route{{
Cidr: netip.MustParsePrefix("10.253.1.0/24"),
Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)},
Install: true,
}}
tn.Routes.Store(&routes)
assert.NoError(t, tn.addRoutes(true, nil))
assert.NoError(t, tn.removeRoutes(routes, nil))
}
// A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that
// belongs to someone else
func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating a utun and adding routes needs root")
}
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
cidr := "10.250.0.128/25"
require.NoError(t, exec.Command("route", "-q", "-n", "add", "-net", cidr, "-interface", "lo0").Run())
t.Cleanup(func() { _ = exec.Command("route", "-q", "-n", "delete", "-net", cidr).Run() })
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("tun:\n routes:\n - route: "+cidr+"\n mtu: 1300\n"))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
res := c.ReloadConfigString("tun: {}\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed, "removing a route we never installed failed the reload")
msgs, err := fetchRoutes()
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
const (
// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
// With no interface address, which x/net/route skips
interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
)
func TestRoutesMayHaveMoved(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
routes := []Route{{Cidr: cidr, Via: via, Install: true}}
// A request like route(8) sends, the kernel echoes it to every routing socket
request := func(typ, flags int, dst, mask [4]byte) []byte {
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: typ,
Flags: flags,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: dst},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
},
}
b, err := m.Marshal()
require.NoError(t, err)
return b
}
raw := func(s string) []byte {
b, err := hex.DecodeString(s)
require.NoError(t, err)
return b
}
lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
routesMayHaveMoved := func(b []byte, routes []Route) bool {
moved, _, _ := routesMayHaveMoved(b, routes)
return moved
}
assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
neighbor := raw(kernelSubnetDelete)
binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
}
// A request names its prefix and who made it, the kernel's own and interface changes name neither
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: unix.RTM_CHANGE,
Flags: unix.RTF_UP,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
},
}
b, err := m.Marshal()
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved)
assert.Equal(t, cidr, p)
assert.Equal(t, int32(1234), pid)
for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
b, err := hex.DecodeString(s)
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved, name)
assert.False(t, p.IsValid(), name)
assert.Zero(t, pid, name)
}
}
func TestProcessName(t *testing.T) {
assert.Equal(t, "launchd", processName(1))
self := filepath.Base(os.Args[0])
assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
assert.Empty(t, processName(-1))
}
// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
// the minute between sweeps when it isn't zero.
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
requireRoot(t)
cfg := "tun:\n unsafe_routes:\n"
for _, cidr := range cidrs {
via := "10.250.0.2"
if netip.MustParsePrefix(cidr).Addr().Is6() {
via = "fd00:250::2"
}
cfg += " - route: " + cidr + "\n via: " + via + "\n"
}
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString(cfg))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
require.NoError(t, err)
if sweep != 0 {
tn.routeSweepEvery = sweep
}
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
return tn, c, cfg
}
func requireRoot(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating interfaces and adding routes needs root")
}
}
// routeVia is the index of the interface that routes cidr, 0 if none does
func routeVia(t *testing.T, cidr string) int {
msgs, err := fetchRoutes()
require.NoError(t, err)
return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
}
// routeStateOf is what tn made of cidr when it last looked
func routeStateOf(tn *tun, cidr string) routeState {
tn.routesMu.Lock()
defer tn.routesMu.Unlock()
return tn.routeStates[netip.MustParsePrefix(cidr)]
}
func routeArgs(action, cidr string, extra ...string) []string {
args := []string{"-q", "-n", action}
if netip.MustParsePrefix(cidr).Addr().Is6() {
args = append(args, "-inet6")
}
return append(append(args, "-net", cidr), extra...)
}
func routeCmd(t *testing.T, action, cidr string, extra ...string) {
requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
}
// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
// says so
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
require.NotContains(t, string(out), "writing to routing socket")
}
// cleanupRoute deletes cidr's route when the test ends, whoever holds it
func cleanupRoute(t *testing.T, cidr string) {
t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
}
// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
func routeThroughLo0(t *testing.T, cidr string) int {
requireRoot(t)
routeCmd(t, "add", cidr, "-interface", "lo0")
cleanupRoute(t, cidr)
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
return lo0.Index
}
// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
t.Run(cidr, func(t *testing.T) {
lo0 := routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
})
}
}
// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
cidr := "10.251.1.0/24"
lo0 := routeThroughLo0(t, cidr)
_, c, _ := newRouteTestTun(t, 0, cidr)
res := c.ReloadConfigString("tun: {}\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, lo0, routeVia(t, cidr))
}
// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
cidr := "10.251.3.0/24"
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
time.Sleep(2 * routeRetryDelay)
require.Equal(t, lo0.Index, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
}
// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
cidr := "10.251.4.0/24"
tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
// A reload with nothing changed is how to ask for it back
res := c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, ours, routeVia(t, cidr))
}
// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
for name, tc := range map[string]struct {
cidr string
sweep time.Duration
misfile func(t *testing.T, tn *tun, cidr string)
}{
// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
}},
// The sweep that caps a table that never settles
"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
tn.reconcileRoutes()
require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
}},
} {
t.Run(name, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, tc.cidr))
routeCmd(t, "delete", tc.cidr)
tc.misfile(t, tn, tc.cidr)
lo0 := routeThroughLo0(t, tc.cidr)
require.Eventually(t, func() bool {
s := routeStateOf(tn, tc.cidr)
return s.kind == routeYielded && s.holder == lo0
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", tc.cidr)
assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
// taken for a hand delete
func TestTunYieldsToARouteChangedAway(t *testing.T) {
for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
t.Run(cidr, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
routeCmd(t, "change", cidr, "-interface", "lo0")
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
require.Equal(t, lo0.Index, routeVia(t, cidr))
require.Eventually(t, func() bool {
s := routeStateOf(tn, cidr)
return s.kind == routeYielded && s.holder == lo0.Index
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
// ethernet configd routed the lan through
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
requireRoot(t)
var feth string
for i := 100; i < 200 && feth == ""; i++ {
if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
feth = fmt.Sprintf("feth%d", i)
}
}
require.NotEmpty(t, feth)
require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
// refuses one named by the feth itself
cidr := "10.251.2.0/24"
routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
cleanupRoute(t, cidr)
ifc, err := net.InterfaceByName(feth)
require.NoError(t, err)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Losing the address first leaves the route, the reconcile finds it still held
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
time.Sleep(2 * routeRetryDelay)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Only RTM_IFINFO says the interface went, its route goes with no message of its own
require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
cidr := "10.251.5.0/24"
routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
tn.routesMu.Lock()
require.NoError(t, tn.routeWatch.Close())
tn.routesMu.Unlock()
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
func TestTunDevNeedsRestart(t *testing.T) {
for name, want := range map[string]bool{
"": false,
"utun": false,
"nope": false,
"utun7": false,
"utun3": true,
} {
assert.Equal(t, want, tunDevNeedsRestart(name, "utun7"), name)
}
}
-121
View File
@@ -1,121 +0,0 @@
//go:build !ios && !e2e_testing
package overlay
import (
"bytes"
"os"
"syscall"
"testing"
"github.com/stretchr/testify/require"
)
func TestTun_WriteFraming(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: w}
for _, tc := range []struct {
name string
pkt []byte
af byte
}{
// Literal families rather than syscall.AF_*, so a wrong constant in Write can't also fix the expectation.
{"ipv4", []byte{0x45, 1, 2, 3, 4, 5}, 2},
{"ipv6", []byte{0x60, 1, 2, 3, 4, 5}, 30},
} {
t.Run(tc.name, func(t *testing.T) {
n, err := tn.Write(tc.pkt)
require.NoError(t, err)
require.Equal(t, len(tc.pkt), n)
got := make([]byte, 64)
n, err = r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, tc.af}, tc.pkt...), got[:n])
})
}
_, err = tn.Write([]byte{0x10, 1, 2, 3})
require.Error(t, err)
_, err = tn.Write([]byte{})
require.ErrorIs(t, err, syscall.EIO)
// The rejected writes must not have put anything on the fd, so the next read sees only this packet.
pkt := []byte{0x45, 9, 9}
_, err = tn.Write(pkt)
require.NoError(t, err)
got := make([]byte, 64)
n, err := r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, 2}, pkt...), got[:n])
}
func TestTun_ReadStripsHeader(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
pkt := []byte{0x45, 1, 2, 3, 4, 5}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err := tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
func TestTun_ReadShort(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
// Fewer bytes than the protocol header is not a packet.
_, err = w.Write([]byte{0, 0})
require.NoError(t, err)
n, err := tn.Read(make([]byte, 64))
require.NoError(t, err)
require.Zero(t, n)
// The next packet reads cleanly only if the short read consumed exactly what was written.
pkt := []byte{0x45, 1, 2, 3}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err = tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
// A heap allocation per packet rarely shows in a short throughput run, only across GC pauses, so
// this pins Read and Write to zero allocations directly.
func TestTun_ReadWriteDoNotAllocate(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tw := &tun{f: w}
tr := &tun{f: r}
pkts := [][]byte{bytes.Repeat([]byte{0x45}, 1300), bytes.Repeat([]byte{0x60}, 1300)}
buf := make([]byte, 1500)
i := 0
allocs := testing.AllocsPerRun(1000, func() {
i++
if _, err := tw.Write(pkts[i%len(pkts)]); err != nil {
t.Fatal(err)
}
if _, err := tr.Read(buf); err != nil {
t.Fatal(err)
}
})
require.Zero(t, allocs)
}
+13 -23
View File
@@ -370,7 +370,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
t := &tun{ t := &tun{
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
l: l, l: l,
fd: fd, fd: fd,
shutdownR: shutdownR, shutdownR: shutdownR,
@@ -385,18 +385,16 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
}, },
} }
err = t.reload(c, true, nil) err = t.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
closeOnErr = false closeOnErr = false
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("tun.dev", "tun.mtu") err := t.reload(c, false)
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -495,7 +493,7 @@ func (t *tun) Activate() error {
} }
} }
return t.addRoutes(false, nil) return t.addRoutes(false)
} }
func (t *tun) setMTU() error { func (t *tun) setMTU() error {
@@ -511,8 +509,8 @@ func (t *tun) setMTU() error {
return err return err
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -532,18 +530,16 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -567,7 +563,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
} }
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { func (t *tun) addRoutes(logErrors bool) error {
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
@@ -580,7 +576,6 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -592,20 +587,15 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.linkAddr) err := delRoute(r.Cidr, t.linkAddr)
if errors.Is(err, unix.ESRCH) { if err != nil {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
+5 -6
View File
@@ -47,17 +47,16 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
l: l, l: l,
} }
err := t.reload(c, true, nil) err := t.reload(c, true)
if err != nil { if err != nil {
_ = file.Close() _ = file.Close()
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
err := t.reload(c, false, r) err := t.reload(c, false)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -68,8 +67,8 @@ func (t *tun) Activate() error {
return nil return nil
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
+19 -47
View File
@@ -15,12 +15,12 @@ import (
"time" "time"
"unsafe" "unsafe"
"github.com/DefinedNet/netlink"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
@@ -185,10 +185,6 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
} }
t.Device = name t.Device = name
// Only read when we create the device, an fd handed to us already has its name and offloads
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.dev", "tun.use_offloads")
})
return t, nil return t, nil
} }
@@ -225,32 +221,30 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
vnetHdr: vnetHdr, vnetHdr: vnetHdr,
offloadFlags: offloadFlags, offloadFlags: offloadFlags,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
TXQueueLen: c.GetInt("tun.tx_queue", 5000), TXQueueLen: c.GetInt("tun.tx_queue", 500),
useSystemRoutes: c.GetBool("tun.use_system_route_table", false), useSystemRoutes: c.GetBool("tun.use_system_route_table", false),
useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0), useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
routesFromSystem: map[netip.Prefix]routing.Gateways{}, routesFromSystem: map[netip.Prefix]routing.Gateways{},
l: l, l: l,
} }
if err = t.reload(c, true, nil); err != nil { if err = t.reload(c, true); err != nil {
_ = t.Close() _ = t.Close()
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("tun.tx_queue", "tun.use_system_route_table", "tun.use_system_route_table_buffer_size") err := t.reload(c, false)
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
return t, nil return t, nil
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
routeChange, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -259,15 +253,6 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
return nil return nil
} }
// Only the mtu changed. The routes are the same, parse them again so the ones without their own mtu pick up the
// new default. Without this an mtu change would remove them all
if !routeChange {
_, routes, err = getAllRoutesFromConfig(t.l, c, t.vpnNetworks, true)
if err != nil {
return err
}
}
routeTree, err := makeRouteTree(t.l, routes, true) routeTree, err := makeRouteTree(t.l, routes, true)
if err != nil { if err != nil {
return err return err
@@ -282,14 +267,14 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
oldDefaultMTU := t.DefaultMTU oldDefaultMTU := t.DefaultMTU
oldMaxMTU := t.MaxMTU oldMaxMTU := t.MaxMTU
newDefaultMTU := getMTU(t.l, c) newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU)
newMaxMTU := newDefaultMTU newMaxMTU := newDefaultMTU
for i, r := range routes { for i, r := range routes {
if r.MTU == 0 { if r.MTU == 0 {
routes[i].MTU = newDefaultMTU routes[i].MTU = newDefaultMTU
} }
if r.MTU > newMaxMTU { if r.MTU > t.MaxMTU {
newMaxMTU = r.MTU newMaxMTU = r.MTU
} }
} }
@@ -303,7 +288,7 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
if oldMaxMTU != newMaxMTU { if oldMaxMTU != newMaxMTU {
t.setMTU(r) t.setMTU()
t.l.Info("Set max MTU", "mtu", t.MaxMTU, "oldMTU", oldMaxMTU) t.l.Info("Set max MTU", "mtu", t.MaxMTU, "oldMTU", oldMaxMTU)
} }
@@ -311,8 +296,7 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
for i := range t.vpnNetworks { for i := range t.vpnNetworks {
err := t.setDefaultRoute(t.vpnNetworks[i]) err := t.setDefaultRoute(t.vpnNetworks[i])
if err != nil { if err != nil {
t.l.Error(err.Error()) t.l.Warn(err.Error())
r.Failed()
} else { } else {
t.l.Info("Set default MTU", "mtu", t.DefaultMTU, "oldMTU", oldDefaultMTU) t.l.Info("Set default MTU", "mtu", t.DefaultMTU, "oldMTU", oldDefaultMTU)
} }
@@ -320,14 +304,13 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
} }
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// This should never be called since addRoutes should log its own errors in a reload condition // This should never be called since addRoutes should log its own errors in a reload condition
util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l) util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l)
r.Failed()
} }
} }
@@ -482,7 +465,7 @@ func (t *tun) Activate() error {
t.deviceIndex = link.Attrs().Index t.deviceIndex = link.Attrs().Index
// Setup our default MTU // Setup our default MTU
t.setMTU(nil) t.setMTU()
// Set the transmit queue length // Set the transmit queue length
ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)} ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)}
@@ -514,7 +497,7 @@ func (t *tun) Activate() error {
} }
// Set the routes // Set the routes
if err = t.addRoutes(false, nil); err != nil { if err = t.addRoutes(false); err != nil {
return err return err
} }
@@ -527,13 +510,12 @@ func (t *tun) Activate() error {
return nil return nil
} }
func (t *tun) setMTU(r *config.Reload) { func (t *tun) setMTU() {
// Set the MTU on the device // Set the MTU on the device
ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)} ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)}
if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil { if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
// This is currently a non fatal condition because the route table must have the MTU set appropriately as well // This is currently a non fatal condition because the route table must have the MTU set appropriately as well
t.l.Error("Failed to set tun mtu", "error", err) t.l.Error("Failed to set tun mtu", "error", err)
r.Failed()
} }
} }
@@ -586,7 +568,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
return nil return nil
} }
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { func (t *tun) addRoutes(logErrors bool) error {
// Path routes // Path routes
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -616,7 +598,6 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -628,7 +609,7 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) { func (t *tun) removeRoutes(routes []Route) {
for _, r := range routes { for _, r := range routes {
if !r.Install { if !r.Install {
continue continue
@@ -652,12 +633,8 @@ func (t *tun) removeRoutes(routes []Route, rl *config.Reload) {
} }
err := netlink.RouteDel(&nr) err := netlink.RouteDel(&nr)
if errors.Is(err, unix.ESRCH) { if err != nil {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
@@ -832,11 +809,6 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
t.routeTree.Store(newTree) t.routeTree.Store(newTree)
} }
// Shutdown wakes the tun readers without releasing their fds, Close releases them
func (t *tun) Shutdown() error {
return t.readers.Shutdown()
}
func (t *tun) Close() error { func (t *tun) Close() error {
t.closeLock.Lock() t.closeLock.Lock()
defer t.closeLock.Unlock() defer t.closeLock.Unlock()
+27 -27
View File
@@ -8,6 +8,7 @@ import (
"log/slog" "log/slog"
"net/netip" "net/netip"
"os" "os"
"regexp"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -66,12 +67,24 @@ type tun struct {
fd int fd int
} }
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
return nil, fmt.Errorf("newTunFromFd not supported in NetBSD") return nil, fmt.Errorf("newTunFromFd not supported in NetBSD")
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
fd, deviceName, err := openTun(l, c.GetString("tun.dev", "")) // Try to open tun device
var err error
deviceName := c.GetString("tun.dev", "")
if deviceName == "" {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
if !deviceNameRE.MatchString(deviceName) {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -86,24 +99,19 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
fd: fd, fd: fd,
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
l: l, l: l,
} }
err = t.reload(c, true, nil) err = t.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("tun.mtu") err := t.reload(c, false)
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -316,7 +324,7 @@ func (t *tun) Activate() error {
} }
} }
return t.addRoutes(false, nil) return t.addRoutes(false)
} }
func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
@@ -331,8 +339,8 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
return err return err
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -352,18 +360,16 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -387,7 +393,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
} }
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { func (t *tun) addRoutes(logErrors bool) error {
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -401,7 +407,6 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -413,20 +418,15 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.vpnNetworks) err := delRoute(r.Cidr, t.vpnNetworks)
if errors.Is(err, unix.ESRCH) { if err != nil {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
+27 -27
View File
@@ -8,6 +8,7 @@ import (
"log/slog" "log/slog"
"net/netip" "net/netip"
"os" "os"
"regexp"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -57,12 +58,24 @@ type tun struct {
fd int fd int
} }
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
return nil, fmt.Errorf("newTunFromFd not supported in openbsd") return nil, fmt.Errorf("newTunFromFd not supported in openbsd")
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
fd, deviceName, err := openTun(l, c.GetString("tun.dev", "")) // Try to open tun device
var err error
deviceName := c.GetString("tun.dev", "")
if deviceName == "" {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
if !deviceNameRE.MatchString(deviceName) {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -77,24 +90,19 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
fd: fd, fd: fd,
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
l: l, l: l,
} }
err = t.reload(c, true, nil) err = t.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("tun.mtu") err := t.reload(c, false)
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
@@ -295,7 +303,7 @@ func (t *tun) Activate() error {
} }
} }
return t.addRoutes(false, nil) return t.addRoutes(false)
} }
func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
@@ -310,8 +318,8 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
return err return err
} }
func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -331,18 +339,16 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -366,7 +372,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
} }
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { func (t *tun) addRoutes(logErrors bool) error {
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -380,7 +386,6 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
} else { } else {
return retErr return retErr
} }
@@ -392,20 +397,15 @@ func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
return nil return nil
} }
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.vpnNetworks) err := delRoute(r.Cidr, t.vpnNetworks)
if errors.Is(err, unix.ESRCH) { if err != nil {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
-107
View File
@@ -1,107 +0,0 @@
//go:build !e2e_testing
package overlay
import (
"net/netip"
"os"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Only tun.mtu changes. The system calls fail without a real device, what matters is the routes nebula keeps
func TestReloadMTUOnlyKeepsRoutes(t *testing.T) {
routes := " unsafe_routes:\n - route: 192.168.100.0/24\n via: 10.1.0.1\n"
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n mtu: 1300\n"+routes))
tn := &tun{
l: test.NewLogger(),
vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")},
ioctlFd: ^uintptr(0),
routesFromSystem: map[netip.Prefix]routing.Gateways{},
}
require.NoError(t, tn.reload(c, true, nil))
require.NoError(t, c.ReloadConfigString("tun:\n mtu: 1400\n"+routes).Err)
_ = tn.reload(c, false, nil)
got := *tn.Routes.Load()
require.Len(t, got, 1)
assert.Equal(t, netip.MustParsePrefix("192.168.100.0/24"), got[0].Cidr)
assert.Equal(t, 1400, got[0].MTU)
assert.Equal(t, 1400, tn.DefaultMTU)
}
// The device mtu has to fit the largest route mtu, wherever it sits in the list
func TestReloadMaxMTUIsTheLargestRoute(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n mtu: 1300\n unsafe_routes:\n"+
" - route: 192.168.100.0/24\n via: 10.1.0.1\n mtu: 1500\n"+
" - route: 192.168.101.0/24\n via: 10.1.0.1\n mtu: 1400\n"))
tn := &tun{
l: test.NewLogger(),
vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")},
ioctlFd: ^uintptr(0),
routesFromSystem: map[netip.Prefix]routing.Gateways{},
}
require.NoError(t, tn.reload(c, true, nil))
assert.Equal(t, 1500, tn.MaxMTU)
assert.Equal(t, 1300, tn.DefaultMTU)
}
// There is no device behind this tun, so adding a route always fails. Removing one is refused without root, as root the
// kernel answers that the route isn't there, which is what we wanted
func TestReloadRouteFailures(t *testing.T) {
route := "tun:\n mtu: 1300\n unsafe_routes:\n - route: 192.168.100.0/24\n via: 10.1.0.1\n"
cases := []struct {
name string
from, to string
wantFails bool
}{
{"added", "tun:\n mtu: 1300\n", route, true},
{"removed", route, "tun:\n mtu: 1300\n", os.Geteuid() != 0},
{"unchanged", route, route + "punchy:\n punch: true\n", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(tc.from))
tn := &tun{
l: test.NewLogger(),
vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")},
ioctlFd: ^uintptr(0),
routesFromSystem: map[netip.Prefix]routing.Gateways{},
}
require.NoError(t, tn.reload(c, true, nil))
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
require.NoError(t, tn.reload(c, false, r))
})
res := c.ReloadConfigString(tc.to)
require.NoError(t, res.Err)
assert.Equal(t, tc.wantFails, res.Failed)
})
}
}
// Keys only read when the device is created need a restart to apply
func TestTunReloadRestartKeys(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n dev: nebrk0\n"))
tn, err := NewDeviceFromConfig(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, 1)
if err != nil {
t.Skipf("can't create a tun device here: %v", err)
}
t.Cleanup(func() { _ = tn.Close() })
res := c.ReloadConfigString("tun:\n disabled: true\n dev: nebrk1\n tx_queue: 100\n")
require.NoError(t, res.Err)
assert.ElementsMatch(t, []string{"tun.disabled", "tun.dev", "tun.tx_queue"}, res.RestartKeys)
}
-451
View File
@@ -1,451 +0,0 @@
//go:build !ios && !e2e_testing
package overlay
import (
"errors"
"fmt"
"net"
"net/netip"
"os"
"strconv"
"sync/atomic"
"time"
"unsafe"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/util"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
// or until another interface takes the prefix, after which ours follows theirs like any other.
const (
// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
// replaces a route by deleting and adding, looking in between would find the prefix free
routeRetryDelay = time.Second
// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
// a message for each
routeSweepInterval = time.Minute
// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
routeSocketRcvBuf = 1 << 20
)
type routeStateKind int
const (
routeUnknown routeStateKind = iota
// routeInstalled is our route holding the prefix
routeInstalled
// routeYielded is another interface holding the prefix, ours is added once it lets go
routeYielded
// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
// another interface takes the prefix
routeRemoved
// routeFailed is adding our route failing for some other reason, it is retried
routeFailed
)
type routeState struct {
kind routeStateKind
// holder is the index of the interface holding the prefix when kind is routeYielded
holder int
}
// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
// is looked up as the message arrives, route(8) has exited by the time the table settles
type routeMover struct {
pid int32
name string
}
// logArgs appends m to a log line's args, when we know who it was
func (m routeMover) logArgs(args ...any) []any {
if m.pid == 0 {
return args
}
args = append(args, "pid", m.pid)
if m.name != "" {
args = append(args, "process", m.name)
}
return args
}
// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
func processName(pid int32) string {
kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
if err != nil || kp.Proc.P_pid != pid {
return ""
}
return unix.ByteSliceToString(kp.Proc.P_comm[:])
}
// installRoutes returns the routes we put in the system routing table, one per prefix
func installRoutes(routes []Route) []Route {
seen := make(map[netip.Prefix]struct{}, len(routes))
var out []Route
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
p := r.Cidr.Masked()
if _, ok := seen[p]; ok {
continue
}
seen[p] = struct{}{}
out = append(out, r)
}
return out
}
// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
// interface holds its prefix. It runs on start and on every reload.
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
var msgs []netroute.Message
var msgsErr error
fetched := false
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
err := addRoute(p, linkAddr)
if err == nil {
t.l.Info("Added route", "route", r)
states[p] = routeState{kind: routeInstalled}
continue
}
if !errors.Is(err, unix.EEXIST) {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if !logErrors {
return retErr
}
retErr.Log(t.l)
rl.Failed()
states[p] = routeState{kind: routeFailed}
continue
}
// Every route we added already exists on a reload, only warn about someone else's
if !fetched {
msgs, msgsErr = fetchRoutes()
fetched = true
}
if msgsErr != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
states[p] = prev
continue
}
switch holder := routeInterfaceIn(msgs, p); holder {
case linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case 0:
// Removed since the add, the watcher will see to it
states[p] = prev
default:
if prev.kind != routeYielded || prev.holder != holder {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
}
states[p] = routeState{kind: routeYielded, holder: holder}
}
}
t.routeStates = states
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
msgs, err := fetchRoutes()
if err != nil {
var left []netip.Prefix
for _, r := range installRoutes(routes) {
left = append(left, r.Cidr)
}
return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
}
for _, r := range installRoutes(routes) {
p := r.Cidr.Masked()
// Whatever happens next, the prefix is no longer one we know about
delete(t.routeStates, p)
if routeInterfaceIn(msgs, p) != linkAddr.Index {
continue
}
err = delRoute(p, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
// changed since it last looked, it runs every routeSweepEvery.
func (t *tun) reconcileRoutes() {
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return
}
t.routeMoversMu.Lock()
movers := t.routeMovers
t.routeMovers = nil
t.routeMoversMu.Unlock()
msgs, err := fetchRoutes()
if err != nil {
t.l.Error("Failed to read the system routing table", "error", err)
return
}
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
by := movers[p]
holder := routeInterfaceIn(msgs, p)
switch {
case holder == linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case holder != 0:
// Another interface taking a prefix we left alone puts ours back in line behind theirs
if prev.kind != routeYielded || prev.holder != holder {
t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
}
states[p] = routeState{kind: routeYielded, holder: holder}
case prev.kind == routeInstalled:
t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
states[p] = routeState{kind: routeRemoved}
case prev.kind == routeRemoved:
states[p] = prev
default:
err := addRoute(p, linkAddr)
switch {
case err == nil:
if prev.kind == routeYielded {
t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
} else {
t.l.Info("Added route", "route", r)
}
states[p] = routeState{kind: routeInstalled}
case errors.Is(err, unix.EEXIST):
// Taken since the fetch, the next look will see by whom
states[p] = prev
default:
if prev.kind != routeFailed {
util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
}
states[p] = routeState{kind: routeFailed}
}
}
}
t.routeStates = states
}
// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
// routing table settles after anything that may have moved a prefix
func (t *tun) startRouteWatch(watch *os.File) {
t.routesMu.Lock()
defer t.routesMu.Unlock()
if t.closed {
if watch != nil {
_ = watch.Close()
}
return
}
t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
if watch != nil {
t.routeWatch = watch
go t.watchRoutes(watch)
}
}
func (t *tun) sweepRoutes() {
// A table still changing gets reconciled once it settles
if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
t.reconcileRoutes()
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
if !t.closed {
t.routeSweep.Reset(t.routeSweepEvery)
}
}
func openRouteSocket() (*os.File, error) {
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
}
// Fewer drops, the sweep catches the rest
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
// Non-blocking puts it on the poller, so Close interrupts a Read
if err = unix.SetNonblock(fd, true); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
}
return os.NewFile(uintptr(fd), "route"), nil
}
// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
func (t *tun) watchRoutes(f *os.File) {
var reconciled atomic.Int64
settle := time.AfterFunc(time.Hour, func() {
reconciled.Store(time.Now().UnixNano())
t.reconcileRoutes()
})
settle.Stop()
defer settle.Stop()
var burst time.Time
self := int32(os.Getpid())
b := make([]byte, os.Getpagesize())
for {
n, err := f.Read(b)
if err != nil {
if !errors.Is(err, os.ErrClosed) {
t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
return
}
moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
if !moved {
continue
}
// Our own are already in routeStates
if p.IsValid() && pid != self {
t.routeMoversMu.Lock()
if t.routeMovers == nil {
t.routeMovers = make(map[netip.Prefix]routeMover)
}
t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
t.routeMoversMu.Unlock()
}
now := time.Now()
t.lastRouteEvent.Store(now.UnixNano())
if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
burst = now
}
settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
}
}
// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
// and the pid of the process that made it.
func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
if len(b) < 4 {
return false, p, 0
}
switch int(b[3]) {
case unix.RTM_IFINFO, unix.RTM_DELADDR:
return true, p, 0
case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
default:
return false, p, 0
}
if len(b) < unix.SizeofRtMsghdr {
return false, p, 0
}
hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
// A failed request is reported too, and a scoped route never held the prefix
if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
return false, p, 0
}
if hdr.Pid == 0 {
// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
// could be holding a prefix
return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
}
// Replies to a request carry the requester's padded sockaddrs
msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
if err != nil || len(msgs) != 1 {
return false, p, 0
}
m, ok := msgs[0].(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
return false, p, 0
}
p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
if !ok {
return false, netip.Prefix{}, 0
}
p = p.Masked()
for _, r := range installRoutes(routes) {
if r.Cidr.Masked() == p {
return true, p, hdr.Pid
}
}
return false, netip.Prefix{}, 0
}
// interfaceName is the name of the interface with index i, for logs
func interfaceName(i int) string {
if ifc, err := net.InterfaceByIndex(i); err == nil {
return ifc.Name
}
return strconv.Itoa(i)
}
-24
View File
@@ -1,24 +0,0 @@
package overlay
import (
"net/netip"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// tun.disabled and the disabled tun's settings are only read when the device is made
func TestDisabledTunRestartKeys(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n disabled: true\n"))
d, err := NewDeviceFromConfig(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, 1)
require.NoError(t, err)
t.Cleanup(func() { _ = d.Close() })
res := c.ReloadConfigString("tun:\n disabled: false\n tx_queue: 100\n")
require.NoError(t, res.Err)
assert.ElementsMatch(t, []string{"tun.disabled", "tun.tx_queue"}, res.RestartKeys)
}
+1 -1
View File
@@ -31,7 +31,7 @@ type TestTun struct {
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) {
_, routes, err := getAllRoutesFromConfig(l, c, vpnNetworks, true) _, routes, err := getAllRoutesFromConfig(c, vpnNetworks, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
+14 -106
View File
@@ -4,7 +4,6 @@ package overlay
import ( import (
"crypto" "crypto"
"errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net/netip" "net/netip"
@@ -76,7 +75,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
t := &winTun{ t := &winTun{
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
guid: *guid, guid: *guid,
networkCategory: cat, networkCategory: cat,
setCategory: setCat, setCategory: setCat,
@@ -84,7 +83,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
l: l, l: l,
} }
err = t.reload(c, true, nil) err = t.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -105,32 +104,18 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
} }
t.tun = tunDevice.(*wintun.NativeTun) t.tun = tunDevice.(*wintun.NativeTun)
// Disable registration before Activate assigns addresses and triggers it. c.RegisterReloadCallback(func(c *config.C) {
disableDNSRegistration(l, *guid) err := t.reload(c, false)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.dev", "tun.mtu")
// Compare with the running category, not the last config. A restart with one we can't parse would fail, so say
// that now rather than ask for one, and going back to the running one after a bad reload needs no restart
if cat, setCat, err := parseNetworkCategory(c.GetString("tun.network_category", "private")); err != nil {
t.l.Error("Invalid tun.network_category, a restart with it would fail", "error", err)
r.Failed()
} else if cat != t.networkCategory || setCat != t.setCategory {
r.RestartRequiredIfChanged("tun.network_category")
}
r.RestartRequiredIfChanged(interfaceBypassKeys...)
err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
r.Failed()
} }
}) })
return t, nil return t, nil
} }
func (t *winTun) reload(c *config.C, initial bool, r *config.Reload) error { func (t *winTun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -150,18 +135,16 @@ func (t *winTun) reload(c *config.C, initial bool, r *config.Reload) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes), r) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
r.Failed()
} }
// Ensure any routes we actually want are installed // Ensure any routes we actually want are installed
err = t.addRoutes(true, r) err = t.addRoutes(true)
if err != nil { if err != nil {
// Catch any stray logs // Catch any stray logs
util.LogWithContextIfNeeded("Failed to add routes", err, t.l) util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
r.Failed()
} }
} }
@@ -176,7 +159,7 @@ func (t *winTun) Activate() error {
return fmt.Errorf("failed to set address: %w", err) return fmt.Errorf("failed to set address: %w", err)
} }
err = t.addRoutes(false, nil) err = t.addRoutes(false)
if err != nil { if err != nil {
return err return err
} }
@@ -195,49 +178,33 @@ func (t *winTun) Activate() error {
return nil return nil
} }
func (t *winTun) addRoutes(logErrors bool, rl *config.Reload) error { func (t *winTun) addRoutes(logErrors bool) error {
luid := winipcfg.LUID(t.tun.LUID()) luid := winipcfg.LUID(t.tun.LUID())
routes := *t.Routes.Load() routes := *t.Routes.Load()
foundDefault4 := false foundDefault4 := false
carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() }) carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() })
// A CIDR listed twice routes by its last entry, install only that one or each reload would flip between them. The for _, r := range routes {
// same rule as systemRoutes
last := make(map[netip.Prefix]int, len(routes))
for i, r := range routes {
last[r.Cidr] = i
}
for i, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via // We don't allow route MTUs so only install routes with a via
continue continue
} }
if last[r.Cidr] != i {
continue
}
// A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter. // A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter.
carriesV6 = carriesV6 || r.Cidr.Addr().Is6() carriesV6 = carriesV6 || r.Cidr.Addr().Is6()
// Add our unsafe route as an on-link route to the nebula tun device. // Add our unsafe route as an on-link route to the nebula tun device.
result, err := addRoute(luid, r) err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric))
if err != nil { if err != nil {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
retErr.Log(t.l) retErr.Log(t.l)
rl.Failed()
continue continue
} else { } else {
return retErr return retErr
} }
} else { } else {
switch result {
case routeAdded:
t.l.Info("Added route", "route", r) t.l.Info("Added route", "route", r)
case routeMetricUpdated:
t.l.Info("Updated route metric", "route", r)
}
} }
if !foundDefault4 { if !foundDefault4 {
@@ -288,7 +255,7 @@ func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error
return nil return nil
} }
func (t *winTun) removeRoutes(routes []Route, rl *config.Reload) error { func (t *winTun) removeRoutes(routes []Route) error {
luid := winipcfg.LUID(t.tun.LUID()) luid := winipcfg.LUID(t.tun.LUID())
for _, r := range routes { for _, r := range routes {
@@ -298,12 +265,8 @@ func (t *winTun) removeRoutes(routes []Route, rl *config.Reload) error {
// See comment on luid.AddRoute // See comment on luid.AddRoute
err := luid.DeleteRoute(r.Cidr, unspecifiedNextHop(r.Cidr)) err := luid.DeleteRoute(r.Cidr, unspecifiedNextHop(r.Cidr))
if errors.Is(err, windows.ERROR_NOT_FOUND) { if err != nil {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else { } else {
t.l.Info("Removed route", "route", r) t.l.Info("Removed route", "route", r)
} }
@@ -353,61 +316,6 @@ func (t *winTun) Close() error {
return t.tun.Close() return t.tun.Close()
} }
// systemRoutes maps each CIDR to the route that puts it in the system table. A CIDR's last entry decides, including
// one that says not to install it
func systemRoutes(routes []Route) map[netip.Prefix]Route {
out := make(map[netip.Prefix]Route, len(routes))
for _, r := range routes {
if len(r.Via) > 0 && r.Install {
out[r.Cidr] = r
} else {
delete(out, r.Cidr)
}
}
return out
}
// findRemovedSystemRoutes is the routes to delete from the system table. Windows deletes by destination and next hop,
// so only a CIDR nothing installs anymore goes, a changed metric is set on the route in place by addRoute
func findRemovedSystemRoutes(newRoutes, oldRoutes []Route) []Route {
keep := systemRoutes(newRoutes)
var removed []Route
for _, r := range systemRoutes(oldRoutes) {
if _, ok := keep[r.Cidr]; !ok {
removed = append(removed, r)
}
}
return removed
}
type routeResult int
const (
routeAdded routeResult = iota
routeMetricUpdated
routeAlreadyInstalled
)
// addRoute installs r on the interface. A reload re-adds every route to catch any the system dropped, and Windows
// refuses one it has with the same destination and next hop whatever its metric. One that is there with another
// metric, ours from before a metric change or someone else's, is brought to ours in place
func addRoute(luid winipcfg.LUID, r Route) (routeResult, error) {
nextHop := unspecifiedNextHop(r.Cidr)
err := luid.AddRoute(r.Cidr, nextHop, uint32(r.Metric))
if !errors.Is(err, windows.ERROR_OBJECT_ALREADY_EXISTS) {
return routeAdded, err
}
row, err := luid.Route(r.Cidr, nextHop)
if err != nil {
return routeAlreadyInstalled, err
}
if row.Metric != uint32(r.Metric) {
row.Metric = uint32(r.Metric)
return routeMetricUpdated, row.Set()
}
return routeAlreadyInstalled, nil
}
func unspecifiedNextHop(p netip.Prefix) netip.Addr { func unspecifiedNextHop(p netip.Prefix) netip.Addr {
if p.Addr().Is4() { if p.Addr().Is4() {
return netip.IPv4Unspecified() return netip.IPv4Unspecified()
-209
View File
@@ -1,209 +0,0 @@
//go:build windows && !e2e_testing
package overlay
import (
"bytes"
"fmt"
"net/netip"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/windows"
"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
)
type routeTest struct {
c *config.C
tun *winTun
luid winipcfg.LUID
log *bytes.Buffer
}
// newRouteTest makes a wintun device of its own, before installing tun.unsafe_routes from cfg
func newRouteTest(t *testing.T, cfg string) *routeTest {
t.Helper()
if !windows.GetCurrentProcessToken().IsElevated() {
t.Skip("creating a wintun device needs an elevated process")
}
if err := checkWinTunExists(); err != nil {
t.Skipf("wintun.dll is not where nebula looks for it: %v", err)
}
rt := &routeTest{log: &bytes.Buffer{}}
l := test.NewLoggerWithOutput(rt.log)
rt.c = config.NewC(l)
require.NoError(t, rt.c.LoadString(cfg))
var err error
rt.tun, err = newTun(rt.c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = rt.tun.Close() })
rt.luid = winipcfg.LUID(rt.tun.tun.LUID())
return rt
}
func (rt *routeTest) metric(t *testing.T, cidr string) uint32 {
t.Helper()
p := netip.MustParsePrefix(cidr)
row, err := rt.luid.Route(p, unspecifiedNextHop(p))
require.NoError(t, err)
return row.Metric
}
// reload applies cfg and expects nothing in it to fail
func (rt *routeTest) reload(t *testing.T, cfg string) {
t.Helper()
res := rt.c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed, "the reload reported a failure")
}
func unsafeRoutes(routes ...string) string {
var b strings.Builder
b.WriteString("tun:\n dev: nebroutetest\n unsafe_routes:\n")
// cidr[@metric][!], the ! for install: false
for _, r := range routes {
r, noInstall := strings.CutSuffix(r, "!")
cidr, metric, _ := strings.Cut(r, "@")
fmt.Fprintf(&b, " - route: %s\n via: 10.250.0.2\n", cidr)
if metric != "" {
fmt.Fprintf(&b, " metric: %s\n", metric)
}
if noInstall {
b.WriteString(" install: false\n")
}
}
return b.String()
}
// A reload that adds a route re-adds the ones already installed, Windows refusing those as already there isn't a failure
func TestWinTun_ReloadKeepsInstalledRoutes(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a))
require.NoError(t, rt.tun.Activate())
rt.reload(t, unsafeRoutes(a, b))
assert.NotContains(t, rt.log.String(), "Failed to add route")
rt.metric(t, a)
rt.metric(t, b)
})
}
// Windows calls a route the same by destination and next hop, so a changed metric is set on the installed route in
// place rather than deleted and added, which would leave a moment with no route
func TestWinTun_ReloadMetricChangeApplies(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
require.NoError(t, rt.tun.Activate())
require.Equal(t, uint32(100), rt.metric(t, a))
rt.reload(t, unsafeRoutes(a+"@200"))
assert.NotContains(t, rt.log.String(), "Failed to add route")
assert.NotContains(t, rt.log.String(), "Removed route")
assert.Equal(t, uint32(200), rt.metric(t, a))
})
}
// A route someone else put on the device with another metric is brought to ours, at startup too
func TestWinTun_ForeignRouteGetsOurMetric(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
p := netip.MustParsePrefix(a)
require.NoError(t, rt.luid.AddRoute(p, unspecifiedNextHop(p), 50))
require.NoError(t, rt.tun.Activate())
assert.Equal(t, uint32(100), rt.metric(t, a))
assert.Contains(t, rt.log.String(), "level=INFO msg=\"Updated route metric\"")
})
}
// A CIDR listed twice is installed once, by its last entry, and reloads leave it there instead of flipping between the
// two metrics
func TestWinTun_DuplicateCidrInstallsTheLastEntry(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200"))
require.NoError(t, rt.tun.Activate())
assert.Equal(t, uint32(200), rt.metric(t, a))
rt.reload(t, unsafeRoutes(a+"@100", a+"@200", b))
assert.Equal(t, uint32(200), rt.metric(t, a))
assert.NotContains(t, rt.log.String(), "Updated route metric")
assert.NotContains(t, rt.log.String(), "Failed to add route")
})
}
// Dropping the duplicate that wasn't installed leaves the installed route alone. Windows deletes by destination and
// next hop, deleting the dropped one would take the live route with it
func TestWinTun_DroppingAnUninstalledDuplicateKeepsTheRoute(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200"))
require.NoError(t, rt.tun.Activate())
rt.reload(t, unsafeRoutes(a+"@200"))
assert.NotContains(t, rt.log.String(), "Removed route")
assert.Equal(t, uint32(200), rt.metric(t, a))
})
}
// A reload ends where a fresh start with the same config would, a CIDR whose last entry says not to install it is
// removed
func TestWinTun_ReloadMatchesAFreshStart(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
cidr := netip.MustParsePrefix(a)
routes := unsafeRoutes(a+"@100", a+"@100!")
fresh := newRouteTest(t, routes)
require.NoError(t, fresh.tun.Activate())
_, err := fresh.luid.Route(cidr, unspecifiedNextHop(cidr))
require.Error(t, err, "a fresh start installed it")
require.NoError(t, fresh.tun.Close())
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
require.NoError(t, rt.tun.Activate())
rt.reload(t, routes)
_, err = rt.luid.Route(cidr, unspecifiedNextHop(cidr))
assert.Error(t, err, "the reload left it installed")
})
}
// Removing a route that is already gone is what we wanted, not a failure
func TestWinTun_ReloadRouteAlreadyGone(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a, b))
require.NoError(t, rt.tun.Activate())
p := netip.MustParsePrefix(b)
require.NoError(t, rt.luid.DeleteRoute(p, unspecifiedNextHop(p)))
rt.reload(t, unsafeRoutes(a))
})
}
// A network category we can't parse would fail the restart, so it fails the reload instead of asking for one
func TestWinTun_ReloadInvalidNetworkCategoryFails(t *testing.T) {
rt := newRouteTest(t, unsafeRoutes())
for range 2 {
res := rt.c.ReloadConfigString(unsafeRoutes() + " network_category: nope\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed, "a bad category that didn't change since the last reload is still bad")
assert.False(t, res.RestartRequired)
}
// Fixing it back to the running category needs nothing
res := rt.c.ReloadConfigString(unsafeRoutes() + " network_category: private\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired)
res = rt.c.ReloadConfigString(unsafeRoutes() + " network_category: public\n")
require.NoError(t, res.Err)
assert.Equal(t, []string{"tun.network_category"}, res.RestartKeys)
}
// forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4
func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) {
t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") })
t.Run("ipv6", func(t *testing.T) { f(t, "fd00:251:1::/64", "fd00:251:2::/64") })
}
+18 -70
View File
@@ -2,9 +2,7 @@ package nebula
import ( import (
"crypto/fips140" "crypto/fips140"
"crypto/sha256"
"encoding/binary" "encoding/binary"
"encoding/hex"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@@ -52,30 +50,15 @@ type CertState struct {
myVpnBroadcastAddrsTable *bart.Lite myVpnBroadcastAddrsTable *bart.Lite
} }
// errCertNeedsRestart marks a new cert that is refused on reload because only a restart can apply it
var errCertNeedsRestart = errors.New("a restart is required to use the new cert")
func NewPKIFromConfig(l *slog.Logger, c *config.C) (*PKI, error) { func NewPKIFromConfig(l *slog.Logger, c *config.C) (*PKI, error) {
pki := &PKI{l: l} pki := &PKI{l: l}
err := pki.reload(c, true, nil) err := pki.reload(c, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
// Compare with the running cipher, not the last config. A restart with one we can't run would fail, so say that rErr := pki.reload(c, false)
// now rather than ask for one, and going back to the running one after a bad reload needs no restart
if cipher := c.GetString("cipher", "aes"); !validCipher(cipher) {
l.Error("Unknown cipher, a restart with it would fail", "cipher", cipher)
r.Failed()
} else if cipher == "chachapoly" && fipsOnly() {
// newCipherSuite refuses it at startup
l.Error("ChaChaPoly is not allowed in FIPS 140-only mode, a restart with it would fail", "cipher", cipher)
r.Failed()
} else if cipher != pki.getCertState().cipher {
r.RestartRequiredIfChanged("cipher")
}
rErr := pki.reload(c, false, r)
if rErr != nil { if rErr != nil {
util.LogWithContextIfNeeded("Failed to reload PKI from config", rErr, l) util.LogWithContextIfNeeded("Failed to reload PKI from config", rErr, l)
} }
@@ -92,29 +75,13 @@ func (p *PKI) getCertState() *CertState {
return p.cs.Load() return p.cs.Load()
} }
// r is nil on the initial load func (p *PKI) reload(c *config.C, initial bool) error {
func (p *PKI) reload(c *config.C, initial bool, r *config.Reload) error {
before := p.cs.Load()
err := p.reloadCerts(c, initial) err := p.reloadCerts(c, initial)
if err != nil { if err != nil {
if initial { if initial {
return err return err
} }
err.Log(p.l) err.Log(p.l)
if errors.Is(err, errCertNeedsRestart) {
r.RestartRequired()
} else {
r.Failed()
}
} else if !initial {
// A cert for a version we weren't running is accepted without comparing its networks to ours. The tun and the
// lighthouse keep the networks they started with
after := p.cs.Load()
if !slices.Equal(before.myVpnNetworks, after.myVpnNetworks) {
p.l.Warn("The new cert changes our networks, a restart is required to use them",
"old_networks", before.myVpnNetworks, "new_networks", after.myVpnNetworks)
r.RestartRequired()
}
} }
err = p.reloadCAPool(c) err = p.reloadCAPool(c)
@@ -123,27 +90,21 @@ func (p *PKI) reload(c *config.C, initial bool, r *config.Reload) error {
return err return err
} }
err.Log(p.l) err.Log(p.l)
r.Failed()
} }
return nil return nil
} }
// fipsOnly is fips140.Enforced, a variable so a test can run the FIPS 140-only reload check without GODEBUG
var fipsOnly = fips140.Enforced
// validCipher is true for the ciphers nebula can run. Each post-handshake CipherState in noiseutil hardcodes its own
// nonce endianness, so there's nothing to set up for either.
func validCipher(cipher string) bool {
return cipher == "aes" || cipher == "chachapoly"
}
func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError { func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
var cipher string var cipher string
var currentState *CertState var currentState *CertState
if initial { if initial {
cipher = c.GetString("cipher", "aes") cipher = c.GetString("cipher", "aes")
if !validCipher(cipher) { switch cipher {
case "aes", "chachapoly":
// Each post-handshake CipherState in noiseutil hardcodes its own
// nonce endianness now, so there's nothing to set up here.
default:
return util.NewContextualError( return util.NewContextualError(
"unknown cipher", "unknown cipher",
m{"cipher": cipher}, m{"cipher": cipher},
@@ -171,7 +132,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Networks in new cert was different from old", "Networks in new cert was different from old",
m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks(), "cert_version": cert.Version1}, m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks(), "cert_version": cert.Version1},
errCertNeedsRestart, nil,
) )
} }
@@ -179,7 +140,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Curve in new v1 cert was different from old", "Curve in new v1 cert was different from old",
m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve(), "cert_version": cert.Version1}, m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve(), "cert_version": cert.Version1},
errCertNeedsRestart, nil,
) )
} }
} }
@@ -194,7 +155,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Networks in new cert was different from old", "Networks in new cert was different from old",
m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks(), "cert_version": cert.Version2}, m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks(), "cert_version": cert.Version2},
errCertNeedsRestart, nil,
) )
} }
@@ -202,7 +163,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Curve in new cert was different from old", "Curve in new cert was different from old",
m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve(), "cert_version": cert.Version2}, m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve(), "cert_version": cert.Version2},
errCertNeedsRestart, nil,
) )
} }
} }
@@ -217,7 +178,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
return util.NewContextualError( return util.NewContextualError(
"Removing a V2 cert is not permitted unless it has identical networks to the new V1 cert", "Removing a V2 cert is not permitted unless it has identical networks to the new V1 cert",
m{"new_v1_networks": newState.v1Cert.Networks(), "old_v2_networks": currentState.v2Cert.Networks()}, m{"new_v1_networks": newState.v1Cert.Networks(), "old_v2_networks": currentState.v2Cert.Networks()},
errCertNeedsRestart, nil,
) )
} }
} }
@@ -235,14 +196,12 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError { func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError {
caPool, err := loadCAPoolFromConfig(p.l, c) caPool, err := loadCAPoolFromConfig(p.l, c)
if caPool != nil {
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
}
if err != nil { if err != nil {
return util.NewContextualError("Failed to load ca from config", nil, err) return util.NewContextualError("Failed to load ca from config", nil, err)
} }
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
return nil return nil
} }
@@ -617,22 +576,11 @@ func loadCAPoolFromConfig(l *slog.Logger, c *config.C) (*cert.CAPool, error) {
bl := c.GetStringSlice("pki.blocklist", []string{}) bl := c.GetStringSlice("pki.blocklist", []string{})
if len(bl) > 0 { if len(bl) > 0 {
var bad []string for _, fp := range bl {
for i, raw := range bl {
// Fingerprints are compared as lowercase hex strings, a malformed entry would silently block nothing
fp := strings.ToLower(strings.TrimSpace(raw))
if b, err := hex.DecodeString(fp); err != nil || len(b) != sha256.Size {
bad = append(bad, fmt.Sprintf("entry %d %q", i+1, raw))
continue
}
caPool.BlocklistFingerprint(fp) caPool.BlocklistFingerprint(fp)
} }
l.Info("Blocklisted certificates", "fingerprintCount", len(bl)-len(bad)) l.Info("Blocklisted certificates", "fingerprintCount", len(bl))
if len(bad) > 0 {
// The valid entries are still returned so a failed reload blocks them, startup refuses the error
return caPool, fmt.Errorf("pki.blocklist entries are not hex certificate fingerprints: %s", strings.Join(bad, ", "))
}
} }
return caPool, nil return caPool, nil
-168
View File
@@ -1,17 +1,10 @@
package nebula package nebula
import ( import (
"crypto/fips140"
"fmt"
"net/netip"
"strings" "strings"
"testing" "testing"
"time"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -99,164 +92,3 @@ func TestNewCipherSuiteUnsupportedCurve(t *testing.T) {
assert.True(t, strings.HasPrefix(err.Error(), "unsupported curve:"), "got: %v", err) assert.True(t, strings.HasPrefix(err.Error(), "unsupported curve:"), "got: %v", err)
assert.Nil(t, cs) assert.Nil(t, cs)
} }
func TestPKIReloadRestartRequired(t *testing.T) {
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
pkiConfig := func(cipher, network string) string {
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
return fmt.Sprintf("cipher: %s\npki:\n ca: %q\n cert: %q\n key: %q\n", cipher, caPEM, certPEM, keyPEM)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig("aes", "10.1.0.2/24")))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
// A cert that changes our networks is refused and needs a restart, it is not a failure
res := c.ReloadConfigString(pkiConfig("aes", "10.1.0.3/24"))
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
assert.False(t, res.Failed)
assert.Equal(t, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, p.getCertState().myVpnNetworks)
// The cipher only applies at startup
c2 := config.NewC(test.NewLogger())
require.NoError(t, c2.LoadString(pkiConfig("aes", "10.1.0.2/24")))
p, err = NewPKIFromConfig(test.NewLogger(), c2)
require.NoError(t, err)
res = c2.ReloadConfigString(pkiConfig("chachapoly", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
assert.Equal(t, "aes", p.getCertState().cipher)
// A cipher we can't run would fail the restart, so it fails the reload instead of asking for one
c4 := config.NewC(test.NewLogger())
require.NoError(t, c4.LoadString(pkiConfig("aes", "10.1.0.2/24")))
_, err = NewPKIFromConfig(test.NewLogger(), c4)
require.NoError(t, err)
for range 2 {
res = c4.ReloadConfigString(pkiConfig("nope", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.True(t, res.Failed, "a bad cipher that didn't change since the last reload is still bad")
assert.False(t, res.RestartRequired)
}
// Fixing it back to the running cipher needs nothing
res = c4.ReloadConfigString(pkiConfig("aes", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired)
// A cert that keeps our networks applies without a restart
c3 := config.NewC(test.NewLogger())
require.NoError(t, c3.LoadString(pkiConfig("aes", "10.1.0.2/24")))
_, err = NewPKIFromConfig(test.NewLogger(), c3)
require.NoError(t, err)
res = c3.ReloadConfigString(pkiConfig("aes", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.False(t, res.RestartRequired)
// A cert that can't be read is a failure, and a restart wouldn't help
res = c3.ReloadConfigString(strings.Replace(pkiConfig("aes", "10.1.0.2/24"), "cert: \"", "cert: \"garbage", 1))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.False(t, res.RestartRequired)
}
// A cert for a version we weren't running is accepted without comparing networks, so switching to one with different
// networks needs a restart. One with the same networks doesn't
func TestPKIReloadNewVersionWithOtherNetworks(t *testing.T) {
ca1, _, ca1Key, ca1PEM := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
ca2, _, ca2Key, ca2PEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
caPEM := string(ca1PEM) + string(ca2PEM)
pkiConfig := func(v cert.Version, network string) string {
ca, caKey := ca1, ca1Key
if v == cert.Version2 {
ca, caKey = ca2, ca2Key
}
_, _, keyPEM, certPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
return fmt.Sprintf("pki:\n ca: %q\n cert: %q\n key: %q\n", caPEM, certPEM, keyPEM)
}
for _, tc := range []struct {
network string
restart bool
}{
{"10.9.0.7/24", true},
{"10.1.0.2/24", false},
} {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig(cert.Version1, "10.1.0.2/24")))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig(cert.Version2, tc.network))
require.NoError(t, res.Err)
assert.False(t, res.Failed, tc.network)
assert.Equal(t, tc.restart, res.RestartRequired, tc.network)
assert.Equal(t, []netip.Prefix{netip.MustParsePrefix(tc.network)}, p.getCertState().myVpnNetworks, "accepted either way, as before")
}
}
// FIPS 140-only mode refuses chachapoly at startup, so reloading to it fails rather than asks for a restart
func TestPKIReloadChachapolyInFIPSOnlyFails(t *testing.T) {
fipsOnly = func() bool { return true }
t.Cleanup(func() { fipsOnly = fips140.Enforced })
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_P256, time.Time{}, time.Time{}, nil, nil, nil)
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_P256, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil)
pkiConfig := func(cipher string) string {
return fmt.Sprintf("cipher: %s\npki:\n ca: %q\n cert: %q\n key: %q\n", cipher, caPEM, certPEM, keyPEM)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig("aes")))
_, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig("chachapoly"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.False(t, res.RestartRequired)
}
func TestLoadCAPoolBlocklist(t *testing.T) {
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
crt, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil)
fp, err := crt.Fingerprint()
require.NoError(t, err)
load := func(entry string) (*cert.CAPool, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(fmt.Sprintf("pki:\n ca: %q\n blocklist:\n - %q\n", caPEM, entry)))
return loadCAPoolFromConfig(test.NewLogger(), c)
}
for _, entry := range []string{fp, strings.ToUpper(fp), " " + fp + "\t"} {
pool, err := load(entry)
require.NoError(t, err, entry)
assert.True(t, pool.IsBlocklisted(fp), entry)
}
for _, entry := range []string{fp[1:], fp + "00", "sha256:" + fp, "zz" + fp[2:], ""} {
_, err := load(entry)
require.Error(t, err, entry)
}
// A bad entry on reload fails it, the valid entries are still blocked
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "me", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.1/24")}, nil, nil)
pkiConfig := func(entry string) string {
return fmt.Sprintf("pki:\n ca: %q\n cert: %q\n key: %q\n blocklist:\n - %q\n", caPEM, certPEM, keyPEM, entry)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig(fp)))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
other, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "other", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.3/24")}, nil, nil)
otherFp, err := other.Fingerprint()
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig(fp) + fmt.Sprintf(" - %q\n - %q\n", otherFp, fp[1:]))
assert.True(t, res.Failed)
assert.True(t, p.GetCAPool().IsBlocklisted(fp))
assert.True(t, p.GetCAPool().IsBlocklisted(otherFp))
}
+1 -2
View File
@@ -67,8 +67,7 @@ func NewPunchyFromConfig(l *slog.Logger, c *config.C, punchConn udp.Conn) *Punch
} }
p.reload(c, true) p.reload(c, true)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C) {
r.RestartRequiredIfChanged("stats.lighthouse_metrics")
p.reload(c, false) p.reload(c, false)
}) })
+6 -16
View File
@@ -76,7 +76,7 @@ punchy:
punch: true punch: true
delay: 10m delay: 10m
respond: true respond: true
`).Err) `))
p.reload(c, false) p.reload(c, false)
assert.True(t, p.punch.Load()) assert.True(t, p.punch.Load())
assert.Equal(t, newDelay, time.Duration(p.delay.Load())) assert.Equal(t, newDelay, time.Duration(p.delay.Load()))
@@ -170,7 +170,7 @@ func TestPunchy_LogFormat_ReloadPunch(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {punch: true}`).Err) require.NoError(t, c.ReloadConfigString(`punchy: {punch: true}`))
entry := findEntry(t, hook.entries, "punchy.punch changed") entry := findEntry(t, hook.entries, "punchy.punch changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -184,7 +184,7 @@ func TestPunchy_LogFormat_ReloadRespond(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {respond: true}`).Err) require.NoError(t, c.ReloadConfigString(`punchy: {respond: true}`))
entry := findEntry(t, hook.entries, "punchy.respond changed") entry := findEntry(t, hook.entries, "punchy.respond changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -198,7 +198,7 @@ func TestPunchy_LogFormat_ReloadDelay(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {delay: 10s}`).Err) require.NoError(t, c.ReloadConfigString(`punchy: {delay: 10s}`))
entry := findEntry(t, hook.entries, "punchy.delay changed") entry := findEntry(t, hook.entries, "punchy.delay changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -212,7 +212,7 @@ func TestPunchy_LogFormat_ReloadTargetAllRemotes(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {target_all_remotes: true}`).Err) require.NoError(t, c.ReloadConfigString(`punchy: {target_all_remotes: true}`))
entry := findEntry(t, hook.entries, "punchy.target_all_remotes changed") entry := findEntry(t, hook.entries, "punchy.target_all_remotes changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
@@ -226,19 +226,9 @@ func TestPunchy_LogFormat_ReloadRespondDelay(t *testing.T) {
NewPunchyFromConfig(l, c, nil) NewPunchyFromConfig(l, c, nil)
hook.entries = nil hook.entries = nil
require.NoError(t, c.ReloadConfigString(`punchy: {respond_delay: 15s}`).Err) require.NoError(t, c.ReloadConfigString(`punchy: {respond_delay: 15s}`))
entry := findEntry(t, hook.entries, "punchy.respond_delay changed") entry := findEntry(t, hook.entries, "punchy.respond_delay changed")
assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, slog.LevelInfo, entry.Level)
assert.Equal(t, map[string]any{"respond_delay": 15 * time.Second}, entry.Attrs) assert.Equal(t, map[string]any{"respond_delay": 15 * time.Second}, entry.Attrs)
} }
func TestPunchyReloadRestartRequired(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("stats:\n lighthouse_metrics: false\n"))
NewPunchyFromConfig(test.NewLogger(), c, nil)
res := c.ReloadConfigString("stats:\n lighthouse_metrics: true\n")
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
}
-91
View File
@@ -1,91 +0,0 @@
package ratelimit
import (
"fmt"
"math"
"sync/atomic"
"time"
)
// Limiter is a token bucket: `burst` calls are allowed at once, then one more per `every`.
// It counts the calls it denies and hands that count to the next allowed call.
//
// The budget belongs to the Limiter instance.
// Every Allow that returns true spends from it, whatever the caller then does.
//
// Time follows Go's monotonic clock.
//
// Create one Limiter for each thing you want to limit,
// sharing one between unrelated events lets the noisy one starve the quiet one.
//
// Check Allow before doing the work being limited, so a denied call stays cheap:
//
// if denied, ok := probeLimiter.Allow(); ok {
// sendProbe()
// probesDenied.Inc(int64(denied))
// }
//
// For log lines use logging.RateLimited, which attaches the denied count to the logger as "suppressed".
type Limiter struct {
every int64
slack int64
tat atomic.Int64
denied atomic.Uint64
}
// maxSpan bounds `burst` * `every` to half of int64, leaving the other half for process uptime so the bucket
// math can't overflow.
const maxSpan = math.MaxInt64 / 2
// New returns a Limiter allowing `burst` calls at once and one more per `every`.
// A `burst` under 1 is 1, and an `every` of 0 or less never limits.
// It returns an error if `burst` * `every` is over about 146 years.
func New(burst int, every time.Duration) (*Limiter, error) {
l := &Limiter{}
if every > 0 {
burst = max(burst, 1)
if int64(burst) > maxSpan/int64(every) {
return nil, fmt.Errorf("ratelimit: burst %d * every %v is too long", burst, every)
}
l.every = int64(every)
l.slack = int64(burst-1) * l.every
}
return l, nil
}
var monoStart = time.Now()
func monoNow() int64 { return int64(time.Since(monoStart)) }
// Allow reports whether this call is allowed.
// If a token is available it spends it and returns the calls denied since the last allowed one, and true.
// Otherwise it returns 0 and false, and counts the denial.
// A nil Limiter allows everything.
func (l *Limiter) Allow() (uint64, bool) {
if l == nil {
return 0, true
}
return l.allow(monoNow())
}
// allow is Allow at a given monotonic time.
func (l *Limiter) allow(now int64) (uint64, bool) {
// Concurrent Allow calls read the clock before racing for the bucket, so one can see a time another already spent
// past and be denied. An unlimited Limiter must never deny, so it skips the bucket.
if l.every == 0 {
return 0, true
}
for {
tat := l.tat.Load()
next := max(tat, now)
if next-now > l.slack {
l.denied.Add(1)
return 0, false
}
if l.tat.CompareAndSwap(tat, next+l.every) {
return l.denied.Swap(0), true
}
}
}
Loaded 100 of 133 files, more files were not shown because too many files have changed in this diff. Show more