Compare commits

...
Author SHA1 Message Date
Ryan Huber d3e21a8fc8 relay: Batch forwarded packets on the relay host 2026-10-17 18:53:36 +00:00
Wade Simmons cda3710d5a fix error return in handleOutsideMessagePacket (#1970)
smoke-extra / freebsd-amd64 (push) Failing after 17s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 16s
smoke-extra / netbsd-amd64 (push) Failing after 16s
smoke-extra / openbsd-amd64 (push) Failing after 16s
smoke-extra / linux-386 (push) Failing after 17s
smoke / Run multi node smoke test (push) Failing after 1m39s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m18s
Build and test / Test linux-pkcs11 (push) Failing after 2m5s
Build and test / Test linux-boringcrypto (push) Failing after 3m1s
Build and test / Test linux-fips140 (push) Failing after 2m55s
Build and test / Cross-build linux-arm (push) Successful in 3m0s
Build and test / Cross-build linux-mips (push) Successful in 3m42s
Build and test / Cross-build linux-other (push) Successful in 3m5s
Build and test / Cross-build windows (push) Successful in 1m2s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m31s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m21s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
Introduced by #1961, we should still return early for invalid packets
2026-10-17 11:09:25 -04:00
John Maguire c26871fe08 service: Close listeners and tear down the netstack on shutdown (#1954) 2026-10-17 11:08:29 -04:00
Caleb Jasik 7daacc6a48 overlay/darwin: test tun framing and zero-alloc Read/Write (#1906)
smoke-extra / freebsd-amd64 (push) Failing after 16s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 16s
smoke-extra / netbsd-amd64 (push) Failing after 17s
smoke-extra / openbsd-amd64 (push) Failing after 17s
smoke-extra / linux-386 (push) Failing after 16s
smoke / Run multi node smoke test (push) Failing after 1m41s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m47s
Build and test / Test linux-pkcs11 (push) Failing after 2m6s
Build and test / Test linux-boringcrypto (push) Failing after 2m57s
Build and test / Test linux-fips140 (push) Failing after 2m53s
Build and test / Cross-build linux-arm (push) Successful in 3m7s
Build and test / Cross-build linux-mips (push) Successful in 3m55s
Build and test / Cross-build linux-other (push) Successful in 3m13s
Build and test / Cross-build windows (push) Successful in 1m4s
Build and test / Cross-build freebsd (push) Successful in 1m35s
Build and test / Cross-build netbsd (push) Successful in 1m35s
Build and test / Cross-build openbsd (push) Successful in 1m34s
Build and test / Cross-build mobile (push) Successful in 3m26s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-16 18:12:58 -05:00
John Maguire 428dad94b3 Reject v2 cert names and groups at signing the parser rejects (#1967)
smoke-extra / freebsd-amd64 (push) Failing after 18s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 18s
smoke-extra / netbsd-amd64 (push) Failing after 16s
smoke-extra / openbsd-amd64 (push) Failing after 17s
smoke-extra / linux-386 (push) Failing after 17s
smoke / Run multi node smoke test (push) Failing after 1m41s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m19s
Build and test / Test linux-pkcs11 (push) Failing after 2m5s
Build and test / Test linux-boringcrypto (push) Failing after 2m56s
Build and test / Test linux-fips140 (push) Failing after 2m51s
Build and test / Cross-build linux-arm (push) Successful in 3m0s
Build and test / Cross-build linux-mips (push) Successful in 3m43s
Build and test / Cross-build linux-other (push) Successful in 3m7s
Build and test / Cross-build windows (push) Successful in 1m2s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m33s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m23s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-16 18:05:51 -04:00
John Maguire 5bf666dd90 Treat uppercase blocked fingerprints the same as lowercase (#1964)
- Error for invalid on startup 
- Still block the valid blocklist entries when a reload has a bad one
2026-10-16 16:28:37 -04:00
John Maguire 49207da077 Remove the ssh debug profiling commands (#1965)
We plan to move this functionality into the debug socket.
2026-10-16 15:47:56 -04:00
Jack Doan bbc49669b7 fix noisy log line (#1961) 2026-10-16 12:08:55 -05:00
Nate Brown 915495d5f4 Fail the reload on a startup only value that would fail a restart, and stop asking for tun.dev restarts that change nothing (#1953)
smoke-extra / freebsd-amd64 (push) Failing after 25s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 13s
smoke-extra / netbsd-amd64 (push) Failing after 11s
smoke-extra / openbsd-amd64 (push) Failing after 12s
smoke-extra / linux-386 (push) Failing after 14s
smoke / Run multi node smoke test (push) Failing after 1m43s
Build and test / Static checks (push) Successful in 2m25s
Build and test / Test linux (push) Failing after 1m28s
Build and test / Test linux-pkcs11 (push) Failing after 2m5s
Build and test / Test linux-boringcrypto (push) Failing after 2m56s
Build and test / Test linux-fips140 (push) Failing after 3m3s
Build and test / Cross-build linux-arm (push) Successful in 3m14s
Build and test / Cross-build linux-mips (push) Successful in 3m48s
Build and test / Cross-build linux-other (push) Successful in 3m13s
Build and test / Cross-build windows (push) Successful in 1m3s
Build and test / Cross-build freebsd (push) Successful in 1m34s
Build and test / Cross-build netbsd (push) Successful in 1m36s
Build and test / Cross-build openbsd (push) Successful in 1m37s
Build and test / Cross-build mobile (push) Successful in 3m33s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
smoke-extra / Run windows smoke test (push) Canceled after 0s
2026-10-15 23:34:09 -05:00
Nate Brown a203d044bb Fail the reload when a config entry is skipped with a warning (#1952) 2026-10-15 23:22:47 -05:00
Nate Brown 316bf87780 Don't restart sshd when only sshd.sandbox_dir changes (#1948) 2026-10-15 22:12:12 -05:00
32 changed files with 1265 additions and 269 deletions

No files matched your search

+7
View File
@@ -7,8 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
### Removed
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
`mutex-profile-fraction` commands, and `sshd.sandbox_dir`, which is now ignored if set.
### Fixed ### Fixed
- Signing a v2 certificate with an empty name, a name longer than 253 bytes, or an empty group now fails.
These certificates were signed successfully but could not be loaded.
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula - `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
+10 -1
View File
@@ -389,7 +389,16 @@ func (c *certificateV2) fromTBSCertificate(t *TBSCertificate) error {
} }
func (c *certificateV2) validate() error { func (c *certificateV2) validate() error {
// Empty names are allowed // Mirror the name and group rules in unmarshalDetails so signing fails instead of producing an unparseable certificate
if c.details.name == "" || len(c.details.name) > MaxNameLength {
return NewErrInvalidCertificateProperties("name must be between 1 and %d bytes", MaxNameLength)
}
for _, group := range c.details.groups {
if group == "" {
return NewErrInvalidCertificateProperties("groups may not be empty")
}
}
if len(c.publicKey) == 0 { if len(c.publicKey) == 0 {
return ErrInvalidPublicKey return ErrInvalidPublicKey
+45
View File
@@ -6,6 +6,7 @@ import (
"crypto/elliptic" "crypto/elliptic"
"crypto/rand" "crypto/rand"
"net/netip" "net/netip"
"strings"
"testing" "testing"
"time" "time"
@@ -135,3 +136,47 @@ func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
assert.True(t, normie) assert.True(t, normie)
} }
} }
func TestCertificateV2_SignRejectsUnparseable(t *testing.T) {
_, priv, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
var invalid *ErrInvalidCertificateProperties
newTBS := func() TBSCertificate {
return TBSCertificate{
Version: Version2,
Name: "testing",
Networks: []netip.Prefix{mustParsePrefixUnmapped("10.1.1.1/24")},
Groups: []string{"test-group1"},
NotBefore: time.Now().Add(-time.Minute),
NotAfter: time.Now().Add(time.Minute),
PublicKey: []byte("1234567890abcedfghij1234567890ab"),
Curve: Curve_CURVE25519,
IsCA: true,
}
}
tbs := newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength)
c, err := tbs.Sign(nil, Curve_CURVE25519, priv)
require.NoError(t, err)
b, err := c.Marshal()
require.NoError(t, err)
_, err = unmarshalCertificateV2(b, nil, Curve_CURVE25519)
require.NoError(t, err)
tbs = newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength+1)
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Name = ""
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Groups = []string{"test-group1", ""}
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
}
+2 -8
View File
@@ -8,6 +8,7 @@ pki:
cert: /etc/nebula/host.crt cert: /etc/nebula/host.crt
key: /etc/nebula/host.key key: /etc/nebula/host.key
# blocklist is a list of certificate fingerprints that we will refuse to talk to # blocklist is a list of certificate fingerprints that we will refuse to talk to
# Entries are the hex fingerprints shown by `nebula-cert print`, an invalid entry fails startup and reload.
#blocklist: #blocklist:
# - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72 # - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72
# disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid. # disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid.
@@ -232,13 +233,6 @@ punchy:
# Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access. # Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
#trusted_cas: #trusted_cas:
#- "ssh public key string" #- "ssh public key string"
# sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
# save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
# and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
# The directory is NOT automatically created.
# Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
# Not reloadable.
#sandbox_dir: /var/tmp/nebula-debug
# EXPERIMENTAL: relay support for networks that can't establish direct connections. # EXPERIMENTAL: relay support for networks that can't establish direct connections.
relay: relay:
@@ -261,7 +255,7 @@ tun:
disabled: false disabled: false
# Name of the device. If not set, a default will be chosen by the OS. # Name of the device. If not set, a default will be chosen by the OS.
# For macOS: if set, must be in the form `utun[0-9]+`. # For macOS: if set, must be in the form `utun[0-9]+`.
# For NetBSD: Required to be set, must be in the form `tun[0-9]+` # For OpenBSD and NetBSD: if set, must be in the form `tun[0-9]+`, otherwise the first free one is used.
# Not reloadable. # Not reloadable.
dev: nebula1 dev: nebula1
# Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert # Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert
+7 -5
View File
@@ -316,9 +316,9 @@ type cachedPacketMetrics struct {
func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap { func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap {
hm := newHostMap(l) hm := newHostMap(l)
hm.reload(c, true) hm.reload(c, true, nil)
c.RegisterReloadCallback(func(c *config.C, _ *config.Reload) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
hm.reload(c, false) hm.reload(c, false, r)
}) })
l.Info("Main HostMap created", "preferredRanges", hm.GetPreferredRanges()) l.Info("Main HostMap created", "preferredRanges", hm.GetPreferredRanges())
@@ -337,7 +337,8 @@ func newHostMap(l *slog.Logger) *HostMap {
} }
} }
func (hm *HostMap) reload(c *config.C, initial bool) { // r is nil on the initial load
func (hm *HostMap) reload(c *config.C, initial bool, r *config.Reload) {
if initial || c.HasChanged("preferred_ranges") { if initial || c.HasChanged("preferred_ranges") {
var preferredRanges []netip.Prefix var preferredRanges []netip.Prefix
rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{}) rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{})
@@ -348,8 +349,9 @@ func (hm *HostMap) reload(c *config.C, initial bool) {
if err != nil { if err != nil {
hm.l.Warn("Failed to parse preferred ranges, ignoring", hm.l.Warn("Failed to parse preferred ranges, ignoring",
"error", err, "error", err,
"range", rawPreferredRanges, "range", rawPreferredRange,
) )
r.Failed()
continue continue
} }
+13
View File
@@ -447,3 +447,16 @@ func TestHostInfo_markOutOnly(t *testing.T) {
assert.True(t, h.markOut(6), "a relayed send must not consume the epoch edge") assert.True(t, h.markOut(6), "a relayed send must not consume the epoch edge")
assert.False(t, h.markOut(6)) assert.False(t, h.markOut(6))
} }
// A preferred range that doesn't parse is skipped and the rest are used, the reload says one was skipped
func TestHostMap_reloadBadPreferredRangeFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("preferred_ranges: [10.0.0.0/8]\n"))
hm := NewHostMapFromConfig(l, c)
res := c.ReloadConfigString("preferred_ranges: [10.0.0.0/8, nope]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, hm.GetPreferredRanges())
}
+3
View File
@@ -384,6 +384,8 @@ type rxContext struct {
hostmapCache map[uint32]*HostInfo hostmapCache map[uint32]*HostInfo
lhh *LightHouseHandler lhh *LightHouseHandler
ctCache *firewall.ConntrackCacheTicker ctCache *firewall.ConntrackCacheTicker
relayBatch *batch.SendBatch
relayRemote netip.AddrPort
} }
func newRxContext(f *Interface, q int) *rxContext { func newRxContext(f *Interface, q int) *rxContext {
@@ -414,6 +416,7 @@ func (f *Interface) listenOut(i int) {
} }
flusher := func() { flusher := func() {
f.flushRelayBatch(rxc)
if err := f.batchers[i].Flush(); err != nil { if err := f.batchers[i].Flush(); err != nil {
f.l.Error("Failed to flush tun coalescer", "error", err) f.l.Error("Failed to flush tun coalescer", "error", err)
} }
+6 -3
View File
@@ -138,14 +138,14 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
h.metrics = newLighthouseMetrics() h.metrics = newLighthouseMetrics()
} }
err := h.reload(c, true) err := h.reload(c, true, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("lighthouse.am_lighthouse", "listen.port", "handshakes.query_buffer", "stats.lighthouse_metrics") r.RestartRequiredIfChanged("lighthouse.am_lighthouse", "listen.port", "handshakes.query_buffer", "stats.lighthouse_metrics")
err := h.reload(c, false) err := h.reload(c, false, r)
switch v := err.(type) { switch v := err.(type) {
case *util.ContextualError: case *util.ContextualError:
v.Log(l) v.Log(l)
@@ -193,7 +193,8 @@ func (lh *LightHouse) GetUpdateInterval() int64 {
return lh.interval.Load() return lh.interval.Load()
} }
func (lh *LightHouse) reload(c *config.C, initial bool) error { // r is nil on the initial load
func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error {
if initial || c.HasChanged("lighthouse.advertise_addrs") { if initial || c.HasChanged("lighthouse.advertise_addrs") {
rawAdvAddrs := c.GetStringSlice("lighthouse.advertise_addrs", []string{}) rawAdvAddrs := c.GetStringSlice("lighthouse.advertise_addrs", []string{})
advAddrs := make([]netip.AddrPort, 0) advAddrs := make([]netip.AddrPort, 0)
@@ -228,6 +229,7 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
"addr", rawAddr, "addr", rawAddr,
"entry", i+1, "entry", i+1,
) )
r.Failed()
continue continue
} }
@@ -386,6 +388,7 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
"relay", v, "relay", v,
"error", err, "error", err,
) )
r.Failed()
} else { } else {
lh.l.Info("Read relay from config", "relay", v) lh.l.Info("Read relay from config", "relay", v)
relaysForMe = append(relaysForMe, configRIP) relaysForMe = append(relaysForMe, configRIP)
+29 -1
View File
@@ -296,7 +296,7 @@ func TestLighthouse_reload(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
c.ReloadConfigString(string(rc)) c.ReloadConfigString(string(rc))
err = lh.reload(c, false) err = lh.reload(c, false, nil)
require.NoError(t, err) require.NoError(t, err)
} }
@@ -1014,3 +1014,31 @@ func TestLighthouse_reloadRestartRequired(t *testing.T) {
require.NoError(t, res.Err) require.NoError(t, res.Err)
assert.True(t, res.RestartRequired) assert.True(t, res.RestartRequired)
} }
// A relay that doesn't parse is skipped and the rest are used, the reload says one was skipped
func TestLighthouse_reloadBadRelayFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("listen:\n port: 4242\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
res := c.ReloadConfigString("listen:\n port: 4242\nrelay:\n relays: [10.128.0.2, nope]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.2")}, lh.GetRelaysForMe())
}
// An advertise_addrs entry inside our own network is skipped and the rest are used, the reload says one was skipped
func TestLighthouse_reloadAdvertiseAddrInNetworkFails(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("listen:\n port: 4242\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
res := c.ReloadConfigString("listen:\n port: 4242\nlighthouse:\n advertise_addrs: [\"1.2.3.4:4242\", \"10.128.0.5:4242\"]\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.2.3.4:4242")}, lh.GetAdvertiseAddrs())
}
+1 -1
View File
@@ -78,7 +78,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
wireSSHReload(l, ssh, c) wireSSHReload(l, ssh, c)
var sshStart func() var sshStart func()
if c.GetBool("sshd.enabled", false) { if c.GetBool("sshd.enabled", false) {
sshStart, err = configSSH(l, ssh, c) sshStart, err = configSSH(l, ssh, c, nil)
if err != nil { if err != nil {
l.Warn("Failed to configure sshd, ssh debugging will not be available", "error", err) l.Warn("Failed to configure sshd, ssh debugging will not be available", "error", err)
sshStart = nil sshStart = nil
+37 -7
View File
@@ -25,7 +25,7 @@ var ErrOutOfWindow = errors.New("out of window packet")
// readOutsidePackets processes one received underlay packet. // readOutsidePackets processes one received underlay packet.
// Message payloads are decrypted IN PLACE, so packet must stay untouched // Message payloads are decrypted IN PLACE, so packet must stay untouched
// by the caller until the batcher for queue q has been flushed // by the caller until the TUN and relay batches for queue q have been flushed.
func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) { func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) {
h := rxc.h h := rxc.h
err := h.Parse(packet) err := h.Parse(packet)
@@ -77,6 +77,12 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
f.messageMetrics.Rx(h.Type, h.Subtype, 1) f.messageMetrics.Rx(h.Type, h.Subtype, 1)
} }
// Earlier forwards must leave before this packet can generate a response.
isMessageRelay := h.Type == header.Message && h.Subtype == header.MessageRelay
if !isMessageRelay {
f.flushRelayBatch(rxc)
}
// Unencrypted packets // Unencrypted packets
switch h.Type { switch h.Type {
case header.Handshake: case header.Handshake:
@@ -88,9 +94,6 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
return return
} }
// Relay packets are special
isMessageRelay := (h.Type == header.Message && h.Subtype == header.MessageRelay)
var hostinfo *HostInfo var hostinfo *HostInfo
if isMessageRelay { if isMessageRelay {
hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex) hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex)
@@ -102,6 +105,8 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
// recvError if necessary // recvError if necessary
if hostinfo == nil || hostinfo.ConnectionState == nil { if hostinfo == nil || hostinfo.ConnectionState == nil {
if !via.IsRelayed { if !via.IsRelayed {
// An unknown relay index can also produce a response.
f.flushRelayBatch(rxc)
f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex) f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex)
} }
return return
@@ -233,8 +238,10 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
// Forward this packet through the relay tunnel, rebuilding it in place. // Forward this packet through the relay tunnel, rebuilding it in place.
// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was // Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
fwdBuf := packet[:0] fwdBuf := packet[:0]
//todo it would potentially be nice to batch these out, err := f.prepareSendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true)
f.SendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true, rxc.q) if err == nil {
f.queueRelayPacket(out, targetHI.GetRemote(), rxc)
}
case TerminalType: case TerminalType:
hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal") hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
return return
@@ -259,6 +266,27 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
} }
} }
// queueRelayPacket borrows a receive buffer until the batch is flushed. Destination changes and the send batch
// limit end a run, preserving arrival order while allowing WriteBatch to use UDP segmentation offload.
func (f *Interface) queueRelayPacket(packet []byte, remote netip.AddrPort, rxc *rxContext) {
if rxc.relayBatch == nil {
rxc.relayBatch = batch.NewSendBatch(f.writers[rxc.q], batch.SendBatchCap, 0)
} else if rxc.relayRemote != remote {
f.flushRelayBatch(rxc)
}
rxc.relayRemote = remote
rxc.relayBatch.Commit(packet, remote)
if rxc.relayBatch.Len() >= batch.SendBatchCap {
f.flushRelayBatch(rxc)
}
}
func (f *Interface) flushRelayBatch(rxc *rxContext) {
if rxc.relayBatch != nil {
f.flushSendBatch(rxc.relayBatch, rxc.q)
}
}
// closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
func (f *Interface) closeTunnel(hostInfo *HostInfo) { func (f *Interface) closeTunnel(hostInfo *HostInfo) {
// Hold the hostmap lock across both the delete and the lighthouse eviction, // Hold the hostmap lock across both the delete and the lighthouse eviction,
@@ -476,7 +504,9 @@ func parseV4(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounter uint64, out []byte, rxc *rxContext) { func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounter uint64, out []byte, rxc *rxContext) {
err := newPacket(out, true, rxc.fwPacket) err := newPacket(out, true, rxc.fwPacket)
if err != nil { if err != nil {
hostinfo.logger(f.l).Warn("Error while validating inbound packet", "error", err, "packet", out) if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out)
}
return return
} }
+291
View File
@@ -0,0 +1,291 @@
package nebula
import (
"bytes"
"net/netip"
"sync"
"testing"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/require"
)
type relayBatchWrite struct {
packets [][]byte
remotes []netip.AddrPort
batched bool
}
type relayBatchConn struct {
udp.NoopConn
read func(udp.EncReader, func())
writes []relayBatchWrite
}
func (c *relayBatchConn) ListenOut(r udp.EncReader, flush func()) error {
c.read(r, flush)
return nil
}
func (c *relayBatchConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
w := relayBatchWrite{remotes: append([]netip.AddrPort(nil), addrs...), batched: true}
for _, b := range bufs {
w.packets = append(w.packets, bytes.Clone(b))
}
c.writes = append(c.writes, w)
return len(bufs), nil
}
func (c *relayBatchConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.writes = append(c.writes, relayBatchWrite{packets: [][]byte{bytes.Clone(b)}, remotes: []netip.AddrPort{addr}})
return nil
}
func (c *relayBatchConn) packets() [][]byte {
var packets [][]byte
for _, w := range c.writes {
packets = append(packets, w.packets...)
}
return packets
}
type relayBatchPath struct {
f *Interface
conn *relayBatchConn
source, target *HostInfo
incoming, outgoing *Relay
sourceKey noiseutil.CipherState
cipher noise.CipherFunc
counter uint64
}
// Separate cipher instances let the reference SendVia path use the same keys and
// counters without rewinding an AEAD instance (including FIPS nonce checks).
func relayBatchKey(cipher noise.CipherFunc, id byte) noiseutil.CipherState {
suite := noise.NewCipherSuite(noise.DH25519, cipher, noise.HashSHA256)
return noiseutil.NewCipherState(noise.UnsafeNewCipherState(suite, [32]byte{id}, 0), cipher)
}
func newRelayBatchPath(t *testing.T, cipher noise.CipherFunc) *relayBatchPath {
t.Helper()
l := test.NewLogger()
conn := &relayBatchConn{}
f := &Interface{
ctx: t.Context(), l: l, hostMap: newHostMap(l), outside: conn,
writers: []udp.Conn{conn}, lightHouse: &LightHouse{l: l},
myVpnNetworksTable: &bart.Lite{}, metricTxDropped: metrics.NewCounter(),
batchers: []*batch.MultiCoalescer{batch.NewMultiCoalescer(&capturingTun{}, l)},
}
f.connectionManager = &connectionManager{
intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{},
}
newHost := func(addr string, remote string, id byte) *HostInfo {
h := &HostInfo{
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
ConnectionState: &ConnectionState{
eKey: relayBatchKey(cipher, id), dKey: relayBatchKey(cipher, id), window: NewBits(ReplayWindow),
},
relayState: RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}},
}
u := netip.MustParseAddrPort(remote)
h.remote.Store(&u)
f.hostMap.Hosts[h.vpnAddrs[0]] = h
return h
}
source := newHost("192.0.2.1", "198.51.100.1:4242", 1)
target := newHost("192.0.2.2", "198.51.100.2:4242", 2)
incoming := &Relay{Type: ForwardingType, State: Established, LocalIndex: 301, RemoteIndex: 302, PeerAddr: target.vpnAddrs[0]}
outgoing := &Relay{Type: ForwardingType, State: Established, LocalIndex: 303, RemoteIndex: 304, PeerAddr: source.vpnAddrs[0]}
source.relayState.InsertRelay(incoming.PeerAddr, incoming.LocalIndex, incoming)
target.relayState.InsertRelay(outgoing.PeerAddr, outgoing.LocalIndex, outgoing)
f.hostMap.Relays[incoming.LocalIndex] = source
f.hostMap.Indexes[100] = source
return &relayBatchPath{f: f, conn: conn, source: source, target: target, incoming: incoming, outgoing: outgoing, sourceKey: relayBatchKey(cipher, 1), cipher: cipher}
}
func (p *relayBatchPath) wrap(t *testing.T, payload []byte) []byte {
t.Helper()
p.counter++
b := header.Encode(make([]byte, header.Len), header.Version, header.Message, header.MessageRelay, p.incoming.LocalIndex, p.counter)
b = append(b, payload...)
b, err := p.sourceKey.EncryptDanger(b, b, nil, p.counter, make([]byte, 12))
require.NoError(t, err)
return b
}
func (p *relayBatchPath) reference(payloads [][]byte) [][]byte {
conn := &relayBatchConn{}
f := &Interface{writers: []udp.Conn{conn}, connectionManager: p.f.connectionManager, l: p.f.l}
h := &HostInfo{ConnectionState: &ConnectionState{eKey: relayBatchKey(p.cipher, 2)}}
u := p.target.GetRemote()
h.remote.Store(&u)
for _, payload := range payloads {
f.SendVia(h, p.outgoing, payload, make([]byte, 12), make([]byte, 0, len(payload)+header.MaxOverhead), false, 0)
}
return conn.packets()
}
func TestRelayReceiveBatch(t *testing.T) {
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
t.Run(cipher.CipherName(), func(t *testing.T) {
p := newRelayBatchPath(t, cipher)
const count = 8
var payloads, wire [][]byte
for i := range count * 2 {
payload := bytes.Repeat([]byte{byte(i)}, 64)
payloads = append(payloads, payload)
wire = append(wire, p.wrap(t, payload))
}
want := p.reference(payloads)
// Model a listener reusing the same slots for the next receive batch.
buffers := make([][]byte, count)
for i := range buffers {
buffers[i] = make([]byte, len(wire[i]))
}
p.conn.read = func(r udp.EncReader, flush func()) {
for b := range 2 {
for i := range count {
copy(buffers[i], wire[b*count+i])
r(p.source.GetRemote(), buffers[i])
require.Len(t, p.conn.writes, b, "forwarding waits for the batch boundary")
}
flush()
require.Len(t, p.conn.writes, b+1, "flush must send before receive slots are reused")
for _, buffer := range buffers {
clear(buffer)
}
}
}
p.f.listenOut(0)
require.Equal(t, want, p.conn.packets())
for _, w := range p.conn.writes {
require.True(t, w.batched)
require.Len(t, w.packets, count)
for _, remote := range w.remotes {
require.Equal(t, p.target.GetRemote(), remote)
}
}
})
}
}
func TestRelayBatchDestinationChange(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, second := p.target.GetRemote(), netip.MustParseAddrPort("198.51.100.3:4242")
remotes := []netip.AddrPort{first, first, second, second, first}
var payloads, wire [][]byte
for i := range remotes {
payload := []byte{byte(i)}
payloads = append(payloads, payload)
wire = append(wire, p.wrap(t, payload))
}
want := p.reference(payloads)
p.conn.read = func(r udp.EncReader, flush func()) {
for i, remote := range remotes {
p.target.remote.Store(&remote)
r(p.source.GetRemote(), wire[i])
}
flush()
}
p.f.listenOut(0)
require.Equal(t, want, p.conn.packets())
require.Len(t, p.conn.writes, 3)
for i, wantRemotes := range [][]netip.AddrPort{{first, first}, {second, second}, {first}} {
require.Equal(t, wantRemotes, p.conn.writes[i].remotes)
}
}
func TestRelayBatchCapacity(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
p.conn.read = func(r udp.EncReader, flush func()) {
for i := range batch.SendBatchCap + 1 {
r(p.source.GetRemote(), p.wrap(t, []byte{byte(i)}))
}
require.Len(t, p.conn.writes, 1)
require.Len(t, p.conn.writes[0].packets, batch.SendBatchCap)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 2)
require.Len(t, p.conn.writes[1].packets, 1)
}
func TestRelayBatchBeforeResponse(t *testing.T) {
for _, unknownRelay := range []bool{false, true} {
name := "test request"
if unknownRelay {
name = "unknown relay"
}
t.Run(name, func(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, last := p.wrap(t, []byte("first")), p.wrap(t, []byte("last"))
p.counter++
request := header.Encode(make([]byte, header.Len), header.Version, header.Test, header.TestRequest, 100, p.counter)
request, err := p.sourceKey.EncryptDanger(request, request, []byte("ping"), p.counter, make([]byte, 12))
require.NoError(t, err)
if unknownRelay {
request = header.Encode(make([]byte, header.Len), header.Version, header.Message, header.MessageRelay, 999, p.counter)
}
p.conn.read = func(r udp.EncReader, flush func()) {
r(p.source.GetRemote(), first)
require.Empty(t, p.conn.writes)
r(p.source.GetRemote(), request)
require.Len(t, p.conn.writes, 2)
r(p.source.GetRemote(), last)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 3)
require.True(t, p.conn.writes[0].batched)
require.False(t, p.conn.writes[1].batched)
require.True(t, p.conn.writes[2].batched)
require.Equal(t, p.source.GetRemote(), p.conn.writes[1].remotes[0])
h := &header.H{}
require.NoError(t, h.Parse(p.conn.writes[1].packets[0]))
if unknownRelay {
require.Equal(t, header.RecvError, h.Type)
require.EqualValues(t, 999, h.RemoteIndex)
} else {
require.Equal(t, header.Test, h.Type)
require.Equal(t, header.TestReply, h.Subtype)
reply := p.conn.writes[1].packets[0]
out, err := p.sourceKey.DecryptDanger(nil, reply[:header.Len], reply[header.Len:], h.MessageCounter, make([]byte, 12))
require.NoError(t, err)
require.Equal(t, []byte("ping"), out)
}
})
}
}
func TestRelayBatchRemovedTarget(t *testing.T) {
for _, remove := range []string{"host", "relay", "disestablished"} {
t.Run(remove, func(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, removed := p.wrap(t, []byte("first")), p.wrap(t, []byte("removed"))
want := p.reference([][]byte{[]byte("first")})
p.conn.read = func(r udp.EncReader, flush func()) {
r(p.source.GetRemote(), first)
switch remove {
case "host":
delete(p.f.hostMap.Hosts, p.target.vpnAddrs[0])
case "relay":
delete(p.target.relayState.relayForByAddr, p.source.vpnAddrs[0])
case "disestablished":
p.target.relayState.UpdateRelayForByIpState(p.source.vpnAddrs[0], Disestablished)
}
r(p.source.GetRemote(), removed)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 1)
require.Equal(t, want, p.conn.packets(), "only the packet accepted before removal is sent")
})
}
}
+2 -2
View File
@@ -10,8 +10,8 @@ type batchWriter interface {
} }
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch. // SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
// One SendBatch is owned by each listenIn goroutine; no locking is needed. // Each reader goroutine owns its own SendBatch; no locking is needed.
// Slots are backed by an Arena (see its docs) // Slots can borrow receive buffers or be backed by an Arena (see its docs).
type SendBatch struct { type SendBatch struct {
out batchWriter out batchWriter
bufs [][]byte bufs [][]byte
+45
View File
@@ -0,0 +1,45 @@
//go:build (openbsd || netbsd) && !e2e_testing
package overlay
import (
"errors"
"fmt"
"log/slog"
"os"
"regexp"
"runtime"
"golang.org/x/sys/unix"
)
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
// tunDevNeedsRestart is true when tun.dev asks for a particular tun that isn't the running one. Anything else takes the
// first free one, and the one we have will do
func tunDevNeedsRestart(name, running string) bool {
return deviceNameRE.MatchString(name) && name != running
}
// openTun opens /dev/name, or the first /dev/tunN nothing else has open when name is empty or not tunN
func openTun(l *slog.Logger, name string) (int, string, error) {
if name != "" && !deviceNameRE.MatchString(name) {
// The name doesn't mean anything here, so like darwin a bad one isn't worth refusing to start over
l.Warn("interface name must be tun[0-9]+ on "+runtime.GOOS+", ignoring", "name", name)
name = ""
}
if name != "" {
fd, err := unix.Open("/dev/"+name, os.O_RDWR, 0)
return fd, name, err
}
for i := 0; ; i++ {
name = fmt.Sprintf("tun%d", i)
fd, err := unix.Open("/dev/"+name, os.O_RDWR, 0)
if err == nil {
return fd, name, nil
}
if !errors.Is(err, unix.EBUSY) {
return -1, "", fmt.Errorf("no free tun device, %s: %w", name, err)
}
}
}
+80
View File
@@ -0,0 +1,80 @@
//go:build (openbsd || netbsd) && !e2e_testing
package overlay
import (
"net/netip"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// A tun.dev that isn't tunN is ignored like darwin does, and the first free tun is used. Only naming a different tun
// asks for a restart
func TestTunReloadDevRestart(t *testing.T) {
newWithDev := func(dev string) (*config.C, *tun, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n dev: "+dev+"\n"))
tn, err := newTun(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, false)
if err == nil {
t.Cleanup(func() { _ = tn.Close() })
}
return c, tn, err
}
if _, _, err := newWithDev(""); err != nil {
t.Skipf("can't create a tun device here: %v", err)
}
c, tn, err := newWithDev("nope")
require.NoError(t, err, "a bad name should fall back to a free tun")
require.Regexp(t, `^tun[0-9]+$`, tn.Device)
for _, dev := range []string{"nope", "", tn.Device, "nope"} {
res := c.ReloadConfigString("tun:\n dev: " + dev + "\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed, dev)
assert.False(t, res.RestartRequired, dev)
}
other := "tun0"
if tn.Device == other {
other = "tun1"
}
res := c.ReloadConfigString("tun:\n dev: " + other + "\n")
require.NoError(t, res.Err)
assert.Equal(t, []string{"tun.dev"}, res.RestartKeys)
}
// An empty tun.dev takes the first tun nothing else has open. Reloading it empty, or naming the one we have, needs
// no restart
func TestTunEmptyDevPicksAFreeOne(t *testing.T) {
newEmpty := func() (*config.C, *tun, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun: {}\n"))
tn, err := newTun(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, false)
if err == nil {
t.Cleanup(func() { _ = tn.Close() })
}
return c, tn, err
}
c, first, err := newEmpty()
if err != nil {
t.Skipf("can't create a tun device here: %v", err)
}
_, second, err := newEmpty()
require.NoError(t, err, "the second tun should skip the one the first has open")
assert.NotEqual(t, first.Device, second.Device)
for _, cfg := range []string{
"tun: {}\npunchy:\n punch: true\n",
"tun:\n dev: " + first.Device + "\n",
"tun: {}\n",
} {
res := c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired, cfg)
}
}
+26 -9
View File
@@ -80,17 +80,31 @@ type ifreqAlias6 struct {
Lifetime addrLifetime Lifetime addrLifetime
} }
// utunIndex is the utun tun.dev asks for, or -1 for whichever is free. ok is false for a name that isn't utunN, which
// also gets whichever is free
func utunIndex(name string) (index int, ok bool) {
if name == "" || name == "utun" {
return -1, true
}
if _, err := fmt.Sscanf(name, "utun%d", &index); err != nil || index < 0 {
return -1, false
}
return index, true
}
// tunDevNeedsRestart is true when tun.dev asks for a particular utun that isn't the running one. Anything else takes
// whichever is free, and the one we have will do
func tunDevNeedsRestart(name, running string) bool {
index, _ := utunIndex(name)
return index >= 0 && fmt.Sprintf("utun%d", index) != running
}
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
name := c.GetString("tun.dev", "") ifIndex, ok := utunIndex(c.GetString("tun.dev", ""))
ifIndex := -1 if !ok {
if name != "" && name != "utun" {
_, err := fmt.Sscanf(name, "utun%d", &ifIndex)
if err != nil || ifIndex < 0 {
// NOTE: we don't make this error so we don't break existing // NOTE: we don't make this error so we don't break existing
// configs that set a name before it was used. // configs that set a name before it was used.
l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring") l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring")
ifIndex = -1
}
} }
fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL) fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL)
@@ -114,7 +128,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
return nil, fmt.Errorf("SYS_CONNECT: %v", err) return nil, fmt.Errorf("SYS_CONNECT: %v", err)
} }
name, err = unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME) name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to retrieve tun name: %w", err) return nil, fmt.Errorf("failed to retrieve tun name: %w", err)
} }
@@ -138,7 +152,10 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.dev", "tun.mtu") r.RestartRequiredIfChanged("tun.mtu")
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r) err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+13
View File
@@ -103,3 +103,16 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))) assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
} }
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
func TestTunDevNeedsRestart(t *testing.T) {
for name, want := range map[string]bool{
"": false,
"utun": false,
"nope": false,
"utun7": false,
"utun3": true,
} {
assert.Equal(t, want, tunDevNeedsRestart(name, "utun7"), name)
}
}
+121
View File
@@ -0,0 +1,121 @@
//go:build !ios && !e2e_testing
package overlay
import (
"bytes"
"os"
"syscall"
"testing"
"github.com/stretchr/testify/require"
)
func TestTun_WriteFraming(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: w}
for _, tc := range []struct {
name string
pkt []byte
af byte
}{
// Literal families rather than syscall.AF_*, so a wrong constant in Write can't also fix the expectation.
{"ipv4", []byte{0x45, 1, 2, 3, 4, 5}, 2},
{"ipv6", []byte{0x60, 1, 2, 3, 4, 5}, 30},
} {
t.Run(tc.name, func(t *testing.T) {
n, err := tn.Write(tc.pkt)
require.NoError(t, err)
require.Equal(t, len(tc.pkt), n)
got := make([]byte, 64)
n, err = r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, tc.af}, tc.pkt...), got[:n])
})
}
_, err = tn.Write([]byte{0x10, 1, 2, 3})
require.Error(t, err)
_, err = tn.Write([]byte{})
require.ErrorIs(t, err, syscall.EIO)
// The rejected writes must not have put anything on the fd, so the next read sees only this packet.
pkt := []byte{0x45, 9, 9}
_, err = tn.Write(pkt)
require.NoError(t, err)
got := make([]byte, 64)
n, err := r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, 2}, pkt...), got[:n])
}
func TestTun_ReadStripsHeader(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
pkt := []byte{0x45, 1, 2, 3, 4, 5}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err := tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
func TestTun_ReadShort(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
// Fewer bytes than the protocol header is not a packet.
_, err = w.Write([]byte{0, 0})
require.NoError(t, err)
n, err := tn.Read(make([]byte, 64))
require.NoError(t, err)
require.Zero(t, n)
// The next packet reads cleanly only if the short read consumed exactly what was written.
pkt := []byte{0x45, 1, 2, 3}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err = tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
// A heap allocation per packet rarely shows in a short throughput run, only across GC pauses, so
// this pins Read and Write to zero allocations directly.
func TestTun_ReadWriteDoNotAllocate(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tw := &tun{f: w}
tr := &tun{f: r}
pkts := [][]byte{bytes.Repeat([]byte{0x45}, 1300), bytes.Repeat([]byte{0x60}, 1300)}
buf := make([]byte, 1500)
i := 0
allocs := testing.AllocsPerRun(1000, func() {
i++
if _, err := tw.Write(pkts[i%len(pkts)]); err != nil {
t.Fatal(err)
}
if _, err := tr.Read(buf); err != nil {
t.Fatal(err)
}
})
require.Zero(t, allocs)
}
+5 -15
View File
@@ -8,7 +8,6 @@ import (
"log/slog" "log/slog"
"net/netip" "net/netip"
"os" "os"
"regexp"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -67,24 +66,12 @@ type tun struct {
fd int fd int
} }
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
return nil, fmt.Errorf("newTunFromFd not supported in NetBSD") return nil, fmt.Errorf("newTunFromFd not supported in NetBSD")
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
// Try to open tun device fd, deviceName, err := openTun(l, c.GetString("tun.dev", ""))
var err error
deviceName := c.GetString("tun.dev", "")
if deviceName == "" {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
if !deviceNameRE.MatchString(deviceName) {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -109,7 +96,10 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.dev", "tun.mtu") r.RestartRequiredIfChanged("tun.mtu")
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r) err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+5 -15
View File
@@ -8,7 +8,6 @@ import (
"log/slog" "log/slog"
"net/netip" "net/netip"
"os" "os"
"regexp"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -58,24 +57,12 @@ type tun struct {
fd int fd int
} }
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
return nil, fmt.Errorf("newTunFromFd not supported in openbsd") return nil, fmt.Errorf("newTunFromFd not supported in openbsd")
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
// Try to open tun device fd, deviceName, err := openTun(l, c.GetString("tun.dev", ""))
var err error
deviceName := c.GetString("tun.dev", "")
if deviceName == "" {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
if !deviceNameRE.MatchString(deviceName) {
return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
}
fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -100,7 +87,10 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.dev", "tun.mtu") r.RestartRequiredIfChanged("tun.mtu")
if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) {
r.RestartRequiredIfChanged("tun.dev")
}
err := t.reload(c, false, r) err := t.reload(c, false, r)
if err != nil { if err != nil {
util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+9 -1
View File
@@ -109,7 +109,15 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
disableDNSRegistration(l, *guid) disableDNSRegistration(l, *guid)
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
r.RestartRequiredIfChanged("tun.dev", "tun.mtu", "tun.network_category") r.RestartRequiredIfChanged("tun.dev", "tun.mtu")
// Compare with the running category, not the last config. A restart with one we can't parse would fail, so say
// that now rather than ask for one, and going back to the running one after a bad reload needs no restart
if cat, setCat, err := parseNetworkCategory(c.GetString("tun.network_category", "private")); err != nil {
t.l.Error("Invalid tun.network_category, a restart with it would fail", "error", err)
r.Failed()
} else if cat != t.networkCategory || setCat != t.setCategory {
r.RestartRequiredIfChanged("tun.network_category")
}
r.RestartRequiredIfChanged(interfaceBypassKeys...) r.RestartRequiredIfChanged(interfaceBypassKeys...)
err := t.reload(c, false, r) err := t.reload(c, false, r)
if err != nil { if err != nil {
+21
View File
@@ -181,6 +181,27 @@ func TestWinTun_ReloadRouteAlreadyGone(t *testing.T) {
}) })
} }
// A network category we can't parse would fail the restart, so it fails the reload instead of asking for one
func TestWinTun_ReloadInvalidNetworkCategoryFails(t *testing.T) {
rt := newRouteTest(t, unsafeRoutes())
for range 2 {
res := rt.c.ReloadConfigString(unsafeRoutes() + " network_category: nope\n")
require.NoError(t, res.Err)
assert.True(t, res.Failed, "a bad category that didn't change since the last reload is still bad")
assert.False(t, res.RestartRequired)
}
// Fixing it back to the running category needs nothing
res := rt.c.ReloadConfigString(unsafeRoutes() + " network_category: private\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired)
res = rt.c.ReloadConfigString(unsafeRoutes() + " network_category: public\n")
require.NoError(t, res.Err)
assert.Equal(t, []string{"tun.network_category"}, res.RestartKeys)
}
// forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4 // forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4
func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) { func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) {
t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") }) t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") })
+40 -9
View File
@@ -2,7 +2,9 @@ package nebula
import ( import (
"crypto/fips140" "crypto/fips140"
"crypto/sha256"
"encoding/binary" "encoding/binary"
"encoding/hex"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@@ -61,7 +63,18 @@ func NewPKIFromConfig(l *slog.Logger, c *config.C) (*PKI, error) {
} }
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
// Compare with the running cipher, not the last config. A restart with one we can't run would fail, so say that
// now rather than ask for one, and going back to the running one after a bad reload needs no restart
if cipher := c.GetString("cipher", "aes"); !validCipher(cipher) {
l.Error("Unknown cipher, a restart with it would fail", "cipher", cipher)
r.Failed()
} else if cipher == "chachapoly" && fipsOnly() {
// newCipherSuite refuses it at startup
l.Error("ChaChaPoly is not allowed in FIPS 140-only mode, a restart with it would fail", "cipher", cipher)
r.Failed()
} else if cipher != pki.getCertState().cipher {
r.RestartRequiredIfChanged("cipher") r.RestartRequiredIfChanged("cipher")
}
rErr := pki.reload(c, false, r) rErr := pki.reload(c, false, r)
if rErr != nil { if rErr != nil {
util.LogWithContextIfNeeded("Failed to reload PKI from config", rErr, l) util.LogWithContextIfNeeded("Failed to reload PKI from config", rErr, l)
@@ -116,16 +129,21 @@ func (p *PKI) reload(c *config.C, initial bool, r *config.Reload) error {
return nil return nil
} }
// fipsOnly is fips140.Enforced, a variable so a test can run the FIPS 140-only reload check without GODEBUG
var fipsOnly = fips140.Enforced
// validCipher is true for the ciphers nebula can run. Each post-handshake CipherState in noiseutil hardcodes its own
// nonce endianness, so there's nothing to set up for either.
func validCipher(cipher string) bool {
return cipher == "aes" || cipher == "chachapoly"
}
func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError { func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
var cipher string var cipher string
var currentState *CertState var currentState *CertState
if initial { if initial {
cipher = c.GetString("cipher", "aes") cipher = c.GetString("cipher", "aes")
switch cipher { if !validCipher(cipher) {
case "aes", "chachapoly":
// Each post-handshake CipherState in noiseutil hardcodes its own
// nonce endianness now, so there's nothing to set up here.
default:
return util.NewContextualError( return util.NewContextualError(
"unknown cipher", "unknown cipher",
m{"cipher": cipher}, m{"cipher": cipher},
@@ -217,12 +235,14 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError { func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError {
caPool, err := loadCAPoolFromConfig(p.l, c) caPool, err := loadCAPoolFromConfig(p.l, c)
if caPool != nil {
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
}
if err != nil { if err != nil {
return util.NewContextualError("Failed to load ca from config", nil, err) return util.NewContextualError("Failed to load ca from config", nil, err)
} }
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
return nil return nil
} }
@@ -597,11 +617,22 @@ func loadCAPoolFromConfig(l *slog.Logger, c *config.C) (*cert.CAPool, error) {
bl := c.GetStringSlice("pki.blocklist", []string{}) bl := c.GetStringSlice("pki.blocklist", []string{})
if len(bl) > 0 { if len(bl) > 0 {
for _, fp := range bl { var bad []string
for i, raw := range bl {
// Fingerprints are compared as lowercase hex strings, a malformed entry would silently block nothing
fp := strings.ToLower(strings.TrimSpace(raw))
if b, err := hex.DecodeString(fp); err != nil || len(b) != sha256.Size {
bad = append(bad, fmt.Sprintf("entry %d %q", i+1, raw))
continue
}
caPool.BlocklistFingerprint(fp) caPool.BlocklistFingerprint(fp)
} }
l.Info("Blocklisted certificates", "fingerprintCount", len(bl)) l.Info("Blocklisted certificates", "fingerprintCount", len(bl)-len(bad))
if len(bad) > 0 {
// The valid entries are still returned so a failed reload blocks them, startup refuses the error
return caPool, fmt.Errorf("pki.blocklist entries are not hex certificate fingerprints: %s", strings.Join(bad, ", "))
}
} }
return caPool, nil return caPool, nil
+82
View File
@@ -1,6 +1,7 @@
package nebula package nebula
import ( import (
"crypto/fips140"
"fmt" "fmt"
"net/netip" "net/netip"
"strings" "strings"
@@ -128,6 +129,23 @@ func TestPKIReloadRestartRequired(t *testing.T) {
assert.True(t, res.RestartRequired) assert.True(t, res.RestartRequired)
assert.Equal(t, "aes", p.getCertState().cipher) assert.Equal(t, "aes", p.getCertState().cipher)
// A cipher we can't run would fail the restart, so it fails the reload instead of asking for one
c4 := config.NewC(test.NewLogger())
require.NoError(t, c4.LoadString(pkiConfig("aes", "10.1.0.2/24")))
_, err = NewPKIFromConfig(test.NewLogger(), c4)
require.NoError(t, err)
for range 2 {
res = c4.ReloadConfigString(pkiConfig("nope", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.True(t, res.Failed, "a bad cipher that didn't change since the last reload is still bad")
assert.False(t, res.RestartRequired)
}
// Fixing it back to the running cipher needs nothing
res = c4.ReloadConfigString(pkiConfig("aes", "10.1.0.2/24"))
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.False(t, res.RestartRequired)
// A cert that keeps our networks applies without a restart // A cert that keeps our networks applies without a restart
c3 := config.NewC(test.NewLogger()) c3 := config.NewC(test.NewLogger())
require.NoError(t, c3.LoadString(pkiConfig("aes", "10.1.0.2/24"))) require.NoError(t, c3.LoadString(pkiConfig("aes", "10.1.0.2/24")))
@@ -178,3 +196,67 @@ func TestPKIReloadNewVersionWithOtherNetworks(t *testing.T) {
assert.Equal(t, []netip.Prefix{netip.MustParsePrefix(tc.network)}, p.getCertState().myVpnNetworks, "accepted either way, as before") assert.Equal(t, []netip.Prefix{netip.MustParsePrefix(tc.network)}, p.getCertState().myVpnNetworks, "accepted either way, as before")
} }
} }
// FIPS 140-only mode refuses chachapoly at startup, so reloading to it fails rather than asks for a restart
func TestPKIReloadChachapolyInFIPSOnlyFails(t *testing.T) {
fipsOnly = func() bool { return true }
t.Cleanup(func() { fipsOnly = fips140.Enforced })
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_P256, time.Time{}, time.Time{}, nil, nil, nil)
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_P256, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil)
pkiConfig := func(cipher string) string {
return fmt.Sprintf("cipher: %s\npki:\n ca: %q\n cert: %q\n key: %q\n", cipher, caPEM, certPEM, keyPEM)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig("aes")))
_, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig("chachapoly"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.False(t, res.RestartRequired)
}
func TestLoadCAPoolBlocklist(t *testing.T) {
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
crt, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil)
fp, err := crt.Fingerprint()
require.NoError(t, err)
load := func(entry string) (*cert.CAPool, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(fmt.Sprintf("pki:\n ca: %q\n blocklist:\n - %q\n", caPEM, entry)))
return loadCAPoolFromConfig(test.NewLogger(), c)
}
for _, entry := range []string{fp, strings.ToUpper(fp), " " + fp + "\t"} {
pool, err := load(entry)
require.NoError(t, err, entry)
assert.True(t, pool.IsBlocklisted(fp), entry)
}
for _, entry := range []string{fp[1:], fp + "00", "sha256:" + fp, "zz" + fp[2:], ""} {
_, err := load(entry)
require.Error(t, err, entry)
}
// A bad entry on reload fails it, the valid entries are still blocked
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "me", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.1/24")}, nil, nil)
pkiConfig := func(entry string) string {
return fmt.Sprintf("pki:\n ca: %q\n cert: %q\n key: %q\n blocklist:\n - %q\n", caPEM, certPEM, keyPEM, entry)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig(fp)))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
other, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "other", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.3/24")}, nil, nil)
otherFp, err := other.Fingerprint()
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig(fp) + fmt.Sprintf(" - %q\n - %q\n", otherFp, fp[1:]))
assert.True(t, res.Failed)
assert.True(t, p.GetCAPool().IsBlocklisted(fp))
assert.True(t, p.GetCAPool().IsBlocklisted(otherFp))
}
+4 -2
View File
@@ -36,11 +36,13 @@ func (l *tcpListener) Accept() (net.Conn, error) {
func (l *tcpListener) Close() error { func (l *tcpListener) Close() error {
l.closeOnce.Do(func() { l.closeOnce.Do(func() {
// Close before leaving the map so a concurrent closeListeners that
// no longer sees this listener knows Accept is already released.
close(l.closed)
l.s.mu.Lock() l.s.mu.Lock()
delete(l.s.mu.listeners, l.port) delete(l.s.mu.listeners, l.port)
l.s.mu.Unlock() l.s.mu.Unlock()
close(l.closed)
}) })
return nil return nil
} }
+70 -7
View File
@@ -32,13 +32,18 @@ import (
const nicID = 1 const nicID = 1
type Service struct { type Service struct {
eg *errgroup.Group
control *nebula.Control control *nebula.Control
ipstack *stack.Stack ipstack *stack.Stack
// done is closed once the forwarding goroutines have exited and the
// netstack is torn down, at which point err holds the first fatal error.
done chan struct{}
err error
mu struct { mu struct {
sync.Mutex sync.Mutex
closed bool
listeners map[uint16]*tcpListener listeners map[uint16]*tcpListener
} }
} }
@@ -65,7 +70,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
ctx := control.Context() ctx := control.Context()
eg, ctx := errgroup.WithContext(ctx) eg, ctx := errgroup.WithContext(ctx)
s := Service{ s := Service{
eg: eg,
control: control, control: control,
} }
s.mu.listeners = map[uint16]*tcpListener{} s.mu.listeners = map[uint16]*tcpListener{}
@@ -74,6 +78,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol}, NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6}, TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6},
}) })
// The teardown goroutine only owns the stack once New succeeds
defer func() {
if reterr != nil {
s.ipstack.Destroy()
}
}()
sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default
tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt) tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt)
if tcpipErr != nil { if tcpipErr != nil {
@@ -123,6 +133,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
// this will read exactly one packet // this will read exactly one packet
n, err := reader.Read(buf) n, err := reader.Read(buf)
if err != nil { if err != nil {
// An error after shutdown began is noise from the device pipes
// closing, anything else is fatal and must stop nebula too
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err return err
} }
packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{ packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{
@@ -130,8 +146,8 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
}) })
linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf) linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf)
if err := ctx.Err(); err != nil { if ctx.Err() != nil {
return err return nil
} }
} }
}) })
@@ -139,13 +155,18 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
for { for {
packet := linkEP.ReadContext(ctx) packet := linkEP.ReadContext(ctx)
if packet == nil { if packet == nil {
if err := ctx.Err(); err != nil { if ctx.Err() != nil {
return err return nil
} }
continue continue
} }
bufView := packet.ToView() bufView := packet.ToView()
if _, err := bufView.WriteTo(writer); err != nil { if _, err := bufView.WriteTo(writer); err != nil {
// Same shutdown noise handling as the inbound reader
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err return err
} }
bufView.Release() bufView.Release()
@@ -158,6 +179,22 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
return control.Wait() return control.Wait()
}) })
// Tear down here rather than in Close so it also happens when nebula stops
// on its own. The forwarding goroutines must exit before the stack they feed
// is destroyed.
s.done = make(chan struct{})
go func() {
s.err = eg.Wait()
s.closeListeners()
// Destroy aborts the endpoints that exist when it starts, then waits on
// every endpoint, so a Dial registering in between would hold Wait until
// its connect timed out. Without the NIC new dials fail route lookup.
// RemoveNIC also closes linkEP.
_ = s.ipstack.RemoveNIC(nicID)
s.ipstack.Destroy()
close(s.done)
}()
return &s, nil return &s, nil
} }
@@ -237,6 +274,9 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
if s.mu.closed {
return nil, net.ErrClosed
}
if _, ok := s.mu.listeners[port]; ok { if _, ok := s.mu.listeners[port]; ok {
return nil, fmt.Errorf("already listening on port %d", port) return nil, fmt.Errorf("already listening on port %d", port)
} }
@@ -245,15 +285,38 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
return l, nil return l, nil
} }
// Wait blocks until the service has fully stopped, either via Close or a fatal
// error, and returns the first fatal error if there was one.
func (s *Service) Wait() error { func (s *Service) Wait() error {
return s.eg.Wait() <-s.done
return s.err
} }
// Close closes all listeners and stops nebula. Use Wait to block until
// shutdown has completed.
func (s *Service) Close() error { func (s *Service) Close() error {
s.closeListeners()
s.control.Stop() s.control.Stop()
return nil return nil
} }
// closeListeners unblocks pending Accept calls and rejects new listeners.
func (s *Service) closeListeners() {
// Every caller closes every listener rather than returning early once
// closed is set, so none returns while another is still closing them.
s.mu.Lock()
s.mu.closed = true
listeners := make([]*tcpListener, 0, len(s.mu.listeners))
for _, l := range s.mu.listeners {
listeners = append(listeners, l)
}
s.mu.Unlock()
for _, l := range listeners {
_ = l.Close()
}
}
func (s *Service) tcpHandler(r *tcp.ForwarderRequest) { func (s *Service) tcpHandler(r *tcp.ForwarderRequest) {
endpointID := r.ID() endpointID := r.ID()
+159
View File
@@ -327,3 +327,162 @@ func TestListenerAcceptAfterClose(t *testing.T) {
} }
} }
} }
// Close must release blocked Accept calls and let Wait report a clean shutdown.
func TestServiceClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
if err := a.Close(); err != nil {
t.Fatal(err)
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after Close")
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after Close")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
if err := a.Close(); err != nil {
t.Fatalf("second close: %v", err)
}
if _, err := a.Listen("tcp", ":1235"); !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Listen after Close, got %v", err)
}
}
// If nebula stops without Close being called, the service must still release
// Accept and tear down the netstack.
func TestServiceNebulaStopped(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
a.control.Stop()
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after nebula stopped")
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after nebula stopped")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
}
// Dials racing Close must not hold Wait until their connect times out.
func TestServiceDialDuringClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
stop := make(chan struct{})
var eg errgroup.Group
for range 8 {
eg.Go(func() error {
for {
select {
case <-stop:
return nil
default:
}
// 10.0.0.2 has no tunnel, so each dial sits in SYN-SENT until aborted
if c, err := a.Dial("tcp", "10.0.0.2:80"); err == nil {
_ = c.Close()
}
}
})
}
time.Sleep(50 * time.Millisecond)
if err := a.Close(); err != nil {
t.Fatal(err)
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait blocked behind a dial that raced Close")
}
close(stop)
dialersDone := make(chan error, 1)
go func() { dialersDone <- eg.Wait() }()
select {
case <-dialersDone:
case <-time.After(5 * time.Second):
t.Fatal("Dial still blocked after Wait")
}
}
+15 -176
View File
@@ -11,11 +11,8 @@ import (
"net" "net"
"net/netip" "net/netip"
"os" "os"
"path/filepath" "slices"
"runtime"
"runtime/pprof"
"sort" "sort"
"strconv"
"strings" "strings"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
@@ -59,13 +56,11 @@ type sshDeviceInfoFlags struct {
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
// sshd.sandbox_dir is read once, when the commands are attached
r.RestartRequiredIfChanged("sshd.sandbox_dir")
if c.GetBool("sshd.enabled", false) { if c.GetBool("sshd.enabled", false) {
// Restarting the server drops every open session, the one that asked for this reload included. Only restart // Restarting the server drops every open session, the one that asked for this reload included. Only restart
// when it isn't running or its config changed. The host key file can change under the same config, new // when it isn't running or its config changed. The host key file can change under the same config, new
// handshakes use the new key without a restart // handshakes use the new key without a restart
if ssh.Running() && !c.HasChanged("sshd") { if ssh.Running() && !slices.ContainsFunc(sshdServerKeys, c.HasChanged) {
key, err := loadSSHHostKey(c) key, err := loadSSHHostKey(c)
if err == nil { if err == nil {
err = ssh.SetHostKey(key) err = ssh.SetHostKey(key)
@@ -77,7 +72,7 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
} }
return return
} }
sshRun, err := configSSH(l, ssh, c) sshRun, err := configSSH(l, ssh, c, r)
if err != nil { if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err) l.Error("Failed to reconfigure the sshd", "error", err)
ssh.Stop() ssh.Stop()
@@ -92,6 +87,9 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
}) })
} }
// sshdServerKeys are what configSSH reads
var sshdServerKeys = []string{"sshd.enabled", "sshd.listen", "sshd.host_key", "sshd.trusted_cas", "sshd.authorized_users"}
// loadSSHHostKey returns sshd.host_key, read from its file when it names one. // loadSSHHostKey returns sshd.host_key, read from its file when it names one.
func loadSSHHostKey(c *config.C) ([]byte, error) { func loadSSHHostKey(c *config.C) ([]byte, error) {
hostKeyPathOrKey := c.GetString("sshd.host_key", "") hostKeyPathOrKey := c.GetString("sshd.host_key", "")
@@ -112,7 +110,8 @@ func loadSSHHostKey(c *config.C) ([]byte, error) {
// updates the passed-in SSHServer. On success, it returns a function // updates the passed-in SSHServer. On success, it returns a function
// that callers may invoke to run the configured ssh server. On // that callers may invoke to run the configured ssh server. On
// failure, it returns nil, error. // failure, it returns nil, error.
func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) { // r is nil at startup, entries that are skipped with a warning fail it on a reload
func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reload) (func(), error) {
listen := c.GetString("sshd.listen", "") listen := c.GetString("sshd.listen", "")
if listen == "" { if listen == "" {
return nil, fmt.Errorf("sshd.listen must be provided") return nil, fmt.Errorf("sshd.listen must be provided")
@@ -145,6 +144,7 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
err := ssh.AddTrustedCA(caAuthorizedKey) err := ssh.AddTrustedCA(caAuthorizedKey)
if err != nil { if err != nil {
l.Warn("SSH CA had an error, ignoring", "error", err, "sshCA", caAuthorizedKey) l.Warn("SSH CA had an error, ignoring", "error", err, "sshCA", caAuthorizedKey)
r.Failed()
continue continue
} }
} }
@@ -156,12 +156,14 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
kDef, ok := rk.(map[string]any) kDef, ok := rk.(map[string]any)
if !ok { if !ok {
l.Warn("Authorized user had an error, ignoring", "sshKeyConfig", rk) l.Warn("Authorized user had an error, ignoring", "sshKeyConfig", rk)
r.Failed()
continue continue
} }
user, ok := kDef["user"].(string) user, ok := kDef["user"].(string)
if !ok { if !ok {
l.Warn("Authorized user is missing the user field", "sshKeyConfig", rk) l.Warn("Authorized user is missing the user field", "sshKeyConfig", rk)
r.Failed()
continue continue
} }
@@ -175,6 +177,7 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
"sshKeyConfig", rk, "sshKeyConfig", rk,
"sshKey", v, "sshKey", v,
) )
r.Failed()
continue continue
} }
@@ -186,6 +189,7 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
"sshKeyConfig", rk, "sshKeyConfig", rk,
"sshKey", subK, "sshKey", subK,
) )
r.Failed()
continue continue
} }
@@ -195,12 +199,14 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
"error", err, "error", err,
"sshKeyConfig", sk, "sshKeyConfig", sk,
) )
r.Failed()
continue continue
} }
} }
default: default:
l.Warn("Authorized user is missing the keys field or was not understood", "sshKeyConfig", rk) l.Warn("Authorized user is missing the keys field or was not understood", "sshKeyConfig", rk)
r.Failed()
} }
} }
} else { } else {
@@ -223,12 +229,6 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
} }
func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) { func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
// sandboxDir defaults to a dir in temp. The intention is that end user will
// create this dir as needed. Overriding this config value to "" allows
// writing to anywhere in the system.
defaultDir := filepath.Join(os.TempDir(), "nebula-debug")
sandboxDir := c.GetString("sshd.sandbox_dir", defaultDir)
ssh.RegisterCommand(&sshd.Command{ ssh.RegisterCommand(&sshd.Command{
Name: "list-hostmap", Name: "list-hostmap",
ShortDescription: "List all known previously connected hosts", ShortDescription: "List all known previously connected hosts",
@@ -284,45 +284,6 @@ func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interfa
}, },
}) })
ssh.RegisterCommand(&sshd.Command{
Name: "start-cpu-profile",
ShortDescription: "Starts a cpu profile and write output to the provided file, ex: `cpu-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshStartCpuProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "stop-cpu-profile",
ShortDescription: "Stops a cpu profile and writes output to the previously provided file",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
pprof.StopCPUProfile()
return w.WriteLine("If a CPU profile was running it is now stopped")
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "save-heap-profile",
ShortDescription: "Saves a heap profile to the provided path, ex: `heap-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetHeapProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "mutex-profile-fraction",
ShortDescription: "Gets or sets runtime.SetMutexProfileFraction",
Callback: sshMutexProfileFraction,
})
ssh.RegisterCommand(&sshd.Command{
Name: "save-mutex-profile",
ShortDescription: "Saves a mutex profile to the provided path, ex: `mutex-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetMutexProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{ ssh.RegisterCommand(&sshd.Command{
Name: "log-level", Name: "log-level",
ShortDescription: "Gets or sets the current log level", ShortDescription: "Gets or sets the current log level",
@@ -553,58 +514,6 @@ func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) er
return nil return nil
} }
// sshSanitizeFilePath validates that the given file path is within the sandbox directory.
// If sandboxDir is empty, the path is returned as-is for backwards compatibility.
func sshSanitizeFilePath(sandboxDir, filePath string) (string, error) {
if sandboxDir == "" {
return filePath, nil
}
// Clean and resolve the path relative to the sandbox directory
if !filepath.IsAbs(filePath) {
filePath = filepath.Join(sandboxDir, filePath)
}
cleaned := filepath.Clean(filePath)
// Ensure the resolved path is within the sandbox directory
cleanedSandbox := filepath.Clean(sandboxDir)
if cleaned == cleanedSandbox {
return "", fmt.Errorf("path %q resolves to the sandbox directory itself %q", filePath, sandboxDir)
}
if !strings.HasPrefix(cleaned, cleanedSandbox+string(filepath.Separator)) {
return "", fmt.Errorf("path %q is outside the sandbox directory %q", filePath, sandboxDir)
}
return cleaned, nil
}
func sshStartCpuProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
err := w.WriteLine("No path to write profile provided")
return err
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
}
err = pprof.StartCPUProfile(file)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to start cpu profile: %s", err))
return err
}
err = w.WriteLine(fmt.Sprintf("Started cpu profile, issue stop-cpu-profile to write the output to %s", a))
return err
}
func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
return w.WriteLine(fmt.Sprintf("%s", ifce.version)) return w.WriteLine(fmt.Sprintf("%s", ifce.version))
} }
@@ -757,76 +666,6 @@ func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) e
return w.WriteLine("Changed") return w.WriteLine("Changed")
} }
func sshGetHeapProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
}
err = pprof.WriteHeapProfile(file)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
return err
}
err = w.WriteLine(fmt.Sprintf("Mem profile created at %s", a))
return err
}
func sshMutexProfileFraction(fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
rate := runtime.SetMutexProfileFraction(-1)
return w.WriteLine(fmt.Sprintf("Current value: %d", rate))
}
newRate, err := strconv.Atoi(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("Invalid argument: %s", a[0]))
}
oldRate := runtime.SetMutexProfileFraction(newRate)
return w.WriteLine(fmt.Sprintf("New value: %d. Old value: %d", newRate, oldRate))
}
func sshGetMutexProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
return w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
}
defer file.Close()
mutexProfile := pprof.Lookup("mutex")
if mutexProfile == nil {
return w.WriteLine("Unable to get pprof.Lookup(\"mutex\")")
}
err = mutexProfile.WriteTo(file, 0)
if err != nil {
return w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
}
return w.WriteLine(fmt.Sprintf("Mutex profile created at %s", a))
}
func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error { func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
ctrl, ok := l.Handler().(interface { ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level GetLevel() slog.Level
+49 -5
View File
@@ -26,13 +26,15 @@ type sshdReloadHarness struct {
server *sshd.SSHServer server *sshd.SSHServer
keyFile string keyFile string
signer ssh.Signer signer ssh.Signer
listen string
users []string
extra string extra string
} }
func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness { func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
t.Helper() t.Helper()
l := test.NewLogger() l := test.NewLogger()
h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key")} h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key"), listen: "127.0.0.1:0", users: []string{"test"}}
h.writeHostKey(t, newEd25519Key(t)) h.writeHostKey(t, newEd25519Key(t))
_, userKey, err := ed25519.GenerateKey(rand.Reader) _, userKey, err := ed25519.GenerateKey(rand.Reader)
@@ -45,7 +47,7 @@ func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
h.server, err = sshd.NewSSHServer(t.Context(), l) h.server, err = sshd.NewSSHServer(t.Context(), l)
require.NoError(t, err) require.NoError(t, err)
wireSSHReload(l, h.server, h.c) wireSSHReload(l, h.server, h.c)
run, err := configSSH(l, h.server, h.c) run, err := configSSH(l, h.server, h.c, nil)
require.NoError(t, err) require.NoError(t, err)
go run() go run()
t.Cleanup(h.server.Stop) t.Cleanup(h.server.Stop)
@@ -71,8 +73,12 @@ func (h *sshdReloadHarness) writeHostKey(t *testing.T, key crypto.Signer) ssh.Pu
} }
func (h *sshdReloadHarness) config() string { func (h *sshdReloadHarness) config() string {
return fmt.Sprintf("sshd:\n enabled: true\n listen: 127.0.0.1:0\n host_key: %s\n authorized_users:\n - user: test\n keys: [%q]\n%s", users := ""
h.keyFile, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())), h.extra) for _, u := range h.users {
users += fmt.Sprintf(" - user: %s\n keys: [%q]\n", u, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())))
}
return fmt.Sprintf("sshd:\n enabled: true\n listen: %s\n host_key: %s\n authorized_users:\n%s%s",
h.listen, h.keyFile, users, h.extra)
} }
// dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys // dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys
@@ -134,15 +140,29 @@ func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) {
assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted") assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted")
} }
// Every key the server reads restarts it when it changes
func TestSSHDReload_ChangedConfigRestarts(t *testing.T) { func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
cases := map[string]func(t *testing.T, h *sshdReloadHarness){
"listen": func(t *testing.T, h *sshdReloadHarness) { h.listen = "localhost:0" },
"authorized_users": func(t *testing.T, h *sshdReloadHarness) { h.users = append(h.users, "other") },
"trusted_cas": func(t *testing.T, h *sshdReloadHarness) { h.extra = " trusted_cas: []\n" },
"host_key": func(t *testing.T, h *sshdReloadHarness) {
h.keyFile = filepath.Join(t.TempDir(), "other_host_key")
h.writeHostKey(t, newEd25519Key(t))
},
}
for name, change := range cases {
t.Run(name, func(t *testing.T) {
h := newSSHDReloadHarness(t) h := newSSHDReloadHarness(t)
client, _ := h.connect(t) client, _ := h.connect(t)
h.extra = " trusted_cas: []\n" change(t, h)
require.NoError(t, h.c.ReloadConfigString(h.config()).Err) require.NoError(t, h.c.ReloadConfigString(h.config()).Err)
assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied") assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied")
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond) require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t) h.connect(t)
})
}
} }
// The config can stay the same while the host key file it names changes. New connections get the new key, open // The config can stay the same while the host key file it names changes. New connections get the new key, open
@@ -194,3 +214,27 @@ func TestSSHDReload_BadHostKeyFileStops(t *testing.T) {
assert.True(t, h.c.ReloadConfigString(h.config()).Failed) assert.True(t, h.c.ReloadConfigString(h.config()).Failed)
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond) require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
} }
// A trusted CA or authorized user that is skipped with a warning fails the reload, the valid ones are still used
func TestSSHDReload_SkippedEntriesFail(t *testing.T) {
cases := map[string]string{
"bad trusted ca": " trusted_cas: [nope]\n",
"user entry not a map": " - nope\n",
"user missing": " - keys: [nope]\n",
"bad key": " - user: other\n keys: nope\n",
"key in a list not a string": " - user: other\n keys: [5]\n",
"bad key in a list": " - user: other\n keys: [nope]\n",
"keys missing": " - user: other\n",
}
for name, extra := range cases {
t.Run(name, func(t *testing.T) {
h := newSSHDReloadHarness(t)
h.extra = extra
res := h.c.ReloadConfigString(h.config())
require.NoError(t, res.Err)
assert.True(t, res.Failed)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
})
}
}
+1 -1
View File
@@ -41,7 +41,7 @@ type Conn interface {
// //
// Returns the number of packets successfully written. A destination the kernel rejects costs only // Returns the number of packets successfully written. A destination the kernel rejects costs only
// its own packet, so a short count means some peers were undeliverable, not that the batch failed. // its own packet, so a short count means some peers were undeliverable, not that the batch failed.
// Not safe for concurrent use on the same Conn. // Safe for concurrent use on the same Conn.
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
// ReloadConfig applies the listen settings that can change, r is nil at startup // ReloadConfig applies the listen settings that can change, r is nil at startup
ReloadConfig(c *config.C, r *config.Reload) ReloadConfig(c *config.C, r *config.Reload)
+6 -1
View File
@@ -12,6 +12,7 @@ import (
"net/netip" "net/netip"
"strconv" "strconv"
"strings" "strings"
"sync"
"unsafe" "unsafe"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
@@ -20,7 +21,7 @@ import (
// batchWriter owns the sendmmsg(2)/UDP-GSO transmit path for a StdConn: the // batchWriter owns the sendmmsg(2)/UDP-GSO transmit path for a StdConn: the
// scratch WriteBatch packs mmsghdr entries into, plus the GSO capability // scratch WriteBatch packs mmsghdr entries into, plus the GSO capability
// state probed at socket creation. Each queue has its own StdConn and // state probed at socket creation. Each queue has its own StdConn and
// batchWriter, so no locking is needed. // batchWriter; mu serializes batches from the TUN reader and relay forwarding.
// //
// Terminology, smallest to largest: // Terminology, smallest to largest:
// //
@@ -39,6 +40,7 @@ import (
// chunk the entries packed for one sendmmsg call, at most MaxWriteBatch. // chunk the entries packed for one sendmmsg call, at most MaxWriteBatch.
// batch the caller's whole bufs/addrs pair, processed as one or more chunks. // batch the caller's whole bufs/addrs pair, processed as one or more chunks.
type batchWriter struct { type batchWriter struct {
mu sync.Mutex
fd int fd int
isV4 bool isV4 bool
@@ -192,6 +194,9 @@ func parseRelease(r string) (major, minor int) {
// Returns the number of packets sent. An error means the call itself failed. // Returns the number of packets sent. An error means the call itself failed.
// A short count means some destinations were undeliverable. // A short count means some destinations were undeliverable.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()
if len(bufs) != len(addrs) { if len(bufs) != len(addrs) {
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs)) return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
} }
@@ -0,0 +1,61 @@
//go:build linux && !android && !e2e_testing
package udp
import (
"net/netip"
"runtime"
"sync"
"testing"
"github.com/stretchr/testify/require"
)
// The TUN reader and relay forwarding share a socket. Exercise their concurrent
// batches against the actual GSO planner and scratch, without relying on a NIC.
func TestWriteBatchConcurrent(t *testing.T) {
const batches = 64
w := newRewindTestWriter()
sent := make(chan [][]byte, 2*batches)
w.sendFn = func(start, n int) (int, error) {
runtime.Gosched()
sent <- capturePrepared(w, start, n)
return n, nil
}
conn := &StdConn{bw: w}
remote := netip.MustParseAddrPort("192.0.2.1:4242")
var wg sync.WaitGroup
start := make(chan struct{})
for producer := range 2 {
wg.Go(func() {
<-start
for i := range batches {
var bufs [][]byte
var addrs []netip.AddrPort
for j := range 4 {
bufs = append(bufs, []byte{byte(producer), byte(i), byte(j), 0})
addrs = append(addrs, remote)
}
n, err := conn.WriteBatch(bufs, addrs)
if err != nil || n != len(bufs) {
t.Errorf("WriteBatch = %d, %v; want %d, nil", n, err, len(bufs))
return
}
}
})
}
close(start)
wg.Wait()
close(sent)
var counts [2]int
for packets := range sent {
require.Len(t, packets, 4)
producer := packets[0][0]
require.Less(t, int(producer), len(counts))
for j, packet := range packets {
require.Equal(t, []byte{producer, byte(counts[producer]), byte(j), 0}, packet)
}
counts[producer]++
}
require.Equal(t, [2]int{batches, batches}, counts)
}