Compare commits

..
Author SHA1 Message Date
JackDoan 3db4c89e9e relay: Let WriteBatch handle destination changes and drop the flush before responses 2026-10-19 16:25:36 -05:00
Ryan Huber d3e21a8fc8 relay: Batch forwarded packets on the relay host 2026-10-17 18:53:36 +00:00
Wade Simmons cda3710d5a fix error return in handleOutsideMessagePacket (#1970)
smoke-extra / freebsd-amd64 (push) Failing after 17s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 16s
smoke-extra / netbsd-amd64 (push) Failing after 16s
smoke-extra / openbsd-amd64 (push) Failing after 16s
smoke-extra / linux-386 (push) Failing after 17s
smoke / Run multi node smoke test (push) Failing after 1m39s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m18s
Build and test / Test linux-pkcs11 (push) Failing after 2m5s
Build and test / Test linux-boringcrypto (push) Failing after 3m1s
Build and test / Test linux-fips140 (push) Failing after 2m55s
Build and test / Cross-build linux-arm (push) Successful in 3m0s
Build and test / Cross-build linux-mips (push) Successful in 3m42s
Build and test / Cross-build linux-other (push) Successful in 3m5s
Build and test / Cross-build windows (push) Successful in 1m2s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m31s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m21s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
Introduced by #1961, we should still return early for invalid packets
2026-10-17 11:09:25 -04:00
John Maguire c26871fe08 service: Close listeners and tear down the netstack on shutdown (#1954) 2026-10-17 11:08:29 -04:00
Caleb Jasik 7daacc6a48 overlay/darwin: test tun framing and zero-alloc Read/Write (#1906)
smoke-extra / freebsd-amd64 (push) Failing after 16s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 16s
smoke-extra / netbsd-amd64 (push) Failing after 17s
smoke-extra / openbsd-amd64 (push) Failing after 17s
smoke-extra / linux-386 (push) Failing after 16s
smoke / Run multi node smoke test (push) Failing after 1m41s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m47s
Build and test / Test linux-pkcs11 (push) Failing after 2m6s
Build and test / Test linux-boringcrypto (push) Failing after 2m57s
Build and test / Test linux-fips140 (push) Failing after 2m53s
Build and test / Cross-build linux-arm (push) Successful in 3m7s
Build and test / Cross-build linux-mips (push) Successful in 3m55s
Build and test / Cross-build linux-other (push) Successful in 3m13s
Build and test / Cross-build windows (push) Successful in 1m4s
Build and test / Cross-build freebsd (push) Successful in 1m35s
Build and test / Cross-build netbsd (push) Successful in 1m35s
Build and test / Cross-build openbsd (push) Successful in 1m34s
Build and test / Cross-build mobile (push) Successful in 3m26s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-16 18:12:58 -05:00
John Maguire 428dad94b3 Reject v2 cert names and groups at signing the parser rejects (#1967)
smoke-extra / freebsd-amd64 (push) Failing after 18s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 18s
smoke-extra / netbsd-amd64 (push) Failing after 16s
smoke-extra / openbsd-amd64 (push) Failing after 17s
smoke-extra / linux-386 (push) Failing after 17s
smoke / Run multi node smoke test (push) Failing after 1m41s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m19s
Build and test / Test linux-pkcs11 (push) Failing after 2m5s
Build and test / Test linux-boringcrypto (push) Failing after 2m56s
Build and test / Test linux-fips140 (push) Failing after 2m51s
Build and test / Cross-build linux-arm (push) Successful in 3m0s
Build and test / Cross-build linux-mips (push) Successful in 3m43s
Build and test / Cross-build linux-other (push) Successful in 3m7s
Build and test / Cross-build windows (push) Successful in 1m2s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m33s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m23s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-16 18:05:51 -04:00
John Maguire 5bf666dd90 Treat uppercase blocked fingerprints the same as lowercase (#1964)
- Error for invalid on startup 
- Still block the valid blocklist entries when a reload has a bad one
2026-10-16 16:28:37 -04:00
John Maguire 49207da077 Remove the ssh debug profiling commands (#1965)
We plan to move this functionality into the debug socket.
2026-10-16 15:47:56 -04:00
26 changed files with 844 additions and 648 deletions

No files matched your search

+7
View File
@@ -7,8 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
### Removed
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
`mutex-profile-fraction` commands, and `sshd.sandbox_dir`, which is now ignored if set.
### Fixed ### Fixed
- Signing a v2 certificate with an empty name, a name longer than 253 bytes, or an empty group now fails.
These certificates were signed successfully but could not be loaded.
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula - `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
+10 -1
View File
@@ -389,7 +389,16 @@ func (c *certificateV2) fromTBSCertificate(t *TBSCertificate) error {
} }
func (c *certificateV2) validate() error { func (c *certificateV2) validate() error {
// Empty names are allowed // Mirror the name and group rules in unmarshalDetails so signing fails instead of producing an unparseable certificate
if c.details.name == "" || len(c.details.name) > MaxNameLength {
return NewErrInvalidCertificateProperties("name must be between 1 and %d bytes", MaxNameLength)
}
for _, group := range c.details.groups {
if group == "" {
return NewErrInvalidCertificateProperties("groups may not be empty")
}
}
if len(c.publicKey) == 0 { if len(c.publicKey) == 0 {
return ErrInvalidPublicKey return ErrInvalidPublicKey
+45
View File
@@ -6,6 +6,7 @@ import (
"crypto/elliptic" "crypto/elliptic"
"crypto/rand" "crypto/rand"
"net/netip" "net/netip"
"strings"
"testing" "testing"
"time" "time"
@@ -135,3 +136,47 @@ func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
assert.True(t, normie) assert.True(t, normie)
} }
} }
func TestCertificateV2_SignRejectsUnparseable(t *testing.T) {
_, priv, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
var invalid *ErrInvalidCertificateProperties
newTBS := func() TBSCertificate {
return TBSCertificate{
Version: Version2,
Name: "testing",
Networks: []netip.Prefix{mustParsePrefixUnmapped("10.1.1.1/24")},
Groups: []string{"test-group1"},
NotBefore: time.Now().Add(-time.Minute),
NotAfter: time.Now().Add(time.Minute),
PublicKey: []byte("1234567890abcedfghij1234567890ab"),
Curve: Curve_CURVE25519,
IsCA: true,
}
}
tbs := newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength)
c, err := tbs.Sign(nil, Curve_CURVE25519, priv)
require.NoError(t, err)
b, err := c.Marshal()
require.NoError(t, err)
_, err = unmarshalCertificateV2(b, nil, Curve_CURVE25519)
require.NoError(t, err)
tbs = newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength+1)
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Name = ""
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Groups = []string{"test-group1", ""}
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
}
+1 -7
View File
@@ -8,6 +8,7 @@ pki:
cert: /etc/nebula/host.crt cert: /etc/nebula/host.crt
key: /etc/nebula/host.key key: /etc/nebula/host.key
# blocklist is a list of certificate fingerprints that we will refuse to talk to # blocklist is a list of certificate fingerprints that we will refuse to talk to
# Entries are the hex fingerprints shown by `nebula-cert print`, an invalid entry fails startup and reload.
#blocklist: #blocklist:
# - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72 # - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72
# disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid. # disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid.
@@ -232,13 +233,6 @@ punchy:
# Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access. # Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
#trusted_cas: #trusted_cas:
#- "ssh public key string" #- "ssh public key string"
# sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
# save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
# and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
# The directory is NOT automatically created.
# Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
# Not reloadable.
#sandbox_dir: /var/tmp/nebula-debug
# EXPERIMENTAL: relay support for networks that can't establish direct connections. # EXPERIMENTAL: relay support for networks that can't establish direct connections.
relay: relay:
+34 -101
View File
@@ -32,21 +32,16 @@ type conn struct {
Expires time.Time // Time when this conntrack entry will expire Expires time.Time // Time when this conntrack entry will expire
// record why the original connection passed the firewall, so we can re-validate // record why the original connection passed the firewall, so we can re-validate
// after ruleset changes or when the peer handshakes again, which may present a // after ruleset changes. Note, rulesVersion is a uint16 so that these two
// cert with different groups. These pack into the word after Expires. // fields pack for free after the uint32 above
incoming bool incoming bool
rulesVersion uint16 rulesVersion uint16
localIndex uint32
} }
// TODO: need conntrack max tracked connections handling // TODO: need conntrack max tracked connections handling
type Firewall struct { type Firewall struct {
Conntrack *FirewallConntrack Conntrack *FirewallConntrack
// siblings finds the other gateways we route an unsafe network through, so a reply that comes back through a
// different gateway than the flow went out through can be matched to it. Nil turns that off.
siblings gatewaySiblings
InRules *FirewallTable InRules *FirewallTable
OutRules *FirewallTable OutRules *FirewallTable
@@ -87,8 +82,8 @@ type firewallMetrics struct {
type FirewallConntrack struct { type FirewallConntrack struct {
sync.Mutex sync.Mutex
Conns map[firewall.ConntrackKey]*conn Conns map[firewall.Packet]*conn
TimerWheel *TimerWheel[firewall.ConntrackKey] TimerWheel *TimerWheel[firewall.Packet]
} }
// FirewallTable is the entry point for a rule, the evaluation order is: // FirewallTable is the entry point for a rule, the evaluation order is:
@@ -172,8 +167,8 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
return &Firewall{ return &Firewall{
Conntrack: &FirewallConntrack{ Conntrack: &FirewallConntrack{
Conns: make(map[firewall.ConntrackKey]*conn), Conns: make(map[firewall.Packet]*conn),
TimerWheel: NewTimerWheel[firewall.ConntrackKey](tmin, tmax), TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax),
}, },
InRules: newFirewallTable(), InRules: newFirewallTable(),
OutRules: newFirewallTable(), OutRules: newFirewallTable(),
@@ -430,7 +425,6 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
// returns nil if the packet should not be dropped. // returns nil if the packet should not be dropped.
func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error { func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
// Make sure remote address matches nebula certificate, and determine how to treat it // Make sure remote address matches nebula certificate, and determine how to treat it
unsafe := false
if h.networks == nil { if h.networks == nil {
// Simple case: Certificate has one address and no unsafe networks // Simple case: Certificate has one address and no unsafe networks
if h.vpnAddrs[0] != fp.RemoteAddr { if h.vpnAddrs[0] != fp.RemoteAddr {
@@ -450,7 +444,7 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
f.metrics(incoming).droppedRemoteAddr.Inc(1) f.metrics(incoming).droppedRemoteAddr.Inc(1)
return ErrPeerRejected // reject for now, one day this may have different FW rules return ErrPeerRejected // reject for now, one day this may have different FW rules
case NetworkTypeUnsafe: case NetworkTypeUnsafe:
unsafe = true // nothing else special, one day this may have different FW rules break // nothing special, one day this may have different FW rules
default: default:
f.metrics(incoming).droppedRemoteAddr.Inc(1) f.metrics(incoming).droppedRemoteAddr.Inc(1)
return ErrUnknownNetworkType //should never happen return ErrUnknownNetworkType //should never happen
@@ -463,11 +457,8 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
return ErrInvalidLocalIP return ErrInvalidLocalIP
} }
// The first vpn address identifies the peer, as it does elsewhere. A host's v1 and v2 certs must agree on it
key := firewall.ConntrackKey{Packet: fp, Peer: h.vpnAddrs[0]}
// Check if we spoke to this tuple, if we did then allow this packet // Check if we spoke to this tuple, if we did then allow this packet
if f.inConns(key, h, caPool, localCache) { if f.inConns(fp, h, caPool, localCache) {
return nil return nil
} }
@@ -478,15 +469,12 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
// We now know which firewall table to check against // We now know which firewall table to check against
if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) { if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
if unsafe && f.inheritConn(key, incoming, h, caPool) {
return nil
}
f.metrics(incoming).droppedNoRule.Inc(1) f.metrics(incoming).droppedNoRule.Inc(1)
return ErrNoMatchingRule return ErrNoMatchingRule
} }
// We always want to conntrack since it is a faster operation // We always want to conntrack since it is a faster operation
f.addConn(key, incoming, h) f.addConn(fp, incoming)
return nil return nil
} }
@@ -515,9 +503,9 @@ func (f *Firewall) EmitStats() {
metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV())) metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
} }
func (f *Firewall) inConns(key firewall.ConntrackKey, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool { func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
if localCache != nil { if localCache != nil {
if _, ok := localCache[key]; ok { if _, ok := localCache[fp]; ok {
return true return true
} }
} }
@@ -530,52 +518,49 @@ func (f *Firewall) inConns(key firewall.ConntrackKey, h *HostInfo, caPool *cert.
f.evict(ep) f.evict(ep)
} }
c, ok := conntrack.Conns[key] c, ok := conntrack.Conns[fp]
if !ok { if !ok {
conntrack.Unlock() conntrack.Unlock()
return false return false
} }
if c.rulesVersion != f.rulesVersion || c.localIndex != h.localIndexId { if c.rulesVersion != f.rulesVersion {
// This conntrack entry was allowed by an older rule set or for an earlier tunnel to the peer, validate // This conntrack entry was for an older rule set, validate
// it still passes with the current rule set and the peer's current certificate // it still passes with the current rule set
table := f.OutRules table := f.OutRules
if c.incoming { if c.incoming {
table = f.InRules table = f.InRules
} }
// We now know which firewall table to check against // We now know which firewall table to check against
if !table.match(key.Packet, c.incoming, h.ConnectionState.peerCert, caPool) { if !table.match(fp, c.incoming, h.ConnectionState.peerCert, caPool) {
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
h.logger(f.l).Debug("dropping old conntrack entry, does not match current ruleset and certificate", h.logger(f.l).Debug("dropping old conntrack entry, does not match new ruleset",
"fwPacket", key.Packet, "fwPacket", fp,
"incoming", c.incoming, "incoming", c.incoming,
"rulesVersion", f.rulesVersion, "rulesVersion", f.rulesVersion,
"oldRulesVersion", c.rulesVersion, "oldRulesVersion", c.rulesVersion,
"oldLocalIndex", c.localIndex,
) )
} }
delete(conntrack.Conns, key) delete(conntrack.Conns, fp)
conntrack.Unlock() conntrack.Unlock()
return false return false
} }
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
h.logger(f.l).Debug("keeping old conntrack entry, does match current ruleset and certificate", h.logger(f.l).Debug("keeping old conntrack entry, does match new ruleset",
"fwPacket", key.Packet, "fwPacket", fp,
"incoming", c.incoming, "incoming", c.incoming,
"rulesVersion", f.rulesVersion, "rulesVersion", f.rulesVersion,
"oldRulesVersion", c.rulesVersion, "oldRulesVersion", c.rulesVersion,
"oldLocalIndex", c.localIndex,
) )
} }
c.rulesVersion = f.rulesVersion c.rulesVersion = f.rulesVersion
c.localIndex = h.localIndexId
} }
switch key.Protocol { switch fp.Protocol {
case iputil.IPProtocolTCP: case iputil.IPProtocolTCP:
c.Expires = time.Now().Add(f.TCPTimeout) c.Expires = time.Now().Add(f.TCPTimeout)
case iputil.IPProtocolUDP: case iputil.IPProtocolUDP:
@@ -587,17 +572,17 @@ func (f *Firewall) inConns(key firewall.ConntrackKey, h *HostInfo, caPool *cert.
conntrack.Unlock() conntrack.Unlock()
if localCache != nil { if localCache != nil {
localCache[key] = struct{}{} localCache[fp] = struct{}{}
} }
return true return true
} }
func (f *Firewall) addConn(key firewall.ConntrackKey, incoming bool, h *HostInfo) { func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
var timeout time.Duration var timeout time.Duration
c := &conn{} c := &conn{}
switch key.Protocol { switch fp.Protocol {
case iputil.IPProtocolTCP: case iputil.IPProtocolTCP:
timeout = f.TCPTimeout timeout = f.TCPTimeout
case iputil.IPProtocolUDP: case iputil.IPProtocolUDP:
@@ -608,78 +593,26 @@ func (f *Firewall) addConn(key firewall.ConntrackKey, incoming bool, h *HostInfo
conntrack := f.Conntrack conntrack := f.Conntrack
conntrack.Lock() conntrack.Lock()
if _, ok := conntrack.Conns[key]; !ok { if _, ok := conntrack.Conns[fp]; !ok {
conntrack.TimerWheel.Advance(time.Now()) conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(key, timeout) conntrack.TimerWheel.Add(fp, timeout)
} }
// Record which rulesVersion and tunnel allowed this connection, so we can retest after // Record which rulesVersion allowed this connection, so we can retest after
// firewall reload or when the peer handshakes again // firewall reload
c.incoming = incoming c.incoming = incoming
c.rulesVersion = f.rulesVersion c.rulesVersion = f.rulesVersion
c.localIndex = h.localIndexId
c.Expires = time.Now().Add(timeout) c.Expires = time.Now().Add(timeout)
conntrack.Conns[key] = c conntrack.Conns[fp] = c
conntrack.Unlock() conntrack.Unlock()
} }
// gatewaySiblings finds the other gateways we route an unsafe network through
type gatewaySiblings interface {
// siblingPeers appends to buf the conntrack peer of every other gateway we route remote through, if h is one of
// those gateways
siblingPeers(remote netip.Addr, h *HostInfo, buf []netip.Addr) []netip.Addr
}
// inheritConn lets h carry a flow that another of our gateways for the same unsafe network already carries. We
// spread the flows we open across the gateways, but the hosts behind them answer through whichever gateway their own
// routing picks, so a reply can come back through a different gateway than the flow went out through. The flow is
// checked against the rules that allowed it in the first place with h's certificate, the same check an entry gets
// when its peer reconnects. Peers that aren't one of our gateways for the network never inherit a flow.
func (f *Firewall) inheritConn(key firewall.ConntrackKey, incoming bool, h *HostInfo, caPool *cert.CAPool) bool {
if f.siblings == nil {
return false
}
var buf [4]netip.Addr
now := time.Now()
for _, peer := range f.siblings.siblingPeers(key.RemoteAddr, h, buf[:0]) {
sk := key
sk.Peer = peer
conntrack := f.Conntrack
conntrack.Lock()
c, ok := conntrack.Conns[sk]
live := ok && c.Expires.After(now)
var allowedIncoming bool
if live {
allowedIncoming = c.incoming
}
conntrack.Unlock()
// The rules for this packet's own direction already said no, only a flow allowed the other way can help
if !live || allowedIncoming == incoming {
continue
}
table := f.OutRules
if allowedIncoming {
table = f.InRules
}
if !table.match(key.Packet, allowedIncoming, h.ConnectionState.peerCert, caPool) {
continue
}
f.addConn(key, allowedIncoming, h)
return true
}
return false
}
// Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel // Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel
// Caller must own the connMutex lock! // Caller must own the connMutex lock!
func (f *Firewall) evict(key firewall.ConntrackKey) { func (f *Firewall) evict(p firewall.Packet) {
// Are we still tracking this conn? // Are we still tracking this conn?
conntrack := f.Conntrack conntrack := f.Conntrack
t, ok := conntrack.Conns[key] t, ok := conntrack.Conns[p]
if !ok { if !ok {
return return
} }
@@ -689,12 +622,12 @@ func (f *Firewall) evict(key firewall.ConntrackKey) {
// Timeout is in the future, re-add the timer // Timeout is in the future, re-add the timer
if newT > 0 { if newT > 0 {
conntrack.TimerWheel.Advance(time.Now()) conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(key, newT) conntrack.TimerWheel.Add(p, newT)
return return
} }
// This conn is done // This conn is done
delete(conntrack.Conns, key) delete(conntrack.Conns, p)
} }
func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool { func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
+1 -9
View File
@@ -3,23 +3,15 @@ package firewall
import ( import (
"context" "context"
"log/slog" "log/slog"
"net/netip"
"sync/atomic" "sync/atomic"
"time" "time"
"github.com/slackhq/nebula/logging" "github.com/slackhq/nebula/logging"
) )
// ConntrackKey identifies a tracked flow by its packet and the peer it was allowed for, so peers that route
// the same unsafe network never share a conntrack entry.
type ConntrackKey struct {
Peer netip.Addr
Packet
}
// ConntrackCache is used as a local routine cache to know if a given flow // ConntrackCache is used as a local routine cache to know if a given flow
// has been seen in the conntrack table. // has been seen in the conntrack table.
type ConntrackCache map[ConntrackKey]struct{} type ConntrackCache map[Packet]struct{}
type ConntrackCacheTicker struct { type ConntrackCacheTicker struct {
cacheV uint64 cacheV uint64
+1 -1
View File
@@ -23,7 +23,7 @@ func newFixedTicker(t *testing.T, l *slog.Logger, cacheLen int) *ConntrackCacheT
cache: make(ConntrackCache, cacheLen), cache: make(ConntrackCache, cacheLen),
} }
for i := range cacheLen { for i := range cacheLen {
c.cache[ConntrackKey{Packet: Packet{LocalPort: uint16(i) + 1}}] = struct{}{} c.cache[Packet{LocalPort: uint16(i) + 1}] = struct{}{}
} }
c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path
return c return c
-4
View File
@@ -26,10 +26,6 @@ type Packet struct {
RemotePort uint16 RemotePort uint16
Protocol uint8 Protocol uint8
Fragment bool Fragment bool
// pad fills what would otherwise be trailing padding so maps hash and compare a ConntrackKey as one
// block of memory instead of field by field. It must always be zero.
pad [2]byte
} }
func (fp *Packet) Copy() *Packet { func (fp *Packet) Copy() *Packet {
+1 -248
View File
@@ -3,7 +3,6 @@ package nebula
import ( import (
"bytes" "bytes"
"errors" "errors"
"fmt"
"log/slog" "log/slog"
"math" "math"
"net/netip" "net/netip"
@@ -980,252 +979,6 @@ func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) {
"SECURITY: attacker spoofed victim's overlay source IP (192.0.2.2) by reusing an existing conntrack 4-tuple; Drop returned %v instead of rejecting", got) "SECURITY: attacker spoofed victim's overlay source IP (192.0.2.2) by reusing an existing conntrack 4-tuple; Drop returned %v instead of rejecting", got)
} }
func TestFirewall_ConntrackUnsafeNetworkPeers(t *testing.T) {
l := test.NewLoggerWithOutput(&bytes.Buffer{})
myVpnNetworksTable := new(bart.Lite)
myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
owner := &dummyCert{
name: "owner",
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
}
// Two routers for the same unsafe network, only one of them is trusted inbound
router := func(name string, addr string, groups ...string) *HostInfo {
inverted := map[string]struct{}{}
for _, g := range groups {
inverted[g] = struct{}{}
}
c := &cert.CachedCertificate{
Certificate: &dummyCert{
name: name,
groups: groups,
networks: []netip.Prefix{netip.MustParsePrefix(addr + "/24")},
unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
},
InvertedGroups: inverted,
}
h := &HostInfo{
ConnectionState: &ConnectionState{peerCert: c},
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
}
h.buildNetworks(myVpnNetworksTable, c.Certificate)
return h
}
trusted := router("trusted", "192.0.2.2", "trusted")
other := router("other", "192.0.2.3")
cp := cert.NewCAPool()
flow := firewall.Packet{
LocalAddr: netip.MustParseAddr("192.0.2.1"),
RemoteAddr: netip.MustParseAddr("198.51.100.5"),
LocalPort: 443,
RemotePort: 55000,
Protocol: iputil.IPProtocolTCP,
}
for _, useCache := range []bool{false, true} {
var cache firewall.ConntrackCache
if useCache {
cache = firewall.ConntrackCache{}
}
t.Run(fmt.Sprintf("inbound flow allowed for one peer/cache=%v", useCache), func(t *testing.T) {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"trusted"}, "", "", "", "", ""))
clear(cache)
require.NoError(t, fw.Drop(flow, true, trusted, cp, cache))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, other, cp, cache))
require.NoError(t, fw.Drop(flow, true, trusted, cp, cache))
})
t.Run(fmt.Sprintf("reply to a flow opened via one peer/cache=%v", useCache), func(t *testing.T) {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
clear(cache)
require.NoError(t, fw.Drop(flow, false, trusted, cp, cache))
require.NoError(t, fw.Drop(flow, true, trusted, cp, cache))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, other, cp, cache))
})
}
}
// fakeSiblings maps each gateway to the conntrack peers of the gateways it shares a route with
type fakeSiblings map[*HostInfo][]netip.Addr
func (s fakeSiblings) siblingPeers(_ netip.Addr, h *HostInfo, buf []netip.Addr) []netip.Addr {
return append(buf, s[h]...)
}
func TestFirewall_ConntrackGatewaySiblings(t *testing.T) {
l := test.NewLoggerWithOutput(&bytes.Buffer{})
myVpnNetworksTable := new(bart.Lite)
myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
owner := &dummyCert{
name: "owner",
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
}
// Routers for the same unsafe network. a and b are both our gateways for it, c only claims it in its cert
router := func(name string, addr string, groups ...string) *HostInfo {
inverted := map[string]struct{}{}
for _, g := range groups {
inverted[g] = struct{}{}
}
c := &cert.CachedCertificate{
Certificate: &dummyCert{
name: name,
groups: groups,
networks: []netip.Prefix{netip.MustParsePrefix(addr + "/24")},
unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
},
InvertedGroups: inverted,
}
h := &HostInfo{
ConnectionState: &ConnectionState{peerCert: c},
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
}
h.buildNetworks(myVpnNetworksTable, c.Certificate)
return h
}
a := router("a", "192.0.2.2", "trusted")
b := router("b", "192.0.2.3", "trusted")
c := router("c", "192.0.2.4", "trusted")
siblings := fakeSiblings{a: {b.vpnAddrs[0]}, b: {a.vpnAddrs[0]}}
cp := cert.NewCAPool()
flow := firewall.Packet{
LocalAddr: netip.MustParseAddr("192.0.2.1"),
RemoteAddr: netip.MustParseAddr("198.51.100.5"),
LocalPort: 55000,
RemotePort: 443,
Protocol: iputil.IPProtocolTCP,
}
// We open flows through our gateways, nothing is allowed in on its own
newFirewall := func(outGroup string) *Firewall {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{outGroup}, "", "", "", "", ""))
fw.siblings = siblings
return fw
}
t.Run("reply through the other gateway", func(t *testing.T) {
fw := newFirewall("trusted")
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
require.NoError(t, fw.Drop(flow, true, a, cp, nil))
// a has its own entry now, and keeps the flow when b's is gone
delete(fw.Conntrack.Conns, firewall.ConntrackKey{Packet: flow, Peer: b.vpnAddrs[0]})
require.NoError(t, fw.Drop(flow, true, a, cp, nil))
})
t.Run("not our gateway", func(t *testing.T) {
fw := newFirewall("trusted")
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, c, cp, nil))
})
t.Run("off without siblings", func(t *testing.T) {
fw := newFirewall("trusted")
fw.siblings = nil
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, a, cp, nil))
})
t.Run("expired sibling entry", func(t *testing.T) {
fw := newFirewall("trusted")
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
fw.Conntrack.Conns[firewall.ConntrackKey{Packet: flow, Peer: b.vpnAddrs[0]}].Expires = time.Now().Add(-time.Second)
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, a, cp, nil))
})
t.Run("the original rules must allow the new gateway", func(t *testing.T) {
fw := newFirewall("b-only")
b.ConnectionState.peerCert.InvertedGroups["b-only"] = struct{}{}
defer delete(b.ConnectionState.peerCert.InvertedGroups, "b-only")
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, a, cp, nil))
})
}
func TestFirewall_ConntrackPeerRehandshake(t *testing.T) {
l := test.NewLoggerWithOutput(&bytes.Buffer{})
myVpnNetworksTable := new(bart.Lite)
myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
owner := &dummyCert{
name: "owner",
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
}
// Each call is a new tunnel to the same peer, presenting a cert with the given groups
tunnel := func(localIndex uint32, groups ...string) *HostInfo {
inverted := map[string]struct{}{}
for _, g := range groups {
inverted[g] = struct{}{}
}
c := &cert.CachedCertificate{
Certificate: &dummyCert{
name: "peer",
groups: groups,
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24"), netip.MustParsePrefix("2001:db8::2/64")},
},
InvertedGroups: inverted,
}
h := &HostInfo{
ConnectionState: &ConnectionState{peerCert: c},
vpnAddrs: []netip.Addr{netip.MustParseAddr("192.0.2.2"), netip.MustParseAddr("2001:db8::2")},
localIndexId: localIndex,
}
h.buildNetworks(myVpnNetworksTable, c.Certificate)
return h
}
cp := cert.NewCAPool()
flow := firewall.Packet{
LocalAddr: netip.MustParseAddr("192.0.2.1"),
RemoteAddr: netip.MustParseAddr("192.0.2.2"),
LocalPort: 443,
RemotePort: 55000,
Protocol: iputil.IPProtocolTCP,
}
key := firewall.ConntrackKey{Packet: flow, Peer: netip.MustParseAddr("192.0.2.2")}
t.Run("inbound flow follows the peer's groups", func(t *testing.T) {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"trusted"}, "", "", "", "", ""))
require.NoError(t, fw.Drop(flow, true, tunnel(1, "trusted"), cp, nil))
// Same groups on a new tunnel keeps the entry and moves it to the new tunnel
require.NoError(t, fw.Drop(flow, true, tunnel(2, "trusted"), cp, nil))
require.Contains(t, fw.Conntrack.Conns, key)
assert.Equal(t, uint32(2), fw.Conntrack.Conns[key].localIndex)
// Losing the group on a new tunnel removes the entry
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, tunnel(3), cp, nil))
assert.NotContains(t, fw.Conntrack.Conns, key)
})
t.Run("reply to a flow we opened survives a new tunnel", func(t *testing.T) {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
require.NoError(t, fw.Drop(flow, false, tunnel(1, "trusted"), cp, nil))
require.NoError(t, fw.Drop(flow, true, tunnel(2), cp, nil))
assert.Equal(t, uint32(2), fw.Conntrack.Conns[key].localIndex)
})
}
// BenchmarkFirewallDropConntrackHit measures Drop on an already-established flow // BenchmarkFirewallDropConntrackHit measures Drop on an already-established flow
// (a conntrack hit). This is the fast path that the source-IP<->cert binding // (a conntrack hit). This is the fast path that the source-IP<->cert binding
// reordering adds work to, so it quantifies the cost of moving the address checks // reordering adds work to, so it quantifies the cost of moving the address checks
@@ -1989,6 +1742,6 @@ func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, end
func resetConntrack(fw *Firewall) { func resetConntrack(fw *Firewall) {
fw.Conntrack.Lock() fw.Conntrack.Lock()
fw.Conntrack.Conns = map[firewall.ConntrackKey]*conn{} fw.Conntrack.Conns = map[firewall.Packet]*conn{}
fw.Conntrack.Unlock() fw.Conntrack.Unlock()
} }
+4 -26
View File
@@ -25,7 +25,6 @@ import (
"github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/ratelimit" "github.com/slackhq/nebula/ratelimit"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
) )
@@ -269,9 +268,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
ifce.reQueryWait.Store(int64(c.reQueryWait)) ifce.reQueryWait.Store(int64(c.reQueryWait))
ifce.connectionManager.intf = ifce ifce.connectionManager.intf = ifce
if ifce.firewall != nil {
ifce.firewall.siblings = ifce
}
// Held until Close so waiting on the interface blocks until the resources are actually released // Held until Close so waiting on the interface blocks until the resources are actually released
ifce.wg.Add(1) ifce.wg.Add(1)
@@ -388,6 +384,8 @@ type rxContext struct {
hostmapCache map[uint32]*HostInfo hostmapCache map[uint32]*HostInfo
lhh *LightHouseHandler lhh *LightHouseHandler
ctCache *firewall.ConntrackCacheTicker ctCache *firewall.ConntrackCacheTicker
// relayBatch holds forwarded relay packets, which borrow their receive buffers until flushed
relayBatch *batch.SendBatch
} }
func newRxContext(f *Interface, q int) *rxContext { func newRxContext(f *Interface, q int) *rxContext {
@@ -400,6 +398,7 @@ func newRxContext(f *Interface, q int) *rxContext {
hostmapCache: map[uint32]*HostInfo{}, hostmapCache: map[uint32]*HostInfo{},
lhh: f.lightHouse.NewRequestHandler(), lhh: f.lightHouse.NewRequestHandler(),
ctCache: firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout), ctCache: firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout),
relayBatch: batch.NewSendBatch(f.writers[q], batch.SendBatchCap, 0),
} }
} }
@@ -418,6 +417,7 @@ func (f *Interface) listenOut(i int) {
} }
flusher := func() { flusher := func() {
f.flushSendBatch(rxc.relayBatch, i)
if err := f.batchers[i].Flush(); err != nil { if err := f.batchers[i].Flush(); err != nil {
f.l.Error("Failed to flush tun coalescer", "error", err) f.l.Error("Failed to flush tun coalescer", "error", err)
} }
@@ -558,7 +558,6 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
} }
oldFw := f.firewall oldFw := f.firewall
fw.siblings = f
conntrack := oldFw.Conntrack conntrack := oldFw.Conntrack
conntrack.Lock() conntrack.Lock()
defer conntrack.Unlock() defer conntrack.Unlock()
@@ -716,24 +715,3 @@ func (f *Interface) Close() error {
f.wg.Done() f.wg.Done()
return errors.Join(errs...) return errors.Join(errs...)
} }
// siblingPeers implements gatewaySiblings from the routes and tunnels we have now. Routes name a gateway by any of its
// vpn addresses, conntrack keys a peer by its first one, so each gateway is resolved through its tunnel. A gateway
// we have no tunnel to can't be carrying a live flow.
func (f *Interface) siblingPeers(remote netip.Addr, h *HostInfo, buf []netip.Addr) []netip.Addr {
gateways := f.inside.RoutesFor(remote)
isGateway := func(g routing.Gateway) bool { return slices.Contains(h.vpnAddrs, g.Addr()) }
if len(gateways) < 2 || !slices.ContainsFunc(gateways, isGateway) {
return buf
}
for _, g := range gateways {
if isGateway(g) {
continue
}
if s := f.hostMap.QueryVpnAddr(g.Addr()); s != nil && s != h {
buf = append(buf, s.vpnAddrs[0])
}
}
return buf
}
+1 -1
View File
@@ -39,7 +39,7 @@ func Test_statsEmitter_setsGauges(t *testing.T) {
hostMap: hostMap, hostMap: hostMap,
inside: &overlaytest.NoopTun{}, inside: &overlaytest.NoopTun{},
outside: &udp.NoopConn{}, outside: &udp.NoopConn{},
firewall: &Firewall{Conntrack: &FirewallConntrack{Conns: map[firewall.ConntrackKey]*conn{}}}, firewall: &Firewall{Conntrack: &FirewallConntrack{Conns: map[firewall.Packet]*conn{}}},
lightHouse: lh, lightHouse: lh,
pki: &PKI{}, pki: &PKI{},
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig), handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
-42
View File
@@ -4,11 +4,8 @@ import (
"net/netip" "net/netip"
"testing" "testing"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/overlay/overlaytest"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -143,42 +140,3 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
assert.True(t, res.Failed) assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall) assert.Same(t, fw, f.firewall)
} }
// routesTun is a NoopTun with a route table
type routesTun struct {
overlaytest.NoopTun
routes *bart.Table[routing.Gateways]
}
func (t routesTun) RoutesFor(ip netip.Addr) routing.Gateways {
r, _ := t.routes.Lookup(ip)
return r
}
func TestInterface_SiblingPeers(t *testing.T) {
addr := netip.MustParseAddr
routes := new(bart.Table[routing.Gateways])
// a and b carry 198.51.100.0/24, the route names b by its v6 address
routes.Insert(netip.MustParsePrefix("198.51.100.0/24"), routing.Gateways{
routing.NewGateway(addr("192.0.2.2"), 1),
routing.NewGateway(addr("fd00::3"), 1),
routing.NewGateway(addr("192.0.2.5"), 1), // no tunnel to this one
})
routes.Insert(netip.MustParsePrefix("203.0.113.0/24"), routing.Gateways{routing.NewGateway(addr("192.0.2.2"), 1)})
hm := newHostMap(test.NewLogger())
a := &HostInfo{vpnAddrs: []netip.Addr{addr("192.0.2.2"), addr("fd00::2")}, localIndexId: 1}
b := &HostInfo{vpnAddrs: []netip.Addr{addr("192.0.2.3"), addr("fd00::3")}, localIndexId: 2}
c := &HostInfo{vpnAddrs: []netip.Addr{addr("192.0.2.4")}, localIndexId: 3}
for _, h := range []*HostInfo{a, b, c} {
hm.unlockedAddHostInfo(h, &Interface{})
}
f := &Interface{inside: routesTun{routes: routes}, hostMap: hm}
// Siblings come back as their first vpn address, the one conntrack keys on, whichever address the route names
assert.Equal(t, []netip.Addr{addr("192.0.2.3")}, f.siblingPeers(addr("198.51.100.5"), a, nil))
assert.Equal(t, []netip.Addr{addr("192.0.2.2")}, f.siblingPeers(addr("198.51.100.5"), b, nil))
assert.Empty(t, f.siblingPeers(addr("198.51.100.5"), c, nil), "not one of our gateways for the network")
assert.Empty(t, f.siblingPeers(addr("203.0.113.5"), a, nil), "a single gateway route has no siblings")
assert.Empty(t, f.siblingPeers(addr("192.0.2.3"), a, nil), "no route")
}
+10 -4
View File
@@ -25,7 +25,7 @@ var ErrOutOfWindow = errors.New("out of window packet")
// readOutsidePackets processes one received underlay packet. // readOutsidePackets processes one received underlay packet.
// Message payloads are decrypted IN PLACE, so packet must stay untouched // Message payloads are decrypted IN PLACE, so packet must stay untouched
// by the caller until the batcher for queue q has been flushed // by the caller until the TUN and relay batches for queue q have been flushed.
func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) { func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) {
h := rxc.h h := rxc.h
err := h.Parse(packet) err := h.Parse(packet)
@@ -233,8 +233,14 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
// Forward this packet through the relay tunnel, rebuilding it in place. // Forward this packet through the relay tunnel, rebuilding it in place.
// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was // Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
fwdBuf := packet[:0] fwdBuf := packet[:0]
//todo it would potentially be nice to batch these out, err := f.prepareSendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true)
f.SendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true, rxc.q) if err == nil {
// Borrows the receive buffer, the listener's flush callback drains it before the buffer is reused
rxc.relayBatch.Commit(out, targetHI.GetRemote())
if rxc.relayBatch.Len() >= batch.SendBatchCap {
f.flushSendBatch(rxc.relayBatch, rxc.q)
}
}
case TerminalType: case TerminalType:
hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal") hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
return return
@@ -478,8 +484,8 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounte
if err != nil { if err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out) hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out)
return
} }
return
} }
dropReason := f.firewall.Drop(rxc.fwPacket.Packet, true, hostinfo, f.pki.GetCAPool(), rxc.ctCache.Get()) dropReason := f.firewall.Drop(rxc.fwPacket.Packet, true, hostinfo, f.pki.GetCAPool(), rxc.ctCache.Get())
+242
View File
@@ -0,0 +1,242 @@
package nebula
import (
"bytes"
"net/netip"
"sync"
"testing"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/require"
)
type relayBatchWrite struct {
packets [][]byte
remotes []netip.AddrPort
batched bool
}
type relayBatchConn struct {
udp.NoopConn
read func(udp.EncReader, func())
writes []relayBatchWrite
}
func (c *relayBatchConn) ListenOut(r udp.EncReader, flush func()) error {
c.read(r, flush)
return nil
}
func (c *relayBatchConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
w := relayBatchWrite{remotes: append([]netip.AddrPort(nil), addrs...), batched: true}
for _, b := range bufs {
w.packets = append(w.packets, bytes.Clone(b))
}
c.writes = append(c.writes, w)
return len(bufs), nil
}
func (c *relayBatchConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.writes = append(c.writes, relayBatchWrite{packets: [][]byte{bytes.Clone(b)}, remotes: []netip.AddrPort{addr}})
return nil
}
func (c *relayBatchConn) packets() [][]byte {
var packets [][]byte
for _, w := range c.writes {
packets = append(packets, w.packets...)
}
return packets
}
type relayBatchPath struct {
f *Interface
conn *relayBatchConn
source, target *HostInfo
incoming, outgoing *Relay
sourceKey noiseutil.CipherState
cipher noise.CipherFunc
counter uint64
}
// Separate cipher instances let the reference SendVia path use the same keys and
// counters without rewinding an AEAD instance (including FIPS nonce checks).
func relayBatchKey(cipher noise.CipherFunc, id byte) noiseutil.CipherState {
suite := noise.NewCipherSuite(noise.DH25519, cipher, noise.HashSHA256)
return noiseutil.NewCipherState(noise.UnsafeNewCipherState(suite, [32]byte{id}, 0), cipher)
}
func newRelayBatchPath(t *testing.T, cipher noise.CipherFunc) *relayBatchPath {
t.Helper()
l := test.NewLogger()
conn := &relayBatchConn{}
f := &Interface{
ctx: t.Context(), l: l, hostMap: newHostMap(l), outside: conn,
writers: []udp.Conn{conn}, lightHouse: &LightHouse{l: l},
myVpnNetworksTable: &bart.Lite{}, metricTxDropped: metrics.NewCounter(),
batchers: []*batch.MultiCoalescer{batch.NewMultiCoalescer(&capturingTun{}, l)},
}
f.connectionManager = &connectionManager{
intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{},
}
newHost := func(addr string, remote string, id byte) *HostInfo {
h := &HostInfo{
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
ConnectionState: &ConnectionState{
eKey: relayBatchKey(cipher, id), dKey: relayBatchKey(cipher, id), window: NewBits(ReplayWindow),
},
relayState: RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}},
}
u := netip.MustParseAddrPort(remote)
h.remote.Store(&u)
f.hostMap.Hosts[h.vpnAddrs[0]] = h
return h
}
source := newHost("192.0.2.1", "198.51.100.1:4242", 1)
target := newHost("192.0.2.2", "198.51.100.2:4242", 2)
incoming := &Relay{Type: ForwardingType, State: Established, LocalIndex: 301, RemoteIndex: 302, PeerAddr: target.vpnAddrs[0]}
outgoing := &Relay{Type: ForwardingType, State: Established, LocalIndex: 303, RemoteIndex: 304, PeerAddr: source.vpnAddrs[0]}
source.relayState.InsertRelay(incoming.PeerAddr, incoming.LocalIndex, incoming)
target.relayState.InsertRelay(outgoing.PeerAddr, outgoing.LocalIndex, outgoing)
f.hostMap.Relays[incoming.LocalIndex] = source
f.hostMap.Indexes[100] = source
return &relayBatchPath{f: f, conn: conn, source: source, target: target, incoming: incoming, outgoing: outgoing, sourceKey: relayBatchKey(cipher, 1), cipher: cipher}
}
func (p *relayBatchPath) wrap(t *testing.T, payload []byte) []byte {
t.Helper()
p.counter++
b := header.Encode(make([]byte, header.Len), header.Version, header.Message, header.MessageRelay, p.incoming.LocalIndex, p.counter)
b = append(b, payload...)
b, err := p.sourceKey.EncryptDanger(b, b, nil, p.counter, make([]byte, 12))
require.NoError(t, err)
return b
}
func (p *relayBatchPath) reference(payloads [][]byte) [][]byte {
conn := &relayBatchConn{}
f := &Interface{writers: []udp.Conn{conn}, connectionManager: p.f.connectionManager, l: p.f.l}
h := &HostInfo{ConnectionState: &ConnectionState{eKey: relayBatchKey(p.cipher, 2)}}
u := p.target.GetRemote()
h.remote.Store(&u)
for _, payload := range payloads {
f.SendVia(h, p.outgoing, payload, make([]byte, 12), make([]byte, 0, len(payload)+header.MaxOverhead), false, 0)
}
return conn.packets()
}
func TestRelayReceiveBatch(t *testing.T) {
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
t.Run(cipher.CipherName(), func(t *testing.T) {
p := newRelayBatchPath(t, cipher)
const count = 8
var payloads, wire [][]byte
for i := range count * 2 {
payload := bytes.Repeat([]byte{byte(i)}, 64)
payloads = append(payloads, payload)
wire = append(wire, p.wrap(t, payload))
}
want := p.reference(payloads)
// Model a listener reusing the same slots for the next receive batch.
buffers := make([][]byte, count)
for i := range buffers {
buffers[i] = make([]byte, len(wire[i]))
}
p.conn.read = func(r udp.EncReader, flush func()) {
for b := range 2 {
for i := range count {
copy(buffers[i], wire[b*count+i])
r(p.source.GetRemote(), buffers[i])
require.Len(t, p.conn.writes, b, "forwarding waits for the batch boundary")
}
flush()
require.Len(t, p.conn.writes, b+1, "flush must send before receive slots are reused")
for _, buffer := range buffers {
clear(buffer)
}
}
}
p.f.listenOut(0)
require.Equal(t, want, p.conn.packets())
for _, w := range p.conn.writes {
require.True(t, w.batched)
require.Len(t, w.packets, count)
for _, remote := range w.remotes {
require.Equal(t, p.target.GetRemote(), remote)
}
}
})
}
}
func TestRelayBatchMixedDestinations(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, second := p.target.GetRemote(), netip.MustParseAddrPort("198.51.100.3:4242")
remotes := []netip.AddrPort{first, first, second, second, first}
var payloads, wire [][]byte
for i := range remotes {
payload := []byte{byte(i)}
payloads = append(payloads, payload)
wire = append(wire, p.wrap(t, payload))
}
want := p.reference(payloads)
p.conn.read = func(r udp.EncReader, flush func()) {
for i, remote := range remotes {
p.target.remote.Store(&remote)
r(p.source.GetRemote(), wire[i])
}
flush()
}
p.f.listenOut(0)
require.Equal(t, want, p.conn.packets())
require.Len(t, p.conn.writes, 1, "WriteBatch splits runs by destination, the batch does not need to")
require.Equal(t, remotes, p.conn.writes[0].remotes)
}
func TestRelayBatchCapacity(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
p.conn.read = func(r udp.EncReader, flush func()) {
for i := range batch.SendBatchCap + 1 {
r(p.source.GetRemote(), p.wrap(t, []byte{byte(i)}))
}
require.Len(t, p.conn.writes, 1)
require.Len(t, p.conn.writes[0].packets, batch.SendBatchCap)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 2)
require.Len(t, p.conn.writes[1].packets, 1)
}
func TestRelayBatchRemovedTarget(t *testing.T) {
for _, remove := range []string{"host", "relay", "disestablished"} {
t.Run(remove, func(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, removed := p.wrap(t, []byte("first")), p.wrap(t, []byte("removed"))
want := p.reference([][]byte{[]byte("first")})
p.conn.read = func(r udp.EncReader, flush func()) {
r(p.source.GetRemote(), first)
switch remove {
case "host":
delete(p.f.hostMap.Hosts, p.target.vpnAddrs[0])
case "relay":
delete(p.target.relayState.relayForByAddr, p.source.vpnAddrs[0])
case "disestablished":
p.target.relayState.UpdateRelayForByIpState(p.source.vpnAddrs[0], Disestablished)
}
r(p.source.GetRemote(), removed)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 1)
require.Equal(t, want, p.conn.packets(), "only the packet accepted before removal is sent")
})
}
}
+2 -2
View File
@@ -10,8 +10,8 @@ type batchWriter interface {
} }
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch. // SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
// One SendBatch is owned by each listenIn goroutine; no locking is needed. // Each reader goroutine owns its own SendBatch; no locking is needed.
// Slots are backed by an Arena (see its docs) // Slots can borrow receive buffers or be backed by an Arena (see its docs).
type SendBatch struct { type SendBatch struct {
out batchWriter out batchWriter
bufs [][]byte bufs [][]byte
+121
View File
@@ -0,0 +1,121 @@
//go:build !ios && !e2e_testing
package overlay
import (
"bytes"
"os"
"syscall"
"testing"
"github.com/stretchr/testify/require"
)
func TestTun_WriteFraming(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: w}
for _, tc := range []struct {
name string
pkt []byte
af byte
}{
// Literal families rather than syscall.AF_*, so a wrong constant in Write can't also fix the expectation.
{"ipv4", []byte{0x45, 1, 2, 3, 4, 5}, 2},
{"ipv6", []byte{0x60, 1, 2, 3, 4, 5}, 30},
} {
t.Run(tc.name, func(t *testing.T) {
n, err := tn.Write(tc.pkt)
require.NoError(t, err)
require.Equal(t, len(tc.pkt), n)
got := make([]byte, 64)
n, err = r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, tc.af}, tc.pkt...), got[:n])
})
}
_, err = tn.Write([]byte{0x10, 1, 2, 3})
require.Error(t, err)
_, err = tn.Write([]byte{})
require.ErrorIs(t, err, syscall.EIO)
// The rejected writes must not have put anything on the fd, so the next read sees only this packet.
pkt := []byte{0x45, 9, 9}
_, err = tn.Write(pkt)
require.NoError(t, err)
got := make([]byte, 64)
n, err := r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, 2}, pkt...), got[:n])
}
func TestTun_ReadStripsHeader(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
pkt := []byte{0x45, 1, 2, 3, 4, 5}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err := tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
func TestTun_ReadShort(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
// Fewer bytes than the protocol header is not a packet.
_, err = w.Write([]byte{0, 0})
require.NoError(t, err)
n, err := tn.Read(make([]byte, 64))
require.NoError(t, err)
require.Zero(t, n)
// The next packet reads cleanly only if the short read consumed exactly what was written.
pkt := []byte{0x45, 1, 2, 3}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err = tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
// A heap allocation per packet rarely shows in a short throughput run, only across GC pauses, so
// this pins Read and Write to zero allocations directly.
func TestTun_ReadWriteDoNotAllocate(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tw := &tun{f: w}
tr := &tun{f: r}
pkts := [][]byte{bytes.Repeat([]byte{0x45}, 1300), bytes.Repeat([]byte{0x60}, 1300)}
buf := make([]byte, 1500)
i := 0
allocs := testing.AllocsPerRun(1000, func() {
i++
if _, err := tw.Write(pkts[i%len(pkts)]); err != nil {
t.Fatal(err)
}
if _, err := tr.Read(buf); err != nil {
t.Fatal(err)
}
})
require.Zero(t, allocs)
}
+19 -4
View File
@@ -2,7 +2,9 @@ package nebula
import ( import (
"crypto/fips140" "crypto/fips140"
"crypto/sha256"
"encoding/binary" "encoding/binary"
"encoding/hex"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@@ -233,12 +235,14 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError { func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError {
caPool, err := loadCAPoolFromConfig(p.l, c) caPool, err := loadCAPoolFromConfig(p.l, c)
if caPool != nil {
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
}
if err != nil { if err != nil {
return util.NewContextualError("Failed to load ca from config", nil, err) return util.NewContextualError("Failed to load ca from config", nil, err)
} }
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
return nil return nil
} }
@@ -613,11 +617,22 @@ func loadCAPoolFromConfig(l *slog.Logger, c *config.C) (*cert.CAPool, error) {
bl := c.GetStringSlice("pki.blocklist", []string{}) bl := c.GetStringSlice("pki.blocklist", []string{})
if len(bl) > 0 { if len(bl) > 0 {
for _, fp := range bl { var bad []string
for i, raw := range bl {
// Fingerprints are compared as lowercase hex strings, a malformed entry would silently block nothing
fp := strings.ToLower(strings.TrimSpace(raw))
if b, err := hex.DecodeString(fp); err != nil || len(b) != sha256.Size {
bad = append(bad, fmt.Sprintf("entry %d %q", i+1, raw))
continue
}
caPool.BlocklistFingerprint(fp) caPool.BlocklistFingerprint(fp)
} }
l.Info("Blocklisted certificates", "fingerprintCount", len(bl)) l.Info("Blocklisted certificates", "fingerprintCount", len(bl)-len(bad))
if len(bad) > 0 {
// The valid entries are still returned so a failed reload blocks them, startup refuses the error
return caPool, fmt.Errorf("pki.blocklist entries are not hex certificate fingerprints: %s", strings.Join(bad, ", "))
}
} }
return caPool, nil return caPool, nil
+43
View File
@@ -217,3 +217,46 @@ func TestPKIReloadChachapolyInFIPSOnlyFails(t *testing.T) {
assert.True(t, res.Failed) assert.True(t, res.Failed)
assert.False(t, res.RestartRequired) assert.False(t, res.RestartRequired)
} }
func TestLoadCAPoolBlocklist(t *testing.T) {
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
crt, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil)
fp, err := crt.Fingerprint()
require.NoError(t, err)
load := func(entry string) (*cert.CAPool, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(fmt.Sprintf("pki:\n ca: %q\n blocklist:\n - %q\n", caPEM, entry)))
return loadCAPoolFromConfig(test.NewLogger(), c)
}
for _, entry := range []string{fp, strings.ToUpper(fp), " " + fp + "\t"} {
pool, err := load(entry)
require.NoError(t, err, entry)
assert.True(t, pool.IsBlocklisted(fp), entry)
}
for _, entry := range []string{fp[1:], fp + "00", "sha256:" + fp, "zz" + fp[2:], ""} {
_, err := load(entry)
require.Error(t, err, entry)
}
// A bad entry on reload fails it, the valid entries are still blocked
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "me", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.1/24")}, nil, nil)
pkiConfig := func(entry string) string {
return fmt.Sprintf("pki:\n ca: %q\n cert: %q\n key: %q\n blocklist:\n - %q\n", caPEM, certPEM, keyPEM, entry)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig(fp)))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
other, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "other", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.3/24")}, nil, nil)
otherFp, err := other.Fingerprint()
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig(fp) + fmt.Sprintf(" - %q\n - %q\n", otherFp, fp[1:]))
assert.True(t, res.Failed)
assert.True(t, p.GetCAPool().IsBlocklisted(fp))
assert.True(t, p.GetCAPool().IsBlocklisted(otherFp))
}
+4 -2
View File
@@ -36,11 +36,13 @@ func (l *tcpListener) Accept() (net.Conn, error) {
func (l *tcpListener) Close() error { func (l *tcpListener) Close() error {
l.closeOnce.Do(func() { l.closeOnce.Do(func() {
// Close before leaving the map so a concurrent closeListeners that
// no longer sees this listener knows Accept is already released.
close(l.closed)
l.s.mu.Lock() l.s.mu.Lock()
delete(l.s.mu.listeners, l.port) delete(l.s.mu.listeners, l.port)
l.s.mu.Unlock() l.s.mu.Unlock()
close(l.closed)
}) })
return nil return nil
} }
+70 -7
View File
@@ -32,13 +32,18 @@ import (
const nicID = 1 const nicID = 1
type Service struct { type Service struct {
eg *errgroup.Group
control *nebula.Control control *nebula.Control
ipstack *stack.Stack ipstack *stack.Stack
// done is closed once the forwarding goroutines have exited and the
// netstack is torn down, at which point err holds the first fatal error.
done chan struct{}
err error
mu struct { mu struct {
sync.Mutex sync.Mutex
closed bool
listeners map[uint16]*tcpListener listeners map[uint16]*tcpListener
} }
} }
@@ -65,7 +70,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
ctx := control.Context() ctx := control.Context()
eg, ctx := errgroup.WithContext(ctx) eg, ctx := errgroup.WithContext(ctx)
s := Service{ s := Service{
eg: eg,
control: control, control: control,
} }
s.mu.listeners = map[uint16]*tcpListener{} s.mu.listeners = map[uint16]*tcpListener{}
@@ -74,6 +78,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol}, NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6}, TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6},
}) })
// The teardown goroutine only owns the stack once New succeeds
defer func() {
if reterr != nil {
s.ipstack.Destroy()
}
}()
sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default
tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt) tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt)
if tcpipErr != nil { if tcpipErr != nil {
@@ -123,6 +133,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
// this will read exactly one packet // this will read exactly one packet
n, err := reader.Read(buf) n, err := reader.Read(buf)
if err != nil { if err != nil {
// An error after shutdown began is noise from the device pipes
// closing, anything else is fatal and must stop nebula too
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err return err
} }
packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{ packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{
@@ -130,8 +146,8 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
}) })
linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf) linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf)
if err := ctx.Err(); err != nil { if ctx.Err() != nil {
return err return nil
} }
} }
}) })
@@ -139,13 +155,18 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
for { for {
packet := linkEP.ReadContext(ctx) packet := linkEP.ReadContext(ctx)
if packet == nil { if packet == nil {
if err := ctx.Err(); err != nil { if ctx.Err() != nil {
return err return nil
} }
continue continue
} }
bufView := packet.ToView() bufView := packet.ToView()
if _, err := bufView.WriteTo(writer); err != nil { if _, err := bufView.WriteTo(writer); err != nil {
// Same shutdown noise handling as the inbound reader
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err return err
} }
bufView.Release() bufView.Release()
@@ -158,6 +179,22 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
return control.Wait() return control.Wait()
}) })
// Tear down here rather than in Close so it also happens when nebula stops
// on its own. The forwarding goroutines must exit before the stack they feed
// is destroyed.
s.done = make(chan struct{})
go func() {
s.err = eg.Wait()
s.closeListeners()
// Destroy aborts the endpoints that exist when it starts, then waits on
// every endpoint, so a Dial registering in between would hold Wait until
// its connect timed out. Without the NIC new dials fail route lookup.
// RemoveNIC also closes linkEP.
_ = s.ipstack.RemoveNIC(nicID)
s.ipstack.Destroy()
close(s.done)
}()
return &s, nil return &s, nil
} }
@@ -237,6 +274,9 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
if s.mu.closed {
return nil, net.ErrClosed
}
if _, ok := s.mu.listeners[port]; ok { if _, ok := s.mu.listeners[port]; ok {
return nil, fmt.Errorf("already listening on port %d", port) return nil, fmt.Errorf("already listening on port %d", port)
} }
@@ -245,15 +285,38 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
return l, nil return l, nil
} }
// Wait blocks until the service has fully stopped, either via Close or a fatal
// error, and returns the first fatal error if there was one.
func (s *Service) Wait() error { func (s *Service) Wait() error {
return s.eg.Wait() <-s.done
return s.err
} }
// Close closes all listeners and stops nebula. Use Wait to block until
// shutdown has completed.
func (s *Service) Close() error { func (s *Service) Close() error {
s.closeListeners()
s.control.Stop() s.control.Stop()
return nil return nil
} }
// closeListeners unblocks pending Accept calls and rejects new listeners.
func (s *Service) closeListeners() {
// Every caller closes every listener rather than returning early once
// closed is set, so none returns while another is still closing them.
s.mu.Lock()
s.mu.closed = true
listeners := make([]*tcpListener, 0, len(s.mu.listeners))
for _, l := range s.mu.listeners {
listeners = append(listeners, l)
}
s.mu.Unlock()
for _, l := range listeners {
_ = l.Close()
}
}
func (s *Service) tcpHandler(r *tcp.ForwarderRequest) { func (s *Service) tcpHandler(r *tcp.ForwarderRequest) {
endpointID := r.ID() endpointID := r.ID()
+159
View File
@@ -327,3 +327,162 @@ func TestListenerAcceptAfterClose(t *testing.T) {
} }
} }
} }
// Close must release blocked Accept calls and let Wait report a clean shutdown.
func TestServiceClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
if err := a.Close(); err != nil {
t.Fatal(err)
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after Close")
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after Close")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
if err := a.Close(); err != nil {
t.Fatalf("second close: %v", err)
}
if _, err := a.Listen("tcp", ":1235"); !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Listen after Close, got %v", err)
}
}
// If nebula stops without Close being called, the service must still release
// Accept and tear down the netstack.
func TestServiceNebulaStopped(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
a.control.Stop()
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after nebula stopped")
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after nebula stopped")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
}
// Dials racing Close must not hold Wait until their connect times out.
func TestServiceDialDuringClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
stop := make(chan struct{})
var eg errgroup.Group
for range 8 {
eg.Go(func() error {
for {
select {
case <-stop:
return nil
default:
}
// 10.0.0.2 has no tunnel, so each dial sits in SYN-SENT until aborted
if c, err := a.Dial("tcp", "10.0.0.2:80"); err == nil {
_ = c.Close()
}
}
})
}
time.Sleep(50 * time.Millisecond)
if err := a.Close(); err != nil {
t.Fatal(err)
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait blocked behind a dial that raced Close")
}
close(stop)
dialersDone := make(chan error, 1)
go func() { dialersDone <- eg.Wait() }()
select {
case <-dialersDone:
case <-time.After(5 * time.Second):
t.Fatal("Dial still blocked after Wait")
}
}
+1 -175
View File
@@ -11,12 +11,8 @@ import (
"net" "net"
"net/netip" "net/netip"
"os" "os"
"path/filepath"
"runtime"
"runtime/pprof"
"slices" "slices"
"sort" "sort"
"strconv"
"strings" "strings"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
@@ -60,8 +56,6 @@ type sshDeviceInfoFlags struct {
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
// sshd.sandbox_dir is read once, when the commands are attached
r.RestartRequiredIfChanged("sshd.sandbox_dir")
if c.GetBool("sshd.enabled", false) { if c.GetBool("sshd.enabled", false) {
// Restarting the server drops every open session, the one that asked for this reload included. Only restart // Restarting the server drops every open session, the one that asked for this reload included. Only restart
// when it isn't running or its config changed. The host key file can change under the same config, new // when it isn't running or its config changed. The host key file can change under the same config, new
@@ -93,8 +87,7 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
}) })
} }
// sshdServerKeys are what configSSH reads. sshd.sandbox_dir isn't one, it's read once at startup and restarting the // sshdServerKeys are what configSSH reads
// server doesn't apply it
var sshdServerKeys = []string{"sshd.enabled", "sshd.listen", "sshd.host_key", "sshd.trusted_cas", "sshd.authorized_users"} var sshdServerKeys = []string{"sshd.enabled", "sshd.listen", "sshd.host_key", "sshd.trusted_cas", "sshd.authorized_users"}
// loadSSHHostKey returns sshd.host_key, read from its file when it names one. // loadSSHHostKey returns sshd.host_key, read from its file when it names one.
@@ -236,12 +229,6 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
} }
func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) { func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
// sandboxDir defaults to a dir in temp. The intention is that end user will
// create this dir as needed. Overriding this config value to "" allows
// writing to anywhere in the system.
defaultDir := filepath.Join(os.TempDir(), "nebula-debug")
sandboxDir := c.GetString("sshd.sandbox_dir", defaultDir)
ssh.RegisterCommand(&sshd.Command{ ssh.RegisterCommand(&sshd.Command{
Name: "list-hostmap", Name: "list-hostmap",
ShortDescription: "List all known previously connected hosts", ShortDescription: "List all known previously connected hosts",
@@ -297,45 +284,6 @@ func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interfa
}, },
}) })
ssh.RegisterCommand(&sshd.Command{
Name: "start-cpu-profile",
ShortDescription: "Starts a cpu profile and write output to the provided file, ex: `cpu-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshStartCpuProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "stop-cpu-profile",
ShortDescription: "Stops a cpu profile and writes output to the previously provided file",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
pprof.StopCPUProfile()
return w.WriteLine("If a CPU profile was running it is now stopped")
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "save-heap-profile",
ShortDescription: "Saves a heap profile to the provided path, ex: `heap-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetHeapProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "mutex-profile-fraction",
ShortDescription: "Gets or sets runtime.SetMutexProfileFraction",
Callback: sshMutexProfileFraction,
})
ssh.RegisterCommand(&sshd.Command{
Name: "save-mutex-profile",
ShortDescription: "Saves a mutex profile to the provided path, ex: `mutex-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetMutexProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{ ssh.RegisterCommand(&sshd.Command{
Name: "log-level", Name: "log-level",
ShortDescription: "Gets or sets the current log level", ShortDescription: "Gets or sets the current log level",
@@ -566,58 +514,6 @@ func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) er
return nil return nil
} }
// sshSanitizeFilePath validates that the given file path is within the sandbox directory.
// If sandboxDir is empty, the path is returned as-is for backwards compatibility.
func sshSanitizeFilePath(sandboxDir, filePath string) (string, error) {
if sandboxDir == "" {
return filePath, nil
}
// Clean and resolve the path relative to the sandbox directory
if !filepath.IsAbs(filePath) {
filePath = filepath.Join(sandboxDir, filePath)
}
cleaned := filepath.Clean(filePath)
// Ensure the resolved path is within the sandbox directory
cleanedSandbox := filepath.Clean(sandboxDir)
if cleaned == cleanedSandbox {
return "", fmt.Errorf("path %q resolves to the sandbox directory itself %q", filePath, sandboxDir)
}
if !strings.HasPrefix(cleaned, cleanedSandbox+string(filepath.Separator)) {
return "", fmt.Errorf("path %q is outside the sandbox directory %q", filePath, sandboxDir)
}
return cleaned, nil
}
func sshStartCpuProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
err := w.WriteLine("No path to write profile provided")
return err
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
}
err = pprof.StartCPUProfile(file)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to start cpu profile: %s", err))
return err
}
err = w.WriteLine(fmt.Sprintf("Started cpu profile, issue stop-cpu-profile to write the output to %s", a))
return err
}
func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
return w.WriteLine(fmt.Sprintf("%s", ifce.version)) return w.WriteLine(fmt.Sprintf("%s", ifce.version))
} }
@@ -770,76 +666,6 @@ func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) e
return w.WriteLine("Changed") return w.WriteLine("Changed")
} }
func sshGetHeapProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
}
err = pprof.WriteHeapProfile(file)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
return err
}
err = w.WriteLine(fmt.Sprintf("Mem profile created at %s", a))
return err
}
func sshMutexProfileFraction(fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
rate := runtime.SetMutexProfileFraction(-1)
return w.WriteLine(fmt.Sprintf("Current value: %d", rate))
}
newRate, err := strconv.Atoi(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("Invalid argument: %s", a[0]))
}
oldRate := runtime.SetMutexProfileFraction(newRate)
return w.WriteLine(fmt.Sprintf("New value: %d. Old value: %d", newRate, oldRate))
}
func sshGetMutexProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
return w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
}
defer file.Close()
mutexProfile := pprof.Lookup("mutex")
if mutexProfile == nil {
return w.WriteLine("Unable to get pprof.Lookup(\"mutex\")")
}
err = mutexProfile.WriteTo(file, 0)
if err != nil {
return w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
}
return w.WriteLine(fmt.Sprintf("Mutex profile created at %s", a))
}
func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error { func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
ctrl, ok := l.Handler().(interface { ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level GetLevel() slog.Level
-12
View File
@@ -165,18 +165,6 @@ func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
} }
} }
// sandbox_dir is only read at startup, restarting sshd for it would drop every session and still not apply it
func TestSSHDReload_SandboxDirKeepsSessions(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
h.extra = " sandbox_dir: " + t.TempDir() + "\n"
res := h.c.ReloadConfigString(h.config())
require.NoError(t, res.Err)
assert.Equal(t, []string{"sshd.sandbox_dir"}, res.RestartKeys)
assert.False(t, dropped(client, 500*time.Millisecond), "sshd was restarted")
}
// The config can stay the same while the host key file it names changes. New connections get the new key, open // The config can stay the same while the host key file it names changes. New connections get the new key, open
// sessions are left alone // sessions are left alone
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) { func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
+1 -1
View File
@@ -41,7 +41,7 @@ type Conn interface {
// //
// Returns the number of packets successfully written. A destination the kernel rejects costs only // Returns the number of packets successfully written. A destination the kernel rejects costs only
// its own packet, so a short count means some peers were undeliverable, not that the batch failed. // its own packet, so a short count means some peers were undeliverable, not that the batch failed.
// Not safe for concurrent use on the same Conn. // Safe for concurrent use on the same Conn.
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
// ReloadConfig applies the listen settings that can change, r is nil at startup // ReloadConfig applies the listen settings that can change, r is nil at startup
ReloadConfig(c *config.C, r *config.Reload) ReloadConfig(c *config.C, r *config.Reload)
+6 -1
View File
@@ -12,6 +12,7 @@ import (
"net/netip" "net/netip"
"strconv" "strconv"
"strings" "strings"
"sync"
"unsafe" "unsafe"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
@@ -20,7 +21,7 @@ import (
// batchWriter owns the sendmmsg(2)/UDP-GSO transmit path for a StdConn: the // batchWriter owns the sendmmsg(2)/UDP-GSO transmit path for a StdConn: the
// scratch WriteBatch packs mmsghdr entries into, plus the GSO capability // scratch WriteBatch packs mmsghdr entries into, plus the GSO capability
// state probed at socket creation. Each queue has its own StdConn and // state probed at socket creation. Each queue has its own StdConn and
// batchWriter, so no locking is needed. // batchWriter; mu serializes batches from the TUN reader and relay forwarding.
// //
// Terminology, smallest to largest: // Terminology, smallest to largest:
// //
@@ -39,6 +40,7 @@ import (
// chunk the entries packed for one sendmmsg call, at most MaxWriteBatch. // chunk the entries packed for one sendmmsg call, at most MaxWriteBatch.
// batch the caller's whole bufs/addrs pair, processed as one or more chunks. // batch the caller's whole bufs/addrs pair, processed as one or more chunks.
type batchWriter struct { type batchWriter struct {
mu sync.Mutex
fd int fd int
isV4 bool isV4 bool
@@ -192,6 +194,9 @@ func parseRelease(r string) (major, minor int) {
// Returns the number of packets sent. An error means the call itself failed. // Returns the number of packets sent. An error means the call itself failed.
// A short count means some destinations were undeliverable. // A short count means some destinations were undeliverable.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()
if len(bufs) != len(addrs) { if len(bufs) != len(addrs) {
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs)) return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
} }
@@ -0,0 +1,61 @@
//go:build linux && !android && !e2e_testing
package udp
import (
"net/netip"
"runtime"
"sync"
"testing"
"github.com/stretchr/testify/require"
)
// The TUN reader and relay forwarding share a socket. Exercise their concurrent
// batches against the actual GSO planner and scratch, without relying on a NIC.
func TestWriteBatchConcurrent(t *testing.T) {
const batches = 64
w := newRewindTestWriter()
sent := make(chan [][]byte, 2*batches)
w.sendFn = func(start, n int) (int, error) {
runtime.Gosched()
sent <- capturePrepared(w, start, n)
return n, nil
}
conn := &StdConn{bw: w}
remote := netip.MustParseAddrPort("192.0.2.1:4242")
var wg sync.WaitGroup
start := make(chan struct{})
for producer := range 2 {
wg.Go(func() {
<-start
for i := range batches {
var bufs [][]byte
var addrs []netip.AddrPort
for j := range 4 {
bufs = append(bufs, []byte{byte(producer), byte(i), byte(j), 0})
addrs = append(addrs, remote)
}
n, err := conn.WriteBatch(bufs, addrs)
if err != nil || n != len(bufs) {
t.Errorf("WriteBatch = %d, %v; want %d, nil", n, err, len(bufs))
return
}
}
})
}
close(start)
wg.Wait()
close(sent)
var counts [2]int
for packets := range sent {
require.Len(t, packets, 4)
producer := packets[0][0]
require.Less(t, int(producer), len(counts))
for j, packet := range packets {
require.Equal(t, []byte{producer, byte(counts[producer]), byte(j), 0}, packet)
}
counts[producer]++
}
require.Equal(t, [2]int{batches, batches}, counts)
}