mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 17:46:52 +01:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d3e21a8fc8 | ||
|
|
cda3710d5a | ||
|
|
c26871fe08 | ||
|
|
7daacc6a48 | ||
|
|
428dad94b3 | ||
|
|
5bf666dd90 | ||
|
|
49207da077 |
No files matched your search
@@ -7,8 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Removed
|
||||||
|
|
||||||
|
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
|
||||||
|
`mutex-profile-fraction` commands, and `sshd.sandbox_dir`, which is now ignored if set.
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
- Signing a v2 certificate with an empty name, a name longer than 253 bytes, or an empty group now fails.
|
||||||
|
These certificates were signed successfully but could not be loaded.
|
||||||
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
|
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
|
||||||
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
|
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
|
||||||
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
|
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
|
||||||
|
|||||||
+10
-1
@@ -389,7 +389,16 @@ func (c *certificateV2) fromTBSCertificate(t *TBSCertificate) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *certificateV2) validate() error {
|
func (c *certificateV2) validate() error {
|
||||||
// Empty names are allowed
|
// Mirror the name and group rules in unmarshalDetails so signing fails instead of producing an unparseable certificate
|
||||||
|
if c.details.name == "" || len(c.details.name) > MaxNameLength {
|
||||||
|
return NewErrInvalidCertificateProperties("name must be between 1 and %d bytes", MaxNameLength)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, group := range c.details.groups {
|
||||||
|
if group == "" {
|
||||||
|
return NewErrInvalidCertificateProperties("groups may not be empty")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if len(c.publicKey) == 0 {
|
if len(c.publicKey) == 0 {
|
||||||
return ErrInvalidPublicKey
|
return ErrInvalidPublicKey
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"crypto/elliptic"
|
"crypto/elliptic"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -135,3 +136,47 @@ func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
|
|||||||
assert.True(t, normie)
|
assert.True(t, normie)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCertificateV2_SignRejectsUnparseable(t *testing.T) {
|
||||||
|
_, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
var invalid *ErrInvalidCertificateProperties
|
||||||
|
|
||||||
|
newTBS := func() TBSCertificate {
|
||||||
|
return TBSCertificate{
|
||||||
|
Version: Version2,
|
||||||
|
Name: "testing",
|
||||||
|
Networks: []netip.Prefix{mustParsePrefixUnmapped("10.1.1.1/24")},
|
||||||
|
Groups: []string{"test-group1"},
|
||||||
|
NotBefore: time.Now().Add(-time.Minute),
|
||||||
|
NotAfter: time.Now().Add(time.Minute),
|
||||||
|
PublicKey: []byte("1234567890abcedfghij1234567890ab"),
|
||||||
|
Curve: Curve_CURVE25519,
|
||||||
|
IsCA: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tbs := newTBS()
|
||||||
|
tbs.Name = strings.Repeat("a", MaxNameLength)
|
||||||
|
c, err := tbs.Sign(nil, Curve_CURVE25519, priv)
|
||||||
|
require.NoError(t, err)
|
||||||
|
b, err := c.Marshal()
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = unmarshalCertificateV2(b, nil, Curve_CURVE25519)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
tbs = newTBS()
|
||||||
|
tbs.Name = strings.Repeat("a", MaxNameLength+1)
|
||||||
|
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
|
||||||
|
require.ErrorAs(t, err, &invalid)
|
||||||
|
|
||||||
|
tbs = newTBS()
|
||||||
|
tbs.Name = ""
|
||||||
|
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
|
||||||
|
require.ErrorAs(t, err, &invalid)
|
||||||
|
|
||||||
|
tbs = newTBS()
|
||||||
|
tbs.Groups = []string{"test-group1", ""}
|
||||||
|
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
|
||||||
|
require.ErrorAs(t, err, &invalid)
|
||||||
|
}
|
||||||
+1
-7
@@ -8,6 +8,7 @@ pki:
|
|||||||
cert: /etc/nebula/host.crt
|
cert: /etc/nebula/host.crt
|
||||||
key: /etc/nebula/host.key
|
key: /etc/nebula/host.key
|
||||||
# blocklist is a list of certificate fingerprints that we will refuse to talk to
|
# blocklist is a list of certificate fingerprints that we will refuse to talk to
|
||||||
|
# Entries are the hex fingerprints shown by `nebula-cert print`, an invalid entry fails startup and reload.
|
||||||
#blocklist:
|
#blocklist:
|
||||||
# - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72
|
# - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72
|
||||||
# disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid.
|
# disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid.
|
||||||
@@ -232,13 +233,6 @@ punchy:
|
|||||||
# Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
|
# Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
|
||||||
#trusted_cas:
|
#trusted_cas:
|
||||||
#- "ssh public key string"
|
#- "ssh public key string"
|
||||||
# sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
|
|
||||||
# save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
|
|
||||||
# and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
|
|
||||||
# The directory is NOT automatically created.
|
|
||||||
# Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
|
|
||||||
# Not reloadable.
|
|
||||||
#sandbox_dir: /var/tmp/nebula-debug
|
|
||||||
|
|
||||||
# EXPERIMENTAL: relay support for networks that can't establish direct connections.
|
# EXPERIMENTAL: relay support for networks that can't establish direct connections.
|
||||||
relay:
|
relay:
|
||||||
|
|||||||
+34
-101
@@ -32,21 +32,16 @@ type conn struct {
|
|||||||
Expires time.Time // Time when this conntrack entry will expire
|
Expires time.Time // Time when this conntrack entry will expire
|
||||||
|
|
||||||
// record why the original connection passed the firewall, so we can re-validate
|
// record why the original connection passed the firewall, so we can re-validate
|
||||||
// after ruleset changes or when the peer handshakes again, which may present a
|
// after ruleset changes. Note, rulesVersion is a uint16 so that these two
|
||||||
// cert with different groups. These pack into the word after Expires.
|
// fields pack for free after the uint32 above
|
||||||
incoming bool
|
incoming bool
|
||||||
rulesVersion uint16
|
rulesVersion uint16
|
||||||
localIndex uint32
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: need conntrack max tracked connections handling
|
// TODO: need conntrack max tracked connections handling
|
||||||
type Firewall struct {
|
type Firewall struct {
|
||||||
Conntrack *FirewallConntrack
|
Conntrack *FirewallConntrack
|
||||||
|
|
||||||
// siblings finds the other gateways we route an unsafe network through, so a reply that comes back through a
|
|
||||||
// different gateway than the flow went out through can be matched to it. Nil turns that off.
|
|
||||||
siblings gatewaySiblings
|
|
||||||
|
|
||||||
InRules *FirewallTable
|
InRules *FirewallTable
|
||||||
OutRules *FirewallTable
|
OutRules *FirewallTable
|
||||||
|
|
||||||
@@ -87,8 +82,8 @@ type firewallMetrics struct {
|
|||||||
type FirewallConntrack struct {
|
type FirewallConntrack struct {
|
||||||
sync.Mutex
|
sync.Mutex
|
||||||
|
|
||||||
Conns map[firewall.ConntrackKey]*conn
|
Conns map[firewall.Packet]*conn
|
||||||
TimerWheel *TimerWheel[firewall.ConntrackKey]
|
TimerWheel *TimerWheel[firewall.Packet]
|
||||||
}
|
}
|
||||||
|
|
||||||
// FirewallTable is the entry point for a rule, the evaluation order is:
|
// FirewallTable is the entry point for a rule, the evaluation order is:
|
||||||
@@ -172,8 +167,8 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
|
|||||||
|
|
||||||
return &Firewall{
|
return &Firewall{
|
||||||
Conntrack: &FirewallConntrack{
|
Conntrack: &FirewallConntrack{
|
||||||
Conns: make(map[firewall.ConntrackKey]*conn),
|
Conns: make(map[firewall.Packet]*conn),
|
||||||
TimerWheel: NewTimerWheel[firewall.ConntrackKey](tmin, tmax),
|
TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax),
|
||||||
},
|
},
|
||||||
InRules: newFirewallTable(),
|
InRules: newFirewallTable(),
|
||||||
OutRules: newFirewallTable(),
|
OutRules: newFirewallTable(),
|
||||||
@@ -430,7 +425,6 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
|
|||||||
// returns nil if the packet should not be dropped.
|
// returns nil if the packet should not be dropped.
|
||||||
func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
|
func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
|
||||||
// Make sure remote address matches nebula certificate, and determine how to treat it
|
// Make sure remote address matches nebula certificate, and determine how to treat it
|
||||||
unsafe := false
|
|
||||||
if h.networks == nil {
|
if h.networks == nil {
|
||||||
// Simple case: Certificate has one address and no unsafe networks
|
// Simple case: Certificate has one address and no unsafe networks
|
||||||
if h.vpnAddrs[0] != fp.RemoteAddr {
|
if h.vpnAddrs[0] != fp.RemoteAddr {
|
||||||
@@ -450,7 +444,7 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
|
|||||||
f.metrics(incoming).droppedRemoteAddr.Inc(1)
|
f.metrics(incoming).droppedRemoteAddr.Inc(1)
|
||||||
return ErrPeerRejected // reject for now, one day this may have different FW rules
|
return ErrPeerRejected // reject for now, one day this may have different FW rules
|
||||||
case NetworkTypeUnsafe:
|
case NetworkTypeUnsafe:
|
||||||
unsafe = true // nothing else special, one day this may have different FW rules
|
break // nothing special, one day this may have different FW rules
|
||||||
default:
|
default:
|
||||||
f.metrics(incoming).droppedRemoteAddr.Inc(1)
|
f.metrics(incoming).droppedRemoteAddr.Inc(1)
|
||||||
return ErrUnknownNetworkType //should never happen
|
return ErrUnknownNetworkType //should never happen
|
||||||
@@ -463,11 +457,8 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
|
|||||||
return ErrInvalidLocalIP
|
return ErrInvalidLocalIP
|
||||||
}
|
}
|
||||||
|
|
||||||
// The first vpn address identifies the peer, as it does elsewhere. A host's v1 and v2 certs must agree on it
|
|
||||||
key := firewall.ConntrackKey{Packet: fp, Peer: h.vpnAddrs[0]}
|
|
||||||
|
|
||||||
// Check if we spoke to this tuple, if we did then allow this packet
|
// Check if we spoke to this tuple, if we did then allow this packet
|
||||||
if f.inConns(key, h, caPool, localCache) {
|
if f.inConns(fp, h, caPool, localCache) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -478,15 +469,12 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
|
|||||||
|
|
||||||
// We now know which firewall table to check against
|
// We now know which firewall table to check against
|
||||||
if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
|
if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
|
||||||
if unsafe && f.inheritConn(key, incoming, h, caPool) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
f.metrics(incoming).droppedNoRule.Inc(1)
|
f.metrics(incoming).droppedNoRule.Inc(1)
|
||||||
return ErrNoMatchingRule
|
return ErrNoMatchingRule
|
||||||
}
|
}
|
||||||
|
|
||||||
// We always want to conntrack since it is a faster operation
|
// We always want to conntrack since it is a faster operation
|
||||||
f.addConn(key, incoming, h)
|
f.addConn(fp, incoming)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -515,9 +503,9 @@ func (f *Firewall) EmitStats() {
|
|||||||
metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
|
metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Firewall) inConns(key firewall.ConntrackKey, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
|
func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
|
||||||
if localCache != nil {
|
if localCache != nil {
|
||||||
if _, ok := localCache[key]; ok {
|
if _, ok := localCache[fp]; ok {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -530,52 +518,49 @@ func (f *Firewall) inConns(key firewall.ConntrackKey, h *HostInfo, caPool *cert.
|
|||||||
f.evict(ep)
|
f.evict(ep)
|
||||||
}
|
}
|
||||||
|
|
||||||
c, ok := conntrack.Conns[key]
|
c, ok := conntrack.Conns[fp]
|
||||||
|
|
||||||
if !ok {
|
if !ok {
|
||||||
conntrack.Unlock()
|
conntrack.Unlock()
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.rulesVersion != f.rulesVersion || c.localIndex != h.localIndexId {
|
if c.rulesVersion != f.rulesVersion {
|
||||||
// This conntrack entry was allowed by an older rule set or for an earlier tunnel to the peer, validate
|
// This conntrack entry was for an older rule set, validate
|
||||||
// it still passes with the current rule set and the peer's current certificate
|
// it still passes with the current rule set
|
||||||
table := f.OutRules
|
table := f.OutRules
|
||||||
if c.incoming {
|
if c.incoming {
|
||||||
table = f.InRules
|
table = f.InRules
|
||||||
}
|
}
|
||||||
|
|
||||||
// We now know which firewall table to check against
|
// We now know which firewall table to check against
|
||||||
if !table.match(key.Packet, c.incoming, h.ConnectionState.peerCert, caPool) {
|
if !table.match(fp, c.incoming, h.ConnectionState.peerCert, caPool) {
|
||||||
if f.l.Enabled(context.Background(), slog.LevelDebug) {
|
if f.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||||
h.logger(f.l).Debug("dropping old conntrack entry, does not match current ruleset and certificate",
|
h.logger(f.l).Debug("dropping old conntrack entry, does not match new ruleset",
|
||||||
"fwPacket", key.Packet,
|
"fwPacket", fp,
|
||||||
"incoming", c.incoming,
|
"incoming", c.incoming,
|
||||||
"rulesVersion", f.rulesVersion,
|
"rulesVersion", f.rulesVersion,
|
||||||
"oldRulesVersion", c.rulesVersion,
|
"oldRulesVersion", c.rulesVersion,
|
||||||
"oldLocalIndex", c.localIndex,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
delete(conntrack.Conns, key)
|
delete(conntrack.Conns, fp)
|
||||||
conntrack.Unlock()
|
conntrack.Unlock()
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
if f.l.Enabled(context.Background(), slog.LevelDebug) {
|
if f.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||||
h.logger(f.l).Debug("keeping old conntrack entry, does match current ruleset and certificate",
|
h.logger(f.l).Debug("keeping old conntrack entry, does match new ruleset",
|
||||||
"fwPacket", key.Packet,
|
"fwPacket", fp,
|
||||||
"incoming", c.incoming,
|
"incoming", c.incoming,
|
||||||
"rulesVersion", f.rulesVersion,
|
"rulesVersion", f.rulesVersion,
|
||||||
"oldRulesVersion", c.rulesVersion,
|
"oldRulesVersion", c.rulesVersion,
|
||||||
"oldLocalIndex", c.localIndex,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
c.rulesVersion = f.rulesVersion
|
c.rulesVersion = f.rulesVersion
|
||||||
c.localIndex = h.localIndexId
|
|
||||||
}
|
}
|
||||||
|
|
||||||
switch key.Protocol {
|
switch fp.Protocol {
|
||||||
case iputil.IPProtocolTCP:
|
case iputil.IPProtocolTCP:
|
||||||
c.Expires = time.Now().Add(f.TCPTimeout)
|
c.Expires = time.Now().Add(f.TCPTimeout)
|
||||||
case iputil.IPProtocolUDP:
|
case iputil.IPProtocolUDP:
|
||||||
@@ -587,17 +572,17 @@ func (f *Firewall) inConns(key firewall.ConntrackKey, h *HostInfo, caPool *cert.
|
|||||||
conntrack.Unlock()
|
conntrack.Unlock()
|
||||||
|
|
||||||
if localCache != nil {
|
if localCache != nil {
|
||||||
localCache[key] = struct{}{}
|
localCache[fp] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Firewall) addConn(key firewall.ConntrackKey, incoming bool, h *HostInfo) {
|
func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
|
||||||
var timeout time.Duration
|
var timeout time.Duration
|
||||||
c := &conn{}
|
c := &conn{}
|
||||||
|
|
||||||
switch key.Protocol {
|
switch fp.Protocol {
|
||||||
case iputil.IPProtocolTCP:
|
case iputil.IPProtocolTCP:
|
||||||
timeout = f.TCPTimeout
|
timeout = f.TCPTimeout
|
||||||
case iputil.IPProtocolUDP:
|
case iputil.IPProtocolUDP:
|
||||||
@@ -608,78 +593,26 @@ func (f *Firewall) addConn(key firewall.ConntrackKey, incoming bool, h *HostInfo
|
|||||||
|
|
||||||
conntrack := f.Conntrack
|
conntrack := f.Conntrack
|
||||||
conntrack.Lock()
|
conntrack.Lock()
|
||||||
if _, ok := conntrack.Conns[key]; !ok {
|
if _, ok := conntrack.Conns[fp]; !ok {
|
||||||
conntrack.TimerWheel.Advance(time.Now())
|
conntrack.TimerWheel.Advance(time.Now())
|
||||||
conntrack.TimerWheel.Add(key, timeout)
|
conntrack.TimerWheel.Add(fp, timeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record which rulesVersion and tunnel allowed this connection, so we can retest after
|
// Record which rulesVersion allowed this connection, so we can retest after
|
||||||
// firewall reload or when the peer handshakes again
|
// firewall reload
|
||||||
c.incoming = incoming
|
c.incoming = incoming
|
||||||
c.rulesVersion = f.rulesVersion
|
c.rulesVersion = f.rulesVersion
|
||||||
c.localIndex = h.localIndexId
|
|
||||||
c.Expires = time.Now().Add(timeout)
|
c.Expires = time.Now().Add(timeout)
|
||||||
conntrack.Conns[key] = c
|
conntrack.Conns[fp] = c
|
||||||
conntrack.Unlock()
|
conntrack.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// gatewaySiblings finds the other gateways we route an unsafe network through
|
|
||||||
type gatewaySiblings interface {
|
|
||||||
// siblingPeers appends to buf the conntrack peer of every other gateway we route remote through, if h is one of
|
|
||||||
// those gateways
|
|
||||||
siblingPeers(remote netip.Addr, h *HostInfo, buf []netip.Addr) []netip.Addr
|
|
||||||
}
|
|
||||||
|
|
||||||
// inheritConn lets h carry a flow that another of our gateways for the same unsafe network already carries. We
|
|
||||||
// spread the flows we open across the gateways, but the hosts behind them answer through whichever gateway their own
|
|
||||||
// routing picks, so a reply can come back through a different gateway than the flow went out through. The flow is
|
|
||||||
// checked against the rules that allowed it in the first place with h's certificate, the same check an entry gets
|
|
||||||
// when its peer reconnects. Peers that aren't one of our gateways for the network never inherit a flow.
|
|
||||||
func (f *Firewall) inheritConn(key firewall.ConntrackKey, incoming bool, h *HostInfo, caPool *cert.CAPool) bool {
|
|
||||||
if f.siblings == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf [4]netip.Addr
|
|
||||||
now := time.Now()
|
|
||||||
for _, peer := range f.siblings.siblingPeers(key.RemoteAddr, h, buf[:0]) {
|
|
||||||
sk := key
|
|
||||||
sk.Peer = peer
|
|
||||||
conntrack := f.Conntrack
|
|
||||||
conntrack.Lock()
|
|
||||||
c, ok := conntrack.Conns[sk]
|
|
||||||
live := ok && c.Expires.After(now)
|
|
||||||
var allowedIncoming bool
|
|
||||||
if live {
|
|
||||||
allowedIncoming = c.incoming
|
|
||||||
}
|
|
||||||
conntrack.Unlock()
|
|
||||||
|
|
||||||
// The rules for this packet's own direction already said no, only a flow allowed the other way can help
|
|
||||||
if !live || allowedIncoming == incoming {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
table := f.OutRules
|
|
||||||
if allowedIncoming {
|
|
||||||
table = f.InRules
|
|
||||||
}
|
|
||||||
if !table.match(key.Packet, allowedIncoming, h.ConnectionState.peerCert, caPool) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
f.addConn(key, allowedIncoming, h)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel
|
// Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel
|
||||||
// Caller must own the connMutex lock!
|
// Caller must own the connMutex lock!
|
||||||
func (f *Firewall) evict(key firewall.ConntrackKey) {
|
func (f *Firewall) evict(p firewall.Packet) {
|
||||||
// Are we still tracking this conn?
|
// Are we still tracking this conn?
|
||||||
conntrack := f.Conntrack
|
conntrack := f.Conntrack
|
||||||
t, ok := conntrack.Conns[key]
|
t, ok := conntrack.Conns[p]
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -689,12 +622,12 @@ func (f *Firewall) evict(key firewall.ConntrackKey) {
|
|||||||
// Timeout is in the future, re-add the timer
|
// Timeout is in the future, re-add the timer
|
||||||
if newT > 0 {
|
if newT > 0 {
|
||||||
conntrack.TimerWheel.Advance(time.Now())
|
conntrack.TimerWheel.Advance(time.Now())
|
||||||
conntrack.TimerWheel.Add(key, newT)
|
conntrack.TimerWheel.Add(p, newT)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// This conn is done
|
// This conn is done
|
||||||
delete(conntrack.Conns, key)
|
delete(conntrack.Conns, p)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
|
func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
|
||||||
|
|||||||
+1
-9
@@ -3,23 +3,15 @@ package firewall
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/netip"
|
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/slackhq/nebula/logging"
|
"github.com/slackhq/nebula/logging"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ConntrackKey identifies a tracked flow by its packet and the peer it was allowed for, so peers that route
|
|
||||||
// the same unsafe network never share a conntrack entry.
|
|
||||||
type ConntrackKey struct {
|
|
||||||
Peer netip.Addr
|
|
||||||
Packet
|
|
||||||
}
|
|
||||||
|
|
||||||
// ConntrackCache is used as a local routine cache to know if a given flow
|
// ConntrackCache is used as a local routine cache to know if a given flow
|
||||||
// has been seen in the conntrack table.
|
// has been seen in the conntrack table.
|
||||||
type ConntrackCache map[ConntrackKey]struct{}
|
type ConntrackCache map[Packet]struct{}
|
||||||
|
|
||||||
type ConntrackCacheTicker struct {
|
type ConntrackCacheTicker struct {
|
||||||
cacheV uint64
|
cacheV uint64
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ func newFixedTicker(t *testing.T, l *slog.Logger, cacheLen int) *ConntrackCacheT
|
|||||||
cache: make(ConntrackCache, cacheLen),
|
cache: make(ConntrackCache, cacheLen),
|
||||||
}
|
}
|
||||||
for i := range cacheLen {
|
for i := range cacheLen {
|
||||||
c.cache[ConntrackKey{Packet: Packet{LocalPort: uint16(i) + 1}}] = struct{}{}
|
c.cache[Packet{LocalPort: uint16(i) + 1}] = struct{}{}
|
||||||
}
|
}
|
||||||
c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path
|
c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path
|
||||||
return c
|
return c
|
||||||
|
|||||||
@@ -26,10 +26,6 @@ type Packet struct {
|
|||||||
RemotePort uint16
|
RemotePort uint16
|
||||||
Protocol uint8
|
Protocol uint8
|
||||||
Fragment bool
|
Fragment bool
|
||||||
|
|
||||||
// pad fills what would otherwise be trailing padding so maps hash and compare a ConntrackKey as one
|
|
||||||
// block of memory instead of field by field. It must always be zero.
|
|
||||||
pad [2]byte
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fp *Packet) Copy() *Packet {
|
func (fp *Packet) Copy() *Packet {
|
||||||
|
|||||||
+1
-248
@@ -3,7 +3,6 @@ package nebula
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"math"
|
"math"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
@@ -980,252 +979,6 @@ func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) {
|
|||||||
"SECURITY: attacker spoofed victim's overlay source IP (192.0.2.2) by reusing an existing conntrack 4-tuple; Drop returned %v instead of rejecting", got)
|
"SECURITY: attacker spoofed victim's overlay source IP (192.0.2.2) by reusing an existing conntrack 4-tuple; Drop returned %v instead of rejecting", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFirewall_ConntrackUnsafeNetworkPeers(t *testing.T) {
|
|
||||||
l := test.NewLoggerWithOutput(&bytes.Buffer{})
|
|
||||||
|
|
||||||
myVpnNetworksTable := new(bart.Lite)
|
|
||||||
myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
|
|
||||||
|
|
||||||
owner := &dummyCert{
|
|
||||||
name: "owner",
|
|
||||||
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
|
|
||||||
}
|
|
||||||
|
|
||||||
// Two routers for the same unsafe network, only one of them is trusted inbound
|
|
||||||
router := func(name string, addr string, groups ...string) *HostInfo {
|
|
||||||
inverted := map[string]struct{}{}
|
|
||||||
for _, g := range groups {
|
|
||||||
inverted[g] = struct{}{}
|
|
||||||
}
|
|
||||||
c := &cert.CachedCertificate{
|
|
||||||
Certificate: &dummyCert{
|
|
||||||
name: name,
|
|
||||||
groups: groups,
|
|
||||||
networks: []netip.Prefix{netip.MustParsePrefix(addr + "/24")},
|
|
||||||
unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
|
|
||||||
},
|
|
||||||
InvertedGroups: inverted,
|
|
||||||
}
|
|
||||||
h := &HostInfo{
|
|
||||||
ConnectionState: &ConnectionState{peerCert: c},
|
|
||||||
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
|
|
||||||
}
|
|
||||||
h.buildNetworks(myVpnNetworksTable, c.Certificate)
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
trusted := router("trusted", "192.0.2.2", "trusted")
|
|
||||||
other := router("other", "192.0.2.3")
|
|
||||||
|
|
||||||
cp := cert.NewCAPool()
|
|
||||||
flow := firewall.Packet{
|
|
||||||
LocalAddr: netip.MustParseAddr("192.0.2.1"),
|
|
||||||
RemoteAddr: netip.MustParseAddr("198.51.100.5"),
|
|
||||||
LocalPort: 443,
|
|
||||||
RemotePort: 55000,
|
|
||||||
Protocol: iputil.IPProtocolTCP,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, useCache := range []bool{false, true} {
|
|
||||||
var cache firewall.ConntrackCache
|
|
||||||
if useCache {
|
|
||||||
cache = firewall.ConntrackCache{}
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run(fmt.Sprintf("inbound flow allowed for one peer/cache=%v", useCache), func(t *testing.T) {
|
|
||||||
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
|
|
||||||
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"trusted"}, "", "", "", "", ""))
|
|
||||||
clear(cache)
|
|
||||||
|
|
||||||
require.NoError(t, fw.Drop(flow, true, trusted, cp, cache))
|
|
||||||
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, other, cp, cache))
|
|
||||||
require.NoError(t, fw.Drop(flow, true, trusted, cp, cache))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run(fmt.Sprintf("reply to a flow opened via one peer/cache=%v", useCache), func(t *testing.T) {
|
|
||||||
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
|
|
||||||
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
|
|
||||||
clear(cache)
|
|
||||||
|
|
||||||
require.NoError(t, fw.Drop(flow, false, trusted, cp, cache))
|
|
||||||
require.NoError(t, fw.Drop(flow, true, trusted, cp, cache))
|
|
||||||
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, other, cp, cache))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fakeSiblings maps each gateway to the conntrack peers of the gateways it shares a route with
|
|
||||||
type fakeSiblings map[*HostInfo][]netip.Addr
|
|
||||||
|
|
||||||
func (s fakeSiblings) siblingPeers(_ netip.Addr, h *HostInfo, buf []netip.Addr) []netip.Addr {
|
|
||||||
return append(buf, s[h]...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFirewall_ConntrackGatewaySiblings(t *testing.T) {
|
|
||||||
l := test.NewLoggerWithOutput(&bytes.Buffer{})
|
|
||||||
|
|
||||||
myVpnNetworksTable := new(bart.Lite)
|
|
||||||
myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
|
|
||||||
|
|
||||||
owner := &dummyCert{
|
|
||||||
name: "owner",
|
|
||||||
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
|
|
||||||
}
|
|
||||||
|
|
||||||
// Routers for the same unsafe network. a and b are both our gateways for it, c only claims it in its cert
|
|
||||||
router := func(name string, addr string, groups ...string) *HostInfo {
|
|
||||||
inverted := map[string]struct{}{}
|
|
||||||
for _, g := range groups {
|
|
||||||
inverted[g] = struct{}{}
|
|
||||||
}
|
|
||||||
c := &cert.CachedCertificate{
|
|
||||||
Certificate: &dummyCert{
|
|
||||||
name: name,
|
|
||||||
groups: groups,
|
|
||||||
networks: []netip.Prefix{netip.MustParsePrefix(addr + "/24")},
|
|
||||||
unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
|
|
||||||
},
|
|
||||||
InvertedGroups: inverted,
|
|
||||||
}
|
|
||||||
h := &HostInfo{
|
|
||||||
ConnectionState: &ConnectionState{peerCert: c},
|
|
||||||
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
|
|
||||||
}
|
|
||||||
h.buildNetworks(myVpnNetworksTable, c.Certificate)
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
a := router("a", "192.0.2.2", "trusted")
|
|
||||||
b := router("b", "192.0.2.3", "trusted")
|
|
||||||
c := router("c", "192.0.2.4", "trusted")
|
|
||||||
siblings := fakeSiblings{a: {b.vpnAddrs[0]}, b: {a.vpnAddrs[0]}}
|
|
||||||
|
|
||||||
cp := cert.NewCAPool()
|
|
||||||
flow := firewall.Packet{
|
|
||||||
LocalAddr: netip.MustParseAddr("192.0.2.1"),
|
|
||||||
RemoteAddr: netip.MustParseAddr("198.51.100.5"),
|
|
||||||
LocalPort: 55000,
|
|
||||||
RemotePort: 443,
|
|
||||||
Protocol: iputil.IPProtocolTCP,
|
|
||||||
}
|
|
||||||
|
|
||||||
// We open flows through our gateways, nothing is allowed in on its own
|
|
||||||
newFirewall := func(outGroup string) *Firewall {
|
|
||||||
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
|
|
||||||
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{outGroup}, "", "", "", "", ""))
|
|
||||||
fw.siblings = siblings
|
|
||||||
return fw
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("reply through the other gateway", func(t *testing.T) {
|
|
||||||
fw := newFirewall("trusted")
|
|
||||||
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
|
|
||||||
require.NoError(t, fw.Drop(flow, true, a, cp, nil))
|
|
||||||
|
|
||||||
// a has its own entry now, and keeps the flow when b's is gone
|
|
||||||
delete(fw.Conntrack.Conns, firewall.ConntrackKey{Packet: flow, Peer: b.vpnAddrs[0]})
|
|
||||||
require.NoError(t, fw.Drop(flow, true, a, cp, nil))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("not our gateway", func(t *testing.T) {
|
|
||||||
fw := newFirewall("trusted")
|
|
||||||
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
|
|
||||||
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, c, cp, nil))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("off without siblings", func(t *testing.T) {
|
|
||||||
fw := newFirewall("trusted")
|
|
||||||
fw.siblings = nil
|
|
||||||
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
|
|
||||||
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, a, cp, nil))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("expired sibling entry", func(t *testing.T) {
|
|
||||||
fw := newFirewall("trusted")
|
|
||||||
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
|
|
||||||
fw.Conntrack.Conns[firewall.ConntrackKey{Packet: flow, Peer: b.vpnAddrs[0]}].Expires = time.Now().Add(-time.Second)
|
|
||||||
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, a, cp, nil))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("the original rules must allow the new gateway", func(t *testing.T) {
|
|
||||||
fw := newFirewall("b-only")
|
|
||||||
b.ConnectionState.peerCert.InvertedGroups["b-only"] = struct{}{}
|
|
||||||
defer delete(b.ConnectionState.peerCert.InvertedGroups, "b-only")
|
|
||||||
|
|
||||||
require.NoError(t, fw.Drop(flow, false, b, cp, nil))
|
|
||||||
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, a, cp, nil))
|
|
||||||
})
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFirewall_ConntrackPeerRehandshake(t *testing.T) {
|
|
||||||
l := test.NewLoggerWithOutput(&bytes.Buffer{})
|
|
||||||
|
|
||||||
myVpnNetworksTable := new(bart.Lite)
|
|
||||||
myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
|
|
||||||
|
|
||||||
owner := &dummyCert{
|
|
||||||
name: "owner",
|
|
||||||
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
|
|
||||||
}
|
|
||||||
|
|
||||||
// Each call is a new tunnel to the same peer, presenting a cert with the given groups
|
|
||||||
tunnel := func(localIndex uint32, groups ...string) *HostInfo {
|
|
||||||
inverted := map[string]struct{}{}
|
|
||||||
for _, g := range groups {
|
|
||||||
inverted[g] = struct{}{}
|
|
||||||
}
|
|
||||||
c := &cert.CachedCertificate{
|
|
||||||
Certificate: &dummyCert{
|
|
||||||
name: "peer",
|
|
||||||
groups: groups,
|
|
||||||
networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24"), netip.MustParsePrefix("2001:db8::2/64")},
|
|
||||||
},
|
|
||||||
InvertedGroups: inverted,
|
|
||||||
}
|
|
||||||
h := &HostInfo{
|
|
||||||
ConnectionState: &ConnectionState{peerCert: c},
|
|
||||||
vpnAddrs: []netip.Addr{netip.MustParseAddr("192.0.2.2"), netip.MustParseAddr("2001:db8::2")},
|
|
||||||
localIndexId: localIndex,
|
|
||||||
}
|
|
||||||
h.buildNetworks(myVpnNetworksTable, c.Certificate)
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
|
|
||||||
cp := cert.NewCAPool()
|
|
||||||
flow := firewall.Packet{
|
|
||||||
LocalAddr: netip.MustParseAddr("192.0.2.1"),
|
|
||||||
RemoteAddr: netip.MustParseAddr("192.0.2.2"),
|
|
||||||
LocalPort: 443,
|
|
||||||
RemotePort: 55000,
|
|
||||||
Protocol: iputil.IPProtocolTCP,
|
|
||||||
}
|
|
||||||
key := firewall.ConntrackKey{Packet: flow, Peer: netip.MustParseAddr("192.0.2.2")}
|
|
||||||
|
|
||||||
t.Run("inbound flow follows the peer's groups", func(t *testing.T) {
|
|
||||||
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
|
|
||||||
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"trusted"}, "", "", "", "", ""))
|
|
||||||
|
|
||||||
require.NoError(t, fw.Drop(flow, true, tunnel(1, "trusted"), cp, nil))
|
|
||||||
|
|
||||||
// Same groups on a new tunnel keeps the entry and moves it to the new tunnel
|
|
||||||
require.NoError(t, fw.Drop(flow, true, tunnel(2, "trusted"), cp, nil))
|
|
||||||
require.Contains(t, fw.Conntrack.Conns, key)
|
|
||||||
assert.Equal(t, uint32(2), fw.Conntrack.Conns[key].localIndex)
|
|
||||||
|
|
||||||
// Losing the group on a new tunnel removes the entry
|
|
||||||
assert.Equal(t, ErrNoMatchingRule, fw.Drop(flow, true, tunnel(3), cp, nil))
|
|
||||||
assert.NotContains(t, fw.Conntrack.Conns, key)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("reply to a flow we opened survives a new tunnel", func(t *testing.T) {
|
|
||||||
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
|
|
||||||
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
|
|
||||||
|
|
||||||
require.NoError(t, fw.Drop(flow, false, tunnel(1, "trusted"), cp, nil))
|
|
||||||
require.NoError(t, fw.Drop(flow, true, tunnel(2), cp, nil))
|
|
||||||
assert.Equal(t, uint32(2), fw.Conntrack.Conns[key].localIndex)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// BenchmarkFirewallDropConntrackHit measures Drop on an already-established flow
|
// BenchmarkFirewallDropConntrackHit measures Drop on an already-established flow
|
||||||
// (a conntrack hit). This is the fast path that the source-IP<->cert binding
|
// (a conntrack hit). This is the fast path that the source-IP<->cert binding
|
||||||
// reordering adds work to, so it quantifies the cost of moving the address checks
|
// reordering adds work to, so it quantifies the cost of moving the address checks
|
||||||
@@ -1989,6 +1742,6 @@ func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, end
|
|||||||
|
|
||||||
func resetConntrack(fw *Firewall) {
|
func resetConntrack(fw *Firewall) {
|
||||||
fw.Conntrack.Lock()
|
fw.Conntrack.Lock()
|
||||||
fw.Conntrack.Conns = map[firewall.ConntrackKey]*conn{}
|
fw.Conntrack.Conns = map[firewall.Packet]*conn{}
|
||||||
fw.Conntrack.Unlock()
|
fw.Conntrack.Unlock()
|
||||||
}
|
}
|
||||||
+3
-26
@@ -25,7 +25,6 @@ import (
|
|||||||
"github.com/slackhq/nebula/overlay/batch"
|
"github.com/slackhq/nebula/overlay/batch"
|
||||||
"github.com/slackhq/nebula/overlay/tio"
|
"github.com/slackhq/nebula/overlay/tio"
|
||||||
"github.com/slackhq/nebula/ratelimit"
|
"github.com/slackhq/nebula/ratelimit"
|
||||||
"github.com/slackhq/nebula/routing"
|
|
||||||
"github.com/slackhq/nebula/udp"
|
"github.com/slackhq/nebula/udp"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -269,9 +268,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
|
|||||||
ifce.reQueryWait.Store(int64(c.reQueryWait))
|
ifce.reQueryWait.Store(int64(c.reQueryWait))
|
||||||
|
|
||||||
ifce.connectionManager.intf = ifce
|
ifce.connectionManager.intf = ifce
|
||||||
if ifce.firewall != nil {
|
|
||||||
ifce.firewall.siblings = ifce
|
|
||||||
}
|
|
||||||
|
|
||||||
// Held until Close so waiting on the interface blocks until the resources are actually released
|
// Held until Close so waiting on the interface blocks until the resources are actually released
|
||||||
ifce.wg.Add(1)
|
ifce.wg.Add(1)
|
||||||
@@ -388,6 +384,8 @@ type rxContext struct {
|
|||||||
hostmapCache map[uint32]*HostInfo
|
hostmapCache map[uint32]*HostInfo
|
||||||
lhh *LightHouseHandler
|
lhh *LightHouseHandler
|
||||||
ctCache *firewall.ConntrackCacheTicker
|
ctCache *firewall.ConntrackCacheTicker
|
||||||
|
relayBatch *batch.SendBatch
|
||||||
|
relayRemote netip.AddrPort
|
||||||
}
|
}
|
||||||
|
|
||||||
func newRxContext(f *Interface, q int) *rxContext {
|
func newRxContext(f *Interface, q int) *rxContext {
|
||||||
@@ -418,6 +416,7 @@ func (f *Interface) listenOut(i int) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
flusher := func() {
|
flusher := func() {
|
||||||
|
f.flushRelayBatch(rxc)
|
||||||
if err := f.batchers[i].Flush(); err != nil {
|
if err := f.batchers[i].Flush(); err != nil {
|
||||||
f.l.Error("Failed to flush tun coalescer", "error", err)
|
f.l.Error("Failed to flush tun coalescer", "error", err)
|
||||||
}
|
}
|
||||||
@@ -558,7 +557,6 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
oldFw := f.firewall
|
oldFw := f.firewall
|
||||||
fw.siblings = f
|
|
||||||
conntrack := oldFw.Conntrack
|
conntrack := oldFw.Conntrack
|
||||||
conntrack.Lock()
|
conntrack.Lock()
|
||||||
defer conntrack.Unlock()
|
defer conntrack.Unlock()
|
||||||
@@ -716,24 +714,3 @@ func (f *Interface) Close() error {
|
|||||||
f.wg.Done()
|
f.wg.Done()
|
||||||
return errors.Join(errs...)
|
return errors.Join(errs...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// siblingPeers implements gatewaySiblings from the routes and tunnels we have now. Routes name a gateway by any of its
|
|
||||||
// vpn addresses, conntrack keys a peer by its first one, so each gateway is resolved through its tunnel. A gateway
|
|
||||||
// we have no tunnel to can't be carrying a live flow.
|
|
||||||
func (f *Interface) siblingPeers(remote netip.Addr, h *HostInfo, buf []netip.Addr) []netip.Addr {
|
|
||||||
gateways := f.inside.RoutesFor(remote)
|
|
||||||
isGateway := func(g routing.Gateway) bool { return slices.Contains(h.vpnAddrs, g.Addr()) }
|
|
||||||
if len(gateways) < 2 || !slices.ContainsFunc(gateways, isGateway) {
|
|
||||||
return buf
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, g := range gateways {
|
|
||||||
if isGateway(g) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if s := f.hostMap.QueryVpnAddr(g.Addr()); s != nil && s != h {
|
|
||||||
buf = append(buf, s.vpnAddrs[0])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return buf
|
|
||||||
}
|
|
||||||
@@ -39,7 +39,7 @@ func Test_statsEmitter_setsGauges(t *testing.T) {
|
|||||||
hostMap: hostMap,
|
hostMap: hostMap,
|
||||||
inside: &overlaytest.NoopTun{},
|
inside: &overlaytest.NoopTun{},
|
||||||
outside: &udp.NoopConn{},
|
outside: &udp.NoopConn{},
|
||||||
firewall: &Firewall{Conntrack: &FirewallConntrack{Conns: map[firewall.ConntrackKey]*conn{}}},
|
firewall: &Firewall{Conntrack: &FirewallConntrack{Conns: map[firewall.Packet]*conn{}}},
|
||||||
lightHouse: lh,
|
lightHouse: lh,
|
||||||
pki: &PKI{},
|
pki: &PKI{},
|
||||||
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
|
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
|
||||||
|
|||||||
@@ -4,11 +4,8 @@ import (
|
|||||||
"net/netip"
|
"net/netip"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/gaissmai/bart"
|
|
||||||
"github.com/slackhq/nebula/cert"
|
"github.com/slackhq/nebula/cert"
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
"github.com/slackhq/nebula/overlay/overlaytest"
|
|
||||||
"github.com/slackhq/nebula/routing"
|
|
||||||
"github.com/slackhq/nebula/test"
|
"github.com/slackhq/nebula/test"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -143,42 +140,3 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
|
|||||||
assert.True(t, res.Failed)
|
assert.True(t, res.Failed)
|
||||||
assert.Same(t, fw, f.firewall)
|
assert.Same(t, fw, f.firewall)
|
||||||
}
|
}
|
||||||
|
|
||||||
// routesTun is a NoopTun with a route table
|
|
||||||
type routesTun struct {
|
|
||||||
overlaytest.NoopTun
|
|
||||||
routes *bart.Table[routing.Gateways]
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t routesTun) RoutesFor(ip netip.Addr) routing.Gateways {
|
|
||||||
r, _ := t.routes.Lookup(ip)
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInterface_SiblingPeers(t *testing.T) {
|
|
||||||
addr := netip.MustParseAddr
|
|
||||||
routes := new(bart.Table[routing.Gateways])
|
|
||||||
// a and b carry 198.51.100.0/24, the route names b by its v6 address
|
|
||||||
routes.Insert(netip.MustParsePrefix("198.51.100.0/24"), routing.Gateways{
|
|
||||||
routing.NewGateway(addr("192.0.2.2"), 1),
|
|
||||||
routing.NewGateway(addr("fd00::3"), 1),
|
|
||||||
routing.NewGateway(addr("192.0.2.5"), 1), // no tunnel to this one
|
|
||||||
})
|
|
||||||
routes.Insert(netip.MustParsePrefix("203.0.113.0/24"), routing.Gateways{routing.NewGateway(addr("192.0.2.2"), 1)})
|
|
||||||
|
|
||||||
hm := newHostMap(test.NewLogger())
|
|
||||||
a := &HostInfo{vpnAddrs: []netip.Addr{addr("192.0.2.2"), addr("fd00::2")}, localIndexId: 1}
|
|
||||||
b := &HostInfo{vpnAddrs: []netip.Addr{addr("192.0.2.3"), addr("fd00::3")}, localIndexId: 2}
|
|
||||||
c := &HostInfo{vpnAddrs: []netip.Addr{addr("192.0.2.4")}, localIndexId: 3}
|
|
||||||
for _, h := range []*HostInfo{a, b, c} {
|
|
||||||
hm.unlockedAddHostInfo(h, &Interface{})
|
|
||||||
}
|
|
||||||
f := &Interface{inside: routesTun{routes: routes}, hostMap: hm}
|
|
||||||
|
|
||||||
// Siblings come back as their first vpn address, the one conntrack keys on, whichever address the route names
|
|
||||||
assert.Equal(t, []netip.Addr{addr("192.0.2.3")}, f.siblingPeers(addr("198.51.100.5"), a, nil))
|
|
||||||
assert.Equal(t, []netip.Addr{addr("192.0.2.2")}, f.siblingPeers(addr("198.51.100.5"), b, nil))
|
|
||||||
assert.Empty(t, f.siblingPeers(addr("198.51.100.5"), c, nil), "not one of our gateways for the network")
|
|
||||||
assert.Empty(t, f.siblingPeers(addr("203.0.113.5"), a, nil), "a single gateway route has no siblings")
|
|
||||||
assert.Empty(t, f.siblingPeers(addr("192.0.2.3"), a, nil), "no route")
|
|
||||||
}
|
|
||||||
+35
-7
@@ -25,7 +25,7 @@ var ErrOutOfWindow = errors.New("out of window packet")
|
|||||||
|
|
||||||
// readOutsidePackets processes one received underlay packet.
|
// readOutsidePackets processes one received underlay packet.
|
||||||
// Message payloads are decrypted IN PLACE, so packet must stay untouched
|
// Message payloads are decrypted IN PLACE, so packet must stay untouched
|
||||||
// by the caller until the batcher for queue q has been flushed
|
// by the caller until the TUN and relay batches for queue q have been flushed.
|
||||||
func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) {
|
func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) {
|
||||||
h := rxc.h
|
h := rxc.h
|
||||||
err := h.Parse(packet)
|
err := h.Parse(packet)
|
||||||
@@ -77,6 +77,12 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
|
|||||||
f.messageMetrics.Rx(h.Type, h.Subtype, 1)
|
f.messageMetrics.Rx(h.Type, h.Subtype, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Earlier forwards must leave before this packet can generate a response.
|
||||||
|
isMessageRelay := h.Type == header.Message && h.Subtype == header.MessageRelay
|
||||||
|
if !isMessageRelay {
|
||||||
|
f.flushRelayBatch(rxc)
|
||||||
|
}
|
||||||
|
|
||||||
// Unencrypted packets
|
// Unencrypted packets
|
||||||
switch h.Type {
|
switch h.Type {
|
||||||
case header.Handshake:
|
case header.Handshake:
|
||||||
@@ -88,9 +94,6 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Relay packets are special
|
|
||||||
isMessageRelay := (h.Type == header.Message && h.Subtype == header.MessageRelay)
|
|
||||||
|
|
||||||
var hostinfo *HostInfo
|
var hostinfo *HostInfo
|
||||||
if isMessageRelay {
|
if isMessageRelay {
|
||||||
hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex)
|
hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex)
|
||||||
@@ -102,6 +105,8 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
|
|||||||
// recvError if necessary
|
// recvError if necessary
|
||||||
if hostinfo == nil || hostinfo.ConnectionState == nil {
|
if hostinfo == nil || hostinfo.ConnectionState == nil {
|
||||||
if !via.IsRelayed {
|
if !via.IsRelayed {
|
||||||
|
// An unknown relay index can also produce a response.
|
||||||
|
f.flushRelayBatch(rxc)
|
||||||
f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex)
|
f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@@ -233,8 +238,10 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
|
|||||||
// Forward this packet through the relay tunnel, rebuilding it in place.
|
// Forward this packet through the relay tunnel, rebuilding it in place.
|
||||||
// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
|
// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
|
||||||
fwdBuf := packet[:0]
|
fwdBuf := packet[:0]
|
||||||
//todo it would potentially be nice to batch these
|
out, err := f.prepareSendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true)
|
||||||
f.SendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true, rxc.q)
|
if err == nil {
|
||||||
|
f.queueRelayPacket(out, targetHI.GetRemote(), rxc)
|
||||||
|
}
|
||||||
case TerminalType:
|
case TerminalType:
|
||||||
hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
|
hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
|
||||||
return
|
return
|
||||||
@@ -259,6 +266,27 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// queueRelayPacket borrows a receive buffer until the batch is flushed. Destination changes and the send batch
|
||||||
|
// limit end a run, preserving arrival order while allowing WriteBatch to use UDP segmentation offload.
|
||||||
|
func (f *Interface) queueRelayPacket(packet []byte, remote netip.AddrPort, rxc *rxContext) {
|
||||||
|
if rxc.relayBatch == nil {
|
||||||
|
rxc.relayBatch = batch.NewSendBatch(f.writers[rxc.q], batch.SendBatchCap, 0)
|
||||||
|
} else if rxc.relayRemote != remote {
|
||||||
|
f.flushRelayBatch(rxc)
|
||||||
|
}
|
||||||
|
rxc.relayRemote = remote
|
||||||
|
rxc.relayBatch.Commit(packet, remote)
|
||||||
|
if rxc.relayBatch.Len() >= batch.SendBatchCap {
|
||||||
|
f.flushRelayBatch(rxc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Interface) flushRelayBatch(rxc *rxContext) {
|
||||||
|
if rxc.relayBatch != nil {
|
||||||
|
f.flushSendBatch(rxc.relayBatch, rxc.q)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
|
// closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
|
||||||
func (f *Interface) closeTunnel(hostInfo *HostInfo) {
|
func (f *Interface) closeTunnel(hostInfo *HostInfo) {
|
||||||
// Hold the hostmap lock across both the delete and the lighthouse eviction,
|
// Hold the hostmap lock across both the delete and the lighthouse eviction,
|
||||||
@@ -478,8 +506,8 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounte
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
if f.l.Enabled(context.Background(), slog.LevelDebug) {
|
if f.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||||
hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out)
|
hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
dropReason := f.firewall.Drop(rxc.fwPacket.Packet, true, hostinfo, f.pki.GetCAPool(), rxc.ctCache.Get())
|
dropReason := f.firewall.Drop(rxc.fwPacket.Packet, true, hostinfo, f.pki.GetCAPool(), rxc.ctCache.Get())
|
||||||
|
|||||||
@@ -0,0 +1,291 @@
|
|||||||
|
package nebula
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"net/netip"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/flynn/noise"
|
||||||
|
"github.com/gaissmai/bart"
|
||||||
|
"github.com/rcrowley/go-metrics"
|
||||||
|
"github.com/slackhq/nebula/header"
|
||||||
|
"github.com/slackhq/nebula/noiseutil"
|
||||||
|
"github.com/slackhq/nebula/overlay/batch"
|
||||||
|
"github.com/slackhq/nebula/test"
|
||||||
|
"github.com/slackhq/nebula/udp"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
type relayBatchWrite struct {
|
||||||
|
packets [][]byte
|
||||||
|
remotes []netip.AddrPort
|
||||||
|
batched bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type relayBatchConn struct {
|
||||||
|
udp.NoopConn
|
||||||
|
read func(udp.EncReader, func())
|
||||||
|
writes []relayBatchWrite
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *relayBatchConn) ListenOut(r udp.EncReader, flush func()) error {
|
||||||
|
c.read(r, flush)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *relayBatchConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||||
|
w := relayBatchWrite{remotes: append([]netip.AddrPort(nil), addrs...), batched: true}
|
||||||
|
for _, b := range bufs {
|
||||||
|
w.packets = append(w.packets, bytes.Clone(b))
|
||||||
|
}
|
||||||
|
c.writes = append(c.writes, w)
|
||||||
|
return len(bufs), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *relayBatchConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
||||||
|
c.writes = append(c.writes, relayBatchWrite{packets: [][]byte{bytes.Clone(b)}, remotes: []netip.AddrPort{addr}})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *relayBatchConn) packets() [][]byte {
|
||||||
|
var packets [][]byte
|
||||||
|
for _, w := range c.writes {
|
||||||
|
packets = append(packets, w.packets...)
|
||||||
|
}
|
||||||
|
return packets
|
||||||
|
}
|
||||||
|
|
||||||
|
type relayBatchPath struct {
|
||||||
|
f *Interface
|
||||||
|
conn *relayBatchConn
|
||||||
|
source, target *HostInfo
|
||||||
|
incoming, outgoing *Relay
|
||||||
|
sourceKey noiseutil.CipherState
|
||||||
|
cipher noise.CipherFunc
|
||||||
|
counter uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Separate cipher instances let the reference SendVia path use the same keys and
|
||||||
|
// counters without rewinding an AEAD instance (including FIPS nonce checks).
|
||||||
|
func relayBatchKey(cipher noise.CipherFunc, id byte) noiseutil.CipherState {
|
||||||
|
suite := noise.NewCipherSuite(noise.DH25519, cipher, noise.HashSHA256)
|
||||||
|
return noiseutil.NewCipherState(noise.UnsafeNewCipherState(suite, [32]byte{id}, 0), cipher)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRelayBatchPath(t *testing.T, cipher noise.CipherFunc) *relayBatchPath {
|
||||||
|
t.Helper()
|
||||||
|
l := test.NewLogger()
|
||||||
|
conn := &relayBatchConn{}
|
||||||
|
f := &Interface{
|
||||||
|
ctx: t.Context(), l: l, hostMap: newHostMap(l), outside: conn,
|
||||||
|
writers: []udp.Conn{conn}, lightHouse: &LightHouse{l: l},
|
||||||
|
myVpnNetworksTable: &bart.Lite{}, metricTxDropped: metrics.NewCounter(),
|
||||||
|
batchers: []*batch.MultiCoalescer{batch.NewMultiCoalescer(&capturingTun{}, l)},
|
||||||
|
}
|
||||||
|
f.connectionManager = &connectionManager{
|
||||||
|
intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{},
|
||||||
|
}
|
||||||
|
newHost := func(addr string, remote string, id byte) *HostInfo {
|
||||||
|
h := &HostInfo{
|
||||||
|
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
|
||||||
|
ConnectionState: &ConnectionState{
|
||||||
|
eKey: relayBatchKey(cipher, id), dKey: relayBatchKey(cipher, id), window: NewBits(ReplayWindow),
|
||||||
|
},
|
||||||
|
relayState: RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}},
|
||||||
|
}
|
||||||
|
u := netip.MustParseAddrPort(remote)
|
||||||
|
h.remote.Store(&u)
|
||||||
|
f.hostMap.Hosts[h.vpnAddrs[0]] = h
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
source := newHost("192.0.2.1", "198.51.100.1:4242", 1)
|
||||||
|
target := newHost("192.0.2.2", "198.51.100.2:4242", 2)
|
||||||
|
incoming := &Relay{Type: ForwardingType, State: Established, LocalIndex: 301, RemoteIndex: 302, PeerAddr: target.vpnAddrs[0]}
|
||||||
|
outgoing := &Relay{Type: ForwardingType, State: Established, LocalIndex: 303, RemoteIndex: 304, PeerAddr: source.vpnAddrs[0]}
|
||||||
|
source.relayState.InsertRelay(incoming.PeerAddr, incoming.LocalIndex, incoming)
|
||||||
|
target.relayState.InsertRelay(outgoing.PeerAddr, outgoing.LocalIndex, outgoing)
|
||||||
|
f.hostMap.Relays[incoming.LocalIndex] = source
|
||||||
|
f.hostMap.Indexes[100] = source
|
||||||
|
return &relayBatchPath{f: f, conn: conn, source: source, target: target, incoming: incoming, outgoing: outgoing, sourceKey: relayBatchKey(cipher, 1), cipher: cipher}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *relayBatchPath) wrap(t *testing.T, payload []byte) []byte {
|
||||||
|
t.Helper()
|
||||||
|
p.counter++
|
||||||
|
b := header.Encode(make([]byte, header.Len), header.Version, header.Message, header.MessageRelay, p.incoming.LocalIndex, p.counter)
|
||||||
|
b = append(b, payload...)
|
||||||
|
b, err := p.sourceKey.EncryptDanger(b, b, nil, p.counter, make([]byte, 12))
|
||||||
|
require.NoError(t, err)
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *relayBatchPath) reference(payloads [][]byte) [][]byte {
|
||||||
|
conn := &relayBatchConn{}
|
||||||
|
f := &Interface{writers: []udp.Conn{conn}, connectionManager: p.f.connectionManager, l: p.f.l}
|
||||||
|
h := &HostInfo{ConnectionState: &ConnectionState{eKey: relayBatchKey(p.cipher, 2)}}
|
||||||
|
u := p.target.GetRemote()
|
||||||
|
h.remote.Store(&u)
|
||||||
|
for _, payload := range payloads {
|
||||||
|
f.SendVia(h, p.outgoing, payload, make([]byte, 12), make([]byte, 0, len(payload)+header.MaxOverhead), false, 0)
|
||||||
|
}
|
||||||
|
return conn.packets()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRelayReceiveBatch(t *testing.T) {
|
||||||
|
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
|
||||||
|
t.Run(cipher.CipherName(), func(t *testing.T) {
|
||||||
|
p := newRelayBatchPath(t, cipher)
|
||||||
|
const count = 8
|
||||||
|
var payloads, wire [][]byte
|
||||||
|
for i := range count * 2 {
|
||||||
|
payload := bytes.Repeat([]byte{byte(i)}, 64)
|
||||||
|
payloads = append(payloads, payload)
|
||||||
|
wire = append(wire, p.wrap(t, payload))
|
||||||
|
}
|
||||||
|
want := p.reference(payloads)
|
||||||
|
// Model a listener reusing the same slots for the next receive batch.
|
||||||
|
buffers := make([][]byte, count)
|
||||||
|
for i := range buffers {
|
||||||
|
buffers[i] = make([]byte, len(wire[i]))
|
||||||
|
}
|
||||||
|
p.conn.read = func(r udp.EncReader, flush func()) {
|
||||||
|
for b := range 2 {
|
||||||
|
for i := range count {
|
||||||
|
copy(buffers[i], wire[b*count+i])
|
||||||
|
r(p.source.GetRemote(), buffers[i])
|
||||||
|
require.Len(t, p.conn.writes, b, "forwarding waits for the batch boundary")
|
||||||
|
}
|
||||||
|
flush()
|
||||||
|
require.Len(t, p.conn.writes, b+1, "flush must send before receive slots are reused")
|
||||||
|
for _, buffer := range buffers {
|
||||||
|
clear(buffer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.f.listenOut(0)
|
||||||
|
require.Equal(t, want, p.conn.packets())
|
||||||
|
for _, w := range p.conn.writes {
|
||||||
|
require.True(t, w.batched)
|
||||||
|
require.Len(t, w.packets, count)
|
||||||
|
for _, remote := range w.remotes {
|
||||||
|
require.Equal(t, p.target.GetRemote(), remote)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRelayBatchDestinationChange(t *testing.T) {
|
||||||
|
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
|
||||||
|
first, second := p.target.GetRemote(), netip.MustParseAddrPort("198.51.100.3:4242")
|
||||||
|
remotes := []netip.AddrPort{first, first, second, second, first}
|
||||||
|
var payloads, wire [][]byte
|
||||||
|
for i := range remotes {
|
||||||
|
payload := []byte{byte(i)}
|
||||||
|
payloads = append(payloads, payload)
|
||||||
|
wire = append(wire, p.wrap(t, payload))
|
||||||
|
}
|
||||||
|
want := p.reference(payloads)
|
||||||
|
p.conn.read = func(r udp.EncReader, flush func()) {
|
||||||
|
for i, remote := range remotes {
|
||||||
|
p.target.remote.Store(&remote)
|
||||||
|
r(p.source.GetRemote(), wire[i])
|
||||||
|
}
|
||||||
|
flush()
|
||||||
|
}
|
||||||
|
p.f.listenOut(0)
|
||||||
|
require.Equal(t, want, p.conn.packets())
|
||||||
|
require.Len(t, p.conn.writes, 3)
|
||||||
|
for i, wantRemotes := range [][]netip.AddrPort{{first, first}, {second, second}, {first}} {
|
||||||
|
require.Equal(t, wantRemotes, p.conn.writes[i].remotes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRelayBatchCapacity(t *testing.T) {
|
||||||
|
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
|
||||||
|
p.conn.read = func(r udp.EncReader, flush func()) {
|
||||||
|
for i := range batch.SendBatchCap + 1 {
|
||||||
|
r(p.source.GetRemote(), p.wrap(t, []byte{byte(i)}))
|
||||||
|
}
|
||||||
|
require.Len(t, p.conn.writes, 1)
|
||||||
|
require.Len(t, p.conn.writes[0].packets, batch.SendBatchCap)
|
||||||
|
flush()
|
||||||
|
}
|
||||||
|
p.f.listenOut(0)
|
||||||
|
require.Len(t, p.conn.writes, 2)
|
||||||
|
require.Len(t, p.conn.writes[1].packets, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRelayBatchBeforeResponse(t *testing.T) {
|
||||||
|
for _, unknownRelay := range []bool{false, true} {
|
||||||
|
name := "test request"
|
||||||
|
if unknownRelay {
|
||||||
|
name = "unknown relay"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
|
||||||
|
first, last := p.wrap(t, []byte("first")), p.wrap(t, []byte("last"))
|
||||||
|
p.counter++
|
||||||
|
request := header.Encode(make([]byte, header.Len), header.Version, header.Test, header.TestRequest, 100, p.counter)
|
||||||
|
request, err := p.sourceKey.EncryptDanger(request, request, []byte("ping"), p.counter, make([]byte, 12))
|
||||||
|
require.NoError(t, err)
|
||||||
|
if unknownRelay {
|
||||||
|
request = header.Encode(make([]byte, header.Len), header.Version, header.Message, header.MessageRelay, 999, p.counter)
|
||||||
|
}
|
||||||
|
p.conn.read = func(r udp.EncReader, flush func()) {
|
||||||
|
r(p.source.GetRemote(), first)
|
||||||
|
require.Empty(t, p.conn.writes)
|
||||||
|
r(p.source.GetRemote(), request)
|
||||||
|
require.Len(t, p.conn.writes, 2)
|
||||||
|
r(p.source.GetRemote(), last)
|
||||||
|
flush()
|
||||||
|
}
|
||||||
|
p.f.listenOut(0)
|
||||||
|
require.Len(t, p.conn.writes, 3)
|
||||||
|
require.True(t, p.conn.writes[0].batched)
|
||||||
|
require.False(t, p.conn.writes[1].batched)
|
||||||
|
require.True(t, p.conn.writes[2].batched)
|
||||||
|
require.Equal(t, p.source.GetRemote(), p.conn.writes[1].remotes[0])
|
||||||
|
h := &header.H{}
|
||||||
|
require.NoError(t, h.Parse(p.conn.writes[1].packets[0]))
|
||||||
|
if unknownRelay {
|
||||||
|
require.Equal(t, header.RecvError, h.Type)
|
||||||
|
require.EqualValues(t, 999, h.RemoteIndex)
|
||||||
|
} else {
|
||||||
|
require.Equal(t, header.Test, h.Type)
|
||||||
|
require.Equal(t, header.TestReply, h.Subtype)
|
||||||
|
reply := p.conn.writes[1].packets[0]
|
||||||
|
out, err := p.sourceKey.DecryptDanger(nil, reply[:header.Len], reply[header.Len:], h.MessageCounter, make([]byte, 12))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, []byte("ping"), out)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRelayBatchRemovedTarget(t *testing.T) {
|
||||||
|
for _, remove := range []string{"host", "relay", "disestablished"} {
|
||||||
|
t.Run(remove, func(t *testing.T) {
|
||||||
|
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
|
||||||
|
first, removed := p.wrap(t, []byte("first")), p.wrap(t, []byte("removed"))
|
||||||
|
want := p.reference([][]byte{[]byte("first")})
|
||||||
|
p.conn.read = func(r udp.EncReader, flush func()) {
|
||||||
|
r(p.source.GetRemote(), first)
|
||||||
|
switch remove {
|
||||||
|
case "host":
|
||||||
|
delete(p.f.hostMap.Hosts, p.target.vpnAddrs[0])
|
||||||
|
case "relay":
|
||||||
|
delete(p.target.relayState.relayForByAddr, p.source.vpnAddrs[0])
|
||||||
|
case "disestablished":
|
||||||
|
p.target.relayState.UpdateRelayForByIpState(p.source.vpnAddrs[0], Disestablished)
|
||||||
|
}
|
||||||
|
r(p.source.GetRemote(), removed)
|
||||||
|
flush()
|
||||||
|
}
|
||||||
|
p.f.listenOut(0)
|
||||||
|
require.Len(t, p.conn.writes, 1)
|
||||||
|
require.Equal(t, want, p.conn.packets(), "only the packet accepted before removal is sent")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,8 +10,8 @@ type batchWriter interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
|
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
|
||||||
// One SendBatch is owned by each listenIn goroutine; no locking is needed.
|
// Each reader goroutine owns its own SendBatch; no locking is needed.
|
||||||
// Slots are backed by an Arena (see its docs)
|
// Slots can borrow receive buffers or be backed by an Arena (see its docs).
|
||||||
type SendBatch struct {
|
type SendBatch struct {
|
||||||
out batchWriter
|
out batchWriter
|
||||||
bufs [][]byte
|
bufs [][]byte
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
//go:build !ios && !e2e_testing
|
||||||
|
|
||||||
|
package overlay
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTun_WriteFraming(t *testing.T) {
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer r.Close()
|
||||||
|
defer w.Close()
|
||||||
|
tn := &tun{f: w}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
pkt []byte
|
||||||
|
af byte
|
||||||
|
}{
|
||||||
|
// Literal families rather than syscall.AF_*, so a wrong constant in Write can't also fix the expectation.
|
||||||
|
{"ipv4", []byte{0x45, 1, 2, 3, 4, 5}, 2},
|
||||||
|
{"ipv6", []byte{0x60, 1, 2, 3, 4, 5}, 30},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
n, err := tn.Write(tc.pkt)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, len(tc.pkt), n)
|
||||||
|
|
||||||
|
got := make([]byte, 64)
|
||||||
|
n, err = r.Read(got)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, append([]byte{0, 0, 0, tc.af}, tc.pkt...), got[:n])
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = tn.Write([]byte{0x10, 1, 2, 3})
|
||||||
|
require.Error(t, err)
|
||||||
|
_, err = tn.Write([]byte{})
|
||||||
|
require.ErrorIs(t, err, syscall.EIO)
|
||||||
|
|
||||||
|
// The rejected writes must not have put anything on the fd, so the next read sees only this packet.
|
||||||
|
pkt := []byte{0x45, 9, 9}
|
||||||
|
_, err = tn.Write(pkt)
|
||||||
|
require.NoError(t, err)
|
||||||
|
got := make([]byte, 64)
|
||||||
|
n, err := r.Read(got)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, append([]byte{0, 0, 0, 2}, pkt...), got[:n])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTun_ReadStripsHeader(t *testing.T) {
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer r.Close()
|
||||||
|
defer w.Close()
|
||||||
|
tn := &tun{f: r}
|
||||||
|
|
||||||
|
pkt := []byte{0x45, 1, 2, 3, 4, 5}
|
||||||
|
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
got := make([]byte, 64)
|
||||||
|
n, err := tn.Read(got)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, pkt, got[:n])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTun_ReadShort(t *testing.T) {
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer r.Close()
|
||||||
|
defer w.Close()
|
||||||
|
tn := &tun{f: r}
|
||||||
|
|
||||||
|
// Fewer bytes than the protocol header is not a packet.
|
||||||
|
_, err = w.Write([]byte{0, 0})
|
||||||
|
require.NoError(t, err)
|
||||||
|
n, err := tn.Read(make([]byte, 64))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Zero(t, n)
|
||||||
|
|
||||||
|
// The next packet reads cleanly only if the short read consumed exactly what was written.
|
||||||
|
pkt := []byte{0x45, 1, 2, 3}
|
||||||
|
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
|
||||||
|
require.NoError(t, err)
|
||||||
|
got := make([]byte, 64)
|
||||||
|
n, err = tn.Read(got)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, pkt, got[:n])
|
||||||
|
}
|
||||||
|
|
||||||
|
// A heap allocation per packet rarely shows in a short throughput run, only across GC pauses, so
|
||||||
|
// this pins Read and Write to zero allocations directly.
|
||||||
|
func TestTun_ReadWriteDoNotAllocate(t *testing.T) {
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer r.Close()
|
||||||
|
defer w.Close()
|
||||||
|
tw := &tun{f: w}
|
||||||
|
tr := &tun{f: r}
|
||||||
|
|
||||||
|
pkts := [][]byte{bytes.Repeat([]byte{0x45}, 1300), bytes.Repeat([]byte{0x60}, 1300)}
|
||||||
|
buf := make([]byte, 1500)
|
||||||
|
i := 0
|
||||||
|
allocs := testing.AllocsPerRun(1000, func() {
|
||||||
|
i++
|
||||||
|
if _, err := tw.Write(pkts[i%len(pkts)]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := tr.Read(buf); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
require.Zero(t, allocs)
|
||||||
|
}
|
||||||
@@ -2,7 +2,9 @@ package nebula
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/fips140"
|
"crypto/fips140"
|
||||||
|
"crypto/sha256"
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -233,12 +235,14 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
|
|||||||
|
|
||||||
func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError {
|
func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError {
|
||||||
caPool, err := loadCAPoolFromConfig(p.l, c)
|
caPool, err := loadCAPoolFromConfig(p.l, c)
|
||||||
|
if caPool != nil {
|
||||||
|
p.caPool.Store(caPool)
|
||||||
|
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return util.NewContextualError("Failed to load ca from config", nil, err)
|
return util.NewContextualError("Failed to load ca from config", nil, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
p.caPool.Store(caPool)
|
|
||||||
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -613,11 +617,22 @@ func loadCAPoolFromConfig(l *slog.Logger, c *config.C) (*cert.CAPool, error) {
|
|||||||
|
|
||||||
bl := c.GetStringSlice("pki.blocklist", []string{})
|
bl := c.GetStringSlice("pki.blocklist", []string{})
|
||||||
if len(bl) > 0 {
|
if len(bl) > 0 {
|
||||||
for _, fp := range bl {
|
var bad []string
|
||||||
|
for i, raw := range bl {
|
||||||
|
// Fingerprints are compared as lowercase hex strings, a malformed entry would silently block nothing
|
||||||
|
fp := strings.ToLower(strings.TrimSpace(raw))
|
||||||
|
if b, err := hex.DecodeString(fp); err != nil || len(b) != sha256.Size {
|
||||||
|
bad = append(bad, fmt.Sprintf("entry %d %q", i+1, raw))
|
||||||
|
continue
|
||||||
|
}
|
||||||
caPool.BlocklistFingerprint(fp)
|
caPool.BlocklistFingerprint(fp)
|
||||||
}
|
}
|
||||||
|
|
||||||
l.Info("Blocklisted certificates", "fingerprintCount", len(bl))
|
l.Info("Blocklisted certificates", "fingerprintCount", len(bl)-len(bad))
|
||||||
|
if len(bad) > 0 {
|
||||||
|
// The valid entries are still returned so a failed reload blocks them, startup refuses the error
|
||||||
|
return caPool, fmt.Errorf("pki.blocklist entries are not hex certificate fingerprints: %s", strings.Join(bad, ", "))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return caPool, nil
|
return caPool, nil
|
||||||
|
|||||||
+43
@@ -217,3 +217,46 @@ func TestPKIReloadChachapolyInFIPSOnlyFails(t *testing.T) {
|
|||||||
assert.True(t, res.Failed)
|
assert.True(t, res.Failed)
|
||||||
assert.False(t, res.RestartRequired)
|
assert.False(t, res.RestartRequired)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadCAPoolBlocklist(t *testing.T) {
|
||||||
|
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
|
||||||
|
crt, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil)
|
||||||
|
fp, err := crt.Fingerprint()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
load := func(entry string) (*cert.CAPool, error) {
|
||||||
|
c := config.NewC(test.NewLogger())
|
||||||
|
require.NoError(t, c.LoadString(fmt.Sprintf("pki:\n ca: %q\n blocklist:\n - %q\n", caPEM, entry)))
|
||||||
|
return loadCAPoolFromConfig(test.NewLogger(), c)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range []string{fp, strings.ToUpper(fp), " " + fp + "\t"} {
|
||||||
|
pool, err := load(entry)
|
||||||
|
require.NoError(t, err, entry)
|
||||||
|
assert.True(t, pool.IsBlocklisted(fp), entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range []string{fp[1:], fp + "00", "sha256:" + fp, "zz" + fp[2:], ""} {
|
||||||
|
_, err := load(entry)
|
||||||
|
require.Error(t, err, entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A bad entry on reload fails it, the valid entries are still blocked
|
||||||
|
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "me", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.1/24")}, nil, nil)
|
||||||
|
pkiConfig := func(entry string) string {
|
||||||
|
return fmt.Sprintf("pki:\n ca: %q\n cert: %q\n key: %q\n blocklist:\n - %q\n", caPEM, certPEM, keyPEM, entry)
|
||||||
|
}
|
||||||
|
c := config.NewC(test.NewLogger())
|
||||||
|
require.NoError(t, c.LoadString(pkiConfig(fp)))
|
||||||
|
p, err := NewPKIFromConfig(test.NewLogger(), c)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
other, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "other", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.3/24")}, nil, nil)
|
||||||
|
otherFp, err := other.Fingerprint()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
res := c.ReloadConfigString(pkiConfig(fp) + fmt.Sprintf(" - %q\n - %q\n", otherFp, fp[1:]))
|
||||||
|
assert.True(t, res.Failed)
|
||||||
|
assert.True(t, p.GetCAPool().IsBlocklisted(fp))
|
||||||
|
assert.True(t, p.GetCAPool().IsBlocklisted(otherFp))
|
||||||
|
}
|
||||||
+4
-2
@@ -36,11 +36,13 @@ func (l *tcpListener) Accept() (net.Conn, error) {
|
|||||||
|
|
||||||
func (l *tcpListener) Close() error {
|
func (l *tcpListener) Close() error {
|
||||||
l.closeOnce.Do(func() {
|
l.closeOnce.Do(func() {
|
||||||
|
// Close before leaving the map so a concurrent closeListeners that
|
||||||
|
// no longer sees this listener knows Accept is already released.
|
||||||
|
close(l.closed)
|
||||||
|
|
||||||
l.s.mu.Lock()
|
l.s.mu.Lock()
|
||||||
delete(l.s.mu.listeners, l.port)
|
delete(l.s.mu.listeners, l.port)
|
||||||
l.s.mu.Unlock()
|
l.s.mu.Unlock()
|
||||||
|
|
||||||
close(l.closed)
|
|
||||||
})
|
})
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
+70
-7
@@ -32,13 +32,18 @@ import (
|
|||||||
const nicID = 1
|
const nicID = 1
|
||||||
|
|
||||||
type Service struct {
|
type Service struct {
|
||||||
eg *errgroup.Group
|
|
||||||
control *nebula.Control
|
control *nebula.Control
|
||||||
ipstack *stack.Stack
|
ipstack *stack.Stack
|
||||||
|
|
||||||
|
// done is closed once the forwarding goroutines have exited and the
|
||||||
|
// netstack is torn down, at which point err holds the first fatal error.
|
||||||
|
done chan struct{}
|
||||||
|
err error
|
||||||
|
|
||||||
mu struct {
|
mu struct {
|
||||||
sync.Mutex
|
sync.Mutex
|
||||||
|
|
||||||
|
closed bool
|
||||||
listeners map[uint16]*tcpListener
|
listeners map[uint16]*tcpListener
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -65,7 +70,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
|||||||
ctx := control.Context()
|
ctx := control.Context()
|
||||||
eg, ctx := errgroup.WithContext(ctx)
|
eg, ctx := errgroup.WithContext(ctx)
|
||||||
s := Service{
|
s := Service{
|
||||||
eg: eg,
|
|
||||||
control: control,
|
control: control,
|
||||||
}
|
}
|
||||||
s.mu.listeners = map[uint16]*tcpListener{}
|
s.mu.listeners = map[uint16]*tcpListener{}
|
||||||
@@ -74,6 +78,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
|||||||
NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
|
NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
|
||||||
TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6},
|
TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6},
|
||||||
})
|
})
|
||||||
|
// The teardown goroutine only owns the stack once New succeeds
|
||||||
|
defer func() {
|
||||||
|
if reterr != nil {
|
||||||
|
s.ipstack.Destroy()
|
||||||
|
}
|
||||||
|
}()
|
||||||
sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default
|
sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default
|
||||||
tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt)
|
tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt)
|
||||||
if tcpipErr != nil {
|
if tcpipErr != nil {
|
||||||
@@ -123,6 +133,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
|||||||
// this will read exactly one packet
|
// this will read exactly one packet
|
||||||
n, err := reader.Read(buf)
|
n, err := reader.Read(buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
// An error after shutdown began is noise from the device pipes
|
||||||
|
// closing, anything else is fatal and must stop nebula too
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
s.control.Stop()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{
|
packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{
|
||||||
@@ -130,8 +146,8 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
|||||||
})
|
})
|
||||||
linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf)
|
linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf)
|
||||||
|
|
||||||
if err := ctx.Err(); err != nil {
|
if ctx.Err() != nil {
|
||||||
return err
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -139,13 +155,18 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
|||||||
for {
|
for {
|
||||||
packet := linkEP.ReadContext(ctx)
|
packet := linkEP.ReadContext(ctx)
|
||||||
if packet == nil {
|
if packet == nil {
|
||||||
if err := ctx.Err(); err != nil {
|
if ctx.Err() != nil {
|
||||||
return err
|
return nil
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
bufView := packet.ToView()
|
bufView := packet.ToView()
|
||||||
if _, err := bufView.WriteTo(writer); err != nil {
|
if _, err := bufView.WriteTo(writer); err != nil {
|
||||||
|
// Same shutdown noise handling as the inbound reader
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
s.control.Stop()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bufView.Release()
|
bufView.Release()
|
||||||
@@ -158,6 +179,22 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
|||||||
return control.Wait()
|
return control.Wait()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Tear down here rather than in Close so it also happens when nebula stops
|
||||||
|
// on its own. The forwarding goroutines must exit before the stack they feed
|
||||||
|
// is destroyed.
|
||||||
|
s.done = make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
s.err = eg.Wait()
|
||||||
|
s.closeListeners()
|
||||||
|
// Destroy aborts the endpoints that exist when it starts, then waits on
|
||||||
|
// every endpoint, so a Dial registering in between would hold Wait until
|
||||||
|
// its connect timed out. Without the NIC new dials fail route lookup.
|
||||||
|
// RemoveNIC also closes linkEP.
|
||||||
|
_ = s.ipstack.RemoveNIC(nicID)
|
||||||
|
s.ipstack.Destroy()
|
||||||
|
close(s.done)
|
||||||
|
}()
|
||||||
|
|
||||||
return &s, nil
|
return &s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -237,6 +274,9 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
|
|||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
if s.mu.closed {
|
||||||
|
return nil, net.ErrClosed
|
||||||
|
}
|
||||||
if _, ok := s.mu.listeners[port]; ok {
|
if _, ok := s.mu.listeners[port]; ok {
|
||||||
return nil, fmt.Errorf("already listening on port %d", port)
|
return nil, fmt.Errorf("already listening on port %d", port)
|
||||||
}
|
}
|
||||||
@@ -245,15 +285,38 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
|
|||||||
return l, nil
|
return l, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Wait blocks until the service has fully stopped, either via Close or a fatal
|
||||||
|
// error, and returns the first fatal error if there was one.
|
||||||
func (s *Service) Wait() error {
|
func (s *Service) Wait() error {
|
||||||
return s.eg.Wait()
|
<-s.done
|
||||||
|
return s.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Close closes all listeners and stops nebula. Use Wait to block until
|
||||||
|
// shutdown has completed.
|
||||||
func (s *Service) Close() error {
|
func (s *Service) Close() error {
|
||||||
|
s.closeListeners()
|
||||||
s.control.Stop()
|
s.control.Stop()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// closeListeners unblocks pending Accept calls and rejects new listeners.
|
||||||
|
func (s *Service) closeListeners() {
|
||||||
|
// Every caller closes every listener rather than returning early once
|
||||||
|
// closed is set, so none returns while another is still closing them.
|
||||||
|
s.mu.Lock()
|
||||||
|
s.mu.closed = true
|
||||||
|
listeners := make([]*tcpListener, 0, len(s.mu.listeners))
|
||||||
|
for _, l := range s.mu.listeners {
|
||||||
|
listeners = append(listeners, l)
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
for _, l := range listeners {
|
||||||
|
_ = l.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Service) tcpHandler(r *tcp.ForwarderRequest) {
|
func (s *Service) tcpHandler(r *tcp.ForwarderRequest) {
|
||||||
endpointID := r.ID()
|
endpointID := r.ID()
|
||||||
|
|
||||||
|
|||||||
@@ -327,3 +327,162 @@ func TestListenerAcceptAfterClose(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Close must release blocked Accept calls and let Wait report a clean shutdown.
|
||||||
|
func TestServiceClose(t *testing.T) {
|
||||||
|
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||||
|
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
|
||||||
|
"static_host_map": m{},
|
||||||
|
"listen": m{
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
"port": 0,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
ln, err := a.Listen("tcp", ":1234")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
accepted := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := ln.Accept()
|
||||||
|
accepted <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
if err := a.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-accepted:
|
||||||
|
if !errors.Is(err, net.ErrClosed) {
|
||||||
|
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Accept still blocked after Close")
|
||||||
|
}
|
||||||
|
|
||||||
|
waited := make(chan error, 1)
|
||||||
|
go func() { waited <- a.Wait() }()
|
||||||
|
select {
|
||||||
|
case err := <-waited:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected clean shutdown from Wait, got %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Wait still blocked after Close")
|
||||||
|
}
|
||||||
|
|
||||||
|
if a.ipstack.HasNIC(nicID) {
|
||||||
|
t.Fatal("netstack not torn down after Wait")
|
||||||
|
}
|
||||||
|
if err := a.Close(); err != nil {
|
||||||
|
t.Fatalf("second close: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := a.Listen("tcp", ":1235"); !errors.Is(err, net.ErrClosed) {
|
||||||
|
t.Fatalf("expected net.ErrClosed from Listen after Close, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If nebula stops without Close being called, the service must still release
|
||||||
|
// Accept and tear down the netstack.
|
||||||
|
func TestServiceNebulaStopped(t *testing.T) {
|
||||||
|
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||||
|
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
|
||||||
|
"static_host_map": m{},
|
||||||
|
"listen": m{
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
"port": 0,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
ln, err := a.Listen("tcp", ":1234")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
accepted := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := ln.Accept()
|
||||||
|
accepted <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
a.control.Stop()
|
||||||
|
|
||||||
|
waited := make(chan error, 1)
|
||||||
|
go func() { waited <- a.Wait() }()
|
||||||
|
select {
|
||||||
|
case err := <-waited:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected clean shutdown from Wait, got %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Wait still blocked after nebula stopped")
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-accepted:
|
||||||
|
if !errors.Is(err, net.ErrClosed) {
|
||||||
|
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Accept still blocked after nebula stopped")
|
||||||
|
}
|
||||||
|
|
||||||
|
if a.ipstack.HasNIC(nicID) {
|
||||||
|
t.Fatal("netstack not torn down after Wait")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dials racing Close must not hold Wait until their connect times out.
|
||||||
|
func TestServiceDialDuringClose(t *testing.T) {
|
||||||
|
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||||
|
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
|
||||||
|
"static_host_map": m{},
|
||||||
|
"listen": m{
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
"port": 0,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
stop := make(chan struct{})
|
||||||
|
var eg errgroup.Group
|
||||||
|
for range 8 {
|
||||||
|
eg.Go(func() error {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
// 10.0.0.2 has no tunnel, so each dial sits in SYN-SENT until aborted
|
||||||
|
if c, err := a.Dial("tcp", "10.0.0.2:80"); err == nil {
|
||||||
|
_ = c.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
if err := a.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waited := make(chan error, 1)
|
||||||
|
go func() { waited <- a.Wait() }()
|
||||||
|
select {
|
||||||
|
case err := <-waited:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected clean shutdown from Wait, got %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Wait blocked behind a dial that raced Close")
|
||||||
|
}
|
||||||
|
|
||||||
|
close(stop)
|
||||||
|
dialersDone := make(chan error, 1)
|
||||||
|
go func() { dialersDone <- eg.Wait() }()
|
||||||
|
select {
|
||||||
|
case <-dialersDone:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Dial still blocked after Wait")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,12 +11,8 @@ import (
|
|||||||
"net"
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"runtime/pprof"
|
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
@@ -60,8 +56,6 @@ type sshDeviceInfoFlags struct {
|
|||||||
|
|
||||||
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
|
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
|
||||||
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
|
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
|
||||||
// sshd.sandbox_dir is read once, when the commands are attached
|
|
||||||
r.RestartRequiredIfChanged("sshd.sandbox_dir")
|
|
||||||
if c.GetBool("sshd.enabled", false) {
|
if c.GetBool("sshd.enabled", false) {
|
||||||
// Restarting the server drops every open session, the one that asked for this reload included. Only restart
|
// Restarting the server drops every open session, the one that asked for this reload included. Only restart
|
||||||
// when it isn't running or its config changed. The host key file can change under the same config, new
|
// when it isn't running or its config changed. The host key file can change under the same config, new
|
||||||
@@ -93,8 +87,7 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// sshdServerKeys are what configSSH reads. sshd.sandbox_dir isn't one, it's read once at startup and restarting the
|
// sshdServerKeys are what configSSH reads
|
||||||
// server doesn't apply it
|
|
||||||
var sshdServerKeys = []string{"sshd.enabled", "sshd.listen", "sshd.host_key", "sshd.trusted_cas", "sshd.authorized_users"}
|
var sshdServerKeys = []string{"sshd.enabled", "sshd.listen", "sshd.host_key", "sshd.trusted_cas", "sshd.authorized_users"}
|
||||||
|
|
||||||
// loadSSHHostKey returns sshd.host_key, read from its file when it names one.
|
// loadSSHHostKey returns sshd.host_key, read from its file when it names one.
|
||||||
@@ -236,12 +229,6 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
|
|||||||
}
|
}
|
||||||
|
|
||||||
func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
|
func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
|
||||||
// sandboxDir defaults to a dir in temp. The intention is that end user will
|
|
||||||
// create this dir as needed. Overriding this config value to "" allows
|
|
||||||
// writing to anywhere in the system.
|
|
||||||
defaultDir := filepath.Join(os.TempDir(), "nebula-debug")
|
|
||||||
sandboxDir := c.GetString("sshd.sandbox_dir", defaultDir)
|
|
||||||
|
|
||||||
ssh.RegisterCommand(&sshd.Command{
|
ssh.RegisterCommand(&sshd.Command{
|
||||||
Name: "list-hostmap",
|
Name: "list-hostmap",
|
||||||
ShortDescription: "List all known previously connected hosts",
|
ShortDescription: "List all known previously connected hosts",
|
||||||
@@ -297,45 +284,6 @@ func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interfa
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
ssh.RegisterCommand(&sshd.Command{
|
|
||||||
Name: "start-cpu-profile",
|
|
||||||
ShortDescription: "Starts a cpu profile and write output to the provided file, ex: `cpu-profile.pb.gz`",
|
|
||||||
Callback: func(fs any, a []string, w sshd.StringWriter) error {
|
|
||||||
return sshStartCpuProfile(sandboxDir, fs, a, w)
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
ssh.RegisterCommand(&sshd.Command{
|
|
||||||
Name: "stop-cpu-profile",
|
|
||||||
ShortDescription: "Stops a cpu profile and writes output to the previously provided file",
|
|
||||||
Callback: func(fs any, a []string, w sshd.StringWriter) error {
|
|
||||||
pprof.StopCPUProfile()
|
|
||||||
return w.WriteLine("If a CPU profile was running it is now stopped")
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
ssh.RegisterCommand(&sshd.Command{
|
|
||||||
Name: "save-heap-profile",
|
|
||||||
ShortDescription: "Saves a heap profile to the provided path, ex: `heap-profile.pb.gz`",
|
|
||||||
Callback: func(fs any, a []string, w sshd.StringWriter) error {
|
|
||||||
return sshGetHeapProfile(sandboxDir, fs, a, w)
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
ssh.RegisterCommand(&sshd.Command{
|
|
||||||
Name: "mutex-profile-fraction",
|
|
||||||
ShortDescription: "Gets or sets runtime.SetMutexProfileFraction",
|
|
||||||
Callback: sshMutexProfileFraction,
|
|
||||||
})
|
|
||||||
|
|
||||||
ssh.RegisterCommand(&sshd.Command{
|
|
||||||
Name: "save-mutex-profile",
|
|
||||||
ShortDescription: "Saves a mutex profile to the provided path, ex: `mutex-profile.pb.gz`",
|
|
||||||
Callback: func(fs any, a []string, w sshd.StringWriter) error {
|
|
||||||
return sshGetMutexProfile(sandboxDir, fs, a, w)
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
ssh.RegisterCommand(&sshd.Command{
|
ssh.RegisterCommand(&sshd.Command{
|
||||||
Name: "log-level",
|
Name: "log-level",
|
||||||
ShortDescription: "Gets or sets the current log level",
|
ShortDescription: "Gets or sets the current log level",
|
||||||
@@ -566,58 +514,6 @@ func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) er
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// sshSanitizeFilePath validates that the given file path is within the sandbox directory.
|
|
||||||
// If sandboxDir is empty, the path is returned as-is for backwards compatibility.
|
|
||||||
func sshSanitizeFilePath(sandboxDir, filePath string) (string, error) {
|
|
||||||
if sandboxDir == "" {
|
|
||||||
return filePath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clean and resolve the path relative to the sandbox directory
|
|
||||||
if !filepath.IsAbs(filePath) {
|
|
||||||
filePath = filepath.Join(sandboxDir, filePath)
|
|
||||||
}
|
|
||||||
cleaned := filepath.Clean(filePath)
|
|
||||||
|
|
||||||
// Ensure the resolved path is within the sandbox directory
|
|
||||||
cleanedSandbox := filepath.Clean(sandboxDir)
|
|
||||||
if cleaned == cleanedSandbox {
|
|
||||||
return "", fmt.Errorf("path %q resolves to the sandbox directory itself %q", filePath, sandboxDir)
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(cleaned, cleanedSandbox+string(filepath.Separator)) {
|
|
||||||
return "", fmt.Errorf("path %q is outside the sandbox directory %q", filePath, sandboxDir)
|
|
||||||
}
|
|
||||||
|
|
||||||
return cleaned, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func sshStartCpuProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
|
|
||||||
if len(a) == 0 {
|
|
||||||
err := w.WriteLine("No path to write profile provided")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
|
|
||||||
if err != nil {
|
|
||||||
return w.WriteLine(err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.Create(filePath)
|
|
||||||
if err != nil {
|
|
||||||
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = pprof.StartCPUProfile(file)
|
|
||||||
if err != nil {
|
|
||||||
err = w.WriteLine(fmt.Sprintf("Unable to start cpu profile: %s", err))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = w.WriteLine(fmt.Sprintf("Started cpu profile, issue stop-cpu-profile to write the output to %s", a))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
|
func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
|
||||||
return w.WriteLine(fmt.Sprintf("%s", ifce.version))
|
return w.WriteLine(fmt.Sprintf("%s", ifce.version))
|
||||||
}
|
}
|
||||||
@@ -770,76 +666,6 @@ func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) e
|
|||||||
return w.WriteLine("Changed")
|
return w.WriteLine("Changed")
|
||||||
}
|
}
|
||||||
|
|
||||||
func sshGetHeapProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
|
|
||||||
if len(a) == 0 {
|
|
||||||
return w.WriteLine("No path to write profile provided")
|
|
||||||
}
|
|
||||||
|
|
||||||
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
|
|
||||||
if err != nil {
|
|
||||||
return w.WriteLine(err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.Create(filePath)
|
|
||||||
if err != nil {
|
|
||||||
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = pprof.WriteHeapProfile(file)
|
|
||||||
if err != nil {
|
|
||||||
err = w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = w.WriteLine(fmt.Sprintf("Mem profile created at %s", a))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func sshMutexProfileFraction(fs any, a []string, w sshd.StringWriter) error {
|
|
||||||
if len(a) == 0 {
|
|
||||||
rate := runtime.SetMutexProfileFraction(-1)
|
|
||||||
return w.WriteLine(fmt.Sprintf("Current value: %d", rate))
|
|
||||||
}
|
|
||||||
|
|
||||||
newRate, err := strconv.Atoi(a[0])
|
|
||||||
if err != nil {
|
|
||||||
return w.WriteLine(fmt.Sprintf("Invalid argument: %s", a[0]))
|
|
||||||
}
|
|
||||||
|
|
||||||
oldRate := runtime.SetMutexProfileFraction(newRate)
|
|
||||||
return w.WriteLine(fmt.Sprintf("New value: %d. Old value: %d", newRate, oldRate))
|
|
||||||
}
|
|
||||||
|
|
||||||
func sshGetMutexProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
|
|
||||||
if len(a) == 0 {
|
|
||||||
return w.WriteLine("No path to write profile provided")
|
|
||||||
}
|
|
||||||
|
|
||||||
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
|
|
||||||
if err != nil {
|
|
||||||
return w.WriteLine(err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.Create(filePath)
|
|
||||||
if err != nil {
|
|
||||||
return w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
mutexProfile := pprof.Lookup("mutex")
|
|
||||||
if mutexProfile == nil {
|
|
||||||
return w.WriteLine("Unable to get pprof.Lookup(\"mutex\")")
|
|
||||||
}
|
|
||||||
|
|
||||||
err = mutexProfile.WriteTo(file, 0)
|
|
||||||
if err != nil {
|
|
||||||
return w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return w.WriteLine(fmt.Sprintf("Mutex profile created at %s", a))
|
|
||||||
}
|
|
||||||
|
|
||||||
func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
|
func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
|
||||||
ctrl, ok := l.Handler().(interface {
|
ctrl, ok := l.Handler().(interface {
|
||||||
GetLevel() slog.Level
|
GetLevel() slog.Level
|
||||||
|
|||||||
@@ -165,18 +165,6 @@ func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sandbox_dir is only read at startup, restarting sshd for it would drop every session and still not apply it
|
|
||||||
func TestSSHDReload_SandboxDirKeepsSessions(t *testing.T) {
|
|
||||||
h := newSSHDReloadHarness(t)
|
|
||||||
client, _ := h.connect(t)
|
|
||||||
|
|
||||||
h.extra = " sandbox_dir: " + t.TempDir() + "\n"
|
|
||||||
res := h.c.ReloadConfigString(h.config())
|
|
||||||
require.NoError(t, res.Err)
|
|
||||||
assert.Equal(t, []string{"sshd.sandbox_dir"}, res.RestartKeys)
|
|
||||||
assert.False(t, dropped(client, 500*time.Millisecond), "sshd was restarted")
|
|
||||||
}
|
|
||||||
|
|
||||||
// The config can stay the same while the host key file it names changes. New connections get the new key, open
|
// The config can stay the same while the host key file it names changes. New connections get the new key, open
|
||||||
// sessions are left alone
|
// sessions are left alone
|
||||||
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
|
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
|
||||||
|
|||||||
+1
-1
@@ -41,7 +41,7 @@ type Conn interface {
|
|||||||
//
|
//
|
||||||
// Returns the number of packets successfully written. A destination the kernel rejects costs only
|
// Returns the number of packets successfully written. A destination the kernel rejects costs only
|
||||||
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
|
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
|
||||||
// Not safe for concurrent use on the same Conn.
|
// Safe for concurrent use on the same Conn.
|
||||||
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
|
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
|
||||||
// ReloadConfig applies the listen settings that can change, r is nil at startup
|
// ReloadConfig applies the listen settings that can change, r is nil at startup
|
||||||
ReloadConfig(c *config.C, r *config.Reload)
|
ReloadConfig(c *config.C, r *config.Reload)
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"net/netip"
|
"net/netip"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
@@ -20,7 +21,7 @@ import (
|
|||||||
// batchWriter owns the sendmmsg(2)/UDP-GSO transmit path for a StdConn: the
|
// batchWriter owns the sendmmsg(2)/UDP-GSO transmit path for a StdConn: the
|
||||||
// scratch WriteBatch packs mmsghdr entries into, plus the GSO capability
|
// scratch WriteBatch packs mmsghdr entries into, plus the GSO capability
|
||||||
// state probed at socket creation. Each queue has its own StdConn and
|
// state probed at socket creation. Each queue has its own StdConn and
|
||||||
// batchWriter, so no locking is needed.
|
// batchWriter; mu serializes batches from the TUN reader and relay forwarding.
|
||||||
//
|
//
|
||||||
// Terminology, smallest to largest:
|
// Terminology, smallest to largest:
|
||||||
//
|
//
|
||||||
@@ -39,6 +40,7 @@ import (
|
|||||||
// chunk the entries packed for one sendmmsg call, at most MaxWriteBatch.
|
// chunk the entries packed for one sendmmsg call, at most MaxWriteBatch.
|
||||||
// batch the caller's whole bufs/addrs pair, processed as one or more chunks.
|
// batch the caller's whole bufs/addrs pair, processed as one or more chunks.
|
||||||
type batchWriter struct {
|
type batchWriter struct {
|
||||||
|
mu sync.Mutex
|
||||||
fd int
|
fd int
|
||||||
isV4 bool
|
isV4 bool
|
||||||
|
|
||||||
@@ -192,6 +194,9 @@ func parseRelease(r string) (major, minor int) {
|
|||||||
// Returns the number of packets sent. An error means the call itself failed.
|
// Returns the number of packets sent. An error means the call itself failed.
|
||||||
// A short count means some destinations were undeliverable.
|
// A short count means some destinations were undeliverable.
|
||||||
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
|
||||||
if len(bufs) != len(addrs) {
|
if len(bufs) != len(addrs) {
|
||||||
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
|
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
//go:build linux && !android && !e2e_testing
|
||||||
|
|
||||||
|
package udp
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/netip"
|
||||||
|
"runtime"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The TUN reader and relay forwarding share a socket. Exercise their concurrent
|
||||||
|
// batches against the actual GSO planner and scratch, without relying on a NIC.
|
||||||
|
func TestWriteBatchConcurrent(t *testing.T) {
|
||||||
|
const batches = 64
|
||||||
|
w := newRewindTestWriter()
|
||||||
|
sent := make(chan [][]byte, 2*batches)
|
||||||
|
w.sendFn = func(start, n int) (int, error) {
|
||||||
|
runtime.Gosched()
|
||||||
|
sent <- capturePrepared(w, start, n)
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
conn := &StdConn{bw: w}
|
||||||
|
remote := netip.MustParseAddrPort("192.0.2.1:4242")
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
start := make(chan struct{})
|
||||||
|
for producer := range 2 {
|
||||||
|
wg.Go(func() {
|
||||||
|
<-start
|
||||||
|
for i := range batches {
|
||||||
|
var bufs [][]byte
|
||||||
|
var addrs []netip.AddrPort
|
||||||
|
for j := range 4 {
|
||||||
|
bufs = append(bufs, []byte{byte(producer), byte(i), byte(j), 0})
|
||||||
|
addrs = append(addrs, remote)
|
||||||
|
}
|
||||||
|
n, err := conn.WriteBatch(bufs, addrs)
|
||||||
|
if err != nil || n != len(bufs) {
|
||||||
|
t.Errorf("WriteBatch = %d, %v; want %d, nil", n, err, len(bufs))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
close(start)
|
||||||
|
wg.Wait()
|
||||||
|
close(sent)
|
||||||
|
var counts [2]int
|
||||||
|
for packets := range sent {
|
||||||
|
require.Len(t, packets, 4)
|
||||||
|
producer := packets[0][0]
|
||||||
|
require.Less(t, int(producer), len(counts))
|
||||||
|
for j, packet := range packets {
|
||||||
|
require.Equal(t, []byte{producer, byte(counts[producer]), byte(j), 0}, packet)
|
||||||
|
}
|
||||||
|
counts[producer]++
|
||||||
|
}
|
||||||
|
require.Equal(t, [2]int{batches, batches}, counts)
|
||||||
|
}
|
||||||
Reference in new issue
Block a user