mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 08:46:52 +01:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a274c7b9bd | ||
|
|
9a5bdb508e | ||
|
|
48ba879198 | ||
|
|
fb2c153dfa | ||
|
|
f552cebc51 | ||
|
|
b04d71d96a | ||
|
|
cacfa8fac4 |
No files matched your search
@@ -86,6 +86,21 @@ func (c *C) InitialLoad() bool {
|
|||||||
return c.oldSettings == nil
|
return c.oldSettings == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Render returns keys and their values as yaml, for a subsystem to remember what it applied. Empty means they couldn't
|
||||||
|
// be rendered and should never match.
|
||||||
|
func (c *C) Render(keys ...string) string {
|
||||||
|
m := make(map[string]any, len(keys))
|
||||||
|
for _, k := range keys {
|
||||||
|
m[k] = c.Get(k)
|
||||||
|
}
|
||||||
|
b, err := yaml.Marshal(m)
|
||||||
|
if err != nil {
|
||||||
|
c.l.Debug("Failed to render config", "keys", keys, "error", err)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
|
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
|
||||||
// k in both the old and new settings will be serialized, the result of the string comparison is returned.
|
// k in both the old and new settings will be serialized, the result of the string comparison is returned.
|
||||||
// If k is an empty string the entire config is tested.
|
// If k is an empty string the entire config is tested.
|
||||||
|
|||||||
+6
-2
@@ -39,8 +39,9 @@ static_host_map:
|
|||||||
# public address. Other valid options are "ip6" and "ip" (returns both.)
|
# public address. Other valid options are "ip6" and "ip" (returns both.)
|
||||||
#network: ip4
|
#network: ip4
|
||||||
|
|
||||||
# lookup_timeout is the DNS query timeout.
|
# lookup_timeout is the DNS query timeout. A failed lookup keeps the last known good addresses, only a not found
|
||||||
#lookup_timeout: 250ms
|
# answer removes them.
|
||||||
|
#lookup_timeout: 5s
|
||||||
|
|
||||||
lighthouse:
|
lighthouse:
|
||||||
# am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
|
# am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
|
||||||
@@ -331,6 +332,9 @@ tun:
|
|||||||
# `mtu`: will default to tun mtu if this option is not specified
|
# `mtu`: will default to tun mtu if this option is not specified
|
||||||
# `metric`: will default to 0 if this option is not specified
|
# `metric`: will default to 0 if this option is not specified
|
||||||
# `install`: will default to true, controls whether this route is installed in the systems routing table.
|
# `install`: will default to true, controls whether this route is installed in the systems routing table.
|
||||||
|
# On macOS, a route whose destination another interface already routes, like the local network, is installed once
|
||||||
|
# that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another
|
||||||
|
# interface routes its destination and lets it go.
|
||||||
# This setting is reloadable.
|
# This setting is reloadable.
|
||||||
unsafe_routes:
|
unsafe_routes:
|
||||||
#- route: 172.16.1.0/24
|
#- route: 172.16.1.0/24
|
||||||
|
|||||||
@@ -66,6 +66,9 @@ type Firewall struct {
|
|||||||
rules string
|
rules string
|
||||||
rulesVersion uint16
|
rulesVersion uint16
|
||||||
|
|
||||||
|
// source is the rendered firewall config this firewall was built from, the cert networks are kept above
|
||||||
|
source string
|
||||||
|
|
||||||
defaultLocalCIDRAny bool
|
defaultLocalCIDRAny bool
|
||||||
incomingMetrics firewallMetrics
|
incomingMetrics firewallMetrics
|
||||||
outgoingMetrics firewallMetrics
|
outgoingMetrics firewallMetrics
|
||||||
@@ -246,6 +249,7 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fw.source = c.Render("firewall")
|
||||||
return fw, nil
|
return fw, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+35
-23
@@ -61,16 +61,17 @@ type HandshakeManager struct {
|
|||||||
vpnIps map[netip.Addr]*HandshakeHostInfo
|
vpnIps map[netip.Addr]*HandshakeHostInfo
|
||||||
indexes map[uint32]*HandshakeHostInfo
|
indexes map[uint32]*HandshakeHostInfo
|
||||||
|
|
||||||
mainHostMap *HostMap
|
mainHostMap *HostMap
|
||||||
lightHouse *LightHouse
|
lightHouse *LightHouse
|
||||||
outside udp.Conn
|
outside udp.Conn
|
||||||
config HandshakeConfig
|
config HandshakeConfig
|
||||||
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
|
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
|
||||||
messageMetrics *MessageMetrics
|
messageMetrics *MessageMetrics
|
||||||
metricInitiated metrics.Counter
|
metricInitiated metrics.Counter
|
||||||
metricTimedOut metrics.Counter
|
metricTimedOut metrics.Counter
|
||||||
f *Interface
|
metricRejectedBlocklisted metrics.Counter
|
||||||
l *slog.Logger
|
f *Interface
|
||||||
|
l *slog.Logger
|
||||||
|
|
||||||
// can be used to trigger outbound handshake for the given vpnIp
|
// can be used to trigger outbound handshake for the given vpnIp
|
||||||
trigger chan netip.Addr
|
trigger chan netip.Addr
|
||||||
@@ -118,18 +119,19 @@ func (hh *HandshakeHostInfo) cachePacket(l *slog.Logger, t header.MessageType, s
|
|||||||
|
|
||||||
func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager {
|
func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager {
|
||||||
return &HandshakeManager{
|
return &HandshakeManager{
|
||||||
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
|
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
|
||||||
indexes: map[uint32]*HandshakeHostInfo{},
|
indexes: map[uint32]*HandshakeHostInfo{},
|
||||||
mainHostMap: mainHostMap,
|
mainHostMap: mainHostMap,
|
||||||
lightHouse: lightHouse,
|
lightHouse: lightHouse,
|
||||||
outside: outside,
|
outside: outside,
|
||||||
config: config,
|
config: config,
|
||||||
trigger: make(chan netip.Addr, config.triggerBuffer),
|
trigger: make(chan netip.Addr, config.triggerBuffer),
|
||||||
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
|
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
|
||||||
messageMetrics: config.messageMetrics,
|
messageMetrics: config.messageMetrics,
|
||||||
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
|
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
|
||||||
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
|
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
|
||||||
l: l,
|
metricRejectedBlocklisted: metrics.GetOrRegisterCounter("handshake_manager.rejected.blocklisted", nil),
|
||||||
|
l: l,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -403,6 +405,8 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
|
|||||||
|
|
||||||
hm.Unlock()
|
hm.Unlock()
|
||||||
hm.lightHouse.QueryServer(vpnAddr)
|
hm.lightHouse.QueryServer(vpnAddr)
|
||||||
|
// If this host came from a static_map hostname its addresses may have moved, retries will pick up anything new
|
||||||
|
hostinfo.remotes.ResolveHostnames()
|
||||||
return hostinfo
|
return hostinfo
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -717,7 +721,12 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
|
|||||||
|
|
||||||
response, result, err := machine.ProcessPacket(nil, packet)
|
response, result, err := machine.ProcessPacket(nil, packet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.l.Error("Failed to process handshake packet", "from", via, "error", err)
|
level := slog.LevelError
|
||||||
|
if errors.Is(err, cert.ErrBlockListed) {
|
||||||
|
level = slog.LevelDebug
|
||||||
|
hm.metricRejectedBlocklisted.Inc(1)
|
||||||
|
}
|
||||||
|
f.l.Log(context.Background(), level, "Failed to process handshake packet", "from", via, "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -849,6 +858,9 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
|
|||||||
|
|
||||||
response, result, err := machine.ProcessPacket(nil, packet)
|
response, result, err := machine.ProcessPacket(nil, packet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, cert.ErrBlockListed) {
|
||||||
|
hm.metricRejectedBlocklisted.Inc(1)
|
||||||
|
}
|
||||||
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
|
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
|
||||||
if machine.Failed() {
|
if machine.Failed() {
|
||||||
f.l.Warn("Failed to process handshake packet, abandoning",
|
f.l.Warn("Failed to process handshake packet, abandoning",
|
||||||
|
|||||||
@@ -1,12 +1,17 @@
|
|||||||
package nebula
|
package nebula
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"log/slog"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/flynn/noise"
|
||||||
"github.com/gaissmai/bart"
|
"github.com/gaissmai/bart"
|
||||||
|
"github.com/rcrowley/go-metrics"
|
||||||
"github.com/slackhq/nebula/cert"
|
"github.com/slackhq/nebula/cert"
|
||||||
|
ct "github.com/slackhq/nebula/cert_test"
|
||||||
|
"github.com/slackhq/nebula/handshake"
|
||||||
"github.com/slackhq/nebula/header"
|
"github.com/slackhq/nebula/header"
|
||||||
"github.com/slackhq/nebula/test"
|
"github.com/slackhq/nebula/test"
|
||||||
"github.com/slackhq/nebula/udp"
|
"github.com/slackhq/nebula/udp"
|
||||||
@@ -235,6 +240,89 @@ func TestHandleIncomingDispatch(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBlocklistedHandshakeLogLevelAndCounter(t *testing.T) {
|
||||||
|
ca, _, caKey, _ := ct.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
|
||||||
|
pool := ct.NewTestCAPool(ca)
|
||||||
|
|
||||||
|
newCredential := func(name, network string) (cert.Certificate, *handshake.Credential) {
|
||||||
|
c, _, rawKey, _ := ct.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey,
|
||||||
|
name, ca.NotBefore(), ca.NotAfter(), []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
|
||||||
|
privateKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey)
|
||||||
|
require.NoError(t, err)
|
||||||
|
certBytes, err := c.MarshalForHandshakes()
|
||||||
|
require.NoError(t, err)
|
||||||
|
cipherSuite := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
|
||||||
|
return c, handshake.NewCredential(c, certBytes, privateKey, cipherSuite)
|
||||||
|
}
|
||||||
|
|
||||||
|
initiatorCert, initiatorCredential := newCredential("initiator", "10.0.0.1/24")
|
||||||
|
responderCert, responderCredential := newCredential("responder", "10.0.0.2/24")
|
||||||
|
fingerprint, err := initiatorCert.Fingerprint()
|
||||||
|
require.NoError(t, err)
|
||||||
|
pool.BlocklistFingerprint(fingerprint)
|
||||||
|
|
||||||
|
initiator, err := handshake.NewMachine(cert.Version2,
|
||||||
|
func(cert.Version) *handshake.Credential { return initiatorCredential },
|
||||||
|
func(c cert.Certificate) (*cert.CachedCertificate, error) {
|
||||||
|
return pool.VerifyCertificate(time.Now(), c)
|
||||||
|
},
|
||||||
|
func() (uint32, error) { return 1, nil }, true, header.HandshakeIXPSK0)
|
||||||
|
require.NoError(t, err)
|
||||||
|
packet, err := initiator.Initiate(nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
handler := &capturingHandler{}
|
||||||
|
logger := slog.New(handler)
|
||||||
|
pki := &PKI{}
|
||||||
|
pki.cs.Store(&CertState{initiatingVersion: cert.Version2, v2Credential: responderCredential})
|
||||||
|
pki.caPool.Store(pool)
|
||||||
|
hm := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
|
||||||
|
hm.metricRejectedBlocklisted = metrics.NewCounter()
|
||||||
|
hm.f = &Interface{pki: pki, l: logger}
|
||||||
|
via := ViaSender{UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242")}
|
||||||
|
|
||||||
|
hm.beginHandshake(via, packet, nil)
|
||||||
|
entry := findEntry(t, handler.entries, "Failed to process handshake packet")
|
||||||
|
assert.Equal(t, slog.LevelDebug, entry.Level)
|
||||||
|
require.ErrorIs(t, entry.Attrs["error"].(error), cert.ErrBlockListed)
|
||||||
|
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
|
||||||
|
|
||||||
|
handler.entries = nil
|
||||||
|
hm.beginHandshake(via, nil, nil)
|
||||||
|
entry = findEntry(t, handler.entries, "Failed to process handshake packet")
|
||||||
|
assert.Equal(t, slog.LevelError, entry.Level)
|
||||||
|
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
|
||||||
|
|
||||||
|
// The initiator sees the same rejection in continueHandshake, after a
|
||||||
|
// responder has accepted its first packet and sent a valid reply.
|
||||||
|
cleanPool := ct.NewTestCAPool(ca)
|
||||||
|
responder, err := handshake.NewMachine(cert.Version2,
|
||||||
|
func(cert.Version) *handshake.Credential { return responderCredential },
|
||||||
|
func(c cert.Certificate) (*cert.CachedCertificate, error) {
|
||||||
|
return cleanPool.VerifyCertificate(time.Now(), c)
|
||||||
|
},
|
||||||
|
func() (uint32, error) { return 2, nil }, false, header.HandshakeIXPSK0)
|
||||||
|
require.NoError(t, err)
|
||||||
|
response, _, err := responder.ProcessPacket(nil, packet)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
responderFingerprint, err := responderCert.Fingerprint()
|
||||||
|
require.NoError(t, err)
|
||||||
|
pool.BlocklistFingerprint(responderFingerprint)
|
||||||
|
|
||||||
|
initiatingHM := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
|
||||||
|
initiatingHM.metricRejectedBlocklisted = metrics.NewCounter()
|
||||||
|
initiatingHM.f = &Interface{l: logger}
|
||||||
|
hh := &HandshakeHostInfo{
|
||||||
|
machine: initiator,
|
||||||
|
hostinfo: &HostInfo{localIndexId: 1, vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}},
|
||||||
|
}
|
||||||
|
initiatingHM.indexes[1] = hh
|
||||||
|
via.IsRelayed = true // skip the remote allow list before certificate validation
|
||||||
|
initiatingHM.continueHandshake(via, hh, response)
|
||||||
|
assert.Equal(t, int64(1), initiatingHM.metricRejectedBlocklisted.Count())
|
||||||
|
}
|
||||||
|
|
||||||
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
|
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
|
||||||
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
|
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
|
||||||
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
|
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
|
||||||
|
|||||||
+64
-14
@@ -124,6 +124,8 @@ type Interface struct {
|
|||||||
// reused (every udp.Conn ListenOut guarantees that ordering).
|
// reused (every udp.Conn ListenOut guarantees that ordering).
|
||||||
batchers []*batch.MultiCoalescer
|
batchers []*batch.MultiCoalescer
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
|
// readers covers just the listenIn and listenOut goroutines, wg also holds the construction token Close releases
|
||||||
|
readers sync.WaitGroup
|
||||||
|
|
||||||
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
|
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
|
||||||
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
|
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
|
||||||
@@ -341,14 +343,18 @@ func (f *Interface) activate() error {
|
|||||||
func (f *Interface) run() {
|
func (f *Interface) run() {
|
||||||
// Launch n queues to read packets from udp
|
// Launch n queues to read packets from udp
|
||||||
for i := 0; i < f.routines; i++ {
|
for i := 0; i < f.routines; i++ {
|
||||||
|
f.readers.Add(1)
|
||||||
f.wg.Go(func() {
|
f.wg.Go(func() {
|
||||||
|
defer f.readers.Done()
|
||||||
f.listenOut(i)
|
f.listenOut(i)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Launch n queues to read packets from tun dev
|
// Launch n queues to read packets from tun dev
|
||||||
for i := 0; i < f.routines; i++ {
|
for i := 0; i < f.routines; i++ {
|
||||||
|
f.readers.Add(1)
|
||||||
f.wg.Go(func() {
|
f.wg.Go(func() {
|
||||||
|
defer f.readers.Done()
|
||||||
f.listenIn(f.queues[i], i)
|
f.listenIn(f.queues[i], i)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -537,11 +543,12 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
|
|||||||
curCert = cs.getCertificate(cert.Version1)
|
curCert = cs.getCertificate(cert.Version1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction.
|
// Compare with what the running firewall was built from, not the last config, so a firewall that failed to build
|
||||||
// Check to see if that set has changed, and if so, rebuild the firewall.
|
// is tried again. The cert is the one this reload brought, PKI registers its reload callback before this one
|
||||||
certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks)
|
src := c.Render("firewall")
|
||||||
|
if src != "" && src == f.firewall.source && curCert != nil &&
|
||||||
if !c.HasChanged("firewall") && !certUnsafeChanged {
|
slices.Equal(curCert.Networks(), f.firewall.assignedNetworks) &&
|
||||||
|
slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) {
|
||||||
f.l.Debug("No firewall config change detected")
|
f.l.Debug("No firewall config change detected")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -682,6 +689,14 @@ func (f *Interface) GetCertState() *CertState {
|
|||||||
return f.pki.getCertState()
|
return f.pki.getCertState()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// shutdowner is a udp.Conn or overlay.Device whose readers can be woken without releasing their fds
|
||||||
|
type shutdowner interface {
|
||||||
|
Shutdown() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// readerShutdownTimeout bounds how long Close waits on woken readers before releasing their fds anyway
|
||||||
|
const readerShutdownTimeout = 5 * time.Second
|
||||||
|
|
||||||
// Close releases the interface's resources: the udp sockets and the tun device.
|
// Close releases the interface's resources: the udp sockets and the tun device.
|
||||||
// It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated,
|
// It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated,
|
||||||
// calls after the first return nil without doing anything.
|
// calls after the first return nil without doing anything.
|
||||||
@@ -691,20 +706,55 @@ func (f *Interface) Close() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var errs []error
|
var errs []error
|
||||||
|
closeUdp := func(i int, u udp.Conn) {
|
||||||
// Release the udp readers
|
if err := u.Close(); err != nil {
|
||||||
for i, u := range f.writers {
|
|
||||||
err := u.Close()
|
|
||||||
if err != nil {
|
|
||||||
f.l.Error("Error while closing udp socket", "error", err, "writer", i)
|
f.l.Error("Error while closing udp socket", "error", err, "writer", i)
|
||||||
errs = append(errs, err)
|
errs = append(errs, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
closeTun := func() {
|
||||||
|
if err := f.inside.Close(); err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Release the tun device (closing the tun also closes all readers)
|
// Wake every reader before releasing any fd, one that loops back into a syscall on a reused fd number never wakes.
|
||||||
closeErr := f.inside.Close()
|
// Anything without Shutdown is closed right away, as before.
|
||||||
if closeErr != nil {
|
var deferred []func()
|
||||||
errs = append(errs, closeErr)
|
for i, u := range f.writers {
|
||||||
|
if s, ok := u.(shutdowner); ok {
|
||||||
|
if err := s.Shutdown(); err != nil {
|
||||||
|
f.l.Error("Error while shutting down udp socket", "error", err, "writer", i)
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
deferred = append(deferred, func() { closeUdp(i, u) })
|
||||||
|
} else {
|
||||||
|
closeUdp(i, u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if s, ok := f.inside.(shutdowner); ok {
|
||||||
|
if err := s.Shutdown(); err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
deferred = append(deferred, closeTun)
|
||||||
|
} else {
|
||||||
|
closeTun()
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(deferred) > 0 {
|
||||||
|
readersDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
f.readers.Wait()
|
||||||
|
close(readersDone)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-readersDone:
|
||||||
|
case <-time.After(readerShutdownTimeout):
|
||||||
|
f.l.Warn("Readers did not stop, closing their fds anyway", "timeout", readerShutdownTimeout)
|
||||||
|
}
|
||||||
|
for _, closeFn := range deferred {
|
||||||
|
closeFn()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Release the construction token so waiters know the resources are gone
|
// Release the construction token so waiters know the resources are gone
|
||||||
|
|||||||
+58
-3
@@ -135,8 +135,63 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
|
|||||||
f := &Interface{pki: pki, firewall: fw, l: l}
|
f := &Interface{pki: pki, firewall: fw, l: l}
|
||||||
cfg.RegisterReloadCallback(f.reloadFirewall)
|
cfg.RegisterReloadCallback(f.reloadFirewall)
|
||||||
|
|
||||||
res := cfg.ReloadConfigString(rules("nope"))
|
// The running firewall still isn't built from what the config asks for, so the same reload fails again
|
||||||
|
for range 2 {
|
||||||
|
res := cfg.ReloadConfigString(rules("nope"))
|
||||||
|
require.NoError(t, res.Err)
|
||||||
|
assert.True(t, res.Failed)
|
||||||
|
assert.Same(t, fw, f.firewall)
|
||||||
|
}
|
||||||
|
|
||||||
|
res := cfg.ReloadConfigString(rules("tcp"))
|
||||||
require.NoError(t, res.Err)
|
require.NoError(t, res.Err)
|
||||||
assert.True(t, res.Failed)
|
assert.False(t, res.Failed)
|
||||||
assert.Same(t, fw, f.firewall)
|
assert.NotSame(t, fw, f.firewall)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every input the firewall is built from rebuilds it when it changes, and nothing else does
|
||||||
|
func TestReloadFirewall_RebuildsOnAnyInput(t *testing.T) {
|
||||||
|
l := test.NewLogger()
|
||||||
|
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
|
||||||
|
base := "firewall:\n inbound:\n - port: any\n proto: any\n host: any\n"
|
||||||
|
setup := func() (*config.C, *Interface, *PKI) {
|
||||||
|
pki := &PKI{}
|
||||||
|
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks}, initiatingVersion: cert.Version2})
|
||||||
|
cfg := config.NewC(l)
|
||||||
|
require.NoError(t, cfg.LoadString(base))
|
||||||
|
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
|
||||||
|
require.NoError(t, err)
|
||||||
|
f := &Interface{pki: pki, firewall: fw, l: l}
|
||||||
|
cfg.RegisterReloadCallback(f.reloadFirewall)
|
||||||
|
return cfg, f, pki
|
||||||
|
}
|
||||||
|
setCert := func(pki *PKI, networks, unsafe []netip.Prefix) {
|
||||||
|
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks, unsafeNetworks: unsafe}, initiatingVersion: cert.Version2})
|
||||||
|
}
|
||||||
|
|
||||||
|
cases := map[string]struct {
|
||||||
|
cfg string
|
||||||
|
cert func(*PKI)
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
"nothing": {base + "punchy:\n punch: true\n", nil, false},
|
||||||
|
"rules": {base + " outbound:\n - port: any\n proto: any\n host: any\n", nil, true},
|
||||||
|
"inbound_action": {base + " inbound_action: reject\n", nil, true},
|
||||||
|
"conntrack": {base + " conntrack:\n tcp_timeout: 1m\n", nil, true},
|
||||||
|
"networks": {base, func(p *PKI) { setCert(p, []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil) }, true},
|
||||||
|
"unsafe_networks": {base, func(p *PKI) { setCert(p, networks, []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}) }, true},
|
||||||
|
}
|
||||||
|
for name, tc := range cases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
cfg, f, pki := setup()
|
||||||
|
old := f.firewall
|
||||||
|
if tc.cert != nil {
|
||||||
|
tc.cert(pki)
|
||||||
|
}
|
||||||
|
res := cfg.ReloadConfigString(tc.cfg)
|
||||||
|
require.NoError(t, res.Err)
|
||||||
|
assert.False(t, res.Failed)
|
||||||
|
assert.Equal(t, tc.want, f.firewall != old)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
+5
-3
@@ -452,7 +452,7 @@ func getStaticMapCadence(c *config.C) (time.Duration, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func getStaticMapLookupTimeout(c *config.C) (time.Duration, error) {
|
func getStaticMapLookupTimeout(c *config.C) (time.Duration, error) {
|
||||||
lookupTimeout := c.GetString("static_map.lookup_timeout", "250ms")
|
lookupTimeout := c.GetString("static_map.lookup_timeout", "5s")
|
||||||
d, err := time.ParseDuration(lookupTimeout)
|
d, err := time.ParseDuration(lookupTimeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
@@ -630,7 +630,8 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
|
|||||||
ctx := lh.ctx
|
ctx := lh.ctx
|
||||||
lh.Unlock()
|
lh.Unlock()
|
||||||
|
|
||||||
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, func() {
|
// On reload am.hr is the previous resolver, its last known good addresses carry over in case DNS is down right now
|
||||||
|
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, am.hr, func() {
|
||||||
// This callback runs whenever the DNS hostname resolver finds a different set of addr's
|
// This callback runs whenever the DNS hostname resolver finds a different set of addr's
|
||||||
// in its resolution for hostnames.
|
// in its resolution for hostnames.
|
||||||
am.Lock()
|
am.Lock()
|
||||||
@@ -642,7 +643,8 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
|
|||||||
}
|
}
|
||||||
am.unlockedSetHostnamesResults(hr)
|
am.unlockedSetHostnamesResults(hr)
|
||||||
|
|
||||||
for _, addrPort := range hr.GetAddrs() {
|
// Only literals belong in the cache, anything from DNS has to stay in hr where a not found answer can remove it
|
||||||
|
for _, addrPort := range hr.literalAddrs() {
|
||||||
if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) {
|
if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ type offloadQueueSet struct {
|
|||||||
// usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel.
|
// usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel.
|
||||||
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
|
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
|
||||||
usoEnabled bool
|
usoEnabled bool
|
||||||
|
shutdown atomic.Bool
|
||||||
closed atomic.Bool
|
closed atomic.Bool
|
||||||
// l is handed to each queue for its bad-vnet-header drop logging.
|
// l is handed to each queue for its bad-vnet-header drop logging.
|
||||||
l *slog.Logger
|
l *slog.Logger
|
||||||
@@ -69,6 +70,17 @@ func (c *offloadQueueSet) wakeForShutdown() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
|
||||||
|
func (c *offloadQueueSet) Shutdown() error {
|
||||||
|
if c.shutdown.Swap(true) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, x := range c.pq {
|
||||||
|
x.shutdown.Store(true)
|
||||||
|
}
|
||||||
|
return c.wakeForShutdown()
|
||||||
|
}
|
||||||
|
|
||||||
func (c *offloadQueueSet) Close() error {
|
func (c *offloadQueueSet) Close() error {
|
||||||
if c.closed.Swap(true) {
|
if c.closed.Swap(true) {
|
||||||
return nil
|
return nil
|
||||||
@@ -76,9 +88,7 @@ func (c *offloadQueueSet) Close() error {
|
|||||||
|
|
||||||
errs := []error{}
|
errs := []error{}
|
||||||
|
|
||||||
// Signal all readers blocked in poll to wake up and exit.
|
if err := c.Shutdown(); err != nil {
|
||||||
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
|
|
||||||
if err := c.wakeForShutdown(); err != nil {
|
|
||||||
errs = append(errs, err)
|
errs = append(errs, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ type pollQueueSet struct {
|
|||||||
// pqi is exactly the same as pq, but stored as the interface type
|
// pqi is exactly the same as pq, but stored as the interface type
|
||||||
pqi []Queue
|
pqi []Queue
|
||||||
shutdownFd int
|
shutdownFd int
|
||||||
|
shutdown atomic.Bool
|
||||||
closed atomic.Bool
|
closed atomic.Bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -59,6 +60,17 @@ func (c *pollQueueSet) wakeForShutdown() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
|
||||||
|
func (c *pollQueueSet) Shutdown() error {
|
||||||
|
if c.shutdown.Swap(true) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, x := range c.pq {
|
||||||
|
x.shutdown.Store(true)
|
||||||
|
}
|
||||||
|
return c.wakeForShutdown()
|
||||||
|
}
|
||||||
|
|
||||||
func (c *pollQueueSet) Close() error {
|
func (c *pollQueueSet) Close() error {
|
||||||
if c.closed.Swap(true) {
|
if c.closed.Swap(true) {
|
||||||
return nil
|
return nil
|
||||||
@@ -66,9 +78,7 @@ func (c *pollQueueSet) Close() error {
|
|||||||
|
|
||||||
errs := []error{}
|
errs := []error{}
|
||||||
|
|
||||||
// Signal all readers blocked in poll to wake up and exit.
|
if err := c.Shutdown(); err != nil {
|
||||||
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
|
|
||||||
if err := c.wakeForShutdown(); err != nil {
|
|
||||||
errs = append(errs, err)
|
errs = append(errs, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import (
|
|||||||
// QueueSet holds one or many Queue objects and helps close them in an orderly way.
|
// QueueSet holds one or many Queue objects and helps close them in an orderly way.
|
||||||
type QueueSet interface {
|
type QueueSet interface {
|
||||||
io.Closer
|
io.Closer
|
||||||
|
// Shutdown wakes every reader without releasing any fds, Close only once they have exited
|
||||||
|
Shutdown() error
|
||||||
Queues() []Queue
|
Queues() []Queue
|
||||||
|
|
||||||
// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
|
// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
|
||||||
|
|||||||
@@ -66,7 +66,9 @@ type Offload struct {
|
|||||||
// usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD,
|
// usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD,
|
||||||
// so writers can decide whether emitting GSO_UDP_L4 superpackets is safe.
|
// so writers can decide whether emitting GSO_UDP_L4 superpackets is safe.
|
||||||
usoEnabled bool
|
usoEnabled bool
|
||||||
closed atomic.Bool
|
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
|
||||||
|
shutdown atomic.Bool
|
||||||
|
closed atomic.Bool
|
||||||
|
|
||||||
// gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted
|
// gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted
|
||||||
// by WriteGSO. Kept separate from the read-only package-level validVnetHdr
|
// by WriteGSO. Kept separate from the read-only package-level validVnetHdr
|
||||||
@@ -176,6 +178,9 @@ func (r *Offload) readPacket(block bool) (int, error) {
|
|||||||
// This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs).
|
// This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs).
|
||||||
// Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue.
|
// Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue.
|
||||||
func (r *Offload) Read() ([]Packet, error) {
|
func (r *Offload) Read() ([]Packet, error) {
|
||||||
|
if r.shutdown.Load() {
|
||||||
|
return nil, os.ErrClosed
|
||||||
|
}
|
||||||
r.pending = r.pending[:0]
|
r.pending = r.pending[:0]
|
||||||
r.rxOff = 0
|
r.rxOff = 0
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,9 @@ import (
|
|||||||
type Poll struct {
|
type Poll struct {
|
||||||
fd int
|
fd int
|
||||||
shutdownFd int
|
shutdownFd int
|
||||||
closed atomic.Bool
|
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
|
||||||
|
shutdown atomic.Bool
|
||||||
|
closed atomic.Bool
|
||||||
|
|
||||||
readBuf []byte
|
readBuf []byte
|
||||||
batchRet [1]Packet
|
batchRet [1]Packet
|
||||||
@@ -52,6 +54,9 @@ func (t *Poll) blockOnWrite() error {
|
|||||||
// config currently lose that batching. blockOn and the EAGAIN plumbing are
|
// config currently lose that batching. blockOn and the EAGAIN plumbing are
|
||||||
// already shared; kept one-packet-per-Read for now to preserve behavior.
|
// already shared; kept one-packet-per-Read for now to preserve behavior.
|
||||||
func (t *Poll) Read() ([]Packet, error) {
|
func (t *Poll) Read() ([]Packet, error) {
|
||||||
|
if t.shutdown.Load() {
|
||||||
|
return nil, os.ErrClosed
|
||||||
|
}
|
||||||
n, err := t.readOne(t.readBuf)
|
n, err := t.readOne(t.readBuf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -223,3 +223,30 @@ func TestOffloadQueueSet_Close_ClosesShutdownFd(t *testing.T) {
|
|||||||
// Second Close must not touch fds (shutdownFd is now -1) and must return nil.
|
// Second Close must not touch fds (shutdownFd is now -1) and must return nil.
|
||||||
require.NoError(t, qs.Close())
|
require.NoError(t, qs.Close())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A busy reader never polls the shutdown eventfd, so Read has to stop on its own with data still waiting
|
||||||
|
func TestQueueSet_ShutdownStopsReaderWithDataWaiting(t *testing.T) {
|
||||||
|
sets := map[string]func() (QueueSet, error){
|
||||||
|
"poll": NewPollQueueSet,
|
||||||
|
"offload": func() (QueueSet, error) { return NewOffloadQueueSet(false, slog.New(slog.DiscardHandler)) },
|
||||||
|
}
|
||||||
|
for name, newSet := range sets {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
var fds [2]int
|
||||||
|
require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
|
||||||
|
t.Cleanup(func() { _ = unix.Close(fds[1]) })
|
||||||
|
|
||||||
|
qs, err := newSet()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, qs.Add(fds[0]))
|
||||||
|
t.Cleanup(func() { _ = qs.Close() })
|
||||||
|
|
||||||
|
_, err = unix.Write(fds[1], make([]byte, 64))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, qs.Shutdown())
|
||||||
|
|
||||||
|
_, err = qs.Queues()[0].Read()
|
||||||
|
require.ErrorIs(t, err, os.ErrClosed)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+54
-84
@@ -3,7 +3,6 @@
|
|||||||
package overlay
|
package overlay
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"math/bits"
|
"math/bits"
|
||||||
@@ -12,6 +11,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
"time"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
"github.com/gaissmai/bart"
|
"github.com/gaissmai/bart"
|
||||||
@@ -35,6 +35,26 @@ type tun struct {
|
|||||||
// and routes for it. NEPacketTunnelProvider on darwin does this.
|
// and routes for it. NEPacketTunnelProvider on darwin does this.
|
||||||
hostOwned bool
|
hostOwned bool
|
||||||
l *slog.Logger
|
l *slog.Logger
|
||||||
|
|
||||||
|
// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
|
||||||
|
// fields below it
|
||||||
|
routesMu sync.Mutex
|
||||||
|
// routeStates is who held each installable unsafe_route prefix when we last looked
|
||||||
|
routeStates map[netip.Prefix]routeState
|
||||||
|
// routeWatch is the routing socket watchRoutes reads, Close closes it
|
||||||
|
routeWatch *os.File
|
||||||
|
// routeSweep reconciles the unsafe_routes every routeSweepEvery
|
||||||
|
routeSweep *time.Timer
|
||||||
|
routeSweepEvery time.Duration
|
||||||
|
closed bool
|
||||||
|
|
||||||
|
// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
|
||||||
|
// lock so the route watcher never waits on a reconcile
|
||||||
|
routeMoversMu sync.Mutex
|
||||||
|
routeMovers map[netip.Prefix]routeMover
|
||||||
|
|
||||||
|
// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
|
||||||
|
lastRouteEvent atomic.Int64
|
||||||
}
|
}
|
||||||
|
|
||||||
type ifReq struct {
|
type ifReq struct {
|
||||||
@@ -139,11 +159,12 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
|
|||||||
}
|
}
|
||||||
|
|
||||||
t := &tun{
|
t := &tun{
|
||||||
f: os.NewFile(uintptr(fd), ""),
|
f: os.NewFile(uintptr(fd), ""),
|
||||||
Device: name,
|
Device: name,
|
||||||
vpnNetworks: vpnNetworks,
|
vpnNetworks: vpnNetworks,
|
||||||
DefaultMTU: getMTU(l, c),
|
DefaultMTU: getMTU(l, c),
|
||||||
l: l,
|
routeSweepEvery: routeSweepInterval,
|
||||||
|
l: l,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = t.reload(c, true, nil)
|
err = t.reload(c, true, nil)
|
||||||
@@ -219,6 +240,16 @@ func utunNameFromFd(fd int) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) Close() error {
|
func (t *tun) Close() error {
|
||||||
|
t.routesMu.Lock()
|
||||||
|
t.closed = true
|
||||||
|
if t.routeWatch != nil {
|
||||||
|
_ = t.routeWatch.Close()
|
||||||
|
}
|
||||||
|
if t.routeSweep != nil {
|
||||||
|
t.routeSweep.Stop()
|
||||||
|
}
|
||||||
|
t.routesMu.Unlock()
|
||||||
|
|
||||||
if t.f != nil {
|
if t.f != nil {
|
||||||
return t.f.Close()
|
return t.f.Close()
|
||||||
}
|
}
|
||||||
@@ -287,8 +318,22 @@ func (t *tun) Activate() error {
|
|||||||
return fmt.Errorf("failed to run tun device: %s", err)
|
return fmt.Errorf("failed to run tun device: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Watch before adding so a route removed in between is still noticed
|
||||||
|
watch, err := openRouteSocket()
|
||||||
|
if err != nil {
|
||||||
|
t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Unsafe path routes
|
// Unsafe path routes
|
||||||
return t.addRoutes(false, nil)
|
if err = t.addRoutes(false, nil); err != nil {
|
||||||
|
if watch != nil {
|
||||||
|
_ = watch.Close()
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
t.startRouteWatch(watch)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
|
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
|
||||||
@@ -393,7 +438,8 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !initial && !change {
|
if !initial && !change {
|
||||||
return nil
|
// Any reload puts back the unsafe_routes someone deleted
|
||||||
|
return t.addRoutes(true, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
routeTree, err := makeRouteTree(t.l, routes, false)
|
routeTree, err := makeRouteTree(t.l, routes, false)
|
||||||
@@ -511,82 +557,6 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
|
|||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
|
|
||||||
// The route tree is still ours, the system routing table is not
|
|
||||||
if t.hostOwned {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Activate hasn't run yet, it will install the routes
|
|
||||||
linkAddr := t.linkAddr.Load()
|
|
||||||
if linkAddr == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
routes := *t.Routes.Load()
|
|
||||||
systemRoutes := sync.OnceValues(fetchRoutes)
|
|
||||||
|
|
||||||
for _, r := range routes {
|
|
||||||
if len(r.Via) == 0 || !r.Install {
|
|
||||||
// We don't allow route MTUs so only install routes with a via
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err := addRoute(r.Cidr, linkAddr)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, unix.EEXIST) {
|
|
||||||
// Every route we added already exists on a reload, only warn about someone else's
|
|
||||||
msgs, err := systemRoutes()
|
|
||||||
if err != nil {
|
|
||||||
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
|
|
||||||
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
|
|
||||||
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
|
||||||
if logErrors {
|
|
||||||
retErr.Log(t.l)
|
|
||||||
rl.Failed()
|
|
||||||
} else {
|
|
||||||
return retErr
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
t.l.Info("Added route", "route", r)
|
|
||||||
// A cidr listed twice would otherwise find our own route missing from an older fetch
|
|
||||||
systemRoutes = sync.OnceValues(fetchRoutes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
|
|
||||||
linkAddr := t.linkAddr.Load()
|
|
||||||
if t.hostOwned || linkAddr == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, r := range routes {
|
|
||||||
if len(r.Via) == 0 || !r.Install {
|
|
||||||
// addRoutes never installs these
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err := delRoute(r.Cidr, linkAddr)
|
|
||||||
if errors.Is(err, unix.ESRCH) {
|
|
||||||
// Already gone, which is what we wanted
|
|
||||||
t.l.Info("Route was already removed", "route", r)
|
|
||||||
} else if err != nil {
|
|
||||||
t.l.Error("Failed to remove route", "error", err, "route", r)
|
|
||||||
rl.Failed()
|
|
||||||
} else {
|
|
||||||
t.l.Info("Removed route", "route", r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
|
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
|
||||||
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -3,12 +3,17 @@
|
|||||||
package overlay
|
package overlay
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
"github.com/slackhq/nebula/routing"
|
"github.com/slackhq/nebula/routing"
|
||||||
@@ -104,6 +109,367 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
|
|||||||
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
|
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
|
||||||
|
const (
|
||||||
|
// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
|
||||||
|
kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
|
||||||
|
addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
|
||||||
|
// With no interface address, which x/net/route skips
|
||||||
|
interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRoutesMayHaveMoved(t *testing.T) {
|
||||||
|
cidr := netip.MustParsePrefix("192.168.1.0/24")
|
||||||
|
via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
|
||||||
|
routes := []Route{{Cidr: cidr, Via: via, Install: true}}
|
||||||
|
// A request like route(8) sends, the kernel echoes it to every routing socket
|
||||||
|
request := func(typ, flags int, dst, mask [4]byte) []byte {
|
||||||
|
m := &netroute.RouteMessage{
|
||||||
|
Version: unix.RTM_VERSION,
|
||||||
|
Type: typ,
|
||||||
|
Flags: flags,
|
||||||
|
ID: 1234,
|
||||||
|
Addrs: []netroute.Addr{
|
||||||
|
unix.RTAX_DST: &netroute.Inet4Addr{IP: dst},
|
||||||
|
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
|
||||||
|
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
b, err := m.Marshal()
|
||||||
|
require.NoError(t, err)
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
raw := func(s string) []byte {
|
||||||
|
b, err := hex.DecodeString(s)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
|
||||||
|
routesMayHaveMoved := func(b []byte, routes []Route) bool {
|
||||||
|
moved, _, _ := routesMayHaveMoved(b, routes)
|
||||||
|
return moved
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
|
||||||
|
assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
|
||||||
|
assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
|
||||||
|
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
|
||||||
|
|
||||||
|
failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
|
||||||
|
binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
|
||||||
|
assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
|
||||||
|
|
||||||
|
_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
|
||||||
|
require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
|
||||||
|
assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
|
||||||
|
neighbor := raw(kernelSubnetDelete)
|
||||||
|
binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
|
||||||
|
assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
|
||||||
|
|
||||||
|
assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
|
||||||
|
assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A request names its prefix and who made it, the kernel's own and interface changes name neither
|
||||||
|
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
|
||||||
|
cidr := netip.MustParsePrefix("192.168.1.0/24")
|
||||||
|
routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
|
||||||
|
m := &netroute.RouteMessage{
|
||||||
|
Version: unix.RTM_VERSION,
|
||||||
|
Type: unix.RTM_CHANGE,
|
||||||
|
Flags: unix.RTF_UP,
|
||||||
|
ID: 1234,
|
||||||
|
Addrs: []netroute.Addr{
|
||||||
|
unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
|
||||||
|
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
|
||||||
|
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
b, err := m.Marshal()
|
||||||
|
require.NoError(t, err)
|
||||||
|
moved, p, pid := routesMayHaveMoved(b, routes)
|
||||||
|
assert.True(t, moved)
|
||||||
|
assert.Equal(t, cidr, p)
|
||||||
|
assert.Equal(t, int32(1234), pid)
|
||||||
|
|
||||||
|
for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
|
||||||
|
b, err := hex.DecodeString(s)
|
||||||
|
require.NoError(t, err)
|
||||||
|
moved, p, pid := routesMayHaveMoved(b, routes)
|
||||||
|
assert.True(t, moved, name)
|
||||||
|
assert.False(t, p.IsValid(), name)
|
||||||
|
assert.Zero(t, pid, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessName(t *testing.T) {
|
||||||
|
assert.Equal(t, "launchd", processName(1))
|
||||||
|
self := filepath.Base(os.Args[0])
|
||||||
|
assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
|
||||||
|
assert.Empty(t, processName(-1))
|
||||||
|
}
|
||||||
|
|
||||||
|
// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
|
||||||
|
// the minute between sweeps when it isn't zero.
|
||||||
|
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
|
||||||
|
requireRoot(t)
|
||||||
|
|
||||||
|
cfg := "tun:\n unsafe_routes:\n"
|
||||||
|
for _, cidr := range cidrs {
|
||||||
|
via := "10.250.0.2"
|
||||||
|
if netip.MustParsePrefix(cidr).Addr().Is6() {
|
||||||
|
via = "fd00:250::2"
|
||||||
|
}
|
||||||
|
cfg += " - route: " + cidr + "\n via: " + via + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
l := test.NewLogger()
|
||||||
|
c := config.NewC(l)
|
||||||
|
require.NoError(t, c.LoadString(cfg))
|
||||||
|
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
if sweep != 0 {
|
||||||
|
tn.routeSweepEvery = sweep
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = tn.Close() })
|
||||||
|
require.NoError(t, tn.Activate())
|
||||||
|
return tn, c, cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireRoot(t *testing.T) {
|
||||||
|
if os.Geteuid() != 0 {
|
||||||
|
t.Skip("creating interfaces and adding routes needs root")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// routeVia is the index of the interface that routes cidr, 0 if none does
|
||||||
|
func routeVia(t *testing.T, cidr string) int {
|
||||||
|
msgs, err := fetchRoutes()
|
||||||
|
require.NoError(t, err)
|
||||||
|
return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
|
||||||
|
}
|
||||||
|
|
||||||
|
// routeStateOf is what tn made of cidr when it last looked
|
||||||
|
func routeStateOf(tn *tun, cidr string) routeState {
|
||||||
|
tn.routesMu.Lock()
|
||||||
|
defer tn.routesMu.Unlock()
|
||||||
|
return tn.routeStates[netip.MustParsePrefix(cidr)]
|
||||||
|
}
|
||||||
|
|
||||||
|
func routeArgs(action, cidr string, extra ...string) []string {
|
||||||
|
args := []string{"-q", "-n", action}
|
||||||
|
if netip.MustParsePrefix(cidr).Addr().Is6() {
|
||||||
|
args = append(args, "-inet6")
|
||||||
|
}
|
||||||
|
return append(append(args, "-net", cidr), extra...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func routeCmd(t *testing.T, action, cidr string, extra ...string) {
|
||||||
|
requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
|
||||||
|
}
|
||||||
|
|
||||||
|
// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
|
||||||
|
// says so
|
||||||
|
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
require.NoError(t, err, string(out))
|
||||||
|
require.NotContains(t, string(out), "writing to routing socket")
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanupRoute deletes cidr's route when the test ends, whoever holds it
|
||||||
|
func cleanupRoute(t *testing.T, cidr string) {
|
||||||
|
t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
|
||||||
|
}
|
||||||
|
|
||||||
|
// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
|
||||||
|
func routeThroughLo0(t *testing.T, cidr string) int {
|
||||||
|
requireRoot(t)
|
||||||
|
routeCmd(t, "add", cidr, "-interface", "lo0")
|
||||||
|
cleanupRoute(t, cidr)
|
||||||
|
lo0, err := net.InterfaceByName("lo0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
return lo0.Index
|
||||||
|
}
|
||||||
|
|
||||||
|
// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
|
||||||
|
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
|
||||||
|
for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
|
||||||
|
t.Run(cidr, func(t *testing.T) {
|
||||||
|
lo0 := routeThroughLo0(t, cidr)
|
||||||
|
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||||
|
require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
|
||||||
|
|
||||||
|
routeCmd(t, "delete", cidr)
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
|
||||||
|
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
|
||||||
|
cidr := "10.251.1.0/24"
|
||||||
|
lo0 := routeThroughLo0(t, cidr)
|
||||||
|
_, c, _ := newRouteTestTun(t, 0, cidr)
|
||||||
|
|
||||||
|
res := c.ReloadConfigString("tun: {}\n")
|
||||||
|
require.NoError(t, res.Err)
|
||||||
|
assert.False(t, res.Failed)
|
||||||
|
assert.Equal(t, lo0, routeVia(t, cidr))
|
||||||
|
}
|
||||||
|
|
||||||
|
// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
|
||||||
|
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
|
||||||
|
cidr := "10.251.3.0/24"
|
||||||
|
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||||
|
ours := tn.linkAddr.Load().Index
|
||||||
|
require.Equal(t, ours, routeVia(t, cidr))
|
||||||
|
|
||||||
|
cleanupRoute(t, cidr)
|
||||||
|
requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
|
||||||
|
lo0, err := net.InterfaceByName("lo0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
time.Sleep(2 * routeRetryDelay)
|
||||||
|
require.Equal(t, lo0.Index, routeVia(t, cidr))
|
||||||
|
|
||||||
|
routeCmd(t, "delete", cidr)
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
|
||||||
|
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
|
||||||
|
cidr := "10.251.4.0/24"
|
||||||
|
tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
|
||||||
|
ours := tn.linkAddr.Load().Index
|
||||||
|
require.Equal(t, ours, routeVia(t, cidr))
|
||||||
|
|
||||||
|
routeCmd(t, "delete", cidr)
|
||||||
|
assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
|
||||||
|
|
||||||
|
// A reload with nothing changed is how to ask for it back
|
||||||
|
res := c.ReloadConfigString(cfg)
|
||||||
|
require.NoError(t, res.Err)
|
||||||
|
assert.False(t, res.Failed)
|
||||||
|
assert.Equal(t, ours, routeVia(t, cidr))
|
||||||
|
}
|
||||||
|
|
||||||
|
// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
|
||||||
|
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
|
||||||
|
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
|
||||||
|
for name, tc := range map[string]struct {
|
||||||
|
cidr string
|
||||||
|
sweep time.Duration
|
||||||
|
misfile func(t *testing.T, tn *tun, cidr string)
|
||||||
|
}{
|
||||||
|
// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
|
||||||
|
"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
|
||||||
|
require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
}},
|
||||||
|
// The sweep that caps a table that never settles
|
||||||
|
"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
|
||||||
|
tn.reconcileRoutes()
|
||||||
|
require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
|
||||||
|
ours := tn.linkAddr.Load().Index
|
||||||
|
require.Equal(t, ours, routeVia(t, tc.cidr))
|
||||||
|
|
||||||
|
routeCmd(t, "delete", tc.cidr)
|
||||||
|
tc.misfile(t, tn, tc.cidr)
|
||||||
|
|
||||||
|
lo0 := routeThroughLo0(t, tc.cidr)
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
s := routeStateOf(tn, tc.cidr)
|
||||||
|
return s.kind == routeYielded && s.holder == lo0
|
||||||
|
}, 5*time.Second, 50*time.Millisecond)
|
||||||
|
|
||||||
|
routeCmd(t, "delete", tc.cidr)
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
|
||||||
|
// taken for a hand delete
|
||||||
|
func TestTunYieldsToARouteChangedAway(t *testing.T) {
|
||||||
|
for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
|
||||||
|
t.Run(cidr, func(t *testing.T) {
|
||||||
|
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||||
|
ours := tn.linkAddr.Load().Index
|
||||||
|
require.Equal(t, ours, routeVia(t, cidr))
|
||||||
|
|
||||||
|
cleanupRoute(t, cidr)
|
||||||
|
routeCmd(t, "change", cidr, "-interface", "lo0")
|
||||||
|
lo0, err := net.InterfaceByName("lo0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, lo0.Index, routeVia(t, cidr))
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
s := routeStateOf(tn, cidr)
|
||||||
|
return s.kind == routeYielded && s.holder == lo0.Index
|
||||||
|
}, 5*time.Second, 50*time.Millisecond)
|
||||||
|
|
||||||
|
routeCmd(t, "delete", cidr)
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
|
||||||
|
// ethernet configd routed the lan through
|
||||||
|
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
|
||||||
|
requireRoot(t)
|
||||||
|
var feth string
|
||||||
|
for i := 100; i < 200 && feth == ""; i++ {
|
||||||
|
if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
|
||||||
|
feth = fmt.Sprintf("feth%d", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
require.NotEmpty(t, feth)
|
||||||
|
require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
|
||||||
|
t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
|
||||||
|
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
|
||||||
|
|
||||||
|
// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
|
||||||
|
// refuses one named by the feth itself
|
||||||
|
cidr := "10.251.2.0/24"
|
||||||
|
routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
|
||||||
|
cleanupRoute(t, cidr)
|
||||||
|
ifc, err := net.InterfaceByName(feth)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||||
|
require.Equal(t, ifc.Index, routeVia(t, cidr))
|
||||||
|
|
||||||
|
// Losing the address first leaves the route, the reconcile finds it still held
|
||||||
|
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
|
||||||
|
time.Sleep(2 * routeRetryDelay)
|
||||||
|
require.Equal(t, ifc.Index, routeVia(t, cidr))
|
||||||
|
|
||||||
|
// Only RTM_IFINFO says the interface went, its route goes with no message of its own
|
||||||
|
require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
|
||||||
|
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
|
||||||
|
cidr := "10.251.5.0/24"
|
||||||
|
routeThroughLo0(t, cidr)
|
||||||
|
tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
|
||||||
|
|
||||||
|
tn.routesMu.Lock()
|
||||||
|
require.NoError(t, tn.routeWatch.Close())
|
||||||
|
tn.routesMu.Unlock()
|
||||||
|
|
||||||
|
routeCmd(t, "delete", cidr)
|
||||||
|
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
|
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
|
||||||
func TestTunDevNeedsRestart(t *testing.T) {
|
func TestTunDevNeedsRestart(t *testing.T) {
|
||||||
for name, want := range map[string]bool{
|
for name, want := range map[string]bool{
|
||||||
|
|||||||
@@ -832,6 +832,11 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
|
|||||||
t.routeTree.Store(newTree)
|
t.routeTree.Store(newTree)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Shutdown wakes the tun readers without releasing their fds, Close releases them
|
||||||
|
func (t *tun) Shutdown() error {
|
||||||
|
return t.readers.Shutdown()
|
||||||
|
}
|
||||||
|
|
||||||
func (t *tun) Close() error {
|
func (t *tun) Close() error {
|
||||||
t.closeLock.Lock()
|
t.closeLock.Lock()
|
||||||
defer t.closeLock.Unlock()
|
defer t.closeLock.Unlock()
|
||||||
|
|||||||
@@ -0,0 +1,451 @@
|
|||||||
|
//go:build !ios && !e2e_testing
|
||||||
|
|
||||||
|
package overlay
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/netip"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
"unsafe"
|
||||||
|
|
||||||
|
"github.com/slackhq/nebula/config"
|
||||||
|
"github.com/slackhq/nebula/util"
|
||||||
|
netroute "golang.org/x/net/route"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
|
||||||
|
// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
|
||||||
|
// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
|
||||||
|
// or until another interface takes the prefix, after which ours follows theirs like any other.
|
||||||
|
|
||||||
|
const (
|
||||||
|
// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
|
||||||
|
// replaces a route by deleting and adding, looking in between would find the prefix free
|
||||||
|
routeRetryDelay = time.Second
|
||||||
|
|
||||||
|
// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
|
||||||
|
// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
|
||||||
|
// a message for each
|
||||||
|
routeSweepInterval = time.Minute
|
||||||
|
|
||||||
|
// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
|
||||||
|
routeSocketRcvBuf = 1 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
type routeStateKind int
|
||||||
|
|
||||||
|
const (
|
||||||
|
routeUnknown routeStateKind = iota
|
||||||
|
// routeInstalled is our route holding the prefix
|
||||||
|
routeInstalled
|
||||||
|
// routeYielded is another interface holding the prefix, ours is added once it lets go
|
||||||
|
routeYielded
|
||||||
|
// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
|
||||||
|
// another interface takes the prefix
|
||||||
|
routeRemoved
|
||||||
|
// routeFailed is adding our route failing for some other reason, it is retried
|
||||||
|
routeFailed
|
||||||
|
)
|
||||||
|
|
||||||
|
type routeState struct {
|
||||||
|
kind routeStateKind
|
||||||
|
// holder is the index of the interface holding the prefix when kind is routeYielded
|
||||||
|
holder int
|
||||||
|
}
|
||||||
|
|
||||||
|
// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
|
||||||
|
// is looked up as the message arrives, route(8) has exited by the time the table settles
|
||||||
|
type routeMover struct {
|
||||||
|
pid int32
|
||||||
|
name string
|
||||||
|
}
|
||||||
|
|
||||||
|
// logArgs appends m to a log line's args, when we know who it was
|
||||||
|
func (m routeMover) logArgs(args ...any) []any {
|
||||||
|
if m.pid == 0 {
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
args = append(args, "pid", m.pid)
|
||||||
|
if m.name != "" {
|
||||||
|
args = append(args, "process", m.name)
|
||||||
|
}
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
|
||||||
|
// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
|
||||||
|
func processName(pid int32) string {
|
||||||
|
kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
|
||||||
|
if err != nil || kp.Proc.P_pid != pid {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return unix.ByteSliceToString(kp.Proc.P_comm[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// installRoutes returns the routes we put in the system routing table, one per prefix
|
||||||
|
func installRoutes(routes []Route) []Route {
|
||||||
|
seen := make(map[netip.Prefix]struct{}, len(routes))
|
||||||
|
var out []Route
|
||||||
|
for _, r := range routes {
|
||||||
|
if len(r.Via) == 0 || !r.Install {
|
||||||
|
// We don't allow route MTUs so only install routes with a via
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
p := r.Cidr.Masked()
|
||||||
|
if _, ok := seen[p]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[p] = struct{}{}
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
|
||||||
|
// interface holds its prefix. It runs on start and on every reload.
|
||||||
|
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
|
||||||
|
// The route tree is still ours, the system routing table is not
|
||||||
|
if t.hostOwned {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routesMu.Lock()
|
||||||
|
defer t.routesMu.Unlock()
|
||||||
|
|
||||||
|
// Activate hasn't run yet, it will install the routes
|
||||||
|
linkAddr := t.linkAddr.Load()
|
||||||
|
if linkAddr == nil || t.closed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var msgs []netroute.Message
|
||||||
|
var msgsErr error
|
||||||
|
fetched := false
|
||||||
|
states := make(map[netip.Prefix]routeState)
|
||||||
|
for _, r := range installRoutes(*t.Routes.Load()) {
|
||||||
|
p := r.Cidr.Masked()
|
||||||
|
prev := t.routeStates[p]
|
||||||
|
|
||||||
|
err := addRoute(p, linkAddr)
|
||||||
|
if err == nil {
|
||||||
|
t.l.Info("Added route", "route", r)
|
||||||
|
states[p] = routeState{kind: routeInstalled}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !errors.Is(err, unix.EEXIST) {
|
||||||
|
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
||||||
|
if !logErrors {
|
||||||
|
return retErr
|
||||||
|
}
|
||||||
|
retErr.Log(t.l)
|
||||||
|
rl.Failed()
|
||||||
|
states[p] = routeState{kind: routeFailed}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every route we added already exists on a reload, only warn about someone else's
|
||||||
|
if !fetched {
|
||||||
|
msgs, msgsErr = fetchRoutes()
|
||||||
|
fetched = true
|
||||||
|
}
|
||||||
|
if msgsErr != nil {
|
||||||
|
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
|
||||||
|
states[p] = prev
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch holder := routeInterfaceIn(msgs, p); holder {
|
||||||
|
case linkAddr.Index:
|
||||||
|
states[p] = routeState{kind: routeInstalled}
|
||||||
|
case 0:
|
||||||
|
// Removed since the add, the watcher will see to it
|
||||||
|
states[p] = prev
|
||||||
|
default:
|
||||||
|
if prev.kind != routeYielded || prev.holder != holder {
|
||||||
|
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
|
||||||
|
}
|
||||||
|
states[p] = routeState{kind: routeYielded, holder: holder}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routeStates = states
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
|
||||||
|
if t.hostOwned {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routesMu.Lock()
|
||||||
|
defer t.routesMu.Unlock()
|
||||||
|
|
||||||
|
linkAddr := t.linkAddr.Load()
|
||||||
|
if linkAddr == nil || t.closed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
|
||||||
|
msgs, err := fetchRoutes()
|
||||||
|
if err != nil {
|
||||||
|
var left []netip.Prefix
|
||||||
|
for _, r := range installRoutes(routes) {
|
||||||
|
left = append(left, r.Cidr)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, r := range installRoutes(routes) {
|
||||||
|
p := r.Cidr.Masked()
|
||||||
|
// Whatever happens next, the prefix is no longer one we know about
|
||||||
|
delete(t.routeStates, p)
|
||||||
|
if routeInterfaceIn(msgs, p) != linkAddr.Index {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
err = delRoute(p, linkAddr)
|
||||||
|
if errors.Is(err, unix.ESRCH) {
|
||||||
|
// Already gone, which is what we wanted
|
||||||
|
t.l.Info("Route was already removed", "route", r)
|
||||||
|
} else if err != nil {
|
||||||
|
t.l.Error("Failed to remove route", "error", err, "route", r)
|
||||||
|
rl.Failed()
|
||||||
|
} else {
|
||||||
|
t.l.Info("Removed route", "route", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
|
||||||
|
// changed since it last looked, it runs every routeSweepEvery.
|
||||||
|
func (t *tun) reconcileRoutes() {
|
||||||
|
t.routesMu.Lock()
|
||||||
|
defer t.routesMu.Unlock()
|
||||||
|
|
||||||
|
linkAddr := t.linkAddr.Load()
|
||||||
|
if linkAddr == nil || t.closed {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routeMoversMu.Lock()
|
||||||
|
movers := t.routeMovers
|
||||||
|
t.routeMovers = nil
|
||||||
|
t.routeMoversMu.Unlock()
|
||||||
|
|
||||||
|
msgs, err := fetchRoutes()
|
||||||
|
if err != nil {
|
||||||
|
t.l.Error("Failed to read the system routing table", "error", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
states := make(map[netip.Prefix]routeState)
|
||||||
|
for _, r := range installRoutes(*t.Routes.Load()) {
|
||||||
|
p := r.Cidr.Masked()
|
||||||
|
prev := t.routeStates[p]
|
||||||
|
by := movers[p]
|
||||||
|
|
||||||
|
holder := routeInterfaceIn(msgs, p)
|
||||||
|
switch {
|
||||||
|
case holder == linkAddr.Index:
|
||||||
|
states[p] = routeState{kind: routeInstalled}
|
||||||
|
|
||||||
|
case holder != 0:
|
||||||
|
// Another interface taking a prefix we left alone puts ours back in line behind theirs
|
||||||
|
if prev.kind != routeYielded || prev.holder != holder {
|
||||||
|
t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
|
||||||
|
}
|
||||||
|
states[p] = routeState{kind: routeYielded, holder: holder}
|
||||||
|
|
||||||
|
case prev.kind == routeInstalled:
|
||||||
|
t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
|
||||||
|
states[p] = routeState{kind: routeRemoved}
|
||||||
|
|
||||||
|
case prev.kind == routeRemoved:
|
||||||
|
states[p] = prev
|
||||||
|
|
||||||
|
default:
|
||||||
|
err := addRoute(p, linkAddr)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
if prev.kind == routeYielded {
|
||||||
|
t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
|
||||||
|
} else {
|
||||||
|
t.l.Info("Added route", "route", r)
|
||||||
|
}
|
||||||
|
states[p] = routeState{kind: routeInstalled}
|
||||||
|
case errors.Is(err, unix.EEXIST):
|
||||||
|
// Taken since the fetch, the next look will see by whom
|
||||||
|
states[p] = prev
|
||||||
|
default:
|
||||||
|
if prev.kind != routeFailed {
|
||||||
|
util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
|
||||||
|
}
|
||||||
|
states[p] = routeState{kind: routeFailed}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.routeStates = states
|
||||||
|
}
|
||||||
|
|
||||||
|
// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
|
||||||
|
// routing table settles after anything that may have moved a prefix
|
||||||
|
func (t *tun) startRouteWatch(watch *os.File) {
|
||||||
|
t.routesMu.Lock()
|
||||||
|
defer t.routesMu.Unlock()
|
||||||
|
if t.closed {
|
||||||
|
if watch != nil {
|
||||||
|
_ = watch.Close()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
|
||||||
|
if watch != nil {
|
||||||
|
t.routeWatch = watch
|
||||||
|
go t.watchRoutes(watch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tun) sweepRoutes() {
|
||||||
|
// A table still changing gets reconciled once it settles
|
||||||
|
if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
|
||||||
|
t.reconcileRoutes()
|
||||||
|
}
|
||||||
|
|
||||||
|
t.routesMu.Lock()
|
||||||
|
defer t.routesMu.Unlock()
|
||||||
|
if !t.closed {
|
||||||
|
t.routeSweep.Reset(t.routeSweepEvery)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func openRouteSocket() (*os.File, error) {
|
||||||
|
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
|
||||||
|
}
|
||||||
|
// Fewer drops, the sweep catches the rest
|
||||||
|
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
|
||||||
|
// Non-blocking puts it on the poller, so Close interrupts a Read
|
||||||
|
if err = unix.SetNonblock(fd, true); err != nil {
|
||||||
|
_ = unix.Close(fd)
|
||||||
|
return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
|
||||||
|
}
|
||||||
|
return os.NewFile(uintptr(fd), "route"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
|
||||||
|
// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
|
||||||
|
func (t *tun) watchRoutes(f *os.File) {
|
||||||
|
var reconciled atomic.Int64
|
||||||
|
settle := time.AfterFunc(time.Hour, func() {
|
||||||
|
reconciled.Store(time.Now().UnixNano())
|
||||||
|
t.reconcileRoutes()
|
||||||
|
})
|
||||||
|
settle.Stop()
|
||||||
|
defer settle.Stop()
|
||||||
|
|
||||||
|
var burst time.Time
|
||||||
|
self := int32(os.Getpid())
|
||||||
|
b := make([]byte, os.Getpagesize())
|
||||||
|
for {
|
||||||
|
n, err := f.Read(b)
|
||||||
|
if err != nil {
|
||||||
|
if !errors.Is(err, os.ErrClosed) {
|
||||||
|
t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
|
||||||
|
if !moved {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Our own are already in routeStates
|
||||||
|
if p.IsValid() && pid != self {
|
||||||
|
t.routeMoversMu.Lock()
|
||||||
|
if t.routeMovers == nil {
|
||||||
|
t.routeMovers = make(map[netip.Prefix]routeMover)
|
||||||
|
}
|
||||||
|
t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
|
||||||
|
t.routeMoversMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
t.lastRouteEvent.Store(now.UnixNano())
|
||||||
|
if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
|
||||||
|
burst = now
|
||||||
|
}
|
||||||
|
settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
|
||||||
|
// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
|
||||||
|
// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
|
||||||
|
// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
|
||||||
|
// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
|
||||||
|
// and the pid of the process that made it.
|
||||||
|
func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
|
||||||
|
if len(b) < 4 {
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
switch int(b[3]) {
|
||||||
|
case unix.RTM_IFINFO, unix.RTM_DELADDR:
|
||||||
|
return true, p, 0
|
||||||
|
case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
|
||||||
|
default:
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(b) < unix.SizeofRtMsghdr {
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
|
||||||
|
// A failed request is reported too, and a scoped route never held the prefix
|
||||||
|
if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
if hdr.Pid == 0 {
|
||||||
|
// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
|
||||||
|
// could be holding a prefix
|
||||||
|
return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Replies to a request carry the requester's padded sockaddrs
|
||||||
|
msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
|
||||||
|
if err != nil || len(msgs) != 1 {
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
m, ok := msgs[0].(*netroute.RouteMessage)
|
||||||
|
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
|
||||||
|
return false, p, 0
|
||||||
|
}
|
||||||
|
p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
|
||||||
|
if !ok {
|
||||||
|
return false, netip.Prefix{}, 0
|
||||||
|
}
|
||||||
|
p = p.Masked()
|
||||||
|
for _, r := range installRoutes(routes) {
|
||||||
|
if r.Cidr.Masked() == p {
|
||||||
|
return true, p, hdr.Pid
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, netip.Prefix{}, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// interfaceName is the name of the interface with index i, for logs
|
||||||
|
func interfaceName(i int) string {
|
||||||
|
if ifc, err := net.InterfaceByIndex(i); err == nil {
|
||||||
|
return ifc.Name
|
||||||
|
}
|
||||||
|
return strconv.Itoa(i)
|
||||||
|
}
|
||||||
@@ -528,6 +528,10 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
|
|||||||
if !rm.GetAmRelay() {
|
if !rm.GetAmRelay() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if !slices.Contains(h.vpnAddrs, from) {
|
||||||
|
logMsg.Info("Discarding relay request where relayFrom is not the sending peer")
|
||||||
|
return
|
||||||
|
}
|
||||||
peer := rm.hostmap.QueryVpnAddr(target)
|
peer := rm.hostmap.QueryVpnAddr(target)
|
||||||
if peer == nil {
|
if peer == nil {
|
||||||
// Try to establish a connection to this host. If we get a future relay request,
|
// Try to establish a connection to this host. If we get a future relay request,
|
||||||
|
|||||||
@@ -7,8 +7,10 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/gaissmai/bart"
|
"github.com/gaissmai/bart"
|
||||||
|
"github.com/slackhq/nebula/cert"
|
||||||
"github.com/slackhq/nebula/test"
|
"github.com/slackhq/nebula/test"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestStartRelaysLogDedupe verifies that repeated attempts with the same relay set drop the log
|
// TestStartRelaysLogDedupe verifies that repeated attempts with the same relay set drop the log
|
||||||
@@ -95,3 +97,58 @@ func TestStartRelaysLogDedupe(t *testing.T) {
|
|||||||
assert.NotContains(t, buf.String(), msg, "should not log when we shortcut out")
|
assert.NotContains(t, buf.String(), msg, "should not log when we shortcut out")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHandleCreateRelayRequestRejectsSpoofedFrom verifies that a relay only forwards a CreateRelayRequest
|
||||||
|
// when relayFrom is the peer that sent it. Otherwise any peer could reset another host's established
|
||||||
|
// relay back to Requested, or allocate unbounded relay entries on the target's hostinfo.
|
||||||
|
func TestHandleCreateRelayRequestRejectsSpoofedFrom(t *testing.T) {
|
||||||
|
l := test.NewLogger()
|
||||||
|
relayAddr := netip.MustParseAddr("10.128.0.128")
|
||||||
|
malloryAddr := netip.MustParseAddr("10.128.0.3")
|
||||||
|
victimAddr := netip.MustParseAddr("10.128.0.1")
|
||||||
|
targetAddr := netip.MustParseAddr("10.128.0.2")
|
||||||
|
|
||||||
|
addrTable := new(bart.Lite)
|
||||||
|
addrTable.Insert(netip.PrefixFrom(relayAddr, relayAddr.BitLen()))
|
||||||
|
hm := newHostMap(l)
|
||||||
|
f := &Interface{myVpnAddrsTable: addrTable, hostMap: hm}
|
||||||
|
rm := &relayManager{l: l, hostmap: hm}
|
||||||
|
rm.amRelay.Store(true)
|
||||||
|
|
||||||
|
newRelayState := func() RelayState {
|
||||||
|
return RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}}
|
||||||
|
}
|
||||||
|
mallory := &HostInfo{vpnAddrs: []netip.Addr{malloryAddr}, localIndexId: 1, relayState: newRelayState()}
|
||||||
|
target := &HostInfo{
|
||||||
|
vpnAddrs: []netip.Addr{targetAddr},
|
||||||
|
localIndexId: 2,
|
||||||
|
remotes: NewRemoteList([]netip.Addr{targetAddr}, nil),
|
||||||
|
relayState: newRelayState(),
|
||||||
|
}
|
||||||
|
target.SetRemote(netip.MustParseAddrPort("192.0.2.2:4242"))
|
||||||
|
hm.unlockedAddHostInfo(mallory, f)
|
||||||
|
hm.unlockedAddHostInfo(target, f)
|
||||||
|
|
||||||
|
// The victim already has a working relay to the target through us.
|
||||||
|
_, err := AddRelay(l, target, hm, victimAddr, nil, ForwardingType, Established)
|
||||||
|
require.NoError(t, err)
|
||||||
|
relaysBefore := len(hm.Relays)
|
||||||
|
|
||||||
|
for i := range 10 {
|
||||||
|
spoofed := netip.AddrFrom4([4]byte{10, 128, 1, byte(i)})
|
||||||
|
for _, from := range []netip.Addr{victimAddr, spoofed} {
|
||||||
|
rm.handleCreateRelayRequest(cert.Version2, mallory, f, &NebulaControl{
|
||||||
|
Type: NebulaControl_CreateRelayRequest,
|
||||||
|
InitiatorRelayIndex: uint32(100 + i),
|
||||||
|
RelayFromAddr: netAddrToProtoAddr(from),
|
||||||
|
RelayToAddr: netAddrToProtoAddr(targetAddr),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
r, ok := target.relayState.QueryRelayForByIp(victimAddr)
|
||||||
|
require.True(t, ok)
|
||||||
|
assert.Equal(t, Established, r.State, "victim's relay must not be reset by a request from another peer")
|
||||||
|
assert.Len(t, hm.Relays, relaysBefore, "spoofed requests must not allocate relay entries")
|
||||||
|
assert.Empty(t, mallory.relayState.CopyAllRelayFor(), "spoofed requests must not create state on the sender")
|
||||||
|
}
|
||||||
+124
-19
@@ -2,6 +2,7 @@ package nebula
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
@@ -58,6 +59,8 @@ type cacheV6 struct {
|
|||||||
type hostnamePort struct {
|
type hostnamePort struct {
|
||||||
name string
|
name string
|
||||||
port uint16
|
port uint16
|
||||||
|
// addr is set when name is an ip literal, these never need a lookup
|
||||||
|
addr netip.Addr
|
||||||
}
|
}
|
||||||
|
|
||||||
type hostnamesResults struct {
|
type hostnamesResults struct {
|
||||||
@@ -65,15 +68,20 @@ type hostnamesResults struct {
|
|||||||
network string
|
network string
|
||||||
lookupTimeout time.Duration
|
lookupTimeout time.Duration
|
||||||
cancelFn func()
|
cancelFn func()
|
||||||
|
resolveNow chan struct{}
|
||||||
l *slog.Logger
|
l *slog.Logger
|
||||||
ips atomic.Pointer[map[netip.AddrPort]struct{}]
|
ips atomic.Pointer[map[netip.AddrPort]struct{}]
|
||||||
|
// known is the last known good addresses per hostname, kept so a reload can carry them forward
|
||||||
|
known atomic.Pointer[map[hostnamePort][]netip.AddrPort]
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, onUpdate func()) (*hostnamesResults, error) {
|
// NewHostnameResults starts resolving hostPorts. Any hostnames also in prev start with its last known good addresses.
|
||||||
|
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, prev *hostnamesResults, onUpdate func()) (*hostnamesResults, error) {
|
||||||
r := &hostnamesResults{
|
r := &hostnamesResults{
|
||||||
hostnames: make([]hostnamePort, len(hostPorts)),
|
hostnames: make([]hostnamePort, len(hostPorts)),
|
||||||
network: network,
|
network: network,
|
||||||
lookupTimeout: timeout,
|
lookupTimeout: timeout,
|
||||||
|
resolveNow: make(chan struct{}, 1),
|
||||||
l: l,
|
l: l,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -81,6 +89,13 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
|
|||||||
// DNS lookups for hostnames that aren't hardcoded IP's will happen in a background goroutine.
|
// DNS lookups for hostnames that aren't hardcoded IP's will happen in a background goroutine.
|
||||||
performBackgroundLookup := false
|
performBackgroundLookup := false
|
||||||
ips := map[netip.AddrPort]struct{}{}
|
ips := map[netip.AddrPort]struct{}{}
|
||||||
|
known := make([][]netip.AddrPort, len(hostPorts))
|
||||||
|
var prevKnown map[hostnamePort][]netip.AddrPort
|
||||||
|
if prev != nil && prev.network == network {
|
||||||
|
if p := prev.known.Load(); p != nil {
|
||||||
|
prevKnown = *p
|
||||||
|
}
|
||||||
|
}
|
||||||
for idx, hostPort := range hostPorts {
|
for idx, hostPort := range hostPorts {
|
||||||
|
|
||||||
rIp, sPort, err := net.SplitHostPort(hostPort)
|
rIp, sPort, err := net.SplitHostPort(hostPort)
|
||||||
@@ -98,13 +113,19 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
// This address is a hostname, not an IP address
|
// This address is a hostname, not an IP address
|
||||||
performBackgroundLookup = true
|
performBackgroundLookup = true
|
||||||
|
known[idx] = prevKnown[r.hostnames[idx]]
|
||||||
|
for _, a := range known[idx] {
|
||||||
|
ips[a] = struct{}{}
|
||||||
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save the IP address immediately
|
// Save the IP address immediately
|
||||||
|
r.hostnames[idx].addr = addr
|
||||||
ips[netip.AddrPortFrom(addr, uint16(iPort))] = struct{}{}
|
ips[netip.AddrPortFrom(addr, uint16(iPort))] = struct{}{}
|
||||||
}
|
}
|
||||||
r.ips.Store(&ips)
|
r.ips.Store(&ips)
|
||||||
|
r.storeKnown(known)
|
||||||
|
|
||||||
// Time for the DNS lookup goroutine
|
// Time for the DNS lookup goroutine
|
||||||
if performBackgroundLookup {
|
if performBackgroundLookup {
|
||||||
@@ -114,21 +135,12 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
|
|||||||
go func() {
|
go func() {
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
for {
|
for {
|
||||||
|
r.resolve(newCtx, known)
|
||||||
|
r.storeKnown(known)
|
||||||
netipAddrs := map[netip.AddrPort]struct{}{}
|
netipAddrs := map[netip.AddrPort]struct{}{}
|
||||||
for _, hostPort := range r.hostnames {
|
for _, addrs := range known {
|
||||||
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, r.lookupTimeout)
|
|
||||||
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, r.network, hostPort.name)
|
|
||||||
timeoutCancel()
|
|
||||||
if err != nil {
|
|
||||||
l.Error("DNS resolution failed for static_map host",
|
|
||||||
"hostname", hostPort.name,
|
|
||||||
"network", r.network,
|
|
||||||
"error", err,
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, a := range addrs {
|
for _, a := range addrs {
|
||||||
netipAddrs[netip.AddrPortFrom(a.Unmap(), hostPort.port)] = struct{}{}
|
netipAddrs[a] = struct{}{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
origSet := r.ips.Load()
|
origSet := r.ips.Load()
|
||||||
@@ -159,7 +171,7 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
|
|||||||
case <-newCtx.Done():
|
case <-newCtx.Done():
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
continue
|
case <-r.resolveNow:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -168,6 +180,84 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
|
|||||||
return r, nil
|
return r, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// resolve looks up every hostname in parallel and folds the results into known
|
||||||
|
func (hr *hostnamesResults) resolve(ctx context.Context, known [][]netip.AddrPort) {
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i, hostPort := range hr.hostnames {
|
||||||
|
if hostPort.addr.IsValid() {
|
||||||
|
known[i] = []netip.AddrPort{netip.AddrPortFrom(hostPort.addr, hostPort.port)}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Go(func() {
|
||||||
|
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, hr.lookupTimeout)
|
||||||
|
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, hr.network, hostPort.name)
|
||||||
|
timeoutCancel()
|
||||||
|
known[i] = applyLookup(known[i], hostPort.port, addrs, err)
|
||||||
|
// A canceled ctx means we are reloading or shutting down, not that DNS failed
|
||||||
|
if err != nil && ctx.Err() == nil {
|
||||||
|
hr.l.Error("DNS resolution failed for static_map host",
|
||||||
|
"hostname", hostPort.name,
|
||||||
|
"network", hr.network,
|
||||||
|
"error", err,
|
||||||
|
"usingAddrs", known[i],
|
||||||
|
)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyLookup returns the addresses to use for a hostname given the previous ones and a lookup result.
|
||||||
|
// Only a not found answer clears them, a stale lighthouse address costs a handshake but an empty set costs everything.
|
||||||
|
// darwin also reports not found while offline, that's harmless since nothing is reachable and the next handshake
|
||||||
|
// triggers a fresh lookup once we're back.
|
||||||
|
func applyLookup(prev []netip.AddrPort, port uint16, addrs []netip.Addr, err error) []netip.AddrPort {
|
||||||
|
if err != nil {
|
||||||
|
var dnsErr *net.DNSError
|
||||||
|
if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return prev
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]netip.AddrPort, 0, len(addrs))
|
||||||
|
for _, a := range addrs {
|
||||||
|
out = append(out, netip.AddrPortFrom(a.Unmap(), port))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func (hr *hostnamesResults) storeKnown(known [][]netip.AddrPort) {
|
||||||
|
byName := make(map[hostnamePort][]netip.AddrPort, len(known))
|
||||||
|
for i, addrs := range known {
|
||||||
|
byName[hr.hostnames[i]] = addrs
|
||||||
|
}
|
||||||
|
hr.known.Store(&byName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// literalAddrs returns the entries that were configured as ip addresses rather than hostnames
|
||||||
|
func (hr *hostnamesResults) literalAddrs() []netip.AddrPort {
|
||||||
|
var out []netip.AddrPort
|
||||||
|
for _, hostPort := range hr.hostnames {
|
||||||
|
if hostPort.addr.IsValid() {
|
||||||
|
out = append(out, netip.AddrPortFrom(hostPort.addr, hostPort.port))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveNow asks the background goroutine to look up the hostnames again without waiting for the next tick
|
||||||
|
func (hr *hostnamesResults) ResolveNow() {
|
||||||
|
if hr == nil || hr.resolveNow == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case hr.resolveNow <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (hr *hostnamesResults) Cancel() {
|
func (hr *hostnamesResults) Cancel() {
|
||||||
if hr != nil && hr.cancelFn != nil {
|
if hr != nil && hr.cancelFn != nil {
|
||||||
hr.cancelFn()
|
hr.cancelFn()
|
||||||
@@ -256,6 +346,13 @@ func (r *RemoteList) ResetForOwner(ownerVpnAddr netip.Addr) {
|
|||||||
r.shouldRebuild = true
|
r.shouldRebuild = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ResolveHostnames kicks a fresh DNS lookup of any static_map hostnames for this host
|
||||||
|
func (r *RemoteList) ResolveHostnames() {
|
||||||
|
r.RLock()
|
||||||
|
defer r.RUnlock()
|
||||||
|
r.hr.ResolveNow()
|
||||||
|
}
|
||||||
|
|
||||||
// ClearHostnameResults cancels the in-flight DNS resolver goroutine (if any) and drops the resolved IP cache.
|
// ClearHostnameResults cancels the in-flight DNS resolver goroutine (if any) and drops the resolved IP cache.
|
||||||
func (r *RemoteList) ClearHostnameResults() {
|
func (r *RemoteList) ClearHostnameResults() {
|
||||||
r.Lock()
|
r.Lock()
|
||||||
@@ -460,10 +557,14 @@ func (r *RemoteList) unlockedSetV4(ownerVpnIp, vpnIp netip.Addr, to []*V4AddrPor
|
|||||||
// Reset the slice
|
// Reset the slice
|
||||||
c.reported = c.reported[:0]
|
c.reported = c.reported[:0]
|
||||||
|
|
||||||
// We can't take their array but we can take their pointers
|
// We can't take their array but we can take their pointers.
|
||||||
for _, v := range to[:minInt(len(to), MaxRemotes)] {
|
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
|
||||||
|
for _, v := range to {
|
||||||
if check(vpnIp, v) {
|
if check(vpnIp, v) {
|
||||||
c.reported = append(c.reported, v)
|
c.reported = append(c.reported, v)
|
||||||
|
if len(c.reported) == MaxRemotes {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -508,10 +609,14 @@ func (r *RemoteList) unlockedSetV6(ownerVpnIp, vpnIp netip.Addr, to []*V6AddrPor
|
|||||||
// Reset the slice
|
// Reset the slice
|
||||||
c.reported = c.reported[:0]
|
c.reported = c.reported[:0]
|
||||||
|
|
||||||
// We can't take their array but we can take their pointers
|
// We can't take their array but we can take their pointers.
|
||||||
for _, v := range to[:minInt(len(to), MaxRemotes)] {
|
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
|
||||||
|
for _, v := range to {
|
||||||
if check(vpnIp, v) {
|
if check(vpnIp, v) {
|
||||||
c.reported = append(c.reported, v)
|
c.reported = append(c.reported, v)
|
||||||
|
if len(c.reported) == MaxRemotes {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,15 @@
|
|||||||
package nebula
|
package nebula
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/slackhq/nebula/test"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
@@ -328,6 +333,55 @@ func BenchmarkSortRebuild(b *testing.B) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestApplyLookup(t *testing.T) {
|
||||||
|
prev := []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}
|
||||||
|
|
||||||
|
// A successful lookup replaces the previous addresses
|
||||||
|
got := applyLookup(prev, 4242, []netip.Addr{netip.MustParseAddr("::ffff:2.2.2.2")}, nil)
|
||||||
|
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("2.2.2.2:4242")}, got)
|
||||||
|
|
||||||
|
// A not found answer means the record is gone
|
||||||
|
assert.Empty(t, applyLookup(prev, 4242, nil, &net.DNSError{Err: "no such host", IsNotFound: true}))
|
||||||
|
|
||||||
|
// Anything else is not new information, keep what we had
|
||||||
|
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "i/o timeout", IsTimeout: true}))
|
||||||
|
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "server misbehaving", IsTemporary: true}))
|
||||||
|
assert.Equal(t, prev, applyLookup(prev, 4242, nil, context.DeadlineExceeded))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewHostnameResults_CarriesKnown(t *testing.T) {
|
||||||
|
// A canceled context keeps the background lookup from learning anything, as if DNS were down
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
known := netip.MustParseAddrPort("1.1.1.1:4242")
|
||||||
|
prev := &hostnamesResults{network: "ip4"}
|
||||||
|
prev.known.Store(&map[hostnamePort][]netip.AddrPort{{name: "lh.invalid", port: 4242}: {known}})
|
||||||
|
|
||||||
|
hostPorts := []string{"lh.invalid:4242", "10.0.0.1:4242"}
|
||||||
|
literal := netip.MustParseAddrPort("10.0.0.1:4242")
|
||||||
|
|
||||||
|
hr, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, prev, func() {})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr.GetAddrs())
|
||||||
|
assert.Equal(t, []netip.AddrPort{literal}, hr.literalAddrs())
|
||||||
|
|
||||||
|
// And again from the new one, before it has finished a lookup of its own
|
||||||
|
hr2, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, hr, func() {})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr2.GetAddrs())
|
||||||
|
|
||||||
|
// A port change is a different entry
|
||||||
|
hr3, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, []string{"lh.invalid:4243"}, prev, func() {})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, hr3.GetAddrs())
|
||||||
|
|
||||||
|
// So is a network change
|
||||||
|
hr4, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip6", time.Second, hostPorts, prev, func() {})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, []netip.AddrPort{literal}, hr4.GetAddrs())
|
||||||
|
}
|
||||||
|
|
||||||
func newIp4AndPortFromString(s string) *V4AddrPort {
|
func newIp4AndPortFromString(s string) *V4AddrPort {
|
||||||
a := netip.MustParseAddrPort(s)
|
a := netip.MustParseAddrPort(s)
|
||||||
v4Addr := a.Addr().As4()
|
v4Addr := a.Addr().As4()
|
||||||
@@ -346,3 +400,38 @@ func newIp6AndPortFromString(s string) *V6AddrPort {
|
|||||||
Port: uint32(a.Port()),
|
Port: uint32(a.Port()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRemoteList_SetFiltersBeforeCap(t *testing.T) {
|
||||||
|
owner := netip.MustParseAddr("0.0.0.0")
|
||||||
|
rl := NewRemoteList([]netip.Addr{owner}, nil)
|
||||||
|
|
||||||
|
// MaxRemotes denied addrs first, then the only allowed one, then enough allowed to exceed the cap
|
||||||
|
var v4 []*V4AddrPort
|
||||||
|
for i := 0; i < MaxRemotes; i++ {
|
||||||
|
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("172.17.0.%d:4242", i+1)))
|
||||||
|
}
|
||||||
|
for i := 0; i < MaxRemotes+1; i++ {
|
||||||
|
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("70.199.182.%d:4242", i+1)))
|
||||||
|
}
|
||||||
|
rl.unlockedSetV4(owner, owner, v4, func(_ netip.Addr, a *V4AddrPort) bool {
|
||||||
|
return protoV4AddrPortToNetAddrPort(a).Addr().As4()[0] != 172
|
||||||
|
})
|
||||||
|
|
||||||
|
var v6 []*V6AddrPort
|
||||||
|
for i := 0; i < MaxRemotes; i++ {
|
||||||
|
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[fd00::%d]:4242", i+1)))
|
||||||
|
}
|
||||||
|
for i := 0; i < MaxRemotes+1; i++ {
|
||||||
|
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[2001:db8::%d]:4242", i+1)))
|
||||||
|
}
|
||||||
|
rl.unlockedSetV6(owner, owner, v6, func(_ netip.Addr, a *V6AddrPort) bool {
|
||||||
|
return protoV6AddrPortToNetAddrPort(a).Addr().As16()[0] != 0xfd
|
||||||
|
})
|
||||||
|
|
||||||
|
rl.Rebuild([]netip.Prefix{})
|
||||||
|
assert.Len(t, rl.addrs, MaxRemotes*2)
|
||||||
|
for _, a := range rl.addrs {
|
||||||
|
assert.NotEqual(t, byte(172), a.Addr().As16()[12], "denied v4 addr %v was kept", a)
|
||||||
|
assert.NotEqual(t, byte(0xfd), a.Addr().As16()[0], "denied v6 addr %v was kept", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
+12
-4
@@ -255,6 +255,10 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
|
|||||||
msgs, buffers, names, _ := prepareRawMessages(u.batch, bufSize, cmsgSpace)
|
msgs, buffers, names, _ := prepareRawMessages(u.batch, bufSize, cmsgSpace)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
|
// A shut down socket still queues and returns datagrams, a busy reader would never see an empty read
|
||||||
|
if u.closed.Load() {
|
||||||
|
return net.ErrClosed
|
||||||
|
}
|
||||||
if cmsgSpace > 0 {
|
if cmsgSpace > 0 {
|
||||||
for i := range msgs {
|
for i := range msgs {
|
||||||
setMsgControllen(&msgs[i].Hdr, cmsgSpace)
|
setMsgControllen(&msgs[i].Hdr, cmsgSpace)
|
||||||
@@ -446,12 +450,16 @@ func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) Close() error {
|
// Shutdown wakes the reader without releasing the fd, Close only once it has exited
|
||||||
|
func (u *StdConn) Shutdown() error {
|
||||||
u.closed.Store(true)
|
u.closed.Store(true)
|
||||||
// Wake the reader parked in recvmmsg. shutdown(2) on an unconnected socket
|
// shutdown(2) on an unconnected socket returns ENOTCONN but still wakes the reader, so ignore the error
|
||||||
// returns ENOTCONN but still wakes it, so ignore the error.
|
|
||||||
// The reader then sees closed and stops touching the fd, making the Close below safe.
|
|
||||||
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
|
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *StdConn) Close() error {
|
||||||
|
_ = u.Shutdown()
|
||||||
return unix.Close(u.sysFd)
|
return unix.Close(u.sysFd)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+74
-5
@@ -76,7 +76,7 @@ func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
|
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
|
||||||
// reader must tear down cleanly on Close() regardless of what the socket has
|
// reader must tear down cleanly on Shutdown() regardless of what the socket has
|
||||||
// carried. The three cases the report called out:
|
// carried. The three cases the report called out:
|
||||||
//
|
//
|
||||||
// no traffic ever -> works (shutdown wakes recvmmsg with n==0)
|
// no traffic ever -> works (shutdown wakes recvmmsg with n==0)
|
||||||
@@ -162,11 +162,12 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
|
|||||||
// empty queue with the socket already having received a packet.
|
// empty queue with the socket already having received a packet.
|
||||||
time.Sleep(500 * time.Millisecond)
|
time.Sleep(500 * time.Millisecond)
|
||||||
|
|
||||||
|
// Tear down the way Interface.Close does, the fd stays open until the reader is gone
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
if err := sc.Close(); err != nil {
|
if err := sc.Shutdown(); err != nil {
|
||||||
t.Fatalf("Close: %v", err)
|
t.Fatalf("Shutdown: %v", err)
|
||||||
}
|
}
|
||||||
close(stop)
|
defer func() { _ = sc.Close() }()
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case err := <-loopDone:
|
case err := <-loopDone:
|
||||||
@@ -179,7 +180,75 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
|
|||||||
case <-time.After(3 * time.Second):
|
case <-time.After(3 * time.Second):
|
||||||
buf := make([]byte, 1<<20)
|
buf := make([]byte, 1<<20)
|
||||||
n := runtime.Stack(buf, true)
|
n := runtime.Stack(buf, true)
|
||||||
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
|
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Shutdown\n%s", name, buf[:n])
|
||||||
}
|
}
|
||||||
|
close(stop)
|
||||||
<-trafficDone
|
<-trafficDone
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A shut down socket still returns queued datagrams, so a reader that is behind has to stop on its own within the
|
||||||
|
// batch it was working on
|
||||||
|
func TestListenOutShutdown_BusyReader(t *testing.T) {
|
||||||
|
const batch = 64
|
||||||
|
udpSettings := Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Multi: true, Batch: batch}
|
||||||
|
c, err := NewListener(testLogger(), udpSettings)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewListener: %v", err)
|
||||||
|
}
|
||||||
|
sc := c.(*StdConn)
|
||||||
|
defer sc.Close()
|
||||||
|
addr, err := sc.LocalAddr()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LocalAddr: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
stop := make(chan struct{})
|
||||||
|
defer close(stop)
|
||||||
|
for range 8 {
|
||||||
|
send, err := net.Dial("udp", addr.String())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dial: %v", err)
|
||||||
|
}
|
||||||
|
defer send.Close()
|
||||||
|
go func() {
|
||||||
|
b := make([]byte, 1200)
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
_, _ = send.Write(b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
var afterShutdown atomic.Int64
|
||||||
|
loopDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
|
||||||
|
if sc.closed.Load() {
|
||||||
|
afterShutdown.Add(1)
|
||||||
|
}
|
||||||
|
// Slower than the senders so the queue never drains
|
||||||
|
for start := time.Now(); time.Since(start) < 200*time.Microsecond; {
|
||||||
|
}
|
||||||
|
}, func() {})
|
||||||
|
}()
|
||||||
|
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
if err := sc.Shutdown(); err != nil {
|
||||||
|
t.Fatalf("Shutdown: %v", err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-loopDone:
|
||||||
|
if !errors.Is(err, net.ErrClosed) {
|
||||||
|
t.Fatalf("ListenOut returned %v, want net.ErrClosed", err)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatalf("ListenOut still running 3s after Shutdown, delivered %d packets since", afterShutdown.Load())
|
||||||
|
}
|
||||||
|
if n := afterShutdown.Load(); n > batch {
|
||||||
|
t.Fatalf("delivered %d packets after Shutdown, more than the one batch in flight", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user