mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 12:46:52 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ca1e6178a1 |
No files matched your search
@@ -86,21 +86,6 @@ func (c *C) InitialLoad() bool {
|
||||
return c.oldSettings == nil
|
||||
}
|
||||
|
||||
// Render returns keys and their values as yaml, for a subsystem to remember what it applied. Empty means they couldn't
|
||||
// be rendered and should never match.
|
||||
func (c *C) Render(keys ...string) string {
|
||||
m := make(map[string]any, len(keys))
|
||||
for _, k := range keys {
|
||||
m[k] = c.Get(k)
|
||||
}
|
||||
b, err := yaml.Marshal(m)
|
||||
if err != nil {
|
||||
c.l.Debug("Failed to render config", "keys", keys, "error", err)
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
|
||||
// k in both the old and new settings will be serialized, the result of the string comparison is returned.
|
||||
// If k is an empty string the entire config is tested.
|
||||
|
||||
+2
-6
@@ -39,9 +39,8 @@ static_host_map:
|
||||
# public address. Other valid options are "ip6" and "ip" (returns both.)
|
||||
#network: ip4
|
||||
|
||||
# lookup_timeout is the DNS query timeout. A failed lookup keeps the last known good addresses, only a not found
|
||||
# answer removes them.
|
||||
#lookup_timeout: 5s
|
||||
# lookup_timeout is the DNS query timeout.
|
||||
#lookup_timeout: 250ms
|
||||
|
||||
lighthouse:
|
||||
# am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
|
||||
@@ -332,9 +331,6 @@ tun:
|
||||
# `mtu`: will default to tun mtu if this option is not specified
|
||||
# `metric`: will default to 0 if this option is not specified
|
||||
# `install`: will default to true, controls whether this route is installed in the systems routing table.
|
||||
# On macOS, a route whose destination another interface already routes, like the local network, is installed once
|
||||
# that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another
|
||||
# interface routes its destination and lets it go.
|
||||
# This setting is reloadable.
|
||||
unsafe_routes:
|
||||
#- route: 172.16.1.0/24
|
||||
|
||||
@@ -66,9 +66,6 @@ type Firewall struct {
|
||||
rules string
|
||||
rulesVersion uint16
|
||||
|
||||
// source is the rendered firewall config this firewall was built from, the cert networks are kept above
|
||||
source string
|
||||
|
||||
defaultLocalCIDRAny bool
|
||||
incomingMetrics firewallMetrics
|
||||
outgoingMetrics firewallMetrics
|
||||
@@ -249,7 +246,6 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fw.source = c.Render("firewall")
|
||||
return fw, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ require (
|
||||
golang.org/x/term v0.46.0
|
||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
|
||||
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
|
||||
golang.zx2c4.com/wireguard/windows v1.0.1
|
||||
golang.zx2c4.com/wireguard/windows v1.1.1
|
||||
google.golang.org/protobuf v1.36.12
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980
|
||||
|
||||
@@ -218,8 +218,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu
|
||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
|
||||
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
|
||||
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
|
||||
golang.zx2c4.com/wireguard/windows v1.0.1 h1:eOxiDVbywPC+ZQqvdCK7x+ZwWXKbYv50TtH8ysFIbw8=
|
||||
golang.zx2c4.com/wireguard/windows v1.0.1/go.mod h1:+fbT3FFdX4zzYDLwJh5+HPEcNN/3HyNdzhNSVsQM+zs=
|
||||
golang.zx2c4.com/wireguard/windows v1.1.1 h1:8/H97U1v1PNDNcBsMZgU3KFuND9MQdTsU2NOwmCXArE=
|
||||
golang.zx2c4.com/wireguard/windows v1.1.1/go.mod h1:+fbT3FFdX4zzYDLwJh5+HPEcNN/3HyNdzhNSVsQM+zs=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
|
||||
+23
-35
@@ -61,17 +61,16 @@ type HandshakeManager struct {
|
||||
vpnIps map[netip.Addr]*HandshakeHostInfo
|
||||
indexes map[uint32]*HandshakeHostInfo
|
||||
|
||||
mainHostMap *HostMap
|
||||
lightHouse *LightHouse
|
||||
outside udp.Conn
|
||||
config HandshakeConfig
|
||||
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
|
||||
messageMetrics *MessageMetrics
|
||||
metricInitiated metrics.Counter
|
||||
metricTimedOut metrics.Counter
|
||||
metricRejectedBlocklisted metrics.Counter
|
||||
f *Interface
|
||||
l *slog.Logger
|
||||
mainHostMap *HostMap
|
||||
lightHouse *LightHouse
|
||||
outside udp.Conn
|
||||
config HandshakeConfig
|
||||
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
|
||||
messageMetrics *MessageMetrics
|
||||
metricInitiated metrics.Counter
|
||||
metricTimedOut metrics.Counter
|
||||
f *Interface
|
||||
l *slog.Logger
|
||||
|
||||
// can be used to trigger outbound handshake for the given vpnIp
|
||||
trigger chan netip.Addr
|
||||
@@ -119,19 +118,18 @@ func (hh *HandshakeHostInfo) cachePacket(l *slog.Logger, t header.MessageType, s
|
||||
|
||||
func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager {
|
||||
return &HandshakeManager{
|
||||
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
|
||||
indexes: map[uint32]*HandshakeHostInfo{},
|
||||
mainHostMap: mainHostMap,
|
||||
lightHouse: lightHouse,
|
||||
outside: outside,
|
||||
config: config,
|
||||
trigger: make(chan netip.Addr, config.triggerBuffer),
|
||||
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
|
||||
messageMetrics: config.messageMetrics,
|
||||
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
|
||||
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
|
||||
metricRejectedBlocklisted: metrics.GetOrRegisterCounter("handshake_manager.rejected.blocklisted", nil),
|
||||
l: l,
|
||||
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
|
||||
indexes: map[uint32]*HandshakeHostInfo{},
|
||||
mainHostMap: mainHostMap,
|
||||
lightHouse: lightHouse,
|
||||
outside: outside,
|
||||
config: config,
|
||||
trigger: make(chan netip.Addr, config.triggerBuffer),
|
||||
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
|
||||
messageMetrics: config.messageMetrics,
|
||||
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
|
||||
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
|
||||
l: l,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -405,8 +403,6 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
|
||||
|
||||
hm.Unlock()
|
||||
hm.lightHouse.QueryServer(vpnAddr)
|
||||
// If this host came from a static_map hostname its addresses may have moved, retries will pick up anything new
|
||||
hostinfo.remotes.ResolveHostnames()
|
||||
return hostinfo
|
||||
}
|
||||
|
||||
@@ -721,12 +717,7 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
|
||||
|
||||
response, result, err := machine.ProcessPacket(nil, packet)
|
||||
if err != nil {
|
||||
level := slog.LevelError
|
||||
if errors.Is(err, cert.ErrBlockListed) {
|
||||
level = slog.LevelDebug
|
||||
hm.metricRejectedBlocklisted.Inc(1)
|
||||
}
|
||||
f.l.Log(context.Background(), level, "Failed to process handshake packet", "from", via, "error", err)
|
||||
f.l.Error("Failed to process handshake packet", "from", via, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -858,9 +849,6 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
|
||||
|
||||
response, result, err := machine.ProcessPacket(nil, packet)
|
||||
if err != nil {
|
||||
if errors.Is(err, cert.ErrBlockListed) {
|
||||
hm.metricRejectedBlocklisted.Inc(1)
|
||||
}
|
||||
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
|
||||
if machine.Failed() {
|
||||
f.l.Warn("Failed to process handshake packet, abandoning",
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
package nebula
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/flynn/noise"
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/rcrowley/go-metrics"
|
||||
"github.com/slackhq/nebula/cert"
|
||||
ct "github.com/slackhq/nebula/cert_test"
|
||||
"github.com/slackhq/nebula/handshake"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/slackhq/nebula/udp"
|
||||
@@ -240,89 +235,6 @@ func TestHandleIncomingDispatch(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestBlocklistedHandshakeLogLevelAndCounter(t *testing.T) {
|
||||
ca, _, caKey, _ := ct.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
|
||||
pool := ct.NewTestCAPool(ca)
|
||||
|
||||
newCredential := func(name, network string) (cert.Certificate, *handshake.Credential) {
|
||||
c, _, rawKey, _ := ct.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey,
|
||||
name, ca.NotBefore(), ca.NotAfter(), []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
|
||||
privateKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey)
|
||||
require.NoError(t, err)
|
||||
certBytes, err := c.MarshalForHandshakes()
|
||||
require.NoError(t, err)
|
||||
cipherSuite := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
|
||||
return c, handshake.NewCredential(c, certBytes, privateKey, cipherSuite)
|
||||
}
|
||||
|
||||
initiatorCert, initiatorCredential := newCredential("initiator", "10.0.0.1/24")
|
||||
responderCert, responderCredential := newCredential("responder", "10.0.0.2/24")
|
||||
fingerprint, err := initiatorCert.Fingerprint()
|
||||
require.NoError(t, err)
|
||||
pool.BlocklistFingerprint(fingerprint)
|
||||
|
||||
initiator, err := handshake.NewMachine(cert.Version2,
|
||||
func(cert.Version) *handshake.Credential { return initiatorCredential },
|
||||
func(c cert.Certificate) (*cert.CachedCertificate, error) {
|
||||
return pool.VerifyCertificate(time.Now(), c)
|
||||
},
|
||||
func() (uint32, error) { return 1, nil }, true, header.HandshakeIXPSK0)
|
||||
require.NoError(t, err)
|
||||
packet, err := initiator.Initiate(nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
handler := &capturingHandler{}
|
||||
logger := slog.New(handler)
|
||||
pki := &PKI{}
|
||||
pki.cs.Store(&CertState{initiatingVersion: cert.Version2, v2Credential: responderCredential})
|
||||
pki.caPool.Store(pool)
|
||||
hm := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
|
||||
hm.metricRejectedBlocklisted = metrics.NewCounter()
|
||||
hm.f = &Interface{pki: pki, l: logger}
|
||||
via := ViaSender{UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242")}
|
||||
|
||||
hm.beginHandshake(via, packet, nil)
|
||||
entry := findEntry(t, handler.entries, "Failed to process handshake packet")
|
||||
assert.Equal(t, slog.LevelDebug, entry.Level)
|
||||
require.ErrorIs(t, entry.Attrs["error"].(error), cert.ErrBlockListed)
|
||||
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
|
||||
|
||||
handler.entries = nil
|
||||
hm.beginHandshake(via, nil, nil)
|
||||
entry = findEntry(t, handler.entries, "Failed to process handshake packet")
|
||||
assert.Equal(t, slog.LevelError, entry.Level)
|
||||
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
|
||||
|
||||
// The initiator sees the same rejection in continueHandshake, after a
|
||||
// responder has accepted its first packet and sent a valid reply.
|
||||
cleanPool := ct.NewTestCAPool(ca)
|
||||
responder, err := handshake.NewMachine(cert.Version2,
|
||||
func(cert.Version) *handshake.Credential { return responderCredential },
|
||||
func(c cert.Certificate) (*cert.CachedCertificate, error) {
|
||||
return cleanPool.VerifyCertificate(time.Now(), c)
|
||||
},
|
||||
func() (uint32, error) { return 2, nil }, false, header.HandshakeIXPSK0)
|
||||
require.NoError(t, err)
|
||||
response, _, err := responder.ProcessPacket(nil, packet)
|
||||
require.NoError(t, err)
|
||||
|
||||
responderFingerprint, err := responderCert.Fingerprint()
|
||||
require.NoError(t, err)
|
||||
pool.BlocklistFingerprint(responderFingerprint)
|
||||
|
||||
initiatingHM := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
|
||||
initiatingHM.metricRejectedBlocklisted = metrics.NewCounter()
|
||||
initiatingHM.f = &Interface{l: logger}
|
||||
hh := &HandshakeHostInfo{
|
||||
machine: initiator,
|
||||
hostinfo: &HostInfo{localIndexId: 1, vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}},
|
||||
}
|
||||
initiatingHM.indexes[1] = hh
|
||||
via.IsRelayed = true // skip the remote allow list before certificate validation
|
||||
initiatingHM.continueHandshake(via, hh, response)
|
||||
assert.Equal(t, int64(1), initiatingHM.metricRejectedBlocklisted.Count())
|
||||
}
|
||||
|
||||
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
|
||||
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
|
||||
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
|
||||
|
||||
+14
-64
@@ -124,8 +124,6 @@ type Interface struct {
|
||||
// reused (every udp.Conn ListenOut guarantees that ordering).
|
||||
batchers []*batch.MultiCoalescer
|
||||
wg sync.WaitGroup
|
||||
// readers covers just the listenIn and listenOut goroutines, wg also holds the construction token Close releases
|
||||
readers sync.WaitGroup
|
||||
|
||||
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
|
||||
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
|
||||
@@ -343,18 +341,14 @@ func (f *Interface) activate() error {
|
||||
func (f *Interface) run() {
|
||||
// Launch n queues to read packets from udp
|
||||
for i := 0; i < f.routines; i++ {
|
||||
f.readers.Add(1)
|
||||
f.wg.Go(func() {
|
||||
defer f.readers.Done()
|
||||
f.listenOut(i)
|
||||
})
|
||||
}
|
||||
|
||||
// Launch n queues to read packets from tun dev
|
||||
for i := 0; i < f.routines; i++ {
|
||||
f.readers.Add(1)
|
||||
f.wg.Go(func() {
|
||||
defer f.readers.Done()
|
||||
f.listenIn(f.queues[i], i)
|
||||
})
|
||||
}
|
||||
@@ -543,12 +537,11 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
|
||||
curCert = cs.getCertificate(cert.Version1)
|
||||
}
|
||||
|
||||
// Compare with what the running firewall was built from, not the last config, so a firewall that failed to build
|
||||
// is tried again. The cert is the one this reload brought, PKI registers its reload callback before this one
|
||||
src := c.Render("firewall")
|
||||
if src != "" && src == f.firewall.source && curCert != nil &&
|
||||
slices.Equal(curCert.Networks(), f.firewall.assignedNetworks) &&
|
||||
slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) {
|
||||
// The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction.
|
||||
// Check to see if that set has changed, and if so, rebuild the firewall.
|
||||
certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks)
|
||||
|
||||
if !c.HasChanged("firewall") && !certUnsafeChanged {
|
||||
f.l.Debug("No firewall config change detected")
|
||||
return
|
||||
}
|
||||
@@ -689,14 +682,6 @@ func (f *Interface) GetCertState() *CertState {
|
||||
return f.pki.getCertState()
|
||||
}
|
||||
|
||||
// shutdowner is a udp.Conn or overlay.Device whose readers can be woken without releasing their fds
|
||||
type shutdowner interface {
|
||||
Shutdown() error
|
||||
}
|
||||
|
||||
// readerShutdownTimeout bounds how long Close waits on woken readers before releasing their fds anyway
|
||||
const readerShutdownTimeout = 5 * time.Second
|
||||
|
||||
// Close releases the interface's resources: the udp sockets and the tun device.
|
||||
// It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated,
|
||||
// calls after the first return nil without doing anything.
|
||||
@@ -706,55 +691,20 @@ func (f *Interface) Close() error {
|
||||
}
|
||||
|
||||
var errs []error
|
||||
closeUdp := func(i int, u udp.Conn) {
|
||||
if err := u.Close(); err != nil {
|
||||
|
||||
// Release the udp readers
|
||||
for i, u := range f.writers {
|
||||
err := u.Close()
|
||||
if err != nil {
|
||||
f.l.Error("Error while closing udp socket", "error", err, "writer", i)
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
closeTun := func() {
|
||||
if err := f.inside.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Wake every reader before releasing any fd, one that loops back into a syscall on a reused fd number never wakes.
|
||||
// Anything without Shutdown is closed right away, as before.
|
||||
var deferred []func()
|
||||
for i, u := range f.writers {
|
||||
if s, ok := u.(shutdowner); ok {
|
||||
if err := s.Shutdown(); err != nil {
|
||||
f.l.Error("Error while shutting down udp socket", "error", err, "writer", i)
|
||||
errs = append(errs, err)
|
||||
}
|
||||
deferred = append(deferred, func() { closeUdp(i, u) })
|
||||
} else {
|
||||
closeUdp(i, u)
|
||||
}
|
||||
}
|
||||
if s, ok := f.inside.(shutdowner); ok {
|
||||
if err := s.Shutdown(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
deferred = append(deferred, closeTun)
|
||||
} else {
|
||||
closeTun()
|
||||
}
|
||||
|
||||
if len(deferred) > 0 {
|
||||
readersDone := make(chan struct{})
|
||||
go func() {
|
||||
f.readers.Wait()
|
||||
close(readersDone)
|
||||
}()
|
||||
select {
|
||||
case <-readersDone:
|
||||
case <-time.After(readerShutdownTimeout):
|
||||
f.l.Warn("Readers did not stop, closing their fds anyway", "timeout", readerShutdownTimeout)
|
||||
}
|
||||
for _, closeFn := range deferred {
|
||||
closeFn()
|
||||
}
|
||||
// Release the tun device (closing the tun also closes all readers)
|
||||
closeErr := f.inside.Close()
|
||||
if closeErr != nil {
|
||||
errs = append(errs, closeErr)
|
||||
}
|
||||
|
||||
// Release the construction token so waiters know the resources are gone
|
||||
|
||||
+3
-58
@@ -135,63 +135,8 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
|
||||
f := &Interface{pki: pki, firewall: fw, l: l}
|
||||
cfg.RegisterReloadCallback(f.reloadFirewall)
|
||||
|
||||
// The running firewall still isn't built from what the config asks for, so the same reload fails again
|
||||
for range 2 {
|
||||
res := cfg.ReloadConfigString(rules("nope"))
|
||||
require.NoError(t, res.Err)
|
||||
assert.True(t, res.Failed)
|
||||
assert.Same(t, fw, f.firewall)
|
||||
}
|
||||
|
||||
res := cfg.ReloadConfigString(rules("tcp"))
|
||||
res := cfg.ReloadConfigString(rules("nope"))
|
||||
require.NoError(t, res.Err)
|
||||
assert.False(t, res.Failed)
|
||||
assert.NotSame(t, fw, f.firewall)
|
||||
}
|
||||
|
||||
// Every input the firewall is built from rebuilds it when it changes, and nothing else does
|
||||
func TestReloadFirewall_RebuildsOnAnyInput(t *testing.T) {
|
||||
l := test.NewLogger()
|
||||
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
|
||||
base := "firewall:\n inbound:\n - port: any\n proto: any\n host: any\n"
|
||||
setup := func() (*config.C, *Interface, *PKI) {
|
||||
pki := &PKI{}
|
||||
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks}, initiatingVersion: cert.Version2})
|
||||
cfg := config.NewC(l)
|
||||
require.NoError(t, cfg.LoadString(base))
|
||||
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
|
||||
require.NoError(t, err)
|
||||
f := &Interface{pki: pki, firewall: fw, l: l}
|
||||
cfg.RegisterReloadCallback(f.reloadFirewall)
|
||||
return cfg, f, pki
|
||||
}
|
||||
setCert := func(pki *PKI, networks, unsafe []netip.Prefix) {
|
||||
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks, unsafeNetworks: unsafe}, initiatingVersion: cert.Version2})
|
||||
}
|
||||
|
||||
cases := map[string]struct {
|
||||
cfg string
|
||||
cert func(*PKI)
|
||||
want bool
|
||||
}{
|
||||
"nothing": {base + "punchy:\n punch: true\n", nil, false},
|
||||
"rules": {base + " outbound:\n - port: any\n proto: any\n host: any\n", nil, true},
|
||||
"inbound_action": {base + " inbound_action: reject\n", nil, true},
|
||||
"conntrack": {base + " conntrack:\n tcp_timeout: 1m\n", nil, true},
|
||||
"networks": {base, func(p *PKI) { setCert(p, []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil) }, true},
|
||||
"unsafe_networks": {base, func(p *PKI) { setCert(p, networks, []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}) }, true},
|
||||
}
|
||||
for name, tc := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg, f, pki := setup()
|
||||
old := f.firewall
|
||||
if tc.cert != nil {
|
||||
tc.cert(pki)
|
||||
}
|
||||
res := cfg.ReloadConfigString(tc.cfg)
|
||||
require.NoError(t, res.Err)
|
||||
assert.False(t, res.Failed)
|
||||
assert.Equal(t, tc.want, f.firewall != old)
|
||||
})
|
||||
}
|
||||
assert.True(t, res.Failed)
|
||||
assert.Same(t, fw, f.firewall)
|
||||
}
|
||||
+3
-5
@@ -452,7 +452,7 @@ func getStaticMapCadence(c *config.C) (time.Duration, error) {
|
||||
}
|
||||
|
||||
func getStaticMapLookupTimeout(c *config.C) (time.Duration, error) {
|
||||
lookupTimeout := c.GetString("static_map.lookup_timeout", "5s")
|
||||
lookupTimeout := c.GetString("static_map.lookup_timeout", "250ms")
|
||||
d, err := time.ParseDuration(lookupTimeout)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -630,8 +630,7 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
|
||||
ctx := lh.ctx
|
||||
lh.Unlock()
|
||||
|
||||
// On reload am.hr is the previous resolver, its last known good addresses carry over in case DNS is down right now
|
||||
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, am.hr, func() {
|
||||
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, func() {
|
||||
// This callback runs whenever the DNS hostname resolver finds a different set of addr's
|
||||
// in its resolution for hostnames.
|
||||
am.Lock()
|
||||
@@ -643,8 +642,7 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
|
||||
}
|
||||
am.unlockedSetHostnamesResults(hr)
|
||||
|
||||
// Only literals belong in the cache, anything from DNS has to stay in hr where a not found answer can remove it
|
||||
for _, addrPort := range hr.literalAddrs() {
|
||||
for _, addrPort := range hr.GetAddrs() {
|
||||
if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ type offloadQueueSet struct {
|
||||
// usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel.
|
||||
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
|
||||
usoEnabled bool
|
||||
shutdown atomic.Bool
|
||||
closed atomic.Bool
|
||||
// l is handed to each queue for its bad-vnet-header drop logging.
|
||||
l *slog.Logger
|
||||
@@ -70,17 +69,6 @@ func (c *offloadQueueSet) wakeForShutdown() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
|
||||
func (c *offloadQueueSet) Shutdown() error {
|
||||
if c.shutdown.Swap(true) {
|
||||
return nil
|
||||
}
|
||||
for _, x := range c.pq {
|
||||
x.shutdown.Store(true)
|
||||
}
|
||||
return c.wakeForShutdown()
|
||||
}
|
||||
|
||||
func (c *offloadQueueSet) Close() error {
|
||||
if c.closed.Swap(true) {
|
||||
return nil
|
||||
@@ -88,7 +76,9 @@ func (c *offloadQueueSet) Close() error {
|
||||
|
||||
errs := []error{}
|
||||
|
||||
if err := c.Shutdown(); err != nil {
|
||||
// Signal all readers blocked in poll to wake up and exit.
|
||||
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
|
||||
if err := c.wakeForShutdown(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ type pollQueueSet struct {
|
||||
// pqi is exactly the same as pq, but stored as the interface type
|
||||
pqi []Queue
|
||||
shutdownFd int
|
||||
shutdown atomic.Bool
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
@@ -60,17 +59,6 @@ func (c *pollQueueSet) wakeForShutdown() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
|
||||
func (c *pollQueueSet) Shutdown() error {
|
||||
if c.shutdown.Swap(true) {
|
||||
return nil
|
||||
}
|
||||
for _, x := range c.pq {
|
||||
x.shutdown.Store(true)
|
||||
}
|
||||
return c.wakeForShutdown()
|
||||
}
|
||||
|
||||
func (c *pollQueueSet) Close() error {
|
||||
if c.closed.Swap(true) {
|
||||
return nil
|
||||
@@ -78,7 +66,9 @@ func (c *pollQueueSet) Close() error {
|
||||
|
||||
errs := []error{}
|
||||
|
||||
if err := c.Shutdown(); err != nil {
|
||||
// Signal all readers blocked in poll to wake up and exit.
|
||||
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
|
||||
if err := c.wakeForShutdown(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
// QueueSet holds one or many Queue objects and helps close them in an orderly way.
|
||||
type QueueSet interface {
|
||||
io.Closer
|
||||
// Shutdown wakes every reader without releasing any fds, Close only once they have exited
|
||||
Shutdown() error
|
||||
Queues() []Queue
|
||||
|
||||
// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
|
||||
|
||||
@@ -66,9 +66,7 @@ type Offload struct {
|
||||
// usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD,
|
||||
// so writers can decide whether emitting GSO_UDP_L4 superpackets is safe.
|
||||
usoEnabled bool
|
||||
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
|
||||
shutdown atomic.Bool
|
||||
closed atomic.Bool
|
||||
closed atomic.Bool
|
||||
|
||||
// gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted
|
||||
// by WriteGSO. Kept separate from the read-only package-level validVnetHdr
|
||||
@@ -178,9 +176,6 @@ func (r *Offload) readPacket(block bool) (int, error) {
|
||||
// This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs).
|
||||
// Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue.
|
||||
func (r *Offload) Read() ([]Packet, error) {
|
||||
if r.shutdown.Load() {
|
||||
return nil, os.ErrClosed
|
||||
}
|
||||
r.pending = r.pending[:0]
|
||||
r.rxOff = 0
|
||||
|
||||
|
||||
@@ -13,9 +13,7 @@ import (
|
||||
type Poll struct {
|
||||
fd int
|
||||
shutdownFd int
|
||||
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
|
||||
shutdown atomic.Bool
|
||||
closed atomic.Bool
|
||||
closed atomic.Bool
|
||||
|
||||
readBuf []byte
|
||||
batchRet [1]Packet
|
||||
@@ -54,9 +52,6 @@ func (t *Poll) blockOnWrite() error {
|
||||
// config currently lose that batching. blockOn and the EAGAIN plumbing are
|
||||
// already shared; kept one-packet-per-Read for now to preserve behavior.
|
||||
func (t *Poll) Read() ([]Packet, error) {
|
||||
if t.shutdown.Load() {
|
||||
return nil, os.ErrClosed
|
||||
}
|
||||
n, err := t.readOne(t.readBuf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -223,30 +223,3 @@ func TestOffloadQueueSet_Close_ClosesShutdownFd(t *testing.T) {
|
||||
// Second Close must not touch fds (shutdownFd is now -1) and must return nil.
|
||||
require.NoError(t, qs.Close())
|
||||
}
|
||||
|
||||
// A busy reader never polls the shutdown eventfd, so Read has to stop on its own with data still waiting
|
||||
func TestQueueSet_ShutdownStopsReaderWithDataWaiting(t *testing.T) {
|
||||
sets := map[string]func() (QueueSet, error){
|
||||
"poll": NewPollQueueSet,
|
||||
"offload": func() (QueueSet, error) { return NewOffloadQueueSet(false, slog.New(slog.DiscardHandler)) },
|
||||
}
|
||||
for name, newSet := range sets {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var fds [2]int
|
||||
require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
|
||||
t.Cleanup(func() { _ = unix.Close(fds[1]) })
|
||||
|
||||
qs, err := newSet()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, qs.Add(fds[0]))
|
||||
t.Cleanup(func() { _ = qs.Close() })
|
||||
|
||||
_, err = unix.Write(fds[1], make([]byte, 64))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, qs.Shutdown())
|
||||
|
||||
_, err = qs.Queues()[0].Read()
|
||||
require.ErrorIs(t, err, os.ErrClosed)
|
||||
})
|
||||
}
|
||||
}
|
||||
+84
-54
@@ -3,6 +3,7 @@
|
||||
package overlay
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/bits"
|
||||
@@ -11,7 +12,6 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
@@ -35,26 +35,6 @@ type tun struct {
|
||||
// and routes for it. NEPacketTunnelProvider on darwin does this.
|
||||
hostOwned bool
|
||||
l *slog.Logger
|
||||
|
||||
// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
|
||||
// fields below it
|
||||
routesMu sync.Mutex
|
||||
// routeStates is who held each installable unsafe_route prefix when we last looked
|
||||
routeStates map[netip.Prefix]routeState
|
||||
// routeWatch is the routing socket watchRoutes reads, Close closes it
|
||||
routeWatch *os.File
|
||||
// routeSweep reconciles the unsafe_routes every routeSweepEvery
|
||||
routeSweep *time.Timer
|
||||
routeSweepEvery time.Duration
|
||||
closed bool
|
||||
|
||||
// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
|
||||
// lock so the route watcher never waits on a reconcile
|
||||
routeMoversMu sync.Mutex
|
||||
routeMovers map[netip.Prefix]routeMover
|
||||
|
||||
// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
|
||||
lastRouteEvent atomic.Int64
|
||||
}
|
||||
|
||||
type ifReq struct {
|
||||
@@ -159,12 +139,11 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
|
||||
}
|
||||
|
||||
t := &tun{
|
||||
f: os.NewFile(uintptr(fd), ""),
|
||||
Device: name,
|
||||
vpnNetworks: vpnNetworks,
|
||||
DefaultMTU: getMTU(l, c),
|
||||
routeSweepEvery: routeSweepInterval,
|
||||
l: l,
|
||||
f: os.NewFile(uintptr(fd), ""),
|
||||
Device: name,
|
||||
vpnNetworks: vpnNetworks,
|
||||
DefaultMTU: getMTU(l, c),
|
||||
l: l,
|
||||
}
|
||||
|
||||
err = t.reload(c, true, nil)
|
||||
@@ -240,16 +219,6 @@ func utunNameFromFd(fd int) string {
|
||||
}
|
||||
|
||||
func (t *tun) Close() error {
|
||||
t.routesMu.Lock()
|
||||
t.closed = true
|
||||
if t.routeWatch != nil {
|
||||
_ = t.routeWatch.Close()
|
||||
}
|
||||
if t.routeSweep != nil {
|
||||
t.routeSweep.Stop()
|
||||
}
|
||||
t.routesMu.Unlock()
|
||||
|
||||
if t.f != nil {
|
||||
return t.f.Close()
|
||||
}
|
||||
@@ -318,22 +287,8 @@ func (t *tun) Activate() error {
|
||||
return fmt.Errorf("failed to run tun device: %s", err)
|
||||
}
|
||||
|
||||
// Watch before adding so a route removed in between is still noticed
|
||||
watch, err := openRouteSocket()
|
||||
if err != nil {
|
||||
t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
|
||||
}
|
||||
|
||||
// Unsafe path routes
|
||||
if err = t.addRoutes(false, nil); err != nil {
|
||||
if watch != nil {
|
||||
_ = watch.Close()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
t.startRouteWatch(watch)
|
||||
return nil
|
||||
return t.addRoutes(false, nil)
|
||||
}
|
||||
|
||||
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
|
||||
@@ -438,8 +393,7 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
|
||||
}
|
||||
|
||||
if !initial && !change {
|
||||
// Any reload puts back the unsafe_routes someone deleted
|
||||
return t.addRoutes(true, r)
|
||||
return nil
|
||||
}
|
||||
|
||||
routeTree, err := makeRouteTree(t.l, routes, false)
|
||||
@@ -557,6 +511,82 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
|
||||
// The route tree is still ours, the system routing table is not
|
||||
if t.hostOwned {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Activate hasn't run yet, it will install the routes
|
||||
linkAddr := t.linkAddr.Load()
|
||||
if linkAddr == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
routes := *t.Routes.Load()
|
||||
systemRoutes := sync.OnceValues(fetchRoutes)
|
||||
|
||||
for _, r := range routes {
|
||||
if len(r.Via) == 0 || !r.Install {
|
||||
// We don't allow route MTUs so only install routes with a via
|
||||
continue
|
||||
}
|
||||
|
||||
err := addRoute(r.Cidr, linkAddr)
|
||||
if err != nil {
|
||||
if errors.Is(err, unix.EEXIST) {
|
||||
// Every route we added already exists on a reload, only warn about someone else's
|
||||
msgs, err := systemRoutes()
|
||||
if err != nil {
|
||||
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
|
||||
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
|
||||
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
|
||||
}
|
||||
} else {
|
||||
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
||||
if logErrors {
|
||||
retErr.Log(t.l)
|
||||
rl.Failed()
|
||||
} else {
|
||||
return retErr
|
||||
}
|
||||
}
|
||||
} else {
|
||||
t.l.Info("Added route", "route", r)
|
||||
// A cidr listed twice would otherwise find our own route missing from an older fetch
|
||||
systemRoutes = sync.OnceValues(fetchRoutes)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
|
||||
linkAddr := t.linkAddr.Load()
|
||||
if t.hostOwned || linkAddr == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, r := range routes {
|
||||
if len(r.Via) == 0 || !r.Install {
|
||||
// addRoutes never installs these
|
||||
continue
|
||||
}
|
||||
|
||||
err := delRoute(r.Cidr, linkAddr)
|
||||
if errors.Is(err, unix.ESRCH) {
|
||||
// Already gone, which is what we wanted
|
||||
t.l.Info("Route was already removed", "route", r)
|
||||
} else if err != nil {
|
||||
t.l.Error("Failed to remove route", "error", err, "route", r)
|
||||
rl.Failed()
|
||||
} else {
|
||||
t.l.Info("Removed route", "route", r)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
|
||||
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
||||
if err != nil {
|
||||
|
||||
@@ -3,17 +3,12 @@
|
||||
package overlay
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
@@ -109,367 +104,6 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
|
||||
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
|
||||
}
|
||||
|
||||
// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
|
||||
const (
|
||||
// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
|
||||
kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
|
||||
addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
|
||||
// With no interface address, which x/net/route skips
|
||||
interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
|
||||
)
|
||||
|
||||
func TestRoutesMayHaveMoved(t *testing.T) {
|
||||
cidr := netip.MustParsePrefix("192.168.1.0/24")
|
||||
via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
|
||||
routes := []Route{{Cidr: cidr, Via: via, Install: true}}
|
||||
// A request like route(8) sends, the kernel echoes it to every routing socket
|
||||
request := func(typ, flags int, dst, mask [4]byte) []byte {
|
||||
m := &netroute.RouteMessage{
|
||||
Version: unix.RTM_VERSION,
|
||||
Type: typ,
|
||||
Flags: flags,
|
||||
ID: 1234,
|
||||
Addrs: []netroute.Addr{
|
||||
unix.RTAX_DST: &netroute.Inet4Addr{IP: dst},
|
||||
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
|
||||
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
|
||||
},
|
||||
}
|
||||
b, err := m.Marshal()
|
||||
require.NoError(t, err)
|
||||
return b
|
||||
}
|
||||
raw := func(s string) []byte {
|
||||
b, err := hex.DecodeString(s)
|
||||
require.NoError(t, err)
|
||||
return b
|
||||
}
|
||||
lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
|
||||
routesMayHaveMoved := func(b []byte, routes []Route) bool {
|
||||
moved, _, _ := routesMayHaveMoved(b, routes)
|
||||
return moved
|
||||
}
|
||||
|
||||
assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
|
||||
assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
|
||||
assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
|
||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
|
||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
|
||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
|
||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
|
||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
|
||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
|
||||
|
||||
failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
|
||||
binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
|
||||
assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
|
||||
|
||||
_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
|
||||
require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
|
||||
assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
|
||||
neighbor := raw(kernelSubnetDelete)
|
||||
binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
|
||||
assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
|
||||
|
||||
assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
|
||||
assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
|
||||
}
|
||||
|
||||
// A request names its prefix and who made it, the kernel's own and interface changes name neither
|
||||
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
|
||||
cidr := netip.MustParsePrefix("192.168.1.0/24")
|
||||
routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
|
||||
m := &netroute.RouteMessage{
|
||||
Version: unix.RTM_VERSION,
|
||||
Type: unix.RTM_CHANGE,
|
||||
Flags: unix.RTF_UP,
|
||||
ID: 1234,
|
||||
Addrs: []netroute.Addr{
|
||||
unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
|
||||
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
|
||||
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
|
||||
},
|
||||
}
|
||||
b, err := m.Marshal()
|
||||
require.NoError(t, err)
|
||||
moved, p, pid := routesMayHaveMoved(b, routes)
|
||||
assert.True(t, moved)
|
||||
assert.Equal(t, cidr, p)
|
||||
assert.Equal(t, int32(1234), pid)
|
||||
|
||||
for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
|
||||
b, err := hex.DecodeString(s)
|
||||
require.NoError(t, err)
|
||||
moved, p, pid := routesMayHaveMoved(b, routes)
|
||||
assert.True(t, moved, name)
|
||||
assert.False(t, p.IsValid(), name)
|
||||
assert.Zero(t, pid, name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessName(t *testing.T) {
|
||||
assert.Equal(t, "launchd", processName(1))
|
||||
self := filepath.Base(os.Args[0])
|
||||
assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
|
||||
assert.Empty(t, processName(-1))
|
||||
}
|
||||
|
||||
// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
|
||||
// the minute between sweeps when it isn't zero.
|
||||
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
|
||||
requireRoot(t)
|
||||
|
||||
cfg := "tun:\n unsafe_routes:\n"
|
||||
for _, cidr := range cidrs {
|
||||
via := "10.250.0.2"
|
||||
if netip.MustParsePrefix(cidr).Addr().Is6() {
|
||||
via = "fd00:250::2"
|
||||
}
|
||||
cfg += " - route: " + cidr + "\n via: " + via + "\n"
|
||||
}
|
||||
|
||||
l := test.NewLogger()
|
||||
c := config.NewC(l)
|
||||
require.NoError(t, c.LoadString(cfg))
|
||||
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
|
||||
require.NoError(t, err)
|
||||
if sweep != 0 {
|
||||
tn.routeSweepEvery = sweep
|
||||
}
|
||||
t.Cleanup(func() { _ = tn.Close() })
|
||||
require.NoError(t, tn.Activate())
|
||||
return tn, c, cfg
|
||||
}
|
||||
|
||||
func requireRoot(t *testing.T) {
|
||||
if os.Geteuid() != 0 {
|
||||
t.Skip("creating interfaces and adding routes needs root")
|
||||
}
|
||||
}
|
||||
|
||||
// routeVia is the index of the interface that routes cidr, 0 if none does
|
||||
func routeVia(t *testing.T, cidr string) int {
|
||||
msgs, err := fetchRoutes()
|
||||
require.NoError(t, err)
|
||||
return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
|
||||
}
|
||||
|
||||
// routeStateOf is what tn made of cidr when it last looked
|
||||
func routeStateOf(tn *tun, cidr string) routeState {
|
||||
tn.routesMu.Lock()
|
||||
defer tn.routesMu.Unlock()
|
||||
return tn.routeStates[netip.MustParsePrefix(cidr)]
|
||||
}
|
||||
|
||||
func routeArgs(action, cidr string, extra ...string) []string {
|
||||
args := []string{"-q", "-n", action}
|
||||
if netip.MustParsePrefix(cidr).Addr().Is6() {
|
||||
args = append(args, "-inet6")
|
||||
}
|
||||
return append(append(args, "-net", cidr), extra...)
|
||||
}
|
||||
|
||||
func routeCmd(t *testing.T, action, cidr string, extra ...string) {
|
||||
requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
|
||||
}
|
||||
|
||||
// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
|
||||
// says so
|
||||
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
|
||||
out, err := cmd.CombinedOutput()
|
||||
require.NoError(t, err, string(out))
|
||||
require.NotContains(t, string(out), "writing to routing socket")
|
||||
}
|
||||
|
||||
// cleanupRoute deletes cidr's route when the test ends, whoever holds it
|
||||
func cleanupRoute(t *testing.T, cidr string) {
|
||||
t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
|
||||
}
|
||||
|
||||
// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
|
||||
func routeThroughLo0(t *testing.T, cidr string) int {
|
||||
requireRoot(t)
|
||||
routeCmd(t, "add", cidr, "-interface", "lo0")
|
||||
cleanupRoute(t, cidr)
|
||||
lo0, err := net.InterfaceByName("lo0")
|
||||
require.NoError(t, err)
|
||||
return lo0.Index
|
||||
}
|
||||
|
||||
// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
|
||||
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
|
||||
for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
|
||||
t.Run(cidr, func(t *testing.T) {
|
||||
lo0 := routeThroughLo0(t, cidr)
|
||||
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||
require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
|
||||
|
||||
routeCmd(t, "delete", cidr)
|
||||
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
|
||||
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
|
||||
cidr := "10.251.1.0/24"
|
||||
lo0 := routeThroughLo0(t, cidr)
|
||||
_, c, _ := newRouteTestTun(t, 0, cidr)
|
||||
|
||||
res := c.ReloadConfigString("tun: {}\n")
|
||||
require.NoError(t, res.Err)
|
||||
assert.False(t, res.Failed)
|
||||
assert.Equal(t, lo0, routeVia(t, cidr))
|
||||
}
|
||||
|
||||
// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
|
||||
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
|
||||
cidr := "10.251.3.0/24"
|
||||
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||
ours := tn.linkAddr.Load().Index
|
||||
require.Equal(t, ours, routeVia(t, cidr))
|
||||
|
||||
cleanupRoute(t, cidr)
|
||||
requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
|
||||
lo0, err := net.InterfaceByName("lo0")
|
||||
require.NoError(t, err)
|
||||
time.Sleep(2 * routeRetryDelay)
|
||||
require.Equal(t, lo0.Index, routeVia(t, cidr))
|
||||
|
||||
routeCmd(t, "delete", cidr)
|
||||
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
||||
}
|
||||
|
||||
// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
|
||||
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
|
||||
cidr := "10.251.4.0/24"
|
||||
tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
|
||||
ours := tn.linkAddr.Load().Index
|
||||
require.Equal(t, ours, routeVia(t, cidr))
|
||||
|
||||
routeCmd(t, "delete", cidr)
|
||||
assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
|
||||
|
||||
// A reload with nothing changed is how to ask for it back
|
||||
res := c.ReloadConfigString(cfg)
|
||||
require.NoError(t, res.Err)
|
||||
assert.False(t, res.Failed)
|
||||
assert.Equal(t, ours, routeVia(t, cidr))
|
||||
}
|
||||
|
||||
// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
|
||||
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
|
||||
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
cidr string
|
||||
sweep time.Duration
|
||||
misfile func(t *testing.T, tn *tun, cidr string)
|
||||
}{
|
||||
// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
|
||||
"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
|
||||
require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
|
||||
}},
|
||||
// The sweep that caps a table that never settles
|
||||
"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
|
||||
tn.reconcileRoutes()
|
||||
require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
|
||||
}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
|
||||
ours := tn.linkAddr.Load().Index
|
||||
require.Equal(t, ours, routeVia(t, tc.cidr))
|
||||
|
||||
routeCmd(t, "delete", tc.cidr)
|
||||
tc.misfile(t, tn, tc.cidr)
|
||||
|
||||
lo0 := routeThroughLo0(t, tc.cidr)
|
||||
require.Eventually(t, func() bool {
|
||||
s := routeStateOf(tn, tc.cidr)
|
||||
return s.kind == routeYielded && s.holder == lo0
|
||||
}, 5*time.Second, 50*time.Millisecond)
|
||||
|
||||
routeCmd(t, "delete", tc.cidr)
|
||||
assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
|
||||
// taken for a hand delete
|
||||
func TestTunYieldsToARouteChangedAway(t *testing.T) {
|
||||
for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
|
||||
t.Run(cidr, func(t *testing.T) {
|
||||
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||
ours := tn.linkAddr.Load().Index
|
||||
require.Equal(t, ours, routeVia(t, cidr))
|
||||
|
||||
cleanupRoute(t, cidr)
|
||||
routeCmd(t, "change", cidr, "-interface", "lo0")
|
||||
lo0, err := net.InterfaceByName("lo0")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, lo0.Index, routeVia(t, cidr))
|
||||
require.Eventually(t, func() bool {
|
||||
s := routeStateOf(tn, cidr)
|
||||
return s.kind == routeYielded && s.holder == lo0.Index
|
||||
}, 5*time.Second, 50*time.Millisecond)
|
||||
|
||||
routeCmd(t, "delete", cidr)
|
||||
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
|
||||
// ethernet configd routed the lan through
|
||||
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
|
||||
requireRoot(t)
|
||||
var feth string
|
||||
for i := 100; i < 200 && feth == ""; i++ {
|
||||
if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
|
||||
feth = fmt.Sprintf("feth%d", i)
|
||||
}
|
||||
}
|
||||
require.NotEmpty(t, feth)
|
||||
require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
|
||||
t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
|
||||
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
|
||||
|
||||
// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
|
||||
// refuses one named by the feth itself
|
||||
cidr := "10.251.2.0/24"
|
||||
routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
|
||||
cleanupRoute(t, cidr)
|
||||
ifc, err := net.InterfaceByName(feth)
|
||||
require.NoError(t, err)
|
||||
|
||||
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
||||
require.Equal(t, ifc.Index, routeVia(t, cidr))
|
||||
|
||||
// Losing the address first leaves the route, the reconcile finds it still held
|
||||
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
|
||||
time.Sleep(2 * routeRetryDelay)
|
||||
require.Equal(t, ifc.Index, routeVia(t, cidr))
|
||||
|
||||
// Only RTM_IFINFO says the interface went, its route goes with no message of its own
|
||||
require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
|
||||
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
||||
}
|
||||
|
||||
// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
|
||||
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
|
||||
cidr := "10.251.5.0/24"
|
||||
routeThroughLo0(t, cidr)
|
||||
tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
|
||||
|
||||
tn.routesMu.Lock()
|
||||
require.NoError(t, tn.routeWatch.Close())
|
||||
tn.routesMu.Unlock()
|
||||
|
||||
routeCmd(t, "delete", cidr)
|
||||
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
||||
}
|
||||
|
||||
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
|
||||
func TestTunDevNeedsRestart(t *testing.T) {
|
||||
for name, want := range map[string]bool{
|
||||
|
||||
@@ -832,11 +832,6 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
|
||||
t.routeTree.Store(newTree)
|
||||
}
|
||||
|
||||
// Shutdown wakes the tun readers without releasing their fds, Close releases them
|
||||
func (t *tun) Shutdown() error {
|
||||
return t.readers.Shutdown()
|
||||
}
|
||||
|
||||
func (t *tun) Close() error {
|
||||
t.closeLock.Lock()
|
||||
defer t.closeLock.Unlock()
|
||||
|
||||
@@ -1,451 +0,0 @@
|
||||
//go:build !ios && !e2e_testing
|
||||
|
||||
package overlay
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/util"
|
||||
netroute "golang.org/x/net/route"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
|
||||
// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
|
||||
// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
|
||||
// or until another interface takes the prefix, after which ours follows theirs like any other.
|
||||
|
||||
const (
|
||||
// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
|
||||
// replaces a route by deleting and adding, looking in between would find the prefix free
|
||||
routeRetryDelay = time.Second
|
||||
|
||||
// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
|
||||
// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
|
||||
// a message for each
|
||||
routeSweepInterval = time.Minute
|
||||
|
||||
// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
|
||||
routeSocketRcvBuf = 1 << 20
|
||||
)
|
||||
|
||||
type routeStateKind int
|
||||
|
||||
const (
|
||||
routeUnknown routeStateKind = iota
|
||||
// routeInstalled is our route holding the prefix
|
||||
routeInstalled
|
||||
// routeYielded is another interface holding the prefix, ours is added once it lets go
|
||||
routeYielded
|
||||
// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
|
||||
// another interface takes the prefix
|
||||
routeRemoved
|
||||
// routeFailed is adding our route failing for some other reason, it is retried
|
||||
routeFailed
|
||||
)
|
||||
|
||||
type routeState struct {
|
||||
kind routeStateKind
|
||||
// holder is the index of the interface holding the prefix when kind is routeYielded
|
||||
holder int
|
||||
}
|
||||
|
||||
// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
|
||||
// is looked up as the message arrives, route(8) has exited by the time the table settles
|
||||
type routeMover struct {
|
||||
pid int32
|
||||
name string
|
||||
}
|
||||
|
||||
// logArgs appends m to a log line's args, when we know who it was
|
||||
func (m routeMover) logArgs(args ...any) []any {
|
||||
if m.pid == 0 {
|
||||
return args
|
||||
}
|
||||
args = append(args, "pid", m.pid)
|
||||
if m.name != "" {
|
||||
args = append(args, "process", m.name)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
|
||||
func processName(pid int32) string {
|
||||
kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
|
||||
if err != nil || kp.Proc.P_pid != pid {
|
||||
return ""
|
||||
}
|
||||
return unix.ByteSliceToString(kp.Proc.P_comm[:])
|
||||
}
|
||||
|
||||
// installRoutes returns the routes we put in the system routing table, one per prefix
|
||||
func installRoutes(routes []Route) []Route {
|
||||
seen := make(map[netip.Prefix]struct{}, len(routes))
|
||||
var out []Route
|
||||
for _, r := range routes {
|
||||
if len(r.Via) == 0 || !r.Install {
|
||||
// We don't allow route MTUs so only install routes with a via
|
||||
continue
|
||||
}
|
||||
p := r.Cidr.Masked()
|
||||
if _, ok := seen[p]; ok {
|
||||
continue
|
||||
}
|
||||
seen[p] = struct{}{}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
|
||||
// interface holds its prefix. It runs on start and on every reload.
|
||||
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
|
||||
// The route tree is still ours, the system routing table is not
|
||||
if t.hostOwned {
|
||||
return nil
|
||||
}
|
||||
|
||||
t.routesMu.Lock()
|
||||
defer t.routesMu.Unlock()
|
||||
|
||||
// Activate hasn't run yet, it will install the routes
|
||||
linkAddr := t.linkAddr.Load()
|
||||
if linkAddr == nil || t.closed {
|
||||
return nil
|
||||
}
|
||||
|
||||
var msgs []netroute.Message
|
||||
var msgsErr error
|
||||
fetched := false
|
||||
states := make(map[netip.Prefix]routeState)
|
||||
for _, r := range installRoutes(*t.Routes.Load()) {
|
||||
p := r.Cidr.Masked()
|
||||
prev := t.routeStates[p]
|
||||
|
||||
err := addRoute(p, linkAddr)
|
||||
if err == nil {
|
||||
t.l.Info("Added route", "route", r)
|
||||
states[p] = routeState{kind: routeInstalled}
|
||||
continue
|
||||
}
|
||||
|
||||
if !errors.Is(err, unix.EEXIST) {
|
||||
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
||||
if !logErrors {
|
||||
return retErr
|
||||
}
|
||||
retErr.Log(t.l)
|
||||
rl.Failed()
|
||||
states[p] = routeState{kind: routeFailed}
|
||||
continue
|
||||
}
|
||||
|
||||
// Every route we added already exists on a reload, only warn about someone else's
|
||||
if !fetched {
|
||||
msgs, msgsErr = fetchRoutes()
|
||||
fetched = true
|
||||
}
|
||||
if msgsErr != nil {
|
||||
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
|
||||
states[p] = prev
|
||||
continue
|
||||
}
|
||||
|
||||
switch holder := routeInterfaceIn(msgs, p); holder {
|
||||
case linkAddr.Index:
|
||||
states[p] = routeState{kind: routeInstalled}
|
||||
case 0:
|
||||
// Removed since the add, the watcher will see to it
|
||||
states[p] = prev
|
||||
default:
|
||||
if prev.kind != routeYielded || prev.holder != holder {
|
||||
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
|
||||
}
|
||||
states[p] = routeState{kind: routeYielded, holder: holder}
|
||||
}
|
||||
}
|
||||
|
||||
t.routeStates = states
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
|
||||
if t.hostOwned {
|
||||
return nil
|
||||
}
|
||||
|
||||
t.routesMu.Lock()
|
||||
defer t.routesMu.Unlock()
|
||||
|
||||
linkAddr := t.linkAddr.Load()
|
||||
if linkAddr == nil || t.closed {
|
||||
return nil
|
||||
}
|
||||
|
||||
// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
|
||||
msgs, err := fetchRoutes()
|
||||
if err != nil {
|
||||
var left []netip.Prefix
|
||||
for _, r := range installRoutes(routes) {
|
||||
left = append(left, r.Cidr)
|
||||
}
|
||||
return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
|
||||
}
|
||||
|
||||
for _, r := range installRoutes(routes) {
|
||||
p := r.Cidr.Masked()
|
||||
// Whatever happens next, the prefix is no longer one we know about
|
||||
delete(t.routeStates, p)
|
||||
if routeInterfaceIn(msgs, p) != linkAddr.Index {
|
||||
continue
|
||||
}
|
||||
|
||||
err = delRoute(p, linkAddr)
|
||||
if errors.Is(err, unix.ESRCH) {
|
||||
// Already gone, which is what we wanted
|
||||
t.l.Info("Route was already removed", "route", r)
|
||||
} else if err != nil {
|
||||
t.l.Error("Failed to remove route", "error", err, "route", r)
|
||||
rl.Failed()
|
||||
} else {
|
||||
t.l.Info("Removed route", "route", r)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
|
||||
// changed since it last looked, it runs every routeSweepEvery.
|
||||
func (t *tun) reconcileRoutes() {
|
||||
t.routesMu.Lock()
|
||||
defer t.routesMu.Unlock()
|
||||
|
||||
linkAddr := t.linkAddr.Load()
|
||||
if linkAddr == nil || t.closed {
|
||||
return
|
||||
}
|
||||
|
||||
t.routeMoversMu.Lock()
|
||||
movers := t.routeMovers
|
||||
t.routeMovers = nil
|
||||
t.routeMoversMu.Unlock()
|
||||
|
||||
msgs, err := fetchRoutes()
|
||||
if err != nil {
|
||||
t.l.Error("Failed to read the system routing table", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
states := make(map[netip.Prefix]routeState)
|
||||
for _, r := range installRoutes(*t.Routes.Load()) {
|
||||
p := r.Cidr.Masked()
|
||||
prev := t.routeStates[p]
|
||||
by := movers[p]
|
||||
|
||||
holder := routeInterfaceIn(msgs, p)
|
||||
switch {
|
||||
case holder == linkAddr.Index:
|
||||
states[p] = routeState{kind: routeInstalled}
|
||||
|
||||
case holder != 0:
|
||||
// Another interface taking a prefix we left alone puts ours back in line behind theirs
|
||||
if prev.kind != routeYielded || prev.holder != holder {
|
||||
t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
|
||||
}
|
||||
states[p] = routeState{kind: routeYielded, holder: holder}
|
||||
|
||||
case prev.kind == routeInstalled:
|
||||
t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
|
||||
states[p] = routeState{kind: routeRemoved}
|
||||
|
||||
case prev.kind == routeRemoved:
|
||||
states[p] = prev
|
||||
|
||||
default:
|
||||
err := addRoute(p, linkAddr)
|
||||
switch {
|
||||
case err == nil:
|
||||
if prev.kind == routeYielded {
|
||||
t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
|
||||
} else {
|
||||
t.l.Info("Added route", "route", r)
|
||||
}
|
||||
states[p] = routeState{kind: routeInstalled}
|
||||
case errors.Is(err, unix.EEXIST):
|
||||
// Taken since the fetch, the next look will see by whom
|
||||
states[p] = prev
|
||||
default:
|
||||
if prev.kind != routeFailed {
|
||||
util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
|
||||
}
|
||||
states[p] = routeState{kind: routeFailed}
|
||||
}
|
||||
}
|
||||
}
|
||||
t.routeStates = states
|
||||
}
|
||||
|
||||
// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
|
||||
// routing table settles after anything that may have moved a prefix
|
||||
func (t *tun) startRouteWatch(watch *os.File) {
|
||||
t.routesMu.Lock()
|
||||
defer t.routesMu.Unlock()
|
||||
if t.closed {
|
||||
if watch != nil {
|
||||
_ = watch.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
|
||||
if watch != nil {
|
||||
t.routeWatch = watch
|
||||
go t.watchRoutes(watch)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tun) sweepRoutes() {
|
||||
// A table still changing gets reconciled once it settles
|
||||
if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
|
||||
t.reconcileRoutes()
|
||||
}
|
||||
|
||||
t.routesMu.Lock()
|
||||
defer t.routesMu.Unlock()
|
||||
if !t.closed {
|
||||
t.routeSweep.Reset(t.routeSweepEvery)
|
||||
}
|
||||
}
|
||||
|
||||
func openRouteSocket() (*os.File, error) {
|
||||
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
|
||||
}
|
||||
// Fewer drops, the sweep catches the rest
|
||||
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
|
||||
// Non-blocking puts it on the poller, so Close interrupts a Read
|
||||
if err = unix.SetNonblock(fd, true); err != nil {
|
||||
_ = unix.Close(fd)
|
||||
return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
|
||||
}
|
||||
return os.NewFile(uintptr(fd), "route"), nil
|
||||
}
|
||||
|
||||
// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
|
||||
// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
|
||||
func (t *tun) watchRoutes(f *os.File) {
|
||||
var reconciled atomic.Int64
|
||||
settle := time.AfterFunc(time.Hour, func() {
|
||||
reconciled.Store(time.Now().UnixNano())
|
||||
t.reconcileRoutes()
|
||||
})
|
||||
settle.Stop()
|
||||
defer settle.Stop()
|
||||
|
||||
var burst time.Time
|
||||
self := int32(os.Getpid())
|
||||
b := make([]byte, os.Getpagesize())
|
||||
for {
|
||||
n, err := f.Read(b)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrClosed) {
|
||||
t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
|
||||
if !moved {
|
||||
continue
|
||||
}
|
||||
// Our own are already in routeStates
|
||||
if p.IsValid() && pid != self {
|
||||
t.routeMoversMu.Lock()
|
||||
if t.routeMovers == nil {
|
||||
t.routeMovers = make(map[netip.Prefix]routeMover)
|
||||
}
|
||||
t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
|
||||
t.routeMoversMu.Unlock()
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
t.lastRouteEvent.Store(now.UnixNano())
|
||||
if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
|
||||
burst = now
|
||||
}
|
||||
settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
|
||||
}
|
||||
}
|
||||
|
||||
// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
|
||||
// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
|
||||
// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
|
||||
// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
|
||||
// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
|
||||
// and the pid of the process that made it.
|
||||
func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
|
||||
if len(b) < 4 {
|
||||
return false, p, 0
|
||||
}
|
||||
switch int(b[3]) {
|
||||
case unix.RTM_IFINFO, unix.RTM_DELADDR:
|
||||
return true, p, 0
|
||||
case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
|
||||
default:
|
||||
return false, p, 0
|
||||
}
|
||||
|
||||
if len(b) < unix.SizeofRtMsghdr {
|
||||
return false, p, 0
|
||||
}
|
||||
hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
|
||||
// A failed request is reported too, and a scoped route never held the prefix
|
||||
if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
|
||||
return false, p, 0
|
||||
}
|
||||
|
||||
if hdr.Pid == 0 {
|
||||
// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
|
||||
// could be holding a prefix
|
||||
return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
|
||||
}
|
||||
|
||||
// Replies to a request carry the requester's padded sockaddrs
|
||||
msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
|
||||
if err != nil || len(msgs) != 1 {
|
||||
return false, p, 0
|
||||
}
|
||||
m, ok := msgs[0].(*netroute.RouteMessage)
|
||||
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
|
||||
return false, p, 0
|
||||
}
|
||||
p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
|
||||
if !ok {
|
||||
return false, netip.Prefix{}, 0
|
||||
}
|
||||
p = p.Masked()
|
||||
for _, r := range installRoutes(routes) {
|
||||
if r.Cidr.Masked() == p {
|
||||
return true, p, hdr.Pid
|
||||
}
|
||||
}
|
||||
return false, netip.Prefix{}, 0
|
||||
}
|
||||
|
||||
// interfaceName is the name of the interface with index i, for logs
|
||||
func interfaceName(i int) string {
|
||||
if ifc, err := net.InterfaceByIndex(i); err == nil {
|
||||
return ifc.Name
|
||||
}
|
||||
return strconv.Itoa(i)
|
||||
}
|
||||
@@ -528,10 +528,6 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
|
||||
if !rm.GetAmRelay() {
|
||||
return
|
||||
}
|
||||
if !slices.Contains(h.vpnAddrs, from) {
|
||||
logMsg.Info("Discarding relay request where relayFrom is not the sending peer")
|
||||
return
|
||||
}
|
||||
peer := rm.hostmap.QueryVpnAddr(target)
|
||||
if peer == nil {
|
||||
// Try to establish a connection to this host. If we get a future relay request,
|
||||
|
||||
@@ -7,10 +7,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/cert"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestStartRelaysLogDedupe verifies that repeated attempts with the same relay set drop the log
|
||||
@@ -97,58 +95,3 @@ func TestStartRelaysLogDedupe(t *testing.T) {
|
||||
assert.NotContains(t, buf.String(), msg, "should not log when we shortcut out")
|
||||
})
|
||||
}
|
||||
|
||||
// TestHandleCreateRelayRequestRejectsSpoofedFrom verifies that a relay only forwards a CreateRelayRequest
|
||||
// when relayFrom is the peer that sent it. Otherwise any peer could reset another host's established
|
||||
// relay back to Requested, or allocate unbounded relay entries on the target's hostinfo.
|
||||
func TestHandleCreateRelayRequestRejectsSpoofedFrom(t *testing.T) {
|
||||
l := test.NewLogger()
|
||||
relayAddr := netip.MustParseAddr("10.128.0.128")
|
||||
malloryAddr := netip.MustParseAddr("10.128.0.3")
|
||||
victimAddr := netip.MustParseAddr("10.128.0.1")
|
||||
targetAddr := netip.MustParseAddr("10.128.0.2")
|
||||
|
||||
addrTable := new(bart.Lite)
|
||||
addrTable.Insert(netip.PrefixFrom(relayAddr, relayAddr.BitLen()))
|
||||
hm := newHostMap(l)
|
||||
f := &Interface{myVpnAddrsTable: addrTable, hostMap: hm}
|
||||
rm := &relayManager{l: l, hostmap: hm}
|
||||
rm.amRelay.Store(true)
|
||||
|
||||
newRelayState := func() RelayState {
|
||||
return RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}}
|
||||
}
|
||||
mallory := &HostInfo{vpnAddrs: []netip.Addr{malloryAddr}, localIndexId: 1, relayState: newRelayState()}
|
||||
target := &HostInfo{
|
||||
vpnAddrs: []netip.Addr{targetAddr},
|
||||
localIndexId: 2,
|
||||
remotes: NewRemoteList([]netip.Addr{targetAddr}, nil),
|
||||
relayState: newRelayState(),
|
||||
}
|
||||
target.SetRemote(netip.MustParseAddrPort("192.0.2.2:4242"))
|
||||
hm.unlockedAddHostInfo(mallory, f)
|
||||
hm.unlockedAddHostInfo(target, f)
|
||||
|
||||
// The victim already has a working relay to the target through us.
|
||||
_, err := AddRelay(l, target, hm, victimAddr, nil, ForwardingType, Established)
|
||||
require.NoError(t, err)
|
||||
relaysBefore := len(hm.Relays)
|
||||
|
||||
for i := range 10 {
|
||||
spoofed := netip.AddrFrom4([4]byte{10, 128, 1, byte(i)})
|
||||
for _, from := range []netip.Addr{victimAddr, spoofed} {
|
||||
rm.handleCreateRelayRequest(cert.Version2, mallory, f, &NebulaControl{
|
||||
Type: NebulaControl_CreateRelayRequest,
|
||||
InitiatorRelayIndex: uint32(100 + i),
|
||||
RelayFromAddr: netAddrToProtoAddr(from),
|
||||
RelayToAddr: netAddrToProtoAddr(targetAddr),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
r, ok := target.relayState.QueryRelayForByIp(victimAddr)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, Established, r.State, "victim's relay must not be reset by a request from another peer")
|
||||
assert.Len(t, hm.Relays, relaysBefore, "spoofed requests must not allocate relay entries")
|
||||
assert.Empty(t, mallory.relayState.CopyAllRelayFor(), "spoofed requests must not create state on the sender")
|
||||
}
|
||||
+19
-124
@@ -2,7 +2,6 @@ package nebula
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
@@ -59,8 +58,6 @@ type cacheV6 struct {
|
||||
type hostnamePort struct {
|
||||
name string
|
||||
port uint16
|
||||
// addr is set when name is an ip literal, these never need a lookup
|
||||
addr netip.Addr
|
||||
}
|
||||
|
||||
type hostnamesResults struct {
|
||||
@@ -68,20 +65,15 @@ type hostnamesResults struct {
|
||||
network string
|
||||
lookupTimeout time.Duration
|
||||
cancelFn func()
|
||||
resolveNow chan struct{}
|
||||
l *slog.Logger
|
||||
ips atomic.Pointer[map[netip.AddrPort]struct{}]
|
||||
// known is the last known good addresses per hostname, kept so a reload can carry them forward
|
||||
known atomic.Pointer[map[hostnamePort][]netip.AddrPort]
|
||||
}
|
||||
|
||||
// NewHostnameResults starts resolving hostPorts. Any hostnames also in prev start with its last known good addresses.
|
||||
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, prev *hostnamesResults, onUpdate func()) (*hostnamesResults, error) {
|
||||
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, onUpdate func()) (*hostnamesResults, error) {
|
||||
r := &hostnamesResults{
|
||||
hostnames: make([]hostnamePort, len(hostPorts)),
|
||||
network: network,
|
||||
lookupTimeout: timeout,
|
||||
resolveNow: make(chan struct{}, 1),
|
||||
l: l,
|
||||
}
|
||||
|
||||
@@ -89,13 +81,6 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
|
||||
// DNS lookups for hostnames that aren't hardcoded IP's will happen in a background goroutine.
|
||||
performBackgroundLookup := false
|
||||
ips := map[netip.AddrPort]struct{}{}
|
||||
known := make([][]netip.AddrPort, len(hostPorts))
|
||||
var prevKnown map[hostnamePort][]netip.AddrPort
|
||||
if prev != nil && prev.network == network {
|
||||
if p := prev.known.Load(); p != nil {
|
||||
prevKnown = *p
|
||||
}
|
||||
}
|
||||
for idx, hostPort := range hostPorts {
|
||||
|
||||
rIp, sPort, err := net.SplitHostPort(hostPort)
|
||||
@@ -113,19 +98,13 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
|
||||
if err != nil {
|
||||
// This address is a hostname, not an IP address
|
||||
performBackgroundLookup = true
|
||||
known[idx] = prevKnown[r.hostnames[idx]]
|
||||
for _, a := range known[idx] {
|
||||
ips[a] = struct{}{}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Save the IP address immediately
|
||||
r.hostnames[idx].addr = addr
|
||||
ips[netip.AddrPortFrom(addr, uint16(iPort))] = struct{}{}
|
||||
}
|
||||
r.ips.Store(&ips)
|
||||
r.storeKnown(known)
|
||||
|
||||
// Time for the DNS lookup goroutine
|
||||
if performBackgroundLookup {
|
||||
@@ -135,12 +114,21 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
|
||||
go func() {
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
r.resolve(newCtx, known)
|
||||
r.storeKnown(known)
|
||||
netipAddrs := map[netip.AddrPort]struct{}{}
|
||||
for _, addrs := range known {
|
||||
for _, hostPort := range r.hostnames {
|
||||
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, r.lookupTimeout)
|
||||
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, r.network, hostPort.name)
|
||||
timeoutCancel()
|
||||
if err != nil {
|
||||
l.Error("DNS resolution failed for static_map host",
|
||||
"hostname", hostPort.name,
|
||||
"network", r.network,
|
||||
"error", err,
|
||||
)
|
||||
continue
|
||||
}
|
||||
for _, a := range addrs {
|
||||
netipAddrs[a] = struct{}{}
|
||||
netipAddrs[netip.AddrPortFrom(a.Unmap(), hostPort.port)] = struct{}{}
|
||||
}
|
||||
}
|
||||
origSet := r.ips.Load()
|
||||
@@ -171,7 +159,7 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
|
||||
case <-newCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
case <-r.resolveNow:
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -180,84 +168,6 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// resolve looks up every hostname in parallel and folds the results into known
|
||||
func (hr *hostnamesResults) resolve(ctx context.Context, known [][]netip.AddrPort) {
|
||||
var wg sync.WaitGroup
|
||||
for i, hostPort := range hr.hostnames {
|
||||
if hostPort.addr.IsValid() {
|
||||
known[i] = []netip.AddrPort{netip.AddrPortFrom(hostPort.addr, hostPort.port)}
|
||||
continue
|
||||
}
|
||||
|
||||
wg.Go(func() {
|
||||
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, hr.lookupTimeout)
|
||||
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, hr.network, hostPort.name)
|
||||
timeoutCancel()
|
||||
known[i] = applyLookup(known[i], hostPort.port, addrs, err)
|
||||
// A canceled ctx means we are reloading or shutting down, not that DNS failed
|
||||
if err != nil && ctx.Err() == nil {
|
||||
hr.l.Error("DNS resolution failed for static_map host",
|
||||
"hostname", hostPort.name,
|
||||
"network", hr.network,
|
||||
"error", err,
|
||||
"usingAddrs", known[i],
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// applyLookup returns the addresses to use for a hostname given the previous ones and a lookup result.
|
||||
// Only a not found answer clears them, a stale lighthouse address costs a handshake but an empty set costs everything.
|
||||
// darwin also reports not found while offline, that's harmless since nothing is reachable and the next handshake
|
||||
// triggers a fresh lookup once we're back.
|
||||
func applyLookup(prev []netip.AddrPort, port uint16, addrs []netip.Addr, err error) []netip.AddrPort {
|
||||
if err != nil {
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
|
||||
return nil
|
||||
}
|
||||
return prev
|
||||
}
|
||||
|
||||
out := make([]netip.AddrPort, 0, len(addrs))
|
||||
for _, a := range addrs {
|
||||
out = append(out, netip.AddrPortFrom(a.Unmap(), port))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (hr *hostnamesResults) storeKnown(known [][]netip.AddrPort) {
|
||||
byName := make(map[hostnamePort][]netip.AddrPort, len(known))
|
||||
for i, addrs := range known {
|
||||
byName[hr.hostnames[i]] = addrs
|
||||
}
|
||||
hr.known.Store(&byName)
|
||||
}
|
||||
|
||||
// literalAddrs returns the entries that were configured as ip addresses rather than hostnames
|
||||
func (hr *hostnamesResults) literalAddrs() []netip.AddrPort {
|
||||
var out []netip.AddrPort
|
||||
for _, hostPort := range hr.hostnames {
|
||||
if hostPort.addr.IsValid() {
|
||||
out = append(out, netip.AddrPortFrom(hostPort.addr, hostPort.port))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ResolveNow asks the background goroutine to look up the hostnames again without waiting for the next tick
|
||||
func (hr *hostnamesResults) ResolveNow() {
|
||||
if hr == nil || hr.resolveNow == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case hr.resolveNow <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (hr *hostnamesResults) Cancel() {
|
||||
if hr != nil && hr.cancelFn != nil {
|
||||
hr.cancelFn()
|
||||
@@ -346,13 +256,6 @@ func (r *RemoteList) ResetForOwner(ownerVpnAddr netip.Addr) {
|
||||
r.shouldRebuild = true
|
||||
}
|
||||
|
||||
// ResolveHostnames kicks a fresh DNS lookup of any static_map hostnames for this host
|
||||
func (r *RemoteList) ResolveHostnames() {
|
||||
r.RLock()
|
||||
defer r.RUnlock()
|
||||
r.hr.ResolveNow()
|
||||
}
|
||||
|
||||
// ClearHostnameResults cancels the in-flight DNS resolver goroutine (if any) and drops the resolved IP cache.
|
||||
func (r *RemoteList) ClearHostnameResults() {
|
||||
r.Lock()
|
||||
@@ -557,14 +460,10 @@ func (r *RemoteList) unlockedSetV4(ownerVpnIp, vpnIp netip.Addr, to []*V4AddrPor
|
||||
// Reset the slice
|
||||
c.reported = c.reported[:0]
|
||||
|
||||
// We can't take their array but we can take their pointers.
|
||||
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
|
||||
for _, v := range to {
|
||||
// We can't take their array but we can take their pointers
|
||||
for _, v := range to[:minInt(len(to), MaxRemotes)] {
|
||||
if check(vpnIp, v) {
|
||||
c.reported = append(c.reported, v)
|
||||
if len(c.reported) == MaxRemotes {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -609,14 +508,10 @@ func (r *RemoteList) unlockedSetV6(ownerVpnIp, vpnIp netip.Addr, to []*V6AddrPor
|
||||
// Reset the slice
|
||||
c.reported = c.reported[:0]
|
||||
|
||||
// We can't take their array but we can take their pointers.
|
||||
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
|
||||
for _, v := range to {
|
||||
// We can't take their array but we can take their pointers
|
||||
for _, v := range to[:minInt(len(to), MaxRemotes)] {
|
||||
if check(vpnIp, v) {
|
||||
c.reported = append(c.reported, v)
|
||||
if len(c.reported) == MaxRemotes {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,15 +1,10 @@
|
||||
package nebula
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -333,55 +328,6 @@ func BenchmarkSortRebuild(b *testing.B) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestApplyLookup(t *testing.T) {
|
||||
prev := []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}
|
||||
|
||||
// A successful lookup replaces the previous addresses
|
||||
got := applyLookup(prev, 4242, []netip.Addr{netip.MustParseAddr("::ffff:2.2.2.2")}, nil)
|
||||
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("2.2.2.2:4242")}, got)
|
||||
|
||||
// A not found answer means the record is gone
|
||||
assert.Empty(t, applyLookup(prev, 4242, nil, &net.DNSError{Err: "no such host", IsNotFound: true}))
|
||||
|
||||
// Anything else is not new information, keep what we had
|
||||
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "i/o timeout", IsTimeout: true}))
|
||||
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "server misbehaving", IsTemporary: true}))
|
||||
assert.Equal(t, prev, applyLookup(prev, 4242, nil, context.DeadlineExceeded))
|
||||
}
|
||||
|
||||
func TestNewHostnameResults_CarriesKnown(t *testing.T) {
|
||||
// A canceled context keeps the background lookup from learning anything, as if DNS were down
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
known := netip.MustParseAddrPort("1.1.1.1:4242")
|
||||
prev := &hostnamesResults{network: "ip4"}
|
||||
prev.known.Store(&map[hostnamePort][]netip.AddrPort{{name: "lh.invalid", port: 4242}: {known}})
|
||||
|
||||
hostPorts := []string{"lh.invalid:4242", "10.0.0.1:4242"}
|
||||
literal := netip.MustParseAddrPort("10.0.0.1:4242")
|
||||
|
||||
hr, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, prev, func() {})
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr.GetAddrs())
|
||||
assert.Equal(t, []netip.AddrPort{literal}, hr.literalAddrs())
|
||||
|
||||
// And again from the new one, before it has finished a lookup of its own
|
||||
hr2, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, hr, func() {})
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr2.GetAddrs())
|
||||
|
||||
// A port change is a different entry
|
||||
hr3, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, []string{"lh.invalid:4243"}, prev, func() {})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, hr3.GetAddrs())
|
||||
|
||||
// So is a network change
|
||||
hr4, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip6", time.Second, hostPorts, prev, func() {})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []netip.AddrPort{literal}, hr4.GetAddrs())
|
||||
}
|
||||
|
||||
func newIp4AndPortFromString(s string) *V4AddrPort {
|
||||
a := netip.MustParseAddrPort(s)
|
||||
v4Addr := a.Addr().As4()
|
||||
@@ -400,38 +346,3 @@ func newIp6AndPortFromString(s string) *V6AddrPort {
|
||||
Port: uint32(a.Port()),
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteList_SetFiltersBeforeCap(t *testing.T) {
|
||||
owner := netip.MustParseAddr("0.0.0.0")
|
||||
rl := NewRemoteList([]netip.Addr{owner}, nil)
|
||||
|
||||
// MaxRemotes denied addrs first, then the only allowed one, then enough allowed to exceed the cap
|
||||
var v4 []*V4AddrPort
|
||||
for i := 0; i < MaxRemotes; i++ {
|
||||
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("172.17.0.%d:4242", i+1)))
|
||||
}
|
||||
for i := 0; i < MaxRemotes+1; i++ {
|
||||
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("70.199.182.%d:4242", i+1)))
|
||||
}
|
||||
rl.unlockedSetV4(owner, owner, v4, func(_ netip.Addr, a *V4AddrPort) bool {
|
||||
return protoV4AddrPortToNetAddrPort(a).Addr().As4()[0] != 172
|
||||
})
|
||||
|
||||
var v6 []*V6AddrPort
|
||||
for i := 0; i < MaxRemotes; i++ {
|
||||
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[fd00::%d]:4242", i+1)))
|
||||
}
|
||||
for i := 0; i < MaxRemotes+1; i++ {
|
||||
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[2001:db8::%d]:4242", i+1)))
|
||||
}
|
||||
rl.unlockedSetV6(owner, owner, v6, func(_ netip.Addr, a *V6AddrPort) bool {
|
||||
return protoV6AddrPortToNetAddrPort(a).Addr().As16()[0] != 0xfd
|
||||
})
|
||||
|
||||
rl.Rebuild([]netip.Prefix{})
|
||||
assert.Len(t, rl.addrs, MaxRemotes*2)
|
||||
for _, a := range rl.addrs {
|
||||
assert.NotEqual(t, byte(172), a.Addr().As16()[12], "denied v4 addr %v was kept", a)
|
||||
assert.NotEqual(t, byte(0xfd), a.Addr().As16()[0], "denied v6 addr %v was kept", a)
|
||||
}
|
||||
}
|
||||
+5
-13
@@ -255,10 +255,6 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
|
||||
msgs, buffers, names, _ := prepareRawMessages(u.batch, bufSize, cmsgSpace)
|
||||
|
||||
for {
|
||||
// A shut down socket still queues and returns datagrams, a busy reader would never see an empty read
|
||||
if u.closed.Load() {
|
||||
return net.ErrClosed
|
||||
}
|
||||
if cmsgSpace > 0 {
|
||||
for i := range msgs {
|
||||
setMsgControllen(&msgs[i].Hdr, cmsgSpace)
|
||||
@@ -450,16 +446,12 @@ func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown wakes the reader without releasing the fd, Close only once it has exited
|
||||
func (u *StdConn) Shutdown() error {
|
||||
u.closed.Store(true)
|
||||
// shutdown(2) on an unconnected socket returns ENOTCONN but still wakes the reader, so ignore the error
|
||||
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *StdConn) Close() error {
|
||||
_ = u.Shutdown()
|
||||
u.closed.Store(true)
|
||||
// Wake the reader parked in recvmmsg. shutdown(2) on an unconnected socket
|
||||
// returns ENOTCONN but still wakes it, so ignore the error.
|
||||
// The reader then sees closed and stops touching the fd, making the Close below safe.
|
||||
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
|
||||
return unix.Close(u.sysFd)
|
||||
}
|
||||
|
||||
|
||||
+5
-74
@@ -76,7 +76,7 @@ func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
|
||||
// reader must tear down cleanly on Shutdown() regardless of what the socket has
|
||||
// reader must tear down cleanly on Close() regardless of what the socket has
|
||||
// carried. The three cases the report called out:
|
||||
//
|
||||
// no traffic ever -> works (shutdown wakes recvmmsg with n==0)
|
||||
@@ -162,12 +162,11 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
|
||||
// empty queue with the socket already having received a packet.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
// Tear down the way Interface.Close does, the fd stays open until the reader is gone
|
||||
start := time.Now()
|
||||
if err := sc.Shutdown(); err != nil {
|
||||
t.Fatalf("Shutdown: %v", err)
|
||||
if err := sc.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
defer func() { _ = sc.Close() }()
|
||||
close(stop)
|
||||
|
||||
select {
|
||||
case err := <-loopDone:
|
||||
@@ -180,75 +179,7 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
|
||||
case <-time.After(3 * time.Second):
|
||||
buf := make([]byte, 1<<20)
|
||||
n := runtime.Stack(buf, true)
|
||||
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Shutdown\n%s", name, buf[:n])
|
||||
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
|
||||
}
|
||||
close(stop)
|
||||
<-trafficDone
|
||||
}
|
||||
|
||||
// A shut down socket still returns queued datagrams, so a reader that is behind has to stop on its own within the
|
||||
// batch it was working on
|
||||
func TestListenOutShutdown_BusyReader(t *testing.T) {
|
||||
const batch = 64
|
||||
udpSettings := Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Multi: true, Batch: batch}
|
||||
c, err := NewListener(testLogger(), udpSettings)
|
||||
if err != nil {
|
||||
t.Fatalf("NewListener: %v", err)
|
||||
}
|
||||
sc := c.(*StdConn)
|
||||
defer sc.Close()
|
||||
addr, err := sc.LocalAddr()
|
||||
if err != nil {
|
||||
t.Fatalf("LocalAddr: %v", err)
|
||||
}
|
||||
|
||||
stop := make(chan struct{})
|
||||
defer close(stop)
|
||||
for range 8 {
|
||||
send, err := net.Dial("udp", addr.String())
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
defer send.Close()
|
||||
go func() {
|
||||
b := make([]byte, 1200)
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
default:
|
||||
_, _ = send.Write(b)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
var afterShutdown atomic.Int64
|
||||
loopDone := make(chan error, 1)
|
||||
go func() {
|
||||
loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
|
||||
if sc.closed.Load() {
|
||||
afterShutdown.Add(1)
|
||||
}
|
||||
// Slower than the senders so the queue never drains
|
||||
for start := time.Now(); time.Since(start) < 200*time.Microsecond; {
|
||||
}
|
||||
}, func() {})
|
||||
}()
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
if err := sc.Shutdown(); err != nil {
|
||||
t.Fatalf("Shutdown: %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-loopDone:
|
||||
if !errors.Is(err, net.ErrClosed) {
|
||||
t.Fatalf("ListenOut returned %v, want net.ErrClosed", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatalf("ListenOut still running 3s after Shutdown, delivered %d packets since", afterShutdown.Load())
|
||||
}
|
||||
if n := afterShutdown.Load(); n > batch {
|
||||
t.Fatalf("delivered %d packets after Shutdown, more than the one batch in flight", n)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user