Compare commits

..
Author SHA1 Message Date
dependabot[bot] ca1e6178a1 Bump golang.zx2c4.com/wireguard/windows
Bumps the zx2c4-dependencies group with 1 update in the / directory: golang.zx2c4.com/wireguard/windows.


Updates `golang.zx2c4.com/wireguard/windows` from 1.0.1 to 1.1.1

---
updated-dependencies:
- dependency-name: golang.zx2c4.com/wireguard/windows
  dependency-version: 1.1.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: zx2c4-dependencies
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-10-19 19:53:12 +00:00
26 changed files with 169 additions and 1582 deletions

No files matched your search

-15
View File
@@ -86,21 +86,6 @@ func (c *C) InitialLoad() bool {
return c.oldSettings == nil
}
// Render returns keys and their values as yaml, for a subsystem to remember what it applied. Empty means they couldn't
// be rendered and should never match.
func (c *C) Render(keys ...string) string {
m := make(map[string]any, len(keys))
for _, k := range keys {
m[k] = c.Get(k)
}
b, err := yaml.Marshal(m)
if err != nil {
c.l.Debug("Failed to render config", "keys", keys, "error", err)
return ""
}
return string(b)
}
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
// k in both the old and new settings will be serialized, the result of the string comparison is returned.
// If k is an empty string the entire config is tested.
+2 -6
View File
@@ -39,9 +39,8 @@ static_host_map:
# public address. Other valid options are "ip6" and "ip" (returns both.)
#network: ip4
# lookup_timeout is the DNS query timeout. A failed lookup keeps the last known good addresses, only a not found
# answer removes them.
#lookup_timeout: 5s
# lookup_timeout is the DNS query timeout.
#lookup_timeout: 250ms
lighthouse:
# am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
@@ -332,9 +331,6 @@ tun:
# `mtu`: will default to tun mtu if this option is not specified
# `metric`: will default to 0 if this option is not specified
# `install`: will default to true, controls whether this route is installed in the systems routing table.
# On macOS, a route whose destination another interface already routes, like the local network, is installed once
# that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another
# interface routes its destination and lets it go.
# This setting is reloadable.
unsafe_routes:
#- route: 172.16.1.0/24
-4
View File
@@ -66,9 +66,6 @@ type Firewall struct {
rules string
rulesVersion uint16
// source is the rendered firewall config this firewall was built from, the cert networks are kept above
source string
defaultLocalCIDRAny bool
incomingMetrics firewallMetrics
outgoingMetrics firewallMetrics
@@ -249,7 +246,6 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
return nil, err
}
fw.source = c.Render("firewall")
return fw, nil
}
+1 -1
View File
@@ -29,7 +29,7 @@ require (
golang.org/x/term v0.46.0
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
golang.zx2c4.com/wireguard/windows v1.0.1
golang.zx2c4.com/wireguard/windows v1.1.1
google.golang.org/protobuf v1.36.12
gopkg.in/yaml.v3 v3.0.1
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980
+2 -2
View File
@@ -218,8 +218,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
golang.zx2c4.com/wireguard/windows v1.0.1 h1:eOxiDVbywPC+ZQqvdCK7x+ZwWXKbYv50TtH8ysFIbw8=
golang.zx2c4.com/wireguard/windows v1.0.1/go.mod h1:+fbT3FFdX4zzYDLwJh5+HPEcNN/3HyNdzhNSVsQM+zs=
golang.zx2c4.com/wireguard/windows v1.1.1 h1:8/H97U1v1PNDNcBsMZgU3KFuND9MQdTsU2NOwmCXArE=
golang.zx2c4.com/wireguard/windows v1.1.1/go.mod h1:+fbT3FFdX4zzYDLwJh5+HPEcNN/3HyNdzhNSVsQM+zs=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+23 -35
View File
@@ -61,17 +61,16 @@ type HandshakeManager struct {
vpnIps map[netip.Addr]*HandshakeHostInfo
indexes map[uint32]*HandshakeHostInfo
mainHostMap *HostMap
lightHouse *LightHouse
outside udp.Conn
config HandshakeConfig
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
messageMetrics *MessageMetrics
metricInitiated metrics.Counter
metricTimedOut metrics.Counter
metricRejectedBlocklisted metrics.Counter
f *Interface
l *slog.Logger
mainHostMap *HostMap
lightHouse *LightHouse
outside udp.Conn
config HandshakeConfig
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
messageMetrics *MessageMetrics
metricInitiated metrics.Counter
metricTimedOut metrics.Counter
f *Interface
l *slog.Logger
// can be used to trigger outbound handshake for the given vpnIp
trigger chan netip.Addr
@@ -119,19 +118,18 @@ func (hh *HandshakeHostInfo) cachePacket(l *slog.Logger, t header.MessageType, s
func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager {
return &HandshakeManager{
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
indexes: map[uint32]*HandshakeHostInfo{},
mainHostMap: mainHostMap,
lightHouse: lightHouse,
outside: outside,
config: config,
trigger: make(chan netip.Addr, config.triggerBuffer),
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
metricRejectedBlocklisted: metrics.GetOrRegisterCounter("handshake_manager.rejected.blocklisted", nil),
l: l,
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
indexes: map[uint32]*HandshakeHostInfo{},
mainHostMap: mainHostMap,
lightHouse: lightHouse,
outside: outside,
config: config,
trigger: make(chan netip.Addr, config.triggerBuffer),
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
l: l,
}
}
@@ -405,8 +403,6 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
hm.Unlock()
hm.lightHouse.QueryServer(vpnAddr)
// If this host came from a static_map hostname its addresses may have moved, retries will pick up anything new
hostinfo.remotes.ResolveHostnames()
return hostinfo
}
@@ -721,12 +717,7 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
response, result, err := machine.ProcessPacket(nil, packet)
if err != nil {
level := slog.LevelError
if errors.Is(err, cert.ErrBlockListed) {
level = slog.LevelDebug
hm.metricRejectedBlocklisted.Inc(1)
}
f.l.Log(context.Background(), level, "Failed to process handshake packet", "from", via, "error", err)
f.l.Error("Failed to process handshake packet", "from", via, "error", err)
return
}
@@ -858,9 +849,6 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
response, result, err := machine.ProcessPacket(nil, packet)
if err != nil {
if errors.Is(err, cert.ErrBlockListed) {
hm.metricRejectedBlocklisted.Inc(1)
}
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
if machine.Failed() {
f.l.Warn("Failed to process handshake packet, abandoning",
-88
View File
@@ -1,17 +1,12 @@
package nebula
import (
"log/slog"
"net/netip"
"testing"
"time"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
@@ -240,89 +235,6 @@ func TestHandleIncomingDispatch(t *testing.T) {
})
}
func TestBlocklistedHandshakeLogLevelAndCounter(t *testing.T) {
ca, _, caKey, _ := ct.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
pool := ct.NewTestCAPool(ca)
newCredential := func(name, network string) (cert.Certificate, *handshake.Credential) {
c, _, rawKey, _ := ct.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey,
name, ca.NotBefore(), ca.NotAfter(), []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
privateKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey)
require.NoError(t, err)
certBytes, err := c.MarshalForHandshakes()
require.NoError(t, err)
cipherSuite := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
return c, handshake.NewCredential(c, certBytes, privateKey, cipherSuite)
}
initiatorCert, initiatorCredential := newCredential("initiator", "10.0.0.1/24")
responderCert, responderCredential := newCredential("responder", "10.0.0.2/24")
fingerprint, err := initiatorCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(fingerprint)
initiator, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return initiatorCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return pool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 1, nil }, true, header.HandshakeIXPSK0)
require.NoError(t, err)
packet, err := initiator.Initiate(nil)
require.NoError(t, err)
handler := &capturingHandler{}
logger := slog.New(handler)
pki := &PKI{}
pki.cs.Store(&CertState{initiatingVersion: cert.Version2, v2Credential: responderCredential})
pki.caPool.Store(pool)
hm := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
hm.metricRejectedBlocklisted = metrics.NewCounter()
hm.f = &Interface{pki: pki, l: logger}
via := ViaSender{UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242")}
hm.beginHandshake(via, packet, nil)
entry := findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelDebug, entry.Level)
require.ErrorIs(t, entry.Attrs["error"].(error), cert.ErrBlockListed)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
handler.entries = nil
hm.beginHandshake(via, nil, nil)
entry = findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelError, entry.Level)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
// The initiator sees the same rejection in continueHandshake, after a
// responder has accepted its first packet and sent a valid reply.
cleanPool := ct.NewTestCAPool(ca)
responder, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return responderCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return cleanPool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 2, nil }, false, header.HandshakeIXPSK0)
require.NoError(t, err)
response, _, err := responder.ProcessPacket(nil, packet)
require.NoError(t, err)
responderFingerprint, err := responderCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(responderFingerprint)
initiatingHM := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
initiatingHM.metricRejectedBlocklisted = metrics.NewCounter()
initiatingHM.f = &Interface{l: logger}
hh := &HandshakeHostInfo{
machine: initiator,
hostinfo: &HostInfo{localIndexId: 1, vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}},
}
initiatingHM.indexes[1] = hh
via.IsRelayed = true // skip the remote allow list before certificate validation
initiatingHM.continueHandshake(via, hh, response)
assert.Equal(t, int64(1), initiatingHM.metricRejectedBlocklisted.Count())
}
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
+14 -64
View File
@@ -124,8 +124,6 @@ type Interface struct {
// reused (every udp.Conn ListenOut guarantees that ordering).
batchers []*batch.MultiCoalescer
wg sync.WaitGroup
// readers covers just the listenIn and listenOut goroutines, wg also holds the construction token Close releases
readers sync.WaitGroup
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
@@ -343,18 +341,14 @@ func (f *Interface) activate() error {
func (f *Interface) run() {
// Launch n queues to read packets from udp
for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() {
defer f.readers.Done()
f.listenOut(i)
})
}
// Launch n queues to read packets from tun dev
for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() {
defer f.readers.Done()
f.listenIn(f.queues[i], i)
})
}
@@ -543,12 +537,11 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
curCert = cs.getCertificate(cert.Version1)
}
// Compare with what the running firewall was built from, not the last config, so a firewall that failed to build
// is tried again. The cert is the one this reload brought, PKI registers its reload callback before this one
src := c.Render("firewall")
if src != "" && src == f.firewall.source && curCert != nil &&
slices.Equal(curCert.Networks(), f.firewall.assignedNetworks) &&
slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) {
// The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction.
// Check to see if that set has changed, and if so, rebuild the firewall.
certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks)
if !c.HasChanged("firewall") && !certUnsafeChanged {
f.l.Debug("No firewall config change detected")
return
}
@@ -689,14 +682,6 @@ func (f *Interface) GetCertState() *CertState {
return f.pki.getCertState()
}
// shutdowner is a udp.Conn or overlay.Device whose readers can be woken without releasing their fds
type shutdowner interface {
Shutdown() error
}
// readerShutdownTimeout bounds how long Close waits on woken readers before releasing their fds anyway
const readerShutdownTimeout = 5 * time.Second
// Close releases the interface's resources: the udp sockets and the tun device.
// It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated,
// calls after the first return nil without doing anything.
@@ -706,55 +691,20 @@ func (f *Interface) Close() error {
}
var errs []error
closeUdp := func(i int, u udp.Conn) {
if err := u.Close(); err != nil {
// Release the udp readers
for i, u := range f.writers {
err := u.Close()
if err != nil {
f.l.Error("Error while closing udp socket", "error", err, "writer", i)
errs = append(errs, err)
}
}
closeTun := func() {
if err := f.inside.Close(); err != nil {
errs = append(errs, err)
}
}
// Wake every reader before releasing any fd, one that loops back into a syscall on a reused fd number never wakes.
// Anything without Shutdown is closed right away, as before.
var deferred []func()
for i, u := range f.writers {
if s, ok := u.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
f.l.Error("Error while shutting down udp socket", "error", err, "writer", i)
errs = append(errs, err)
}
deferred = append(deferred, func() { closeUdp(i, u) })
} else {
closeUdp(i, u)
}
}
if s, ok := f.inside.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
errs = append(errs, err)
}
deferred = append(deferred, closeTun)
} else {
closeTun()
}
if len(deferred) > 0 {
readersDone := make(chan struct{})
go func() {
f.readers.Wait()
close(readersDone)
}()
select {
case <-readersDone:
case <-time.After(readerShutdownTimeout):
f.l.Warn("Readers did not stop, closing their fds anyway", "timeout", readerShutdownTimeout)
}
for _, closeFn := range deferred {
closeFn()
}
// Release the tun device (closing the tun also closes all readers)
closeErr := f.inside.Close()
if closeErr != nil {
errs = append(errs, closeErr)
}
// Release the construction token so waiters know the resources are gone
+3 -58
View File
@@ -135,63 +135,8 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
// The running firewall still isn't built from what the config asks for, so the same reload fails again
for range 2 {
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
res := cfg.ReloadConfigString(rules("tcp"))
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.NotSame(t, fw, f.firewall)
}
// Every input the firewall is built from rebuilds it when it changes, and nothing else does
func TestReloadFirewall_RebuildsOnAnyInput(t *testing.T) {
l := test.NewLogger()
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
base := "firewall:\n inbound:\n - port: any\n proto: any\n host: any\n"
setup := func() (*config.C, *Interface, *PKI) {
pki := &PKI{}
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks}, initiatingVersion: cert.Version2})
cfg := config.NewC(l)
require.NoError(t, cfg.LoadString(base))
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
require.NoError(t, err)
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
return cfg, f, pki
}
setCert := func(pki *PKI, networks, unsafe []netip.Prefix) {
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks, unsafeNetworks: unsafe}, initiatingVersion: cert.Version2})
}
cases := map[string]struct {
cfg string
cert func(*PKI)
want bool
}{
"nothing": {base + "punchy:\n punch: true\n", nil, false},
"rules": {base + " outbound:\n - port: any\n proto: any\n host: any\n", nil, true},
"inbound_action": {base + " inbound_action: reject\n", nil, true},
"conntrack": {base + " conntrack:\n tcp_timeout: 1m\n", nil, true},
"networks": {base, func(p *PKI) { setCert(p, []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil) }, true},
"unsafe_networks": {base, func(p *PKI) { setCert(p, networks, []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}) }, true},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
cfg, f, pki := setup()
old := f.firewall
if tc.cert != nil {
tc.cert(pki)
}
res := cfg.ReloadConfigString(tc.cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, tc.want, f.firewall != old)
})
}
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
+3 -5
View File
@@ -452,7 +452,7 @@ func getStaticMapCadence(c *config.C) (time.Duration, error) {
}
func getStaticMapLookupTimeout(c *config.C) (time.Duration, error) {
lookupTimeout := c.GetString("static_map.lookup_timeout", "5s")
lookupTimeout := c.GetString("static_map.lookup_timeout", "250ms")
d, err := time.ParseDuration(lookupTimeout)
if err != nil {
return 0, err
@@ -630,8 +630,7 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
ctx := lh.ctx
lh.Unlock()
// On reload am.hr is the previous resolver, its last known good addresses carry over in case DNS is down right now
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, am.hr, func() {
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, func() {
// This callback runs whenever the DNS hostname resolver finds a different set of addr's
// in its resolution for hostnames.
am.Lock()
@@ -643,8 +642,7 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
}
am.unlockedSetHostnamesResults(hr)
// Only literals belong in the cache, anything from DNS has to stay in hr where a not found answer can remove it
for _, addrPort := range hr.literalAddrs() {
for _, addrPort := range hr.GetAddrs() {
if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) {
continue
}
+3 -13
View File
@@ -20,7 +20,6 @@ type offloadQueueSet struct {
// usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel.
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
usoEnabled bool
shutdown atomic.Bool
closed atomic.Bool
// l is handed to each queue for its bad-vnet-header drop logging.
l *slog.Logger
@@ -70,17 +69,6 @@ func (c *offloadQueueSet) wakeForShutdown() error {
return err
}
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *offloadQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *offloadQueueSet) Close() error {
if c.closed.Swap(true) {
return nil
@@ -88,7 +76,9 @@ func (c *offloadQueueSet) Close() error {
errs := []error{}
if err := c.Shutdown(); err != nil {
// Signal all readers blocked in poll to wake up and exit.
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
errs = append(errs, err)
}
+3 -13
View File
@@ -16,7 +16,6 @@ type pollQueueSet struct {
// pqi is exactly the same as pq, but stored as the interface type
pqi []Queue
shutdownFd int
shutdown atomic.Bool
closed atomic.Bool
}
@@ -60,17 +59,6 @@ func (c *pollQueueSet) wakeForShutdown() error {
return err
}
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *pollQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *pollQueueSet) Close() error {
if c.closed.Swap(true) {
return nil
@@ -78,7 +66,9 @@ func (c *pollQueueSet) Close() error {
errs := []error{}
if err := c.Shutdown(); err != nil {
// Signal all readers blocked in poll to wake up and exit.
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
errs = append(errs, err)
}
-2
View File
@@ -7,8 +7,6 @@ import (
// QueueSet holds one or many Queue objects and helps close them in an orderly way.
type QueueSet interface {
io.Closer
// Shutdown wakes every reader without releasing any fds, Close only once they have exited
Shutdown() error
Queues() []Queue
// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
+1 -6
View File
@@ -66,9 +66,7 @@ type Offload struct {
// usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD,
// so writers can decide whether emitting GSO_UDP_L4 superpackets is safe.
usoEnabled bool
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool
closed atomic.Bool
// gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted
// by WriteGSO. Kept separate from the read-only package-level validVnetHdr
@@ -178,9 +176,6 @@ func (r *Offload) readPacket(block bool) (int, error) {
// This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs).
// Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue.
func (r *Offload) Read() ([]Packet, error) {
if r.shutdown.Load() {
return nil, os.ErrClosed
}
r.pending = r.pending[:0]
r.rxOff = 0
+1 -6
View File
@@ -13,9 +13,7 @@ import (
type Poll struct {
fd int
shutdownFd int
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool
closed atomic.Bool
readBuf []byte
batchRet [1]Packet
@@ -54,9 +52,6 @@ func (t *Poll) blockOnWrite() error {
// config currently lose that batching. blockOn and the EAGAIN plumbing are
// already shared; kept one-packet-per-Read for now to preserve behavior.
func (t *Poll) Read() ([]Packet, error) {
if t.shutdown.Load() {
return nil, os.ErrClosed
}
n, err := t.readOne(t.readBuf)
if err != nil {
return nil, err
-27
View File
@@ -223,30 +223,3 @@ func TestOffloadQueueSet_Close_ClosesShutdownFd(t *testing.T) {
// Second Close must not touch fds (shutdownFd is now -1) and must return nil.
require.NoError(t, qs.Close())
}
// A busy reader never polls the shutdown eventfd, so Read has to stop on its own with data still waiting
func TestQueueSet_ShutdownStopsReaderWithDataWaiting(t *testing.T) {
sets := map[string]func() (QueueSet, error){
"poll": NewPollQueueSet,
"offload": func() (QueueSet, error) { return NewOffloadQueueSet(false, slog.New(slog.DiscardHandler)) },
}
for name, newSet := range sets {
t.Run(name, func(t *testing.T) {
var fds [2]int
require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
t.Cleanup(func() { _ = unix.Close(fds[1]) })
qs, err := newSet()
require.NoError(t, err)
require.NoError(t, qs.Add(fds[0]))
t.Cleanup(func() { _ = qs.Close() })
_, err = unix.Write(fds[1], make([]byte, 64))
require.NoError(t, err)
require.NoError(t, qs.Shutdown())
_, err = qs.Queues()[0].Read()
require.ErrorIs(t, err, os.ErrClosed)
})
}
}
+84 -54
View File
@@ -3,6 +3,7 @@
package overlay
import (
"errors"
"fmt"
"log/slog"
"math/bits"
@@ -11,7 +12,6 @@ import (
"sync"
"sync/atomic"
"syscall"
"time"
"unsafe"
"github.com/gaissmai/bart"
@@ -35,26 +35,6 @@ type tun struct {
// and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool
l *slog.Logger
// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
// fields below it
routesMu sync.Mutex
// routeStates is who held each installable unsafe_route prefix when we last looked
routeStates map[netip.Prefix]routeState
// routeWatch is the routing socket watchRoutes reads, Close closes it
routeWatch *os.File
// routeSweep reconciles the unsafe_routes every routeSweepEvery
routeSweep *time.Timer
routeSweepEvery time.Duration
closed bool
// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
// lock so the route watcher never waits on a reconcile
routeMoversMu sync.Mutex
routeMovers map[netip.Prefix]routeMover
// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
lastRouteEvent atomic.Int64
}
type ifReq struct {
@@ -159,12 +139,11 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
}
t := &tun{
f: os.NewFile(uintptr(fd), ""),
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c),
routeSweepEvery: routeSweepInterval,
l: l,
f: os.NewFile(uintptr(fd), ""),
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c),
l: l,
}
err = t.reload(c, true, nil)
@@ -240,16 +219,6 @@ func utunNameFromFd(fd int) string {
}
func (t *tun) Close() error {
t.routesMu.Lock()
t.closed = true
if t.routeWatch != nil {
_ = t.routeWatch.Close()
}
if t.routeSweep != nil {
t.routeSweep.Stop()
}
t.routesMu.Unlock()
if t.f != nil {
return t.f.Close()
}
@@ -318,22 +287,8 @@ func (t *tun) Activate() error {
return fmt.Errorf("failed to run tun device: %s", err)
}
// Watch before adding so a route removed in between is still noticed
watch, err := openRouteSocket()
if err != nil {
t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
// Unsafe path routes
if err = t.addRoutes(false, nil); err != nil {
if watch != nil {
_ = watch.Close()
}
return err
}
t.startRouteWatch(watch)
return nil
return t.addRoutes(false, nil)
}
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
@@ -438,8 +393,7 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
}
if !initial && !change {
// Any reload puts back the unsafe_routes someone deleted
return t.addRoutes(true, r)
return nil
}
routeTree, err := makeRouteTree(t.l, routes, false)
@@ -557,6 +511,82 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
return nil, nil
}
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
err := addRoute(r.Cidr, linkAddr)
if err != nil {
if errors.Is(err, unix.EEXIST) {
// Every route we added already exists on a reload, only warn about someone else's
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
retErr.Log(t.l)
rl.Failed()
} else {
return retErr
}
}
} else {
t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
}
}
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
linkAddr := t.linkAddr.Load()
if t.hostOwned || linkAddr == nil {
return nil
}
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue
}
err := delRoute(r.Cidr, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
-366
View File
@@ -3,17 +3,12 @@
package overlay
import (
"encoding/binary"
"encoding/hex"
"fmt"
"net"
"net/netip"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
@@ -109,367 +104,6 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
const (
// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
// With no interface address, which x/net/route skips
interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
)
func TestRoutesMayHaveMoved(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
routes := []Route{{Cidr: cidr, Via: via, Install: true}}
// A request like route(8) sends, the kernel echoes it to every routing socket
request := func(typ, flags int, dst, mask [4]byte) []byte {
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: typ,
Flags: flags,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: dst},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
},
}
b, err := m.Marshal()
require.NoError(t, err)
return b
}
raw := func(s string) []byte {
b, err := hex.DecodeString(s)
require.NoError(t, err)
return b
}
lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
routesMayHaveMoved := func(b []byte, routes []Route) bool {
moved, _, _ := routesMayHaveMoved(b, routes)
return moved
}
assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
neighbor := raw(kernelSubnetDelete)
binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
}
// A request names its prefix and who made it, the kernel's own and interface changes name neither
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: unix.RTM_CHANGE,
Flags: unix.RTF_UP,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
},
}
b, err := m.Marshal()
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved)
assert.Equal(t, cidr, p)
assert.Equal(t, int32(1234), pid)
for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
b, err := hex.DecodeString(s)
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved, name)
assert.False(t, p.IsValid(), name)
assert.Zero(t, pid, name)
}
}
func TestProcessName(t *testing.T) {
assert.Equal(t, "launchd", processName(1))
self := filepath.Base(os.Args[0])
assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
assert.Empty(t, processName(-1))
}
// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
// the minute between sweeps when it isn't zero.
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
requireRoot(t)
cfg := "tun:\n unsafe_routes:\n"
for _, cidr := range cidrs {
via := "10.250.0.2"
if netip.MustParsePrefix(cidr).Addr().Is6() {
via = "fd00:250::2"
}
cfg += " - route: " + cidr + "\n via: " + via + "\n"
}
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString(cfg))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
require.NoError(t, err)
if sweep != 0 {
tn.routeSweepEvery = sweep
}
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
return tn, c, cfg
}
func requireRoot(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating interfaces and adding routes needs root")
}
}
// routeVia is the index of the interface that routes cidr, 0 if none does
func routeVia(t *testing.T, cidr string) int {
msgs, err := fetchRoutes()
require.NoError(t, err)
return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
}
// routeStateOf is what tn made of cidr when it last looked
func routeStateOf(tn *tun, cidr string) routeState {
tn.routesMu.Lock()
defer tn.routesMu.Unlock()
return tn.routeStates[netip.MustParsePrefix(cidr)]
}
func routeArgs(action, cidr string, extra ...string) []string {
args := []string{"-q", "-n", action}
if netip.MustParsePrefix(cidr).Addr().Is6() {
args = append(args, "-inet6")
}
return append(append(args, "-net", cidr), extra...)
}
func routeCmd(t *testing.T, action, cidr string, extra ...string) {
requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
}
// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
// says so
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
require.NotContains(t, string(out), "writing to routing socket")
}
// cleanupRoute deletes cidr's route when the test ends, whoever holds it
func cleanupRoute(t *testing.T, cidr string) {
t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
}
// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
func routeThroughLo0(t *testing.T, cidr string) int {
requireRoot(t)
routeCmd(t, "add", cidr, "-interface", "lo0")
cleanupRoute(t, cidr)
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
return lo0.Index
}
// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
t.Run(cidr, func(t *testing.T) {
lo0 := routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
})
}
}
// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
cidr := "10.251.1.0/24"
lo0 := routeThroughLo0(t, cidr)
_, c, _ := newRouteTestTun(t, 0, cidr)
res := c.ReloadConfigString("tun: {}\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, lo0, routeVia(t, cidr))
}
// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
cidr := "10.251.3.0/24"
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
time.Sleep(2 * routeRetryDelay)
require.Equal(t, lo0.Index, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
}
// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
cidr := "10.251.4.0/24"
tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
// A reload with nothing changed is how to ask for it back
res := c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, ours, routeVia(t, cidr))
}
// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
for name, tc := range map[string]struct {
cidr string
sweep time.Duration
misfile func(t *testing.T, tn *tun, cidr string)
}{
// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
}},
// The sweep that caps a table that never settles
"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
tn.reconcileRoutes()
require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
}},
} {
t.Run(name, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, tc.cidr))
routeCmd(t, "delete", tc.cidr)
tc.misfile(t, tn, tc.cidr)
lo0 := routeThroughLo0(t, tc.cidr)
require.Eventually(t, func() bool {
s := routeStateOf(tn, tc.cidr)
return s.kind == routeYielded && s.holder == lo0
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", tc.cidr)
assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
// taken for a hand delete
func TestTunYieldsToARouteChangedAway(t *testing.T) {
for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
t.Run(cidr, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
routeCmd(t, "change", cidr, "-interface", "lo0")
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
require.Equal(t, lo0.Index, routeVia(t, cidr))
require.Eventually(t, func() bool {
s := routeStateOf(tn, cidr)
return s.kind == routeYielded && s.holder == lo0.Index
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
// ethernet configd routed the lan through
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
requireRoot(t)
var feth string
for i := 100; i < 200 && feth == ""; i++ {
if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
feth = fmt.Sprintf("feth%d", i)
}
}
require.NotEmpty(t, feth)
require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
// refuses one named by the feth itself
cidr := "10.251.2.0/24"
routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
cleanupRoute(t, cidr)
ifc, err := net.InterfaceByName(feth)
require.NoError(t, err)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Losing the address first leaves the route, the reconcile finds it still held
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
time.Sleep(2 * routeRetryDelay)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Only RTM_IFINFO says the interface went, its route goes with no message of its own
require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
cidr := "10.251.5.0/24"
routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
tn.routesMu.Lock()
require.NoError(t, tn.routeWatch.Close())
tn.routesMu.Unlock()
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
func TestTunDevNeedsRestart(t *testing.T) {
for name, want := range map[string]bool{
-5
View File
@@ -832,11 +832,6 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
t.routeTree.Store(newTree)
}
// Shutdown wakes the tun readers without releasing their fds, Close releases them
func (t *tun) Shutdown() error {
return t.readers.Shutdown()
}
func (t *tun) Close() error {
t.closeLock.Lock()
defer t.closeLock.Unlock()
-451
View File
@@ -1,451 +0,0 @@
//go:build !ios && !e2e_testing
package overlay
import (
"errors"
"fmt"
"net"
"net/netip"
"os"
"strconv"
"sync/atomic"
"time"
"unsafe"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/util"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
// or until another interface takes the prefix, after which ours follows theirs like any other.
const (
// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
// replaces a route by deleting and adding, looking in between would find the prefix free
routeRetryDelay = time.Second
// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
// a message for each
routeSweepInterval = time.Minute
// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
routeSocketRcvBuf = 1 << 20
)
type routeStateKind int
const (
routeUnknown routeStateKind = iota
// routeInstalled is our route holding the prefix
routeInstalled
// routeYielded is another interface holding the prefix, ours is added once it lets go
routeYielded
// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
// another interface takes the prefix
routeRemoved
// routeFailed is adding our route failing for some other reason, it is retried
routeFailed
)
type routeState struct {
kind routeStateKind
// holder is the index of the interface holding the prefix when kind is routeYielded
holder int
}
// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
// is looked up as the message arrives, route(8) has exited by the time the table settles
type routeMover struct {
pid int32
name string
}
// logArgs appends m to a log line's args, when we know who it was
func (m routeMover) logArgs(args ...any) []any {
if m.pid == 0 {
return args
}
args = append(args, "pid", m.pid)
if m.name != "" {
args = append(args, "process", m.name)
}
return args
}
// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
func processName(pid int32) string {
kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
if err != nil || kp.Proc.P_pid != pid {
return ""
}
return unix.ByteSliceToString(kp.Proc.P_comm[:])
}
// installRoutes returns the routes we put in the system routing table, one per prefix
func installRoutes(routes []Route) []Route {
seen := make(map[netip.Prefix]struct{}, len(routes))
var out []Route
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
p := r.Cidr.Masked()
if _, ok := seen[p]; ok {
continue
}
seen[p] = struct{}{}
out = append(out, r)
}
return out
}
// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
// interface holds its prefix. It runs on start and on every reload.
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
var msgs []netroute.Message
var msgsErr error
fetched := false
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
err := addRoute(p, linkAddr)
if err == nil {
t.l.Info("Added route", "route", r)
states[p] = routeState{kind: routeInstalled}
continue
}
if !errors.Is(err, unix.EEXIST) {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if !logErrors {
return retErr
}
retErr.Log(t.l)
rl.Failed()
states[p] = routeState{kind: routeFailed}
continue
}
// Every route we added already exists on a reload, only warn about someone else's
if !fetched {
msgs, msgsErr = fetchRoutes()
fetched = true
}
if msgsErr != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
states[p] = prev
continue
}
switch holder := routeInterfaceIn(msgs, p); holder {
case linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case 0:
// Removed since the add, the watcher will see to it
states[p] = prev
default:
if prev.kind != routeYielded || prev.holder != holder {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
}
states[p] = routeState{kind: routeYielded, holder: holder}
}
}
t.routeStates = states
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
msgs, err := fetchRoutes()
if err != nil {
var left []netip.Prefix
for _, r := range installRoutes(routes) {
left = append(left, r.Cidr)
}
return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
}
for _, r := range installRoutes(routes) {
p := r.Cidr.Masked()
// Whatever happens next, the prefix is no longer one we know about
delete(t.routeStates, p)
if routeInterfaceIn(msgs, p) != linkAddr.Index {
continue
}
err = delRoute(p, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
// changed since it last looked, it runs every routeSweepEvery.
func (t *tun) reconcileRoutes() {
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return
}
t.routeMoversMu.Lock()
movers := t.routeMovers
t.routeMovers = nil
t.routeMoversMu.Unlock()
msgs, err := fetchRoutes()
if err != nil {
t.l.Error("Failed to read the system routing table", "error", err)
return
}
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
by := movers[p]
holder := routeInterfaceIn(msgs, p)
switch {
case holder == linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case holder != 0:
// Another interface taking a prefix we left alone puts ours back in line behind theirs
if prev.kind != routeYielded || prev.holder != holder {
t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
}
states[p] = routeState{kind: routeYielded, holder: holder}
case prev.kind == routeInstalled:
t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
states[p] = routeState{kind: routeRemoved}
case prev.kind == routeRemoved:
states[p] = prev
default:
err := addRoute(p, linkAddr)
switch {
case err == nil:
if prev.kind == routeYielded {
t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
} else {
t.l.Info("Added route", "route", r)
}
states[p] = routeState{kind: routeInstalled}
case errors.Is(err, unix.EEXIST):
// Taken since the fetch, the next look will see by whom
states[p] = prev
default:
if prev.kind != routeFailed {
util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
}
states[p] = routeState{kind: routeFailed}
}
}
}
t.routeStates = states
}
// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
// routing table settles after anything that may have moved a prefix
func (t *tun) startRouteWatch(watch *os.File) {
t.routesMu.Lock()
defer t.routesMu.Unlock()
if t.closed {
if watch != nil {
_ = watch.Close()
}
return
}
t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
if watch != nil {
t.routeWatch = watch
go t.watchRoutes(watch)
}
}
func (t *tun) sweepRoutes() {
// A table still changing gets reconciled once it settles
if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
t.reconcileRoutes()
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
if !t.closed {
t.routeSweep.Reset(t.routeSweepEvery)
}
}
func openRouteSocket() (*os.File, error) {
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
}
// Fewer drops, the sweep catches the rest
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
// Non-blocking puts it on the poller, so Close interrupts a Read
if err = unix.SetNonblock(fd, true); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
}
return os.NewFile(uintptr(fd), "route"), nil
}
// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
func (t *tun) watchRoutes(f *os.File) {
var reconciled atomic.Int64
settle := time.AfterFunc(time.Hour, func() {
reconciled.Store(time.Now().UnixNano())
t.reconcileRoutes()
})
settle.Stop()
defer settle.Stop()
var burst time.Time
self := int32(os.Getpid())
b := make([]byte, os.Getpagesize())
for {
n, err := f.Read(b)
if err != nil {
if !errors.Is(err, os.ErrClosed) {
t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
return
}
moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
if !moved {
continue
}
// Our own are already in routeStates
if p.IsValid() && pid != self {
t.routeMoversMu.Lock()
if t.routeMovers == nil {
t.routeMovers = make(map[netip.Prefix]routeMover)
}
t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
t.routeMoversMu.Unlock()
}
now := time.Now()
t.lastRouteEvent.Store(now.UnixNano())
if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
burst = now
}
settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
}
}
// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
// and the pid of the process that made it.
func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
if len(b) < 4 {
return false, p, 0
}
switch int(b[3]) {
case unix.RTM_IFINFO, unix.RTM_DELADDR:
return true, p, 0
case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
default:
return false, p, 0
}
if len(b) < unix.SizeofRtMsghdr {
return false, p, 0
}
hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
// A failed request is reported too, and a scoped route never held the prefix
if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
return false, p, 0
}
if hdr.Pid == 0 {
// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
// could be holding a prefix
return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
}
// Replies to a request carry the requester's padded sockaddrs
msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
if err != nil || len(msgs) != 1 {
return false, p, 0
}
m, ok := msgs[0].(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
return false, p, 0
}
p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
if !ok {
return false, netip.Prefix{}, 0
}
p = p.Masked()
for _, r := range installRoutes(routes) {
if r.Cidr.Masked() == p {
return true, p, hdr.Pid
}
}
return false, netip.Prefix{}, 0
}
// interfaceName is the name of the interface with index i, for logs
func interfaceName(i int) string {
if ifc, err := net.InterfaceByIndex(i); err == nil {
return ifc.Name
}
return strconv.Itoa(i)
}
-4
View File
@@ -528,10 +528,6 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
if !rm.GetAmRelay() {
return
}
if !slices.Contains(h.vpnAddrs, from) {
logMsg.Info("Discarding relay request where relayFrom is not the sending peer")
return
}
peer := rm.hostmap.QueryVpnAddr(target)
if peer == nil {
// Try to establish a connection to this host. If we get a future relay request,
-57
View File
@@ -7,10 +7,8 @@ import (
"testing"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestStartRelaysLogDedupe verifies that repeated attempts with the same relay set drop the log
@@ -97,58 +95,3 @@ func TestStartRelaysLogDedupe(t *testing.T) {
assert.NotContains(t, buf.String(), msg, "should not log when we shortcut out")
})
}
// TestHandleCreateRelayRequestRejectsSpoofedFrom verifies that a relay only forwards a CreateRelayRequest
// when relayFrom is the peer that sent it. Otherwise any peer could reset another host's established
// relay back to Requested, or allocate unbounded relay entries on the target's hostinfo.
func TestHandleCreateRelayRequestRejectsSpoofedFrom(t *testing.T) {
l := test.NewLogger()
relayAddr := netip.MustParseAddr("10.128.0.128")
malloryAddr := netip.MustParseAddr("10.128.0.3")
victimAddr := netip.MustParseAddr("10.128.0.1")
targetAddr := netip.MustParseAddr("10.128.0.2")
addrTable := new(bart.Lite)
addrTable.Insert(netip.PrefixFrom(relayAddr, relayAddr.BitLen()))
hm := newHostMap(l)
f := &Interface{myVpnAddrsTable: addrTable, hostMap: hm}
rm := &relayManager{l: l, hostmap: hm}
rm.amRelay.Store(true)
newRelayState := func() RelayState {
return RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}}
}
mallory := &HostInfo{vpnAddrs: []netip.Addr{malloryAddr}, localIndexId: 1, relayState: newRelayState()}
target := &HostInfo{
vpnAddrs: []netip.Addr{targetAddr},
localIndexId: 2,
remotes: NewRemoteList([]netip.Addr{targetAddr}, nil),
relayState: newRelayState(),
}
target.SetRemote(netip.MustParseAddrPort("192.0.2.2:4242"))
hm.unlockedAddHostInfo(mallory, f)
hm.unlockedAddHostInfo(target, f)
// The victim already has a working relay to the target through us.
_, err := AddRelay(l, target, hm, victimAddr, nil, ForwardingType, Established)
require.NoError(t, err)
relaysBefore := len(hm.Relays)
for i := range 10 {
spoofed := netip.AddrFrom4([4]byte{10, 128, 1, byte(i)})
for _, from := range []netip.Addr{victimAddr, spoofed} {
rm.handleCreateRelayRequest(cert.Version2, mallory, f, &NebulaControl{
Type: NebulaControl_CreateRelayRequest,
InitiatorRelayIndex: uint32(100 + i),
RelayFromAddr: netAddrToProtoAddr(from),
RelayToAddr: netAddrToProtoAddr(targetAddr),
})
}
}
r, ok := target.relayState.QueryRelayForByIp(victimAddr)
require.True(t, ok)
assert.Equal(t, Established, r.State, "victim's relay must not be reset by a request from another peer")
assert.Len(t, hm.Relays, relaysBefore, "spoofed requests must not allocate relay entries")
assert.Empty(t, mallory.relayState.CopyAllRelayFor(), "spoofed requests must not create state on the sender")
}
+19 -124
View File
@@ -2,7 +2,6 @@ package nebula
import (
"context"
"errors"
"log/slog"
"net"
"net/netip"
@@ -59,8 +58,6 @@ type cacheV6 struct {
type hostnamePort struct {
name string
port uint16
// addr is set when name is an ip literal, these never need a lookup
addr netip.Addr
}
type hostnamesResults struct {
@@ -68,20 +65,15 @@ type hostnamesResults struct {
network string
lookupTimeout time.Duration
cancelFn func()
resolveNow chan struct{}
l *slog.Logger
ips atomic.Pointer[map[netip.AddrPort]struct{}]
// known is the last known good addresses per hostname, kept so a reload can carry them forward
known atomic.Pointer[map[hostnamePort][]netip.AddrPort]
}
// NewHostnameResults starts resolving hostPorts. Any hostnames also in prev start with its last known good addresses.
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, prev *hostnamesResults, onUpdate func()) (*hostnamesResults, error) {
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, onUpdate func()) (*hostnamesResults, error) {
r := &hostnamesResults{
hostnames: make([]hostnamePort, len(hostPorts)),
network: network,
lookupTimeout: timeout,
resolveNow: make(chan struct{}, 1),
l: l,
}
@@ -89,13 +81,6 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
// DNS lookups for hostnames that aren't hardcoded IP's will happen in a background goroutine.
performBackgroundLookup := false
ips := map[netip.AddrPort]struct{}{}
known := make([][]netip.AddrPort, len(hostPorts))
var prevKnown map[hostnamePort][]netip.AddrPort
if prev != nil && prev.network == network {
if p := prev.known.Load(); p != nil {
prevKnown = *p
}
}
for idx, hostPort := range hostPorts {
rIp, sPort, err := net.SplitHostPort(hostPort)
@@ -113,19 +98,13 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
if err != nil {
// This address is a hostname, not an IP address
performBackgroundLookup = true
known[idx] = prevKnown[r.hostnames[idx]]
for _, a := range known[idx] {
ips[a] = struct{}{}
}
continue
}
// Save the IP address immediately
r.hostnames[idx].addr = addr
ips[netip.AddrPortFrom(addr, uint16(iPort))] = struct{}{}
}
r.ips.Store(&ips)
r.storeKnown(known)
// Time for the DNS lookup goroutine
if performBackgroundLookup {
@@ -135,12 +114,21 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
go func() {
defer ticker.Stop()
for {
r.resolve(newCtx, known)
r.storeKnown(known)
netipAddrs := map[netip.AddrPort]struct{}{}
for _, addrs := range known {
for _, hostPort := range r.hostnames {
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, r.lookupTimeout)
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, r.network, hostPort.name)
timeoutCancel()
if err != nil {
l.Error("DNS resolution failed for static_map host",
"hostname", hostPort.name,
"network", r.network,
"error", err,
)
continue
}
for _, a := range addrs {
netipAddrs[a] = struct{}{}
netipAddrs[netip.AddrPortFrom(a.Unmap(), hostPort.port)] = struct{}{}
}
}
origSet := r.ips.Load()
@@ -171,7 +159,7 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
case <-newCtx.Done():
return
case <-ticker.C:
case <-r.resolveNow:
continue
}
}
}()
@@ -180,84 +168,6 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
return r, nil
}
// resolve looks up every hostname in parallel and folds the results into known
func (hr *hostnamesResults) resolve(ctx context.Context, known [][]netip.AddrPort) {
var wg sync.WaitGroup
for i, hostPort := range hr.hostnames {
if hostPort.addr.IsValid() {
known[i] = []netip.AddrPort{netip.AddrPortFrom(hostPort.addr, hostPort.port)}
continue
}
wg.Go(func() {
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, hr.lookupTimeout)
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, hr.network, hostPort.name)
timeoutCancel()
known[i] = applyLookup(known[i], hostPort.port, addrs, err)
// A canceled ctx means we are reloading or shutting down, not that DNS failed
if err != nil && ctx.Err() == nil {
hr.l.Error("DNS resolution failed for static_map host",
"hostname", hostPort.name,
"network", hr.network,
"error", err,
"usingAddrs", known[i],
)
}
})
}
wg.Wait()
}
// applyLookup returns the addresses to use for a hostname given the previous ones and a lookup result.
// Only a not found answer clears them, a stale lighthouse address costs a handshake but an empty set costs everything.
// darwin also reports not found while offline, that's harmless since nothing is reachable and the next handshake
// triggers a fresh lookup once we're back.
func applyLookup(prev []netip.AddrPort, port uint16, addrs []netip.Addr, err error) []netip.AddrPort {
if err != nil {
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
return nil
}
return prev
}
out := make([]netip.AddrPort, 0, len(addrs))
for _, a := range addrs {
out = append(out, netip.AddrPortFrom(a.Unmap(), port))
}
return out
}
func (hr *hostnamesResults) storeKnown(known [][]netip.AddrPort) {
byName := make(map[hostnamePort][]netip.AddrPort, len(known))
for i, addrs := range known {
byName[hr.hostnames[i]] = addrs
}
hr.known.Store(&byName)
}
// literalAddrs returns the entries that were configured as ip addresses rather than hostnames
func (hr *hostnamesResults) literalAddrs() []netip.AddrPort {
var out []netip.AddrPort
for _, hostPort := range hr.hostnames {
if hostPort.addr.IsValid() {
out = append(out, netip.AddrPortFrom(hostPort.addr, hostPort.port))
}
}
return out
}
// ResolveNow asks the background goroutine to look up the hostnames again without waiting for the next tick
func (hr *hostnamesResults) ResolveNow() {
if hr == nil || hr.resolveNow == nil {
return
}
select {
case hr.resolveNow <- struct{}{}:
default:
}
}
func (hr *hostnamesResults) Cancel() {
if hr != nil && hr.cancelFn != nil {
hr.cancelFn()
@@ -346,13 +256,6 @@ func (r *RemoteList) ResetForOwner(ownerVpnAddr netip.Addr) {
r.shouldRebuild = true
}
// ResolveHostnames kicks a fresh DNS lookup of any static_map hostnames for this host
func (r *RemoteList) ResolveHostnames() {
r.RLock()
defer r.RUnlock()
r.hr.ResolveNow()
}
// ClearHostnameResults cancels the in-flight DNS resolver goroutine (if any) and drops the resolved IP cache.
func (r *RemoteList) ClearHostnameResults() {
r.Lock()
@@ -557,14 +460,10 @@ func (r *RemoteList) unlockedSetV4(ownerVpnIp, vpnIp netip.Addr, to []*V4AddrPor
// Reset the slice
c.reported = c.reported[:0]
// We can't take their array but we can take their pointers.
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
for _, v := range to {
// We can't take their array but we can take their pointers
for _, v := range to[:minInt(len(to), MaxRemotes)] {
if check(vpnIp, v) {
c.reported = append(c.reported, v)
if len(c.reported) == MaxRemotes {
break
}
}
}
}
@@ -609,14 +508,10 @@ func (r *RemoteList) unlockedSetV6(ownerVpnIp, vpnIp netip.Addr, to []*V6AddrPor
// Reset the slice
c.reported = c.reported[:0]
// We can't take their array but we can take their pointers.
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
for _, v := range to {
// We can't take their array but we can take their pointers
for _, v := range to[:minInt(len(to), MaxRemotes)] {
if check(vpnIp, v) {
c.reported = append(c.reported, v)
if len(c.reported) == MaxRemotes {
break
}
}
}
}
-89
View File
@@ -1,15 +1,10 @@
package nebula
import (
"context"
"encoding/binary"
"fmt"
"net"
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -333,55 +328,6 @@ func BenchmarkSortRebuild(b *testing.B) {
})
}
func TestApplyLookup(t *testing.T) {
prev := []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}
// A successful lookup replaces the previous addresses
got := applyLookup(prev, 4242, []netip.Addr{netip.MustParseAddr("::ffff:2.2.2.2")}, nil)
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("2.2.2.2:4242")}, got)
// A not found answer means the record is gone
assert.Empty(t, applyLookup(prev, 4242, nil, &net.DNSError{Err: "no such host", IsNotFound: true}))
// Anything else is not new information, keep what we had
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "i/o timeout", IsTimeout: true}))
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "server misbehaving", IsTemporary: true}))
assert.Equal(t, prev, applyLookup(prev, 4242, nil, context.DeadlineExceeded))
}
func TestNewHostnameResults_CarriesKnown(t *testing.T) {
// A canceled context keeps the background lookup from learning anything, as if DNS were down
ctx, cancel := context.WithCancel(context.Background())
cancel()
known := netip.MustParseAddrPort("1.1.1.1:4242")
prev := &hostnamesResults{network: "ip4"}
prev.known.Store(&map[hostnamePort][]netip.AddrPort{{name: "lh.invalid", port: 4242}: {known}})
hostPorts := []string{"lh.invalid:4242", "10.0.0.1:4242"}
literal := netip.MustParseAddrPort("10.0.0.1:4242")
hr, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, prev, func() {})
require.NoError(t, err)
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr.GetAddrs())
assert.Equal(t, []netip.AddrPort{literal}, hr.literalAddrs())
// And again from the new one, before it has finished a lookup of its own
hr2, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, hr, func() {})
require.NoError(t, err)
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr2.GetAddrs())
// A port change is a different entry
hr3, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, []string{"lh.invalid:4243"}, prev, func() {})
require.NoError(t, err)
assert.Empty(t, hr3.GetAddrs())
// So is a network change
hr4, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip6", time.Second, hostPorts, prev, func() {})
require.NoError(t, err)
assert.Equal(t, []netip.AddrPort{literal}, hr4.GetAddrs())
}
func newIp4AndPortFromString(s string) *V4AddrPort {
a := netip.MustParseAddrPort(s)
v4Addr := a.Addr().As4()
@@ -400,38 +346,3 @@ func newIp6AndPortFromString(s string) *V6AddrPort {
Port: uint32(a.Port()),
}
}
func TestRemoteList_SetFiltersBeforeCap(t *testing.T) {
owner := netip.MustParseAddr("0.0.0.0")
rl := NewRemoteList([]netip.Addr{owner}, nil)
// MaxRemotes denied addrs first, then the only allowed one, then enough allowed to exceed the cap
var v4 []*V4AddrPort
for i := 0; i < MaxRemotes; i++ {
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("172.17.0.%d:4242", i+1)))
}
for i := 0; i < MaxRemotes+1; i++ {
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("70.199.182.%d:4242", i+1)))
}
rl.unlockedSetV4(owner, owner, v4, func(_ netip.Addr, a *V4AddrPort) bool {
return protoV4AddrPortToNetAddrPort(a).Addr().As4()[0] != 172
})
var v6 []*V6AddrPort
for i := 0; i < MaxRemotes; i++ {
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[fd00::%d]:4242", i+1)))
}
for i := 0; i < MaxRemotes+1; i++ {
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[2001:db8::%d]:4242", i+1)))
}
rl.unlockedSetV6(owner, owner, v6, func(_ netip.Addr, a *V6AddrPort) bool {
return protoV6AddrPortToNetAddrPort(a).Addr().As16()[0] != 0xfd
})
rl.Rebuild([]netip.Prefix{})
assert.Len(t, rl.addrs, MaxRemotes*2)
for _, a := range rl.addrs {
assert.NotEqual(t, byte(172), a.Addr().As16()[12], "denied v4 addr %v was kept", a)
assert.NotEqual(t, byte(0xfd), a.Addr().As16()[0], "denied v6 addr %v was kept", a)
}
}
+5 -13
View File
@@ -255,10 +255,6 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
msgs, buffers, names, _ := prepareRawMessages(u.batch, bufSize, cmsgSpace)
for {
// A shut down socket still queues and returns datagrams, a busy reader would never see an empty read
if u.closed.Load() {
return net.ErrClosed
}
if cmsgSpace > 0 {
for i := range msgs {
setMsgControllen(&msgs[i].Hdr, cmsgSpace)
@@ -450,16 +446,12 @@ func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
return nil
}
// Shutdown wakes the reader without releasing the fd, Close only once it has exited
func (u *StdConn) Shutdown() error {
u.closed.Store(true)
// shutdown(2) on an unconnected socket returns ENOTCONN but still wakes the reader, so ignore the error
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
return nil
}
func (u *StdConn) Close() error {
_ = u.Shutdown()
u.closed.Store(true)
// Wake the reader parked in recvmmsg. shutdown(2) on an unconnected socket
// returns ENOTCONN but still wakes it, so ignore the error.
// The reader then sees closed and stops touching the fd, making the Close below safe.
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
return unix.Close(u.sysFd)
}
+5 -74
View File
@@ -76,7 +76,7 @@ func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
}
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
// reader must tear down cleanly on Shutdown() regardless of what the socket has
// reader must tear down cleanly on Close() regardless of what the socket has
// carried. The three cases the report called out:
//
// no traffic ever -> works (shutdown wakes recvmmsg with n==0)
@@ -162,12 +162,11 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
// empty queue with the socket already having received a packet.
time.Sleep(500 * time.Millisecond)
// Tear down the way Interface.Close does, the fd stays open until the reader is gone
start := time.Now()
if err := sc.Shutdown(); err != nil {
t.Fatalf("Shutdown: %v", err)
if err := sc.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
defer func() { _ = sc.Close() }()
close(stop)
select {
case err := <-loopDone:
@@ -180,75 +179,7 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
case <-time.After(3 * time.Second):
buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true)
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Shutdown\n%s", name, buf[:n])
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
}
close(stop)
<-trafficDone
}
// A shut down socket still returns queued datagrams, so a reader that is behind has to stop on its own within the
// batch it was working on
func TestListenOutShutdown_BusyReader(t *testing.T) {
const batch = 64
udpSettings := Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Multi: true, Batch: batch}
c, err := NewListener(testLogger(), udpSettings)
if err != nil {
t.Fatalf("NewListener: %v", err)
}
sc := c.(*StdConn)
defer sc.Close()
addr, err := sc.LocalAddr()
if err != nil {
t.Fatalf("LocalAddr: %v", err)
}
stop := make(chan struct{})
defer close(stop)
for range 8 {
send, err := net.Dial("udp", addr.String())
if err != nil {
t.Fatalf("dial: %v", err)
}
defer send.Close()
go func() {
b := make([]byte, 1200)
for {
select {
case <-stop:
return
default:
_, _ = send.Write(b)
}
}
}()
}
var afterShutdown atomic.Int64
loopDone := make(chan error, 1)
go func() {
loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
if sc.closed.Load() {
afterShutdown.Add(1)
}
// Slower than the senders so the queue never drains
for start := time.Now(); time.Since(start) < 200*time.Microsecond; {
}
}, func() {})
}()
time.Sleep(200 * time.Millisecond)
if err := sc.Shutdown(); err != nil {
t.Fatalf("Shutdown: %v", err)
}
select {
case err := <-loopDone:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("ListenOut returned %v, want net.ErrClosed", err)
}
case <-time.After(3 * time.Second):
t.Fatalf("ListenOut still running 3s after Shutdown, delivered %d packets since", afterShutdown.Load())
}
if n := afterShutdown.Load(); n > batch {
t.Fatalf("delivered %d packets after Shutdown, more than the one batch in flight", n)
}
}