Compare commits

...
Author SHA1 Message Date
Nate Brown a274c7b9bd Keep static_host_map DNS results when a lookup fails 2026-10-21 09:28:03 -05:00
John Maguire 9a5bdb508e Apply remote_allow_list before capping addrs at MaxRemotes (#1985)
smoke-extra / freebsd-amd64 (push) Failing after 16s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 17s
smoke-extra / netbsd-amd64 (push) Failing after 17s
smoke-extra / openbsd-amd64 (push) Failing after 17s
smoke-extra / linux-386 (push) Failing after 18s
smoke / Run multi node smoke test (push) Failing after 1m43s
Build and test / Static checks (push) Successful in 48s
Build and test / Test linux (push) Failing after 1m28s
Build and test / Test linux-pkcs11 (push) Failing after 2m18s
Build and test / Test linux-boringcrypto (push) Failing after 2m59s
Build and test / Test linux-fips140 (push) Failing after 3m3s
Build and test / Cross-build linux-arm (push) Successful in 3m9s
Build and test / Cross-build linux-mips (push) Successful in 3m56s
Build and test / Cross-build linux-other (push) Successful in 3m15s
Build and test / Cross-build windows (push) Successful in 1m5s
Build and test / Cross-build freebsd (push) Successful in 1m36s
Build and test / Cross-build netbsd (push) Successful in 1m34s
Build and test / Cross-build openbsd (push) Successful in 1m33s
Build and test / Cross-build mobile (push) Successful in 3m34s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
smoke-extra / Run windows smoke test (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
unlockedSetV4/V6 truncated the reported list to MaxRemotes and only then
ran the allow list check. A host reporting more than MaxRemotes addrs
(docker bridges, temporary v6 addrs, etc) could have every allowed addr
discarded if the denied ones sorted first. Filter the full list and stop
once MaxRemotes allowed addrs are collected. This affects lighthouses
processing HostUpdateNotification and clients processing HostQueryReply.
2026-10-20 22:08:55 -04:00
Nate Brown 48ba879198 Wait for readers to exit before closing their fds on linux (#1979)
smoke-extra / freebsd-amd64 (push) Failing after 16s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 17s
smoke-extra / netbsd-amd64 (push) Failing after 17s
smoke-extra / openbsd-amd64 (push) Failing after 16s
smoke-extra / linux-386 (push) Failing after 15s
smoke / Run multi node smoke test (push) Failing after 1m40s
Build and test / Static checks (push) Successful in 47s
Build and test / Test linux (push) Failing after 1m52s
Build and test / Test linux-pkcs11 (push) Failing after 2m7s
Build and test / Test linux-boringcrypto (push) Failing after 3m3s
Build and test / Test linux-fips140 (push) Failing after 2m57s
Build and test / Cross-build linux-arm (push) Successful in 3m2s
Build and test / Cross-build linux-mips (push) Successful in 3m42s
Build and test / Cross-build linux-other (push) Successful in 3m12s
Build and test / Cross-build windows (push) Successful in 1m4s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m32s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m24s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-20 14:27:31 -04:00
brad-defined fb2c153dfa Relays always speak directly with peers; enforced RelayRequest msgs (#1980)
* Relays always speak directly with peers; enforce this in RelayRequest messages
2026-10-20 14:00:46 -04:00
17 changed files with 500 additions and 50 deletions

No files matched your search

+3 -2
View File
@@ -39,8 +39,9 @@ static_host_map:
# public address. Other valid options are "ip6" and "ip" (returns both.) # public address. Other valid options are "ip6" and "ip" (returns both.)
#network: ip4 #network: ip4
# lookup_timeout is the DNS query timeout. # lookup_timeout is the DNS query timeout. A failed lookup keeps the last known good addresses, only a not found
#lookup_timeout: 250ms # answer removes them.
#lookup_timeout: 5s
lighthouse: lighthouse:
# am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes # am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
+2
View File
@@ -405,6 +405,8 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
hm.Unlock() hm.Unlock()
hm.lightHouse.QueryServer(vpnAddr) hm.lightHouse.QueryServer(vpnAddr)
// If this host came from a static_map hostname its addresses may have moved, retries will pick up anything new
hostinfo.remotes.ResolveHostnames()
return hostinfo return hostinfo
} }
+58 -9
View File
@@ -124,6 +124,8 @@ type Interface struct {
// reused (every udp.Conn ListenOut guarantees that ordering). // reused (every udp.Conn ListenOut guarantees that ordering).
batchers []*batch.MultiCoalescer batchers []*batch.MultiCoalescer
wg sync.WaitGroup wg sync.WaitGroup
// readers covers just the listenIn and listenOut goroutines, wg also holds the construction token Close releases
readers sync.WaitGroup
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
@@ -341,14 +343,18 @@ func (f *Interface) activate() error {
func (f *Interface) run() { func (f *Interface) run() {
// Launch n queues to read packets from udp // Launch n queues to read packets from udp
for i := 0; i < f.routines; i++ { for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() { f.wg.Go(func() {
defer f.readers.Done()
f.listenOut(i) f.listenOut(i)
}) })
} }
// Launch n queues to read packets from tun dev // Launch n queues to read packets from tun dev
for i := 0; i < f.routines; i++ { for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() { f.wg.Go(func() {
defer f.readers.Done()
f.listenIn(f.queues[i], i) f.listenIn(f.queues[i], i)
}) })
} }
@@ -683,6 +689,14 @@ func (f *Interface) GetCertState() *CertState {
return f.pki.getCertState() return f.pki.getCertState()
} }
// shutdowner is a udp.Conn or overlay.Device whose readers can be woken without releasing their fds
type shutdowner interface {
Shutdown() error
}
// readerShutdownTimeout bounds how long Close waits on woken readers before releasing their fds anyway
const readerShutdownTimeout = 5 * time.Second
// Close releases the interface's resources: the udp sockets and the tun device. // Close releases the interface's resources: the udp sockets and the tun device.
// It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated, // It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated,
// calls after the first return nil without doing anything. // calls after the first return nil without doing anything.
@@ -692,20 +706,55 @@ func (f *Interface) Close() error {
} }
var errs []error var errs []error
closeUdp := func(i int, u udp.Conn) {
// Release the udp readers if err := u.Close(); err != nil {
for i, u := range f.writers {
err := u.Close()
if err != nil {
f.l.Error("Error while closing udp socket", "error", err, "writer", i) f.l.Error("Error while closing udp socket", "error", err, "writer", i)
errs = append(errs, err) errs = append(errs, err)
} }
} }
closeTun := func() {
if err := f.inside.Close(); err != nil {
errs = append(errs, err)
}
}
// Release the tun device (closing the tun also closes all readers) // Wake every reader before releasing any fd, one that loops back into a syscall on a reused fd number never wakes.
closeErr := f.inside.Close() // Anything without Shutdown is closed right away, as before.
if closeErr != nil { var deferred []func()
errs = append(errs, closeErr) for i, u := range f.writers {
if s, ok := u.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
f.l.Error("Error while shutting down udp socket", "error", err, "writer", i)
errs = append(errs, err)
}
deferred = append(deferred, func() { closeUdp(i, u) })
} else {
closeUdp(i, u)
}
}
if s, ok := f.inside.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
errs = append(errs, err)
}
deferred = append(deferred, closeTun)
} else {
closeTun()
}
if len(deferred) > 0 {
readersDone := make(chan struct{})
go func() {
f.readers.Wait()
close(readersDone)
}()
select {
case <-readersDone:
case <-time.After(readerShutdownTimeout):
f.l.Warn("Readers did not stop, closing their fds anyway", "timeout", readerShutdownTimeout)
}
for _, closeFn := range deferred {
closeFn()
}
} }
// Release the construction token so waiters know the resources are gone // Release the construction token so waiters know the resources are gone
+5 -3
View File
@@ -452,7 +452,7 @@ func getStaticMapCadence(c *config.C) (time.Duration, error) {
} }
func getStaticMapLookupTimeout(c *config.C) (time.Duration, error) { func getStaticMapLookupTimeout(c *config.C) (time.Duration, error) {
lookupTimeout := c.GetString("static_map.lookup_timeout", "250ms") lookupTimeout := c.GetString("static_map.lookup_timeout", "5s")
d, err := time.ParseDuration(lookupTimeout) d, err := time.ParseDuration(lookupTimeout)
if err != nil { if err != nil {
return 0, err return 0, err
@@ -630,7 +630,8 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
ctx := lh.ctx ctx := lh.ctx
lh.Unlock() lh.Unlock()
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, func() { // On reload am.hr is the previous resolver, its last known good addresses carry over in case DNS is down right now
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, am.hr, func() {
// This callback runs whenever the DNS hostname resolver finds a different set of addr's // This callback runs whenever the DNS hostname resolver finds a different set of addr's
// in its resolution for hostnames. // in its resolution for hostnames.
am.Lock() am.Lock()
@@ -642,7 +643,8 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
} }
am.unlockedSetHostnamesResults(hr) am.unlockedSetHostnamesResults(hr)
for _, addrPort := range hr.GetAddrs() { // Only literals belong in the cache, anything from DNS has to stay in hr where a not found answer can remove it
for _, addrPort := range hr.literalAddrs() {
if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) { if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) {
continue continue
} }
+13 -3
View File
@@ -20,6 +20,7 @@ type offloadQueueSet struct {
// usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel. // usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel.
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission. // Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
usoEnabled bool usoEnabled bool
shutdown atomic.Bool
closed atomic.Bool closed atomic.Bool
// l is handed to each queue for its bad-vnet-header drop logging. // l is handed to each queue for its bad-vnet-header drop logging.
l *slog.Logger l *slog.Logger
@@ -69,6 +70,17 @@ func (c *offloadQueueSet) wakeForShutdown() error {
return err return err
} }
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *offloadQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *offloadQueueSet) Close() error { func (c *offloadQueueSet) Close() error {
if c.closed.Swap(true) { if c.closed.Swap(true) {
return nil return nil
@@ -76,9 +88,7 @@ func (c *offloadQueueSet) Close() error {
errs := []error{} errs := []error{}
// Signal all readers blocked in poll to wake up and exit. if err := c.Shutdown(); err != nil {
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
errs = append(errs, err) errs = append(errs, err)
} }
+13 -3
View File
@@ -16,6 +16,7 @@ type pollQueueSet struct {
// pqi is exactly the same as pq, but stored as the interface type // pqi is exactly the same as pq, but stored as the interface type
pqi []Queue pqi []Queue
shutdownFd int shutdownFd int
shutdown atomic.Bool
closed atomic.Bool closed atomic.Bool
} }
@@ -59,6 +60,17 @@ func (c *pollQueueSet) wakeForShutdown() error {
return err return err
} }
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *pollQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *pollQueueSet) Close() error { func (c *pollQueueSet) Close() error {
if c.closed.Swap(true) { if c.closed.Swap(true) {
return nil return nil
@@ -66,9 +78,7 @@ func (c *pollQueueSet) Close() error {
errs := []error{} errs := []error{}
// Signal all readers blocked in poll to wake up and exit. if err := c.Shutdown(); err != nil {
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
errs = append(errs, err) errs = append(errs, err)
} }
+2
View File
@@ -7,6 +7,8 @@ import (
// QueueSet holds one or many Queue objects and helps close them in an orderly way. // QueueSet holds one or many Queue objects and helps close them in an orderly way.
type QueueSet interface { type QueueSet interface {
io.Closer io.Closer
// Shutdown wakes every reader without releasing any fds, Close only once they have exited
Shutdown() error
Queues() []Queue Queues() []Queue
// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue. // Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
+6 -1
View File
@@ -66,7 +66,9 @@ type Offload struct {
// usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD, // usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD,
// so writers can decide whether emitting GSO_UDP_L4 superpackets is safe. // so writers can decide whether emitting GSO_UDP_L4 superpackets is safe.
usoEnabled bool usoEnabled bool
closed atomic.Bool // shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool
// gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted // gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted
// by WriteGSO. Kept separate from the read-only package-level validVnetHdr // by WriteGSO. Kept separate from the read-only package-level validVnetHdr
@@ -176,6 +178,9 @@ func (r *Offload) readPacket(block bool) (int, error) {
// This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs). // This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs).
// Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue. // Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue.
func (r *Offload) Read() ([]Packet, error) { func (r *Offload) Read() ([]Packet, error) {
if r.shutdown.Load() {
return nil, os.ErrClosed
}
r.pending = r.pending[:0] r.pending = r.pending[:0]
r.rxOff = 0 r.rxOff = 0
+6 -1
View File
@@ -13,7 +13,9 @@ import (
type Poll struct { type Poll struct {
fd int fd int
shutdownFd int shutdownFd int
closed atomic.Bool // shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool
readBuf []byte readBuf []byte
batchRet [1]Packet batchRet [1]Packet
@@ -52,6 +54,9 @@ func (t *Poll) blockOnWrite() error {
// config currently lose that batching. blockOn and the EAGAIN plumbing are // config currently lose that batching. blockOn and the EAGAIN plumbing are
// already shared; kept one-packet-per-Read for now to preserve behavior. // already shared; kept one-packet-per-Read for now to preserve behavior.
func (t *Poll) Read() ([]Packet, error) { func (t *Poll) Read() ([]Packet, error) {
if t.shutdown.Load() {
return nil, os.ErrClosed
}
n, err := t.readOne(t.readBuf) n, err := t.readOne(t.readBuf)
if err != nil { if err != nil {
return nil, err return nil, err
+27
View File
@@ -223,3 +223,30 @@ func TestOffloadQueueSet_Close_ClosesShutdownFd(t *testing.T) {
// Second Close must not touch fds (shutdownFd is now -1) and must return nil. // Second Close must not touch fds (shutdownFd is now -1) and must return nil.
require.NoError(t, qs.Close()) require.NoError(t, qs.Close())
} }
// A busy reader never polls the shutdown eventfd, so Read has to stop on its own with data still waiting
func TestQueueSet_ShutdownStopsReaderWithDataWaiting(t *testing.T) {
sets := map[string]func() (QueueSet, error){
"poll": NewPollQueueSet,
"offload": func() (QueueSet, error) { return NewOffloadQueueSet(false, slog.New(slog.DiscardHandler)) },
}
for name, newSet := range sets {
t.Run(name, func(t *testing.T) {
var fds [2]int
require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
t.Cleanup(func() { _ = unix.Close(fds[1]) })
qs, err := newSet()
require.NoError(t, err)
require.NoError(t, qs.Add(fds[0]))
t.Cleanup(func() { _ = qs.Close() })
_, err = unix.Write(fds[1], make([]byte, 64))
require.NoError(t, err)
require.NoError(t, qs.Shutdown())
_, err = qs.Queues()[0].Read()
require.ErrorIs(t, err, os.ErrClosed)
})
}
}
+5
View File
@@ -832,6 +832,11 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
t.routeTree.Store(newTree) t.routeTree.Store(newTree)
} }
// Shutdown wakes the tun readers without releasing their fds, Close releases them
func (t *tun) Shutdown() error {
return t.readers.Shutdown()
}
func (t *tun) Close() error { func (t *tun) Close() error {
t.closeLock.Lock() t.closeLock.Lock()
defer t.closeLock.Unlock() defer t.closeLock.Unlock()
+4
View File
@@ -528,6 +528,10 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
if !rm.GetAmRelay() { if !rm.GetAmRelay() {
return return
} }
if !slices.Contains(h.vpnAddrs, from) {
logMsg.Info("Discarding relay request where relayFrom is not the sending peer")
return
}
peer := rm.hostmap.QueryVpnAddr(target) peer := rm.hostmap.QueryVpnAddr(target)
if peer == nil { if peer == nil {
// Try to establish a connection to this host. If we get a future relay request, // Try to establish a connection to this host. If we get a future relay request,
+57
View File
@@ -7,8 +7,10 @@ import (
"testing" "testing"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
// TestStartRelaysLogDedupe verifies that repeated attempts with the same relay set drop the log // TestStartRelaysLogDedupe verifies that repeated attempts with the same relay set drop the log
@@ -95,3 +97,58 @@ func TestStartRelaysLogDedupe(t *testing.T) {
assert.NotContains(t, buf.String(), msg, "should not log when we shortcut out") assert.NotContains(t, buf.String(), msg, "should not log when we shortcut out")
}) })
} }
// TestHandleCreateRelayRequestRejectsSpoofedFrom verifies that a relay only forwards a CreateRelayRequest
// when relayFrom is the peer that sent it. Otherwise any peer could reset another host's established
// relay back to Requested, or allocate unbounded relay entries on the target's hostinfo.
func TestHandleCreateRelayRequestRejectsSpoofedFrom(t *testing.T) {
l := test.NewLogger()
relayAddr := netip.MustParseAddr("10.128.0.128")
malloryAddr := netip.MustParseAddr("10.128.0.3")
victimAddr := netip.MustParseAddr("10.128.0.1")
targetAddr := netip.MustParseAddr("10.128.0.2")
addrTable := new(bart.Lite)
addrTable.Insert(netip.PrefixFrom(relayAddr, relayAddr.BitLen()))
hm := newHostMap(l)
f := &Interface{myVpnAddrsTable: addrTable, hostMap: hm}
rm := &relayManager{l: l, hostmap: hm}
rm.amRelay.Store(true)
newRelayState := func() RelayState {
return RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}}
}
mallory := &HostInfo{vpnAddrs: []netip.Addr{malloryAddr}, localIndexId: 1, relayState: newRelayState()}
target := &HostInfo{
vpnAddrs: []netip.Addr{targetAddr},
localIndexId: 2,
remotes: NewRemoteList([]netip.Addr{targetAddr}, nil),
relayState: newRelayState(),
}
target.SetRemote(netip.MustParseAddrPort("192.0.2.2:4242"))
hm.unlockedAddHostInfo(mallory, f)
hm.unlockedAddHostInfo(target, f)
// The victim already has a working relay to the target through us.
_, err := AddRelay(l, target, hm, victimAddr, nil, ForwardingType, Established)
require.NoError(t, err)
relaysBefore := len(hm.Relays)
for i := range 10 {
spoofed := netip.AddrFrom4([4]byte{10, 128, 1, byte(i)})
for _, from := range []netip.Addr{victimAddr, spoofed} {
rm.handleCreateRelayRequest(cert.Version2, mallory, f, &NebulaControl{
Type: NebulaControl_CreateRelayRequest,
InitiatorRelayIndex: uint32(100 + i),
RelayFromAddr: netAddrToProtoAddr(from),
RelayToAddr: netAddrToProtoAddr(targetAddr),
})
}
}
r, ok := target.relayState.QueryRelayForByIp(victimAddr)
require.True(t, ok)
assert.Equal(t, Established, r.State, "victim's relay must not be reset by a request from another peer")
assert.Len(t, hm.Relays, relaysBefore, "spoofed requests must not allocate relay entries")
assert.Empty(t, mallory.relayState.CopyAllRelayFor(), "spoofed requests must not create state on the sender")
}
+124 -19
View File
@@ -2,6 +2,7 @@ package nebula
import ( import (
"context" "context"
"errors"
"log/slog" "log/slog"
"net" "net"
"net/netip" "net/netip"
@@ -58,6 +59,8 @@ type cacheV6 struct {
type hostnamePort struct { type hostnamePort struct {
name string name string
port uint16 port uint16
// addr is set when name is an ip literal, these never need a lookup
addr netip.Addr
} }
type hostnamesResults struct { type hostnamesResults struct {
@@ -65,15 +68,20 @@ type hostnamesResults struct {
network string network string
lookupTimeout time.Duration lookupTimeout time.Duration
cancelFn func() cancelFn func()
resolveNow chan struct{}
l *slog.Logger l *slog.Logger
ips atomic.Pointer[map[netip.AddrPort]struct{}] ips atomic.Pointer[map[netip.AddrPort]struct{}]
// known is the last known good addresses per hostname, kept so a reload can carry them forward
known atomic.Pointer[map[hostnamePort][]netip.AddrPort]
} }
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, onUpdate func()) (*hostnamesResults, error) { // NewHostnameResults starts resolving hostPorts. Any hostnames also in prev start with its last known good addresses.
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, prev *hostnamesResults, onUpdate func()) (*hostnamesResults, error) {
r := &hostnamesResults{ r := &hostnamesResults{
hostnames: make([]hostnamePort, len(hostPorts)), hostnames: make([]hostnamePort, len(hostPorts)),
network: network, network: network,
lookupTimeout: timeout, lookupTimeout: timeout,
resolveNow: make(chan struct{}, 1),
l: l, l: l,
} }
@@ -81,6 +89,13 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
// DNS lookups for hostnames that aren't hardcoded IP's will happen in a background goroutine. // DNS lookups for hostnames that aren't hardcoded IP's will happen in a background goroutine.
performBackgroundLookup := false performBackgroundLookup := false
ips := map[netip.AddrPort]struct{}{} ips := map[netip.AddrPort]struct{}{}
known := make([][]netip.AddrPort, len(hostPorts))
var prevKnown map[hostnamePort][]netip.AddrPort
if prev != nil && prev.network == network {
if p := prev.known.Load(); p != nil {
prevKnown = *p
}
}
for idx, hostPort := range hostPorts { for idx, hostPort := range hostPorts {
rIp, sPort, err := net.SplitHostPort(hostPort) rIp, sPort, err := net.SplitHostPort(hostPort)
@@ -98,13 +113,19 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
if err != nil { if err != nil {
// This address is a hostname, not an IP address // This address is a hostname, not an IP address
performBackgroundLookup = true performBackgroundLookup = true
known[idx] = prevKnown[r.hostnames[idx]]
for _, a := range known[idx] {
ips[a] = struct{}{}
}
continue continue
} }
// Save the IP address immediately // Save the IP address immediately
r.hostnames[idx].addr = addr
ips[netip.AddrPortFrom(addr, uint16(iPort))] = struct{}{} ips[netip.AddrPortFrom(addr, uint16(iPort))] = struct{}{}
} }
r.ips.Store(&ips) r.ips.Store(&ips)
r.storeKnown(known)
// Time for the DNS lookup goroutine // Time for the DNS lookup goroutine
if performBackgroundLookup { if performBackgroundLookup {
@@ -114,21 +135,12 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
go func() { go func() {
defer ticker.Stop() defer ticker.Stop()
for { for {
r.resolve(newCtx, known)
r.storeKnown(known)
netipAddrs := map[netip.AddrPort]struct{}{} netipAddrs := map[netip.AddrPort]struct{}{}
for _, hostPort := range r.hostnames { for _, addrs := range known {
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, r.lookupTimeout)
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, r.network, hostPort.name)
timeoutCancel()
if err != nil {
l.Error("DNS resolution failed for static_map host",
"hostname", hostPort.name,
"network", r.network,
"error", err,
)
continue
}
for _, a := range addrs { for _, a := range addrs {
netipAddrs[netip.AddrPortFrom(a.Unmap(), hostPort.port)] = struct{}{} netipAddrs[a] = struct{}{}
} }
} }
origSet := r.ips.Load() origSet := r.ips.Load()
@@ -159,7 +171,7 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
case <-newCtx.Done(): case <-newCtx.Done():
return return
case <-ticker.C: case <-ticker.C:
continue case <-r.resolveNow:
} }
} }
}() }()
@@ -168,6 +180,84 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
return r, nil return r, nil
} }
// resolve looks up every hostname in parallel and folds the results into known
func (hr *hostnamesResults) resolve(ctx context.Context, known [][]netip.AddrPort) {
var wg sync.WaitGroup
for i, hostPort := range hr.hostnames {
if hostPort.addr.IsValid() {
known[i] = []netip.AddrPort{netip.AddrPortFrom(hostPort.addr, hostPort.port)}
continue
}
wg.Go(func() {
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, hr.lookupTimeout)
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, hr.network, hostPort.name)
timeoutCancel()
known[i] = applyLookup(known[i], hostPort.port, addrs, err)
// A canceled ctx means we are reloading or shutting down, not that DNS failed
if err != nil && ctx.Err() == nil {
hr.l.Error("DNS resolution failed for static_map host",
"hostname", hostPort.name,
"network", hr.network,
"error", err,
"usingAddrs", known[i],
)
}
})
}
wg.Wait()
}
// applyLookup returns the addresses to use for a hostname given the previous ones and a lookup result.
// Only a not found answer clears them, a stale lighthouse address costs a handshake but an empty set costs everything.
// darwin also reports not found while offline, that's harmless since nothing is reachable and the next handshake
// triggers a fresh lookup once we're back.
func applyLookup(prev []netip.AddrPort, port uint16, addrs []netip.Addr, err error) []netip.AddrPort {
if err != nil {
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
return nil
}
return prev
}
out := make([]netip.AddrPort, 0, len(addrs))
for _, a := range addrs {
out = append(out, netip.AddrPortFrom(a.Unmap(), port))
}
return out
}
func (hr *hostnamesResults) storeKnown(known [][]netip.AddrPort) {
byName := make(map[hostnamePort][]netip.AddrPort, len(known))
for i, addrs := range known {
byName[hr.hostnames[i]] = addrs
}
hr.known.Store(&byName)
}
// literalAddrs returns the entries that were configured as ip addresses rather than hostnames
func (hr *hostnamesResults) literalAddrs() []netip.AddrPort {
var out []netip.AddrPort
for _, hostPort := range hr.hostnames {
if hostPort.addr.IsValid() {
out = append(out, netip.AddrPortFrom(hostPort.addr, hostPort.port))
}
}
return out
}
// ResolveNow asks the background goroutine to look up the hostnames again without waiting for the next tick
func (hr *hostnamesResults) ResolveNow() {
if hr == nil || hr.resolveNow == nil {
return
}
select {
case hr.resolveNow <- struct{}{}:
default:
}
}
func (hr *hostnamesResults) Cancel() { func (hr *hostnamesResults) Cancel() {
if hr != nil && hr.cancelFn != nil { if hr != nil && hr.cancelFn != nil {
hr.cancelFn() hr.cancelFn()
@@ -256,6 +346,13 @@ func (r *RemoteList) ResetForOwner(ownerVpnAddr netip.Addr) {
r.shouldRebuild = true r.shouldRebuild = true
} }
// ResolveHostnames kicks a fresh DNS lookup of any static_map hostnames for this host
func (r *RemoteList) ResolveHostnames() {
r.RLock()
defer r.RUnlock()
r.hr.ResolveNow()
}
// ClearHostnameResults cancels the in-flight DNS resolver goroutine (if any) and drops the resolved IP cache. // ClearHostnameResults cancels the in-flight DNS resolver goroutine (if any) and drops the resolved IP cache.
func (r *RemoteList) ClearHostnameResults() { func (r *RemoteList) ClearHostnameResults() {
r.Lock() r.Lock()
@@ -460,10 +557,14 @@ func (r *RemoteList) unlockedSetV4(ownerVpnIp, vpnIp netip.Addr, to []*V4AddrPor
// Reset the slice // Reset the slice
c.reported = c.reported[:0] c.reported = c.reported[:0]
// We can't take their array but we can take their pointers // We can't take their array but we can take their pointers.
for _, v := range to[:minInt(len(to), MaxRemotes)] { // Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
for _, v := range to {
if check(vpnIp, v) { if check(vpnIp, v) {
c.reported = append(c.reported, v) c.reported = append(c.reported, v)
if len(c.reported) == MaxRemotes {
break
}
} }
} }
} }
@@ -508,10 +609,14 @@ func (r *RemoteList) unlockedSetV6(ownerVpnIp, vpnIp netip.Addr, to []*V6AddrPor
// Reset the slice // Reset the slice
c.reported = c.reported[:0] c.reported = c.reported[:0]
// We can't take their array but we can take their pointers // We can't take their array but we can take their pointers.
for _, v := range to[:minInt(len(to), MaxRemotes)] { // Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
for _, v := range to {
if check(vpnIp, v) { if check(vpnIp, v) {
c.reported = append(c.reported, v) c.reported = append(c.reported, v)
if len(c.reported) == MaxRemotes {
break
}
} }
} }
} }
+89
View File
@@ -1,10 +1,15 @@
package nebula package nebula
import ( import (
"context"
"encoding/binary" "encoding/binary"
"fmt"
"net"
"net/netip" "net/netip"
"testing" "testing"
"time"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -328,6 +333,55 @@ func BenchmarkSortRebuild(b *testing.B) {
}) })
} }
func TestApplyLookup(t *testing.T) {
prev := []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}
// A successful lookup replaces the previous addresses
got := applyLookup(prev, 4242, []netip.Addr{netip.MustParseAddr("::ffff:2.2.2.2")}, nil)
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("2.2.2.2:4242")}, got)
// A not found answer means the record is gone
assert.Empty(t, applyLookup(prev, 4242, nil, &net.DNSError{Err: "no such host", IsNotFound: true}))
// Anything else is not new information, keep what we had
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "i/o timeout", IsTimeout: true}))
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "server misbehaving", IsTemporary: true}))
assert.Equal(t, prev, applyLookup(prev, 4242, nil, context.DeadlineExceeded))
}
func TestNewHostnameResults_CarriesKnown(t *testing.T) {
// A canceled context keeps the background lookup from learning anything, as if DNS were down
ctx, cancel := context.WithCancel(context.Background())
cancel()
known := netip.MustParseAddrPort("1.1.1.1:4242")
prev := &hostnamesResults{network: "ip4"}
prev.known.Store(&map[hostnamePort][]netip.AddrPort{{name: "lh.invalid", port: 4242}: {known}})
hostPorts := []string{"lh.invalid:4242", "10.0.0.1:4242"}
literal := netip.MustParseAddrPort("10.0.0.1:4242")
hr, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, prev, func() {})
require.NoError(t, err)
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr.GetAddrs())
assert.Equal(t, []netip.AddrPort{literal}, hr.literalAddrs())
// And again from the new one, before it has finished a lookup of its own
hr2, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, hr, func() {})
require.NoError(t, err)
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr2.GetAddrs())
// A port change is a different entry
hr3, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, []string{"lh.invalid:4243"}, prev, func() {})
require.NoError(t, err)
assert.Empty(t, hr3.GetAddrs())
// So is a network change
hr4, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip6", time.Second, hostPorts, prev, func() {})
require.NoError(t, err)
assert.Equal(t, []netip.AddrPort{literal}, hr4.GetAddrs())
}
func newIp4AndPortFromString(s string) *V4AddrPort { func newIp4AndPortFromString(s string) *V4AddrPort {
a := netip.MustParseAddrPort(s) a := netip.MustParseAddrPort(s)
v4Addr := a.Addr().As4() v4Addr := a.Addr().As4()
@@ -346,3 +400,38 @@ func newIp6AndPortFromString(s string) *V6AddrPort {
Port: uint32(a.Port()), Port: uint32(a.Port()),
} }
} }
func TestRemoteList_SetFiltersBeforeCap(t *testing.T) {
owner := netip.MustParseAddr("0.0.0.0")
rl := NewRemoteList([]netip.Addr{owner}, nil)
// MaxRemotes denied addrs first, then the only allowed one, then enough allowed to exceed the cap
var v4 []*V4AddrPort
for i := 0; i < MaxRemotes; i++ {
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("172.17.0.%d:4242", i+1)))
}
for i := 0; i < MaxRemotes+1; i++ {
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("70.199.182.%d:4242", i+1)))
}
rl.unlockedSetV4(owner, owner, v4, func(_ netip.Addr, a *V4AddrPort) bool {
return protoV4AddrPortToNetAddrPort(a).Addr().As4()[0] != 172
})
var v6 []*V6AddrPort
for i := 0; i < MaxRemotes; i++ {
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[fd00::%d]:4242", i+1)))
}
for i := 0; i < MaxRemotes+1; i++ {
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[2001:db8::%d]:4242", i+1)))
}
rl.unlockedSetV6(owner, owner, v6, func(_ netip.Addr, a *V6AddrPort) bool {
return protoV6AddrPortToNetAddrPort(a).Addr().As16()[0] != 0xfd
})
rl.Rebuild([]netip.Prefix{})
assert.Len(t, rl.addrs, MaxRemotes*2)
for _, a := range rl.addrs {
assert.NotEqual(t, byte(172), a.Addr().As16()[12], "denied v4 addr %v was kept", a)
assert.NotEqual(t, byte(0xfd), a.Addr().As16()[0], "denied v6 addr %v was kept", a)
}
}
+12 -4
View File
@@ -255,6 +255,10 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
msgs, buffers, names, _ := prepareRawMessages(u.batch, bufSize, cmsgSpace) msgs, buffers, names, _ := prepareRawMessages(u.batch, bufSize, cmsgSpace)
for { for {
// A shut down socket still queues and returns datagrams, a busy reader would never see an empty read
if u.closed.Load() {
return net.ErrClosed
}
if cmsgSpace > 0 { if cmsgSpace > 0 {
for i := range msgs { for i := range msgs {
setMsgControllen(&msgs[i].Hdr, cmsgSpace) setMsgControllen(&msgs[i].Hdr, cmsgSpace)
@@ -446,12 +450,16 @@ func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
return nil return nil
} }
func (u *StdConn) Close() error { // Shutdown wakes the reader without releasing the fd, Close only once it has exited
func (u *StdConn) Shutdown() error {
u.closed.Store(true) u.closed.Store(true)
// Wake the reader parked in recvmmsg. shutdown(2) on an unconnected socket // shutdown(2) on an unconnected socket returns ENOTCONN but still wakes the reader, so ignore the error
// returns ENOTCONN but still wakes it, so ignore the error.
// The reader then sees closed and stops touching the fd, making the Close below safe.
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR) _ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
return nil
}
func (u *StdConn) Close() error {
_ = u.Shutdown()
return unix.Close(u.sysFd) return unix.Close(u.sysFd)
} }
+74 -5
View File
@@ -76,7 +76,7 @@ func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
} }
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking // TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
// reader must tear down cleanly on Close() regardless of what the socket has // reader must tear down cleanly on Shutdown() regardless of what the socket has
// carried. The three cases the report called out: // carried. The three cases the report called out:
// //
// no traffic ever -> works (shutdown wakes recvmmsg with n==0) // no traffic ever -> works (shutdown wakes recvmmsg with n==0)
@@ -162,11 +162,12 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
// empty queue with the socket already having received a packet. // empty queue with the socket already having received a packet.
time.Sleep(500 * time.Millisecond) time.Sleep(500 * time.Millisecond)
// Tear down the way Interface.Close does, the fd stays open until the reader is gone
start := time.Now() start := time.Now()
if err := sc.Close(); err != nil { if err := sc.Shutdown(); err != nil {
t.Fatalf("Close: %v", err) t.Fatalf("Shutdown: %v", err)
} }
close(stop) defer func() { _ = sc.Close() }()
select { select {
case err := <-loopDone: case err := <-loopDone:
@@ -179,7 +180,75 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
case <-time.After(3 * time.Second): case <-time.After(3 * time.Second):
buf := make([]byte, 1<<20) buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true) n := runtime.Stack(buf, true)
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n]) t.Fatalf("%s: HANG, ListenOut did not return within 3s of Shutdown\n%s", name, buf[:n])
} }
close(stop)
<-trafficDone <-trafficDone
} }
// A shut down socket still returns queued datagrams, so a reader that is behind has to stop on its own within the
// batch it was working on
func TestListenOutShutdown_BusyReader(t *testing.T) {
const batch = 64
udpSettings := Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Multi: true, Batch: batch}
c, err := NewListener(testLogger(), udpSettings)
if err != nil {
t.Fatalf("NewListener: %v", err)
}
sc := c.(*StdConn)
defer sc.Close()
addr, err := sc.LocalAddr()
if err != nil {
t.Fatalf("LocalAddr: %v", err)
}
stop := make(chan struct{})
defer close(stop)
for range 8 {
send, err := net.Dial("udp", addr.String())
if err != nil {
t.Fatalf("dial: %v", err)
}
defer send.Close()
go func() {
b := make([]byte, 1200)
for {
select {
case <-stop:
return
default:
_, _ = send.Write(b)
}
}
}()
}
var afterShutdown atomic.Int64
loopDone := make(chan error, 1)
go func() {
loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
if sc.closed.Load() {
afterShutdown.Add(1)
}
// Slower than the senders so the queue never drains
for start := time.Now(); time.Since(start) < 200*time.Microsecond; {
}
}, func() {})
}()
time.Sleep(200 * time.Millisecond)
if err := sc.Shutdown(); err != nil {
t.Fatalf("Shutdown: %v", err)
}
select {
case err := <-loopDone:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("ListenOut returned %v, want net.ErrClosed", err)
}
case <-time.After(3 * time.Second):
t.Fatalf("ListenOut still running 3s after Shutdown, delivered %d packets since", afterShutdown.Load())
}
if n := afterShutdown.Load(); n > batch {
t.Fatalf("delivered %d packets after Shutdown, more than the one batch in flight", n)
}
}