Compare commits

..
Author SHA1 Message Date
Caleb JasikandClaude Opus 5.5 3bb5918e7c udp/darwin: send to busy established peers on connected sockets
On darwin an unconnected sendto runs the address and MAC policy checks
and the EndpointSecurity send check for every datagram, and while
xprotectd throttles it each send takes about 185us, which caps a
tunnel's upload and, through its ACKs, its download. A connected socket
runs those checks once, at connect, and it is the only UDP send that
reports the interface queue's flow control, as ENOBUFS.

Only the current direct remote of an established tunnel is eligible.
udp.EstablishedPeerConn is an optional Conn interface, implemented on
darwin; establishedRemotes refcounts remotes over the main hostmap and
tells each writer on the first and last reference: handshake completion
adds one, deleting or retiring a hostinfo removes it, SetRemote (roaming,
SetRemoteIfPreferred, control, ssh) moves it. Pending and relayed
hostinfos hold none, and a rehandshake's two hostinfos share one.

An eligible peer that Nebula sends NEBULA_CONNSOCK_RUN (64) datagrams to
within NEBULA_CONNSOCK_WINDOW (1s) gets a connected socket: SO_REUSEPORT,
bound to the source address the listener's route to the peer picks on
the listener's port, carrying the listener's IP_BOUND_IF. The peer sees
the same address and port as from the listener, and the socket's reader
delivers through the listener's reader. The listener itself never sets
SO_REUSEPORT (listen routines still do), so no other uid can bind the
port; with a specific listen.host and no routines connected sockets stay
off, since sharing an exact address needs SO_REUSEPORT on both.

All routines listening on one address share one set of connected
sockets: at most NEBULA_CONNSOCKS (8), one per peer, with opens from a
bucket that refills one per NEBULA_CONNSOCK_OPEN_EVERY (1s). A socket
closes when its peer stops being eligible, after NEBULA_CONNSOCK_IDLE
(30s) without traffic, on Rebind (of every routine) and Close, and on an
error that means the connected path is gone. Closing never runs under
the hostmap's or the eligibility lock; the peer can't reopen until the
close finishes. A send that gets ENOBUFS or EAGAIN waits within one
10ms budget shared by every writer since the last send that went
through, then drops; nothing waits in the netpoller without a deadline.
When nothing is eligible or open, a send costs one atomic load before
the listener path.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-28 16:28:07 -05:00
56 changed files with 2505 additions and 2696 deletions

No files matched your search

-7
View File
@@ -7,13 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
### Fixed
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
the handshake starts, so a pending hostinfo never exists without one.
## [1.11.1] - 2026-08-21 ## [1.11.1] - 2026-08-21
See the [v1.11.1](https://github.com/slackhq/nebula/milestone/30?closed=1) milestone for a complete list of changes. See the [v1.11.1](https://github.com/slackhq/nebula/milestone/30?closed=1) milestone for a complete list of changes.
+4 -2
View File
@@ -174,8 +174,10 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
switch decision { switch decision {
case deleteTunnel: case deleteTunnel:
// Local delete only, clearing the lighthouse cache if this was the last hostinfo for the vpn addr if cm.hostMap.DeleteHostInfo(hostinfo) {
cm.intf.closeTunnel(hostinfo) // Only clearing the lighthouse cache if this is the last hostinfo for this vpn ip in the hostmap
cm.intf.lightHouse.DeleteVpnAddrs(hostinfo.vpnAddrs)
}
case closeTunnel: case closeTunnel:
cm.intf.sendCloseTunnel(hostinfo) cm.intf.sendCloseTunnel(hostinfo)
-3
View File
@@ -7,7 +7,6 @@ import (
"testing" "testing"
"time" "time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/overlay/overlaytest" "github.com/slackhq/nebula/overlay/overlaytest"
@@ -22,8 +21,6 @@ func newTestLighthouse() *LightHouse {
l: test.NewLogger(), l: test.NewLogger(),
addrMap: map[netip.Addr]*RemoteList{}, addrMap: map[netip.Addr]*RemoteList{},
queryChan: make(chan netip.Addr, 10), queryChan: make(chan netip.Addr, 10),
// Nothing drains queryChan here, so an 11th query is dropped.
queryDropped: metrics.NilCounter{},
} }
lighthouses := []netip.Addr{} lighthouses := []netip.Addr{}
staticList := map[netip.Addr]struct{}{} staticList := map[netip.Addr]struct{}{}
-119
View File
@@ -2,8 +2,6 @@ package nebula
import ( import (
"net/netip" "net/netip"
"strconv"
"sync"
"testing" "testing"
"time" "time"
@@ -13,7 +11,6 @@ import (
ct "github.com/slackhq/nebula/cert_test" ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake" "github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -150,122 +147,6 @@ func TestSendNoMetricsCloseTunnelKeepsRebindEpoch(t *testing.T) {
assert.True(t, hostinfo.markOut(1), "a CloseTunnel send must not consume the rebind epoch") assert.True(t, hostinfo.markOut(1), "a CloseTunnel send must not consume the rebind epoch")
} }
// capturingConn is a udp.Conn that records every WriteTo.
type capturingConn struct {
udp.NoopConn
writes [][]byte
addrs []netip.AddrPort
}
func (c *capturingConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.writes = append(c.writes, append([]byte(nil), b...))
c.addrs = append(c.addrs, addr)
return nil
}
// TestSendNoMetricsViaRelay sends through a relay from the mtu sized buffer the cached packet flush uses. Payloads up
// to overlay.MaxMTU go out intact, anything bigger is dropped rather than outgrowing the buffer, which once panicked.
func TestSendNoMetricsViaRelay(t *testing.T) {
tests := []struct {
n int
sent bool
}{
{0, true},
{1300, true},
{overlay.MaxMTU, true},
{overlay.MaxMTU + 1, false}, // too big for the relay's tag
{mtu - 47, false}, // too big for the inner tag, this panicked
{9000, false}, // a full packet at tun.mtu 9000
}
for _, tt := range tests {
t.Run(strconv.Itoa(tt.n), func(t *testing.T) {
// One tunnel to the target, carried end to end, and one to the relay that wraps it
targetInit, targetResp := runTestHandshake(t)
relayInit, relayResp := runTestHandshake(t)
ci, err := newConnectionStateFromResult(targetInit)
require.NoError(t, err)
relayCI, err := newConnectionStateFromResult(relayInit)
require.NoError(t, err)
targetAddr := netip.MustParseAddr("10.0.0.2")
relayAddr := netip.MustParseAddr("10.0.0.3")
relayRemote := netip.MustParseAddrPort("192.0.2.1:4242")
relay := &Relay{Type: TerminalType, State: Established, LocalIndex: 5, RemoteIndex: 6, PeerAddr: targetAddr}
relayHI := &HostInfo{
vpnAddrs: []netip.Addr{relayAddr},
ConnectionState: relayCI,
relayState: RelayState{
relayForByAddr: map[netip.Addr]*Relay{targetAddr: relay},
relayForByIdx: map[uint32]*Relay{relay.LocalIndex: relay},
},
}
relayHI.remote.Store(&relayRemote)
// No remote of its own, so the send has to go via the relay
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{targetAddr},
ConnectionState: ci,
remoteIndexId: 7,
relayState: RelayState{
relays: []netip.Addr{relayAddr},
relayForByAddr: map[netip.Addr]*Relay{},
relayForByIdx: map[uint32]*Relay{},
},
}
l := test.NewLogger()
hm := newHostMap(l)
hm.Hosts[relayAddr] = relayHI
conn := &capturingConn{}
f := &Interface{
l: l,
hostMap: hm,
messageMetrics: &MessageMetrics{txExhausted: metrics.NewCounter()},
writers: []udp.Conn{conn},
}
f.connectionManager = &connectionManager{intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{}}
payload := make([]byte, tt.n)
for i := range payload {
payload[i] = byte(i)
}
counter := ci.messageCounter.Load()
f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, payload, make([]byte, 12), make([]byte, mtu), 0)
if !tt.sent {
assert.Empty(t, conn.writes)
assert.Equal(t, counter, ci.messageCounter.Load(), "a dropped packet must not spend a counter")
return
}
require.Len(t, conn.writes, 1)
assert.Equal(t, relayRemote, conn.addrs[0])
pkt := conn.writes[0]
// The relay authenticates the whole packet under its own header
outer := &header.H{}
require.NoError(t, outer.Parse(pkt))
assert.Equal(t, header.MessageRelay, outer.Subtype)
assert.Equal(t, relay.RemoteIndex, outer.RemoteIndex)
relayPeer, err := newConnectionStateFromResult(relayResp)
require.NoError(t, err)
require.NoError(t, relayPeer.VerifyRelay(l, outer.MessageCounter, pkt, make([]byte, 12)))
// Inside it, the target decrypts the end to end packet back to the payload
inner := pkt[header.Len : len(pkt)-relayPeer.dKey.Overhead()]
h := &header.H{}
require.NoError(t, h.Parse(inner))
assert.Equal(t, header.Message, h.Type)
assert.Equal(t, hostinfo.remoteIndexId, h.RemoteIndex)
targetPeer, err := newConnectionStateFromResult(targetResp)
require.NoError(t, err)
got, err := targetPeer.Decrypt(l, h.MessageCounter, inner, make([]byte, 12))
require.NoError(t, err)
assert.Equal(t, payload, got)
})
}
}
func TestNewConnectionStateFromResult(t *testing.T) { func TestNewConnectionStateFromResult(t *testing.T) {
initR, respR := runTestHandshake(t) initR, respR := runTestHandshake(t)
+2 -1
View File
@@ -203,7 +203,8 @@ func (c *Control) RebindUDPServer() {
} }
// A failure here means we are likely still pinned to the interface we came up on, so the rest of this is // A failure here means we are likely still pinned to the interface we came up on, so the rest of this is
// unlikely to help. Say so instead of silently carrying on as if we rebound. // unlikely to help. Say so instead of silently carrying on as if we rebound. Only the first writer is rebound;
// darwin's Rebind clears every listen routine's listener.
if err := c.f.outside.Rebind(); err != nil { if err := c.f.outside.Rebind(); err != nil {
c.l.Error("Failed to rebind udp socket", "error", err) c.l.Error("Failed to rebind udp socket", "error", err)
} }
-68
View File
@@ -1,68 +0,0 @@
# vpnemu
Reproduces, on macOS and without Tailscale, the way a VPN with an exit node takes nebula's vpn network route
away from its utun (#1931).
A macOS network extension never touches the routing table. Its routes are published in configd's dynamic store
under `State:/Network/Service/<id>/IPv4` and `/IPv6`, and configd's IPMonitor installs them. vpnemu creates a
utun and publishes those keys itself, so configd does exactly what it would do for the real app. It publishes
the routes Tailscale would with an exit node: every interface prefix that isn't a single IP, the rule in
Tailscale's `ipnlocal.internalAndExternalInterfacesFrom`.
| mode | Tailscale setting | published as | where nebula's prefix goes |
| --- | --- | --- | --- |
| `claim` | exit node, no LAN access | `AdditionalRoutes` | into vpnemu's utun |
| `exclude` | exit node, LAN access | `ExcludedRoutes`, service overrides primary | out the next interface (en0) |
configd answers `EEXIST` with a delete and an add, so whatever held the prefix loses it. When the service is
withdrawn, configd deletes its routes by prefix alone, whoever holds them by then.
## Build
It is its own module, so `purego` stays out of nebula's `go.mod`. purego reaches CoreFoundation and
SystemConfiguration without cgo, so it cross compiles and runs on a mac with no developer tools.
```sh
cd e2e/darwin/vpnemu
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o vpnemu .
# a cross compiled binary copied to a mac may need: codesign -s - -f vpnemu
```
## Run
```sh
sudo ./vpnemu -mode claim -keep en0 # runs until ctrl-c, then withdraws and closes its utun
sudo ./vpnemu -clear # withdraw a service a crashed run left behind
```
`-keep` names interfaces whose prefixes are never published. Keep the one your ssh session arrives on, or
`claim` takes your LAN into a utun nobody reads. `exclude` makes vpnemu primary, so anything off a kept
interface's subnet, internet included, goes into its utun and is dropped while it runs.
`-follow` (default on) republishes when interface prefixes change, as Tailscale does on link changes. With
`-follow=false` a route stays published after its interface is gone, which stages the route configd leaves
behind for a nebula that restarts before the VPN has caught up.
## A/B a nebula build
`repro.sh` starts nebula, then runs vpnemu, and reports egress and ping per peer before, during and after:
```sh
sudo ./repro.sh -n ./nebula -c config.yml -p 192.168.212.1 -p fd19:31::1 -m claim -k en0
```
Before #1931 (utun holds `/24` and `/64`):
```
during 192.168.212.1 egress=utun2 ping=FAIL
after 192.168.212.1 egress=en0 ping=FAIL
```
With #1931 (utun holds `/32` and `/128`, network routes added explicitly) every phase stays on nebula's utun.
### Leftover route
Run an old nebula, start `vpnemu -follow=false`, then restart onto a new nebula while vpnemu is still up. The
new nebula finds the vpn network routed to vpnemu's utun, replaces it and starts. Stop vpnemu and configd deletes
the prefix, taking nebula's route with it until nebula restarts. With real Tailscale this needs nebula to restart
before Tailscale notices the old utun is gone, a one time window when upgrading from a nebula without #1931.
-8
View File
@@ -1,8 +0,0 @@
module github.com/slackhq/nebula/e2e/darwin/vpnemu
go 1.26.0
require (
github.com/ebitengine/purego v0.9.1
golang.org/x/sys v0.48.0
)
-4
View File
@@ -1,4 +0,0 @@
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
-292
View File
@@ -1,292 +0,0 @@
//go:build darwin
// vpnemu stands in for a macOS VPN app with an exit node in use, so route hijacks like the one
// Tailscale causes can be reproduced without Tailscale.
//
// A network extension does not touch the routing table itself. It hands its routes to
// nesessionmanager, which publishes them in configd's dynamic store under
// State:/Network/Service/<id>/IPv4 and /IPv6, and configd's IPMonitor installs them. vpnemu
// creates a utun and publishes the same keys directly, so configd does exactly what it would do
// for the real thing.
//
// Which routes it publishes follows Tailscale's rule for exit nodes
// (ipnlocal.internalAndExternalInterfacesFrom): every interface address prefix that is not a
// single IP counts as an attached LAN. With LAN access off those prefixes are claimed into the
// tunnel (-mode claim, published as AdditionalRoutes). With LAN access on they are excluded and
// leave through the next interface (-mode exclude, published as ExcludedRoutes on a service that
// overrides the primary).
//
// It runs until interrupted, then withdraws the service and closes its utun. Needs root.
package main
import (
"flag"
"fmt"
"log"
"net"
"net/netip"
"os"
"os/exec"
"os/signal"
"slices"
"strings"
"syscall"
"time"
"github.com/ebitengine/purego"
"golang.org/x/sys/unix"
)
const (
serviceID = "VPNEMU"
// Tailscale's own addresses, single IPs, so the rule skips them like Tailscale skips its own
addr4 = "100.100.0.1"
addr6 = "fd7a:115c:a1e0::1"
utunControlName = "com.apple.net.utun_control"
utunOptIfname = 2
cfStringUTF8 = 0x08000100
)
var (
cfStringCreateWithCString func(alloc uintptr, s *byte, enc uint32) uintptr
cfDataCreate func(alloc uintptr, b *byte, n int) uintptr
cfPropertyListCreateWithData func(alloc, data uintptr, opts uint, format *int, err *uintptr) uintptr
cfRelease func(ref uintptr)
scDynamicStoreCreate func(alloc, name, callout, ctx uintptr) uintptr
scDynamicStoreSetValue func(store, key, value uintptr) bool
scDynamicStoreRemoveValue func(store, key uintptr) bool
scError func() int32
)
// loadFrameworks binds the few CoreFoundation and SystemConfiguration calls we need. purego does
// it without cgo, so this cross compiles and runs on a mac with no developer tools. JXA can read
// the dynamic store but its writes fail, and scutil cannot build the arrays of dictionaries the
// route keys hold.
func loadFrameworks() {
cf, err := purego.Dlopen("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", purego.RTLD_NOW|purego.RTLD_GLOBAL)
if err != nil {
log.Fatal(err)
}
sc, err := purego.Dlopen("/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration", purego.RTLD_NOW|purego.RTLD_GLOBAL)
if err != nil {
log.Fatal(err)
}
purego.RegisterLibFunc(&cfStringCreateWithCString, cf, "CFStringCreateWithCString")
purego.RegisterLibFunc(&cfDataCreate, cf, "CFDataCreate")
purego.RegisterLibFunc(&cfPropertyListCreateWithData, cf, "CFPropertyListCreateWithData")
purego.RegisterLibFunc(&cfRelease, cf, "CFRelease")
purego.RegisterLibFunc(&scDynamicStoreCreate, sc, "SCDynamicStoreCreate")
purego.RegisterLibFunc(&scDynamicStoreSetValue, sc, "SCDynamicStoreSetValue")
purego.RegisterLibFunc(&scDynamicStoreRemoveValue, sc, "SCDynamicStoreRemoveValue")
purego.RegisterLibFunc(&scError, sc, "SCError")
}
func cfString(s string) uintptr {
b := append([]byte(s), 0)
return cfStringCreateWithCString(0, &b[0], cfStringUTF8)
}
// cfPlist builds a property list from its xml form, the least code way to make nested CF values
func cfPlist(xml string) uintptr {
b := []byte(xml)
d := cfDataCreate(0, &b[0], len(b))
defer cfRelease(d)
var errRef uintptr
p := cfPropertyListCreateWithData(0, d, 0, nil, &errRef)
if p == 0 {
log.Fatalf("invalid property list:\n%s", xml)
}
return p
}
type store struct {
ref uintptr
k4, k6 uintptr
}
func openStore() *store {
return &store{
ref: scDynamicStoreCreate(0, cfString("vpnemu"), 0, 0),
k4: cfString("State:/Network/Service/" + serviceID + "/IPv4"),
k6: cfString("State:/Network/Service/" + serviceID + "/IPv6"),
}
}
func (s *store) set(key uintptr, xml string) {
v := cfPlist(xml)
defer cfRelease(v)
if !scDynamicStoreSetValue(s.ref, key, v) {
log.Fatalf("SCDynamicStoreSetValue failed: SCError %d", scError())
}
}
func (s *store) withdraw() {
scDynamicStoreRemoveValue(s.ref, s.k4)
scDynamicStoreRemoveValue(s.ref, s.k6)
}
// createUtun opens a utun the kernel numbers for us and gives it Tailscale-like single addresses.
// The interface lives as long as the returned fd.
func createUtun() (int, string) {
fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL)
if err != nil {
log.Fatalf("system socket: %v", err)
}
info := &unix.CtlInfo{}
copy(info.Name[:], utunControlName)
if err = unix.IoctlCtlInfo(fd, info); err != nil {
log.Fatalf("CTLIOCGINFO: %v", err)
}
if err = unix.Connect(fd, &unix.SockaddrCtl{ID: info.Id, Unit: 0}); err != nil {
log.Fatalf("connect utun control: %v", err)
}
name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, utunOptIfname)
if err != nil {
log.Fatalf("utun name: %v", err)
}
for _, args := range [][]string{
{name, "inet", addr4, addr4, "netmask", "255.255.255.255", "up"},
{name, "inet6", addr6, "prefixlen", "128"},
} {
if out, err := exec.Command("ifconfig", args...).CombinedOutput(); err != nil {
log.Fatalf("ifconfig %s: %v: %s", strings.Join(args, " "), err, out)
}
}
return fd, name
}
// lanPrefixes applies Tailscale's attached-LAN rule to every interface but those in skip
func lanPrefixes(skip []string) (v4, v6 []netip.Prefix) {
ifs, err := net.Interfaces()
if err != nil {
log.Fatal(err)
}
for _, i := range ifs {
if slices.Contains(skip, i.Name) || i.Flags&net.FlagLoopback != 0 {
continue
}
addrs, _ := i.Addrs()
for _, a := range addrs {
n, ok := a.(*net.IPNet)
if !ok {
continue
}
ip, _ := netip.AddrFromSlice(n.IP)
ip = ip.Unmap()
ones, _ := n.Mask.Size()
pfx := netip.PrefixFrom(ip, ones).Masked()
// Tailscale skips single IPs. Link-local is our addition, every interface has the
// same fe80::/64 and claiming it would only break the harness's own host.
if pfx.IsSingleIP() || ip.IsLinkLocalUnicast() {
continue
}
if ip.Is4() {
v4 = append(v4, pfx)
} else {
v6 = append(v6, pfx)
}
}
}
slices.SortFunc(v4, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) })
slices.SortFunc(v6, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) })
return v4, v6
}
func routes4(ps []netip.Prefix) string {
var b strings.Builder
for _, p := range ps {
mask := net.IP(net.CIDRMask(p.Bits(), 32)).String()
fmt.Fprintf(&b, "<dict><key>DestinationAddress</key><string>%s</string><key>SubnetMask</key><string>%s</string></dict>", p.Addr(), mask)
}
return b.String()
}
func routes6(ps []netip.Prefix) string {
var b strings.Builder
for _, p := range ps {
fmt.Fprintf(&b, "<dict><key>DestinationAddress</key><string>%s</string><key>PrefixLength</key><integer>%d</integer></dict>", p.Addr(), p.Bits())
}
return b.String()
}
func (s *store) publish(ifName string, exclude bool, v4, v6 []netip.Prefix) {
routesKey, primary := "AdditionalRoutes", ""
if exclude {
// configd only steers excluded routes for a service with a default route that is primary.
// While this runs, everything not on a kept interface's subnet goes into the utun and dies.
routesKey = "ExcludedRoutes"
primary = "<key>Router</key><string>" + addr4 + "</string><key>OverridePrimary</key><integer>1</integer>"
}
const head = `<?xml version="1.0" encoding="UTF-8"?><plist version="1.0"><dict>`
s.set(s.k4, head+
"<key>InterfaceName</key><string>"+ifName+"</string>"+
"<key>Addresses</key><array><string>"+addr4+"</string></array>"+
"<key>SubnetMasks</key><array><string>255.255.255.255</string></array>"+primary+
"<key>"+routesKey+"</key><array>"+routes4(v4)+"</array></dict></plist>")
s.set(s.k6, head+
"<key>InterfaceName</key><string>"+ifName+"</string>"+
"<key>Addresses</key><array><string>"+addr6+"</string></array>"+
"<key>PrefixLength</key><array><integer>128</integer></array>"+
"<key>"+routesKey+"</key><array>"+routes6(v6)+"</array></dict></plist>")
log.Printf("published %s on %s, %s: %v %v", serviceID, ifName, routesKey, v4, v6)
}
func main() {
mode := flag.String("mode", "claim", "claim: exit node without LAN access, the prefixes are routed into the vpn\n"+
"exclude: exit node with LAN access, the prefixes leave via the next interface and the vpn becomes primary")
keep := flag.String("keep", "en0", "comma separated interfaces whose prefixes are never published, keep the one your ssh session uses")
follow := flag.Bool("follow", true, "republish when interface prefixes change, as Tailscale does on link changes.\n"+
"Off, a route stays published after its interface is gone, which is how a leftover route is staged")
clear := flag.Bool("clear", false, "withdraw a service a crashed run left behind, then exit")
flag.Parse()
if *mode != "claim" && *mode != "exclude" {
log.Fatalf("unknown -mode %q", *mode)
}
if os.Geteuid() != 0 {
log.Fatal("needs root, to create the utun and write configd's dynamic store")
}
loadFrameworks()
s := openStore()
if *clear {
s.withdraw()
log.Print("withdrawn")
return
}
fd, ifName := createUtun()
exclude := *mode == "exclude"
skip := append(strings.Split(*keep, ","), ifName)
v4, v6 := lanPrefixes(skip)
s.publish(ifName, exclude, v4, v6)
sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
tick := time.NewTicker(time.Second)
defer tick.Stop()
for {
select {
case <-sig:
s.withdraw()
_ = unix.Close(fd)
log.Print("withdrawn")
return
case <-tick.C:
if !*follow {
continue
}
n4, n6 := lanPrefixes(skip)
if !slices.Equal(n4, v4) || !slices.Equal(n6, v6) {
v4, v6 = n4, n6
s.publish(ifName, exclude, v4, v6)
}
}
}
}
-76
View File
@@ -1,76 +0,0 @@
#!/bin/sh
# Reports where traffic to nebula peers goes before, during and after a vpnemu exit node.
#
# usage: sudo ./repro.sh -n <nebula binary> -c <nebula config> -p <peer vpn addr> [-p <peer>...]
# [-m claim|exclude] [-k <keep interfaces>]
#
# Each peer should answer pings over the overlay. A healthy run shows the nebula utun as egress
# and the ping working in all three phases.
set -u
MODE=claim
KEEP=en0
PEERS=""
NEBULA=""
CONFIG=""
while getopts n:c:p:m:k: o; do
case $o in
n) NEBULA=$OPTARG ;;
c) CONFIG=$OPTARG ;;
p) PEERS="$PEERS $OPTARG" ;;
m) MODE=$OPTARG ;;
k) KEEP=$OPTARG ;;
*) exit 2 ;;
esac
done
if [ -z "$NEBULA" ] || [ -z "$CONFIG" ] || [ -z "$PEERS" ]; then
sed -n '4,5p' "$0"
exit 2
fi
HERE=$(cd "$(dirname "$0")" && pwd)
VPNEMU=${VPNEMU:-$HERE/vpnemu}
LOG=$(mktemp -t nebula-repro)
report() {
for p in $PEERS; do
case $p in
*:*) fam=-inet6 ping=ping6 ;;
*) fam=-inet ping="ping -t 3" ;;
esac
egress=$(route -n get $fam "$p" 2>/dev/null | awk '/interface:/{print $2}')
if $ping -c 2 "$p" >/dev/null 2>&1; then ok=ok; else ok=FAIL; fi
printf '%-8s %-24s egress=%-8s ping=%s\n' "$1" "$p" "${egress:-none}" "$ok"
done
}
cleanup() {
[ -n "${EMU:-}" ] && kill "$EMU" 2>/dev/null
[ -n "${NEB:-}" ] && kill "$NEB" 2>/dev/null
wait 2>/dev/null
}
trap cleanup EXIT INT TERM
"$NEBULA" -config "$CONFIG" >"$LOG" 2>&1 &
NEB=$!
sleep 3
if ! kill -0 $NEB 2>/dev/null; then
echo "nebula exited:"
tail -5 "$LOG"
exit 1
fi
echo "nebula on $(grep -o 'interface=utun[0-9]*' "$LOG" | head -1 | cut -d= -f2), log in $LOG"
report before
"$VPNEMU" -mode "$MODE" -keep "$KEEP" &
EMU=$!
sleep 3
report during
kill $EMU
wait $EMU 2>/dev/null
EMU=
sleep 2
report after
grep -E 'level=(WARN|ERROR).*(route|Route)' "$LOG"
+90
View File
@@ -0,0 +1,90 @@
package nebula
import (
"net/netip"
"sync"
"github.com/slackhq/nebula/udp"
)
// establishedRemotes tells the udp conns that implement udp.EstablishedPeerConn which addresses are the current
// direct remote of at least one established tunnel. A hostinfo holds a reference on its remote while it is in the
// main hostmap, and SetRemote moves the reference; a relayed tunnel has no direct remote and holds none.
type establishedRemotes struct {
mu sync.Mutex
conns []udp.EstablishedPeerConn
refs map[netip.AddrPort]int
}
// newEstablishedRemotes returns nil when no conn wants to know, which every method accepts.
func newEstablishedRemotes(conns []udp.Conn) *establishedRemotes {
var pc []udp.EstablishedPeerConn
for _, c := range conns {
if p, ok := c.(udp.EstablishedPeerConn); ok {
pc = append(pc, p)
}
}
if len(pc) == 0 {
return nil
}
return &establishedRemotes{conns: pc, refs: map[netip.AddrPort]int{}}
}
// add makes hostinfo hold a reference on its remote, as it joins the main hostmap. A hostinfo holds references in
// one establishedRemotes at a time, so an add while it is in another is ignored.
func (e *establishedRemotes) add(hostinfo *HostInfo) {
if e == nil {
return
}
e.mu.Lock()
defer e.mu.Unlock()
if !hostinfo.establishedIn.CompareAndSwap(nil, e) && hostinfo.establishedIn.Load() != e {
return
}
e.syncLocked(hostinfo)
}
// remove drops hostinfo's reference, as it leaves the main hostmap.
func (e *establishedRemotes) remove(hostinfo *HostInfo) {
if e == nil {
return
}
e.mu.Lock()
defer e.mu.Unlock()
if hostinfo.establishedIn.CompareAndSwap(e, nil) {
e.syncLocked(hostinfo)
}
}
// syncLocked moves hostinfo's reference to the remote it should hold now: its current remote while it is
// established, none otherwise. Reading both under mu makes the last call win however add, remove and SetRemote
// interleave. Only the establishedRemotes that hostinfo is in, or has just left, may call it, since
// hostinfo.establishedRemote is a reference in that one's refs.
func (e *establishedRemotes) syncLocked(hostinfo *HostInfo) {
var want netip.AddrPort
if hostinfo.establishedIn.Load() == e {
want = hostinfo.GetRemote()
}
held := hostinfo.establishedRemote
if want == held {
return
}
if held.IsValid() {
e.refs[held]--
if e.refs[held] == 0 {
delete(e.refs, held)
for _, c := range e.conns {
c.SetEstablishedPeer(held, false)
}
}
}
if want.IsValid() {
e.refs[want]++
if e.refs[want] == 1 {
for _, c := range e.conns {
c.SetEstablishedPeer(want, true)
}
}
}
hostinfo.establishedRemote = want
}
+173
View File
@@ -0,0 +1,173 @@
package nebula
import (
"net/netip"
"sync"
"testing"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// establishedPeerConn records which destinations the hostmap has told it are established peers, and fails the test
// on a call that doesn't change anything, which would mean a reference count went wrong.
type establishedPeerConn struct {
udp.NoopConn
t *testing.T
mu sync.Mutex
peers map[netip.AddrPort]bool
}
func (c *establishedPeerConn) SetEstablishedPeer(dst netip.AddrPort, established bool) {
c.mu.Lock()
defer c.mu.Unlock()
assert.NotEqual(c.t, established, c.peers[dst], "SetEstablishedPeer(%v, %v) repeated", dst, established)
if established {
c.peers[dst] = true
} else {
delete(c.peers, dst)
}
}
func (c *establishedPeerConn) established() []netip.AddrPort {
c.mu.Lock()
defer c.mu.Unlock()
var out []netip.AddrPort
for dst := range c.peers {
out = append(out, dst)
}
return out
}
// newEstablishedTestHostMap returns a hostmap whose conns are a recording conn and one that doesn't track peers.
func newEstablishedTestHostMap(t *testing.T) (*HostMap, *establishedPeerConn) {
hm := newHostMap(test.NewLogger())
hm.preferredRanges.Store(&[]netip.Prefix{netip.MustParsePrefix("192.168.0.0/16")})
c := &establishedPeerConn{t: t, peers: map[netip.AddrPort]bool{}}
hm.established = newEstablishedRemotes([]udp.Conn{c, udp.NoopConn{}})
require.NotNil(t, hm.established)
return hm, c
}
func newEstablishedTestHostInfo(vpnAddr netip.Addr, index uint32) *HostInfo {
return &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
localIndexId: index,
remotes: NewRemoteList([]netip.Addr{vpnAddr}, nil),
}
}
func TestNewEstablishedRemotesWithoutTrackingConns(t *testing.T) {
assert.Nil(t, newEstablishedRemotes([]udp.Conn{udp.NoopConn{}}))
}
// A conn hears about a remote while at least one hostinfo in the main hostmap has it as its direct remote: from
// the add, through roaming and SetRemoteIfPreferred, until the last such hostinfo is deleted or retired by
// MaxHostInfosPerVpnIp. Pending and relayed hostinfos never count.
func TestHostMap_EstablishedRemotes(t *testing.T) {
hm, c := newEstablishedTestHostMap(t)
f := &Interface{}
a := netip.MustParseAddr("10.0.0.1")
r1 := netip.MustParseAddrPort("203.0.113.1:4242")
r2 := netip.MustParseAddrPort("203.0.113.2:4242")
r3 := netip.MustParseAddrPort("192.168.1.3:4242")
// A pending hostinfo, as a handshake sets its remote, isn't established.
h1 := newEstablishedTestHostInfo(a, 1)
h1.SetRemote(r1)
assert.Empty(t, c.established())
// Handshake completion adds it to the main hostmap.
hm.unlockedAddHostInfo(h1, f)
assert.ElementsMatch(t, []netip.AddrPort{r1}, c.established())
// A relayed tunnel has no direct remote.
relayed := newEstablishedTestHostInfo(netip.MustParseAddr("10.0.0.2"), 2)
hm.unlockedAddHostInfo(relayed, f)
assert.ElementsMatch(t, []netip.AddrPort{r1}, c.established())
// A rehandshake's new hostinfo shares the remote, which stays established until both are gone.
h3 := newEstablishedTestHostInfo(a, 3)
h3.SetRemote(r1)
hm.unlockedAddHostInfo(h3, f)
hm.DeleteHostInfo(h1)
assert.ElementsMatch(t, []netip.AddrPort{r1}, c.established())
// Roaming moves the reference.
h3.SetRemote(r2)
assert.ElementsMatch(t, []netip.AddrPort{r2}, c.established())
require.True(t, h3.SetRemoteIfPreferred(hm, ViaSender{UdpAddr: r3}))
assert.ElementsMatch(t, []netip.AddrPort{r3}, c.established())
// A relayed tunnel that hears from its peer directly roams onto a direct remote.
relayed.SetRemote(r1)
assert.ElementsMatch(t, []netip.AddrPort{r1, r3}, c.established())
hm.DeleteHostInfo(relayed)
// A deleted hostinfo lets go, and a SetRemote after that changes nothing.
hm.DeleteHostInfo(h3)
assert.Empty(t, c.established())
h3.SetRemote(r2)
assert.Empty(t, c.established())
hm.DeleteHostInfo(h3)
// Adding past MaxHostInfosPerVpnIp retires the oldest hostinfo and its remote.
var hs []*HostInfo
for i := range MaxHostInfosPerVpnIp + 1 {
h := newEstablishedTestHostInfo(a, uint32(10+i))
h.SetRemote(netip.AddrPortFrom(netip.MustParseAddr("198.51.100.1"), uint16(1000+i)))
hm.unlockedAddHostInfo(h, f)
hs = append(hs, h)
}
assert.Len(t, c.established(), MaxHostInfosPerVpnIp)
assert.NotContains(t, c.established(), hs[0].GetRemote())
}
// Roaming on the receive path races tunnel teardown on the connection manager; whatever the order, a deleted
// hostinfo holds no reference.
func TestHostMap_EstablishedRemotesRoamRacesDelete(t *testing.T) {
hm, c := newEstablishedTestHostMap(t)
f := &Interface{}
a := netip.MustParseAddr("10.0.0.1")
for i := range 200 {
h := newEstablishedTestHostInfo(a, uint32(i+1))
h.SetRemote(netip.MustParseAddrPort("203.0.113.1:4242"))
hm.Lock()
hm.unlockedAddHostInfo(h, f)
hm.Unlock()
var wg sync.WaitGroup
wg.Go(func() { h.SetRemote(netip.AddrPortFrom(netip.MustParseAddr("203.0.113.2"), uint16(i+1))) })
wg.Go(func() { hm.DeleteHostInfo(h) })
wg.Wait()
require.Empty(t, c.established(), "round %d", i)
}
}
// A hostinfo holds its reference in the first tracking hostmap it joins; another's add and delete of it change
// nothing, so neither hostmap's counts go wrong.
func TestHostMap_EstablishedRemotesSingleOwner(t *testing.T) {
hm1, c1 := newEstablishedTestHostMap(t)
hm2, c2 := newEstablishedTestHostMap(t)
f := &Interface{}
r := netip.MustParseAddrPort("203.0.113.1:4242")
h := newEstablishedTestHostInfo(netip.MustParseAddr("10.0.0.1"), 1)
h.SetRemote(r)
hm1.unlockedAddHostInfo(h, f)
hm2.unlockedAddHostInfo(h, f)
assert.ElementsMatch(t, []netip.AddrPort{r}, c1.established())
assert.Empty(t, c2.established())
hm2.DeleteHostInfo(h)
assert.Equal(t, map[netip.AddrPort]int{r: 1}, hm1.established.refs)
assert.Empty(t, hm2.established.refs)
assert.ElementsMatch(t, []netip.AddrPort{r}, c1.established())
hm1.DeleteHostInfo(h)
assert.Empty(t, hm1.established.refs)
assert.Empty(t, hm2.established.refs)
assert.Empty(t, c1.established())
assert.Empty(t, c2.established())
}
+7 -8
View File
@@ -135,12 +135,12 @@ listen:
# superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the # superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the
# default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts. # default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts.
#batch: 64 #batch: 64
# Linux only. Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel # Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
# Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default) # Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
# Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide # Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide
# max, net.core.rmem_max and net.core.wmem_max # max, net.core.rmem_max and net.core.wmem_max
read_buffer: 3412512 #read_buffer: 10485760
write_buffer: 3412512 #write_buffer: 10485760
# On Windows only # On Windows only
# When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port. # When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port.
@@ -262,10 +262,9 @@ tun:
drop_local_broadcast: false drop_local_broadcast: false
# Toggles forwarding of multicast packets # Toggles forwarding of multicast packets
drop_multicast: false drop_multicast: false
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 5000 # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500
tx_queue: 5000 tx_queue: 500
# Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
# Anything above 8937, here or on a route below, is capped to 8937 with a warning
mtu: 1300 mtu: 1300
# the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable. # the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable.
@@ -411,8 +410,8 @@ logging:
#try_interval: 100ms #try_interval: 100ms
#retries: 20 #retries: 20
# query_buffer is the size of the buffer channel for querying lighthouses, drops are recorded in the lighthouse.query.dropped metric. # query_buffer is the size of the buffer channel for querying lighthouses
#query_buffer: 1024 #query_buffer: 64
# trigger_buffer is the size of the buffer channel for quickly sending handshakes # trigger_buffer is the size of the buffer channel for quickly sending handshakes
# after receiving the response for lighthouse queries # after receiving the response for lighthouse queries
+5 -18
View File
@@ -31,7 +31,6 @@ package nebula
import ( import (
"bufio" "bufio"
"context"
"fmt" "fmt"
"net" "net"
"strconv" "strconv"
@@ -52,29 +51,17 @@ type graphiteConfigExport struct {
Percentiles []float64 // Percentiles to export from timers and histograms Percentiles []float64 // Percentiles to export from timers and histograms
} }
// graphiteTimeout bounds one submission, the connect and the write together. An export is a few KB, a host that can't
// take that in this long isn't going to.
const graphiteTimeout = 30 * time.Second
// graphiteOnce performs a single submission to Graphite, returning a non-nil // graphiteOnce performs a single submission to Graphite, returning a non-nil
// error if it could not connect or write all of it within timeout. It gives up // error on failed connections.
// as soon as ctx is done. func graphiteOnce(c graphiteConfigExport) error {
func graphiteOnce(ctx context.Context, c graphiteConfigExport, timeout time.Duration) error {
now := time.Now().Unix() now := time.Now().Unix()
du := float64(c.DurationUnit) du := float64(c.DurationUnit)
flushSeconds := float64(c.FlushInterval) / float64(time.Second) flushSeconds := float64(c.FlushInterval) / float64(time.Second)
deadline := time.Now().Add(timeout) conn, err := net.DialTCP("tcp", nil, c.Addr)
d := net.Dialer{Deadline: deadline}
conn, err := d.DialContext(ctx, "tcp", c.Addr.String())
if err != nil { if err != nil {
return err return err
} }
defer conn.Close() defer conn.Close()
if err := conn.SetDeadline(deadline); err != nil {
return err
}
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
defer stop()
w := bufio.NewWriter(conn) w := bufio.NewWriter(conn)
c.Registry.Each(func(name string, i any) { c.Registry.Each(func(name string, i any) {
switch metric := i.(type) { switch metric := i.(type) {
@@ -124,7 +111,7 @@ func graphiteOnce(ctx context.Context, c graphiteConfigExport, timeout time.Dura
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now) fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now)
fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now) fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now)
} }
w.Flush()
}) })
// A failed write sticks, so this reports the first one return nil
return w.Flush()
} }
-315
View File
@@ -1,315 +0,0 @@
package nebula
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"strings"
"sync"
"testing"
"time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// graphiteHost accepts connections and hands each one to the test
type graphiteHost struct {
ln net.Listener
conns chan net.Conn
mu sync.Mutex
closed bool
open []net.Conn
}
// newGraphiteHost accepts connections for the test. A non-zero window shrinks each connection's receive buffer, so a
// sender that is not read from blocks after a little whatever the host's TCP tuning. Only for tests that need a stall,
// a small window makes reading very slow on Linux.
func newGraphiteHost(t *testing.T, window int) *graphiteHost {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
h := &graphiteHost{ln: ln, conns: make(chan net.Conn, 8)}
done := make(chan struct{})
t.Cleanup(func() {
_ = ln.Close()
h.mu.Lock()
h.closed = true
for _, c := range h.open {
_ = c.Close()
}
h.mu.Unlock()
close(done)
})
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
h.mu.Lock()
if h.closed {
h.mu.Unlock()
_ = c.Close()
return
}
h.open = append(h.open, c)
h.mu.Unlock()
if tc, ok := c.(*net.TCPConn); ok && window > 0 {
_ = tc.SetReadBuffer(window)
}
select {
case h.conns <- c:
case <-done:
return
}
}
}()
return h
}
func (h *graphiteHost) addr(t *testing.T) *net.TCPAddr {
a, err := net.ResolveTCPAddr("tcp", h.ln.Addr().String())
require.NoError(t, err)
return a
}
// readExport reads what one send wrote, bounded so a send that never finishes fails the test rather than hanging it
func readExport(t *testing.T, c net.Conn) []byte {
t.Helper()
require.NoError(t, c.SetReadDeadline(time.Now().Add(10*time.Second)))
b, err := io.ReadAll(c)
require.NoError(t, err)
return b
}
func (h *graphiteHost) next(t *testing.T, wait time.Duration) net.Conn {
t.Helper()
select {
case c := <-h.conns:
return c
case <-time.After(wait):
t.Fatal("no connection")
return nil
}
}
// lockedBuffer is a log destination a test can read while the code under test writes to it from another goroutine
type lockedBuffer struct {
mu sync.Mutex
buf bytes.Buffer
}
func (b *lockedBuffer) Write(p []byte) (int, error) {
b.mu.Lock()
defer b.mu.Unlock()
return b.buf.Write(p)
}
func (b *lockedBuffer) String() string {
b.mu.Lock()
defer b.mu.Unlock()
return b.buf.String()
}
func testGraphiteConfig(addr *net.TCPAddr, r metrics.Registry) graphiteConfigExport {
return graphiteConfigExport{Addr: addr, Registry: r, FlushInterval: time.Second, DurationUnit: time.Nanosecond, Prefix: "nebula"}
}
// Enough metrics that one export overflows the socket buffers on any platform, so a send to a host that stops
// reading blocks
func bigGraphiteRegistry(r metrics.Registry, t *testing.T) {
for i := range 200000 {
name := fmt.Sprintf("big.gauge.%d", i)
metrics.GetOrRegisterGauge(name, r).Update(int64(i))
t.Cleanup(func() { r.Unregister(name) })
}
}
func TestGraphiteOnce_sends(t *testing.T) {
r := metrics.NewRegistry()
metrics.GetOrRegisterGauge("g", r).Update(7)
metrics.GetOrRegisterCounter("c", r).Inc(3)
h := newGraphiteHost(t, 0)
require.NoError(t, graphiteOnce(t.Context(), testGraphiteConfig(h.addr(t), r), time.Minute))
b := readExport(t, h.next(t, 5*time.Second))
assert.Contains(t, string(b), "nebula.g.value 7 ")
assert.Contains(t, string(b), "nebula.c.count 3 ")
assert.Contains(t, string(b), "nebula.c.count_ps 3.00 ")
}
// A host that accepts and never reads is given up on, and the failed write is reported
func TestGraphiteOnce_stalledHostTimesOut(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 4096)
done := make(chan error, 1)
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(t.Context(), cfg, 300*time.Millisecond) }()
h.next(t, 5*time.Second) // accepted, never read
select {
case err := <-done:
require.Error(t, err)
case <-time.After(10 * time.Second):
t.Fatal("the send did not give up on a host that stopped reading")
}
}
// The timeout covers the whole send, a host that keeps reading but too slowly is still given up on
func TestGraphiteOnce_timeoutCoversTheWholeSend(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 0)
done := make(chan error, 1)
start := time.Now()
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(t.Context(), cfg, 500*time.Millisecond) }()
c := h.next(t, 5*time.Second)
stopReading := make(chan struct{})
defer close(stopReading)
// 32KB every 10ms keeps every write well inside the timeout, the whole export takes seconds
go func() {
buf := make([]byte, 32<<10)
for {
select {
case <-stopReading:
return
case <-time.After(10 * time.Millisecond):
}
if _, err := c.Read(buf); err != nil {
return
}
}
}()
select {
case err := <-done:
require.Error(t, err)
assert.Less(t, time.Since(start), 5*time.Second)
case <-time.After(10 * time.Second):
t.Fatal("a host that kept reading slowly held the send past its timeout")
}
}
func TestGraphiteOnce_cancelledCtxDoesNotConnect(t *testing.T) {
h := newGraphiteHost(t, 0)
ctx, cancel := context.WithCancel(t.Context())
cancel()
require.Error(t, graphiteOnce(ctx, testGraphiteConfig(h.addr(t), metrics.NewRegistry()), time.Minute))
select {
case <-h.conns:
t.Fatal("connected on a cancelled ctx")
case <-time.After(200 * time.Millisecond):
}
}
// A stop or reload abandons a stuck send rather than waiting out its timeout
func TestGraphiteOnce_cancelAbandonsSend(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 4096)
ctx, cancel := context.WithCancel(t.Context())
done := make(chan error, 1)
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(ctx, cfg, time.Minute) }()
h.next(t, 5*time.Second)
select {
case err := <-done:
t.Fatalf("the send should be stuck on a host that is not reading, it ended with %v", err)
case <-time.After(200 * time.Millisecond):
}
cancel()
select {
case err := <-done:
require.Error(t, err)
case <-time.After(5 * time.Second):
t.Fatal("the send is still stuck after its ctx was cancelled")
}
}
// buildRuntime hands graphite the configured prefix and interval
func TestStatsServer_graphiteExportUsesConfig(t *testing.T) {
const name = "graphite.wiring.test"
metrics.GetOrRegisterCounter(name, nil).Inc(8)
t.Cleanup(func() { metrics.DefaultRegistry.Unregister(name) })
h := newGraphiteHost(t, 0)
s, c := newTestStatsServer(t)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "4s", "prefix": "pfx", "host": h.ln.Addr().String()})
cfg, err := loadStatsConfig(c)
require.NoError(t, err)
s.runMu.Lock()
fns, _ := s.buildRuntime(t.Context(), cfg)
s.runMu.Unlock()
for _, fn := range fns {
fn()
}
b := readExport(t, h.next(t, 5*time.Second))
assert.Contains(t, string(b), "pfx."+name+".count 8 ")
assert.Contains(t, string(b), "pfx."+name+".count_ps 2.00 ")
}
func TestStatsServer_graphiteFailureIsLogged(t *testing.T) {
logs := &lockedBuffer{}
s, c := newTestStatsServer(t)
s.l = test.NewLoggerWithOutput(logs)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)})
cfg, err := loadStatsConfig(c)
require.NoError(t, err)
s.runMu.Lock()
fns, _ := s.buildRuntime(t.Context(), cfg)
s.runMu.Unlock()
for _, fn := range fns {
fn()
}
assert.Contains(t, logs.String(), "Graphite export failed")
}
// Stop abandons the running export, so it goes through Start rather than handing buildRuntime a ctx, and a stop is
// not logged as a failure
func TestStatsServer_stopAbandonsGraphiteSend(t *testing.T) {
bigGraphiteRegistry(metrics.DefaultRegistry, t)
h := newGraphiteHost(t, 4096)
logs := &lockedBuffer{}
s, c := newTestStatsServer(t)
s.l = test.NewLoggerWithOutput(logs)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "50ms", "host": h.ln.Addr().String()})
require.NoError(t, s.reload(c, true))
started := make(chan struct{})
go func() {
s.Start()
close(started)
}()
stuck := h.next(t, 30*time.Second)
s.Stop()
select {
case <-started:
case <-time.After(5 * time.Second):
t.Fatal("Start did not return after Stop")
}
// A closed sender resets the connection once it sees this, one still sending just buffers it. Either way the
// read below doesn't have to drain the export through the small window to find out
_, err := stuck.Write([]byte("x"))
require.NoError(t, err)
require.NoError(t, stuck.SetReadDeadline(time.Now().Add(10*time.Second)))
n, err := io.Copy(io.Discard, stuck)
var ne net.Error
if errors.As(err, &ne) && ne.Timeout() {
t.Fatalf("the send kept the connection after Stop, read %d bytes", n)
}
assert.Less(t, n, int64(1<<20), "the send kept going after Stop")
// The capture loop logs on its own goroutine after Start has returned, give it the chance
assert.Never(t, func() bool { return strings.Contains(logs.String(), "Graphite export failed") }, 500*time.Millisecond, 10*time.Millisecond, "a stop is not a failure")
}
+11 -8
View File
@@ -54,8 +54,7 @@ type HandshakeConfig struct {
} }
type HandshakeManager struct { type HandshakeManager struct {
// Mutex for interacting with the vpnIps and indexes maps. // Mutex for interacting with the vpnIps and indexes maps
// Lock order is hostmap, then this lock, then the lighthouse, then a RemoteList.
sync.RWMutex sync.RWMutex
vpnIps map[netip.Addr]*HandshakeHostInfo vpnIps map[netip.Addr]*HandshakeHostInfo
@@ -260,6 +259,14 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
"style": header.SubTypeName(header.Handshake, hh.machine.Subtype()), "style": header.SubTypeName(header.Handshake, hh.machine.Subtype()),
} }
// Get a remotes object if we don't already have one.
// This is mainly to protect us as this should never be the case
// NB ^ This comment doesn't jive. It's how the thing gets initialized.
// It's the common path. Should it update every time, in case a future LH query/queries give us more info?
if hostinfo.remotes == nil {
hostinfo.remotes = hm.lightHouse.QueryCache([]netip.Addr{vpnIp})
}
remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges()) remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges())
remotesHaveChanged := !slices.Equal(remotes, hh.lastRemotes) remotesHaveChanged := !slices.Equal(remotes, hh.lastRemotes)
@@ -364,9 +371,6 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
hostinfo := &HostInfo{ hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr}, vpnAddrs: []netip.Addr{vpnAddr},
HandshakePacket: make(map[uint8][]byte, 0), HandshakePacket: make(map[uint8][]byte, 0),
// Attached here so a pending hostinfo never exists without one: SetRemote dereferences
// it, and create-tunnel -address can reach SetRemote before the handshake loop runs.
remotes: hm.lightHouse.QueryCache([]netip.Addr{vpnAddr}),
relayState: RelayState{ relayState: RelayState{
relays: nil, relays: nil,
relayForByAddr: map[netip.Addr]*Relay{}, relayForByAddr: map[netip.Addr]*Relay{},
@@ -942,14 +946,13 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
hm.DeleteHostInfo(hostinfo) hm.DeleteHostInfo(hostinfo)
hm.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) { hm.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) {
newHH.hostinfo.remotes = hostinfo.remotes
newHH.hostinfo.remotes.BlockRemote(via) newHH.hostinfo.remotes.BlockRemote(via)
newHH.packetStore = hh.packetStore newHH.packetStore = hh.packetStore
hh.packetStore = []*cachedPacket{} hh.packetStore = []*cachedPacket{}
})
// sendCloseTunnel may take the hostmap lock, so it must not run under the handshake
// manager lock that StartHandshake holds around the callback.
hostinfo.vpnAddrs = vpnAddrs hostinfo.vpnAddrs = vpnAddrs
f.sendCloseTunnel(hostinfo) f.sendCloseTunnel(hostinfo)
})
return return
} }
+2 -68
View File
@@ -11,7 +11,6 @@ import (
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func Test_NewHandshakeManagerVpnIp(t *testing.T) { func Test_NewHandshakeManagerVpnIp(t *testing.T) {
@@ -43,6 +42,8 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
i2 := blah.StartHandshake(ip, nil) i2 := blah.StartHandshake(ip, nil)
assert.Same(t, i, i2) assert.Same(t, i, i2)
i.remotes = NewRemoteList([]netip.Addr{}, nil)
// Adding something to pending should not affect the main hostmap // Adding something to pending should not affect the main hostmap
assert.Empty(t, mainHM.Hosts) assert.Empty(t, mainHM.Hosts)
@@ -234,70 +235,3 @@ func TestHandleIncomingDispatch(t *testing.T) {
assert.Empty(t, hm.indexes, "orphan stage-2 must not create state") assert.Empty(t, hm.indexes, "orphan stage-2 must not create state")
}) })
} }
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
func Test_PendingHandshakeHasRemotes(t *testing.T) {
l := test.NewLogger()
mainHM := newHostMap(l)
preferredRanges := []netip.Prefix{}
mainHM.preferredRanges.Store(&preferredRanges)
hm := NewHandshakeManager(l, mainHM, newTestLighthouse(), &udp.NoopConn{}, defaultHandshakeConfig)
vpnIp := netip.MustParseAddr("172.1.1.2")
remote := netip.MustParseAddrPort("192.168.1.9:4242")
hi := hm.StartHandshake(vpnIp, func(hh *HandshakeHostInfo) {
require.NotNil(t, hh.hostinfo.remotes, "a pending handshake must never be without its RemoteList")
hh.hostinfo.SetRemote(remote)
})
assert.Contains(t, hi.remotes.CopyAddrs(preferredRanges), remote, "the handshake must be sent to the supplied remote")
assert.Equal(t, remote, hi.GetRemote())
// A RemoteList keeps one learned address per host, so a second remote replaces the
// first — the same behaviour change-remote has on an established tunnel.
other := netip.MustParseAddrPort("192.168.1.10:4242")
hi.SetRemote(other)
assert.Equal(t, []netip.AddrPort{other}, hi.remotes.CopyAddrs(preferredRanges))
assert.Equal(t, other, hi.GetRemote())
}
// newRemotesRaceFixture builds a handshake manager with an established tunnel to vpnAddr whose
// RemoteList is the lighthouse's cached list, the state every teardown-during-handshake race starts from.
func newRemotesRaceFixture(t *testing.T, vpnAddr netip.Addr) (*HandshakeManager, *Interface, *HostInfo) {
t.Helper()
l := test.NewLogger()
mainHM := newHostMap(l)
preferredRanges := []netip.Prefix{}
mainHM.preferredRanges.Store(&preferredRanges)
lh := newTestLighthouse()
hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig)
f := &Interface{hostMap: mainHM, lightHouse: lh, handshakeManager: hm, l: l}
hm.f = f
established := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
localIndexId: 1,
remoteIndexId: 101,
remotes: lh.QueryCache([]netip.Addr{vpnAddr}),
HandshakePacket: map[uint8][]byte{},
}
mainHM.Lock()
mainHM.unlockedAddHostInfo(established, f)
mainHM.Unlock()
return hm, f, established
}
// With no handshake pending, tearing down the last tunnel still evicts the list.
func Test_CloseTunnelEvictsWithoutPendingHandshake(t *testing.T) {
vpnAddr := netip.MustParseAddr("172.1.1.2")
_, f, established := newRemotesRaceFixture(t, vpnAddr)
f.closeTunnel(established)
assert.NotContains(t, f.hostMap.Hosts, vpnAddr)
assert.NotContains(t, f.lightHouse.addrMap, vpnAddr)
}
-5
View File
@@ -24,11 +24,6 @@ type m = map[string]any
const ( const (
Version uint8 = 1 Version uint8 = 1
Len = 16 Len = 16
// MaxOverhead is the most a payload grows on its way to the underlay.
// A relayed packet has two headers and two AEAD tags, one for the inner tunnel and another for the relay tunnel.
// Every supported cipher (today) has a 16 byte tag.
MaxOverhead = 2 * (Len + 16)
) )
type MessageType uint8 type MessageType uint8
+16
View File
@@ -69,6 +69,8 @@ type HostMap struct {
Hosts map[netip.Addr]*HostInfo Hosts map[netip.Addr]*HostInfo
moreHosts map[netip.Addr][]*HostInfo moreHosts map[netip.Addr][]*HostInfo
preferredRanges atomic.Pointer[[]netip.Prefix] preferredRanges atomic.Pointer[[]netip.Prefix]
// established hears about each hostinfo that joins or leaves the hostmap; nil when no conn wants to know.
established *establishedRemotes
l *slog.Logger l *slog.Logger
} }
@@ -276,6 +278,11 @@ type HostInfo struct {
// This value will be behind against actual tunnel utilization in the hot path. // This value will be behind against actual tunnel utilization in the hot path.
// This should only be used by the ConnectionManagers ticker routine. // This should only be used by the ConnectionManagers ticker routine.
lastUsed time.Time lastUsed time.Time
// establishedIn is set while the hostinfo is in a main hostmap whose conns track established remotes, and
// establishedRemote is the remote it holds a reference on there, guarded by establishedIn's mu.
establishedIn atomic.Pointer[establishedRemotes]
establishedRemote netip.AddrPort
} }
type ViaSender struct { type ViaSender struct {
@@ -516,6 +523,7 @@ func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) bool {
if len(hm.Indexes) == 0 { if len(hm.Indexes) == 0 {
hm.Indexes = map[uint32]*HostInfo{} hm.Indexes = map[uint32]*HostInfo{}
} }
hm.established.remove(hostinfo)
if hm.l.Enabled(context.Background(), slog.LevelDebug) { if hm.l.Enabled(context.Background(), slog.LevelDebug) {
hm.l.Debug("Hostmap hostInfo deleted", hm.l.Debug("Hostmap hostInfo deleted",
@@ -662,6 +670,7 @@ func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
hm.Indexes[hostinfo.localIndexId] = hostinfo hm.Indexes[hostinfo.localIndexId] = hostinfo
hm.RemoteIndexes[hostinfo.remoteIndexId] = hostinfo hm.RemoteIndexes[hostinfo.remoteIndexId] = hostinfo
hm.established.add(hostinfo)
hostinfo.markOut(f.rebindEpoch.Load()) hostinfo.markOut(f.rebindEpoch.Load())
if f.connectionManager != nil { // f.connectionManager is only nil in some unit tests if f.connectionManager != nil { // f.connectionManager is only nil in some unit tests
@@ -842,6 +851,13 @@ func (i *HostInfo) SetRemote(remote netip.AddrPort) {
if i.GetRemote() != remote { if i.GetRemote() != remote {
i.remote.Store(&remote) i.remote.Store(&remote)
i.remotes.LearnRemote(i.vpnAddrs[0], remote) i.remotes.LearnRemote(i.vpnAddrs[0], remote)
if e := i.establishedIn.Load(); e != nil {
e.mu.Lock()
if i.establishedIn.Load() == e {
e.syncLocked(i)
}
e.mu.Unlock()
}
} }
} }
+12 -29
View File
@@ -10,7 +10,6 @@ import (
"github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/noiseutil" "github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
@@ -199,8 +198,8 @@ func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt tio.Packet, nb []b
} }
err = tio.SegmentSuperpacket(pkt, func(seg []byte) error { err = tio.SegmentSuperpacket(pkt, func(seg []byte) error {
//relay header + header + plaintext + AEAD tag + relay tag //relay header + header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305) + relay tag
scratch := sendBatch.Reserve(len(seg) + header.MaxOverhead) scratch := sendBatch.Reserve(header.Len + header.Len + len(seg) + 16 + 16)
innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb) innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb)
if innerPacket == nil { if innerPacket == nil {
@@ -518,9 +517,7 @@ func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, noc
err = f.writers[q].WriteTo(toSend, via.GetRemote()) err = f.writers[q].WriteTo(toSend, via.GetRemote())
if err != nil { if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpRelayTxErrLimiter); ok { via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err)
via.logger(l).Info("Failed to WriteTo in sendVia", "error", err)
}
} }
} }
@@ -529,26 +526,16 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
return return
} }
useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid() useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid()
// The encrypt appends to out in place and a relayed packet is then wrapped in place,
// so out has to fit the whole packet up front or the encrypt reallocates out from under SendVia.
// We cap the MTU of tun devices and routes we create ourselves, so only a host that set a bigger MTU than we allow gets here.
need := header.Len + len(p) + ci.eKey.Overhead()
if useRelay {
// The relay tunnel runs the same cipher, and adds its own header and tag
need += header.Len + ci.eKey.Overhead()
}
if cap(out) < need {
if l, ok := logging.RateLimited(f.l, f.txOversizeLimiter); ok {
hostinfo.logger(l).Error("Dropping outbound packet, out buffer not large enough", "outCap", cap(out), "payloadLen", len(p), "relayed", useRelay)
}
return
}
fullOut := out fullOut := out
if useRelay { if useRelay {
if len(out) < header.Len {
// out always has a capacity of mtu, but not always a length greater than the header.Len.
// Grow it to make sure the next operation works.
out = out[:header.Len]
}
// Save a header's worth of data at the front of the 'out' buffer. // Save a header's worth of data at the front of the 'out' buffer.
out = out[header.Len:header.Len] out = out[header.Len:]
} }
if noiseutil.EncryptLockNeeded { if noiseutil.EncryptLockNeeded {
@@ -595,23 +582,19 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
if remote.IsValid() { if remote.IsValid() {
err = f.writers[q].WriteTo(out, remote) err = f.writers[q].WriteTo(out, remote)
if err != nil { if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok { hostinfo.logger(f.l).Error("Failed to write outgoing packet",
hostinfo.logger(l).Error("Failed to write outgoing packet",
"error", err, "error", err,
"udpAddr", remote, "udpAddr", remote,
) )
} }
}
} else if hr := hostinfo.GetRemote(); hr.IsValid() { } else if hr := hostinfo.GetRemote(); hr.IsValid() {
err = f.writers[q].WriteTo(out, hr) err = f.writers[q].WriteTo(out, hr)
if err != nil { if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok { hostinfo.logger(f.l).Error("Failed to write outgoing packet",
hostinfo.logger(l).Error("Failed to write outgoing packet",
"error", err, "error", err,
"udpAddr", hr, "udpAddr", hr,
) )
} }
}
} else { } else {
// Try to send via a relay // Try to send via a relay
for _, relayIP := range hostinfo.relayState.CopyRelayIps() { for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
+18 -24
View File
@@ -24,7 +24,6 @@ import (
"github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/overlay"
"github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/ratelimit"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
) )
@@ -125,10 +124,6 @@ type Interface struct {
batchers []*batch.MultiCoalescer batchers []*batch.MultiCoalescer
wg sync.WaitGroup wg sync.WaitGroup
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
txOversizeLimiter *ratelimit.Limiter // oversized outbound packet drop log limiter
// fatalErr holds the first unexpected reader error that caused shutdown. // fatalErr holds the first unexpected reader error that caused shutdown.
// nil means "no fatal error" (yet) // nil means "no fatal error" (yet)
fatalErr atomic.Pointer[error] fatalErr atomic.Pointer[error]
@@ -203,19 +198,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
return nil, errors.New("no connection manager") return nil, errors.New("no connection manager")
} }
udpTxErrLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
udpRelayTxErrLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
txOversizeLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
if c.routines <= 1 { if c.routines <= 1 {
c.PinThreads = false //pinning is not useful unless there's more than one tun reader c.PinThreads = false //pinning is not useful unless there's more than one tun reader
} }
@@ -248,9 +230,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
conntrackCacheTimeout: c.ConntrackCacheTimeout, conntrackCacheTimeout: c.ConntrackCacheTimeout,
cpuAffinity: c.CpuAffinity, cpuAffinity: c.CpuAffinity,
pinThreads: c.PinThreads, pinThreads: c.PinThreads,
udpTxErrLimiter: udpTxErrLimiter,
udpRelayTxErrLimiter: udpRelayTxErrLimiter,
txOversizeLimiter: txOversizeLimiter,
metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)), metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
metricTxDropped: metrics.GetOrRegisterCounter("udp.tx.dropped", nil), metricTxDropped: metrics.GetOrRegisterCounter("udp.tx.dropped", nil),
@@ -645,15 +624,17 @@ func (f *Interface) reloadMisc(c *config.C) {
} }
} }
// statsEmitter updates the interface's gauges, the stats server runs it on its capture loop. func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
func (f *Interface) statsEmitter() func() { ticker := time.NewTicker(i)
defer ticker.Stop()
udpStats := udp.NewUDPStatsEmitter(f.writers) udpStats := udp.NewUDPStatsEmitter(f.writers)
certExpirationGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil) certExpirationGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil)
certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil) certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil)
certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil) certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil)
return func() { emit := func() {
f.firewall.EmitStats() f.firewall.EmitStats()
f.handshakeManager.EmitStats() f.handshakeManager.EmitStats()
udpStats() udpStats()
@@ -670,6 +651,19 @@ func (f *Interface) statsEmitter() func() {
certMaxVersion.Update(int64(certState.v1Cert.Version())) certMaxVersion.Update(int64(certState.v1Cert.Version()))
} }
} }
// Prime gauges so a Prometheus scrape that lands before the first tick
// sees real values instead of the zero defaults (issue #907).
emit()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
emit()
}
}
} }
func (f *Interface) GetHostInfo(vpnIp netip.Addr) *HostInfo { func (f *Interface) GetHostInfo(vpnIp netip.Addr) *HostInfo {
+14 -5
View File
@@ -3,6 +3,7 @@
package nebula package nebula
import ( import (
"context"
"net/netip" "net/netip"
"testing" "testing"
"time" "time"
@@ -17,8 +18,11 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
// Priming before the first tick (issue #907) is Start's job, see TestStatsServer_Start_primes. // Test_emitStats_primesGauges covers issue #907: a Prometheus scrape that
func Test_statsEmitter_setsGauges(t *testing.T) { // landed before the first ticker fire used to read 0 for the cert gauges.
// emitStats now primes the gauges before entering the ticker loop. We assert
// the gauge is zero before the first call and non-zero after.
func Test_emitStats_primesGauges(t *testing.T) {
defer metrics.DefaultRegistry.UnregisterAll() defer metrics.DefaultRegistry.UnregisterAll()
l := test.NewLogger() l := test.NewLogger()
@@ -52,12 +56,17 @@ func Test_statsEmitter_setsGauges(t *testing.T) {
ifce.pki.cs.Store(cs) ifce.pki.cs.Store(cs)
ttlGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil) ttlGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil)
require.Zero(t, ttlGauge.Value(), "gauge should be zero before the emitter runs") require.Zero(t, ttlGauge.Value(), "gauge should be zero before emitStats runs")
ifce.statsEmitter()() // Pre-cancel the context so emitStats returns after priming the gauges
// without ever reading from ticker.C. The one hour interval is just a
// belt-and-suspenders, the test does not expect the ticker to fire.
ctx, cancel := context.WithCancel(context.Background())
cancel()
ifce.emitStats(ctx, time.Hour)
ttl := ttlGauge.Value() ttl := ttlGauge.Value()
assert.Positive(t, ttl, "ttl gauge should be set by one run of the emitter") assert.Positive(t, ttl, "ttl gauge should be primed by emitStats before its first tick")
assert.LessOrEqual(t, ttl, int64(3600)) assert.LessOrEqual(t, ttl, int64(3600))
assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.initiating_version", nil).Value()) assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.initiating_version", nil).Value())
assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.max_version", nil).Value()) assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.max_version", nil).Value())
+5 -28
View File
@@ -15,7 +15,6 @@ import (
"time" "time"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
@@ -82,8 +81,6 @@ type LightHouse struct {
updateTrigger chan struct{} updateTrigger chan struct{}
queryChan chan netip.Addr queryChan chan netip.Addr
// queryDropped counts queries QueryServer dropped because queryChan was full.
queryDropped metrics.Counter
calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote
@@ -119,8 +116,7 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
nebulaPort: nebulaPort, nebulaPort: nebulaPort,
punchy: p, punchy: p,
updateTrigger: make(chan struct{}, 1), updateTrigger: make(chan struct{}, 1),
queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 1024)), queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
queryDropped: metrics.GetOrRegisterCounter("lighthouse.query.dropped", nil),
l: l, l: l,
} }
h.localAddrsFn = func(al *LocalAllowList) []netip.Addr { h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
@@ -359,13 +355,7 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
} }
} }
// am_relay decides whether relay.relays applies to us at all if initial || c.HasChanged("relay.relays") {
if initial || c.HasChanged("relay.relays") || c.HasChanged("relay.am_relay") {
var before []netip.Addr
if !initial {
before = lh.GetRelaysForMe()
}
switch c.GetBool("relay.am_relay", false) { switch c.GetBool("relay.am_relay", false) {
case true: case true:
// Relays aren't allowed to specify other relays // Relays aren't allowed to specify other relays
@@ -390,11 +380,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
} }
lh.relaysForMe.Store(&relaysForMe) lh.relaysForMe.Store(&relaysForMe)
} }
// Tell the lighthouses now rather than at the next interval, but only when what we advertise changed
if !initial && !slices.Equal(before, lh.GetRelaysForMe()) {
lh.TriggerUpdate()
}
} }
return nil return nil
@@ -527,19 +512,14 @@ func (lh *LightHouse) Query(vpnAddr netip.Addr) *RemoteList {
return nil return nil
} }
// QueryServer is asynchronous and non-blocking, no reply should be expected. // QueryServer is asynchronous so no reply should be expected
// If the query channel is full, the query is dropped.
func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) { func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) {
// Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses // Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses
if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) { if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) {
return return
} }
select { lh.queryChan <- vpnAddr
case lh.queryChan <- vpnAddr:
default:
lh.queryDropped.Inc(1)
}
} }
func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList { func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList {
@@ -1255,13 +1235,10 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []neti
// sendHostPunchNotification signals the other side to punch some zero byte udp packets // sendHostPunchNotification signals the other side to punch some zero byte udp packets
func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, punchNotifDest netip.Addr, w EncWriter) { func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, punchNotifDest netip.Addr, w EncWriter) {
whereToPunch := fromVpnAddrs[0] whereToPunch := fromVpnAddrs[0]
// Resolve the target before taking the querier's RemoteList lock below. GetHostInfo takes the
// hostmap lock, and the hostmap is taken before the lighthouse, which is taken before a
// RemoteList, so looking it up inside the callback would deadlock.
targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest)
found, ln, err := lhh.lh.queryAndPrepMessage(whereToPunch, func(c *cache) (int, error) { found, ln, err := lhh.lh.queryAndPrepMessage(whereToPunch, func(c *cache) (int, error) {
n = lhh.resetMeta() n = lhh.resetMeta()
n.Type = NebulaMeta_HostPunchNotification n.Type = NebulaMeta_HostPunchNotification
targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest)
var useVersion cert.Version var useVersion cert.Version
if targetHI == nil { if targetHI == nil {
useVersion = lhh.lh.ifce.GetCertState().initiatingVersion useVersion = lhh.lh.ifce.GetCertState().initiatingVersion
-202
View File
@@ -6,16 +6,13 @@ import (
"errors" "errors"
"fmt" "fmt"
"net/netip" "net/netip"
"sync"
"testing" "testing"
"time"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.yaml.in/yaml/v3" "go.yaml.in/yaml/v3"
@@ -798,202 +795,3 @@ func TestLightHouse_logLocalAddrsErr(t *testing.T) {
lh.logLocalAddrsErr(errors.New("something else")) lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN") assert.Contains(t, out.String(), "level=WARN")
} }
// heldEncWriter holds the query worker in its first GetHostInfo until release closes.
type heldEncWriter struct {
mockEncWriter
once sync.Once
held chan struct{}
release chan struct{}
}
func (w *heldEncWriter) GetHostInfo(netip.Addr) *HostInfo {
w.once.Do(func() { close(w.held) })
<-w.release
return nil
}
// QueryServer never waits on the query worker, since the packet readers call it and nothing drains the channel at
// shutdown; a query that finds the channel full is dropped and counted. The worker is held on its first query so the
// channel fills.
func TestLighthouse_QueryServerDoesNotBlock(t *testing.T) {
l := test.NewLogger()
cs := testCertState(netip.MustParsePrefix("10.128.0.1/16"))
lh1 := "10.128.0.2"
c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}}
c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
c.Settings["handshakes"] = map[string]any{"query_buffer": 1}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err)
w := &heldEncWriter{held: make(chan struct{}), release: make(chan struct{})}
lh.ifce = w
t.Cleanup(func() { close(w.release) })
lh.QueryServer(netip.MustParseAddr("10.128.0.10"))
select {
case <-w.held:
case <-time.After(5 * time.Second):
t.Fatal("the query worker never took the first query")
}
dropped := lh.queryDropped.Count()
done := make(chan struct{})
go func() {
lh.QueryServer(netip.MustParseAddr("10.128.0.11"))
lh.QueryServer(netip.MustParseAddr("10.128.0.12"))
close(done)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("QueryServer blocked on a full query channel")
}
assert.EqualValues(t, 1, lh.queryDropped.Count()-dropped)
}
// hostmapEncWriter answers GetHostInfo from a real hostmap, the way Interface does.
type hostmapEncWriter struct {
mockEncWriter
hm *HostMap
}
func (w *hostmapEncWriter) GetHostInfo(a netip.Addr) *HostInfo { return w.hm.QueryVpnAddr(a) }
func (w *hostmapEncWriter) GetCertState() *CertState {
return &CertState{initiatingVersion: cert.Version1}
}
// A lighthouse runs four paths that each hold one lock while waiting for the next: CheckAndComplete
// holds the hostmap and takes the handshake manager, StartHandshake holds the handshake manager and
// takes the lighthouse, a host update holds the lighthouse and takes the sender's RemoteList, and a
// punch notification holds the querier's RemoteList while it resolves the target. If that last step
// reaches the hostmap under the RemoteList lock, the four deadlock. Drive them all at once.
func TestLighthouse_PunchNotificationDoesNotCloseLockCycle(t *testing.T) {
l := test.NewLogger()
myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true}
c.Settings["listen"] = map[string]any{"port": 4242}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(myVpnNet), nil, nil)
require.NoError(t, err)
mainHM := newHostMap(l)
pr := []netip.Prefix{}
mainHM.preferredRanges.Store(&pr)
lh.ifce = &hostmapEncWriter{hm: mainHM}
hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig)
f := &Interface{hostMap: mainHM, lightHouse: lh, handshakeManager: hm, l: l}
hm.f = f
peer := netip.MustParseAddr("10.128.0.2")
peerUdp := netip.MustParseAddrPort("10.0.0.2:4242")
target := netip.MustParseAddr("10.128.0.3")
targetUdp := netip.MustParseAddrPort("10.0.0.3:4242")
outbound := netip.MustParseAddr("10.128.0.4")
inbound := netip.MustParseAddr("10.128.0.5")
seed := lh.NewRequestHandler()
newLHHostUpdate(peerUdp, peer, []netip.AddrPort{peerUdp}, seed)
newLHHostUpdate(targetUdp, target, []netip.AddrPort{targetUdp}, seed)
stop := make(chan struct{})
var wg sync.WaitGroup
loop := func(body func()) {
defer wg.Done()
for {
select {
case <-stop:
return
default:
body()
}
}
}
wg.Add(4)
// Outbound handshakes keep starting and being dropped.
go loop(func() {
hm.DeleteHostInfo(hm.StartHandshake(outbound, nil))
})
// Inbound handshakes keep completing and being torn down.
var idx uint32
go loop(func() {
idx++
hi := &HostInfo{
vpnAddrs: []netip.Addr{inbound},
localIndexId: idx,
remoteIndexId: idx,
remotes: lh.QueryCache([]netip.Addr{inbound}),
HandshakePacket: map[uint8][]byte{handshakePacketStage0: {byte(idx)}},
}
if _, err := hm.CheckAndComplete(hi, handshakePacketStage0, f); err == nil {
mainHM.DeleteHostInfo(hi)
}
})
// The peer keeps sending host updates.
updates := lh.NewRequestHandler()
go loop(func() {
newLHHostUpdate(peerUdp, peer, []netip.AddrPort{peerUdp}, updates)
})
// The same peer keeps querying for the target, which ends in a punch notification.
queries := lh.NewRequestHandler()
go loop(func() {
sendLHHostRequest(peerUdp, peer, target, queries, nil)
})
time.Sleep(time.Second)
close(stop)
done := make(chan struct{})
go func() { wg.Wait(); close(done) }()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("deadlock: the lighthouse paths never finished after stop was signalled")
}
}
func TestLighthouse_reloadRelaysTriggersUpdate(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
base := "lighthouse:\n am_lighthouse: true\nlisten:\n port: 4242\n"
require.NoError(t, c.LoadString(base+"relay:\n relays: [10.128.0.5]\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
assert.Empty(t, lh.updateTrigger)
require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6]\n"))
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
<-lh.updateTrigger
// The config changed but what we advertise did not, a bad entry is skipped
require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6, nope]\n"))
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe())
assert.Empty(t, lh.updateTrigger)
}
func TestLighthouse_reloadAmRelayAlone(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
base := "lighthouse:\n am_lighthouse: true\nlisten:\n port: 4242\nrelay:\n relays: [10.128.0.5]\n"
require.NoError(t, c.LoadString(base+" am_relay: false\n"))
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe())
// Relays aren't allowed to have relays, flipping am_relay alone must drop them and tell the lighthouses
require.NoError(t, c.ReloadConfigString(base+" am_relay: true\n"))
assert.Empty(t, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
<-lh.updateTrigger
require.NoError(t, c.ReloadConfigString(base+" am_relay: false\n"))
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
}
-31
View File
@@ -1,31 +0,0 @@
package logging
import "log/slog"
// RateLimiter is what RateLimited needs from a rate limiter, such as a *ratelimit.Limiter.
// Allow reports whether this call may proceed, and how many calls were denied since the last one that did.
type RateLimiter interface {
Allow() (denied uint64, ok bool)
}
// RateLimited checks lim before anything is built.
// When allowed it returns l, with a "suppressed" count attached if lim denied calls since its last allowed one.
// A nil lim allows everything.
//
// if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok {
// hostinfo.logger(l).Error("Failed to write outgoing packet", "error", err)
// }
func RateLimited(l *slog.Logger, lim RateLimiter) (*slog.Logger, bool) {
if lim == nil {
return l, true
}
denied, ok := lim.Allow()
if !ok {
return nil, false
}
if denied > 0 {
return l.With("suppressed", denied), true
}
return l, true
}
-106
View File
@@ -1,106 +0,0 @@
package logging
import (
"bytes"
"errors"
"io"
"log/slog"
"net/netip"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/ratelimit"
)
type fakeLimiter struct {
denied uint64
ok bool
}
func (f fakeLimiter) Allow() (uint64, bool) { return f.denied, f.ok }
func TestRateLimited(t *testing.T) {
var buf bytes.Buffer
base := NewLogger(&buf)
if l, ok := RateLimited(base, fakeLimiter{ok: false}); ok || l != nil {
t.Fatalf("denied call: ok=%v logger=%v", ok, l)
}
if l, ok := RateLimited(base, fakeLimiter{ok: true}); !ok || l != base {
t.Fatal("nothing denied should hand back the same logger")
}
if l, ok := RateLimited(base, nil); !ok || l != base {
t.Fatal("a nil limiter should allow with the same logger")
}
l, ok := RateLimited(base, fakeLimiter{denied: 7, ok: true})
if !ok {
t.Fatal("allowed call denied")
}
l.Info("hello", "k", "v")
if !strings.Contains(buf.String(), "suppressed=7") {
t.Fatalf("want suppressed=7, got %q", buf.String())
}
}
// The benchmarks compare a log line through slog directly against the same line behind RateLimited: logged with
// nothing denied, denied, and logged after denials.
var (
benchErr = errors.New("sendto: no buffer space available")
benchAddr = netip.MustParseAddrPort("192.0.2.1:4242")
)
func benchLogger() *slog.Logger {
return NewLogger(io.Discard).With("localIndex", 1234, "remoteIndex", 5678)
}
func BenchmarkRateLimitedBaseline(b *testing.B) {
lg := benchLogger()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
lg.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
func BenchmarkRateLimitedAllowed(b *testing.B) {
lg, lim := benchLogger(), mustLimiter(b, 1, 0)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func BenchmarkRateLimitedDenied(b *testing.B) {
lg, lim := benchLogger(), mustLimiter(b, 1, time.Hour)
lim.Allow()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func BenchmarkRateLimitedAfterDenied(b *testing.B) {
lg, lim := benchLogger(), fakeLimiter{denied: 5, ok: true}
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func mustLimiter(b *testing.B, burst int, every time.Duration) *ratelimit.Limiter {
l, err := ratelimit.New(burst, every)
if err != nil {
b.Fatal(err)
}
return l
}
+4 -5
View File
@@ -213,6 +213,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
} }
hostMap := NewHostMapFromConfig(l, c) hostMap := NewHostMapFromConfig(l, c)
hostMap.established = newEstablishedRemotes(udpConns)
punchy := NewPunchyFromConfig(l, c, udpConns[0]) punchy := NewPunchyFromConfig(l, c, udpConns[0])
connManager := newConnectionManagerFromConfig(l, c, hostMap, punchy) connManager := newConnectionManagerFromConfig(l, c, hostMap, punchy)
lightHouse, err := NewLightHouseFromConfig(ctx, l, c, pki.getCertState(), udpConns[0], punchy) lightHouse, err := NewLightHouseFromConfig(ctx, l, c, pki.getCertState(), udpConns[0], punchy)
@@ -300,7 +301,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
} }
var ifce *Interface var ifce *Interface
var emitStats func()
if !configTest { if !configTest {
ifce, err = NewInterface(ctx, ifConfig) ifce, err = NewInterface(ctx, ifConfig)
if err != nil { if err != nil {
@@ -319,12 +319,9 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
go handshakeManager.Run(ctx) go handshakeManager.Run(ctx)
punchy.Start(ctx, ifce, hostMap, lightHouse) punchy.Start(ctx, ifce, hostMap, lightHouse)
// On the stats server's loop, so the gauges follow the interval it accepted
emitStats = ifce.statsEmitter()
} }
stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest, emitStats) stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest)
if err != nil { if err != nil {
return nil, util.ContextualizeIfNeeded("Failed to start stats emitter", err) return nil, util.ContextualizeIfNeeded("Failed to start stats emitter", err)
} }
@@ -333,6 +330,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
return nil, nil return nil, nil
} }
go ifce.emitStats(ctx, c.GetDuration("stats.interval", time.Second*10))
attachCommands(l, c, ssh, ifce) attachCommands(l, c, ssh, ifce)
networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c) networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
+6 -9
View File
@@ -160,7 +160,9 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
case header.TestReply: case header.TestReply:
// No-op, useful for the Roaming and connectionManager side-effects above // No-op, useful for the Roaming and connectionManager side-effects above
case header.TestRequest: case header.TestRequest:
if header.MaxOverhead+len(out) > len(rxc.scratch) { const maxCipherOverhead = 16 //todo we use this too often, needs a real importable const
const maxOverhead = header.Len + header.Len + maxCipherOverhead + maxCipherOverhead
if maxOverhead+len(out) > len(rxc.scratch) {
// A reply that cannot fit in scratch is dropped no matter the log level. // A reply that cannot fit in scratch is dropped no matter the log level.
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("dropping oversized test request", "payloadLen", len(out), "from", via) hostinfo.logger(f.l).Debug("dropping oversized test request", "payloadLen", len(out), "from", via)
@@ -261,16 +263,11 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
// closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
func (f *Interface) closeTunnel(hostInfo *HostInfo) { func (f *Interface) closeTunnel(hostInfo *HostInfo) {
// Hold the hostmap lock across both the delete and the lighthouse eviction, final := f.hostMap.DeleteHostInfo(hostInfo)
// to avoid a race with handshake completion. if final {
f.hostMap.Lock()
defer f.hostMap.Unlock()
if !f.hostMap.unlockedDeleteHostInfo(hostInfo) {
return
}
// We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage // We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage
f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs) f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs)
}
} }
// sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote // sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote
-57
View File
@@ -1,57 +0,0 @@
//go:build !e2e_testing
package overlay
import (
"fmt"
"log/slog"
"golang.org/x/sys/windows"
"golang.org/x/sys/windows/registry"
)
var tcpipInterfaceKeys = []string{
`SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces`,
`SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces`,
}
// disableDNSRegistration prevents publishing overlay addresses in the host's
// DNS records, where clients outside the overlay cannot reach them.
// Failures are non-fatal so registry access cannot prevent tunnel creation.
func disableDNSRegistration(l *slog.Logger, guid windows.GUID) {
id := guid.String()
for _, base := range tcpipInterfaceKeys {
path := base + `\` + id
if err := setDNSRegistrationValues(path); err != nil {
l.Warn("Failed to disable DNS registration on the tun adapter", "error", err, "key", path)
continue
}
l.Debug("Disabled DNS registration on the tun adapter", "key", path)
}
}
func setDNSRegistrationValues(path string) error {
// The stack may not have created the adapter's interface key yet.
k, _, err := registry.CreateKey(registry.LOCAL_MACHINE, path, registry.SET_VALUE)
if err != nil {
return fmt.Errorf("open HKLM\\%s: %w", path, err)
}
defer k.Close()
// Neither RegistrationEnabled nor DisableDynamicUpdate alone reliably
// prevents registration, so all three settings are required.
values := []struct {
name string
data uint32
}{
{"RegistrationEnabled", 0},
{"DisableDynamicUpdate", 1},
{"MaxNumberOfAddressesToRegister", 0},
}
for _, v := range values {
if err := k.SetDWordValue(v.name, v.data); err != nil {
return fmt.Errorf("set %s: %w", v.name, err)
}
}
return nil
}
+4 -28
View File
@@ -50,13 +50,7 @@ func (r Route) String() string {
func makeRouteTree(l *slog.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) { func makeRouteTree(l *slog.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) {
routeTree := new(bart.Table[routing.Gateways]) routeTree := new(bart.Table[routing.Gateways])
seen := make(map[netip.Prefix]bool, len(routes))
for _, r := range routes { for _, r := range routes {
if seen[r.Cidr] {
l.Warn("route is listed more than once, only the last entry's via is used", "route", r)
}
seen[r.Cidr] = true
if !allowMTU && r.MTU > 0 { if !allowMTU && r.MTU > 0 {
l.Warn("route MTU is not supported on this platform", l.Warn("route MTU is not supported on this platform",
"goos", runtime.GOOS, "goos", runtime.GOOS,
@@ -104,11 +98,7 @@ func parseRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
mtu, ok := rMtu.(int) mtu, ok := rMtu.(int)
if !ok { if !ok {
s, ok := rMtu.(string) mtu, err = strconv.Atoi(rMtu.(string))
if !ok {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, rMtu)
}
mtu, err = strconv.Atoi(s)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, err) return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, err)
} }
@@ -184,11 +174,7 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
if rMtu, ok := m["mtu"]; ok { if rMtu, ok := m["mtu"]; ok {
mtu, ok = rMtu.(int) mtu, ok = rMtu.(int)
if !ok { if !ok {
s, ok := rMtu.(string) mtu, err = strconv.Atoi(rMtu.(string))
if !ok {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, rMtu)
}
mtu, err = strconv.Atoi(s)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err) return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err)
} }
@@ -206,15 +192,10 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
metric, ok := rMetric.(int) metric, ok := rMetric.(int)
if !ok { if !ok {
s, ok := rMetric.(string) _, err = strconv.ParseInt(rMetric.(string), 10, 32)
if !ok {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, rMetric)
}
n, err := strconv.ParseInt(s, 10, 32)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, err) return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, err)
} }
metric = int(n)
} }
if metric < 0 || metric > math.MaxInt32 { if metric < 0 || metric > math.MaxInt32 {
@@ -267,15 +248,10 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
gatewayWeight, ok := rGatewayWeight.(int) gatewayWeight, ok := rGatewayWeight.(int)
if !ok { if !ok {
s, ok := rGatewayWeight.(string) _, err = strconv.ParseInt(rGatewayWeight.(string), 10, 32)
if !ok {
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
}
w, err := strconv.ParseInt(s, 10, 32)
if err != nil { if err != nil {
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1) return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
} }
gatewayWeight = int(w)
} }
if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 { if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 {
-116
View File
@@ -1,17 +1,13 @@
package overlay package overlay
import ( import (
"bytes"
"fmt" "fmt"
"net/netip" "net/netip"
"strings"
"testing" "testing"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -406,115 +402,3 @@ func Test_makeMultipathUnsafeRouteTree(t *testing.T) {
routing.CalculateBucketsForGateways(expectedGateways) routing.CalculateBucketsForGateways(expectedGateways)
assert.ElementsMatch(t, expectedGateways, r) assert.ElementsMatch(t, expectedGateways, r)
} }
func Test_getMTU(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
// unset
assert.Equal(t, DefaultMTU, getMTU(l, c))
// the largest a relayed packet can carry in one underlay buffer
require.Equal(t, udp.MTU, MaxMTU+header.MaxOverhead)
c.Settings["tun"] = map[string]any{"mtu": MaxMTU}
assert.Equal(t, MaxMTU, getMTU(l, c))
// too big is capped
c.Settings["tun"] = map[string]any{"mtu": 9000}
assert.Equal(t, MaxMTU, getMTU(l, c))
}
func Test_getAllRoutesFromConfigCapsMTU(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
c.Settings["tun"] = map[string]any{
"routes": []any{
map[string]any{"mtu": "9000", "route": "10.0.0.0/29"},
map[string]any{"mtu": "1300", "route": "10.0.0.8/29"},
},
"unsafe_routes": []any{
map[string]any{"via": "127.0.0.1", "mtu": "9000", "route": "1.0.0.0/29"},
},
}
changed, routes, err := getAllRoutesFromConfig(l, c, []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")}, true)
require.NoError(t, err)
assert.True(t, changed)
mtus := map[string]int{}
for _, r := range routes {
mtus[r.Cidr.String()] = r.MTU
}
assert.Equal(t, map[string]int{"10.0.0.0/29": MaxMTU, "10.0.0.8/29": 1300, "1.0.0.0/29": MaxMTU}, mtus)
}
// The route tree keeps one entry per CIDR, so a CIDR listed twice only routes by the last one. Say so instead of
// letting the earlier via, or on Windows the earlier metric, quietly lose
func Test_makeRouteTree_WarnsOnDuplicate(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewLoggerWithOutput(buf)
via := func(s string) routing.Gateways {
return routing.Gateways{routing.NewGateway(netip.MustParseAddr(s), 1)}
}
cidr := netip.MustParsePrefix("10.251.1.0/24")
_, err := makeRouteTree(l, []Route{
{Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100},
{Cidr: netip.MustParsePrefix("10.251.2.0/24"), Via: via("10.250.0.2"), Install: true},
}, false)
require.NoError(t, err)
assert.Empty(t, buf.String())
tree, err := makeRouteTree(l, []Route{
{Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100},
{Cidr: cidr, Via: via("10.250.0.3"), Install: true, Metric: 200},
}, false)
require.NoError(t, err)
assert.Equal(t, 1, strings.Count(buf.String(), "route is listed more than once, only the last entry's via is used"))
gw, ok := tree.Lookup(netip.MustParseAddr("10.251.1.1"))
require.True(t, ok)
assert.Equal(t, netip.MustParseAddr("10.250.0.3"), gw[0].Addr(), "the last entry wins")
}
// A quoted metric or weight is a number written as a string, it means that number. Anything else is an error, not a
// panic
func Test_parseUnsafeRoutes_QuotedNumbers(t *testing.T) {
c := config.NewC(test.NewLogger())
n := netip.MustParsePrefix("10.0.0.1/24")
route := func(metric, weight any) {
c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{
"route": "1.0.0.0/8",
"metric": metric,
"via": []any{map[string]any{"gateway": "10.0.0.2", "weight": weight}},
}}}
}
route("100", "5")
routes, err := parseUnsafeRoutes(c, []netip.Prefix{n})
require.NoError(t, err)
require.Len(t, routes, 1)
assert.Equal(t, 100, routes[0].Metric)
assert.Equal(t, "{addr: 10.0.0.2, weight: 5}", routes[0].Via[0].String())
route(1.5, 1)
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.metric in tun.unsafe_routes is not an integer: 1.5")
route(1, true)
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry .weight in tun.unsafe_routes[1].via[1] is not an integer")
}
// An mtu that is neither a number nor a quoted one is an error, not a panic
func Test_parseRoutes_MTUNotANumber(t *testing.T) {
c := config.NewC(test.NewLogger())
n := netip.MustParsePrefix("10.0.0.1/24")
c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"route": "1.0.0.0/8", "mtu": 1500.5}}}
_, err := parseRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.mtu in tun.routes is not an integer: 1500.5")
c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"route": "1.0.0.0/8", "via": "10.0.0.2", "mtu": 1500.5}}}
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is not an integer: 1500.5")
}
+2 -24
View File
@@ -7,27 +7,11 @@ import (
"net/netip" "net/netip"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
) )
const DefaultMTU = 1300 const DefaultMTU = 1300
// MaxMTU is the largest tun.mtu, or route mtu, the underlay can carry.
// Every underlay buffer is udp.MTU bytes, and a relayed packet adds header.MaxOverhead to its plaintext.
const MaxMTU = udp.MTU - header.MaxOverhead
// getMTU reads tun.mtu, capping a value too big for the underlay to carry.
func getMTU(l *slog.Logger, c *config.C) int {
mtu := c.GetInt("tun.mtu", DefaultMTU)
if mtu > MaxMTU {
l.Warn("tun.mtu is too big for the underlay to carry, capping it", "mtu", mtu, "maxMTU", MaxMTU)
return MaxMTU
}
return mtu
}
type NameError struct { type NameError struct {
Name string Name string
Underlying error Underlying error
@@ -43,7 +27,7 @@ type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix,
func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) { func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
switch { switch {
case c.GetBool("tun.disabled", false): case c.GetBool("tun.disabled", false):
tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 5000), c.GetBool("stats.message_metrics", false), l) tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
return tun, nil return tun, nil
default: default:
@@ -57,7 +41,7 @@ func NewFdDeviceFromConfig(fd *int) DeviceFactory {
} }
} }
func getAllRoutesFromConfig(l *slog.Logger, c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) { func getAllRoutesFromConfig(c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) {
if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") { if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") {
return false, nil, nil return false, nil, nil
} }
@@ -73,12 +57,6 @@ func getAllRoutesFromConfig(l *slog.Logger, c *config.C, vpnNetworks []netip.Pre
} }
routes = append(routes, unsafeRoutes...) routes = append(routes, unsafeRoutes...)
for i, r := range routes {
if r.MTU > MaxMTU {
l.Warn("route mtu is too big for the underlay to carry, capping it", "route", r.Cidr, "mtu", r.MTU, "maxMTU", MaxMTU)
routes[i].MTU = MaxMTU
}
}
return true, routes, nil return true, routes, nil
} }
+1 -1
View File
@@ -68,7 +68,7 @@ func (t *tun) Activate() error {
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
+3 -3
View File
@@ -126,7 +126,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
f: os.NewFile(uintptr(fd), ""), f: os.NewFile(uintptr(fd), ""),
Device: name, Device: name,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c), DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
l: l, l: l,
} }
@@ -167,7 +167,7 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
f: file, f: file,
Device: utunNameFromFd(deviceFd), Device: utunNameFromFd(deviceFd),
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c), DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
hostOwned: true, hostOwned: true,
l: l, l: l,
} }
@@ -350,7 +350,7 @@ func (t *tun) activate6(network netip.Prefix) error {
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
+2 -2
View File
@@ -370,7 +370,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
t := &tun{ t := &tun{
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
l: l, l: l,
fd: fd, fd: fd,
shutdownR: shutdownR, shutdownR: shutdownR,
@@ -510,7 +510,7 @@ func (t *tun) setMTU() error {
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
+1 -1
View File
@@ -68,7 +68,7 @@ func (t *tun) Activate() error {
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
+4 -13
View File
@@ -221,7 +221,7 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
vnetHdr: vnetHdr, vnetHdr: vnetHdr,
offloadFlags: offloadFlags, offloadFlags: offloadFlags,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
TXQueueLen: c.GetInt("tun.tx_queue", 5000), TXQueueLen: c.GetInt("tun.tx_queue", 500),
useSystemRoutes: c.GetBool("tun.use_system_route_table", false), useSystemRoutes: c.GetBool("tun.use_system_route_table", false),
useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0), useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
routesFromSystem: map[netip.Prefix]routing.Gateways{}, routesFromSystem: map[netip.Prefix]routing.Gateways{},
@@ -244,7 +244,7 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool) error {
routeChange, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -253,15 +253,6 @@ func (t *tun) reload(c *config.C, initial bool) error {
return nil return nil
} }
// Only the mtu changed. The routes are the same, parse them again so the ones without their own mtu pick up the
// new default. Without this an mtu change would remove them all
if !routeChange {
_, routes, err = getAllRoutesFromConfig(t.l, c, t.vpnNetworks, true)
if err != nil {
return err
}
}
routeTree, err := makeRouteTree(t.l, routes, true) routeTree, err := makeRouteTree(t.l, routes, true)
if err != nil { if err != nil {
return err return err
@@ -276,14 +267,14 @@ func (t *tun) reload(c *config.C, initial bool) error {
oldDefaultMTU := t.DefaultMTU oldDefaultMTU := t.DefaultMTU
oldMaxMTU := t.MaxMTU oldMaxMTU := t.MaxMTU
newDefaultMTU := getMTU(t.l, c) newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU)
newMaxMTU := newDefaultMTU newMaxMTU := newDefaultMTU
for i, r := range routes { for i, r := range routes {
if r.MTU == 0 { if r.MTU == 0 {
routes[i].MTU = newDefaultMTU routes[i].MTU = newDefaultMTU
} }
if r.MTU > newMaxMTU { if r.MTU > t.MaxMTU {
newMaxMTU = r.MTU newMaxMTU = r.MTU
} }
} }
+2 -2
View File
@@ -99,7 +99,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
fd: fd, fd: fd,
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
l: l, l: l,
} }
@@ -340,7 +340,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
+2 -2
View File
@@ -90,7 +90,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
fd: fd, fd: fd,
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
l: l, l: l,
} }
@@ -319,7 +319,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
} }
func (t *tun) reload(c *config.C, initial bool) error { func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
-56
View File
@@ -1,56 +0,0 @@
//go:build !e2e_testing
package overlay
import (
"net/netip"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Only tun.mtu changes. The system calls fail without a real device, what matters is the routes nebula keeps
func TestReloadMTUOnlyKeepsRoutes(t *testing.T) {
routes := " unsafe_routes:\n - route: 192.168.100.0/24\n via: 10.1.0.1\n"
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n mtu: 1300\n"+routes))
tn := &tun{
l: test.NewLogger(),
vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")},
ioctlFd: ^uintptr(0),
routesFromSystem: map[netip.Prefix]routing.Gateways{},
}
require.NoError(t, tn.reload(c, true))
require.NoError(t, c.ReloadConfigString("tun:\n mtu: 1400\n"+routes))
_ = tn.reload(c, false)
got := *tn.Routes.Load()
require.Len(t, got, 1)
assert.Equal(t, netip.MustParsePrefix("192.168.100.0/24"), got[0].Cidr)
assert.Equal(t, 1400, got[0].MTU)
assert.Equal(t, 1400, tn.DefaultMTU)
}
// The device mtu has to fit the largest route mtu, wherever it sits in the list
func TestReloadMaxMTUIsTheLargestRoute(t *testing.T) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString("tun:\n mtu: 1300\n unsafe_routes:\n"+
" - route: 192.168.100.0/24\n via: 10.1.0.1\n mtu: 1500\n"+
" - route: 192.168.101.0/24\n via: 10.1.0.1\n mtu: 1400\n"))
tn := &tun{
l: test.NewLogger(),
vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")},
ioctlFd: ^uintptr(0),
routesFromSystem: map[netip.Prefix]routing.Gateways{},
}
require.NoError(t, tn.reload(c, true))
assert.Equal(t, 1500, tn.MaxMTU)
assert.Equal(t, 1300, tn.DefaultMTU)
}
+1 -1
View File
@@ -31,7 +31,7 @@ type TestTun struct {
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) {
_, routes, err := getAllRoutesFromConfig(l, c, vpnNetworks, true) _, routes, err := getAllRoutesFromConfig(c, vpnNetworks, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
+5 -79
View File
@@ -4,7 +4,6 @@ package overlay
import ( import (
"crypto" "crypto"
"errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net/netip" "net/netip"
@@ -76,7 +75,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
t := &winTun{ t := &winTun{
Device: deviceName, Device: deviceName,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
MTU: getMTU(l, c), MTU: c.GetInt("tun.mtu", DefaultMTU),
guid: *guid, guid: *guid,
networkCategory: cat, networkCategory: cat,
setCategory: setCat, setCategory: setCat,
@@ -105,9 +104,6 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
} }
t.tun = tunDevice.(*wintun.NativeTun) t.tun = tunDevice.(*wintun.NativeTun)
// Disable registration before Activate assigns addresses and triggers it.
disableDNSRegistration(l, *guid)
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C) {
err := t.reload(c, false) err := t.reload(c, false)
if err != nil { if err != nil {
@@ -119,7 +115,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
} }
func (t *winTun) reload(c *config.C, initial bool) error { func (t *winTun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
if err != nil { if err != nil {
return err return err
} }
@@ -139,7 +135,7 @@ func (t *winTun) reload(c *config.C, initial bool) error {
if !initial { if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next // Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes)) err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
if err != nil { if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
} }
@@ -188,27 +184,17 @@ func (t *winTun) addRoutes(logErrors bool) error {
foundDefault4 := false foundDefault4 := false
carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() }) carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() })
// A CIDR listed twice routes by its last entry, install only that one or each reload would flip between them. The for _, r := range routes {
// same rule as systemRoutes
last := make(map[netip.Prefix]int, len(routes))
for i, r := range routes {
last[r.Cidr] = i
}
for i, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via // We don't allow route MTUs so only install routes with a via
continue continue
} }
if last[r.Cidr] != i {
continue
}
// A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter. // A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter.
carriesV6 = carriesV6 || r.Cidr.Addr().Is6() carriesV6 = carriesV6 || r.Cidr.Addr().Is6()
// Add our unsafe route as an on-link route to the nebula tun device. // Add our unsafe route as an on-link route to the nebula tun device.
result, err := addRoute(luid, r) err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric))
if err != nil { if err != nil {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
@@ -218,12 +204,7 @@ func (t *winTun) addRoutes(logErrors bool) error {
return retErr return retErr
} }
} else { } else {
switch result {
case routeAdded:
t.l.Info("Added route", "route", r) t.l.Info("Added route", "route", r)
case routeMetricUpdated:
t.l.Info("Updated route metric", "route", r)
}
} }
if !foundDefault4 { if !foundDefault4 {
@@ -335,61 +316,6 @@ func (t *winTun) Close() error {
return t.tun.Close() return t.tun.Close()
} }
// systemRoutes maps each CIDR to the route that puts it in the system table. A CIDR's last entry decides, including
// one that says not to install it
func systemRoutes(routes []Route) map[netip.Prefix]Route {
out := make(map[netip.Prefix]Route, len(routes))
for _, r := range routes {
if len(r.Via) > 0 && r.Install {
out[r.Cidr] = r
} else {
delete(out, r.Cidr)
}
}
return out
}
// findRemovedSystemRoutes is the routes to delete from the system table. Windows deletes by destination and next hop,
// so only a CIDR nothing installs anymore goes, a changed metric is set on the route in place by addRoute
func findRemovedSystemRoutes(newRoutes, oldRoutes []Route) []Route {
keep := systemRoutes(newRoutes)
var removed []Route
for _, r := range systemRoutes(oldRoutes) {
if _, ok := keep[r.Cidr]; !ok {
removed = append(removed, r)
}
}
return removed
}
type routeResult int
const (
routeAdded routeResult = iota
routeMetricUpdated
routeAlreadyInstalled
)
// addRoute installs r on the interface. A reload re-adds every route to catch any the system dropped, and Windows
// refuses one it has with the same destination and next hop whatever its metric. One that is there with another
// metric, ours from before a metric change or someone else's, is brought to ours in place
func addRoute(luid winipcfg.LUID, r Route) (routeResult, error) {
nextHop := unspecifiedNextHop(r.Cidr)
err := luid.AddRoute(r.Cidr, nextHop, uint32(r.Metric))
if !errors.Is(err, windows.ERROR_OBJECT_ALREADY_EXISTS) {
return routeAdded, err
}
row, err := luid.Route(r.Cidr, nextHop)
if err != nil {
return routeAlreadyInstalled, err
}
if row.Metric != uint32(r.Metric) {
row.Metric = uint32(r.Metric)
return routeMetricUpdated, row.Set()
}
return routeAlreadyInstalled, nil
}
func unspecifiedNextHop(p netip.Prefix) netip.Addr { func unspecifiedNextHop(p netip.Prefix) netip.Addr {
if p.Addr().Is4() { if p.Addr().Is4() {
return netip.IPv4Unspecified() return netip.IPv4Unspecified()
-168
View File
@@ -1,168 +0,0 @@
//go:build windows && !e2e_testing
package overlay
import (
"bytes"
"fmt"
"net/netip"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/windows"
"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
)
type routeTest struct {
c *config.C
tun *winTun
luid winipcfg.LUID
log *bytes.Buffer
}
// newRouteTest makes a wintun device of its own, before installing tun.unsafe_routes from cfg
func newRouteTest(t *testing.T, cfg string) *routeTest {
t.Helper()
if !windows.GetCurrentProcessToken().IsElevated() {
t.Skip("creating a wintun device needs an elevated process")
}
if err := checkWinTunExists(); err != nil {
t.Skipf("wintun.dll is not where nebula looks for it: %v", err)
}
rt := &routeTest{log: &bytes.Buffer{}}
l := test.NewLoggerWithOutput(rt.log)
rt.c = config.NewC(l)
require.NoError(t, rt.c.LoadString(cfg))
var err error
rt.tun, err = newTun(rt.c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = rt.tun.Close() })
rt.luid = winipcfg.LUID(rt.tun.tun.LUID())
return rt
}
func (rt *routeTest) metric(t *testing.T, cidr string) uint32 {
t.Helper()
p := netip.MustParsePrefix(cidr)
row, err := rt.luid.Route(p, unspecifiedNextHop(p))
require.NoError(t, err)
return row.Metric
}
func unsafeRoutes(routes ...string) string {
var b strings.Builder
b.WriteString("tun:\n dev: nebroutetest\n unsafe_routes:\n")
// cidr[@metric][!], the ! for install: false
for _, r := range routes {
r, noInstall := strings.CutSuffix(r, "!")
cidr, metric, _ := strings.Cut(r, "@")
fmt.Fprintf(&b, " - route: %s\n via: 10.250.0.2\n", cidr)
if metric != "" {
fmt.Fprintf(&b, " metric: %s\n", metric)
}
if noInstall {
b.WriteString(" install: false\n")
}
}
return b.String()
}
// A reload that adds a route re-adds the ones already installed, Windows refusing those as already there isn't a failure
func TestWinTun_ReloadKeepsInstalledRoutes(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a))
require.NoError(t, rt.tun.Activate())
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a, b)))
assert.NotContains(t, rt.log.String(), "Failed to add route")
rt.metric(t, a)
rt.metric(t, b)
})
}
// Windows calls a route the same by destination and next hop, so a changed metric is set on the installed route in
// place rather than deleted and added, which would leave a moment with no route
func TestWinTun_ReloadMetricChangeApplies(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
require.NoError(t, rt.tun.Activate())
require.Equal(t, uint32(100), rt.metric(t, a))
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200")))
assert.NotContains(t, rt.log.String(), "Failed to add route")
assert.NotContains(t, rt.log.String(), "Removed route")
assert.Equal(t, uint32(200), rt.metric(t, a))
})
}
// A route someone else put on the device with another metric is brought to ours, at startup too
func TestWinTun_ForeignRouteGetsOurMetric(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
p := netip.MustParsePrefix(a)
require.NoError(t, rt.luid.AddRoute(p, unspecifiedNextHop(p), 50))
require.NoError(t, rt.tun.Activate())
assert.Equal(t, uint32(100), rt.metric(t, a))
assert.Contains(t, rt.log.String(), "level=INFO msg=\"Updated route metric\"")
})
}
// A CIDR listed twice is installed once, by its last entry, and reloads leave it there instead of flipping between the
// two metrics
func TestWinTun_DuplicateCidrInstallsTheLastEntry(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200"))
require.NoError(t, rt.tun.Activate())
assert.Equal(t, uint32(200), rt.metric(t, a))
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@100", a+"@200", b)))
assert.Equal(t, uint32(200), rt.metric(t, a))
assert.NotContains(t, rt.log.String(), "Updated route metric")
assert.NotContains(t, rt.log.String(), "Failed to add route")
})
}
// Dropping the duplicate that wasn't installed leaves the installed route alone. Windows deletes by destination and
// next hop, deleting the dropped one would take the live route with it
func TestWinTun_DroppingAnUninstalledDuplicateKeepsTheRoute(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200"))
require.NoError(t, rt.tun.Activate())
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200")))
assert.NotContains(t, rt.log.String(), "Removed route")
assert.Equal(t, uint32(200), rt.metric(t, a))
})
}
// A reload ends where a fresh start with the same config would, a CIDR whose last entry says not to install it is
// removed
func TestWinTun_ReloadMatchesAFreshStart(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
cidr := netip.MustParsePrefix(a)
routes := unsafeRoutes(a+"@100", a+"@100!")
fresh := newRouteTest(t, routes)
require.NoError(t, fresh.tun.Activate())
_, err := fresh.luid.Route(cidr, unspecifiedNextHop(cidr))
require.Error(t, err, "a fresh start installed it")
require.NoError(t, fresh.tun.Close())
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
require.NoError(t, rt.tun.Activate())
require.NoError(t, rt.c.ReloadConfigString(routes))
_, err = rt.luid.Route(cidr, unspecifiedNextHop(cidr))
assert.Error(t, err, "the reload left it installed")
})
}
// forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4
func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) {
t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") })
t.Run("ipv6", func(t *testing.T) { f(t, "fd00:251:1::/64", "fd00:251:2::/64") })
}
-91
View File
@@ -1,91 +0,0 @@
package ratelimit
import (
"fmt"
"math"
"sync/atomic"
"time"
)
// Limiter is a token bucket: `burst` calls are allowed at once, then one more per `every`.
// It counts the calls it denies and hands that count to the next allowed call.
//
// The budget belongs to the Limiter instance.
// Every Allow that returns true spends from it, whatever the caller then does.
//
// Time follows Go's monotonic clock.
//
// Create one Limiter for each thing you want to limit,
// sharing one between unrelated events lets the noisy one starve the quiet one.
//
// Check Allow before doing the work being limited, so a denied call stays cheap:
//
// if denied, ok := probeLimiter.Allow(); ok {
// sendProbe()
// probesDenied.Inc(int64(denied))
// }
//
// For log lines use logging.RateLimited, which attaches the denied count to the logger as "suppressed".
type Limiter struct {
every int64
slack int64
tat atomic.Int64
denied atomic.Uint64
}
// maxSpan bounds `burst` * `every` to half of int64, leaving the other half for process uptime so the bucket
// math can't overflow.
const maxSpan = math.MaxInt64 / 2
// New returns a Limiter allowing `burst` calls at once and one more per `every`.
// A `burst` under 1 is 1, and an `every` of 0 or less never limits.
// It returns an error if `burst` * `every` is over about 146 years.
func New(burst int, every time.Duration) (*Limiter, error) {
l := &Limiter{}
if every > 0 {
burst = max(burst, 1)
if int64(burst) > maxSpan/int64(every) {
return nil, fmt.Errorf("ratelimit: burst %d * every %v is too long", burst, every)
}
l.every = int64(every)
l.slack = int64(burst-1) * l.every
}
return l, nil
}
var monoStart = time.Now()
func monoNow() int64 { return int64(time.Since(monoStart)) }
// Allow reports whether this call is allowed.
// If a token is available it spends it and returns the calls denied since the last allowed one, and true.
// Otherwise it returns 0 and false, and counts the denial.
// A nil Limiter allows everything.
func (l *Limiter) Allow() (uint64, bool) {
if l == nil {
return 0, true
}
return l.allow(monoNow())
}
// allow is Allow at a given monotonic time.
func (l *Limiter) allow(now int64) (uint64, bool) {
// Concurrent Allow calls read the clock before racing for the bucket, so one can see a time another already spent
// past and be denied. An unlimited Limiter must never deny, so it skips the bucket.
if l.every == 0 {
return 0, true
}
for {
tat := l.tat.Load()
next := max(tat, now)
if next-now > l.slack {
l.denied.Add(1)
return 0, false
}
if l.tat.CompareAndSwap(tat, next+l.every) {
return l.denied.Swap(0), true
}
}
}
-264
View File
@@ -1,264 +0,0 @@
package ratelimit
import (
"math"
"sync"
"sync/atomic"
"testing"
"time"
)
func mustLimiter(t testing.TB, burst int, every time.Duration) *Limiter {
t.Helper()
l, err := New(burst, every)
if err != nil {
t.Fatal(err)
}
return l
}
func TestLimiterBurstAndRefill(t *testing.T) {
l, now := mustLimiter(t, 3, time.Second), int64(0)
for i := 0; i < 3; i++ {
if n, ok := l.allow(now); !ok || n != 0 {
t.Fatalf("burst line %d: ok=%v denied=%d", i, ok, n)
}
}
for i := 0; i < 5; i++ {
if _, ok := l.allow(now); ok {
t.Fatalf("line %d past the burst logged", i)
}
}
now += int64(time.Second)
if n, ok := l.allow(now); !ok || n != 5 {
t.Fatalf("after one refill: ok=%v denied=%d, want true 5", ok, n)
}
if _, ok := l.allow(now); ok {
t.Fatal("one refill earned more than one line")
}
// A long quiet spell refills to burst, not past it, and the count reported above does not carry over.
now += int64(time.Hour)
for i, want := range []uint64{1, 0, 0} {
if n, ok := l.allow(now); !ok || n != want {
t.Fatalf("refilled line %d: ok=%v denied=%d, want true %d", i, ok, n, want)
}
}
if _, ok := l.allow(now); ok {
t.Fatal("bucket refilled past burst")
}
}
func TestLimiterBounds(t *testing.T) {
for _, every := range []time.Duration{0, -time.Second} {
l := mustLimiter(t, 1, every)
for i := 0; i < 100; i++ {
if _, ok := l.allow(0); !ok {
t.Fatalf("every=%v limited line %d", every, i)
}
}
}
l := mustLimiter(t, 0, time.Second)
if _, ok := l.allow(0); !ok {
t.Fatal("burst 0 should allow one line")
}
if _, ok := l.allow(0); ok {
t.Fatal("burst 0 should allow only one line")
}
}
func TestLimiterOverflow(t *testing.T) {
for _, c := range []struct {
burst int
every time.Duration
}{
{math.MaxInt, time.Second},
{1, math.MaxInt64},
{2, maxSpan/2 + 1},
} {
if _, err := New(c.burst, c.every); err == nil {
t.Fatalf("burst %d every %d should not fit", c.burst, c.every)
}
}
l := mustLimiter(t, 2, maxSpan/2)
for i := 0; i < 2; i++ {
if _, ok := l.allow(0); !ok {
t.Fatalf("the largest span should allow its burst, denied call %d", i)
}
}
if _, ok := l.allow(0); ok {
t.Fatal("the largest span allowed past its burst")
}
}
// Concurrent callers read the clock at slightly different times, which must not limit a Limiter that never limits.
func TestLimiterUnlimitedConcurrent(t *testing.T) {
l := mustLimiter(t, 1, 0)
var denied atomic.Int64
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 10000; j++ {
if _, ok := l.Allow(); !ok {
denied.Add(1)
}
}
}()
}
wg.Wait()
if got := denied.Load(); got != 0 {
t.Fatalf("denied %d lines", got)
}
}
func TestLimiterNil(t *testing.T) {
var l *Limiter
for i := 0; i < 10; i++ {
if _, ok := l.Allow(); !ok {
t.Fatal("nil limiter limited")
}
}
}
func TestLimiterConcurrent(t *testing.T) {
l := mustLimiter(t, 10, time.Hour)
var allowed atomic.Int64
var reported atomic.Uint64
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 1000; j++ {
if n, ok := l.allow(0); ok {
allowed.Add(1)
reported.Add(n)
}
}
}()
}
wg.Wait()
if got := allowed.Load(); got != 10 {
t.Fatalf("allowed %d lines, want the burst of 10", got)
}
// An allowed call that lands after some denials reports them, so the rest are still pending.
if got := reported.Load() + l.denied.Load(); got != 16*1000-10 {
t.Fatalf("denied %d, want %d", got, 16*1000-10)
}
}
func BenchmarkAllowAllowed(b *testing.B) {
l := mustLimiter(b, 1, time.Nanosecond)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
l.tat.Store(0)
l.Allow()
}
}
func BenchmarkAllowDenied(b *testing.B) {
l := mustLimiter(b, 1, time.Hour)
l.Allow()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
l.Allow()
}
}
func BenchmarkAllowDeniedParallel(b *testing.B) {
l := mustLimiter(b, 1, time.Hour)
l.Allow()
b.ReportAllocs()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
l.Allow()
}
})
}
// FuzzLimiter checks the bucket against its exact definition over arbitrary call timings.
// A call is denied if and only if some allowed call a would leave too many calls for the time since a:
// (allowed calls since a, plus this one) * every > burst * every + (now - a).
// Every denied call must also be reported exactly once, by a later allowed call or still pending.
func FuzzLimiter(f *testing.F) {
f.Add(uint8(3), uint16(1000), []byte{0, 0, 0, 0, 16, 0, 0, 200, 1, 1, 1, 15, 17, 0})
f.Add(uint8(1), uint16(1), []byte{0, 16, 15, 16, 0, 255, 0, 0})
f.Fuzz(func(t *testing.T, burst uint8, everyUs uint16, steps []byte) {
if len(steps) > 512 {
steps = steps[:512]
}
b := int64(burst%16) + 1
every := int64(everyUs%1000+1) * int64(time.Microsecond)
l := mustLimiter(t, int(b), time.Duration(every))
var now int64
var allowedAt []int64
var denied, reported uint64
for _, s := range steps {
// Each step advances by a sixteenth of every, so timings land on and around the refill.
now += int64(s) * every / 16
over := false
for i, a := range allowedAt {
if int64(len(allowedAt)-i+1)*every > b*every+now-a {
over = true
break
}
}
n, ok := l.allow(now)
if ok == over {
t.Fatalf("at %d with %d allowed before: ok=%v, the definition says %v", now, len(allowedAt), ok, !over)
}
if !ok {
denied++
continue
}
reported += n
allowedAt = append(allowedAt, now)
}
if got := reported + l.denied.Load(); got != denied {
t.Fatalf("reported %d denied calls, made %d", got, denied)
}
})
}
// Concurrent callers on the real clock must account for every denied call exactly once.
func TestLimiterDeniedConservedConcurrent(t *testing.T) {
l := mustLimiter(t, 4, time.Microsecond)
var denied, reported atomic.Uint64
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 20000; j++ {
if n, ok := l.Allow(); ok {
reported.Add(n)
} else {
denied.Add(1)
}
}
}()
}
wg.Wait()
if denied.Load() == 0 {
t.Fatal("nothing was denied, the test proves nothing")
}
if got := reported.Load() + l.denied.Load(); got != denied.Load() {
t.Fatalf("reported %d denied calls, made %d", got, denied.Load())
}
}
+2 -6
View File
@@ -684,14 +684,10 @@ func sshCreateTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) e
} }
} }
// Set the remote from the callback, which StartHandshake runs under the handshake manager lock hostInfo = ifce.handshakeManager.StartHandshake(vpnAddr, nil)
// before the hostinfo reaches the handshake loop or the static-host trigger, so the address is
// in the RemoteList before the first handshake is sent.
ifce.handshakeManager.StartHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
if addr.IsValid() { if addr.IsValid() {
hh.hostinfo.SetRemote(addr) hostInfo.SetRemote(addr)
} }
})
return w.WriteLine("Created") return w.WriteLine("Created")
} }
+14 -38
View File
@@ -38,11 +38,6 @@ type statsServer struct {
runMu sync.Mutex runMu sync.Mutex
runCfg *statsConfig runCfg *statsConfig
run *statsRuntime // non-nil while a runtime is live run *statsRuntime // non-nil while a runtime is live
// Closed when the last runtime's capture loop has returned. Stop waits on it so nothing captures once it returns,
// and a reload's new runtime never overlaps the old one, go-metrics keeps its GC capture state in unguarded globals
loopDone chan struct{}
// See newStatsServerFromConfig, never changes after it
emitStats func()
} }
// statsRuntime is the live state owned by a single Start invocation. Start // statsRuntime is the live state owned by a single Start invocation. Start
@@ -87,16 +82,12 @@ type promConfig struct {
// //
// Start is safe to call unconditionally: it no-ops when stats are disabled. // Start is safe to call unconditionally: it no-ops when stats are disabled.
// The returned pointer is always non-nil, even on error. // The returned pointer is always non-nil, even on error.
// func newStatsServerFromConfig(ctx context.Context, l *slog.Logger, c *config.C, buildVersion string, configTest bool) (*statsServer, error) {
// emitStats, if set, updates metrics on every capture, ahead of the exporters so each export sees what it just set. It
// is fixed here, before the reload callback exists, so every runtime has it.
func newStatsServerFromConfig(ctx context.Context, l *slog.Logger, c *config.C, buildVersion string, configTest bool, emitStats func()) (*statsServer, error) {
s := &statsServer{ s := &statsServer{
l: l, l: l,
ctx: ctx, ctx: ctx,
buildVersion: buildVersion, buildVersion: buildVersion,
configTest: configTest, configTest: configTest,
emitStats: emitStats,
} }
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C) {
@@ -129,6 +120,7 @@ func (s *statsServer) reload(c *config.C, initial bool) error {
s.runMu.Lock() s.runMu.Lock()
sameCfg := s.runCfg != nil && *s.runCfg == newCfg sameCfg := s.runCfg != nil && *s.runCfg == newCfg
s.runCfg = &newCfg s.runCfg = &newCfg
running := s.run != nil
s.runMu.Unlock() s.runMu.Unlock()
s.enabled.Store(enabled) s.enabled.Store(enabled)
@@ -137,8 +129,9 @@ func (s *statsServer) reload(c *config.C, initial bool) error {
return nil return nil
} }
// Even with nothing running, a runtime whose listener failed can still be finishing its last pass if running {
s.Stop() s.Stop()
}
if enabled && !s.configTest { if enabled && !s.configTest {
go s.Start() go s.Start()
} }
@@ -160,19 +153,13 @@ func (s *statsServer) Start() {
return return
} }
cfg := *s.runCfg cfg := *s.runCfg
captureFns, listener := s.buildRuntime(cfg)
runCtx, cancel := context.WithCancel(s.ctx) runCtx, cancel := context.WithCancel(s.ctx)
captureFns, listener := s.buildRuntime(runCtx, cfg)
rt := &statsRuntime{cancel: cancel, listener: listener} rt := &statsRuntime{cancel: cancel, listener: listener}
s.run = rt s.run = rt
done := make(chan struct{})
s.loopDone = done
s.runMu.Unlock() s.runMu.Unlock()
// Prime before serving, so a scrape that lands before the first tick sees real values instead of zeros (issue #907) go captureStatsLoop(runCtx, cfg.interval, captureFns)
for _, fn := range captureFns {
fn()
}
go captureStatsLoop(runCtx, cfg.interval, captureFns, done)
cleanExit := true cleanExit := true
if listener == nil { if listener == nil {
@@ -231,12 +218,13 @@ func (s *statsServer) serveListener(listener *http.Server) bool {
// Stop tears down the active runtime, if any. Idempotent. // Stop tears down the active runtime, if any. Idempotent.
func (s *statsServer) Stop() { func (s *statsServer) Stop() {
// Held until the last pass is done, a Start that slipped in after s.run cleared would capture alongside it
s.runMu.Lock() s.runMu.Lock()
defer s.runMu.Unlock()
rt := s.run rt := s.run
s.run = nil s.run = nil
if rt != nil { s.runMu.Unlock()
if rt == nil {
return
}
rt.cancel() rt.cancel()
if rt.listener != nil { if rt.listener != nil {
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -245,18 +233,11 @@ func (s *statsServer) Stop() {
} }
cancel() cancel()
} }
}
// Nothing captures once Stop returns. The last runtime is cancelled by now, whoever cleared it, so this is at most
// the pass it was in. Nothing in a pass takes runMu
if s.loopDone != nil {
<-s.loopDone
}
} }
// buildRuntime produces the capture functions and, for prometheus, an un-served // buildRuntime produces the capture functions and, for prometheus, an un-served
// http.Server from cfg. cfg has already been validated by loadStatsConfig. // http.Server from cfg. cfg has already been validated by loadStatsConfig.
// A graphite send in progress is abandoned when ctx is done. func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) {
func (s *statsServer) buildRuntime(ctx context.Context, cfg statsConfig) ([]func(), *http.Server) {
// rcrowley/go-metrics guards these registrations with a private sync.Once, // rcrowley/go-metrics guards these registrations with a private sync.Once,
// so subsequent reloads are no-ops. // so subsequent reloads are no-ops.
metrics.RegisterDebugGCStats(metrics.DefaultRegistry) metrics.RegisterDebugGCStats(metrics.DefaultRegistry)
@@ -266,9 +247,6 @@ func (s *statsServer) buildRuntime(ctx context.Context, cfg statsConfig) ([]func
func() { metrics.CaptureDebugGCStatsOnce(metrics.DefaultRegistry) }, func() { metrics.CaptureDebugGCStatsOnce(metrics.DefaultRegistry) },
func() { metrics.CaptureRuntimeMemStatsOnce(metrics.DefaultRegistry) }, func() { metrics.CaptureRuntimeMemStatsOnce(metrics.DefaultRegistry) },
} }
if s.emitStats != nil {
captureFns = append(captureFns, s.emitStats)
}
switch cfg.typ { switch cfg.typ {
case "graphite": case "graphite":
@@ -284,7 +262,7 @@ func (s *statsServer) buildRuntime(ctx context.Context, cfg statsConfig) ([]func
Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999}, Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999},
} }
captureFns = append(captureFns, func() { captureFns = append(captureFns, func() {
if err := graphiteOnce(ctx, gcfg, graphiteTimeout); err != nil && ctx.Err() == nil { if err := graphiteOnce(gcfg); err != nil {
s.l.Error("Graphite export failed", "error", err) s.l.Error("Graphite export failed", "error", err)
} }
}) })
@@ -331,10 +309,8 @@ func (s *statsServer) buildRuntime(ctx context.Context, cfg statsConfig) ([]func
return captureFns, nil return captureFns, nil
} }
// captureStatsLoop runs each fn on every tick of d until ctx is cancelled, then // captureStatsLoop runs each fn on every tick of d until ctx is cancelled.
// closes done. func captureStatsLoop(ctx context.Context, d time.Duration, fns []func()) {
func captureStatsLoop(ctx context.Context, d time.Duration, fns []func(), done chan struct{}) {
defer close(done)
t := time.NewTicker(d) t := time.NewTicker(d)
defer t.Stop() defer t.Stop()
for { for {
+30 -295
View File
@@ -2,18 +2,13 @@ package nebula
import ( import (
"context" "context"
"fmt"
"io" "io"
"log/slog" "log/slog"
"net" "net"
"net/http"
"net/http/httptest"
"strconv" "strconv"
"sync/atomic"
"testing" "testing"
"time" "time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -115,14 +110,15 @@ func TestStatsServer_reload_initial_prometheus(t *testing.T) {
} }
func TestStatsServer_Start_graphite_blocksUntilStop(t *testing.T) { func TestStatsServer_Start_graphite_blocksUntilStop(t *testing.T) {
h := newGraphiteHost(t, 0) sink := newGraphiteSink(t)
defer sink.Close()
s, c := newTestStatsServer(t) s, c := newTestStatsServer(t)
setStatsConfig(c, map[string]any{ setStatsConfig(c, map[string]any{
"type": "graphite", "type": "graphite",
"interval": "1s", "interval": "1s",
"protocol": "tcp", "protocol": "tcp",
"host": h.ln.Addr().String(), "host": sink.Addr(),
"prefix": "test", "prefix": "test",
}) })
require.NoError(t, s.reload(c, true)) require.NoError(t, s.reload(c, true))
@@ -375,19 +371,35 @@ func waitForListening(t *testing.T, addr string) {
}) })
} }
// stopAtCleanup returns the ctx to build *s with, and stops it when the test ends. Cancelling first turns away a Start // graphiteSink is a minimal TCP accept-and-discard server so graphiteOnce
// a reload spawned that hasn't run yet, then Stop waits out the last pass, so nothing captures into the next test // calls in tests don't spam error logs or wedge on connection refused.
func stopAtCleanup(t *testing.T, s **statsServer) context.Context { type graphiteSink struct {
ctx, cancel := context.WithCancel(context.Background()) ln net.Listener
t.Cleanup(func() {
cancel()
if *s != nil {
(*s).Stop()
}
})
return ctx
} }
func newGraphiteSink(t *testing.T) *graphiteSink {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
g := &graphiteSink{ln: ln}
go func() {
for {
conn, err := ln.Accept()
if err != nil {
return
}
go func(c net.Conn) {
_, _ = io.Copy(io.Discard, c)
_ = c.Close()
}(conn)
}
}()
return g
}
func (g *graphiteSink) Addr() string { return g.ln.Addr().String() }
func (g *graphiteSink) Close() { _ = g.ln.Close() }
func freeTCPPort(t *testing.T) string { func freeTCPPort(t *testing.T) string {
t.Helper() t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0") ln, err := net.Listen("tcp", "127.0.0.1:0")
@@ -396,280 +408,3 @@ func freeTCPPort(t *testing.T) string {
require.NoError(t, ln.Close()) require.NoError(t, ln.Close())
return strconv.Itoa(port) return strconv.Itoa(port)
} }
// Each export has to see what emitStats set in the same pass
func TestStatsServer_emitStatsRunsBeforeExport(t *testing.T) {
defer metrics.DefaultRegistry.Unregister("emitter.order.test")
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{
"type": "prometheus",
"interval": "1s",
"listen": "127.0.0.1:0",
"path": "/metrics",
})
gauge := metrics.GetOrRegisterGauge("emitter.order.test", nil)
s, err := newStatsServerFromConfig(t.Context(), l, c, "", false, func() { gauge.Update(42) })
require.NoError(t, err)
s.runMu.Lock()
fns, srv := s.buildRuntime(t.Context(), *s.runCfg)
s.runMu.Unlock()
require.NotNil(t, srv)
for _, fn := range fns {
fn()
}
rec := httptest.NewRecorder()
srv.Handler.ServeHTTP(rec, httptest.NewRequest("GET", "/metrics", nil))
assert.Contains(t, rec.Body.String(), "emitter_order_test 42")
}
// A scrape that lands the moment the listener is up already sees what emitStats set (issue #907)
func TestStatsServer_Start_primesBeforeServing(t *testing.T) {
defer metrics.DefaultRegistry.Unregister("prime.before.serving")
port := freeTCPPort(t)
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{
"type": "prometheus",
"interval": "1h",
"listen": "127.0.0.1:" + port,
"path": "/metrics",
})
gauge := metrics.GetOrRegisterGauge("prime.before.serving", nil)
// The pause would let a listener that came up before the prime serve the zero value
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() {
time.Sleep(200 * time.Millisecond)
gauge.Update(42)
})
require.NoError(t, err)
go s.Start()
waitForListening(t, "127.0.0.1:"+port)
resp, err := http.Get("http://127.0.0.1:" + port + "/metrics")
require.NoError(t, err)
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
assert.Contains(t, string(body), "prime_before_serving 42")
}
// Start captures once before the first tick, the interval here is an hour
func TestStatsServer_Start_primes(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer ln.Close()
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": ln.Addr().String()})
primed := make(chan struct{}, 1)
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err = newStatsServerFromConfig(ctx, l, c, "", false, func() {
select {
case primed <- struct{}{}:
default:
}
})
require.NoError(t, err)
go s.Start()
select {
case <-primed:
case <-time.After(5 * time.Second):
t.Fatal("nothing was captured before the first tick")
}
}
// Graphite has to ship what emitStats set in the same pass, not the pass before
func TestStatsServer_emitStatsRunsBeforeGraphiteExport(t *testing.T) {
defer metrics.DefaultRegistry.Unregister("emitter.graphite.test")
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer ln.Close()
received := make(chan string, 1)
go func() {
conn, err := ln.Accept()
if err != nil {
return
}
defer conn.Close()
b, _ := io.ReadAll(conn)
received <- string(b)
}()
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": ln.Addr().String()})
gauge := metrics.GetOrRegisterGauge("emitter.graphite.test", nil)
s, err := newStatsServerFromConfig(t.Context(), l, c, "", false, func() { gauge.Update(42) })
require.NoError(t, err)
s.runMu.Lock()
fns, _ := s.buildRuntime(t.Context(), *s.runCfg)
s.runMu.Unlock()
for _, fn := range fns {
fn()
}
select {
case body := <-received:
assert.Contains(t, body, ".emitter.graphite.test.value 42 ")
case <-time.After(5 * time.Second):
t.Fatal("graphite sent nothing")
}
}
// stats.interval reloads, and the runtime a reload starts still has emitStats
func TestStatsServer_reloadMovesTheInterval(t *testing.T) {
var passes atomic.Int64
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
// Nothing listens here, each export fails at once
host := "127.0.0.1:" + freeTCPPort(t)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": host})
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() { passes.Add(1) })
require.NoError(t, err)
go s.Start()
waitFor(t, func() bool { return passes.Load() == 1 })
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "20ms", "host": host})
require.NoError(t, s.reload(c, false))
waitFor(t, func() bool { return passes.Load() >= 6 })
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": host})
require.NoError(t, s.reload(c, false))
// The reload's Stop waited out the 20ms loop. The new runtime's prime may land either side of this read, it is the
// only pass left to come
stopped := passes.Load()
time.Sleep(300 * time.Millisecond)
assert.LessOrEqual(t, passes.Load()-stopped, int64(1), "the 20ms loop kept running after the reload")
}
// A reload's new runtime never captures while the old one is still in a pass, go-metrics keeps its GC capture state
// in unguarded globals
func TestStatsServer_reloadWaitsForTheLastPass(t *testing.T) {
var active, most atomic.Int64
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
host := "127.0.0.1:" + freeTCPPort(t)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "10ms", "host": host})
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() {
n := active.Add(1)
for {
m := most.Load()
if n <= m || most.CompareAndSwap(m, n) {
break
}
}
// Long enough that a reload almost always lands mid pass
time.Sleep(50 * time.Millisecond)
active.Add(-1)
})
require.NoError(t, err)
go s.Start()
for i := range 6 {
setStatsConfig(c, map[string]any{"type": "graphite", "interval": fmt.Sprintf("%dms", 11-i%2), "host": host})
require.NoError(t, s.reload(c, false))
time.Sleep(30 * time.Millisecond)
}
assert.Equal(t, int64(1), most.Load())
}
// Nothing captures once Stop returns, a pass in progress is waited out
func TestStatsServer_Stop_waitsForThePass(t *testing.T) {
var inPass atomic.Bool
entered := make(chan struct{}, 1)
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)})
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() {
inPass.Store(true)
select {
case entered <- struct{}{}:
default:
}
time.Sleep(100 * time.Millisecond)
inPass.Store(false)
})
require.NoError(t, err)
go s.Start()
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("never captured")
}
s.Stop()
assert.False(t, inPass.Load(), "Stop returned with a pass still running")
}
// A Start that arrives while Stop waits out the last pass doesn't capture alongside it
func TestStatsServer_Start_duringStopWaitsForThePass(t *testing.T) {
var active, most atomic.Int64
entered := make(chan struct{}, 1)
l := slog.New(slog.DiscardHandler)
c := config.NewC(l)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)})
var s *statsServer
ctx := stopAtCleanup(t, &s)
s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() {
n := active.Add(1)
for {
m := most.Load()
if n <= m || most.CompareAndSwap(m, n) {
break
}
}
select {
case entered <- struct{}{}:
default:
}
time.Sleep(200 * time.Millisecond)
active.Add(-1)
})
require.NoError(t, err)
go s.Start()
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("never captured")
}
stopped := make(chan struct{})
go func() {
s.Stop()
close(stopped)
}()
// Stop holds runMu for its whole wait, the next Start has to arrive after it took it
waitFor(t, func() bool {
if s.runMu.TryLock() {
s.runMu.Unlock()
return false
}
return true
})
go s.Start()
select {
case <-stopped:
case <-time.After(5 * time.Second):
t.Fatal("Stop did not return")
}
waitFor(t, func() bool { return len(entered) == 1 })
assert.Equal(t, int64(1), most.Load(), "a Start captured alongside the pass Stop was waiting out")
}
+8
View File
@@ -48,6 +48,14 @@ type Conn interface {
Close() error Close() error
} }
// EstablishedPeerConn is a Conn that treats the remotes of established tunnels differently from other
// destinations, as darwin's gives busy ones connected sockets.
type EstablishedPeerConn interface {
// SetEstablishedPeer reports whether dst is the current remote of at least one established tunnel. Calls for
// one dst must not race each other.
SetEstablishedPeer(dst netip.AddrPort, established bool)
}
type NoopConn struct{} type NoopConn struct{}
func (NoopConn) Rebind() error { func (NoopConn) Rebind() error {
+115
View File
@@ -0,0 +1,115 @@
//go:build !e2e_testing
package udp
import (
"log/slog"
"net"
"net/netip"
"testing"
"time"
)
// benchConnSocks opens a wildcard listener with ListenOut running, with the connected socket knobs pinned to
// their defaults and then cfg applied, and a loopback peer that never reads, so its sends cost only the sender.
func benchConnSocks(b *testing.B, s Settings, cfg func(*connSockConfig)) (*StdConn, netip.AddrPort) {
b.Helper()
for k, v := range map[string]string{
"NEBULA_CONNSOCKS": "8",
"NEBULA_CONNSOCK_RUN": "64",
"NEBULA_CONNSOCK_WINDOW": "1s",
"NEBULA_CONNSOCK_IDLE": "30s",
"NEBULA_CONNSOCK_OPEN_EVERY": "1s",
} {
b.Setenv(k, v)
}
c, err := NewListener(slog.New(slog.DiscardHandler), s)
if err != nil {
b.Fatal(err)
}
b.Cleanup(func() { _ = c.Close() })
u := c.(*StdConn)
if cfg != nil {
cfg(&u.socks.cfg)
}
go func() { _ = u.ListenOut(func(netip.AddrPort, []byte) {}, func() {}) }()
for u.reader.Load() == nil {
time.Sleep(time.Millisecond)
}
pc, err := net.ListenPacket("udp4", "127.0.0.1:0")
if err != nil {
b.Fatal(err)
}
b.Cleanup(func() { _ = pc.Close() })
return u, pc.LocalAddr().(*net.UDPAddr).AddrPort()
}
// BenchmarkConnSockWriteTo measures one WriteTo of a full-size datagram down each path: the listener with
// connected sockets off, to a destination that is no tunnel's remote while another is, to an established peer
// that never gets busy, and on a connected socket.
func BenchmarkConnSockWriteTo(b *testing.B) {
pkt := make([]byte, 1400)
cases := []struct {
name string
cfg func(*connSockConfig)
setup func(b *testing.B, u *StdConn, dst netip.AddrPort)
}{
{name: "listener/off", cfg: func(c *connSockConfig) { c.max = 0 }},
{name: "listener/ineligible", setup: func(b *testing.B, u *StdConn, dst netip.AddrPort) {
u.SetEstablishedPeer(netip.AddrPortFrom(dst.Addr(), dst.Port()+1), true)
}},
{name: "listener/eligible-not-busy", cfg: func(c *connSockConfig) { c.run = 1 << 30 },
setup: func(b *testing.B, u *StdConn, dst netip.AddrPort) { u.SetEstablishedPeer(dst, true) }},
{name: "connected", setup: func(b *testing.B, u *StdConn, dst netip.AddrPort) {
u.SetEstablishedPeer(dst, true)
for range u.socks.cfg.run {
_ = u.WriteTo(pkt, dst)
}
if u.connSockFor(dst) == nil {
b.Fatal("no connected socket")
}
}},
}
for _, c := range cases {
b.Run(c.name, func(b *testing.B) {
u, dst := benchConnSocks(b, Settings{Listen: netip.MustParseAddrPort("0.0.0.0:0")}, c.cfg)
if c.setup != nil {
c.setup(b, u, dst)
}
b.ReportAllocs()
b.ResetTimer()
for range b.N {
_ = u.WriteTo(pkt, dst)
}
})
}
}
// BenchmarkConnSockDispatch measures what WriteTo adds before the listener's sendto, without the syscall: a v4
// listener refuses a v6 destination before sending, with connected sockets off or with another peer established.
func BenchmarkConnSockDispatch(b *testing.B) {
pkt := make([]byte, 1400)
dst := netip.MustParseAddrPort("[::1]:4242")
for _, c := range []struct {
name string
cfg func(*connSockConfig)
establish bool
}{
{name: "off", cfg: func(c *connSockConfig) { c.max = 0 }},
{name: "ineligible", establish: true},
} {
b.Run(c.name, func(b *testing.B) {
u, peer := benchConnSocks(b, Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Multi: true}, c.cfg)
if c.establish {
u.SetEstablishedPeer(peer, true)
}
b.ReportAllocs()
b.ResetTimer()
for range b.N {
if u.WriteTo(pkt, dst) == nil {
b.Fatal("sent a v6 datagram on a v4 listener")
}
}
})
}
}
+803
View File
@@ -0,0 +1,803 @@
//go:build !e2e_testing
package udp
import (
"context"
"errors"
"fmt"
"log/slog"
"maps"
"net"
"net/netip"
"os"
"slices"
"strconv"
"sync"
"sync/atomic"
"syscall"
"time"
"golang.org/x/sys/unix"
)
// A connected socket is a UDP socket connected to one busy peer, on the listener's port and bound to the source
// address the listener's route to that peer picks, so the peer sees the same address and port from either. A send
// on a connected socket skips the per-datagram address and MAC policy checks, and the EndpointSecurity send check
// that xprotectd can hold every unconnected sendto on, and it is the only UDP send that reports the interface
// queue's flow control, as ENOBUFS. A connected socket wins xnu's lookup for its peer's datagrams; it never sees
// anyone else's.
//
// A connected socket sets SO_REUSEPORT and binds a specific address beside a wildcard listener, which sets
// SO_REUSEPORT only for listen routines. xnu refuses a wildcard SO_REUSEPORT bind beside a holder without the
// option, and refuses another uid's bind to an address a socket holds specifically, so no other user can join the
// port while connected sockets open and close: uid nobody got EADDRINUSE on every bind against a root listener
// with connected sockets in the 09-25 probes. A connected socket is closed, never disconnected, because xnu widens
// a disconnected socket's local address to the wildcard.
//
// Only the current remote of an established tunnel may get one: Nebula names those with SetEstablishedPeer, so an
// unauthenticated sender can't earn a connected socket by drawing replies such as recv_error or handshake
// responses.
type connSock struct {
dst netip.AddrPort
peer *connSockPeer
conn *net.UDPConn
rc syscall.RawConn
// used is the monotonic time in nanoseconds of the connected socket's last send or receive.
used atomic.Int64
// stallAt is the monotonic time of the first ENOBUFS or EAGAIN since a send last went through, or 0, and
// dropping is set once that stall has dropped a datagram, so each stall logs its drops once.
stallAt atomic.Int64
dropping atomic.Bool
// parkMu serializes the writes that wait for a full send buffer, since the write deadline bounding each wait
// is the descriptor's, not the writer's.
parkMu sync.Mutex
// sys writes one datagram on the descriptor: writeFd, or a stand-in for the interface in tests.
sys func(fd int, b []byte) error
}
func writeFd(fd int, b []byte) error {
_, err := syscall.Write(fd, b)
return err
}
// errStallSpent is a write that found the connected socket's stall budget spent, so its datagram is dropped.
var errStallSpent = errors.New("connected socket stall budget spent")
// connSockWriter carries one write through RawConn.Write. They are pooled with fn bound once, since a closure
// passed through the RawConn interface escapes and would allocate on every send.
type connSockWriter struct {
s *connSock
b []byte
park bool
err error
fn func(fd uintptr) bool
}
var connSockWriters = sync.Pool{New: func() any {
w := &connSockWriter{}
w.fn = w.write
return w
}}
func (w *connSockWriter) write(fd uintptr) bool {
for {
w.err = w.s.sys(int(fd), w.b)
switch {
case w.err == syscall.EINTR:
case w.err == syscall.EAGAIN && w.park:
return false
default:
return true
}
}
}
// try writes b on s once and returns nil, the write's errno, os.ErrDeadlineExceeded, or net.ErrClosed. With park
// set, EAGAIN waits in the netpoller until the socket is writable or the write deadline passes; without it, EAGAIN
// is returned.
func (s *connSock) try(b []byte, park bool) error {
w := connSockWriters.Get().(*connSockWriter)
w.s, w.b, w.park = s, b, park
rerr := s.rc.Write(w.fn)
err := w.err
*w = connSockWriter{fn: w.fn}
connSockWriters.Put(w)
switch {
case rerr == nil:
return err
case errors.Is(rerr, os.ErrDeadlineExceeded):
return os.ErrDeadlineExceeded
default:
return net.ErrClosed
}
}
// connSockConfig holds the knobs. connSockConfigFromEnv reads all but backoff and enobufsMax from the environment,
// so one binary can be tuned without a rebuild.
type connSockConfig struct {
// max is how many connected sockets the listen routines may have open at once; 0 turns connected sockets off.
max int
// run is how many datagrams to one destination within one window earn it a connected socket. darwin's tun
// hands over one packet per read, so a peer's sends arrive one WriteTo or one-datagram batch at a time and
// busyness has to be counted across calls.
run int
window time.Duration
// idle closes a connected socket after it has neither sent nor received for this long.
idle time.Duration
// backoff keeps a destination whose connected socket failed on the listener for this long.
backoff time.Duration
// openEvery is how often the open bucket earns back an open once max are spent, which bounds connected socket
// churn however fast senders earn connected sockets and errors close them.
openEvery time.Duration
// enobufsMax is how long a connected socket's sends may wait out ENOBUFS without one going through before they
// drop instead, so a suspended interface costs one write per datagram rather than stalling the tun reader.
enobufsMax time.Duration
}
func connSockConfigFromEnv() connSockConfig {
c := connSockConfig{
max: 8,
run: 64,
window: time.Second,
idle: 30 * time.Second,
backoff: 30 * time.Second,
openEvery: time.Second,
enobufsMax: 10 * time.Millisecond,
}
if v, err := strconv.Atoi(os.Getenv("NEBULA_CONNSOCKS")); err == nil {
c.max = v
}
if v, err := strconv.Atoi(os.Getenv("NEBULA_CONNSOCK_RUN")); err == nil && v > 0 {
c.run = v
}
if v, err := time.ParseDuration(os.Getenv("NEBULA_CONNSOCK_WINDOW")); err == nil && v > 0 {
c.window = v
}
if v, err := time.ParseDuration(os.Getenv("NEBULA_CONNSOCK_IDLE")); err == nil && v > 0 {
c.idle = v
}
if v, err := time.ParseDuration(os.Getenv("NEBULA_CONNSOCK_OPEN_EVERY")); err == nil && v > 0 {
c.openEvery = v
}
return c
}
// connSockGroups holds the connSocks that the listen routines bound to one address share, so the limits, the
// connected sockets and Rebind cover every routine, and a peer has at most one connected socket.
var connSockGroups = struct {
sync.Mutex
m map[netip.AddrPort]*connSocks
}{m: map[netip.AddrPort]*connSocks{}}
// joinConnSocks returns the connSocks u shares with the other listen routines bound to la, and makes it for the
// first.
func joinConnSocks(u *StdConn, la netip.AddrPort, s Settings) *connSocks {
g := &connSockGroups
g.Lock()
defer g.Unlock()
if cs := g.m[la]; s.Multi && cs != nil {
cs.mu.Lock()
cs.members = append(cs.members, u)
cs.mu.Unlock()
return cs
}
cs := &connSocks{cfg: connSockConfigFromEnv(), key: la, members: []*StdConn{u}, clock: monoNow, sleep: time.Sleep}
if cs.cfg.max > 0 && !u.listenHost.IsUnspecified() && !s.Multi {
// A connected socket on a specific listen.host binds the listener's own address, which xnu allows only when
// both sockets set SO_REUSEPORT, and a listener with it lets another uid bind a dual-stack wildcard on the
// port.
cs.off.Store(true)
u.l.Debug("connected sockets: off for a specific listen.host", "addr", s.Listen)
}
if s.Multi {
g.m[la] = cs
}
return cs
}
type connSocks struct {
cfg connSockConfig
// key is the listeners' bound address, which connSockGroups files this under.
key netip.AddrPort
// off keeps connected sockets from opening, when the listener's address can't be shared with them.
off atomic.Bool
// table maps each destination that is the remote of an established tunnel, and that a connected socket could
// reach, to its peer; it is nil when there are none, so a send with nothing to consider costs one load. Every
// send reads it without a lock, and mu guards copying it. An open that holds mu and still finds its peer here
// can't outlive the tunnel.
table atomic.Pointer[map[netip.AddrPort]*connSockPeer]
// reserved counts the connected sockets that are open, opening or closing, against cfg.max; mu guards changing it.
reserved atomic.Int32
mu sync.Mutex
closed bool
// members are the listen routines' listeners that share these connected sockets.
members []*StdConn
// pending holds each destination whose connected socket is opening or closing, and not on its peer. xnu
// refuses to connect a 4-tuple that is still bound, so a destination in pending can't open another.
pending map[netip.AddrPort]struct{}
// rebinds counts Rebinds, so an open that copied the listener's interface before one is discarded.
rebinds uint64
// closers tracks the goroutines that close detached connected sockets.
closers sync.WaitGroup
// done stops the reaper; it is made when ListenOut starts the connected sockets.
done chan struct{}
// opens bounds connected socket opens to cfg.max at once and one per cfg.openEvery after.
opens gcra
// logs bounds connected socket log lines at info or warn; see connSockLogBurst.
logs gcra
// clock returns monotonic nanoseconds and sleep waits out ENOBUFS: monoNow and time.Sleep, or stand-ins in
// tests.
clock func() int64
sleep func(time.Duration)
// dialHook and closeHook, when set, run just before an open's dials and a close's Close, outside mu; tests
// use them to hold an open or close in flight.
dialHook, closeHook func(dst netip.AddrPort)
}
// connSockPeer is what connSocks keeps about an established tunnel's remote: its connected socket, how busy it is,
// and how long a failed connected socket keeps it on the listener.
type connSockPeer struct {
sock atomic.Pointer[connSock]
// windowMu guards start, the monotonic time the current window began, and n, the datagrams sent in it, so a
// window's reset and its counts can't interleave with another sender's.
windowMu sync.Mutex
start int64
n int
// retryAt is the monotonic time before which a peer whose connected socket failed stays on the listener;
// connSocks.mu guards it.
retryAt int64
}
// busy counts n more datagrams to p and reports whether it has now sent cfg.run within one window. A busy peer's
// count starts over, so a peer that can't get a connected socket retries at most once per cfg.run sends.
func (p *connSockPeer) busy(now int64, n int, cfg *connSockConfig) bool {
p.windowMu.Lock()
defer p.windowMu.Unlock()
if time.Duration(now-p.start) > cfg.window {
p.start, p.n = now, 0
}
p.n += n
if p.n < cfg.run {
return false
}
p.n = 0
return true
}
// connSockLogBurst and connSockLogEvery size the bucket that lets connected socket opens, closes and failures log
// at info or warn; past it they log at debug, so churn can't flood the log.
const (
connSockLogBurst = 32
connSockLogEvery = 10 * time.Second
)
var monoStart = time.Now()
func monoNow() int64 { return int64(time.Since(monoStart)) }
// gcra is a token bucket kept as the monotonic time it is next full (the generic cell rate algorithm), so it needs
// no refill tick.
type gcra struct{ tat int64 }
// take spends a token from a bucket of burst that earns one back per every, and reports whether there was one to
// spend.
func (g *gcra) take(now int64, burst int, every time.Duration) bool {
e := int64(every)
tat := max(g.tat, now)
if tat-now > int64(burst-1)*e {
return false
}
g.tat = tat + e
return true
}
// logLevelLocked returns level while the log bucket has a token, and debug once it is spent.
func (cs *connSocks) logLevelLocked(level slog.Level) slog.Level {
if cs.logs.take(cs.clock(), connSockLogBurst, connSockLogEvery) {
return level
}
return slog.LevelDebug
}
// startConnSocks starts the idle reaper.
func (u *StdConn) startConnSocks() {
cs := u.socks
cs.mu.Lock()
defer cs.mu.Unlock()
if cs.closed || cs.done != nil || cs.cfg.max <= 0 || cs.off.Load() {
return
}
cs.done = make(chan struct{})
go u.reapConnSocks(cs.done)
}
// stopConnSocks closes every connected socket and keeps new ones from opening, once any listen routine's listener
// closes, as they do together at shutdown. It returns once their descriptors are closed.
func (u *StdConn) stopConnSocks() {
cs := u.socks
cs.mu.Lock()
cs.members = slices.DeleteFunc(cs.members, func(m *StdConn) bool { return m == u })
if !cs.closed {
cs.closed = true
if cs.done != nil {
close(cs.done)
}
u.detachAllLocked("close")
}
cs.mu.Unlock()
connSockGroups.Lock()
if connSockGroups.m[cs.key] == cs {
delete(connSockGroups.m, cs.key)
}
connSockGroups.Unlock()
cs.closers.Wait()
}
// siblings returns the other listen routines' listeners.
func (cs *connSocks) siblings(u *StdConn) []*StdConn {
cs.mu.Lock()
defer cs.mu.Unlock()
return slices.DeleteFunc(slices.Clone(cs.members), func(m *StdConn) bool { return m == u })
}
// connSockFor returns dst's connected socket, or nil.
func (u *StdConn) connSockFor(dst netip.AddrPort) *connSock {
if p := u.socks.peer(dst); p != nil {
return p.sock.Load()
}
return nil
}
func (cs *connSocks) peer(dst netip.AddrPort) *connSockPeer {
if t := cs.table.Load(); t != nil {
return (*t)[dst]
}
return nil
}
// SetEstablishedPeer makes dst eligible for a connected socket while it is the remote of an established tunnel,
// and closes the one it has once it no longer is. Nebula calls it for every listen routine under the hostmap's
// write lock, so repeats are no-ops and it makes no syscalls: the descriptor is closed by another goroutine.
func (u *StdConn) SetEstablishedPeer(dst netip.AddrPort, established bool) {
cs := u.socks
if cs.cfg.max <= 0 || cs.off.Load() || !u.connSockable(dst) {
return
}
cs.mu.Lock()
defer cs.mu.Unlock()
t := cs.table.Load()
p := cs.peer(dst)
if cs.closed || (p != nil) == established {
return
}
next := map[netip.AddrPort]*connSockPeer{}
if t != nil {
next = maps.Clone(*t)
}
if established {
p = &connSockPeer{start: cs.clock()}
next[dst] = p
} else {
delete(next, dst)
if s := p.sock.Load(); s != nil {
u.detachLocked(s, "tunnel gone", false)
}
}
cs.storeTableLocked(next)
}
// storeTableLocked publishes t, or nil for an empty one.
func (cs *connSocks) storeTableLocked(t map[netip.AddrPort]*connSockPeer) {
if len(t) == 0 {
cs.table.Store(nil)
return
}
cs.table.Store(&t)
}
// connSockable reports whether a connected socket to dst could send what the listener would. A v4 listener can't
// reach a v6 destination at all, and a connected socket bound to a specific listen.host must share its family.
func (u *StdConn) connSockable(dst netip.AddrPort) bool {
if !dst.IsValid() || dst.Port() == 0 {
return false
}
if u.isV4 {
return dst.Addr().Is4()
}
return u.listenHost.IsUnspecified() || u.listenHost.Unmap().Is4() == dst.Addr().Unmap().Is4()
}
// connSockRun sends bufs, all to p's destination, on p's connected socket, opening one if they make p busy. It
// returns how many went out; when that is short of len(bufs), err is the failure of bufs[sent], or nil to leave
// bufs[sent:] to the listener.
func (u *StdConn) connSockRun(p *connSockPeer, dst netip.AddrPort, bufs [][]byte) (sent int, err error) {
s := p.sock.Load()
if s == nil {
if s = u.maybeOpenConnSock(p, dst, len(bufs)); s == nil {
return 0, nil
}
}
sent, errno, closed := u.connSockWrite(s, bufs)
switch {
case sent == len(bufs):
case connSockDead(errno):
u.closeConnSock(s, errno.Error(), true)
case !closed:
return sent, &net.OpError{Op: "write", Err: errno}
}
return sent, nil
}
// maybeOpenConnSock counts n datagrams to p, and opens a connected socket to it once it is busy and nothing rules
// one out.
func (u *StdConn) maybeOpenConnSock(p *connSockPeer, dst netip.AddrPort, n int) *connSock {
cs := u.socks
if n <= 0 || u.reader.Load() == nil || int(cs.reserved.Load()) >= cs.cfg.max || !p.busy(cs.clock(), n, &cs.cfg) {
return nil
}
cs.mu.Lock()
if cs.closed || cs.peer(dst) != p {
cs.mu.Unlock()
return nil
}
if s := p.sock.Load(); s != nil {
cs.mu.Unlock()
return s
}
now := cs.clock()
if _, ok := cs.pending[dst]; ok || int(cs.reserved.Load()) >= cs.cfg.max || now < p.retryAt ||
!cs.opens.take(now, cs.cfg.max, cs.cfg.openEvery) {
cs.mu.Unlock()
return nil
}
if cs.pending == nil {
cs.pending = map[netip.AddrPort]struct{}{}
}
cs.pending[dst] = struct{}{}
cs.reserved.Add(1)
rebinds := cs.rebinds
cs.mu.Unlock()
if cs.dialHook != nil {
cs.dialHook(dst)
}
s, err := u.dialConnSock(p, dst)
cs.mu.Lock()
if err != nil {
delete(cs.pending, dst)
cs.reserved.Add(-1)
p.retryAt = cs.clock() + int64(cs.cfg.backoff)
level := cs.logLevelLocked(slog.LevelWarn)
cs.mu.Unlock()
u.l.Log(context.Background(), level, "connected socket: open failed, staying on the listener", "udpAddr", dst, "error", err)
return nil
}
if cs.closed || cs.rebinds != rebinds || cs.peer(dst) != p {
// The tunnel went away, or a Rebind or shutdown ran, while the socket was opening; it stays in pending until
// it is closed.
cs.mu.Unlock()
u.finishClose(s)
u.l.Debug("connected socket: discarded, its tunnel or route changed while it opened", "udpAddr", dst)
return nil
}
delete(cs.pending, dst)
p.sock.Store(s)
go u.readConnSock(s)
level := cs.logLevelLocked(slog.LevelInfo)
cs.mu.Unlock()
u.l.Log(context.Background(), level, "connected socket: opened", "udpAddr", dst, "local", s.conn.LocalAddr(), "open", cs.reserved.Load())
return s
}
// dialConnSock opens a socket on the listener's port with SO_REUSEPORT, bound to the configured listen.host or,
// for a wildcard listener, to the source address the route to dst picks, and connected to dst. It carries the
// listener's IP_BOUND_IF, so both route the same way.
func (u *StdConn) dialConnSock(p *connSockPeer, dst netip.AddrPort) (*connSock, error) {
raddr := net.UDPAddrFromAddrPort(netip.AddrPortFrom(dst.Addr().Unmap(), dst.Port()))
network := "udp6"
if raddr.IP.To4() != nil {
network = "udp4"
}
ifindex, err := u.boundIf()
if err != nil {
return nil, err
}
local := &net.UDPAddr{IP: u.listenHost.AsSlice(), Zone: u.listenHost.Zone()}
if u.listenHost.IsUnspecified() {
// A connected socket gets the source address xnu would pick for an unconnected send to dst.
d := net.Dialer{Control: connSockControl(false, ifindex)}
probe, err := d.Dial(network, raddr.String())
if err != nil {
return nil, err
}
local = probe.LocalAddr().(*net.UDPAddr)
_ = probe.Close()
}
local.Port = int(u.port)
d := net.Dialer{LocalAddr: local, Control: connSockControl(true, ifindex)}
c, err := d.Dial(network, raddr.String())
if err != nil {
return nil, err
}
uc := c.(*net.UDPConn)
rc, err := uc.SyscallConn()
if err != nil {
_ = uc.Close()
return nil, err
}
s := &connSock{dst: dst, peer: p, conn: uc, rc: rc, sys: writeFd}
s.used.Store(u.socks.clock())
return s, nil
}
// connSockControl sets SO_REUSEPORT when reuse is set, and IP_BOUND_IF or IPV6_BOUND_IF when ifindex isn't 0,
// before the socket binds.
func connSockControl(reuse bool, ifindex int) func(network, address string, c syscall.RawConn) error {
return func(network, _ string, c syscall.RawConn) error {
var serr error
err := c.Control(func(fd uintptr) {
if reuse {
serr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
}
if serr == nil && ifindex != 0 {
if network == "udp4" {
serr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_BOUND_IF, ifindex)
} else {
serr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_BOUND_IF, ifindex)
}
}
})
if err != nil {
return err
}
return serr
}
}
// boundIf returns the interface index the listener is scoped to, or 0. An error keeps the peer on the listener,
// since a connected socket opened without the listener's scope could route out an interface the listener can't.
func (u *StdConn) boundIf() (int, error) {
level, opt := unix.IPPROTO_IPV6, unix.IPV6_BOUND_IF
if u.isV4 {
level, opt = unix.IPPROTO_IP, unix.IP_BOUND_IF
}
v, err := unix.GetsockoptInt(int(u.sysFd), level, opt)
if err != nil {
return 0, fmt.Errorf("reading the listener's bound interface: %w", err)
}
return v, nil
}
// closeConnSock closes s if it is still open. With backoff set, the peer stays on the listener until the backoff
// runs out.
func (u *StdConn) closeConnSock(s *connSock, why string, backoff bool) {
u.socks.mu.Lock()
u.detachLocked(s, why, backoff)
u.socks.mu.Unlock()
}
// detachLocked takes s off its peer, if it is still there, and closes it on another goroutine, since callers such
// as SetEstablishedPeer hold locks that no socket syscall should run under. s.dst stays in pending until the close
// completes.
func (u *StdConn) detachLocked(s *connSock, why string, backoff bool) {
cs := u.socks
if !s.peer.sock.CompareAndSwap(s, nil) {
return
}
if backoff {
s.peer.retryAt = cs.clock() + int64(cs.cfg.backoff)
}
if cs.pending == nil {
cs.pending = map[netip.AddrPort]struct{}{}
}
cs.pending[s.dst] = struct{}{}
level := cs.logLevelLocked(slog.LevelInfo)
cs.closers.Add(1)
go func() {
defer cs.closers.Done()
open := u.finishClose(s)
u.l.Log(context.Background(), level, "connected socket: closed", "udpAddr", s.dst, "reason", why, "open", open)
}()
}
// finishClose closes s, whose destination is in pending, then lets the destination open again, and returns how
// many connected sockets are left. Closing waits only for a write already in a syscall: a write parked on a full
// buffer is woken, and one waiting out ENOBUFS holds no reference on the descriptor.
func (u *StdConn) finishClose(s *connSock) int32 {
cs := u.socks
if cs.closeHook != nil {
cs.closeHook(s.dst)
}
_ = s.conn.Close()
cs.mu.Lock()
defer cs.mu.Unlock()
delete(cs.pending, s.dst)
return cs.reserved.Add(-1)
}
// detachAllLocked detaches every connected socket, for a network change or shutdown.
func (u *StdConn) detachAllLocked(why string) {
if t := u.socks.table.Load(); t != nil {
for _, p := range *t {
if s := p.sock.Load(); s != nil {
u.detachLocked(s, why, false)
}
}
}
}
// closeConnSocks detaches every connected socket and discards the ones still opening, for a network change.
func (u *StdConn) closeConnSocks(why string) {
u.socks.mu.Lock()
u.socks.rebinds++
u.detachAllLocked(why)
u.socks.mu.Unlock()
}
func (u *StdConn) reapConnSocks(done <-chan struct{}) {
cs := u.socks
idle := cs.cfg.idle
tick := time.NewTicker(max(min(idle/4, 5*time.Second), time.Millisecond))
defer tick.Stop()
for {
select {
case <-done:
return
case <-tick.C:
}
t := cs.table.Load()
if t == nil {
continue
}
now := cs.clock()
for _, p := range *t {
if s := p.sock.Load(); s != nil && time.Duration(now-s.used.Load()) > idle {
u.closeConnSock(s, "idle", false)
}
}
}
}
// connSockDead reports whether a connected socket send or receive error means the connected path to the peer is
// gone, so its traffic belongs back on the listener. A connected UDP socket reports a peer's ICMP unreachable as
// ECONNREFUSED on its next call, which the unconnected listener never sees.
func connSockDead(errno syscall.Errno) bool {
switch errno {
case unix.ECONNREFUSED, unix.EHOSTUNREACH, unix.EHOSTDOWN, unix.ENETUNREACH, unix.ENETDOWN,
unix.EADDRNOTAVAIL, unix.EPIPE, unix.ENOTCONN, unix.EDESTADDRREQ:
return true
}
return false
}
// enobufsWait is how long a connected socket send sleeps when the interface queue is flow controlled. The kernel
// posts no wakeup when the advisory lifts, so this polls; 50us kept a gigabit link full without spinning a core in
// the 09-25 probes.
const enobufsWait = 50 * time.Microsecond
// connSockWrite writes bufs on s, one datagram per write. A full socket buffer parks the caller in the netpoller
// until the socket is writable, and a flow-controlled interface makes it wait out ENOBUFS, so a busy link slows
// the tun reader instead of dropping. Once the connected socket has waited cfg.enobufsMax without a send going
// through, either one drops the datagram instead, and a dropped datagram counts as sent, as the listener's
// unreported interface drops do; it never falls back to the listener, which would reorder it. closed reports that
// the connected socket was closed before bufs[sent] went out; any other error stops the run and is returned with
// how many went out.
func (u *StdConn) connSockWrite(s *connSock, bufs [][]byte) (sent int, errno syscall.Errno, closed bool) {
cs := u.socks
defer func() {
if sent > 0 {
s.used.Store(cs.clock())
}
}()
for sent < len(bufs) {
err := s.try(bufs[sent], false)
if err == syscall.EAGAIN || err == os.ErrDeadlineExceeded {
// os.ErrDeadlineExceeded here is another writer's parked deadline, passed but not yet cleared.
err = u.parkedWrite(s, bufs[sent])
}
switch err {
case nil:
sent++
if s.stallAt.Load() != 0 {
s.stallAt.Store(0)
s.dropping.Store(false)
}
continue
case errStallSpent:
u.dropStalled(s)
sent++
continue
}
errno, ok := err.(syscall.Errno)
if !ok {
return sent, 0, true
}
if errno != syscall.ENOBUFS {
return sent, errno, false
}
// The wait holds no reference on the descriptor, so a close doesn't wait for it.
if s.stalledFor(cs.clock()) < cs.cfg.enobufsMax {
cs.sleep(enobufsWait)
} else {
u.dropStalled(s)
sent++
}
}
return sent, 0, false
}
// parkedWrite writes b after EAGAIN, parked in the netpoller for what is left of s's stall budget, and returns
// errStallSpent once none is. golang/go#73919: a darwin UDP write parked on EAGAIN can wait forever for a
// writability event that never comes, so the wait always has a deadline.
func (u *StdConn) parkedWrite(s *connSock, b []byte) error {
cs := u.socks
s.parkMu.Lock()
defer s.parkMu.Unlock()
left := cs.cfg.enobufsMax - s.stalledFor(cs.clock())
if left <= 0 {
return errStallSpent
}
if err := s.conn.SetWriteDeadline(time.Now().Add(left)); err != nil {
return net.ErrClosed
}
err := s.try(b, true)
_ = s.conn.SetWriteDeadline(time.Time{})
if err == os.ErrDeadlineExceeded {
return errStallSpent
}
return err
}
// dropStalled logs the first datagram a stall drops.
func (u *StdConn) dropStalled(s *connSock) {
if !s.dropping.Swap(true) {
u.l.Debug("connected socket: sends stalled, dropping datagrams until one goes through", "udpAddr", s.dst)
}
}
// stalledFor records an ENOBUFS or EAGAIN on s at now and returns how long its sends have stalled: since the first
// ENOBUFS or EAGAIN after a send last went through. Every writer measures from that one time, so concurrent writers
// spend the budget no faster than one, and a send going through ends the stall for every writer, including one
// asleep in it.
func (s *connSock) stalledFor(now int64) time.Duration {
if s.stallAt.CompareAndSwap(0, now) {
return 0
}
at := s.stallAt.Load()
if at == 0 {
return 0
}
return time.Duration(now - at)
}
// readConnSock delivers a connected socket's datagrams through the listener's reader, serialized with ListenOut,
// until the connected socket is closed. Any receive error closes it: a connected socket reports its peer's
// unreachables here, and retrying an unexpected error could spin.
func (u *StdConn) readConnSock(s *connSock) {
rd := u.reader.Load()
buf := make([]byte, MTU)
for {
n, from, err := s.conn.ReadFromUDPAddrPort(buf)
if err != nil {
if errors.Is(err, net.ErrClosed) {
return
}
why := err.Error()
var errno syscall.Errno
if errors.As(err, &errno) && connSockDead(errno) {
why = errno.Error()
} else {
u.l.Warn("connected socket: unexpected receive error", "udpAddr", s.dst, "error", err)
}
u.closeConnSock(s, why, true)
return
}
s.used.Store(u.socks.clock())
u.readMu.Lock()
rd.r(netip.AddrPortFrom(from.Addr().Unmap(), from.Port()), buf[:n:n])
rd.flush()
u.readMu.Unlock()
}
}
File diff suppressed because it is too large. Load diff
+6
View File
@@ -0,0 +1,6 @@
//go:build !race && !e2e_testing
package udp
// raceEnabled reports a -race build, whose sync.Pool drops a random share of what is put back.
const raceEnabled = false
+6
View File
@@ -0,0 +1,6 @@
//go:build race && !e2e_testing
package udp
// raceEnabled reports a -race build, whose sync.Pool drops a random share of what is put back.
const raceEnabled = true
+104 -7
View File
@@ -10,6 +10,8 @@ import (
"log/slog" "log/slog"
"net" "net"
"net/netip" "net/netip"
"sync"
"sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -22,9 +24,26 @@ type StdConn struct {
isV4 bool isV4 bool
sysFd uintptr sysFd uintptr
l *slog.Logger l *slog.Logger
// listenHost and port are the configured address and the bound port, which a connected socket shares.
listenHost netip.Addr
port uint16
// rc reaches the listener's descriptor for another listen routine's Rebind.
rc syscall.RawConn
socks *connSocks
// reader is ListenOut's reader, which connected socket readers share; readMu serializes them, since an
// EncReader isn't safe for concurrent use.
reader atomic.Pointer[readerPair]
readMu sync.Mutex
}
type readerPair struct {
r EncReader
flush func()
} }
var _ Conn = &StdConn{} var _ Conn = &StdConn{}
var _ EstablishedPeerConn = &StdConn{}
func NewListener(l *slog.Logger, s Settings) (Conn, error) { func NewListener(l *slog.Logger, s Settings) (Conn, error) {
lc := NewListenConfig(s.Multi) lc := NewListenConfig(s.Multi)
@@ -34,12 +53,13 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
} }
if uc, ok := pc.(*net.UDPConn); ok { if uc, ok := pc.(*net.UDPConn); ok {
c := &StdConn{UDPConn: uc, l: l} c := &StdConn{UDPConn: uc, l: l, listenHost: s.Listen.Addr()}
rc, err := uc.SyscallConn() rc, err := uc.SyscallConn()
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to open udp socket: %w", err) return nil, fmt.Errorf("failed to open udp socket: %w", err)
} }
c.rc = rc
err = rc.Control(func(fd uintptr) { err = rc.Control(func(fd uintptr) {
c.sysFd = fd c.sysFd = fd
@@ -53,6 +73,8 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
return nil, err return nil, err
} }
c.isV4 = la.Addr().Is4() c.isV4 = la.Addr().Is4()
c.port = la.Port()
c.socks = joinConnSocks(c, la, s)
return c, nil return c, nil
} }
@@ -89,6 +111,19 @@ func NewListenConfig(multi bool) net.ListenConfig {
func sendto(s int, buf []byte, flags int, to unsafe.Pointer, addrlen int32) (err error) func sendto(s int, buf []byte, flags int, to unsafe.Pointer, addrlen int32) (err error)
func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error { func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
if t := u.socks.table.Load(); t != nil {
if p := (*t)[ap]; p != nil {
bufs := [1][]byte{b}
if sent, err := u.connSockRun(p, ap, bufs[:]); sent == 1 || err != nil {
return err
}
}
}
return u.writeTo(b, ap)
}
// writeTo sends b through the listener.
func (u *StdConn) writeTo(b []byte, ap netip.AddrPort) error {
var sa unsafe.Pointer var sa unsafe.Pointer
var addrLen int32 var addrLen int32
@@ -141,18 +176,50 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch. // An un-sendable destination costs its own packet, never the ones behind it in the batch.
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer // TODO: writeTo maps EWOULDBLOCK to an error, so a full listener send buffer
// silently drops the rest of a burst (linux blocks instead). Poll for // silently drops those packets (linux blocks instead). Poll for
// writability on EAGAIN before giving up on the remainder. // writability on EAGAIN before giving up on them.
written := 0 written := 0
t := u.socks.table.Load()
if t == nil {
for i, b := range bufs { for i, b := range bufs {
if err := u.WriteTo(b, addrs[i]); err == nil { if err := u.writeTo(b, addrs[i]); err == nil {
written++ written++
} else { } else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err) u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
} }
} }
return written, nil return written, nil
}
for off := 0; off < len(bufs); {
// A run of datagrams to one peer goes out together, on its connected socket if it has or earns one.
dst := addrs[off]
end := off + 1
for end < len(bufs) && addrs[end] == dst {
end++
}
if p := (*t)[dst]; p != nil {
for off < end {
sent, err := u.connSockRun(p, dst, bufs[off:end])
written += sent
off += sent
if err == nil {
break
}
u.l.Debug("failed to write packet in batch", "udpAddr", dst, "error", err)
off++
}
}
// The listener sends what no connected socket took.
for ; off < end; off++ {
if err := u.writeTo(bufs[off], dst); err == nil {
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", dst, "error", err)
}
}
}
return written, nil
} }
func (u *StdConn) LocalAddr() (netip.AddrPort, error) { func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
@@ -182,6 +249,8 @@ func NewUDPStatsEmitter(udpConns []Conn) func() {
func (u *StdConn) ListenOut(r EncReader, flush func()) error { func (u *StdConn) ListenOut(r EncReader, flush func()) error {
buffer := make([]byte, MTU) buffer := make([]byte, MTU)
u.reader.Store(&readerPair{r: r, flush: flush})
u.startConnSocks()
for { for {
// Just read one packet at a time // Just read one packet at a time
@@ -194,18 +263,29 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
continue continue
} }
u.readMu.Lock()
r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n:n]) r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n:n])
flush() flush()
u.readMu.Unlock()
} }
} }
// Close closes the connected sockets of every listen routine, then the listener.
func (u *StdConn) Close() error {
u.stopConnSocks()
return u.UDPConn.Close()
}
func (u *StdConn) SupportsMultipleReaders() bool { func (u *StdConn) SupportsMultipleReaders() bool {
return false return false
} }
// Rebind clears the interface the kernel scoped this socket to, so that sends are routed against the current // Rebind clears the interface the kernel scoped this socket to, so that sends are routed against the current
// routing table instead of the interface we happened to be on when the socket was created. Darwin pins sockets // routing table instead of the interface we happened to be on when the socket was created. Darwin pins sockets
// this way on its own, which is what strands us after the underlying network changes. // this way on its own, which is what strands us after the underlying network changes. Control.RebindUDPServer
// rebinds only the first writer, so this clears the other listen routines' listeners too. Connected sockets are
// closed after the listeners are cleared, and later ones copy the cleared scope and pick their source addresses
// afresh.
func (u *StdConn) Rebind() error { func (u *StdConn) Rebind() error {
var err error var err error
if u.isV4 { if u.isV4 {
@@ -213,6 +293,23 @@ func (u *StdConn) Rebind() error {
} else { } else {
err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0) err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0)
} }
for _, m := range u.socks.siblings(u) {
err = errors.Join(err, m.clearBoundIf())
}
u.closeConnSocks("rebind")
return err return err
} }
// clearBoundIf clears the interface the listener is scoped to through its RawConn, which fails on a listener that
// closed concurrently instead of reaching whatever reused its descriptor number.
func (u *StdConn) clearBoundIf() error {
level, opt := syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF
if u.isV4 {
level, opt = syscall.IPPROTO_IP, syscall.IP_BOUND_IF
}
var serr error
if err := u.rc.Control(func(fd uintptr) { serr = syscall.SetsockoptInt(int(fd), level, opt, 0) }); err != nil {
return err
}
return serr
}
+1 -1
View File
@@ -30,7 +30,7 @@ func procyield(cycles uint32)
const ( const (
packetsPerRing = 1024 packetsPerRing = 1024
// Fits a relayed packet at overlay.MaxMTU, which tun.mtu and route mtus are capped to. 17.6MB page locked per socket. // Caps tun.mtu at MTU-32 direct, MTU-64 relayed, unenforced anywhere else. 17.6MB page locked per socket.
bytesPerPacket = MTU bytesPerPacket = MTU
receiveSpins = 15 receiveSpins = 15
) )