mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 15:06:52 +01:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
781a590b8a | ||
|
|
8a2de9b1b5 |
No files matched your search
@@ -0,0 +1,68 @@
|
||||
# vpnemu
|
||||
|
||||
Reproduces, on macOS and without Tailscale, the way a VPN with an exit node takes nebula's vpn network route
|
||||
away from its utun (#1931).
|
||||
|
||||
A macOS network extension never touches the routing table. Its routes are published in configd's dynamic store
|
||||
under `State:/Network/Service/<id>/IPv4` and `/IPv6`, and configd's IPMonitor installs them. vpnemu creates a
|
||||
utun and publishes those keys itself, so configd does exactly what it would do for the real app. It publishes
|
||||
the routes Tailscale would with an exit node: every interface prefix that isn't a single IP, the rule in
|
||||
Tailscale's `ipnlocal.internalAndExternalInterfacesFrom`.
|
||||
|
||||
| mode | Tailscale setting | published as | where nebula's prefix goes |
|
||||
| --- | --- | --- | --- |
|
||||
| `claim` | exit node, no LAN access | `AdditionalRoutes` | into vpnemu's utun |
|
||||
| `exclude` | exit node, LAN access | `ExcludedRoutes`, service overrides primary | out the next interface (en0) |
|
||||
|
||||
configd answers `EEXIST` with a delete and an add, so whatever held the prefix loses it. When the service is
|
||||
withdrawn, configd deletes its routes by prefix alone, whoever holds them by then.
|
||||
|
||||
## Build
|
||||
|
||||
It is its own module, so `purego` stays out of nebula's `go.mod`. purego reaches CoreFoundation and
|
||||
SystemConfiguration without cgo, so it cross compiles and runs on a mac with no developer tools.
|
||||
|
||||
```sh
|
||||
cd e2e/darwin/vpnemu
|
||||
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o vpnemu .
|
||||
# a cross compiled binary copied to a mac may need: codesign -s - -f vpnemu
|
||||
```
|
||||
|
||||
## Run
|
||||
|
||||
```sh
|
||||
sudo ./vpnemu -mode claim -keep en0 # runs until ctrl-c, then withdraws and closes its utun
|
||||
sudo ./vpnemu -clear # withdraw a service a crashed run left behind
|
||||
```
|
||||
|
||||
`-keep` names interfaces whose prefixes are never published. Keep the one your ssh session arrives on, or
|
||||
`claim` takes your LAN into a utun nobody reads. `exclude` makes vpnemu primary, so anything off a kept
|
||||
interface's subnet, internet included, goes into its utun and is dropped while it runs.
|
||||
|
||||
`-follow` (default on) republishes when interface prefixes change, as Tailscale does on link changes. With
|
||||
`-follow=false` a route stays published after its interface is gone, which stages the route configd leaves
|
||||
behind for a nebula that restarts before the VPN has caught up.
|
||||
|
||||
## A/B a nebula build
|
||||
|
||||
`repro.sh` starts nebula, then runs vpnemu, and reports egress and ping per peer before, during and after:
|
||||
|
||||
```sh
|
||||
sudo ./repro.sh -n ./nebula -c config.yml -p 192.168.212.1 -p fd19:31::1 -m claim -k en0
|
||||
```
|
||||
|
||||
Before #1931 (utun holds `/24` and `/64`):
|
||||
|
||||
```
|
||||
during 192.168.212.1 egress=utun2 ping=FAIL
|
||||
after 192.168.212.1 egress=en0 ping=FAIL
|
||||
```
|
||||
|
||||
With #1931 (utun holds `/32` and `/128`, network routes added explicitly) every phase stays on nebula's utun.
|
||||
|
||||
### Leftover route
|
||||
|
||||
Run an old nebula, start `vpnemu -follow=false`, then restart onto a new nebula while vpnemu is still up. The
|
||||
new nebula finds the vpn network routed to vpnemu's utun, replaces it and starts. Stop vpnemu and configd deletes
|
||||
the prefix, taking nebula's route with it until nebula restarts. With real Tailscale this needs nebula to restart
|
||||
before Tailscale notices the old utun is gone, a one time window when upgrading from a nebula without #1931.
|
||||
@@ -0,0 +1,8 @@
|
||||
module github.com/slackhq/nebula/e2e/darwin/vpnemu
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/ebitengine/purego v0.9.1
|
||||
golang.org/x/sys v0.48.0
|
||||
)
|
||||
@@ -0,0 +1,4 @@
|
||||
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
|
||||
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
@@ -0,0 +1,292 @@
|
||||
//go:build darwin
|
||||
|
||||
// vpnemu stands in for a macOS VPN app with an exit node in use, so route hijacks like the one
|
||||
// Tailscale causes can be reproduced without Tailscale.
|
||||
//
|
||||
// A network extension does not touch the routing table itself. It hands its routes to
|
||||
// nesessionmanager, which publishes them in configd's dynamic store under
|
||||
// State:/Network/Service/<id>/IPv4 and /IPv6, and configd's IPMonitor installs them. vpnemu
|
||||
// creates a utun and publishes the same keys directly, so configd does exactly what it would do
|
||||
// for the real thing.
|
||||
//
|
||||
// Which routes it publishes follows Tailscale's rule for exit nodes
|
||||
// (ipnlocal.internalAndExternalInterfacesFrom): every interface address prefix that is not a
|
||||
// single IP counts as an attached LAN. With LAN access off those prefixes are claimed into the
|
||||
// tunnel (-mode claim, published as AdditionalRoutes). With LAN access on they are excluded and
|
||||
// leave through the next interface (-mode exclude, published as ExcludedRoutes on a service that
|
||||
// overrides the primary).
|
||||
//
|
||||
// It runs until interrupted, then withdraws the service and closes its utun. Needs root.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/ebitengine/purego"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const (
|
||||
serviceID = "VPNEMU"
|
||||
// Tailscale's own addresses, single IPs, so the rule skips them like Tailscale skips its own
|
||||
addr4 = "100.100.0.1"
|
||||
addr6 = "fd7a:115c:a1e0::1"
|
||||
|
||||
utunControlName = "com.apple.net.utun_control"
|
||||
utunOptIfname = 2
|
||||
cfStringUTF8 = 0x08000100
|
||||
)
|
||||
|
||||
var (
|
||||
cfStringCreateWithCString func(alloc uintptr, s *byte, enc uint32) uintptr
|
||||
cfDataCreate func(alloc uintptr, b *byte, n int) uintptr
|
||||
cfPropertyListCreateWithData func(alloc, data uintptr, opts uint, format *int, err *uintptr) uintptr
|
||||
cfRelease func(ref uintptr)
|
||||
scDynamicStoreCreate func(alloc, name, callout, ctx uintptr) uintptr
|
||||
scDynamicStoreSetValue func(store, key, value uintptr) bool
|
||||
scDynamicStoreRemoveValue func(store, key uintptr) bool
|
||||
scError func() int32
|
||||
)
|
||||
|
||||
// loadFrameworks binds the few CoreFoundation and SystemConfiguration calls we need. purego does
|
||||
// it without cgo, so this cross compiles and runs on a mac with no developer tools. JXA can read
|
||||
// the dynamic store but its writes fail, and scutil cannot build the arrays of dictionaries the
|
||||
// route keys hold.
|
||||
func loadFrameworks() {
|
||||
cf, err := purego.Dlopen("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", purego.RTLD_NOW|purego.RTLD_GLOBAL)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
sc, err := purego.Dlopen("/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration", purego.RTLD_NOW|purego.RTLD_GLOBAL)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
purego.RegisterLibFunc(&cfStringCreateWithCString, cf, "CFStringCreateWithCString")
|
||||
purego.RegisterLibFunc(&cfDataCreate, cf, "CFDataCreate")
|
||||
purego.RegisterLibFunc(&cfPropertyListCreateWithData, cf, "CFPropertyListCreateWithData")
|
||||
purego.RegisterLibFunc(&cfRelease, cf, "CFRelease")
|
||||
purego.RegisterLibFunc(&scDynamicStoreCreate, sc, "SCDynamicStoreCreate")
|
||||
purego.RegisterLibFunc(&scDynamicStoreSetValue, sc, "SCDynamicStoreSetValue")
|
||||
purego.RegisterLibFunc(&scDynamicStoreRemoveValue, sc, "SCDynamicStoreRemoveValue")
|
||||
purego.RegisterLibFunc(&scError, sc, "SCError")
|
||||
}
|
||||
|
||||
func cfString(s string) uintptr {
|
||||
b := append([]byte(s), 0)
|
||||
return cfStringCreateWithCString(0, &b[0], cfStringUTF8)
|
||||
}
|
||||
|
||||
// cfPlist builds a property list from its xml form, the least code way to make nested CF values
|
||||
func cfPlist(xml string) uintptr {
|
||||
b := []byte(xml)
|
||||
d := cfDataCreate(0, &b[0], len(b))
|
||||
defer cfRelease(d)
|
||||
var errRef uintptr
|
||||
p := cfPropertyListCreateWithData(0, d, 0, nil, &errRef)
|
||||
if p == 0 {
|
||||
log.Fatalf("invalid property list:\n%s", xml)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
type store struct {
|
||||
ref uintptr
|
||||
k4, k6 uintptr
|
||||
}
|
||||
|
||||
func openStore() *store {
|
||||
return &store{
|
||||
ref: scDynamicStoreCreate(0, cfString("vpnemu"), 0, 0),
|
||||
k4: cfString("State:/Network/Service/" + serviceID + "/IPv4"),
|
||||
k6: cfString("State:/Network/Service/" + serviceID + "/IPv6"),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *store) set(key uintptr, xml string) {
|
||||
v := cfPlist(xml)
|
||||
defer cfRelease(v)
|
||||
if !scDynamicStoreSetValue(s.ref, key, v) {
|
||||
log.Fatalf("SCDynamicStoreSetValue failed: SCError %d", scError())
|
||||
}
|
||||
}
|
||||
|
||||
func (s *store) withdraw() {
|
||||
scDynamicStoreRemoveValue(s.ref, s.k4)
|
||||
scDynamicStoreRemoveValue(s.ref, s.k6)
|
||||
}
|
||||
|
||||
// createUtun opens a utun the kernel numbers for us and gives it Tailscale-like single addresses.
|
||||
// The interface lives as long as the returned fd.
|
||||
func createUtun() (int, string) {
|
||||
fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL)
|
||||
if err != nil {
|
||||
log.Fatalf("system socket: %v", err)
|
||||
}
|
||||
info := &unix.CtlInfo{}
|
||||
copy(info.Name[:], utunControlName)
|
||||
if err = unix.IoctlCtlInfo(fd, info); err != nil {
|
||||
log.Fatalf("CTLIOCGINFO: %v", err)
|
||||
}
|
||||
if err = unix.Connect(fd, &unix.SockaddrCtl{ID: info.Id, Unit: 0}); err != nil {
|
||||
log.Fatalf("connect utun control: %v", err)
|
||||
}
|
||||
name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, utunOptIfname)
|
||||
if err != nil {
|
||||
log.Fatalf("utun name: %v", err)
|
||||
}
|
||||
|
||||
for _, args := range [][]string{
|
||||
{name, "inet", addr4, addr4, "netmask", "255.255.255.255", "up"},
|
||||
{name, "inet6", addr6, "prefixlen", "128"},
|
||||
} {
|
||||
if out, err := exec.Command("ifconfig", args...).CombinedOutput(); err != nil {
|
||||
log.Fatalf("ifconfig %s: %v: %s", strings.Join(args, " "), err, out)
|
||||
}
|
||||
}
|
||||
return fd, name
|
||||
}
|
||||
|
||||
// lanPrefixes applies Tailscale's attached-LAN rule to every interface but those in skip
|
||||
func lanPrefixes(skip []string) (v4, v6 []netip.Prefix) {
|
||||
ifs, err := net.Interfaces()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for _, i := range ifs {
|
||||
if slices.Contains(skip, i.Name) || i.Flags&net.FlagLoopback != 0 {
|
||||
continue
|
||||
}
|
||||
addrs, _ := i.Addrs()
|
||||
for _, a := range addrs {
|
||||
n, ok := a.(*net.IPNet)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
ip, _ := netip.AddrFromSlice(n.IP)
|
||||
ip = ip.Unmap()
|
||||
ones, _ := n.Mask.Size()
|
||||
pfx := netip.PrefixFrom(ip, ones).Masked()
|
||||
// Tailscale skips single IPs. Link-local is our addition, every interface has the
|
||||
// same fe80::/64 and claiming it would only break the harness's own host.
|
||||
if pfx.IsSingleIP() || ip.IsLinkLocalUnicast() {
|
||||
continue
|
||||
}
|
||||
if ip.Is4() {
|
||||
v4 = append(v4, pfx)
|
||||
} else {
|
||||
v6 = append(v6, pfx)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.SortFunc(v4, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) })
|
||||
slices.SortFunc(v6, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) })
|
||||
return v4, v6
|
||||
}
|
||||
|
||||
func routes4(ps []netip.Prefix) string {
|
||||
var b strings.Builder
|
||||
for _, p := range ps {
|
||||
mask := net.IP(net.CIDRMask(p.Bits(), 32)).String()
|
||||
fmt.Fprintf(&b, "<dict><key>DestinationAddress</key><string>%s</string><key>SubnetMask</key><string>%s</string></dict>", p.Addr(), mask)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func routes6(ps []netip.Prefix) string {
|
||||
var b strings.Builder
|
||||
for _, p := range ps {
|
||||
fmt.Fprintf(&b, "<dict><key>DestinationAddress</key><string>%s</string><key>PrefixLength</key><integer>%d</integer></dict>", p.Addr(), p.Bits())
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (s *store) publish(ifName string, exclude bool, v4, v6 []netip.Prefix) {
|
||||
routesKey, primary := "AdditionalRoutes", ""
|
||||
if exclude {
|
||||
// configd only steers excluded routes for a service with a default route that is primary.
|
||||
// While this runs, everything not on a kept interface's subnet goes into the utun and dies.
|
||||
routesKey = "ExcludedRoutes"
|
||||
primary = "<key>Router</key><string>" + addr4 + "</string><key>OverridePrimary</key><integer>1</integer>"
|
||||
}
|
||||
|
||||
const head = `<?xml version="1.0" encoding="UTF-8"?><plist version="1.0"><dict>`
|
||||
s.set(s.k4, head+
|
||||
"<key>InterfaceName</key><string>"+ifName+"</string>"+
|
||||
"<key>Addresses</key><array><string>"+addr4+"</string></array>"+
|
||||
"<key>SubnetMasks</key><array><string>255.255.255.255</string></array>"+primary+
|
||||
"<key>"+routesKey+"</key><array>"+routes4(v4)+"</array></dict></plist>")
|
||||
s.set(s.k6, head+
|
||||
"<key>InterfaceName</key><string>"+ifName+"</string>"+
|
||||
"<key>Addresses</key><array><string>"+addr6+"</string></array>"+
|
||||
"<key>PrefixLength</key><array><integer>128</integer></array>"+
|
||||
"<key>"+routesKey+"</key><array>"+routes6(v6)+"</array></dict></plist>")
|
||||
|
||||
log.Printf("published %s on %s, %s: %v %v", serviceID, ifName, routesKey, v4, v6)
|
||||
}
|
||||
|
||||
func main() {
|
||||
mode := flag.String("mode", "claim", "claim: exit node without LAN access, the prefixes are routed into the vpn\n"+
|
||||
"exclude: exit node with LAN access, the prefixes leave via the next interface and the vpn becomes primary")
|
||||
keep := flag.String("keep", "en0", "comma separated interfaces whose prefixes are never published, keep the one your ssh session uses")
|
||||
follow := flag.Bool("follow", true, "republish when interface prefixes change, as Tailscale does on link changes.\n"+
|
||||
"Off, a route stays published after its interface is gone, which is how a leftover route is staged")
|
||||
clear := flag.Bool("clear", false, "withdraw a service a crashed run left behind, then exit")
|
||||
flag.Parse()
|
||||
|
||||
if *mode != "claim" && *mode != "exclude" {
|
||||
log.Fatalf("unknown -mode %q", *mode)
|
||||
}
|
||||
if os.Geteuid() != 0 {
|
||||
log.Fatal("needs root, to create the utun and write configd's dynamic store")
|
||||
}
|
||||
|
||||
loadFrameworks()
|
||||
s := openStore()
|
||||
if *clear {
|
||||
s.withdraw()
|
||||
log.Print("withdrawn")
|
||||
return
|
||||
}
|
||||
|
||||
fd, ifName := createUtun()
|
||||
exclude := *mode == "exclude"
|
||||
skip := append(strings.Split(*keep, ","), ifName)
|
||||
|
||||
v4, v6 := lanPrefixes(skip)
|
||||
s.publish(ifName, exclude, v4, v6)
|
||||
|
||||
sig := make(chan os.Signal, 1)
|
||||
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
|
||||
tick := time.NewTicker(time.Second)
|
||||
defer tick.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-sig:
|
||||
s.withdraw()
|
||||
_ = unix.Close(fd)
|
||||
log.Print("withdrawn")
|
||||
return
|
||||
case <-tick.C:
|
||||
if !*follow {
|
||||
continue
|
||||
}
|
||||
n4, n6 := lanPrefixes(skip)
|
||||
if !slices.Equal(n4, v4) || !slices.Equal(n6, v6) {
|
||||
v4, v6 = n4, n6
|
||||
s.publish(ifName, exclude, v4, v6)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/bin/sh
|
||||
# Reports where traffic to nebula peers goes before, during and after a vpnemu exit node.
|
||||
#
|
||||
# usage: sudo ./repro.sh -n <nebula binary> -c <nebula config> -p <peer vpn addr> [-p <peer>...]
|
||||
# [-m claim|exclude] [-k <keep interfaces>]
|
||||
#
|
||||
# Each peer should answer pings over the overlay. A healthy run shows the nebula utun as egress
|
||||
# and the ping working in all three phases.
|
||||
set -u
|
||||
|
||||
MODE=claim
|
||||
KEEP=en0
|
||||
PEERS=""
|
||||
NEBULA=""
|
||||
CONFIG=""
|
||||
while getopts n:c:p:m:k: o; do
|
||||
case $o in
|
||||
n) NEBULA=$OPTARG ;;
|
||||
c) CONFIG=$OPTARG ;;
|
||||
p) PEERS="$PEERS $OPTARG" ;;
|
||||
m) MODE=$OPTARG ;;
|
||||
k) KEEP=$OPTARG ;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
done
|
||||
if [ -z "$NEBULA" ] || [ -z "$CONFIG" ] || [ -z "$PEERS" ]; then
|
||||
sed -n '4,5p' "$0"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
HERE=$(cd "$(dirname "$0")" && pwd)
|
||||
VPNEMU=${VPNEMU:-$HERE/vpnemu}
|
||||
LOG=$(mktemp -t nebula-repro)
|
||||
|
||||
report() {
|
||||
for p in $PEERS; do
|
||||
case $p in
|
||||
*:*) fam=-inet6 ping=ping6 ;;
|
||||
*) fam=-inet ping="ping -t 3" ;;
|
||||
esac
|
||||
egress=$(route -n get $fam "$p" 2>/dev/null | awk '/interface:/{print $2}')
|
||||
if $ping -c 2 "$p" >/dev/null 2>&1; then ok=ok; else ok=FAIL; fi
|
||||
printf '%-8s %-24s egress=%-8s ping=%s\n' "$1" "$p" "${egress:-none}" "$ok"
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
[ -n "${EMU:-}" ] && kill "$EMU" 2>/dev/null
|
||||
[ -n "${NEB:-}" ] && kill "$NEB" 2>/dev/null
|
||||
wait 2>/dev/null
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
"$NEBULA" -config "$CONFIG" >"$LOG" 2>&1 &
|
||||
NEB=$!
|
||||
sleep 3
|
||||
if ! kill -0 $NEB 2>/dev/null; then
|
||||
echo "nebula exited:"
|
||||
tail -5 "$LOG"
|
||||
exit 1
|
||||
fi
|
||||
echo "nebula on $(grep -o 'interface=utun[0-9]*' "$LOG" | head -1 | cut -d= -f2), log in $LOG"
|
||||
report before
|
||||
|
||||
"$VPNEMU" -mode "$MODE" -keep "$KEEP" &
|
||||
EMU=$!
|
||||
sleep 3
|
||||
report during
|
||||
|
||||
kill $EMU
|
||||
wait $EMU 2>/dev/null
|
||||
EMU=
|
||||
sleep 2
|
||||
report after
|
||||
|
||||
grep -E 'level=(WARN|ERROR).*(route|Route)' "$LOG"
|
||||
+5
-5
@@ -135,12 +135,12 @@ listen:
|
||||
# superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the
|
||||
# default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts.
|
||||
#batch: 64
|
||||
# Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
|
||||
# Linux only. Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
|
||||
# Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
|
||||
# Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide
|
||||
# max, net.core.rmem_max and net.core.wmem_max
|
||||
#read_buffer: 10485760
|
||||
#write_buffer: 10485760
|
||||
read_buffer: 3412512
|
||||
write_buffer: 3412512
|
||||
|
||||
# On Windows only
|
||||
# When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port.
|
||||
@@ -262,8 +262,8 @@ tun:
|
||||
drop_local_broadcast: false
|
||||
# Toggles forwarding of multicast packets
|
||||
drop_multicast: false
|
||||
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500
|
||||
tx_queue: 500
|
||||
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 5000
|
||||
tx_queue: 5000
|
||||
# Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
|
||||
# Anything above 8937, here or on a route below, is capped to 8937 with a warning
|
||||
mtu: 1300
|
||||
|
||||
+1
-1
@@ -43,7 +43,7 @@ type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix,
|
||||
func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
|
||||
switch {
|
||||
case c.GetBool("tun.disabled", false):
|
||||
tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
|
||||
tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 5000), c.GetBool("stats.message_metrics", false), l)
|
||||
return tun, nil
|
||||
|
||||
default:
|
||||
|
||||
@@ -221,7 +221,7 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
|
||||
vnetHdr: vnetHdr,
|
||||
offloadFlags: offloadFlags,
|
||||
vpnNetworks: vpnNetworks,
|
||||
TXQueueLen: c.GetInt("tun.tx_queue", 500),
|
||||
TXQueueLen: c.GetInt("tun.tx_queue", 5000),
|
||||
useSystemRoutes: c.GetBool("tun.use_system_route_table", false),
|
||||
useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
|
||||
routesFromSystem: map[netip.Prefix]routing.Gateways{},
|
||||
|
||||
+4
-24
@@ -135,23 +135,12 @@ func (u *StdConn) Rebind() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetRecvBuffer tries SO_RCVBUFFORCE, which needs CAP_NET_ADMIN. Without it we
|
||||
// fall back to SO_RCVBUF, which the kernel clamps to net.core.rmem_max.
|
||||
func (u *StdConn) SetRecvBuffer(n int) error {
|
||||
err := unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n)
|
||||
if errors.Is(err, unix.EPERM) {
|
||||
err = unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUF, n)
|
||||
}
|
||||
return err
|
||||
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n)
|
||||
}
|
||||
|
||||
// SetSendBuffer is SetRecvBuffer for SO_SNDBUFFORCE/SO_SNDBUF and net.core.wmem_max.
|
||||
func (u *StdConn) SetSendBuffer(n int) error {
|
||||
err := unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
|
||||
if errors.Is(err, unix.EPERM) {
|
||||
err = unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF, n)
|
||||
}
|
||||
return err
|
||||
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
|
||||
}
|
||||
|
||||
func (u *StdConn) SetSoMark(mark int) error {
|
||||
@@ -407,13 +396,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
|
||||
b := c.GetInt("listen.read_buffer", 0)
|
||||
if b > 0 {
|
||||
if err := u.SetRecvBuffer(b); err == nil {
|
||||
// The kernel reports double the requested size
|
||||
if s, err := u.GetRecvBuffer(); err == nil && s >= 2*b {
|
||||
if s, err := u.GetRecvBuffer(); err == nil {
|
||||
u.l.Info("listen.read_buffer was set", "size", s)
|
||||
} else if err == nil {
|
||||
u.l.Warn("listen.read_buffer was limited by the system",
|
||||
"requested", b, "size", s,
|
||||
"hint", "raise net.core.rmem_max or grant CAP_NET_ADMIN")
|
||||
} else {
|
||||
u.l.Warn("Failed to get listen.read_buffer", "error", err)
|
||||
}
|
||||
@@ -425,12 +409,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
|
||||
b = c.GetInt("listen.write_buffer", 0)
|
||||
if b > 0 {
|
||||
if err := u.SetSendBuffer(b); err == nil {
|
||||
if s, err := u.GetSendBuffer(); err == nil && s >= 2*b {
|
||||
if s, err := u.GetSendBuffer(); err == nil {
|
||||
u.l.Info("listen.write_buffer was set", "size", s)
|
||||
} else if err == nil {
|
||||
u.l.Warn("listen.write_buffer was limited by the system",
|
||||
"requested", b, "size", s,
|
||||
"hint", "raise net.core.wmem_max or grant CAP_NET_ADMIN")
|
||||
} else {
|
||||
u.l.Warn("Failed to get listen.write_buffer", "error", err)
|
||||
}
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
//go:build linux && !android && !e2e_testing
|
||||
|
||||
package udp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
)
|
||||
|
||||
func readSysctlInt(t *testing.T, path string) int {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Skipf("can't read %s: %v", path, err)
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(b)))
|
||||
if err != nil {
|
||||
t.Skipf("can't parse %s: %v", path, err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// TestSetBufferAboveSysctlMax asks for more than net.core.{r,w}mem_max. With
|
||||
// CAP_NET_ADMIN the FORCE variant takes it as is; without it we must fall back
|
||||
// to the plain option and get the clamped size instead of EPERM.
|
||||
func TestSetBufferAboveSysctlMax(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
sysctl string
|
||||
set func(*StdConn, int) error
|
||||
get func(*StdConn) (int, error)
|
||||
}{
|
||||
{"read", "/proc/sys/net/core/rmem_max", (*StdConn).SetRecvBuffer, (*StdConn).GetRecvBuffer},
|
||||
{"write", "/proc/sys/net/core/wmem_max", (*StdConn).SetSendBuffer, (*StdConn).GetSendBuffer},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sysMax := readSysctlInt(t, tc.sysctl)
|
||||
c, err := NewListener(testLogger(), Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Batch: 1})
|
||||
if err != nil {
|
||||
t.Skipf("NewListener: %v", err)
|
||||
}
|
||||
sc := c.(*StdConn)
|
||||
defer sc.Close()
|
||||
|
||||
want := sysMax + 1<<20
|
||||
if err := tc.set(sc, want); err != nil {
|
||||
t.Fatalf("set %d: %v", want, err)
|
||||
}
|
||||
got, err := tc.get(sc)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
// The kernel doubles the value; either path gives at least the clamped size.
|
||||
if got < 2*sysMax {
|
||||
t.Fatalf("effective size %d, want >= %d", got, 2*sysMax)
|
||||
}
|
||||
t.Logf("requested %d, %s=%d, effective %d (forced=%v)", want, tc.sysctl, sysMax, got, got >= 2*want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReloadConfigBufferLogLevel checks that a clamped buffer is a Warn, and a
|
||||
// buffer that fits is still the Info it always was.
|
||||
func TestReloadConfigBufferLogLevel(t *testing.T) {
|
||||
sysMax := readSysctlInt(t, "/proc/sys/net/core/rmem_max")
|
||||
if wmax := readSysctlInt(t, "/proc/sys/net/core/wmem_max"); wmax < sysMax {
|
||||
sysMax = wmax
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
size int
|
||||
want []string
|
||||
}{
|
||||
{"fits", 64 * 1024, []string{"level=INFO msg=\"listen.read_buffer was set\"", "level=INFO msg=\"listen.write_buffer was set\""}},
|
||||
{"too big", sysMax + 1<<20, []string{"level=WARN msg=\"listen.read_buffer was limited by the system\"", "level=WARN msg=\"listen.write_buffer was limited by the system\""}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
l := slog.New(slog.NewTextHandler(&buf, nil))
|
||||
c, err := NewListener(l, Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Batch: 1})
|
||||
if err != nil {
|
||||
t.Skipf("NewListener: %v", err)
|
||||
}
|
||||
sc := c.(*StdConn)
|
||||
defer sc.Close()
|
||||
|
||||
if tc.size > sysMax {
|
||||
// With CAP_NET_ADMIN the FORCE variant takes any size, nothing gets limited.
|
||||
if err := sc.SetRecvBuffer(tc.size); err == nil {
|
||||
if got, _ := sc.GetRecvBuffer(); got >= 2*tc.size {
|
||||
t.Skip("FORCE path took the full size, can't observe the limit")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cfg := config.NewC(l)
|
||||
if err := cfg.LoadString(fmt.Sprintf("listen:\n read_buffer: %d\n write_buffer: %d\n", tc.size, tc.size)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buf.Reset()
|
||||
sc.ReloadConfig(cfg)
|
||||
out := buf.String()
|
||||
for _, w := range tc.want {
|
||||
if !strings.Contains(out, w) {
|
||||
t.Errorf("missing %s in:\n%s", w, out)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out, "level=ERROR") {
|
||||
t.Errorf("unexpected error log:\n%s", out)
|
||||
}
|
||||
t.Log(out)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user