mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 09:06:52 +01:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
73d26dd87d |
No files matched your search
@@ -295,6 +295,14 @@ tun:
|
|||||||
# - 2
|
# - 2
|
||||||
# - 4
|
# - 4
|
||||||
|
|
||||||
|
# Linux only. avoid_cpus removes these CPU IDs from the affinity of every thread of the process at startup,
|
||||||
|
# embedding applications included; later threads inherit the remaining set. Applied before cpu_affinity and the
|
||||||
|
# default pin selection, independent of pin_threads. Invalid IDs, or IDs that would leave no CPU, fail
|
||||||
|
# startup. Not reloadable; ignored on other platforms. To keep Nebula off the underlay NIC's receive IRQ CPUs, find
|
||||||
|
# them in /proc/interrupts and /proc/irq/<IRQ>/effective_affinity_list, add their SMT siblings from
|
||||||
|
# /sys/devices/system/cpu/cpu<N>/topology/thread_siblings_list, and revisit the list when IRQ placement changes.
|
||||||
|
#avoid_cpus: []
|
||||||
|
|
||||||
# Route based MTU overrides, you have known vpn ip paths that can support larger MTUs you can increase/decrease them here
|
# Route based MTU overrides, you have known vpn ip paths that can support larger MTUs you can increase/decrease them here
|
||||||
routes:
|
routes:
|
||||||
#- mtu: 8800
|
#- mtu: 8800
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package nebula
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
@@ -31,6 +32,10 @@ const maxRoutines = 1 << 16
|
|||||||
const _ = noiseutil.RejectHeadroom - 4*maxRoutines
|
const _ = noiseutil.RejectHeadroom - 4*maxRoutines
|
||||||
|
|
||||||
func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
|
func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
|
||||||
|
// Apply before starting services or selecting per-reader pin CPUs.
|
||||||
|
if err := configureAvoidCPUs(c, l, configTest); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
// The goroutines started below stop only when this context does, and only a caller holding the
|
// The goroutines started below stop only when this context does, and only a caller holding the
|
||||||
// Control can arrange that. Cancel whenever we are not handing one back, which covers an error
|
// Control can arrange that. Cancel whenever we are not handing one back, which covers an error
|
||||||
@@ -416,6 +421,41 @@ func parseCpuAffinity(c *config.C, l *slog.Logger, routines int) []int {
|
|||||||
return cpus
|
return cpus
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// configureAvoidCPUs applies tun.avoid_cpus, a list of CPU IDs to keep every thread of the process off. Unlike
|
||||||
|
// tun.cpu_affinity, a bad entry fails startup rather than being ignored. Configuration tests only validate it.
|
||||||
|
func configureAvoidCPUs(c *config.C, l *slog.Logger, configTest bool) error {
|
||||||
|
raw := c.Get("tun.avoid_cpus")
|
||||||
|
if raw == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
values, ok := raw.([]any)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("tun.avoid_cpus must be a list of non-negative integers")
|
||||||
|
}
|
||||||
|
cpus := make([]int, 0, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
cpu, ok := value.(int)
|
||||||
|
if !ok || cpu < 0 {
|
||||||
|
return fmt.Errorf("tun.avoid_cpus must contain non-negative integers, got %v", value)
|
||||||
|
}
|
||||||
|
cpus = append(cpus, cpu)
|
||||||
|
}
|
||||||
|
if len(cpus) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := util.AvoidCPUs(cpus, !configTest); err != nil {
|
||||||
|
if errors.Is(err, errors.ErrUnsupported) {
|
||||||
|
l.Warn("tun.avoid_cpus is only supported on Linux; ignoring")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("tun.avoid_cpus: %w", err)
|
||||||
|
}
|
||||||
|
if !configTest {
|
||||||
|
l.Info("Excluded CPUs from process threads", "cpus", cpus)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func moduleVersion() string {
|
func moduleVersion() string {
|
||||||
info, ok := debug.ReadBuildInfo()
|
info, ok := debug.ReadBuildInfo()
|
||||||
if !ok {
|
if !ok {
|
||||||
|
|||||||
@@ -59,6 +59,27 @@ func TestParseCpuAffinity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigureAvoidCPUs(t *testing.T) {
|
||||||
|
l := test.NewLogger()
|
||||||
|
for _, tc := range []struct {
|
||||||
|
value any
|
||||||
|
valid bool
|
||||||
|
}{
|
||||||
|
{nil, true}, {[]any{}, true}, {"auto", false}, {"1,2", false}, {[]any{-1}, false}, {[]any{1.5}, false},
|
||||||
|
{[]any{"2"}, false},
|
||||||
|
} {
|
||||||
|
c := config.NewC(l)
|
||||||
|
if tc.value != nil {
|
||||||
|
c.Settings["tun"] = map[string]any{"avoid_cpus": tc.value}
|
||||||
|
}
|
||||||
|
if err := configureAvoidCPUs(c, l, true); tc.valid {
|
||||||
|
require.NoError(t, err)
|
||||||
|
} else {
|
||||||
|
require.Error(t, err, "%v", tc.value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running
|
// TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running
|
||||||
// when it hands back a Control to stop them with.
|
// when it hands back a Control to stop them with.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
//go:build linux && !android && !e2e_testing
|
||||||
|
|
||||||
|
package util
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"unsafe"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AvoidCPUs removes cpus from the affinity of every thread of the process, for use at startup. New threads inherit
|
||||||
|
// their parent's mask, so it repeats until a pass finds nothing left to change; a thread that exits meanwhile is
|
||||||
|
// skipped. Every mask is checked before any is changed. With apply false it only validates, for configuration tests.
|
||||||
|
// This affects the entire process, including other users of an embedded Nebula.
|
||||||
|
func AvoidCPUs(cpus []int, apply bool) error {
|
||||||
|
const capacity = int(unsafe.Sizeof(unix.CPUSet{})) * 8
|
||||||
|
for _, cpu := range cpus {
|
||||||
|
if cpu < 0 || cpu >= capacity {
|
||||||
|
return fmt.Errorf("CPU %d is out of range", cpu)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(cpus) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
masks, err := avoidingMasks(cpus)
|
||||||
|
if err != nil || !apply || len(masks) == 0 {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for tid, mask := range masks {
|
||||||
|
if err := unix.SchedSetaffinity(tid, &mask); err != nil && !errors.Is(err, unix.ESRCH) {
|
||||||
|
return fmt.Errorf("set thread %d affinity: %w", tid, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// avoidingMasks returns the mask without cpus of every thread whose mask has any of them.
|
||||||
|
func avoidingMasks(cpus []int) (map[int]unix.CPUSet, error) {
|
||||||
|
entries, err := os.ReadDir("/proc/self/task")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
masks := make(map[int]unix.CPUSet, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
tid, err := strconv.Atoi(entry.Name())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var mask unix.CPUSet
|
||||||
|
err = unix.SchedGetaffinity(tid, &mask)
|
||||||
|
if errors.Is(err, unix.ESRCH) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read thread %d affinity: %w", tid, err)
|
||||||
|
}
|
||||||
|
next := mask
|
||||||
|
for _, cpu := range cpus {
|
||||||
|
next.Clear(cpu)
|
||||||
|
}
|
||||||
|
if next.Count() == 0 {
|
||||||
|
return nil, fmt.Errorf("avoid_cpus leaves thread %d with no allowed CPU", tid)
|
||||||
|
}
|
||||||
|
if next != mask {
|
||||||
|
masks[tid] = next
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return masks, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
//go:build linux && !android && !e2e_testing
|
||||||
|
|
||||||
|
package util
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"unsafe"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestAvoidCPUs changes the affinity of the whole process, so it runs in a child, leaving the test runner untouched.
|
||||||
|
func TestAvoidCPUs(t *testing.T) {
|
||||||
|
if os.Getenv("NEBULA_AVOID_CPU_TEST") != "1" {
|
||||||
|
exe, err := os.Executable()
|
||||||
|
require.NoError(t, err)
|
||||||
|
cmd := exec.Command(exe, "-test.run=^TestAvoidCPUs$", "-test.v")
|
||||||
|
cmd.Env = append(os.Environ(), "NEBULA_AVOID_CPU_TEST=1")
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
require.NoError(t, err, "%s", out)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
allowed, err := AllowedCPUs()
|
||||||
|
require.NoError(t, err)
|
||||||
|
if len(allowed) < 2 {
|
||||||
|
t.Skip("requires at least two allowed CPUs")
|
||||||
|
}
|
||||||
|
cpu := allowed[0]
|
||||||
|
release := make(chan struct{})
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
// lockThreads occupies 8 OS threads until release, so the runtime has to start more
|
||||||
|
lockThreads := func() {
|
||||||
|
var ready sync.WaitGroup
|
||||||
|
for range 8 {
|
||||||
|
ready.Add(1)
|
||||||
|
wg.Go(func() {
|
||||||
|
runtime.LockOSThread()
|
||||||
|
ready.Done()
|
||||||
|
<-release
|
||||||
|
})
|
||||||
|
}
|
||||||
|
ready.Wait()
|
||||||
|
}
|
||||||
|
lockThreads()
|
||||||
|
|
||||||
|
require.NoError(t, AvoidCPUs([]int{cpu}, false))
|
||||||
|
after, err := AllowedCPUs()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, allowed, after, "configuration check must not mutate affinity")
|
||||||
|
require.Error(t, AvoidCPUs(allowed, true), "must reject an empty mask before applying")
|
||||||
|
require.NoError(t, AvoidCPUs([]int{cpu, cpu}, true))
|
||||||
|
requireAvoided(t, cpu, allowed)
|
||||||
|
// Existing threads stay locked while new ones inherit the restriction
|
||||||
|
lockThreads()
|
||||||
|
requireAvoided(t, cpu, allowed)
|
||||||
|
close(release)
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
// requireAvoided checks no thread may run on cpu, and every thread still may on some CPU of allowed only.
|
||||||
|
func requireAvoided(t *testing.T, cpu int, allowed []int) {
|
||||||
|
t.Helper()
|
||||||
|
entries, err := os.ReadDir("/proc/self/task")
|
||||||
|
require.NoError(t, err)
|
||||||
|
for _, entry := range entries {
|
||||||
|
tid, err := strconv.Atoi(entry.Name())
|
||||||
|
require.NoError(t, err)
|
||||||
|
var mask unix.CPUSet
|
||||||
|
err = unix.SchedGetaffinity(tid, &mask)
|
||||||
|
if err == unix.ESRCH {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.False(t, mask.IsSet(cpu), "thread %d", tid)
|
||||||
|
require.Positive(t, mask.Count())
|
||||||
|
for id := range len(mask) * 64 {
|
||||||
|
if mask.IsSet(id) {
|
||||||
|
require.Contains(t, allowed, id, "must not widen the inherited mask")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAvoidCPUsRange checks the bounds of unix.CPUSet, whose words are 32 bits wide on 32-bit platforms.
|
||||||
|
func TestAvoidCPUsRange(t *testing.T) {
|
||||||
|
capacity := int(unsafe.Sizeof(unix.CPUSet{})) * 8
|
||||||
|
require.NoError(t, AvoidCPUs([]int{capacity - 1}, false))
|
||||||
|
require.EqualError(t, AvoidCPUs([]int{capacity}, false), fmt.Sprintf("CPU %d is out of range", capacity))
|
||||||
|
require.EqualError(t, AvoidCPUs([]int{-1}, false), "CPU -1 is out of range")
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
//go:build !linux || android || e2e_testing
|
||||||
|
|
||||||
|
package util
|
||||||
|
|
||||||
|
import "errors"
|
||||||
|
|
||||||
|
// AvoidCPUs is unsupported outside Linux: there is no thread affinity API to restrict.
|
||||||
|
func AvoidCPUs(_ []int, _ bool) error { return errors.ErrUnsupported }
|
||||||
Reference in new issue
Block a user