tun: Allow excluding CPUs from process threads at startup

Set tun.avoid_cpus to a list of logical CPUs to remove from the affinity of
every thread of the process on Linux, before services start and reader CPUs
are chosen. Later threads inherit the restricted mask; enumeration repeats so
threads created meanwhile are covered. Inherited taskset and cgroup
restrictions are kept. Keeping Nebula off the underlay NIC's receive IRQ CPUs
and their SMT siblings is what this is for; the example configuration shows
how to find them.

Invalid IDs, a list that would leave a thread no CPU, and syscall failures
fail startup. Configuration tests only validate. Other platforms log and
ignore the setting. The exclusion is process-wide, including embedded use, and
not reloadable. It is independent of thread pinning.

New API: util.AvoidCPUs, next to the existing PinThreadToCPU and AllowedCPUs.

Tests run in a child process and check existing and new threads, preserved
inherited masks, validation without changes, and the unix.CPUSet bounds on
32- and 64-bit platforms. No isolated performance claim is made.

util/cpuavoid_linux.go

@@ -0,0 +1,74 @@
+//go:build linux && !android && !e2e_testing
+
+package util
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+// AvoidCPUs removes cpus from the affinity of every thread of the process, for use at startup. New threads inherit
+// their parent's mask, so it repeats until a pass finds nothing left to change; a thread that exits meanwhile is
+// skipped. Every mask is checked before any is changed. With apply false it only validates, for configuration tests.
+// This affects the entire process, including other users of an embedded Nebula.
+func AvoidCPUs(cpus []int, apply bool) error {
+	const capacity = int(unsafe.Sizeof(unix.CPUSet{})) * 8
+	for _, cpu := range cpus {
+		if cpu < 0 || cpu >= capacity {
+			return fmt.Errorf("CPU %d is out of range", cpu)
+		}
+	}
+	if len(cpus) == 0 {
+		return nil
+	}
+	for {
+		masks, err := avoidingMasks(cpus)
+		if err != nil || !apply || len(masks) == 0 {
+			return err
+		}
+		for tid, mask := range masks {
+			if err := unix.SchedSetaffinity(tid, &mask); err != nil && !errors.Is(err, unix.ESRCH) {
+				return fmt.Errorf("set thread %d affinity: %w", tid, err)
+			}
+		}
+	}
+}
+
+// avoidingMasks returns the mask without cpus of every thread whose mask has any of them.
+func avoidingMasks(cpus []int) (map[int]unix.CPUSet, error) {
+	entries, err := os.ReadDir("/proc/self/task")
+	if err != nil {
+		return nil, err
+	}
+	masks := make(map[int]unix.CPUSet, len(entries))
+	for _, entry := range entries {
+		tid, err := strconv.Atoi(entry.Name())
+		if err != nil {
+			return nil, err
+		}
+		var mask unix.CPUSet
+		err = unix.SchedGetaffinity(tid, &mask)
+		if errors.Is(err, unix.ESRCH) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read thread %d affinity: %w", tid, err)
+		}
+		next := mask
+		for _, cpu := range cpus {
+			next.Clear(cpu)
+		}
+		if next.Count() == 0 {
+			return nil, fmt.Errorf("avoid_cpus leaves thread %d with no allowed CPU", tid)
+		}
+		if next != mask {
+			masks[tid] = next
+		}
+	}
+	return masks, nil
+}

util/cpuavoid_linux_test.go

@@ -0,0 +1,97 @@
+//go:build linux && !android && !e2e_testing
+
+package util
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"sync"
+	"testing"
+	"unsafe"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// TestAvoidCPUs changes the affinity of the whole process, so it runs in a child, leaving the test runner untouched.
+func TestAvoidCPUs(t *testing.T) {
+	if os.Getenv("NEBULA_AVOID_CPU_TEST") != "1" {
+		exe, err := os.Executable()
+		require.NoError(t, err)
+		cmd := exec.Command(exe, "-test.run=^TestAvoidCPUs$", "-test.v")
+		cmd.Env = append(os.Environ(), "NEBULA_AVOID_CPU_TEST=1")
+		out, err := cmd.CombinedOutput()
+		require.NoError(t, err, "%s", out)
+		return
+	}
+	allowed, err := AllowedCPUs()
+	require.NoError(t, err)
+	if len(allowed) < 2 {
+		t.Skip("requires at least two allowed CPUs")
+	}
+	cpu := allowed[0]
+	release := make(chan struct{})
+	var wg sync.WaitGroup
+	// lockThreads occupies 8 OS threads until release, so the runtime has to start more
+	lockThreads := func() {
+		var ready sync.WaitGroup
+		for range 8 {
+			ready.Add(1)
+			wg.Go(func() {
+				runtime.LockOSThread()
+				ready.Done()
+				<-release
+			})
+		}
+		ready.Wait()
+	}
+	lockThreads()
+
+	require.NoError(t, AvoidCPUs([]int{cpu}, false))
+	after, err := AllowedCPUs()
+	require.NoError(t, err)
+	require.Equal(t, allowed, after, "configuration check must not mutate affinity")
+	require.Error(t, AvoidCPUs(allowed, true), "must reject an empty mask before applying")
+	require.NoError(t, AvoidCPUs([]int{cpu, cpu}, true))
+	requireAvoided(t, cpu, allowed)
+	// Existing threads stay locked while new ones inherit the restriction
+	lockThreads()
+	requireAvoided(t, cpu, allowed)
+	close(release)
+	wg.Wait()
+}
+
+// requireAvoided checks no thread may run on cpu, and every thread still may on some CPU of allowed only.
+func requireAvoided(t *testing.T, cpu int, allowed []int) {
+	t.Helper()
+	entries, err := os.ReadDir("/proc/self/task")
+	require.NoError(t, err)
+	for _, entry := range entries {
+		tid, err := strconv.Atoi(entry.Name())
+		require.NoError(t, err)
+		var mask unix.CPUSet
+		err = unix.SchedGetaffinity(tid, &mask)
+		if err == unix.ESRCH {
+			continue
+		}
+		require.NoError(t, err)
+		require.False(t, mask.IsSet(cpu), "thread %d", tid)
+		require.Positive(t, mask.Count())
+		for id := range len(mask) * 64 {
+			if mask.IsSet(id) {
+				require.Contains(t, allowed, id, "must not widen the inherited mask")
+			}
+		}
+	}
+}
+
+// TestAvoidCPUsRange checks the bounds of unix.CPUSet, whose words are 32 bits wide on 32-bit platforms.
+func TestAvoidCPUsRange(t *testing.T) {
+	capacity := int(unsafe.Sizeof(unix.CPUSet{})) * 8
+	require.NoError(t, AvoidCPUs([]int{capacity - 1}, false))
+	require.EqualError(t, AvoidCPUs([]int{capacity}, false), fmt.Sprintf("CPU %d is out of range", capacity))
+	require.EqualError(t, AvoidCPUs([]int{-1}, false), "CPU -1 is out of range")
+}

util/cpuavoid_other.go

@@ -0,0 +1,8 @@
+//go:build !linux || android || e2e_testing
+
+package util
+
+import "errors"
+
+// AvoidCPUs is unsupported outside Linux: there is no thread affinity API to restrict.
+func AvoidCPUs(_ []int, _ bool) error { return errors.ErrUnsupported }