tun: Allow excluding CPUs from process threads at startup

Set tun.avoid_cpus to a list of logical CPUs to remove from the affinity of
every thread of the process on Linux, before services start and reader CPUs
are chosen. Later threads inherit the restricted mask; enumeration repeats so
threads created meanwhile are covered. Inherited taskset and cgroup
restrictions are kept. Keeping Nebula off the underlay NIC's receive IRQ CPUs
and their SMT siblings is what this is for; the example configuration shows
how to find them.

Invalid IDs, a list that would leave a thread no CPU, and syscall failures
fail startup. Configuration tests only validate. Other platforms log and
ignore the setting. The exclusion is process-wide, including embedded use, and
not reloadable. It is independent of thread pinning.

New API: util.AvoidCPUs, next to the existing PinThreadToCPU and AllowedCPUs.

Tests run in a child process and check existing and new threads, preserved
inherited masks, validation without changes, and the unix.CPUSet bounds on
32- and 64-bit platforms. No isolated performance claim is made.

util/cpuavoid_linux_test.go

@@ -0,0 +1,97 @@
+//go:build linux && !android && !e2e_testing
+
+package util
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"sync"
+	"testing"
+	"unsafe"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// TestAvoidCPUs changes the affinity of the whole process, so it runs in a child, leaving the test runner untouched.
+func TestAvoidCPUs(t *testing.T) {
+	if os.Getenv("NEBULA_AVOID_CPU_TEST") != "1" {
+		exe, err := os.Executable()
+		require.NoError(t, err)
+		cmd := exec.Command(exe, "-test.run=^TestAvoidCPUs$", "-test.v")
+		cmd.Env = append(os.Environ(), "NEBULA_AVOID_CPU_TEST=1")
+		out, err := cmd.CombinedOutput()
+		require.NoError(t, err, "%s", out)
+		return
+	}
+	allowed, err := AllowedCPUs()
+	require.NoError(t, err)
+	if len(allowed) < 2 {
+		t.Skip("requires at least two allowed CPUs")
+	}
+	cpu := allowed[0]
+	release := make(chan struct{})
+	var wg sync.WaitGroup
+	// lockThreads occupies 8 OS threads until release, so the runtime has to start more
+	lockThreads := func() {
+		var ready sync.WaitGroup
+		for range 8 {
+			ready.Add(1)
+			wg.Go(func() {
+				runtime.LockOSThread()
+				ready.Done()
+				<-release
+			})
+		}
+		ready.Wait()
+	}
+	lockThreads()
+
+	require.NoError(t, AvoidCPUs([]int{cpu}, false))
+	after, err := AllowedCPUs()
+	require.NoError(t, err)
+	require.Equal(t, allowed, after, "configuration check must not mutate affinity")
+	require.Error(t, AvoidCPUs(allowed, true), "must reject an empty mask before applying")
+	require.NoError(t, AvoidCPUs([]int{cpu, cpu}, true))
+	requireAvoided(t, cpu, allowed)
+	// Existing threads stay locked while new ones inherit the restriction
+	lockThreads()
+	requireAvoided(t, cpu, allowed)
+	close(release)
+	wg.Wait()
+}
+
+// requireAvoided checks no thread may run on cpu, and every thread still may on some CPU of allowed only.
+func requireAvoided(t *testing.T, cpu int, allowed []int) {
+	t.Helper()
+	entries, err := os.ReadDir("/proc/self/task")
+	require.NoError(t, err)
+	for _, entry := range entries {
+		tid, err := strconv.Atoi(entry.Name())
+		require.NoError(t, err)
+		var mask unix.CPUSet
+		err = unix.SchedGetaffinity(tid, &mask)
+		if err == unix.ESRCH {
+			continue
+		}
+		require.NoError(t, err)
+		require.False(t, mask.IsSet(cpu), "thread %d", tid)
+		require.Positive(t, mask.Count())
+		for id := range len(mask) * 64 {
+			if mask.IsSet(id) {
+				require.Contains(t, allowed, id, "must not widen the inherited mask")
+			}
+		}
+	}
+}
+
+// TestAvoidCPUsRange checks the bounds of unix.CPUSet, whose words are 32 bits wide on 32-bit platforms.
+func TestAvoidCPUsRange(t *testing.T) {
+	capacity := int(unsafe.Sizeof(unix.CPUSet{})) * 8
+	require.NoError(t, AvoidCPUs([]int{capacity - 1}, false))
+	require.EqualError(t, AvoidCPUs([]int{capacity}, false), fmt.Sprintf("CPU %d is out of range", capacity))
+	require.EqualError(t, AvoidCPUs([]int{-1}, false), "CPU -1 is out of range")
+}