cpupick: Make tun.avoid_cpus a pin selection rule instead of a process-wide mask

cpupick/avoid_linux.go

@@ -15,8 +15,8 @@ import (
 // Avoid removes cpus from the affinity of every thread of the process, for use at startup. New threads inherit
 // their parent's mask, so it repeats until a pass finds nothing left to change; a thread that exits meanwhile is
 // skipped. Every mask is checked before any is changed. With apply false it only validates, for configuration tests.
-// This affects the entire process, including other users of an embedded Nebula. Run it before Default, which then
-// picks from what remains.
+// This affects the entire process, including other users of an embedded Nebula. Not wired up yet: tun.avoid_cpus
+// only steers Default for now.
 func Avoid(cpus []int, apply bool) error {
 	const capacity = int(unsafe.Sizeof(unix.CPUSet{})) * 8
 	for _, cpu := range cpus {

cpupick/cpupick.go

@@ -17,13 +17,14 @@
 // core demoted to last resort, and the order rotated by a stable per-instance
 // key so co-located instances spread instead of stacking.
 //
-// Avoid narrows the process-wide allowed set before any of that happens
-// (tun.avoid_cpus), so Default and tun.cpu_affinity validation only ever see
-// the CPUs left over.
+// CPUs the operator asked to keep readers off (tun.avoid_cpus, typically the
+// underlay NIC's receive IRQ CPUs) are removed from the allowed set before
+// any of that, so no rule can bring one back.
 package cpupick
 
 import (
 	"log/slog"
+	"slices"
 
 	"github.com/slackhq/nebula/util"
 )
@@ -61,12 +62,17 @@ func flatTopology(cpus []int) topology {
 // Default computes the pin order for `routines` tun readers. key is any
 // stable per-instance value; the bound UDP port is ideal — distinct across
 // co-located instances, stable across restarts so benchmark runs stay
-// comparable. Returns nil when there is nothing useful to say (no affinity
-// support on this platform, lookup failure); callers keep their existing
-// fallback spread.
-func Default(routines int, key uint64, l *slog.Logger) []int {
+// comparable. avoid lists CPUs never to return, matched literally. Returns
+// nil when there is nothing useful to say (no affinity support on this
+// platform, lookup failure, avoid covering every allowed CPU); callers keep
+// their existing fallback spread.
+func Default(routines int, key uint64, avoid []int, l *slog.Logger) []int {
 	allowed, err := util.AllowedCPUs()
-	if err != nil || len(allowed) == 0 {
+	if err != nil {
+		return nil
+	}
+	allowed = slices.DeleteFunc(allowed, func(c int) bool { return slices.Contains(avoid, c) })
+	if len(allowed) == 0 {
 		return nil
 	}
 	perf, signal := perfCPUs(allowed)
@@ -81,7 +87,8 @@ func Default(routines int, key uint64, l *slog.Logger) []int {
 	if l != nil {
 		l.Info("chose default pin CPUs for tun readers",
 			"cpus", cpus[:min(routines, len(cpus))],
-			"perfSignal", signal)
+			"perfSignal", signal,
+			"avoided", avoid)
 	}
 	return cpus
 }

cpupick/default_linux_test.go

@@ -0,0 +1,35 @@
+//go:build linux && !android && !e2e_testing
+
+package cpupick
+
+import (
+	"slices"
+	"testing"
+
+	"github.com/slackhq/nebula/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDefaultSkipsAvoided(t *testing.T) {
+	allowed, err := util.AllowedCPUs()
+	require.NoError(t, err)
+	if len(allowed) < 2 {
+		t.Skip("requires at least two allowed CPUs")
+	}
+	avoid := allowed[:len(allowed)/2]
+	for key := range uint64(16) {
+		got := Default(len(allowed), key, avoid, nil)
+		require.NotEmpty(t, got)
+		for _, cpu := range got {
+			assert.NotContains(t, avoid, cpu, "key %d", key)
+			assert.Contains(t, allowed, cpu, "key %d", key)
+		}
+	}
+	assert.Nil(t, Default(2, 0, allowed, nil), "avoiding everything leaves nothing to suggest")
+
+	// Without avoid every allowed CPU stays in play
+	got := Default(len(allowed), 0, nil, nil)
+	slices.Sort(got)
+	assert.Equal(t, allowed, got)
+}