diff --git a/util/cpuavoid_linux.go b/cpupick/avoid_linux.go similarity index 87% rename from util/cpuavoid_linux.go rename to cpupick/avoid_linux.go index 989e49d3..75919b4d 100644 --- a/util/cpuavoid_linux.go +++ b/cpupick/avoid_linux.go @@ -1,6 +1,6 @@ //go:build linux && !android && !e2e_testing -package util +package cpupick import ( "errors" @@ -12,11 +12,12 @@ import ( "golang.org/x/sys/unix" ) -// AvoidCPUs removes cpus from the affinity of every thread of the process, for use at startup. New threads inherit +// Avoid removes cpus from the affinity of every thread of the process, for use at startup. New threads inherit // their parent's mask, so it repeats until a pass finds nothing left to change; a thread that exits meanwhile is // skipped. Every mask is checked before any is changed. With apply false it only validates, for configuration tests. -// This affects the entire process, including other users of an embedded Nebula. -func AvoidCPUs(cpus []int, apply bool) error { +// This affects the entire process, including other users of an embedded Nebula. Run it before Default, which then +// picks from what remains. +func Avoid(cpus []int, apply bool) error { const capacity = int(unsafe.Sizeof(unix.CPUSet{})) * 8 for _, cpu := range cpus { if cpu < 0 || cpu >= capacity { diff --git a/util/cpuavoid_linux_test.go b/cpupick/avoid_linux_test.go similarity index 65% rename from util/cpuavoid_linux_test.go rename to cpupick/avoid_linux_test.go index 0bed9ad6..b6f22513 100644 --- a/util/cpuavoid_linux_test.go +++ b/cpupick/avoid_linux_test.go @@ -1,6 +1,6 @@ //go:build linux && !android && !e2e_testing -package util +package cpupick import ( "fmt" @@ -12,22 +12,23 @@ import ( "testing" "unsafe" + "github.com/slackhq/nebula/util" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" ) -// TestAvoidCPUs changes the affinity of the whole process, so it runs in a child, leaving the test runner untouched. -func TestAvoidCPUs(t *testing.T) { - if os.Getenv("NEBULA_AVOID_CPU_TEST") != "1" { +// TestAvoid changes the affinity of the whole process, so it runs in a child, leaving the test runner untouched. +func TestAvoid(t *testing.T) { + if os.Getenv("NEBULA_CPUPICK_AVOID_TEST") != "1" { exe, err := os.Executable() require.NoError(t, err) - cmd := exec.Command(exe, "-test.run=^TestAvoidCPUs$", "-test.v") - cmd.Env = append(os.Environ(), "NEBULA_AVOID_CPU_TEST=1") + cmd := exec.Command(exe, "-test.run=^TestAvoid$", "-test.v") + cmd.Env = append(os.Environ(), "NEBULA_CPUPICK_AVOID_TEST=1") out, err := cmd.CombinedOutput() require.NoError(t, err, "%s", out) return } - allowed, err := AllowedCPUs() + allowed, err := util.AllowedCPUs() require.NoError(t, err) if len(allowed) < 2 { t.Skip("requires at least two allowed CPUs") @@ -50,12 +51,12 @@ func TestAvoidCPUs(t *testing.T) { } lockThreads() - require.NoError(t, AvoidCPUs([]int{cpu}, false)) - after, err := AllowedCPUs() + require.NoError(t, Avoid([]int{cpu}, false)) + after, err := util.AllowedCPUs() require.NoError(t, err) require.Equal(t, allowed, after, "configuration check must not mutate affinity") - require.Error(t, AvoidCPUs(allowed, true), "must reject an empty mask before applying") - require.NoError(t, AvoidCPUs([]int{cpu, cpu}, true)) + require.Error(t, Avoid(allowed, true), "must reject an empty mask before applying") + require.NoError(t, Avoid([]int{cpu, cpu}, true)) requireAvoided(t, cpu, allowed) // Existing threads stay locked while new ones inherit the restriction lockThreads() @@ -88,10 +89,10 @@ func requireAvoided(t *testing.T, cpu int, allowed []int) { } } -// TestAvoidCPUsRange checks the bounds of unix.CPUSet, whose words are 32 bits wide on 32-bit platforms. -func TestAvoidCPUsRange(t *testing.T) { +// TestAvoidRange checks the bounds of unix.CPUSet, whose words are 32 bits wide on 32-bit platforms. +func TestAvoidRange(t *testing.T) { capacity := int(unsafe.Sizeof(unix.CPUSet{})) * 8 - require.NoError(t, AvoidCPUs([]int{capacity - 1}, false)) - require.EqualError(t, AvoidCPUs([]int{capacity}, false), fmt.Sprintf("CPU %d is out of range", capacity)) - require.EqualError(t, AvoidCPUs([]int{-1}, false), "CPU -1 is out of range") + require.NoError(t, Avoid([]int{capacity - 1}, false)) + require.EqualError(t, Avoid([]int{capacity}, false), fmt.Sprintf("CPU %d is out of range", capacity)) + require.EqualError(t, Avoid([]int{-1}, false), "CPU -1 is out of range") } diff --git a/cpupick/avoid_other.go b/cpupick/avoid_other.go new file mode 100644 index 00000000..9ee9f187 --- /dev/null +++ b/cpupick/avoid_other.go @@ -0,0 +1,8 @@ +//go:build !linux || android || e2e_testing + +package cpupick + +import "errors" + +// Avoid is unsupported outside Linux: there is no thread affinity API to restrict. +func Avoid(_ []int, _ bool) error { return errors.ErrUnsupported } diff --git a/cpupick/cpupick.go b/cpupick/cpupick.go index 49f791e8..0fd340f1 100644 --- a/cpupick/cpupick.go +++ b/cpupick/cpupick.go @@ -16,6 +16,10 @@ // across distinct physical cores when the topology permits, CPU 0's physical // core demoted to last resort, and the order rotated by a stable per-instance // key so co-located instances spread instead of stacking. +// +// Avoid narrows the process-wide allowed set before any of that happens +// (tun.avoid_cpus), so Default and tun.cpu_affinity validation only ever see +// the CPUs left over. package cpupick import ( diff --git a/main.go b/main.go index f8170c00..2a4c5ed5 100644 --- a/main.go +++ b/main.go @@ -443,7 +443,7 @@ func configureAvoidCPUs(c *config.C, l *slog.Logger, configTest bool) error { if len(cpus) == 0 { return nil } - if err := util.AvoidCPUs(cpus, !configTest); err != nil { + if err := cpupick.Avoid(cpus, !configTest); err != nil { if errors.Is(err, errors.ErrUnsupported) { l.Warn("tun.avoid_cpus is only supported on Linux; ignoring") return nil diff --git a/util/cpuavoid_other.go b/util/cpuavoid_other.go deleted file mode 100644 index 5b473fac..00000000 --- a/util/cpuavoid_other.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build !linux || android || e2e_testing - -package util - -import "errors" - -// AvoidCPUs is unsupported outside Linux: there is no thread affinity API to restrict. -func AvoidCPUs(_ []int, _ bool) error { return errors.ErrUnsupported }