mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-15 13:06:52 +01:00
Ryan Huber
73d26dd87d
Set tun.avoid_cpus to a list of logical CPUs to remove from the affinity of every thread of the process on Linux, before services start and reader CPUs are chosen. Later threads inherit the restricted mask; enumeration repeats so threads created meanwhile are covered. Inherited taskset and cgroup restrictions are kept. Keeping Nebula off the underlay NIC's receive IRQ CPUs and their SMT siblings is what this is for; the example configuration shows how to find them. Invalid IDs, a list that would leave a thread no CPU, and syscall failures fail startup. Configuration tests only validate. Other platforms log and ignore the setting. The exclusion is process-wide, including embedded use, and not reloadable. It is independent of thread pinning. New API: util.AvoidCPUs, next to the existing PinThreadToCPU and AllowedCPUs. Tests run in a child process and check existing and new threads, preserved inherited masks, validation without changes, and the unix.CPUSet bounds on 32- and 64-bit platforms. No isolated performance claim is made.
75 lines
1.9 KiB
Go
75 lines
1.9 KiB
Go
//go:build linux && !android && !e2e_testing
|
|
|
|
package util
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"unsafe"
|
|
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
// AvoidCPUs removes cpus from the affinity of every thread of the process, for use at startup. New threads inherit
|
|
// their parent's mask, so it repeats until a pass finds nothing left to change; a thread that exits meanwhile is
|
|
// skipped. Every mask is checked before any is changed. With apply false it only validates, for configuration tests.
|
|
// This affects the entire process, including other users of an embedded Nebula.
|
|
func AvoidCPUs(cpus []int, apply bool) error {
|
|
const capacity = int(unsafe.Sizeof(unix.CPUSet{})) * 8
|
|
for _, cpu := range cpus {
|
|
if cpu < 0 || cpu >= capacity {
|
|
return fmt.Errorf("CPU %d is out of range", cpu)
|
|
}
|
|
}
|
|
if len(cpus) == 0 {
|
|
return nil
|
|
}
|
|
for {
|
|
masks, err := avoidingMasks(cpus)
|
|
if err != nil || !apply || len(masks) == 0 {
|
|
return err
|
|
}
|
|
for tid, mask := range masks {
|
|
if err := unix.SchedSetaffinity(tid, &mask); err != nil && !errors.Is(err, unix.ESRCH) {
|
|
return fmt.Errorf("set thread %d affinity: %w", tid, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// avoidingMasks returns the mask without cpus of every thread whose mask has any of them.
|
|
func avoidingMasks(cpus []int) (map[int]unix.CPUSet, error) {
|
|
entries, err := os.ReadDir("/proc/self/task")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
masks := make(map[int]unix.CPUSet, len(entries))
|
|
for _, entry := range entries {
|
|
tid, err := strconv.Atoi(entry.Name())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var mask unix.CPUSet
|
|
err = unix.SchedGetaffinity(tid, &mask)
|
|
if errors.Is(err, unix.ESRCH) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read thread %d affinity: %w", tid, err)
|
|
}
|
|
next := mask
|
|
for _, cpu := range cpus {
|
|
next.Clear(cpu)
|
|
}
|
|
if next.Count() == 0 {
|
|
return nil, fmt.Errorf("avoid_cpus leaves thread %d with no allowed CPU", tid)
|
|
}
|
|
if next != mask {
|
|
masks[tid] = next
|
|
}
|
|
}
|
|
return masks, nil
|
|
}
|