mirror of
https://github.com/slackhq/nebula.git
synced 2026-08-15 11:37:02 +02:00
pin tun reader threads to CPUs so per-flow packets keep wire order
Each listenIn goroutine locks its OS thread and pins it to one CPU (sched_setaffinity), so every UDP send from that goroutine leaves through the same XPS-selected NIC TX ring instead of being sprayed across rings and reordered. On by default via tun.pin_threads; queue i pins to the i-th entry of the process's allowed CPU set (respecting cpuset/taskset masks, whose IDs are often not 0..NumCPU-1), or to an explicit tun.cpu_affinity list, validated against that same allowed set. Linux only; pinning is a no-op elsewhere. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
//go:build linux && !android && !e2e_testing
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// PinThreadToCPU restricts the calling OS thread to the given CPU via
|
||||
// sched_setaffinity(2). Combined with runtime.LockOSThread on the
|
||||
// goroutine, this prevents the kernel from migrating us across CPUs and
|
||||
// in turn keeps every UDP send from this goroutine going through the
|
||||
// same XPS-selected TX ring, eliminating the wire-side reorder that
|
||||
// otherwise fragments one nebula flow across multiple rings.
|
||||
func PinThreadToCPU(cpu int) error {
|
||||
runtime.LockOSThread()
|
||||
var set unix.CPUSet
|
||||
set.Zero()
|
||||
set.Set(cpu)
|
||||
return unix.SchedSetaffinity(0, &set)
|
||||
}
|
||||
|
||||
// AllowedCPUs returns the CPU IDs the calling process is currently allowed to
|
||||
// run on, as reported by sched_getaffinity(2). Under a cgroup cpuset or a
|
||||
// `taskset` mask the allowed IDs are frequently not the contiguous range
|
||||
// 0..NumCPU-1 (e.g. pinned to CPUs 4-7: NumCPU reports 4 while the valid IDs
|
||||
// are 4,5,6,7). Callers that need a real CPU to pin to must choose from this
|
||||
// set rather than assuming i % NumCPU is runnable, or every pin fails.
|
||||
func AllowedCPUs() ([]int, error) {
|
||||
var set unix.CPUSet
|
||||
if err := unix.SchedGetaffinity(0, &set); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cpus := make([]int, 0, set.Count())
|
||||
for cpu := 0; cpu < len(set)*64; cpu++ {
|
||||
if set.IsSet(cpu) {
|
||||
cpus = append(cpus, cpu)
|
||||
}
|
||||
}
|
||||
return cpus, nil
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//go:build !linux || android || e2e_testing
|
||||
|
||||
package util
|
||||
|
||||
// PinThreadToCPU is a no-op outside Linux: only Linux exposes a stable
|
||||
// per-thread CPU affinity API and only Linux has XPS-driven TX ring
|
||||
// selection in the first place. On every other platform there's nothing
|
||||
// to fix here.
|
||||
func PinThreadToCPU(_ int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// AllowedCPUs has no meaningful answer off Linux (no sched_getaffinity), so it
|
||||
// reports "unknown" by returning a nil slice and nil error. Callers treat an
|
||||
// empty result as "fall back to the default CPU choice".
|
||||
func AllowedCPUs() ([]int, error) {
|
||||
return nil, nil
|
||||
}
|
||||
Reference in New Issue
Block a user