pin tun reader threads to CPUs so per-flow packets keep wire order

Each listenIn goroutine locks its OS thread and pins it to one CPU
(sched_setaffinity), so every UDP send from that goroutine leaves
through the same XPS-selected NIC TX ring instead of being sprayed
across rings and reordered. On by default via tun.pin_threads; queue i
pins to the i-th entry of the process's allowed CPU set (respecting
cpuset/taskset masks, whose IDs are often not 0..NumCPU-1), or to an
explicit tun.cpu_affinity list, validated against that same allowed
set. Linux only; pinning is a no-op elsewhere.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
JackDoan
2026-07-16 12:25:23 -05:00
parent 913a37cfee
commit 10e9514e44
6 changed files with 242 additions and 2 deletions
+43
View File
@@ -0,0 +1,43 @@
//go:build linux && !android && !e2e_testing
package util
import (
"runtime"
"golang.org/x/sys/unix"
)
// PinThreadToCPU restricts the calling OS thread to the given CPU via
// sched_setaffinity(2). Combined with runtime.LockOSThread on the
// goroutine, this prevents the kernel from migrating us across CPUs and
// in turn keeps every UDP send from this goroutine going through the
// same XPS-selected TX ring, eliminating the wire-side reorder that
// otherwise fragments one nebula flow across multiple rings.
func PinThreadToCPU(cpu int) error {
runtime.LockOSThread()
var set unix.CPUSet
set.Zero()
set.Set(cpu)
return unix.SchedSetaffinity(0, &set)
}
// AllowedCPUs returns the CPU IDs the calling process is currently allowed to
// run on, as reported by sched_getaffinity(2). Under a cgroup cpuset or a
// `taskset` mask the allowed IDs are frequently not the contiguous range
// 0..NumCPU-1 (e.g. pinned to CPUs 4-7: NumCPU reports 4 while the valid IDs
// are 4,5,6,7). Callers that need a real CPU to pin to must choose from this
// set rather than assuming i % NumCPU is runnable, or every pin fails.
func AllowedCPUs() ([]int, error) {
var set unix.CPUSet
if err := unix.SchedGetaffinity(0, &set); err != nil {
return nil, err
}
cpus := make([]int, 0, set.Count())
for cpu := 0; cpu < len(set)*64; cpu++ {
if set.IsSet(cpu) {
cpus = append(cpus, cpu)
}
}
return cpus, nil
}
+18
View File
@@ -0,0 +1,18 @@
//go:build !linux || android || e2e_testing
package util
// PinThreadToCPU is a no-op outside Linux: only Linux exposes a stable
// per-thread CPU affinity API and only Linux has XPS-driven TX ring
// selection in the first place. On every other platform there's nothing
// to fix here.
func PinThreadToCPU(_ int) error {
return nil
}
// AllowedCPUs has no meaningful answer off Linux (no sched_getaffinity), so it
// reports "unknown" by returning a nil slice and nil error. Callers treat an
// empty result as "fall back to the default CPU choice".
func AllowedCPUs() ([]int, error) {
return nil, nil
}