tun: default pin CPUs avoid NIC IRQ cores

When tun.pin_threads is on and tun.cpu_affinity is unset, pick pin CPUs
from the allowed set that do not service any up physical NIC's MSI vectors
(read-only walk of /sys/class/net/*/device/msi_irqs and
/proc/irq/*/effective_affinity_list). The old allowed[i] default pinned
encrypt threads onto exactly the cores drivers affine their first RX queue
IRQs to; a flow whose RSS queue fired there had NAPI fighting encrypt for
the core (measured 8.4 vs 10.2 Gbps REV bimodality). Falls back to the old
spread, with a log, when there aren't enough IRQ-free CPUs - e.g. drivers
that allocate one queue per core until the admin narrows them (ethtool -L).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
JackDoan
2026-07-14 15:14:40 -05:00
parent 37b924945d
commit 7fe4fab167
6 changed files with 448 additions and 3 deletions
+9 -1
View File
@@ -257,13 +257,21 @@ tun:
# Linux only. pin_threads pins each tun reader/encrypt OS thread to a single CPU. This keeps every goroutine's
# batched sends flowing through one XPS-selected NIC TX ring, so packets within a flow stay ordered on the wire
# instead of being sprayed across multiple TX rings and reordered. Not reloadable.
#
# When cpu_affinity is unset, nebula picks CPUs that do NOT service any physical NIC's interrupts (read from
# /sys/class/net/*/device/msi_irqs and /proc/irq/*/effective_affinity_list): an encrypt thread pinned onto a core
# that also runs NAPI for a NIC RX queue fights the softirq for the core and collapses throughput for flows hashed
# to that queue. If the NIC's vectors blanket every allowed CPU (many drivers default to one queue per core) the
# avoidance logs and falls back to the old spread; narrow the NIC's queue/IRQ spread (e.g. `ethtool -X <dev>
# equal N`) or set cpu_affinity explicitly to benefit.
#pin_threads: true
# Linux only. cpu_affinity overrides which CPUs the tun reader threads pin to: a list of CPU IDs, one per routine
# (see the top-level `routines` setting). Lists shorter than `routines` are modulo-cycled across the queues; extra
# entries are ignored. IDs must be within the process's allowed CPU set, so this respects taskset / cgroup cpusets;
# a non-integer or not-allowed entry disables the override and falls back to spreading queues across the allowed
# CPUs. Only meaningful while pin_threads is true. Not reloadable.
# CPUs. Setting this disables the automatic NIC-IRQ avoidance described under pin_threads — prefer CPUs that don't
# service your underlay NIC's RX queue IRQs. Only meaningful while pin_threads is true. Not reloadable.
#cpu_affinity:
# - 2
# - 4
+59 -2
View File
@@ -214,6 +214,12 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
l.Warn("Failed to start DNS responder", "error", err)
}
pinThreads := c.GetBool("tun.pin_threads", true)
cpuAffinity := parseCpuAffinity(c, l, routines)
if pinThreads && len(cpuAffinity) == 0 && !configTest {
cpuAffinity = defaultCPUAffinityAvoidingIRQs(l, routines)
}
ifConfig := &InterfaceConfig{
HostMap: hostMap,
Inside: tun,
@@ -235,8 +241,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
relayManager: NewRelayManager(ctx, l, hostMap, c),
punchy: punchy,
ConntrackCacheTimeout: conntrackCacheTimeout,
CpuAffinity: parseCpuAffinity(c, l, routines),
PinThreads: c.GetBool("tun.pin_threads", true),
CpuAffinity: cpuAffinity,
PinThreads: pinThreads,
l: l,
}
@@ -353,6 +359,57 @@ func parseCpuAffinity(c *config.C, l *slog.Logger, routines int) []int {
return cpus
}
// defaultCPUAffinityAvoidingIRQs picks the default pin set for the tun
// readers when tun.cpu_affinity is unset: allowed CPUs that do NOT service
// any physical NIC's interrupts. The stock allowed[i] spread pins the
// encrypt threads onto exactly the cores most drivers affine their first RX
// queue IRQs to, so whenever a flow's RSS queue fires on a core hosting a
// tun reader, NAPI and encrypt fight for the core and per-flow throughput
// drops (measured: REV 8.4 vs 10.2 Gbps on the same hardware, 2026-07-14).
//
// Returns nil — keeping the old allowed[i] fallback in listenIn — when IRQ
// info is unavailable or when there aren't enough IRQ-free CPUs to give
// every routine its own core: silently doubling readers up on fewer cores
// is worse than the occasional IRQ collision. NICs whose vectors blanket
// every CPU (e.g. mlx5 defaults to one queue per core) make avoidance
// impossible; narrowing the NIC's spread (ethtool -X <dev> equal N, or
// /proc/irq/*/smp_affinity) or setting tun.cpu_affinity explicitly makes it
// effective.
func defaultCPUAffinityAvoidingIRQs(l *slog.Logger, routines int) []int {
irq, err := util.NICIRQCPUs()
if err != nil || len(irq) == 0 {
return nil
}
allowed, err := util.AllowedCPUs()
if err != nil {
return nil
}
cpus := chooseIRQFreeCPUs(allowed, irq, routines)
if cpus == nil {
l.Info("not enough CPUs are free of NIC IRQs to give every tun reader its own; using the default spread",
"routines", routines, "allowed", len(allowed), "irqCPUs", len(irq))
return nil
}
l.Info("pinning tun readers to CPUs clear of NIC IRQs", "cpus", cpus)
return cpus
}
// chooseIRQFreeCPUs returns the first `routines` allowed CPUs not present in
// irq, or nil if fewer than `routines` qualify.
func chooseIRQFreeCPUs(allowed []int, irq map[int]bool, routines int) []int {
free := make([]int, 0, routines)
for _, cpu := range allowed {
if irq[cpu] {
continue
}
free = append(free, cpu)
if len(free) == routines {
return free
}
}
return nil
}
func moduleVersion() string {
info, ok := debug.ReadBuildInfo()
if !ok {
+20
View File
@@ -9,6 +9,26 @@ import (
"github.com/stretchr/testify/assert"
)
func TestChooseIRQFreeCPUs(t *testing.T) {
irq := map[int]bool{0: true, 1: true, 2: true, 3: true}
// Plenty of IRQ-free CPUs: take the first `routines` of them in order.
assert.Equal(t, []int{4, 5}, chooseIRQFreeCPUs([]int{0, 1, 2, 3, 4, 5, 6}, irq, 2))
// Exactly enough.
assert.Equal(t, []int{4, 5, 6}, chooseIRQFreeCPUs([]int{0, 1, 2, 3, 4, 5, 6}, irq, 3))
// Not enough IRQ-free CPUs: nil, caller keeps the old default rather
// than doubling readers up on shared cores.
assert.Nil(t, chooseIRQFreeCPUs([]int{0, 1, 2, 3, 4}, irq, 2))
// No IRQ info at all behaves like a plain prefix of allowed.
assert.Equal(t, []int{0, 1}, chooseIRQFreeCPUs([]int{0, 1, 2}, map[int]bool{}, 2))
// Non-contiguous allowed set (cgroup cpuset) with holes.
assert.Equal(t, []int{9, 12}, chooseIRQFreeCPUs([]int{1, 3, 9, 12}, map[int]bool{1: true, 3: true}, 2))
}
func TestParseCpuAffinity(t *testing.T) {
l := test.NewLogger()
+198
View File
@@ -0,0 +1,198 @@
//go:build linux && !android && !e2e_testing
package util
import (
"os"
"path/filepath"
"strconv"
"strings"
)
// NICIRQCPUs returns the set of CPUs that service interrupts for the ACTIVE
// RX/TX queues of physical network interfaces that are up. Read-only: it
// matches /proc/interrupts action names against each NIC's PCI address and
// interface name, drops vectors whose queue index is beyond the device's
// active queue count (drivers like mlx5 keep handlers registered for
// deactivated queues, so /proc/interrupts alone over-reports), and unions
// /proc/irq/<n>/effective_affinity_list for the survivors.
//
// Callers use this to keep busy pinned threads OFF those CPUs: a thread
// pinned onto a core that also runs NAPI for a NIC RX queue competes with
// softirq processing for the core and measurably collapses throughput for
// flows hashed to that queue.
func NICIRQCPUs() (map[int]bool, error) {
return nicIRQCPUs("/sys/class/net", "/proc/irq", "/proc/interrupts")
}
// irqAction is one row of /proc/interrupts: the IRQ number and the action
// (handler) name in its final column, e.g. "mlx5_comp3@pci:0000:82:00.0".
type irqAction struct {
irq string
action string
}
func nicIRQCPUs(netDir, irqDir, interruptsPath string) (map[int]bool, error) {
actions, err := parseInterrupts(interruptsPath)
if err != nil {
return nil, err
}
devs, err := os.ReadDir(netDir)
if err != nil {
return nil, err
}
cpus := make(map[int]bool)
for _, dev := range devs {
devPath := filepath.Join(netDir, dev.Name())
pciDev, err := filepath.EvalSymlinks(filepath.Join(devPath, "device"))
if err != nil {
continue // virtual device (lo, tun, bridge, vlan, ...)
}
pciAddr := filepath.Base(pciDev)
state, err := os.ReadFile(filepath.Join(devPath, "operstate"))
if err != nil || strings.TrimSpace(string(state)) != "up" {
continue // a down NIC's queue IRQs don't fire
}
nq := countQueues(filepath.Join(devPath, "queues"))
for _, ia := range actions {
if !strings.Contains(ia.action, pciAddr) && !containsWord(ia.action, dev.Name()) {
continue
}
// Vector naming puts the queue index at the end of the handler
// name (mlx5_comp3@pci:..., ice-eth0-TxRx-3, virtio0-input.3).
// An index at or beyond the active queue count is a handler for
// a deactivated queue: registered, but it will not fire.
name, _, _ := strings.Cut(ia.action, "@")
if idx, ok := trailingInt(name); ok && idx >= nq {
continue
}
for _, cpu := range irqAffinity(irqDir, ia.irq) {
cpus[cpu] = true
}
}
}
return cpus, nil
}
// parseInterrupts extracts (irq, action) pairs from /proc/interrupts,
// skipping the header and the non-numeric summary rows (NMI, LOC, ...).
func parseInterrupts(path string) ([]irqAction, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var out []irqAction
for line := range strings.SplitSeq(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
irq, ok := strings.CutSuffix(fields[0], ":")
if !ok {
continue
}
if _, err := strconv.Atoi(irq); err != nil {
continue
}
out = append(out, irqAction{irq: irq, action: fields[len(fields)-1]})
}
return out, nil
}
// irqAffinity returns the CPUs IRQ n actually targets.
// effective_affinity_list is the vector's real target; smp_affinity_list
// (the fallback for kernels without effective affinity reporting) is the
// admin-allowed mask and may be wider.
func irqAffinity(irqDir, irq string) []int {
irqPath := filepath.Join(irqDir, irq)
list, err := os.ReadFile(filepath.Join(irqPath, "effective_affinity_list"))
if err != nil || len(strings.TrimSpace(string(list))) == 0 {
list, err = os.ReadFile(filepath.Join(irqPath, "smp_affinity_list"))
if err != nil {
return nil
}
}
return parseCPUList(strings.TrimSpace(string(list)))
}
// countQueues counts the rx-* entries of a netdev's queues directory — the
// device's ACTIVE RX queues (sysfs removes the directories when a queue is
// deactivated, e.g. by ethtool -L).
func countQueues(queuesDir string) int {
entries, err := os.ReadDir(queuesDir)
if err != nil {
return 0
}
n := 0
for _, e := range entries {
if strings.HasPrefix(e.Name(), "rx-") {
n++
}
}
return n
}
// containsWord reports whether s contains word bounded by non-alphanumeric
// characters (or string edges), so ifname "eth0" doesn't match "eth01".
func containsWord(s, word string) bool {
for start := 0; ; {
i := strings.Index(s[start:], word)
if i < 0 {
return false
}
i += start
before := i == 0 || !isAlnum(s[i-1])
afterIdx := i + len(word)
after := afterIdx == len(s) || !isAlnum(s[afterIdx])
if before && after {
return true
}
start = i + 1
}
}
func isAlnum(b byte) bool {
return b >= '0' && b <= '9' || b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z'
}
// trailingInt parses the decimal digits at the end of s.
func trailingInt(s string) (int, bool) {
i := len(s)
for i > 0 && s[i-1] >= '0' && s[i-1] <= '9' {
i--
}
if i == len(s) {
return 0, false
}
n, err := strconv.Atoi(s[i:])
return n, err == nil
}
// parseCPUList parses the kernel's cpulist format: comma-separated CPU ids
// or inclusive ranges, e.g. "0-3,8,10-12". Malformed elements are skipped —
// this parses trusted kernel output, not user input.
func parseCPUList(s string) []int {
if s == "" {
return nil
}
var cpus []int
for part := range strings.SplitSeq(s, ",") {
lo, hi, ok := strings.Cut(part, "-")
start, err := strconv.Atoi(strings.TrimSpace(lo))
if err != nil {
continue
}
end := start
if ok {
if end, err = strconv.Atoi(strings.TrimSpace(hi)); err != nil {
continue
}
}
for cpu := start; cpu <= end; cpu++ {
cpus = append(cpus, cpu)
}
}
return cpus
}
+153
View File
@@ -0,0 +1,153 @@
//go:build linux && !android && !e2e_testing
package util
import (
"os"
"path/filepath"
"reflect"
"strconv"
"testing"
)
func TestParseCPUList(t *testing.T) {
cases := []struct {
in string
want []int
}{
{"", nil},
{"3", []int{3}},
{"0-3", []int{0, 1, 2, 3}},
{"0-2,8,10-11", []int{0, 1, 2, 8, 10, 11}},
{"garbage", nil},
{"1,garbage,4", []int{1, 4}},
}
for _, c := range cases {
if got := parseCPUList(c.in); !reflect.DeepEqual(got, c.want) {
t.Errorf("parseCPUList(%q) = %v, want %v", c.in, got, c.want)
}
}
}
func TestTrailingInt(t *testing.T) {
cases := []struct {
in string
want int
ok bool
}{
{"mlx5_comp12", 12, true},
{"ice-eth0-TxRx-3", 3, true},
{"virtio0-input.7", 7, true},
{"mlx5_async0", 0, true},
{"no-digits", 0, false},
{"", 0, false},
}
for _, c := range cases {
got, ok := trailingInt(c.in)
if got != c.want || ok != c.ok {
t.Errorf("trailingInt(%q) = (%d, %v), want (%d, %v)", c.in, got, ok, c.want, c.ok)
}
}
}
func TestContainsWord(t *testing.T) {
if !containsWord("ice-eth0-TxRx-3", "eth0") {
t.Error("eth0 should match with boundaries")
}
if containsWord("ice-eth01-TxRx-3", "eth0") {
t.Error("eth0 must not match inside eth01")
}
if !containsWord("eth0", "eth0") {
t.Error("exact match should work")
}
}
// fakeNIC builds /sys/class/net/<name> with operstate, a device symlink to a
// PCI-address-named dir (physical NICs only), and nq rx queue directories.
func fakeNIC(t *testing.T, netDir, name, operstate, pciAddr string, nq int) {
t.Helper()
devPath := filepath.Join(netDir, name)
if err := os.MkdirAll(devPath, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(devPath, "operstate"), []byte(operstate+"\n"), 0o644); err != nil {
t.Fatal(err)
}
if pciAddr == "" {
return
}
pciDir := filepath.Join(netDir, "..", "devices", pciAddr)
if err := os.MkdirAll(pciDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.Symlink(pciDir, filepath.Join(devPath, "device")); err != nil {
t.Fatal(err)
}
for i := 0; i < nq; i++ {
if err := os.MkdirAll(filepath.Join(devPath, "queues", "rx-"+strconv.Itoa(i)), 0o755); err != nil {
t.Fatal(err)
}
}
}
func writeIRQ(t *testing.T, irqDir, irq, affinity string) {
t.Helper()
p := filepath.Join(irqDir, irq)
if err := os.MkdirAll(p, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(p, "effective_affinity_list"), []byte(affinity+"\n"), 0o644); err != nil {
t.Fatal(err)
}
}
func TestNICIRQCPUs(t *testing.T) {
root := t.TempDir()
netDir := filepath.Join(root, "class", "net")
irqDir := filepath.Join(root, "irq")
if err := os.MkdirAll(netDir, 0o755); err != nil {
t.Fatal(err)
}
// eth0: up, 2 active queues at 0000:82:00.0. comp0/comp1 active,
// comp2 is a deactivated queue's leftover handler, async0 always fires.
fakeNIC(t, netDir, "eth0", "up", "0000:82:00.0", 2)
// eth1: physical but down; its vectors must not count.
fakeNIC(t, netDir, "eth1", "down", "0000:83:00.0", 2)
// eth9: up, matched by ifname (intel-style action names), 1 queue.
fakeNIC(t, netDir, "eth9", "up", "0000:84:00.0", 1)
// nebula1: virtual, no device dir.
fakeNIC(t, netDir, "nebula1", "up", "", 0)
interrupts := filepath.Join(root, "interrupts")
content := ` CPU0 CPU1
100: 1 2 IR-PCI-MSIX 1-edge mlx5_comp0@pci:0000:82:00.0
101: 1 2 IR-PCI-MSIX 2-edge mlx5_comp1@pci:0000:82:00.0
102: 1 2 IR-PCI-MSIX 3-edge mlx5_comp2@pci:0000:82:00.0
103: 1 2 IR-PCI-MSIX 4-edge mlx5_async0@pci:0000:82:00.0
200: 1 2 IR-PCI-MSIX 5-edge mlx5_comp0@pci:0000:83:00.0
300: 1 2 IR-PCI-MSIX 6-edge ice-eth9-TxRx-0
301: 1 2 IR-PCI-MSIX 7-edge ice-eth9-TxRx-1
NMI: 0 0 Non-maskable interrupts
`
if err := os.WriteFile(interrupts, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
writeIRQ(t, irqDir, "100", "0-1") // eth0 comp0: counted
writeIRQ(t, irqDir, "101", "2") // eth0 comp1: counted
writeIRQ(t, irqDir, "102", "5") // eth0 comp2: beyond 2 queues, skipped
writeIRQ(t, irqDir, "103", "7") // eth0 async0: counted
writeIRQ(t, irqDir, "200", "9") // eth1 down: skipped
writeIRQ(t, irqDir, "300", "11") // eth9 TxRx-0: counted
writeIRQ(t, irqDir, "301", "12") // eth9 TxRx-1: beyond 1 queue, skipped
got, err := nicIRQCPUs(netDir, irqDir, interrupts)
if err != nil {
t.Fatalf("nicIRQCPUs: %v", err)
}
want := map[int]bool{0: true, 1: true, 2: true, 7: true, 11: true}
if !reflect.DeepEqual(got, want) {
t.Fatalf("nicIRQCPUs = %v, want %v", got, want)
}
}
+9
View File
@@ -0,0 +1,9 @@
//go:build !linux || android || e2e_testing
package util
// NICIRQCPUs reports no IRQ information on platforms without the linux
// sysfs interface; callers fall back to their non-IRQ-aware defaults.
func NICIRQCPUs() (map[int]bool, error) {
return nil, nil
}