From 7fe4fab1678c89d98464121ba92c047850030206 Mon Sep 17 00:00:00 2001 From: JackDoan Date: Tue, 14 Jul 2026 15:14:40 -0500 Subject: [PATCH] tun: default pin CPUs avoid NIC IRQ cores When tun.pin_threads is on and tun.cpu_affinity is unset, pick pin CPUs from the allowed set that do not service any up physical NIC's MSI vectors (read-only walk of /sys/class/net/*/device/msi_irqs and /proc/irq/*/effective_affinity_list). The old allowed[i] default pinned encrypt threads onto exactly the cores drivers affine their first RX queue IRQs to; a flow whose RSS queue fired there had NAPI fighting encrypt for the core (measured 8.4 vs 10.2 Gbps REV bimodality). Falls back to the old spread, with a log, when there aren't enough IRQ-free CPUs - e.g. drivers that allocate one queue per core until the admin narrows them (ethtool -L). Co-Authored-By: Claude Fable 5 --- examples/config.yml | 10 +- main.go | 61 +++++++++++- main_test.go | 20 ++++ util/irqcpus_linux.go | 198 +++++++++++++++++++++++++++++++++++++ util/irqcpus_linux_test.go | 153 ++++++++++++++++++++++++++++ util/irqcpus_other.go | 9 ++ 6 files changed, 448 insertions(+), 3 deletions(-) create mode 100644 util/irqcpus_linux.go create mode 100644 util/irqcpus_linux_test.go create mode 100644 util/irqcpus_other.go diff --git a/examples/config.yml b/examples/config.yml index 4c64f7ab..d30a6e17 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -257,13 +257,21 @@ tun: # Linux only. pin_threads pins each tun reader/encrypt OS thread to a single CPU. This keeps every goroutine's # batched sends flowing through one XPS-selected NIC TX ring, so packets within a flow stay ordered on the wire # instead of being sprayed across multiple TX rings and reordered. Not reloadable. + # + # When cpu_affinity is unset, nebula picks CPUs that do NOT service any physical NIC's interrupts (read from + # /sys/class/net/*/device/msi_irqs and /proc/irq/*/effective_affinity_list): an encrypt thread pinned onto a core + # that also runs NAPI for a NIC RX queue fights the softirq for the core and collapses throughput for flows hashed + # to that queue. If the NIC's vectors blanket every allowed CPU (many drivers default to one queue per core) the + # avoidance logs and falls back to the old spread; narrow the NIC's queue/IRQ spread (e.g. `ethtool -X + # equal N`) or set cpu_affinity explicitly to benefit. #pin_threads: true # Linux only. cpu_affinity overrides which CPUs the tun reader threads pin to: a list of CPU IDs, one per routine # (see the top-level `routines` setting). Lists shorter than `routines` are modulo-cycled across the queues; extra # entries are ignored. IDs must be within the process's allowed CPU set, so this respects taskset / cgroup cpusets; # a non-integer or not-allowed entry disables the override and falls back to spreading queues across the allowed - # CPUs. Only meaningful while pin_threads is true. Not reloadable. + # CPUs. Setting this disables the automatic NIC-IRQ avoidance described under pin_threads — prefer CPUs that don't + # service your underlay NIC's RX queue IRQs. Only meaningful while pin_threads is true. Not reloadable. #cpu_affinity: # - 2 # - 4 diff --git a/main.go b/main.go index 5439b773..ae5ea6e1 100644 --- a/main.go +++ b/main.go @@ -214,6 +214,12 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev l.Warn("Failed to start DNS responder", "error", err) } + pinThreads := c.GetBool("tun.pin_threads", true) + cpuAffinity := parseCpuAffinity(c, l, routines) + if pinThreads && len(cpuAffinity) == 0 && !configTest { + cpuAffinity = defaultCPUAffinityAvoidingIRQs(l, routines) + } + ifConfig := &InterfaceConfig{ HostMap: hostMap, Inside: tun, @@ -235,8 +241,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev relayManager: NewRelayManager(ctx, l, hostMap, c), punchy: punchy, ConntrackCacheTimeout: conntrackCacheTimeout, - CpuAffinity: parseCpuAffinity(c, l, routines), - PinThreads: c.GetBool("tun.pin_threads", true), + CpuAffinity: cpuAffinity, + PinThreads: pinThreads, l: l, } @@ -353,6 +359,57 @@ func parseCpuAffinity(c *config.C, l *slog.Logger, routines int) []int { return cpus } +// defaultCPUAffinityAvoidingIRQs picks the default pin set for the tun +// readers when tun.cpu_affinity is unset: allowed CPUs that do NOT service +// any physical NIC's interrupts. The stock allowed[i] spread pins the +// encrypt threads onto exactly the cores most drivers affine their first RX +// queue IRQs to, so whenever a flow's RSS queue fires on a core hosting a +// tun reader, NAPI and encrypt fight for the core and per-flow throughput +// drops (measured: REV 8.4 vs 10.2 Gbps on the same hardware, 2026-07-14). +// +// Returns nil — keeping the old allowed[i] fallback in listenIn — when IRQ +// info is unavailable or when there aren't enough IRQ-free CPUs to give +// every routine its own core: silently doubling readers up on fewer cores +// is worse than the occasional IRQ collision. NICs whose vectors blanket +// every CPU (e.g. mlx5 defaults to one queue per core) make avoidance +// impossible; narrowing the NIC's spread (ethtool -X equal N, or +// /proc/irq/*/smp_affinity) or setting tun.cpu_affinity explicitly makes it +// effective. +func defaultCPUAffinityAvoidingIRQs(l *slog.Logger, routines int) []int { + irq, err := util.NICIRQCPUs() + if err != nil || len(irq) == 0 { + return nil + } + allowed, err := util.AllowedCPUs() + if err != nil { + return nil + } + cpus := chooseIRQFreeCPUs(allowed, irq, routines) + if cpus == nil { + l.Info("not enough CPUs are free of NIC IRQs to give every tun reader its own; using the default spread", + "routines", routines, "allowed", len(allowed), "irqCPUs", len(irq)) + return nil + } + l.Info("pinning tun readers to CPUs clear of NIC IRQs", "cpus", cpus) + return cpus +} + +// chooseIRQFreeCPUs returns the first `routines` allowed CPUs not present in +// irq, or nil if fewer than `routines` qualify. +func chooseIRQFreeCPUs(allowed []int, irq map[int]bool, routines int) []int { + free := make([]int, 0, routines) + for _, cpu := range allowed { + if irq[cpu] { + continue + } + free = append(free, cpu) + if len(free) == routines { + return free + } + } + return nil +} + func moduleVersion() string { info, ok := debug.ReadBuildInfo() if !ok { diff --git a/main_test.go b/main_test.go index bbaef347..ff62c4b3 100644 --- a/main_test.go +++ b/main_test.go @@ -9,6 +9,26 @@ import ( "github.com/stretchr/testify/assert" ) +func TestChooseIRQFreeCPUs(t *testing.T) { + irq := map[int]bool{0: true, 1: true, 2: true, 3: true} + + // Plenty of IRQ-free CPUs: take the first `routines` of them in order. + assert.Equal(t, []int{4, 5}, chooseIRQFreeCPUs([]int{0, 1, 2, 3, 4, 5, 6}, irq, 2)) + + // Exactly enough. + assert.Equal(t, []int{4, 5, 6}, chooseIRQFreeCPUs([]int{0, 1, 2, 3, 4, 5, 6}, irq, 3)) + + // Not enough IRQ-free CPUs: nil, caller keeps the old default rather + // than doubling readers up on shared cores. + assert.Nil(t, chooseIRQFreeCPUs([]int{0, 1, 2, 3, 4}, irq, 2)) + + // No IRQ info at all behaves like a plain prefix of allowed. + assert.Equal(t, []int{0, 1}, chooseIRQFreeCPUs([]int{0, 1, 2}, map[int]bool{}, 2)) + + // Non-contiguous allowed set (cgroup cpuset) with holes. + assert.Equal(t, []int{9, 12}, chooseIRQFreeCPUs([]int{1, 3, 9, 12}, map[int]bool{1: true, 3: true}, 2)) +} + func TestParseCpuAffinity(t *testing.T) { l := test.NewLogger() diff --git a/util/irqcpus_linux.go b/util/irqcpus_linux.go new file mode 100644 index 00000000..30ae3ab3 --- /dev/null +++ b/util/irqcpus_linux.go @@ -0,0 +1,198 @@ +//go:build linux && !android && !e2e_testing + +package util + +import ( + "os" + "path/filepath" + "strconv" + "strings" +) + +// NICIRQCPUs returns the set of CPUs that service interrupts for the ACTIVE +// RX/TX queues of physical network interfaces that are up. Read-only: it +// matches /proc/interrupts action names against each NIC's PCI address and +// interface name, drops vectors whose queue index is beyond the device's +// active queue count (drivers like mlx5 keep handlers registered for +// deactivated queues, so /proc/interrupts alone over-reports), and unions +// /proc/irq//effective_affinity_list for the survivors. +// +// Callers use this to keep busy pinned threads OFF those CPUs: a thread +// pinned onto a core that also runs NAPI for a NIC RX queue competes with +// softirq processing for the core and measurably collapses throughput for +// flows hashed to that queue. +func NICIRQCPUs() (map[int]bool, error) { + return nicIRQCPUs("/sys/class/net", "/proc/irq", "/proc/interrupts") +} + +// irqAction is one row of /proc/interrupts: the IRQ number and the action +// (handler) name in its final column, e.g. "mlx5_comp3@pci:0000:82:00.0". +type irqAction struct { + irq string + action string +} + +func nicIRQCPUs(netDir, irqDir, interruptsPath string) (map[int]bool, error) { + actions, err := parseInterrupts(interruptsPath) + if err != nil { + return nil, err + } + devs, err := os.ReadDir(netDir) + if err != nil { + return nil, err + } + + cpus := make(map[int]bool) + for _, dev := range devs { + devPath := filepath.Join(netDir, dev.Name()) + pciDev, err := filepath.EvalSymlinks(filepath.Join(devPath, "device")) + if err != nil { + continue // virtual device (lo, tun, bridge, vlan, ...) + } + pciAddr := filepath.Base(pciDev) + state, err := os.ReadFile(filepath.Join(devPath, "operstate")) + if err != nil || strings.TrimSpace(string(state)) != "up" { + continue // a down NIC's queue IRQs don't fire + } + nq := countQueues(filepath.Join(devPath, "queues")) + + for _, ia := range actions { + if !strings.Contains(ia.action, pciAddr) && !containsWord(ia.action, dev.Name()) { + continue + } + // Vector naming puts the queue index at the end of the handler + // name (mlx5_comp3@pci:..., ice-eth0-TxRx-3, virtio0-input.3). + // An index at or beyond the active queue count is a handler for + // a deactivated queue: registered, but it will not fire. + name, _, _ := strings.Cut(ia.action, "@") + if idx, ok := trailingInt(name); ok && idx >= nq { + continue + } + for _, cpu := range irqAffinity(irqDir, ia.irq) { + cpus[cpu] = true + } + } + } + return cpus, nil +} + +// parseInterrupts extracts (irq, action) pairs from /proc/interrupts, +// skipping the header and the non-numeric summary rows (NMI, LOC, ...). +func parseInterrupts(path string) ([]irqAction, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var out []irqAction + for line := range strings.SplitSeq(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + irq, ok := strings.CutSuffix(fields[0], ":") + if !ok { + continue + } + if _, err := strconv.Atoi(irq); err != nil { + continue + } + out = append(out, irqAction{irq: irq, action: fields[len(fields)-1]}) + } + return out, nil +} + +// irqAffinity returns the CPUs IRQ n actually targets. +// effective_affinity_list is the vector's real target; smp_affinity_list +// (the fallback for kernels without effective affinity reporting) is the +// admin-allowed mask and may be wider. +func irqAffinity(irqDir, irq string) []int { + irqPath := filepath.Join(irqDir, irq) + list, err := os.ReadFile(filepath.Join(irqPath, "effective_affinity_list")) + if err != nil || len(strings.TrimSpace(string(list))) == 0 { + list, err = os.ReadFile(filepath.Join(irqPath, "smp_affinity_list")) + if err != nil { + return nil + } + } + return parseCPUList(strings.TrimSpace(string(list))) +} + +// countQueues counts the rx-* entries of a netdev's queues directory — the +// device's ACTIVE RX queues (sysfs removes the directories when a queue is +// deactivated, e.g. by ethtool -L). +func countQueues(queuesDir string) int { + entries, err := os.ReadDir(queuesDir) + if err != nil { + return 0 + } + n := 0 + for _, e := range entries { + if strings.HasPrefix(e.Name(), "rx-") { + n++ + } + } + return n +} + +// containsWord reports whether s contains word bounded by non-alphanumeric +// characters (or string edges), so ifname "eth0" doesn't match "eth01". +func containsWord(s, word string) bool { + for start := 0; ; { + i := strings.Index(s[start:], word) + if i < 0 { + return false + } + i += start + before := i == 0 || !isAlnum(s[i-1]) + afterIdx := i + len(word) + after := afterIdx == len(s) || !isAlnum(s[afterIdx]) + if before && after { + return true + } + start = i + 1 + } +} + +func isAlnum(b byte) bool { + return b >= '0' && b <= '9' || b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' +} + +// trailingInt parses the decimal digits at the end of s. +func trailingInt(s string) (int, bool) { + i := len(s) + for i > 0 && s[i-1] >= '0' && s[i-1] <= '9' { + i-- + } + if i == len(s) { + return 0, false + } + n, err := strconv.Atoi(s[i:]) + return n, err == nil +} + +// parseCPUList parses the kernel's cpulist format: comma-separated CPU ids +// or inclusive ranges, e.g. "0-3,8,10-12". Malformed elements are skipped — +// this parses trusted kernel output, not user input. +func parseCPUList(s string) []int { + if s == "" { + return nil + } + var cpus []int + for part := range strings.SplitSeq(s, ",") { + lo, hi, ok := strings.Cut(part, "-") + start, err := strconv.Atoi(strings.TrimSpace(lo)) + if err != nil { + continue + } + end := start + if ok { + if end, err = strconv.Atoi(strings.TrimSpace(hi)); err != nil { + continue + } + } + for cpu := start; cpu <= end; cpu++ { + cpus = append(cpus, cpu) + } + } + return cpus +} diff --git a/util/irqcpus_linux_test.go b/util/irqcpus_linux_test.go new file mode 100644 index 00000000..5320ca9d --- /dev/null +++ b/util/irqcpus_linux_test.go @@ -0,0 +1,153 @@ +//go:build linux && !android && !e2e_testing + +package util + +import ( + "os" + "path/filepath" + "reflect" + "strconv" + "testing" +) + +func TestParseCPUList(t *testing.T) { + cases := []struct { + in string + want []int + }{ + {"", nil}, + {"3", []int{3}}, + {"0-3", []int{0, 1, 2, 3}}, + {"0-2,8,10-11", []int{0, 1, 2, 8, 10, 11}}, + {"garbage", nil}, + {"1,garbage,4", []int{1, 4}}, + } + for _, c := range cases { + if got := parseCPUList(c.in); !reflect.DeepEqual(got, c.want) { + t.Errorf("parseCPUList(%q) = %v, want %v", c.in, got, c.want) + } + } +} + +func TestTrailingInt(t *testing.T) { + cases := []struct { + in string + want int + ok bool + }{ + {"mlx5_comp12", 12, true}, + {"ice-eth0-TxRx-3", 3, true}, + {"virtio0-input.7", 7, true}, + {"mlx5_async0", 0, true}, + {"no-digits", 0, false}, + {"", 0, false}, + } + for _, c := range cases { + got, ok := trailingInt(c.in) + if got != c.want || ok != c.ok { + t.Errorf("trailingInt(%q) = (%d, %v), want (%d, %v)", c.in, got, ok, c.want, c.ok) + } + } +} + +func TestContainsWord(t *testing.T) { + if !containsWord("ice-eth0-TxRx-3", "eth0") { + t.Error("eth0 should match with boundaries") + } + if containsWord("ice-eth01-TxRx-3", "eth0") { + t.Error("eth0 must not match inside eth01") + } + if !containsWord("eth0", "eth0") { + t.Error("exact match should work") + } +} + +// fakeNIC builds /sys/class/net/ with operstate, a device symlink to a +// PCI-address-named dir (physical NICs only), and nq rx queue directories. +func fakeNIC(t *testing.T, netDir, name, operstate, pciAddr string, nq int) { + t.Helper() + devPath := filepath.Join(netDir, name) + if err := os.MkdirAll(devPath, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(devPath, "operstate"), []byte(operstate+"\n"), 0o644); err != nil { + t.Fatal(err) + } + if pciAddr == "" { + return + } + pciDir := filepath.Join(netDir, "..", "devices", pciAddr) + if err := os.MkdirAll(pciDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(pciDir, filepath.Join(devPath, "device")); err != nil { + t.Fatal(err) + } + for i := 0; i < nq; i++ { + if err := os.MkdirAll(filepath.Join(devPath, "queues", "rx-"+strconv.Itoa(i)), 0o755); err != nil { + t.Fatal(err) + } + } +} + +func writeIRQ(t *testing.T, irqDir, irq, affinity string) { + t.Helper() + p := filepath.Join(irqDir, irq) + if err := os.MkdirAll(p, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(p, "effective_affinity_list"), []byte(affinity+"\n"), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestNICIRQCPUs(t *testing.T) { + root := t.TempDir() + netDir := filepath.Join(root, "class", "net") + irqDir := filepath.Join(root, "irq") + if err := os.MkdirAll(netDir, 0o755); err != nil { + t.Fatal(err) + } + + // eth0: up, 2 active queues at 0000:82:00.0. comp0/comp1 active, + // comp2 is a deactivated queue's leftover handler, async0 always fires. + fakeNIC(t, netDir, "eth0", "up", "0000:82:00.0", 2) + // eth1: physical but down; its vectors must not count. + fakeNIC(t, netDir, "eth1", "down", "0000:83:00.0", 2) + // eth9: up, matched by ifname (intel-style action names), 1 queue. + fakeNIC(t, netDir, "eth9", "up", "0000:84:00.0", 1) + // nebula1: virtual, no device dir. + fakeNIC(t, netDir, "nebula1", "up", "", 0) + + interrupts := filepath.Join(root, "interrupts") + content := ` CPU0 CPU1 + 100: 1 2 IR-PCI-MSIX 1-edge mlx5_comp0@pci:0000:82:00.0 + 101: 1 2 IR-PCI-MSIX 2-edge mlx5_comp1@pci:0000:82:00.0 + 102: 1 2 IR-PCI-MSIX 3-edge mlx5_comp2@pci:0000:82:00.0 + 103: 1 2 IR-PCI-MSIX 4-edge mlx5_async0@pci:0000:82:00.0 + 200: 1 2 IR-PCI-MSIX 5-edge mlx5_comp0@pci:0000:83:00.0 + 300: 1 2 IR-PCI-MSIX 6-edge ice-eth9-TxRx-0 + 301: 1 2 IR-PCI-MSIX 7-edge ice-eth9-TxRx-1 + NMI: 0 0 Non-maskable interrupts +` + if err := os.WriteFile(interrupts, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + writeIRQ(t, irqDir, "100", "0-1") // eth0 comp0: counted + writeIRQ(t, irqDir, "101", "2") // eth0 comp1: counted + writeIRQ(t, irqDir, "102", "5") // eth0 comp2: beyond 2 queues, skipped + writeIRQ(t, irqDir, "103", "7") // eth0 async0: counted + writeIRQ(t, irqDir, "200", "9") // eth1 down: skipped + writeIRQ(t, irqDir, "300", "11") // eth9 TxRx-0: counted + writeIRQ(t, irqDir, "301", "12") // eth9 TxRx-1: beyond 1 queue, skipped + + got, err := nicIRQCPUs(netDir, irqDir, interrupts) + if err != nil { + t.Fatalf("nicIRQCPUs: %v", err) + } + want := map[int]bool{0: true, 1: true, 2: true, 7: true, 11: true} + if !reflect.DeepEqual(got, want) { + t.Fatalf("nicIRQCPUs = %v, want %v", got, want) + } +} diff --git a/util/irqcpus_other.go b/util/irqcpus_other.go new file mode 100644 index 00000000..d7781889 --- /dev/null +++ b/util/irqcpus_other.go @@ -0,0 +1,9 @@ +//go:build !linux || android || e2e_testing + +package util + +// NICIRQCPUs reports no IRQ information on platforms without the linux +// sysfs interface; callers fall back to their non-IRQ-aware defaults. +func NICIRQCPUs() (map[int]bool, error) { + return nil, nil +}