diff --git a/examples/config.yml b/examples/config.yml index f551d5a3..1a6cb393 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -267,7 +267,7 @@ tun: # Linux only. pin_threads pins each tun reader/encrypt OS thread to a single CPU. This keeps every goroutine's # batched sends flowing through one XPS-selected NIC TX ring, so packets within a flow stay ordered on the wire - # instead of being sprayed across multiple TX rings and reordered. Not reloadable. + # instead of being sprayed across multiple TX rings and reordered. Not reloadable. Coerced to false if routines <= 1. #pin_threads: true # Linux only. cpu_affinity overrides which CPUs the tun reader threads pin to: a list of CPU IDs, one per routine diff --git a/interface.go b/interface.go index 7053f1ed..534b511e 100644 --- a/interface.go +++ b/interface.go @@ -202,6 +202,10 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) { return nil, errors.New("no connection manager") } + if c.routines <= 1 { + c.PinThreads = false //pinning is not useful unless there's more than one tun reader + } + cs := c.pki.getCertState() ifce := &Interface{ ctx: ctx, @@ -388,26 +392,30 @@ func (f *Interface) listenOut(i int) { f.l.Debug("underlay reader is done", "reader", i) } +func (f *Interface) pinThisThread(i int) { + var cpu int + if n := len(f.cpuAffinity); n > 0 { + // Explicit tun.cpu_affinity list wins; parseCpuAffinity already + // validated the entries against the allowed CPU set. + cpu = f.cpuAffinity[i%n] + } else if allowed, err := util.AllowedCPUs(); err == nil && len(allowed) > 0 { + // Default: spread queues across the CPUs we're actually allowed to + // run on. Under a cpuset/taskset mask these aren't 0..NumCPU-1, so + // i % NumCPU would pick unrunnable IDs and every pin would fail. + cpu = allowed[i%len(allowed)] + } else { + cpu = i % runtime.NumCPU() + } + if err := util.PinThreadToCPU(cpu); err != nil { + f.l.Warn("failed to pin tun reader to CPU", "queue", i, "cpu", cpu, "err", err) + } +} + func (f *Interface) listenIn(queue tio.Queue, i int) { // Pinning this thread (and goroutine) to a single CPU keeps every sendmmsg from this goroutine going through the // same TX ring on the nic, so the wire sees per-flow order. Skip entirely when tun.pin_threads is false. if f.pinThreads { - var cpu int - if n := len(f.cpuAffinity); n > 0 { - // Explicit tun.cpu_affinity list wins; parseCpuAffinity already - // validated the entries against the allowed CPU set. - cpu = f.cpuAffinity[i%n] - } else if allowed, err := util.AllowedCPUs(); err == nil && len(allowed) > 0 { - // Default: spread queues across the CPUs we're actually allowed to - // run on. Under a cpuset/taskset mask these aren't 0..NumCPU-1, so - // i % NumCPU would pick unrunnable IDs and every pin would fail. - cpu = allowed[i%len(allowed)] - } else { - cpu = i % runtime.NumCPU() - } - if err := util.PinThreadToCPU(cpu); err != nil { - f.l.Warn("failed to pin tun reader to CPU", "queue", i, "cpu", cpu, "err", err) - } + f.pinThisThread(i) } rejectBuf := make([]byte, mtu)