mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-15 06:46:52 +01:00
With staged transmit, an idle reader encrypted and sent a small read itself, while larger reads went to the workers and left from the ordered sender. Each read chose on its own, so under load a routine alternated between the two paths. Their packets left from different CPUs, and the receiver's TCP saw them out of order: on 100 GbE, summed over four hosts, dsack_dups rose from 0-1 on master to about 25k and reord_seen from 0 to 13k-19k. Each routine now holds one mode, inline on the reader or fanout to the workers and the sender. It switches only after tun.staged.gate_hold (default 64) reads in a row want the other mode, and before going inline it waits until every batch in flight is on the wire, so a switch never lets a later packet overtake an earlier one. Going to fanout nothing is in flight. flow.tx.gate_switches counts the switches. On the same hosts dsack_dups fell to 20-52 and reord_seen to 0-2, while throughput matched or beat per-read mixing (many-to-one 34.9 instead of 30.7 Gbit/s). The affinity test waited only for the previous batch's packets to be captured before the next read, but the sender returns the slot after its write, so the reader could still see it in flight and stage the small read. Switching to inline now drains, which makes the inline send deterministic there.
195 lines
6.5 KiB
Go
195 lines
6.5 KiB
Go
package nebula
|
|
|
|
import (
|
|
"fmt"
|
|
"net/netip"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/slackhq/nebula/cert"
|
|
cert_test "github.com/slackhq/nebula/cert_test"
|
|
"github.com/slackhq/nebula/config"
|
|
"github.com/slackhq/nebula/noiseutil"
|
|
"github.com/slackhq/nebula/test"
|
|
"github.com/slackhq/nebula/util"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/goleak"
|
|
)
|
|
|
|
func TestParseCpuAffinity(t *testing.T) {
|
|
l := test.NewLogger()
|
|
|
|
// newConfig returns a config.C with tun.cpu_affinity set to v. A nil v
|
|
// leaves the key unset.
|
|
newConfig := func(v any) *config.C {
|
|
c := config.NewC(l)
|
|
if v != nil {
|
|
c.Settings["tun"] = map[string]any{"cpu_affinity": v}
|
|
}
|
|
return c
|
|
}
|
|
|
|
// unset -> nil (listenIn falls back to spreading across the allowed set)
|
|
assert.Nil(t, parseCpuAffinity(newConfig(nil), l, 1))
|
|
|
|
// Pick a CPU we're actually allowed to run on so a valid list survives
|
|
// validation regardless of the host's affinity mask.
|
|
allowed, _ := util.AllowedCPUs()
|
|
validCPU := 0
|
|
if len(allowed) > 0 {
|
|
validCPU = allowed[0]
|
|
}
|
|
|
|
// valid list -> parsed through unchanged
|
|
assert.Equal(t, []int{validCPU, validCPU}, parseCpuAffinity(newConfig([]any{validCPU, validCPU}), l, 2))
|
|
|
|
// a negative entry is out of range on every platform -> disables the override
|
|
assert.Nil(t, parseCpuAffinity(newConfig([]any{validCPU, -1}), l, 2))
|
|
|
|
// a non-integer entry -> disables the override
|
|
assert.Nil(t, parseCpuAffinity(newConfig([]any{validCPU, "not-a-cpu"}), l, 2))
|
|
|
|
// a CPU id outside the allowed set -> disables the override. Only assertable
|
|
// where we can enumerate the allowed set (e.g. linux); 1<<20 is far beyond
|
|
// any representable CPU id so it can never be in the mask.
|
|
if len(allowed) > 0 {
|
|
assert.Nil(t, parseCpuAffinity(newConfig([]any{1 << 20}), l, 1))
|
|
}
|
|
}
|
|
|
|
// TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running
|
|
// when it hands back a Control to stop them with.
|
|
//
|
|
// A config test gets no Control, so anything it started had nothing to stop it: the lighthouse
|
|
// query worker, and a hostname resolver per dns named static host, ran for the life of the
|
|
// process. That matters to every embedder that validates a config in process rather than by
|
|
// exec'ing, dnclient and the apple clients included, because they do it on each config load and
|
|
// the leak accumulates.
|
|
func TestMain_ConfigTestReleasesItsGoroutines(t *testing.T) {
|
|
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
|
|
|
|
l := test.NewLogger()
|
|
dir := t.TempDir()
|
|
|
|
before := time.Now().Add(-time.Hour)
|
|
after := time.Now().Add(time.Hour)
|
|
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
|
|
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
|
|
_, _, keyPEM, certPEM := cert_test.NewTestCert(
|
|
cert.Version2, cert.Curve_CURVE25519, ca, caKey, "config-test", before, after, networks, nil, nil)
|
|
|
|
caPath := filepath.Join(dir, "ca.pem")
|
|
certPath := filepath.Join(dir, "cert.pem")
|
|
keyPath := filepath.Join(dir, "key.pem")
|
|
require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
|
|
require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
|
|
require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
|
|
|
|
// A static host by address, not by name: the query worker is the goroutine under test and a
|
|
// hostname would drag a real dns lookup into a unit test.
|
|
configBody := fmt.Sprintf(`
|
|
pki:
|
|
ca: %s
|
|
cert: %s
|
|
key: %s
|
|
static_host_map:
|
|
"10.0.0.2": ["192.0.2.1:4242"]
|
|
lighthouse:
|
|
hosts:
|
|
- "10.0.0.2"
|
|
listen:
|
|
host: 127.0.0.1
|
|
port: 0
|
|
tun:
|
|
disabled: true
|
|
firewall:
|
|
outbound:
|
|
- port: any
|
|
proto: any
|
|
host: any
|
|
inbound:
|
|
- port: any
|
|
proto: any
|
|
host: any
|
|
`, caPath, certPath, keyPath)
|
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
|
|
|
|
c := config.NewC(l)
|
|
require.NoError(t, c.Load(dir))
|
|
|
|
ctrl, err := Main(c, true, "config-test", l, nil)
|
|
require.NoError(t, err)
|
|
require.Nil(t, ctrl, "a config test hands back nothing to stop, so it must stop itself")
|
|
}
|
|
|
|
func TestParseStagedConfig(t *testing.T) {
|
|
l := test.NewLogger()
|
|
staged := func(workers any) stagedConfig {
|
|
c := config.NewC(l)
|
|
if workers != nil {
|
|
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": workers}}
|
|
}
|
|
return parseStagedConfig(c, l)
|
|
}
|
|
for _, workers := range []any{nil, -1, 0} {
|
|
require.Equal(t, stagedConfig{}, staged(workers))
|
|
}
|
|
if runtime.GOOS != "linux" {
|
|
require.Equal(t, stagedConfig{}, staged(2))
|
|
return
|
|
}
|
|
want := stagedConfig{workers: 2, tx: !noiseutil.EncryptLockNeeded, rx: true, inlineBelow: 256, gateHold: stagedGateHold,
|
|
txSlots: 8, rxSlots: 6}
|
|
require.Equal(t, want, staged(2))
|
|
|
|
c := config.NewC(l)
|
|
c.Settings["listen"] = map[string]any{"batch": 128}
|
|
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "tx_slots": 3, "rx_slots": 1}}
|
|
got := parseStagedConfig(c, l)
|
|
require.Equal(t, []int{3, 2}, []int{got.txSlots, got.rxSlots}, "overrides apply, down to the minimum")
|
|
|
|
c = config.NewC(l)
|
|
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "tx": false, "inline_below": 0}}
|
|
got = parseStagedConfig(c, l)
|
|
require.False(t, got.tx, "tx: false keeps transmit inline")
|
|
require.True(t, got.rx)
|
|
require.Zero(t, got.inlineBelow, "inline_below: 0 hands every batch off")
|
|
|
|
c = config.NewC(l)
|
|
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "inline_below": 32}}
|
|
require.Equal(t, 32, parseStagedConfig(c, l).inlineBelow)
|
|
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "inline_below": -1}}
|
|
require.Zero(t, parseStagedConfig(c, l).inlineBelow, "a negative threshold is 0")
|
|
|
|
for _, tt := range []struct {
|
|
hold any
|
|
want int
|
|
}{{nil, stagedGateHold}, {8, 8}, {0, 1}, {-3, 1}} {
|
|
c = config.NewC(l)
|
|
settings := map[string]any{"workers": 2}
|
|
if tt.hold != nil {
|
|
settings["gate_hold"] = tt.hold
|
|
}
|
|
c.Settings["tun"] = map[string]any{"staged": settings}
|
|
require.Equal(t, tt.want, parseStagedConfig(c, l).gateHold, "gate_hold %v", tt.hold)
|
|
}
|
|
}
|
|
|
|
func TestStagedSlots(t *testing.T) {
|
|
for _, tt := range []struct{ workers, batch, tx, rx int }{
|
|
{workers: 2, batch: 64, tx: 8, rx: 6},
|
|
{workers: 12, batch: 64, tx: 14, rx: 6},
|
|
{workers: 2, batch: 96, tx: 8, rx: 4},
|
|
{workers: 2, batch: 512, tx: 8, rx: 4},
|
|
{workers: 2, batch: 32, tx: 8, rx: 6},
|
|
{workers: 2, batch: 1, tx: 8, rx: 6},
|
|
} {
|
|
tx, rx := stagedSlots(tt.workers, tt.batch, 0, 0)
|
|
require.Equal(t, []int{tt.tx, tt.rx}, []int{tx, rx}, "workers %d batch %d", tt.workers, tt.batch)
|
|
}
|
|
}
|