Compare commits

..
33 changed files with 412 additions and 3775 deletions

No files matched your search

-5
View File
@@ -7,11 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Removed
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
`mutex-profile-fraction` commands, and `sshd.sandbox_dir`, which is now ignored if set.
### Fixed
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
+1 -10
View File
@@ -14,9 +14,6 @@ import (
)
const (
// ReplayWindow is the build-time receive window and the assumed peer window
// for TX reservation credit. Use a power of two; peers must match or have a
// larger receive window. The bitmap costs ReplayWindow/8 bytes per tunnel.
ReplayWindow = 8192
// RehandshakeAfterMessages rolls keys inside the AES-GCM data-volume margin (~2^-36 advantage at 64KB frames).
@@ -44,7 +41,6 @@ type ConnectionState struct {
peerCert *cert.CachedCertificate
initiator bool
messageCounter atomic.Uint64
txBudget txBudget
window *Bits
decryptLock sync.Mutex
writeLock sync.Mutex
@@ -108,7 +104,7 @@ func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, packet
return nil, ErrAlreadySeen
}
out, err := cs.decryptPacket(messageCounter, packet, nb)
out, err := cs.dKey.DecryptDanger(packet[header.Len:header.Len], packet[:header.Len], packet[header.Len:], messageCounter, nb)
if err != nil {
return nil, err
}
@@ -122,11 +118,6 @@ func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, packet
return out, nil
}
// decryptPacket authenticates and decrypts packet in place, leaving the replay window to the caller.
func (cs *ConnectionState) decryptPacket(messageCounter uint64, packet, nb []byte) ([]byte, error) {
return cs.dKey.DecryptDanger(packet[header.Len:header.Len], packet[:header.Len], packet[header.Len:], messageCounter, nb)
}
func (cs *ConnectionState) VerifyRelay(l *slog.Logger, messageCounter uint64, packet []byte, nb []byte) error {
cs.decryptLock.Lock()
result := cs.window.Check(l, messageCounter)
+3 -12
View File
@@ -23,20 +23,11 @@ import (
// runTestHandshake runs a complete IX handshake between two freshly-built
// peers and returns the initiator and responder Results. Used to produce
// real cipher states for tests that need to exercise post-handshake glue.
func runTestHandshake(t testing.TB) (initR, respR *handshake.Result) {
t.Helper()
return runTestHandshakeWith(t, noise.CipherChaChaPoly)
}
// runTestHandshakeWith is runTestHandshake with the data plane cipher picked by the caller.
func runTestHandshakeWith(t testing.TB, cipher noise.CipherFunc) (initR, respR *handshake.Result) {
func runTestHandshake(t *testing.T) (initR, respR *handshake.Result) {
t.Helper()
// Valid around now rather than the cert_test default, which is fixed at process start and lapses a minute
// later, inside a long -count run
now := time.Now()
ca, _, caKey, _ := ct.NewTestCaCert(
cert.Version2, cert.Curve_CURVE25519, now.Add(-time.Minute), now.Add(time.Hour), nil, nil, nil,
cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
)
caPool := ct.NewTestCAPool(ca)
@@ -49,7 +40,7 @@ func runTestHandshakeWith(t testing.TB, cipher noise.CipherFunc) (initR, respR *
require.NoError(t, err)
hsBytes, err := c.MarshalForHandshakes()
require.NoError(t, err)
ncs := noise.NewCipherSuite(noise.DH25519, cipher, noise.HashSHA256)
ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
cred := handshake.NewCredential(c, hsBytes, priv, ncs)
return func(v cert.Version) *handshake.Credential {
if v == cert.Version2 {
-29
View File
@@ -5,11 +5,9 @@ package e2e
import (
"fmt"
"net/netip"
"runtime"
"testing"
"time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/e2e/router"
@@ -572,30 +570,3 @@ func TestCloseTunnelAuthenticated(t *testing.T) {
myControl.Stop()
theirControl.Stop()
}
// TestStagedTunnel completes a handshake and traffic both ways with staged receive. The test tun cannot hand off
// batches, so transmit stays inline; every received packet, handshakes included, uses the receive pipeline on Linux.
// Other platforms must do the same over the inline fallback.
func TestStagedTunnel(t *testing.T) {
t.Parallel()
received := metrics.GetOrRegisterCounter("flow.rx.inline", nil)
before := received.Count()
staged := m{"tun": m{"staged": m{"workers": 2}}}
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", staged)
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", staged)
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
myControl.Start()
theirControl.Start()
r := router.NewR(t, myControl, theirControl)
defer r.RenderFlow()
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
myControl.Stop()
theirControl.Stop()
if runtime.GOOS != "linux" {
assert.Zero(t, received.Count()-before)
return
}
assert.GreaterOrEqual(t, received.Count()-before, int64(2))
}
+8 -37
View File
@@ -8,7 +8,6 @@ pki:
cert: /etc/nebula/host.crt
key: /etc/nebula/host.key
# blocklist is a list of certificate fingerprints that we will refuse to talk to
# Entries are the hex fingerprints shown by `nebula-cert print`, an invalid entry fails startup and reload.
#blocklist:
# - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72
# disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid.
@@ -233,6 +232,13 @@ punchy:
# Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
#trusted_cas:
#- "ssh public key string"
# sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
# save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
# and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
# The directory is NOT automatically created.
# Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
# Not reloadable.
#sandbox_dir: /var/tmp/nebula-debug
# EXPERIMENTAL: relay support for networks that can't establish direct connections.
relay:
@@ -275,42 +281,9 @@ tun:
# the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable.
#use_offloads: false
# Linux only. staged.workers enables staged packet processing: each routine hands its batches to this many crypto
# workers per direction, then sends or writes them in order. 0, the default, keeps processing inline. Not reloadable.
# Transmit needs use_offloads; BoringCrypto and FIPS builds keep it inline because AES-GCM there needs ordered nonces.
# Receive needs a listener that can hand off its batches, which Linux has.
# IMPORTANT: staged senders require peers with at least the 8192-entry replay window of this release. Released
# v1.11.x peers use 1024 and are incompatible. The window is not negotiated.
# Staged reservations may use half the window, the other half absorbs inline and control traffic, relay forwarding
# and network reordering; unlimited forwarding or delay can still exceed it. Waiting for credit on a busy tunnel
# also delays the other tunnels of the same read, and readers are not guaranteed fairness.
# Small batches, below inline_below TX segments or RX packets (default 256), can skip the workers. On transmit each
# routine holds one mode, inline on the reader or fanout to the workers and the ordered sender, and switches only
# after gate_hold (default 64) reads in a row want the other one. Before going inline it waits for every batch in
# flight to be submitted to the socket; flow.tx.gate_switches counts the switches. This does not drain NIC queues.
# Relay data always stays on the reader, so gate switches and oversized reads cannot move it between NIC queues.
# Direct data retains the adaptive gate and independent sender placement.
# On receive a listener with nothing in flight writes a small batch itself. With multiple routines, inline sends
# still take tunnel reservation credit. 0 hands direct TX and all RX batches to the workers and ordered stages;
# relayed TX and reads too large for a tunnel's whole credit are still sent by the reader.
# tx: false keeps transmit inline and stages only receive.
# tx_slots and rx_slots bound the batches each routine has in flight per direction, minimum 2. A TX slot holds one
# tun read and its ciphertext, about 1.1 MiB; the default is 8, or workers + 2 if more. An RX slot holds one
# recvmmsg batch, listen.batch * 64 KiB with udp_offloads or * 9 KiB without; the default keeps up to 384 receive
# entries in flight in 4 to 6 slots: 6 slots or 24 MiB at the default listen.batch of 64. Lower them to save memory.
#staged:
# workers: 0
# tx: true
# inline_below: 256
# gate_hold: 64
# tx_slots: 8
# rx_slots: 6
# Linux only. pin_threads pins each tun reader/encrypt OS thread to a single CPU. This keeps every goroutine's
# batched sends flowing through one XPS-selected NIC TX ring, so packets within a flow stay ordered on the wire
# instead of being sprayed across multiple TX rings and reordered. Not reloadable. Coerced to false if routines <= 1.
# Staged senders are only pinned with an explicit cpu_affinity, see below; otherwise the scheduler places them.
# Small batches a reader sends itself can use a different TX ring than its sender.
#pin_threads: true
# pin_threads_key helps the CPU-auto-selector shuffle which CPUs are chosen for pinning.
@@ -320,9 +293,7 @@ tun:
# Linux only. cpu_affinity overrides which CPUs the tun reader threads pin to: a list of CPU IDs, one per routine
# (see the top-level `routines` setting). Lists shorter than `routines` are modulo-cycled across the queues; extra
# entries are ignored. Staged sender i pins to entry i+1, modulo-cycled the same way, unless that is its reader's
# CPU, as with a one-entry list: a sender never shares its reader's CPU and stays unpinned instead.
# IDs must be within the process's allowed CPU set, so this respects taskset / cgroup cpusets;
# entries are ignored. IDs must be within the process's allowed CPU set, so this respects taskset / cgroup cpusets;
# a non-integer or not-allowed entry disables the override, leaving the default pin selection described below.
# Only meaningful while pin_threads is true. Not reloadable.
# When unset (or rejected), the default spread prefers performance cores on heterogeneous CPUs (ARM big.LITTLE,
-123
View File
@@ -1,123 +0,0 @@
package nebula
import (
"log/slog"
"runtime"
"sync"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/noiseutil"
)
// Staged processing (tun.staged.workers) moves crypto off each tun reader and udp listener onto worker goroutines,
// behind which an ordered stage sends or writes every batch in read order. It reuses the inline packet handlers:
// staging changes ownership and scheduling, not policy. Each routine has its own workers and ordered stages, so order
// is per routine. Full queues block the reader. A read error drains the batches in flight, while shutdown waits for
// the crypto and then drops packets whose destination has closed. A TX slot holds one tun read and its ciphertext,
// about 1.1 MiB; there are eight, or two more than the workers. An RX slot holds one recvmmsg batch, listen.batch
// times 64 KiB with GRO, and the count keeps up to 384 entries in flight in four to six slots: six and 24 MiB at
// listen.batch=64.
// tun.staged.tx_slots and rx_slots override the counts. Small-MSS TX arenas can grow.
// Relay data stays on the tun reader, regardless of the TX gate, so it never switches between reader and sender
// NIC queues. Direct data keeps its adaptive placement; relay and direct batches both share tunnel counter credit.
//
// Metrics are registered only for active pipelines. flow.{tx,rx}.batches counts reads handed to the ordered stage and
// flow.{tx,rx}.inline those the reader handled itself (TX inline also counts oversized reads); their rates show
// whether staging is used. flow.tx.gate_switches counts transmit switches between inline and fanout mode.
// stagedConfig is tun.staged. workers is per routine and direction; zero keeps processing inline.
type stagedConfig struct {
workers int
tx bool
rx bool
// inlineBelow is the TX segment or RX packet count below which a read may skip the worker handoff, 0 never: a TX
// read in inline mode, an RX read with nothing in flight
inlineBelow int
// txSlots and rxSlots are how many batches each routine can have in flight per direction
txSlots, rxSlots int
// gateHold is how many TX reads in a row must want the other mode before a routine switches between inline and
// fanout transmit
gateHold int
}
// stagedGateHold is the default tun.staged.gate_hold
const stagedGateHold = 64
const (
// stagedTxSlots is the default TX slot count, unless the workers need more
stagedTxSlots = 8
// stagedRxEntries is how many recvmmsg entries the RX slots hold in flight by default
stagedRxEntries = 6 * 64
// stagedMinSlots lets the reader fill one batch while another is in flight
stagedMinSlots = 2
// stagedMinRxSlots keeps the default RX pipeline deep enough for large listen.batch values
stagedMinRxSlots = 4
// stagedMaxRxSlots is the measured default, a smaller listen.batch does not take more slots
stagedMaxRxSlots = 6
)
// stagedSlots sizes the TX and RX slots. TX reads are bounded by the tun queue, so one slot per worker plus the one
// being read and the one being sent, and at least the eight measured. RX slots keep stagedRxEntries in flight, four
// to six, so a larger listen.batch means fewer slots and RX memory never exceeds the six measured. A positive override replaces either, down to stagedMinSlots.
func stagedSlots(workers, batch, txOverride, rxOverride int) (tx, rx int) {
tx = max(stagedTxSlots, workers+2)
rx = min(max(stagedMinRxSlots, (stagedRxEntries+batch-1)/max(batch, 1)), stagedMaxRxSlots)
if txOverride > 0 {
tx = max(txOverride, stagedMinSlots)
}
if rxOverride > 0 {
rx = max(rxOverride, stagedMinSlots)
}
return tx, rx
}
// parseStagedConfig enables staging on Linux only; queues and conns that cannot hand off batches also stay inline.
func parseStagedConfig(c *config.C, l *slog.Logger) stagedConfig {
sc := stagedConfig{workers: c.GetInt("tun.staged.workers", 0)}
if sc.workers <= 0 {
return stagedConfig{}
}
if runtime.GOOS != "linux" {
l.Warn("tun.staged is only supported on Linux; packet processing stays inline")
return stagedConfig{}
}
sc.tx, sc.rx = c.GetBool("tun.staged.tx", true), true
// 256 is the measured crossover, 0 hands every batch to the workers
sc.inlineBelow = max(c.GetInt("tun.staged.inline_below", 256), 0)
if sc.tx && noiseutil.EncryptLockNeeded {
l.Warn("tun.staged cannot encrypt in parallel with boringcrypto or fips140; transmit stays inline")
sc.tx = false
}
sc.gateHold = max(c.GetInt("tun.staged.gate_hold", stagedGateHold), 1)
sc.txSlots, sc.rxSlots = stagedSlots(sc.workers, c.GetInt("listen.batch", 64),
c.GetInt("tun.staged.tx_slots", 0), c.GetInt("tun.staged.rx_slots", 0))
l.Info("Staged packet processing enabled", "workers", sc.workers, "tx", sc.tx, "rx", sc.rx,
"inlineBelow", sc.inlineBelow, "gateHold", sc.gateHold,
"txSlots", sc.txSlots, "rxSlots", sc.rxSlots)
return sc
}
// flowJob is one batch of crypto work. It owns its packet buffers until run returns; jobs and queues are reused.
type flowJob interface{ run(nb []byte) }
// flowPool runs one reader's crypto jobs. The reader is the only producer and closes the pool once its ordered stage
// has drained.
type flowPool struct {
jobs chan flowJob
wg sync.WaitGroup
}
func newFlowPool(workers, queueLen int) *flowPool {
p := &flowPool{jobs: make(chan flowJob, queueLen)}
for range workers {
p.wg.Go(func() {
nb := make([]byte, 12)
for job := range p.jobs {
job.run(nb)
}
})
}
return p
}
func (p *flowPool) close() { close(p.jobs); p.wg.Wait() }
-177
View File
@@ -1,177 +0,0 @@
package nebula
import (
"sync/atomic"
"time"
"github.com/slackhq/nebula/overlay/tio"
)
// txBudget is the reservation credit of one tunnel, shared by every staged reader sending on it. The low 24 bits of
// state are outstanding counters, the next 24 spent credit, and the top 16 the sole ordered reader (q+1); owner zero
// means shared. Readers beyond the owner field's range always take the conservative shared path.
//
// A cap on outstanding counters alone is not enough: a fast reader could repeatedly send and reacquire credit,
// overtaking a paused reader by arbitrarily far. So once readers overlap, or another reader is blocked, spent credit
// is only recycled when ALL reservations have drained. A sole reader instead gets each ordered completion refunded,
// its later batches cannot overtake its earlier ones. All transitions are one CAS, without locks.
//
// The credit is half the peer's replay window. The other half is left for uncredited sends and transport reordering:
// inline and control traffic, and relay forwarding through SendVia, which takes counters on the relay tunnel without
// credit. Unlimited bypass traffic or network delay can still exceed it. The window is not negotiated, so
// ci.window.length, from the build-time ReplayWindow, is also the assumed peer window.
type txBudget struct{ state atomic.Uint64 }
const txBudgetMask = uint64(1<<24 - 1)
// Fail compilation for unsupported build-time windows: a power of two, at least 8192, fitting the credit fields (up
// to 16777216).
const (
_ = uint64(txBudgetMask) - ReplayWindow/2
_ = uint64(ReplayWindow) - 8192
_ = uint64(0) - (ReplayWindow & (ReplayWindow - 1))
)
func txBudgetOwner(q int) uint64 {
if q < 0 || q >= 65535 {
return 0
}
return uint64(q + 1)
}
func (b *txBudget) acquire(n, limit, owner uint64) bool {
for {
old := b.state.Load()
pending, spent, who := old&txBudgetMask, (old>>24)&txBudgetMask, old>>48
if n > limit || spent > limit-n {
// A blocked reader ends sole ownership so ordered completions stop refunding credit. Otherwise a busy
// owner could keep a larger waiter blocked forever without ever sharing a reservation.
if who != 0 && who != owner && !b.state.CompareAndSwap(old, old&^(uint64(0xffff)<<48)) {
continue
}
return false
}
if pending == 0 {
who = owner
} else if who != owner {
who = 0
}
next := who<<48 | (spent+n)<<24 | (pending + n)
if b.state.CompareAndSwap(old, next) {
return true
}
}
}
// release runs only after the socket flush or an explicit drop. A rollback returns credit before any message counter
// was allocated, as when a later tunnel of a relay or multi-tunnel batch could not acquire its own.
func (b *txBudget) release(n uint64, rollback bool) {
for {
old := b.state.Load()
pending, spent, owner := old&txBudgetMask, (old>>24)&txBudgetMask, old>>48
pending -= n
if rollback || owner != 0 {
spent -= n
}
next := owner<<48 | spent<<24 | pending
if pending == 0 {
next = 0
}
if b.state.CompareAndSwap(old, next) {
return
}
}
}
func (ci *ConnectionState) txBudgetLimit() uint64 { return ci.window.length / 2 }
func (r *txReserver) oversized() bool {
for _, t := range r.tunnels {
if t.n > t.ci.txBudgetLimit() {
return true
}
}
return false
}
func (r *txReserver) tryBudget() bool {
for i, t := range r.tunnels {
if !t.ci.txBudget.acquire(t.n, t.ci.txBudgetLimit(), r.owner) {
for _, acquired := range r.tunnels[:i] {
acquired.ci.txBudget.release(acquired.n, true)
}
return false
}
}
return true
}
// waitBudget acquires credit for every tunnel of the batch, never holding a partial acquisition while it waits. Each
// reader first flushes its own oldest batch, flushes need no credit; with none, another reader holds all outstanding
// work and can flush it independently. So there is no circular wait, even with relay tunnels acquired in opposite
// orders. The sleep is only for readers with no local work, not the uncontended hot path. Acquisition is
// all-or-nothing within each direct or relay batch, so pressure on one tunnel also delays packets for idle tunnels
// in that batch. Reader fairness is not guaranteed.
func (p *stagedTx) waitBudget(r *txReserver) bool {
if r.tryBudget() {
return true
}
for {
if p.f.closed.Load() {
return false
}
if !p.flushOldest() {
time.Sleep(50 * time.Microsecond)
}
if r.tryBudget() {
return true
}
}
}
func (s *txSlot) releaseBudget() {
for _, t := range s.reservations {
t.ci.txBudget.release(t.n, false)
}
clear(s.reservations)
s.reservations = s.reservations[:0]
}
// sendLarge handles the unusual reads that exceed a whole tunnel's credit, such as tiny-MSS superpackets. It drains
// the pipeline, then segments and sends them synchronously with credit for each segment, never exceeding the cap and
// never filtering or running connection manager side effects twice. This deliberately slow path avoids a lone-batch
// exception to the shared invariant.
func (p *stagedTx) sendLarge(r *txReserver, items []txItem) {
for p.flushOldest() {
}
for _, it := range items {
err := tio.SegmentSuperpacket(it.pkt, func(seg []byte) error {
if p.f.closed.Load() {
return nil
}
part := it
part.pkt, part.segs = tio.Packet{Bytes: seg}, 1
r.tunnels = r.tunnels[:0]
part.counter = r.need(part.hostinfo, part.ci, 1)
if part.relayHI != nil {
part.relayCounter = r.need(part.relayHI, part.relayHI.ConnectionState, 1)
}
if !p.waitBudget(r) {
return nil
}
one := [1]txItem{part}
if reserved := r.reserve(one[:]); len(reserved) != 0 {
p.f.encryptTxItem(&reserved[0], p.nb, p.sb)
p.f.flushSendBatch(p.sb, p.q)
}
for _, t := range r.tunnels {
t.ci.txBudget.release(t.n, false)
}
return nil
})
if err != nil {
it.hostinfo.logger(p.f.l).Error("Failed to segment superpacket for send", "error", err)
}
}
clear(items)
}
-104
View File
@@ -1,104 +0,0 @@
package nebula
import (
"fmt"
"testing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/require"
)
var budgetWindows = []uint64{8192, 65536}
// TestTXBudgetOvertaking: a fast sender must not recycle credit around a stalled older reservation. A pure in-flight
// count passes the cap assertion but fails this replay test.
func TestTXBudgetOvertaking(t *testing.T) {
for _, window := range budgetWindows {
t.Run(fmt.Sprint(window), func(t *testing.T) {
var b txBudget
limit := window / 2
w := NewBits(window)
l := test.NewLogger()
require.True(t, b.acquire(1, limit, 1)) // counter 1 stalls
for counter := uint64(2); counter <= limit; counter++ {
require.True(t, b.acquire(1, limit, 2))
require.True(t, w.Update(l, counter))
b.release(1, false)
}
require.Equal(t, uint64(1), b.state.Load()&txBudgetMask)
require.False(t, b.acquire(1, limit, 2), "spent credit cannot be recycled while the oldest sender stalls")
// The other half of the window allows this many unreserved sends.
for counter := limit + 1; counter <= window; counter++ {
require.True(t, w.Update(l, counter))
}
require.True(t, w.Update(l, 1), "oldest reservation must still fit after using the whole margin")
b.release(1, false)
require.Zero(t, b.state.Load())
require.True(t, b.acquire(limit, limit, 2))
require.False(t, b.acquire(1, limit, 2))
b.release(limit, false)
})
}
}
// TestTXBudgetBlockedReader models eight 512-segment reads of a busy owner and a 568-segment waiter, without socket
// timing. Alone, the owner gets each completion refunded, so it keeps eight batches in flight forever, even across its
// own failed acquisitions. But even when the owner always retries first, its completions must stop refunding credit
// once the waiter is blocked, and let the budget drain. Waiter 0 is the conservative ID of readers beyond the owner
// field's range.
func TestTXBudgetBlockedReader(t *testing.T) {
for _, waiter := range []uint64{1, 0} {
t.Run(fmt.Sprintf("waiter=%d", waiter), func(t *testing.T) {
var b txBudget
const limit, n, want = uint64(4096), uint64(512), uint64(568)
for range 8 {
require.True(t, b.acquire(n, limit, 2))
}
for range 100 {
require.False(t, b.acquire(n, limit, 2))
b.release(n, false) // the sole sender flushes its oldest batch
require.True(t, b.acquire(n, limit, 2))
require.Equal(t, 8*n, b.state.Load()&txBudgetMask)
}
require.False(t, b.acquire(want, limit, waiter))
for range 7 {
b.release(n, false)
require.False(t, b.acquire(n, limit, 2), "blocked reader must stop the owner's credit recycling")
require.False(t, b.acquire(want, limit, waiter))
}
b.release(n, false)
require.Zero(t, b.state.Load())
// Even if the former owner wins the next acquisition, both now fit.
require.True(t, b.acquire(n, limit, 2))
require.True(t, b.acquire(want, limit, waiter))
b.release(n, false)
b.release(want, false)
require.Zero(t, b.state.Load())
})
}
}
// TestTXBudgetRelayRollback: when a later tunnel of a batch cannot acquire, the earlier ones give their credit back,
// spent credit included.
func TestTXBudgetRelayRollback(t *testing.T) {
for _, window := range budgetWindows {
inner := &ConnectionState{window: NewBits(window)}
outer := &ConnectionState{window: NewBits(window)}
limit := inner.txBudgetLimit()
require.True(t, inner.txBudget.acquire(10, limit, 1))
require.True(t, outer.txBudget.acquire(limit, limit, 1))
r := txReserver{owner: 2, tunnels: []txTunnel{{ci: inner, n: 20}, {ci: outer, n: 20}}}
for range 100 {
require.False(t, r.tryBudget())
require.Equal(t, uint64(10), inner.txBudget.state.Load()&txBudgetMask)
require.Equal(t, uint64(10), (inner.txBudget.state.Load()>>24)&txBudgetMask, "rollback must refund spent credit too")
}
outer.txBudget.release(limit, false)
require.True(t, r.tryBudget())
inner.txBudget.release(20, false)
outer.txBudget.release(20, false)
inner.txBudget.release(10, false)
require.Zero(t, inner.txBudget.state.Load())
require.Zero(t, outer.txBudget.state.Load())
}
}
-135
View File
@@ -1,135 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
import (
"fmt"
"testing"
"time"
"github.com/flynn/noise"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
// gated is staging with an inline threshold of 4 segments and the given gate hold.
func gated(hold int) stagedConfig {
sc := adaptive(2)
sc.inlineBelow, sc.gateHold = 4, hold
return sc
}
// gateCounters snapshots the transmit counters a gate test checks.
type gateCounters struct{ inline, batches, switches int64 }
func readGateCounters() gateCounters {
return gateCounters{
inline: metrics.GetOrRegisterCounter("flow.tx.inline", nil).Count(),
batches: metrics.GetOrRegisterCounter("flow.tx.batches", nil).Count(),
switches: metrics.GetOrRegisterCounter("flow.tx.gate_switches", nil).Count(),
}
}
func (c gateCounters) since(c0 gateCounters) gateCounters {
return gateCounters{c.inline - c0.inline, c.batches - c0.batches, c.switches - c0.switches}
}
// gateReads builds reads of small (3 segments, below the threshold) or large (2 superpackets of 9) shape, in order,
// for one flow, and the segments they carry.
func gateReads(large ...bool) ([][]tio.Packet, []decodedSeg) {
var reads [][]tio.Packet
var want []decodedSeg
seq := uint32(0)
for _, l := range large {
per, segs := 1, 3
if l {
per, segs = 2, 9
}
var b []tio.Packet
for range per {
b = append(b, flowTCPSuperpacket(0, seq, segs))
for s := range segs {
want = append(want, decodedSeg{seq: seq, idx: uint16(s)})
}
seq++
}
reads = append(reads, b)
}
return reads, want
}
func repeatBool(v bool, n int) []bool {
out := make([]bool, n)
for i := range out {
out[i] = v
}
return out
}
// TestFlowStagedTXGateSwitch forces the gate from inline to fanout and back, with a slow socket and slow
// workers so the fanout batches are still in flight when the small reads arrive: wire order is read order, counters
// are contiguous, and each switch happens on exactly the gate_hold-th read that wants it.
func TestFlowStagedTXGateSwitch(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
const hold = 3
for _, routines := range []int{1, 2} {
t.Run(fmt.Sprintf("routines=%d", routines), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, gated(hold))
sender.f.routines = routines
sender.f.writers = []udp.Conn{sender.conn, sender.conn}
delayCrypto(sender, func(n uint64) {
if n%7 == 0 {
time.Sleep(time.Millisecond)
}
})
sender.conn.delay = 4 * time.Millisecond
// 4 small inline, 2 large held inline, 6 large fanout from the switch on, 2 small held in fanout, then
// the switch back drains and 3 small go inline
shape := append(repeatBool(false, 4), repeatBool(true, 8)...)
shape = append(shape, repeatBool(false, 5)...)
reads, want := gateReads(shape...)
ci := sender.hostinfo.ConnectionState
start := ci.messageCounter.Load()
c0 := readGateCounters()
p := newStagedTx(sender.f, newFlowBatchQueue(nil), 0)
for i, r := range reads {
p.send(r)
if i == 13 {
require.Positive(t, p.inFlight, "fanout batches must still be in flight when the small reads start")
}
if i == 14 {
require.Zero(t, p.inFlight, "switching to inline drains the pipeline")
}
}
p.close()
require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured()), "wire order must match read order")
require.Equal(t, start+uint64(len(want)), ci.messageCounter.Load(), "every reserved counter is used once")
require.Equal(t, gateCounters{inline: 4 + 2 + 3, batches: 6 + 2, switches: 2}, readGateCounters().since(c0))
require.Zero(t, ci.txBudget.state.Load())
})
}
}
// TestFlowStagedTXGateHold: the gate never switches before gate_hold consecutive reads want the other mode, and
// a read that wants the current mode restarts the count.
func TestFlowStagedTXGateHold(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
const hold = 5
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, gated(hold))
shape := append(repeatBool(true, hold-1), false)
shape = append(shape, repeatBool(true, hold-1)...)
reads, want := gateReads(append(shape, true)...)
c0 := readGateCounters()
p := newStagedTx(sender.f, newFlowBatchQueue(nil), 0)
for _, r := range reads[:len(reads)-1] {
p.send(r)
}
require.Equal(t, gateCounters{inline: 2*hold - 1}, readGateCounters().since(c0), "no switch before gate_hold")
p.send(reads[len(reads)-1])
require.Equal(t, gateCounters{inline: 2*hold - 1, batches: 1, switches: 1}, readGateCounters().since(c0))
p.close()
require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured()))
}
-371
View File
@@ -1,371 +0,0 @@
package nebula
import (
"encoding/binary"
"net/netip"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
)
// handoffConn is a udp conn that can hand each received batch over. The callback returns replacement buffers with
// the same count and at least the same lengths.
type handoffConn interface {
ListenOutHandoff(udp.EncReader, func([][]byte) [][]byte) error
}
const rxChunkMin = 16
// rxItem is one received packet. A data packet for an established tunnel has ci set; out and err are written by
// whoever decrypts it.
type rxItem struct {
packet []byte
hostinfo *HostInfo
ci *ConnectionState
counter uint64
via netip.AddrPort
out []byte
err error
}
// rxChunk is a contiguous run of a batch's items for one worker to decrypt.
type rxChunk struct {
items []rxItem
done chan struct{}
}
func (c *rxChunk) run(nb []byte) {
decryptRxItems(c.items, nb)
c.done <- struct{}{}
}
// decryptRxItems decrypts the data packets among items in place. The replay window is left to the ordered stage, as
// are packets without a tunnel and the known replays precheck marked.
func decryptRxItems(items []rxItem, nb []byte) {
for i := range items {
it := &items[i]
if it.ci == nil || it.err != nil {
continue
}
it.out, it.err = it.ci.decryptPacket(it.counter, it.packet, nb)
}
}
// rxBatch is one recvmmsg batch moving through the staged receive pipeline.
type rxBatch struct {
// items is every packet of the batch in arrival order. Data packets are decrypted by a worker; anything else is
// handled whole by the writer, in its place.
items []rxItem
data int
// bufs is the set of recvmmsg buffers the items point into, owned by the batch from handoff until it is reused.
bufs [][]byte
// chunks are the decrypt jobs, nChunks of them in use. With none the writer decrypts the batch itself.
chunks []rxChunk
nChunks int
}
func (b *rxBatch) reset() {
clear(b.items)
b.items = b.items[:0]
b.data = 0
for i := range b.nChunks {
b.chunks[i].items = nil
}
b.nChunks = 0
}
// stagedRx is the staged receive pipeline of one udp listener. deliver and handoff run on the listener goroutine.
type stagedRx struct {
f *Interface
q int
pool *flowPool
// h and hostmapCache are the listener's own. wrxc is the commit context: the writer's, or the listener's while
// nothing is in flight.
h *header.H
hostmapCache map[uint32]*HostInfo
wrxc *rxContext
cur *rxBatch
free []*rxBatch
ordered chan *rxBatch
returned chan *rxBatch
inFlight int
writerDone chan struct{}
metricBatches metrics.Counter
metricInline metrics.Counter
}
func newStagedRx(f *Interface, q int, wrxc *rxContext) *stagedRx {
p := &stagedRx{
f: f,
q: q,
h: &header.H{},
hostmapCache: map[uint32]*HostInfo{},
wrxc: wrxc,
ordered: make(chan *rxBatch, f.staged.rxSlots),
returned: make(chan *rxBatch, f.staged.rxSlots),
writerDone: make(chan struct{}),
metricBatches: metrics.GetOrRegisterCounter("flow.rx.batches", nil),
metricInline: metrics.GetOrRegisterCounter("flow.rx.inline", nil),
}
workers := f.staged.workers
p.pool = newFlowPool(workers, f.staged.rxSlots*2*workers)
for range f.staged.rxSlots {
// Two chunks per worker, so the writer can start committing while the workers are still on the rest
b := &rxBatch{items: make([]rxItem, 0, 256), chunks: make([]rxChunk, 2*workers)}
for i := range b.chunks {
b.chunks[i].done = make(chan struct{}, 1)
}
p.free = append(p.free, b)
}
p.cur = p.free[len(p.free)-1]
p.free = p.free[:len(p.free)-1]
go p.writer()
return p
}
// listen runs the conn's receive loop through the pipeline until it fails, then drains the pipeline. The conn hands
// its buffers over, so a batch moves between stages without a copy.
func (p *stagedRx) listen(li handoffConn) error {
defer p.close()
return li.ListenOutHandoff(p.deliver, p.handoff)
}
// deliver is the listener's per-packet work: classify the packet and add it to the current batch. Nothing here
// decrypts or touches the tun.
func (p *stagedRx) deliver(via netip.AddrPort, packet []byte) {
b := p.cur
it := rxItem{packet: packet, via: via}
p.classify(&it)
if it.ci != nil {
b.data++
}
b.items = append(b.items, it)
}
// classify fills in the tunnel for a plain data packet of an established, non-relayed tunnel, leaving anything else
// for the writer to handle whole. The replay pre-check waits for the end of the batch, see precheck.
func (p *stagedRx) classify(it *rxItem) {
f := p.f
h := p.h
packet := it.packet
if h.Parse(packet) != nil || h.Version != header.Version || h.Type != header.Message || h.Subtype != header.MessageNone {
return
}
if f.myVpnNetworksTable.Contains(it.via.Addr()) {
return
}
hostinfo := f.hostMap.QueryIndexCached(h.RemoteIndex, p.hostmapCache)
if hostinfo == nil || hostinfo.ConnectionState == nil {
return
}
ci := hostinfo.ConnectionState
if len(packet) < header.Len+ci.dKey.Overhead() {
return
}
it.hostinfo, it.ci, it.counter = hostinfo, ci, h.MessageCounter
}
// precheck marks obvious replays before anything decrypts them, like ConnectionState.Decrypt does, taking each
// tunnel's lock once per run of its packets rather than once per packet: the writer takes the same lock for its
// window updates, from another core. The writer checks again when it commits. The window only moves forward in
// between, so a stale check here can only let a replay through to that second check. Marked packets keep their
// place: an earlier RecvError may delete their tunnel, and inline receive then replies with a RecvError rather than
// dropping a replay.
func (p *stagedRx) precheck(b *rxBatch) {
f := p.f
items := b.items
for i := 0; i < len(items); {
ci := items[i].ci
if ci == nil {
i++
continue
}
ci.decryptLock.Lock()
for ; i < len(items) && items[i].ci == ci; i++ {
if ci.window.Check(f.l, items[i].counter) {
continue
}
b.data--
items[i].err = ErrAlreadySeen
}
ci.decryptLock.Unlock()
}
}
// handoff ends a batch: the batch takes bufs, and the conn reads the next batch into the buffers of the next free
// batch.
func (p *stagedRx) handoff(bufs [][]byte) [][]byte {
p.cur.bufs = bufs
p.endBatch()
b := p.cur
if len(b.bufs) != len(bufs) || len(b.bufs[0]) < len(bufs[0]) {
b.bufs = make([][]byte, len(bufs))
for i := range b.bufs {
b.bufs[i] = make([]byte, len(bufs[0]))
}
}
return b.bufs
}
// endBatch hands the batch to the workers and the ordered writer, then takes a free batch.
func (p *stagedRx) endBatch() {
clear(p.hostmapCache)
p.reap()
b := p.cur
p.precheck(b)
n := len(b.items)
if n == 0 {
b.reset()
return
}
if p.inFlight == 0 && n < p.f.staged.inlineBelow {
// The writer is idle with nothing in flight, so the listener may commit with its context and flush the tun
p.metricInline.Inc(1)
p.process(b, p.wrxc)
b.reset()
return
}
parts := min(len(b.chunks), b.data/rxChunkMin)
p.metricBatches.Inc(1)
if parts > 0 {
size := (n + parts - 1) / parts
for off := 0; off < n; off += size {
c := &b.chunks[b.nChunks]
c.items = b.items[off:min(off+size, n)]
b.nChunks++
}
}
// The jobs go out before the batch does: once the writer has it, it may finish and reset it at any moment
for i := range b.nChunks {
p.pool.jobs <- &b.chunks[i]
}
p.inFlight++
p.ordered <- b
if len(p.free) == 0 {
// Every batch is in flight, wait for the writer: the socket buffer absorbs the backlog meanwhile.
p.release(<-p.returned)
}
p.cur = p.free[len(p.free)-1]
p.free = p.free[:len(p.free)-1]
}
// reap takes back every batch the writer has finished with, without waiting.
func (p *stagedRx) reap() {
for {
select {
case b := <-p.returned:
p.release(b)
default:
return
}
}
}
func (p *stagedRx) release(b *rxBatch) {
p.inFlight--
p.free = append(p.free, b)
}
// process is the ordered stage for one batch: decrypt whatever no worker is decrypting, commit every item in arrival
// order, and flush the tun batcher.
func (p *stagedRx) process(b *rxBatch, rxc *rxContext) {
if b.nChunks == 0 {
decryptRxItems(b.items, rxc.nb)
p.commit(b.items, rxc)
} else {
// Commit chunk by chunk, so the commit overlaps the workers still on later chunks
for i := range b.nChunks {
c := &b.chunks[i]
<-c.done
p.commit(c.items, rxc)
}
}
p.f.flushOutside(rxc)
}
// commit runs the ordered stage over items in arrival order. A non-data packet goes through readOutsidePackets in
// its place, a run of data packets on one tunnel through commitRun.
func (p *stagedRx) commit(items []rxItem, rxc *rxContext) {
for i := 0; i < len(items); {
ci := items[i].ci
if ci == nil {
p.f.readOutsidePackets(ViaSender{UdpAddr: items[i].via}, items[i].packet, rxc)
i++
continue
}
j := i + 1
for j < len(items) && items[j].ci == ci {
j++
}
p.commitRun(items[i:j], rxc)
i = j
}
}
// commitRun commits a run of decrypted data packets on one tunnel. The window updates go first, under one lock, then
// the rest; handling a data packet never touches the window, so that is the same as one packet at a time.
func (p *stagedRx) commitRun(run []rxItem, rxc *rxContext) {
f := p.f
ci := run[0].ci
// Classification is speculative: earlier control packets may have removed this tunnel. Resolve it through the
// writer's batch-local cache, exactly where inline processing would, before touching replay state.
index := binary.BigEndian.Uint32(run[0].packet[4:8])
hostinfo := f.hostMap.QueryIndexCached(index, rxc.hostmapCache)
if hostinfo != run[0].hostinfo || hostinfo.ConnectionState != ci {
if hostinfo == nil {
for i := range run {
f.maybeSendRecvError(run[i].via, index)
}
}
return
}
ci.decryptLock.Lock()
for i := range run {
// A packet that failed to decrypt does not touch the window, like ConnectionState.Decrypt
if run[i].err == nil && !ci.window.Update(f.l, run[i].counter) {
run[i].err = ErrAlreadySeen
}
}
ci.decryptLock.Unlock()
for i := range run {
it := &run[i]
if f.finishOutsideDecrypt(it.hostinfo, ViaSender{UdpAddr: it.via}, it.counter, it.err) {
f.handleOutsideMessagePacket(it.hostinfo, it.counter, it.out, rxc)
}
}
}
// writer is the ordered receive stage: window, firewall, coalescer and tun write, one batch at a time in arrival
// order.
func (p *stagedRx) writer() {
defer close(p.writerDone)
for b := range p.ordered {
if p.f.closed.Load() {
// The tun is going away, a write would only log errors. Still wait out the workers on the batch.
for i := range b.nChunks {
<-b.chunks[i].done
}
} else {
p.process(b, p.wrxc)
}
b.reset()
p.returned <- b
}
}
// close commits anything still in flight, in order, and stops the writer and the workers.
func (p *stagedRx) close() {
if len(p.cur.items) > 0 {
p.endBatch()
}
close(p.ordered)
<-p.writerDone
p.pool.close()
}
-237
View File
@@ -1,237 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
import (
"fmt"
"net/netip"
"sync/atomic"
"testing"
"time"
"github.com/flynn/noise"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
// TestFlowStagedRXOrderAndReplay feeds every packet behind a forgery of it, from another address, and a replay of
// it, then a test packet that jumps the window and the whole stream again, with shuffled workers and a slow tun,
// through small and large batches: the tun sees each segment once, in order, the window marks them all, the forgeries
// neither consume a counter nor roam, and the jump does not get the earlier data packets of its batch rejected.
func TestFlowStagedRXOrderAndReplay(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
for _, batchLen := range []int{8, 1000} {
t.Run(fmt.Sprintf("%s/batch=%d", cipher.CipherName(), batchLen), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, cipher, stagedOn(4))
delayCrypto(receiver, func(n uint64) {
if n%32 == 0 {
time.Sleep(time.Millisecond)
}
})
receiver.tun.delay = 100 * time.Microsecond
batches, want := flowTestBatches(30, 4, 1, 10)
wire := wireFor(t, sender, batches)
var feed [][]byte
for i, p := range wire {
// Corrupt the tag or the payload
bad := append([]byte(nil), p...)
bad[[]int{len(p) - 1, header.Len + 3}[i%2]] ^= 1
feed = append(feed, bad, p, p)
}
// The jump lands in a batch with fresh data before it, at batch=1000
sender.hostinfo.ConnectionState.messageCounter.Add(2 * ReplayWindow)
feed = append(append(feed, sendControl(sender, header.Test, header.TestReply)), wire...)
forged := func(i int) netip.AddrPort {
if i < 3*len(wire) && i%3 == 0 {
return netip.MustParseAddrPort("198.51.100.9:4242")
}
return flowTestUnder
}
rx0 := rxBatches.Count()
conn := &rxConn{wire: feed, batchLen: batchLen, from: forged}
receiver.receive(conn)
require.Positive(t, rxBatches.Count()-rx0)
require.Equal(t, (len(feed)+batchLen-1)/batchLen, conn.swaps, "every batch is handed off")
require.Equal(t, want, receiver.tun.segments(t), "every packet exactly once, in order")
require.Equal(t, flowTestUnder, receiver.hostinfo.GetRemote())
for _, p := range wire {
h := &header.H{}
require.NoError(t, h.Parse(p))
require.False(t, receiver.hostinfo.ConnectionState.window.Check(receiver.f.l, h.MessageCounter), "counter %d must be marked seen", h.MessageCounter)
}
})
}
}
}
// TestFlowStagedRXClose: a CloseTunnel in the middle of a batch is handled in its place, by the writer. The data after
// it in the batch is still delivered from the batch's hostmap cache, as inline receive does.
func TestFlowStagedRXClose(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
batches, want := flowTestBatches(4, 4, 1, 8)
wire := wireFor(t, sender, batches)
half := len(wire) / 2
feed := append(append(append([][]byte{}, wire[:half]...), sendControl(sender, header.CloseTunnel, 0)), wire[half:]...)
receiver.receive(&rxConn{wire: feed, batchLen: len(feed)})
require.Equal(t, want, receiver.tun.segments(t))
require.Nil(t, receiver.f.hostMap.QueryIndex(receiver.hostinfo.localIndexId))
}
// sendControl returns the wire packet of a control message from sender.
func sendControl(sender *flowTestPeer, typ header.MessageType, st header.MessageSubType) []byte {
sender.f.sendNoMetrics(typ, st, sender.hostinfo.ConnectionState, sender.hostinfo, flowTestUnder, nil, make([]byte, 12), make([]byte, mtu), 0)
wire := sender.conn.captured()
sender.conn.pkts, sender.conn.addrs = nil, nil
return wire[len(wire)-1]
}
// receivePaths is inline, staged and, with the threshold, adaptive receive.
func receivePaths() []stagedConfig {
return []stagedConfig{{}, stagedOn(2), adaptive(2)}
}
// TestFlowStagedRXRecvError feeds a RecvError that deletes the tunnel, then a known replay and new data for it, in one
// batch. Every receive path must drop the data through its own view of the hostmap, without touching the replay window,
// and answer each packet with a RecvError, as inline receive does, without decrypting the replay again.
func TestFlowStagedRXRecvError(t *testing.T) {
for _, sc := range receivePaths() {
t.Run(stagedName(sc), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
receiver.f.messageMetrics = newMessageMetrics()
receiver.f.acceptRecvErrorConfig = recvErrorAlways
receiver.f.sendRecvErrorConfig = recvErrorAlways
batches, want := flowTestBatches(1, 1, 1, 33)
wire := wireFor(t, sender, batches)
rxc := newRxContext(receiver.f, 0)
receiver.f.readOutsidePackets(ViaSender{UdpAddr: flowTestUnder}, append([]byte(nil), wire[0]...), rxc)
receiver.f.flushOutside(rxc)
h := &header.H{}
require.NoError(t, h.Parse(wire[0]))
replay := h.MessageCounter
var decrypts atomic.Int64
delayCrypto(receiver, func(n uint64) {
if n == replay {
decrypts.Add(1)
}
})
ctl := header.Encode(make([]byte, header.Len), header.Version, header.RecvError, 0, receiver.hostinfo.remoteIndexId, 0)
conn := &rxConn{wire: append([][]byte{ctl}, wire...), batchLen: len(wire) + 1}
receiver.receive(conn)
require.Nil(t, receiver.f.hostMap.QueryIndex(receiver.hostinfo.localIndexId))
require.Equal(t, want[:1], receiver.tun.segments(t))
require.Zero(t, decrypts.Load(), "known replays must not be decrypted again")
replies := conn.captured()
require.Len(t, replies, len(wire))
for i, reply := range replies {
require.NoError(t, h.Parse(reply))
require.Equal(t, header.RecvError, h.Type)
require.NoError(t, h.Parse(wire[i]))
require.True(t, i == 0 || receiver.hostinfo.ConnectionState.window.Check(nil, h.MessageCounter), "dropped data must not update replay state")
}
})
}
}
// TestFlowStagedRXReplacedTunnel classifies a data packet, then a close and a new handshake replace its tunnel at the
// same index before the writer commits it, as when the listener runs ahead: it must be dropped untouched.
func TestFlowStagedRXReplacedTunnel(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
receiver.f.messageMetrics = newMessageMetrics()
batches, _ := flowTestBatches(1, 1, 1, 1)
wire := wireFor(t, sender, batches)
closePacket := sendControl(sender, header.CloseTunnel, 0)
p := &stagedRx{f: receiver.f, h: &header.H{}, hostmapCache: map[uint32]*HostInfo{}}
item := rxItem{packet: wire[0], via: flowTestUnder}
p.classify(&item)
require.NotNil(t, item.ci)
rxc := newRxContext(receiver.f, 0)
p.process(&rxBatch{items: []rxItem{{packet: closePacket, via: flowTestUnder}}}, rxc)
require.Nil(t, receiver.f.hostMap.QueryIndex(receiver.hostinfo.localIndexId))
_, replacement := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
receiver.f.hostMap.Lock()
receiver.f.hostMap.unlockedAddHostInfo(replacement.hostinfo, receiver.f)
receiver.f.hostMap.Unlock()
p.process(&rxBatch{items: []rxItem{item}}, rxc)
require.Empty(t, receiver.tun.writes)
require.True(t, item.ci.window.Check(nil, item.counter))
require.True(t, replacement.hostinfo.ConnectionState.window.Check(nil, item.counter))
}
// TestFlowStagedRXRealSocket exercises buffer handoff through the real Linux UDP receive loop.
func TestFlowStagedRXRealSocket(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(3))
batches, want := flowTestBatches(60, 4, 1, 8)
wire := wireFor(t, sender, batches)
l := receiver.f.l
conn, err := udp.NewListener(l, udp.Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Batch: 16, Offloads: true})
require.NoError(t, err)
addr, err := conn.LocalAddr()
require.NoError(t, err)
out, err := udp.NewListener(l, udp.Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Batch: 16, Offloads: true})
require.NoError(t, err)
defer out.Close()
// The sender's address is the tunnel's remote, so no roaming
from, err := out.LocalAddr()
require.NoError(t, err)
receiver.hostinfo.remote.Store(&from)
done := make(chan struct{})
go func() { receiver.receive(conn); close(done) }()
// Send in bursts small enough for the default socket buffer
for off := 0; off < len(wire); off += 32 {
end := min(off+32, len(wire))
addrs := make([]netip.AddrPort, end-off)
for i := range addrs {
addrs[i] = addr
}
_, err := out.WriteBatch(wire[off:end], addrs)
require.NoError(t, err)
require.Eventually(t, func() bool { return len(receiver.tun.segments(t)) >= end }, 5*time.Second, 100*time.Microsecond)
}
receiver.f.closed.Store(true)
require.NoError(t, conn.Close())
<-done
require.Equal(t, want, receiver.tun.segments(t))
}
// TestFlowStagedRXCloseFlushesPartial: packets the listener holds in a batch it never got to end when the conn fails
// are still committed, in order, on a live interface.
func TestFlowStagedRXCloseFlushesPartial(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
batches, want := flowTestBatches(11, 4, 1, 8)
wire := wireFor(t, sender, batches)
p := newStagedRx(receiver.f, 0, newRxContext(receiver.f, 0))
for i, packet := range wire {
p.deliver(flowTestUnder, packet)
if (i+1)%64 == 0 {
p.endBatch()
}
}
p.close()
require.Equal(t, want, receiver.tun.segments(t))
}
// TestFlowStagedRXInlineBelow: small idle batches stay on the listener, larger batches still use the ordered writer.
func TestFlowStagedRXInlineBelow(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sc := adaptive(3)
sc.inlineBelow = 16
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
batches, want := flowTestBatches(40, 4, 1, 16)
wire := wireFor(t, sender, batches)
inline := metrics.GetOrRegisterCounter("flow.rx.inline", nil)
inline0, rx0 := inline.Count(), rxBatches.Count()
receiver.receive(&rxConn{wire: wire, sizes: []int{8, 8, 8, 128, 128, 8, 8, 128, 4, 128}})
require.Equal(t, want, receiver.tun.segments(t))
require.Positive(t, inline.Count()-inline0)
require.Positive(t, rxBatches.Count()-rx0)
}
-536
View File
@@ -1,536 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
import (
"encoding/binary"
"io"
"math"
"net/netip"
"os"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
// flowTestMSS is the segment payload size of the synthetic TSO superpackets.
const flowTestMSS = 1200
var (
flowTestLocal = netip.MustParseAddr("10.0.0.1")
flowTestRemote = netip.MustParseAddr("10.0.0.2")
flowTestUnder = netip.MustParseAddrPort("192.0.2.2:4242")
)
// flowTCPSuperpacket builds an IPv4/TCP TSO superpacket from local to remote carrying segs segments. Segment i's
// payload starts with the flow id, the superpacket seq and i, so a receiver can check what it got and in which order.
func flowTCPSuperpacket(flow uint16, seq uint32, segs int) tio.Packet {
const hdr = 40
pay := segs * flowTestMSS
b := make([]byte, hdr+pay)
b[0] = 0x45
binary.BigEndian.PutUint16(b[2:4], uint16(min(hdr+pay, 65535)))
b[8] = 64
b[9] = iputil.IPProtocolTCP
copy(b[12:16], flowTestLocal.AsSlice())
copy(b[16:20], flowTestRemote.AsSlice())
binary.BigEndian.PutUint16(b[20:22], 10000+flow)
binary.BigEndian.PutUint16(b[22:24], 80)
binary.BigEndian.PutUint32(b[24:28], seq*uint32(pay))
b[32] = 0x50
b[33] = 0x10 // ACK
binary.BigEndian.PutUint16(b[34:36], 65535)
for i := range segs {
p := b[hdr+i*flowTestMSS:]
binary.BigEndian.PutUint16(p[0:2], flow)
binary.BigEndian.PutUint32(p[2:6], seq)
binary.BigEndian.PutUint16(p[6:8], uint16(i))
}
if segs == 1 {
return tio.Packet{Bytes: b}
}
return tio.Packet{Bytes: b, GSO: tio.GSOInfo{Size: flowTestMSS, HdrLen: hdr, CsumStart: 20, Proto: tio.GSOProtoTCP}}
}
// decodedSeg identifies one segment by the marker flowTCPSuperpacket put in its payload.
type decodedSeg struct {
flow uint16
seq uint32
idx uint16
}
func decodeSeg(p []byte) decodedSeg {
return decodedSeg{flow: binary.BigEndian.Uint16(p[0:2]), seq: binary.BigEndian.Uint32(p[2:6]), idx: binary.BigEndian.Uint16(p[6:8])}
}
// flowTestBatches builds n reads of per superpackets of segs segments, across flows interleaved flows, and the
// segments they carry in order.
func flowTestBatches(n, per, flows, segs int) ([][]tio.Packet, []decodedSeg) {
var batches [][]tio.Packet
var want []decodedSeg
seq := make([]uint32, flows)
for range n {
var b []tio.Packet
for j := range per {
flow := uint16(j % flows)
b = append(b, flowTCPSuperpacket(flow, seq[flow], segs))
for s := range segs {
want = append(want, decodedSeg{flow: flow, seq: seq[flow], idx: uint16(s)})
}
seq[flow]++
}
batches = append(batches, b)
}
return batches, want
}
// flowTestPeer is one end of a tunnel: the interface, the hostinfo for the other end, and its capture points.
type flowTestPeer struct {
f *Interface
hostinfo *HostInfo
conn *flowCaptureConn
tun *flowTunCapture
}
// newFlowTestPair builds two interfaces joined by one real tunnel (keys from an actual handshake). sender sends from
// flowTestLocal to flowTestRemote, receiver is the far side.
func newFlowTestPair(t testing.TB, cipher noise.CipherFunc, sc stagedConfig) (sender, receiver *flowTestPeer) {
t.Helper()
initR, respR := runTestHandshakeWith(t, cipher)
sci, err := newConnectionStateFromResult(initR)
require.NoError(t, err)
rci, err := newConnectionStateFromResult(respR)
require.NoError(t, err)
sender = newFlowTestPeer(t, flowTestLocal, flowTestRemote, sci, 100, 200)
receiver = newFlowTestPeer(t, flowTestRemote, flowTestLocal, rci, 200, 100)
sender.f.staged, receiver.f.staged = sc, sc
return sender, receiver
}
func newFlowTestPeer(t testing.TB, me, them netip.Addr, ci *ConnectionState, localIdx, remoteIdx uint32) *flowTestPeer {
l := test.NewLogger()
hm := newHostMap(l)
hm.preferredRanges.Store(&[]netip.Prefix{})
myNets, myAddrs := &bart.Lite{}, &bart.Lite{}
myNets.Insert(netip.PrefixFrom(me, 24))
myAddrs.Insert(netip.PrefixFrom(me, 32))
myCert := &dummyCert{version: cert.Version1, networks: []netip.Prefix{netip.PrefixFrom(me, 24)}}
theirCert := &dummyCert{version: cert.Version1, networks: []netip.Prefix{netip.PrefixFrom(them, 24)}, name: "peer"}
ci.peerCert = &cert.CachedCertificate{Certificate: theirCert, InvertedGroups: map[string]struct{}{}}
fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, myCert)
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
conn, tun := &flowCaptureConn{}, &flowTunCapture{}
lh := newTestLighthouse()
lh.remoteAllowList.Store(&RemoteAllowList{})
f := &Interface{
hostMap: hm,
outside: conn,
firewall: fw,
lightHouse: lh,
pki: &PKI{},
myVpnNetworksTable: myNets,
myVpnAddrsTable: myAddrs,
routines: 1,
writers: []udp.Conn{conn},
messageMetrics: &MessageMetrics{txExhausted: metrics.NewCounter()},
metricTxDropped: metrics.NilCounter{},
l: l,
}
f.ctx = t.Context()
f.pki.caPool.Store(cert.NewCAPool())
// Never promote or requery, those would put extra test packets on the wire
f.tryPromoteEvery.Store(math.MaxUint32)
f.reQueryEvery.Store(math.MaxUint32)
f.batchers = []*batch.MultiCoalescer{batch.NewMultiCoalescer(tun, l)}
f.connectionManager = &connectionManager{
intf: f,
relayUsed: map[uint32]struct{}{},
relayUsedLock: &sync.RWMutex{},
trafficTimer: NewLockingTimerWheel[uint32](time.Second, time.Minute),
}
f.handshakeManager = NewHandshakeManager(l, hm, lh, conn, defaultHandshakeConfig)
f.handshakeManager.f = f
hostinfo := &HostInfo{
ConnectionState: ci,
vpnAddrs: []netip.Addr{them},
localIndexId: localIdx,
remoteIndexId: remoteIdx,
relayState: RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}},
}
hostinfo.remote.Store(&flowTestUnder)
hostinfo.buildNetworks(myNets, theirCert)
hm.unlockedAddHostInfo(hostinfo, f)
return &flowTestPeer{f: f, hostinfo: hostinfo, conn: conn, tun: tun}
}
// flowCaptureConn records every packet sent, in order, as copies.
type flowCaptureConn struct {
udp.NoopConn
mu sync.Mutex
pkts [][]byte
addrs []netip.AddrPort
// delay, when set, slows every WriteBatch down so batches pile up in flight
delay time.Duration
}
func (c *flowCaptureConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
time.Sleep(c.delay)
c.mu.Lock()
defer c.mu.Unlock()
for i, b := range bufs {
c.pkts = append(c.pkts, append([]byte(nil), b...))
c.addrs = append(c.addrs, addrs[i])
}
return len(bufs), nil
}
func (c *flowCaptureConn) WriteTo(b []byte, addr netip.AddrPort) error {
_, err := c.WriteBatch([][]byte{b}, []netip.AddrPort{addr})
return err
}
func (c *flowCaptureConn) captured() [][]byte {
c.mu.Lock()
defer c.mu.Unlock()
return append([][]byte(nil), c.pkts...)
}
// flowTunCapture is a tun queue that records every plaintext write.
type flowTunCapture struct {
mu sync.Mutex
writes [][]byte
// delay, when set, slows every write down
delay time.Duration
}
func (c *flowTunCapture) Read() ([]tio.Packet, error) { return nil, io.EOF }
func (c *flowTunCapture) Close() error { return nil }
func (c *flowTunCapture) Write(b []byte) (int, error) {
time.Sleep(c.delay)
c.mu.Lock()
defer c.mu.Unlock()
c.writes = append(c.writes, append([]byte(nil), b...))
return len(b), nil
}
// segments decodes what reached the tun. The coalescer may have merged segments back into superpackets, so this
// walks each payload in MSS steps.
func (c *flowTunCapture) segments(t testing.TB) []decodedSeg {
t.Helper()
c.mu.Lock()
defer c.mu.Unlock()
var out []decodedSeg
for _, w := range c.writes {
pay := w[40:]
require.Zero(t, len(pay)%flowTestMSS, "tun write payload %d bytes", len(pay))
for off := 0; off < len(pay); off += flowTestMSS {
out = append(out, decodeSeg(pay[off:]))
}
}
return out
}
// decryptWire checks every wire packet decrypts on the receiver, in order, with strictly increasing counters, and
// returns what each one carried. It uses its own replay window and copies, so the wire can still be fed to the
// receiver afterwards.
func decryptWire(t testing.TB, receiver *flowTestPeer, wire [][]byte) []decodedSeg {
t.Helper()
peer := receiver.hostinfo.ConnectionState
ci := &ConnectionState{dKey: peer.dKey, window: NewBits(peer.window.length)}
nb := make([]byte, 12)
var last uint64
out := make([]decodedSeg, 0, len(wire))
for i, pkt := range wire {
h := &header.H{}
require.NoError(t, h.Parse(pkt))
require.Equal(t, header.Message, h.Type)
require.Equal(t, receiver.hostinfo.localIndexId, h.RemoteIndex)
require.Greater(t, h.MessageCounter, last, "wire packet %d: counter %d after %d", i, h.MessageCounter, last)
last = h.MessageCounter
plain, err := ci.Decrypt(receiver.f.l, h.MessageCounter, append([]byte(nil), pkt...), nb)
require.NoError(t, err, "wire packet %d", i)
out = append(out, decodeSeg(plain[40:]))
}
return out
}
// flowBatchQueue is a handoffQueue that hands out scripted Read batches, then fails with io.EOF. Every batch is copied
// into the current receive buffer, so a test sees the buffer ownership the Offload queue has.
type flowBatchQueue struct {
batches [][]tio.Packet
rxBuf []byte
out []tio.Packet
closed chan struct{}
// reads counts the batches handed out
reads atomic.Int64
}
func newFlowBatchQueue(batches [][]tio.Packet) *flowBatchQueue {
return &flowBatchQueue{batches: batches, rxBuf: make([]byte, 8*65536), closed: make(chan struct{})}
}
func (q *flowBatchQueue) Read() ([]tio.Packet, error) {
select {
case <-q.closed:
return nil, os.ErrClosed
default:
}
n := int(q.reads.Load())
if n == len(q.batches) {
return nil, io.EOF
}
q.out = q.out[:0]
off := 0
for _, p := range q.batches[n] {
l := copy(q.rxBuf[off:], p.Bytes)
q.out = append(q.out, tio.Packet{Bytes: q.rxBuf[off : off+l : off+l], GSO: p.GSO})
off += l
}
q.reads.Add(1)
return q.out, nil
}
func (q *flowBatchQueue) SwapRxBuf(buf []byte) []byte {
if buf == nil {
buf = make([]byte, len(q.rxBuf))
}
old := q.rxBuf
q.rxBuf = buf
// Scribble over the buffer we are handed, a staged batch that still pointed into it would be corrupted
for i := range buf {
buf[i] = 0xa5
}
return old
}
func (q *flowBatchQueue) Write(b []byte) (int, error) { return len(b), nil }
func (q *flowBatchQueue) Close() error { close(q.closed); return nil }
// delayedCipher changes scheduling through the real cipher interface, only in tests.
type delayedCipher struct {
noiseutil.CipherState
before func(uint64)
}
func (c delayedCipher) EncryptDanger(out, ad, plain []byte, n uint64, nb []byte) ([]byte, error) {
c.before(n)
return c.CipherState.EncryptDanger(out, ad, plain, n, nb)
}
func (c delayedCipher) DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error) {
c.before(n)
return c.CipherState.DecryptDanger(out, ad, ciphertext, n, nb)
}
func delayCrypto(peer *flowTestPeer, before func(uint64)) {
ci := peer.hostinfo.ConnectionState
ci.eKey = delayedCipher{ci.eKey, before}
ci.dKey = delayedCipher{ci.dKey, before}
}
// stagedOn enables staging with n workers.
func stagedOn(n int) stagedConfig {
return stagedConfig{workers: n, tx: true, rx: true, txSlots: stagedTxSlots, rxSlots: 6}
}
// adaptive is staging with the inline threshold and gate hold parseStagedConfig sets.
func adaptive(n int) stagedConfig {
return stagedConfig{workers: n, tx: true, rx: true, inlineBelow: 256, gateHold: stagedGateHold,
txSlots: stagedTxSlots, rxSlots: 6}
}
func stagedName(sc stagedConfig) string {
switch {
case sc.workers == 0:
return "inline"
case sc.inlineBelow > 0:
return "adaptive"
default:
return "staged"
}
}
// send takes one read through the pipeline, as run does.
func (p *stagedTx) send(pkts []tio.Packet) {
p.handle(pkts, &firewall.ParsedPacket{}, make([]byte, mtu), nil)
}
// drain waits until every batch p has in flight is on the wire.
func (p *stagedTx) drain() {
for p.inFlight > 0 {
p.release(<-p.returned)
}
}
// TestFlowStagedFallback: a tun queue that cannot hand off its buffer, or a conn that cannot hand off its batches,
// keeps that direction on the inline path.
func TestFlowStagedFallback(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(3))
batches, want := flowTestBatches(10, 2, 1, 4)
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
tx0, rx0 := tx.Count(), rxBatches.Count()
sender.f.listenIn(plainQueue{newFlowBatchQueue(batches)}, 0)
wire := sender.conn.captured()
require.Equal(t, want, decryptWire(t, receiver, wire))
require.Equal(t, tx0, tx.Count())
receiver.receive(plainConn{&rxConn{wire: wire, batchLen: 16}})
require.Equal(t, want, receiver.tun.segments(t))
require.Equal(t, rx0, rxBatches.Count())
}
// plainQueue and plainConn hide SwapRxBuf and ListenOutHandoff, so they cannot hand off their batches.
type plainQueue struct{ tio.Queue }
// wireFor produces the wire packets the sender emits for batches, inline.
func wireFor(t testing.TB, sender *flowTestPeer, batches [][]tio.Packet) [][]byte {
t.Helper()
sc := sender.f.staged
sender.f.staged = stagedConfig{}
sender.f.listenIn(newFlowBatchQueue(batches), 0)
sender.f.staged = sc
wire := sender.conn.captured()
sender.conn.pkts, sender.conn.addrs = nil, nil
return wire
}
// rxConn plays wire back to listenOut through the buffer handoff, in batches of batchLen or of the scripted sizes
// (the last one repeating), and captures what is sent back. Every buffer handed back is scribbled over before it is
// read into, so a stage still using one shows corruption. from, when set, picks each packet's source address by its
// index in wire.
type rxConn struct {
flowCaptureConn
wire [][]byte
batchLen int
sizes []int
from func(i int) netip.AddrPort
swaps int
}
func (c *rxConn) ListenOut(r udp.EncReader, flush func()) error {
return c.ListenOutHandoff(r, func(bufs [][]byte) [][]byte {
flush()
return bufs
})
}
func (c *rxConn) ListenOutHandoff(r udp.EncReader, handoff func(bufs [][]byte) [][]byte) error {
n := c.batchLen
for _, size := range c.sizes {
n = max(n, size)
}
bufs := make([][]byte, n)
for i := range bufs {
bufs[i] = make([]byte, udp.MTU)
}
for off, k := 0, 0; off < len(c.wire); k++ {
size := c.batchLen
if len(c.sizes) > 0 {
size = c.sizes[min(k, len(c.sizes)-1)]
}
end := min(off+size, len(c.wire))
for i, p := range c.wire[off:end] {
n := copy(bufs[i], p)
from := flowTestUnder
if c.from != nil {
from = c.from(off + i)
}
r(from, bufs[i][:n:n])
}
next := handoff(bufs)
if len(next) != len(bufs) {
panic("handoff returned the wrong number of buffers")
}
bufs = next
for _, b := range bufs {
for j := range b[:udp.MTU] {
b[j] = 0x5a
}
}
c.swaps++
off = end
}
return os.ErrClosed
}
type plainConn struct{ udp.Conn }
var rxBatches = metrics.GetOrRegisterCounter("flow.rx.batches", nil)
// receive runs the peer's listenOut over conn until conn runs out.
func (p *flowTestPeer) receive(conn udp.Conn) {
p.f.writers = []udp.Conn{conn}
p.f.outside = conn
p.f.listenOut(0)
}
// TestFlowStagedFirewallParity: both paths apply the same outbound and inbound policy. An authenticated inbound
// denial still consumes its replay counter, exactly as inline Decrypt does.
func TestFlowStagedFirewallParity(t *testing.T) {
for _, sc := range []stagedConfig{{}, stagedOn(2)} {
t.Run(stagedName(sc), func(t *testing.T) {
batches, _ := flowTestBatches(4, 3, 1, 8)
sender, _ := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
sender.f.firewall.OutRules = newFirewallTable()
sender.f.listenIn(newFlowBatchQueue(batches), 0)
require.Empty(t, sender.conn.captured())
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
wire := wireFor(t, sender, batches)
receiver.f.firewall.InRules = newFirewallTable()
receiver.receive(&rxConn{wire: wire, batchLen: 64})
require.Empty(t, receiver.tun.writes)
for _, pkt := range wire {
h := &header.H{}
require.NoError(t, h.Parse(pkt))
require.False(t, receiver.hostinfo.ConnectionState.window.Check(receiver.f.l, h.MessageCounter))
}
})
}
}
// TestFlowStagedCipherFallback runs the parsed configuration with the real AES cipher: transmit falls back to inline
// when the build needs nonce-ordered encryption (BoringCrypto, FIPS), receive decryption stays staged.
func TestFlowStagedCipherFallback(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
c.Settings = map[string]any{"tun": map[string]any{"staged": map[string]any{"workers": 2}}}
sender, receiver := newFlowTestPair(t, noiseutil.CipherAESGCM, parseStagedConfig(c, l))
require.Equal(t, !noiseutil.EncryptLockNeeded, sender.f.staged.tx)
require.True(t, sender.f.staged.rx)
batches, want := flowTestBatches(4, 8, 1, 45)
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
tx0, rx0 := tx.Count(), rxBatches.Count()
sender.f.listenIn(newFlowBatchQueue(batches), 0)
if noiseutil.EncryptLockNeeded {
require.Equal(t, tx0, tx.Count())
}
receiver.receive(&rxConn{wire: sender.conn.captured(), batchLen: 512})
require.Greater(t, rxBatches.Count(), rx0)
require.Equal(t, want, receiver.tun.segments(t))
}
-583
View File
@@ -1,583 +0,0 @@
package nebula
import (
"errors"
"net/netip"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/udp"
)
var errFlowSegmentOverrun = errors.New("superpacket has more segments than counters reserved for it")
// txItem is one firewall-approved inside packet (or superpacket) with everything needed to encrypt it. A nonzero
// counter is the first of the segs counters reserved for it on ci, relayCounter the same on the relay tunnel when
// relayHI is set. Zero counters are allocated per segment while encrypting, as inline sends do.
type txItem struct {
pkt tio.Packet
hostinfo *HostInfo
ci *ConnectionState
remote netip.AddrPort
segs uint64
counter uint64
relayHI *HostInfo
relay *Relay
relayCounter uint64
}
// prepareTxItem does the per-packet work sendInsideMessage does before it encrypts, and resolves where the packet goes.
func (f *Interface) prepareTxItem(it *txItem, hostinfo *HostInfo, pkt tio.Packet) bool {
ci := hostinfo.ConnectionState
if ci.eKey == nil {
return false
}
f.recordOutbound(hostinfo)
it.pkt, it.hostinfo, it.ci = pkt, hostinfo, ci
it.remote = hostinfo.GetRemote()
if it.remote.IsValid() {
return true
}
it.relayHI, it.relay = f.findRelay(hostinfo)
if it.relayHI == nil || it.relay == nil {
return false
}
it.remote = it.relayHI.GetRemote()
return true
}
// encryptTxItem segments and encrypts it into sb, for inline and staged sends alike.
func (f *Interface) encryptTxItem(it *txItem, nb []byte, sb *batch.SendBatch) {
reserved := it.counter != 0
c, rc, end := it.counter, it.relayCounter, it.counter+it.segs
err := tio.SegmentSuperpacket(it.pkt, func(seg []byte) error {
if reserved && c >= end {
// More segments than counters reserved for them, never encrypt past the reservation
return errFlowSegmentOverrun
}
if it.relayHI == nil {
// header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305)
if out := f.encryptTxSegment(it, c, seg, nb, sb.Reserve(header.Len+len(seg)+16)); out != nil {
sb.Commit(out, it.remote)
}
} else {
f.encryptTxSegmentVia(it, c, rc, seg, nb, sb)
}
if reserved {
c++
rc++
}
return nil
})
if err != nil {
it.hostinfo.logger(f.l).Error("Failed to segment superpacket for send", "error", err)
}
}
// encryptTxSegment encrypts one segment into out with counter c, or with the next counter of the tunnel when c is
// zero. It returns nil when encryption failed, which is already logged.
func (f *Interface) encryptTxSegment(it *txItem, c uint64, seg, nb, out []byte) []byte {
if c != 0 {
// Reserved counters are unique, so the encryption needs no lock
return f.encryptMessage(header.Message, 0, it.hostinfo, it.ci, c, seg, nb, out, it.remote)
}
if noiseutil.EncryptLockNeeded {
it.ci.writeLock.Lock()
}
c = it.ci.messageCounter.Add(1)
out = f.encryptMessage(header.Message, 0, it.hostinfo, it.ci, c, seg, nb, out, it.remote)
if noiseutil.EncryptLockNeeded {
it.ci.writeLock.Unlock()
}
return out
}
// encryptTxSegmentVia encrypts one segment for the relayed tunnel, then wraps it for the relay with counter rc, both
// allocated on demand when zero like encryptTxSegment does.
func (f *Interface) encryptTxSegmentVia(it *txItem, c, rc uint64, seg, nb []byte, sb *batch.SendBatch) {
//relay header + header + plaintext + AEAD tag + relay tag
scratch := sb.Reserve(len(seg) + header.MaxOverhead)
out := f.encryptTxSegment(it, c, seg, nb, scratch[header.Len:])
if out == nil {
return
}
toSend, err := f.prepareSendVia(it.relayHI, it.relay, out, nb, scratch, true, rc)
if err != nil {
return
}
remote := it.remote
if rc == 0 {
// Unreserved sends follow the relay's current remote, like sendNoMetrics
remote = it.relayHI.GetRemote()
}
sb.Commit(toSend, remote)
}
// txTunnel accumulates one tunnel's counter needs across a batch so the batch reserves them with one atomic add.
type txTunnel struct {
ci *ConnectionState
hostinfo *HostInfo
n uint64
base uint64
exhausted bool
}
// txReserver turns one tun read into txItems with message counters reserved for them. Only the reader goroutine
// collects and reserves counters.
type txReserver struct {
f *Interface
tunnels []txTunnel
owner uint64
// The staged reader separates relay data before reserving direct work. A nil pointer collects both paths.
inlineRelays *[]txItem
}
// collect runs the filter over pkts and appends the sendable ones to items, tallying each tunnel's counter needs.
func (r *txReserver) collect(items []txItem, pkts []tio.Packet, fwPacket *firewall.ParsedPacket, rejectBuf []byte, q int, localCache firewall.ConntrackCache) []txItem {
f := r.f
r.tunnels = r.tunnels[:0]
for _, pkt := range pkts {
hostinfo := f.filterInsidePacket(pkt, fwPacket, rejectBuf, q, localCache)
if hostinfo == nil {
continue
}
var it txItem
if !f.prepareTxItem(&it, hostinfo, pkt) {
continue
}
it.segs = uint64(pkt.SegmentCount())
if it.relayHI != nil {
// The relay bookkeeping inline sends do per segment in prepareSendVia, once per item
f.connectionManager.OutNoRebind(it.relayHI)
f.connectionManager.RelayUsed(it.relay.LocalIndex)
if r.inlineRelays != nil {
*r.inlineRelays = append(*r.inlineRelays, it)
continue
}
it.relayCounter = r.need(it.relayHI, it.relayHI.ConnectionState, it.segs)
}
it.counter = r.need(it.hostinfo, it.ci, it.segs)
items = append(items, it)
}
return items
}
// need adds n counters to ci's tally for this batch and returns the offset of the first one within it.
func (r *txReserver) need(hostinfo *HostInfo, ci *ConnectionState, n uint64) uint64 {
t := r.tunnel(ci)
if t == nil {
r.tunnels = append(r.tunnels, txTunnel{ci: ci, hostinfo: hostinfo})
t = &r.tunnels[len(r.tunnels)-1]
}
off := t.n
t.n += n
return off
}
// tunnel finds ci's tally, a batch rarely touches more than a couple of tunnels so a scan beats a map.
func (r *txReserver) tunnel(ci *ConnectionState) *txTunnel {
for i := range r.tunnels {
if r.tunnels[i].ci == ci {
return &r.tunnels[i]
}
}
return nil
}
// reserve takes the counters collect tallied and stamps them onto items, dropping the items of any tunnel whose
// counter is exhausted. It returns the items that are left.
func (r *txReserver) reserve(items []txItem) []txItem {
f := r.f
// One atomic add per tunnel reserves the whole batch's counters before any worker sees it, so counter order on a
// tunnel is dispatch order, which is the order the batches are flushed in. Reservation and the sender's order are
// both taken on the reader goroutine, so there is no window for another batch to slip in between.
for i := range r.tunnels {
t := &r.tunnels[i]
end := t.ci.messageCounter.Add(t.n)
t.base = end - t.n + 1
if end >= RejectAfterMessages {
t.ci.messageCounter.Store(RejectAfterMessages)
t.exhausted = true
f.messageMetrics.TxExhausted(int64(t.n))
if t.base <= RejectAfterMessages {
t.hostinfo.logger(f.l).Error("Dropping outbound packets, tunnel message counter is exhausted")
}
}
}
n := 0
for _, it := range items {
t := r.tunnel(it.ci)
if t.exhausted {
continue
}
it.counter += t.base
if it.relayHI != nil {
rt := r.tunnel(it.relayHI.ConnectionState)
if rt.exhausted {
continue
}
it.relayCounter += rt.base
}
items[n] = it
n++
}
clear(items[n:])
return items[:n]
}
// handoffQueue is a tun queue that can give up the buffer its last Read filled. SwapRxBuf(nil) allocates a
// correctly sized replacement; later swaps recycle the buffers handed back.
type handoffQueue interface {
tio.Queue
SwapRxBuf([]byte) []byte
}
// txSlot is one staged batch. The reader fills items, a worker encrypts them into sb and signals done, and the
// ordered sender flushes sb.
type txSlot struct {
f *Interface
items []txItem
// buf owns the queue receive buffer the items point into, swapped out at dispatch
buf []byte
sb *batch.SendBatch
done chan struct{}
// reservations hold shared tunnel credit until the flush, including for dropped counters
reservations []txTunnel
}
func (s *txSlot) run(nb []byte) {
for i := range s.items {
s.f.encryptTxItem(&s.items[i], nb, s.sb)
}
s.done <- struct{}{}
}
// stagedTx is the staged transmit pipeline of one tun reader. Everything but the workers and the sender runs on the
// reader goroutine.
type stagedTx struct {
txReserver
// Relay data always leaves on the reader, including small and oversized reads. Its independent reservation
// scratch keeps mixed reads from changing the direct pipeline's counters, gate or sender placement.
relayReserver txReserver
relaySlot txSlot
q int
cpu int // reader CPU, inline sends stay here
senderCPU int // -1 leaves the sender within its inherited affinity
queue handoffQueue
pool *flowPool
// sb and nb are the reader's own, for the sends it does itself
sb *batch.SendBatch
nb []byte
// free holds the reader's idle slots. A dispatched slot goes to the workers and, in the same order, to the
// sender on ordered; the sender gives it back on returned once it is on the wire.
free []*txSlot
ordered chan *txSlot
returned chan *txSlot
inFlight int
senderDone chan struct{}
// gateInline is the routine's transmit mode, gateStreak how many reads in a row wanted the other one
gateInline bool
gateStreak int
metricGateSwitches metrics.Counter
metricBatches metrics.Counter
metricInline metrics.Counter
}
// newStagedTx needs a queue that hands over whole read batches, the offload (TSO/USO) queue on linux. On any other
// queue a read is one packet, and staging it would cost a handoff per packet.
func newStagedTx(f *Interface, queue handoffQueue, q int) *stagedTx {
p := &stagedTx{
txReserver: txReserver{f: f, tunnels: make([]txTunnel, 0, 4), owner: txBudgetOwner(q)},
q: q,
senderCPU: -1,
queue: queue,
sb: batch.NewSendBatch(f.writers[q], batch.SendBatchCap, batch.SendBatchCap*(udp.MTU+32)),
nb: make([]byte, 12),
ordered: make(chan *txSlot, f.staged.txSlots),
returned: make(chan *txSlot, f.staged.txSlots),
senderDone: make(chan struct{}),
metricBatches: metrics.GetOrRegisterCounter("flow.tx.batches", nil),
metricInline: metrics.GetOrRegisterCounter("flow.tx.inline", nil),
// Starts inline, the pipeline is empty, unless the threshold never allows it
gateInline: f.staged.inlineBelow > 0,
metricGateSwitches: metrics.GetOrRegisterCounter("flow.tx.gate_switches", nil),
}
// Inline relay sends can share a tunnel with older direct work on this routine. They are not the ordered
// sender's sole owner, so they must not recycle credit around a stalled direct reservation.
p.relayReserver = txReserver{f: f, tunnels: make([]txTunnel, 0, 4)}
p.inlineRelays = &p.relaySlot.items
if f.pinThreads {
p.cpu = f.threadCPU(q)
// Only an explicit list pins the sender, and never onto its reader's CPU, where the two would serialise.
// Automatic placement by queue index can put a busy sender on a slow core; unpinned, the scheduler moves it.
if n := len(f.cpuAffinity); n > 0 && f.cpuAffinity[(q+1)%n] != p.cpu {
p.senderCPU = f.cpuAffinity[(q+1)%n]
}
}
p.pool = newFlowPool(f.staged.workers, f.staged.txSlots)
// Ciphertext adds per-segment headers and tags; the arena can grow for small segments.
buf := queue.SwapRxBuf(nil)
bufSize := len(buf)
for i := range f.staged.txSlots {
if i > 0 {
buf = make([]byte, bufSize)
}
p.free = append(p.free, &txSlot{
f: f,
buf: buf,
sb: batch.NewSendBatch(f.writers[q], 4*batch.SendBatchCap, bufSize+bufSize/4),
done: make(chan struct{}, 1),
})
}
go p.sender()
return p
}
// run is listenIn's loop with the staged pipeline: read, then hand each batch on, until the queue fails.
func (p *stagedTx) run(fwPacket *firewall.ParsedPacket, rejectBuf []byte, conntrackCache *firewall.ConntrackCacheTicker) error {
defer p.close()
if p.f.pinThreads {
p.f.pinThreadToCPU(p.q, p.cpu)
}
for {
pkts, err := p.queue.Read()
if err != nil {
return err
}
p.ready()
p.handle(pkts, fwPacket, rejectBuf, conntrackCache.Get())
}
}
// handle takes one tun read through the pipeline. It returns once the batch is queued, or sent when the reader sent
// it itself.
func (p *stagedTx) handle(pkts []tio.Packet, fwPacket *firewall.ParsedPacket, rejectBuf []byte, localCache firewall.ConntrackCache) {
p.ready()
inline := p.gate(p.small(pkts))
if inline && p.f.routines == 1 {
// With only one reader and an empty pipeline, no other staged reservation can be overtaken. Keep this common
// case inexpensive.
p.inlineSoleReader(pkts, fwPacket, rejectBuf, localCache)
return
}
slot := p.free[len(p.free)-1]
p.free = p.free[:len(p.free)-1]
items := p.collect(slot.items[:0], pkts, fwPacket, rejectBuf, p.q, localCache)
if len(p.relaySlot.items) != 0 {
p.sendRelays()
}
if len(items) == 0 {
slot.items = items
p.free = append(p.free, slot)
return
}
if p.oversized() {
p.metricInline.Inc(1)
p.sendLarge(&p.txReserver, items)
slot.items = items[:0]
p.free = append(p.free, slot)
return
}
if !p.waitBudget(&p.txReserver) {
clear(items)
slot.items = items[:0]
p.free = append(p.free, slot)
return
}
slot.reservations = append(slot.reservations[:0], p.tunnels...)
items = p.reserve(items)
slot.items = items
if len(items) == 0 {
slot.releaseBudget()
p.free = append(p.free, slot)
return
}
if inline {
p.inline(slot)
p.free = append(p.free, slot)
return
}
// The packets now live in the slot, and the queue reads into the slot's old buffer
slot.buf = p.queue.SwapRxBuf(slot.buf)
p.inFlight++
p.metricBatches.Inc(1)
// Counters were reserved above on this goroutine and the slot takes its place in the sender's order right here,
// before any other batch can, so the sender puts them on the wire in counter order.
p.ordered <- slot
p.pool.jobs <- slot
}
// sendRelays keeps relay data on one send path for the lifetime of this routine. The gate's software drain only
// waits for socket submission; switching CPUs can still overtake packets on another NIC queue. Direct data retains
// the existing gate and sender placement. Relay credit is still shared with all other users of both tunnels.
func (p *stagedTx) sendRelays() {
r, slot := &p.relayReserver, &p.relaySlot
r.tunnels = r.tunnels[:0]
for i := range slot.items {
it := &slot.items[i]
it.counter = r.need(it.hostinfo, it.ci, it.segs)
it.relayCounter = r.need(it.relayHI, it.relayHI.ConnectionState, it.segs)
}
if r.oversized() {
p.metricInline.Inc(1)
p.sendLarge(r, slot.items)
slot.items = slot.items[:0]
return
}
if !p.waitBudget(r) {
clear(slot.items)
slot.items = slot.items[:0]
return
}
slot.reservations = append(slot.reservations[:0], r.tunnels...)
slot.items = r.reserve(slot.items)
p.inline(slot)
}
// gate reports whether a read goes inline. Each routine holds one
// transmit mode, inline on the reader or fanout to the workers and the sender, until gateHold reads in a row want the
// other one, the last of which goes the new way. Choosing per read let small inline sends leave from the reader's CPU
// between staged ones leaving from the sender's, and the receiver's TCP saw them reordered. Going inline the reader
// first waits until everything in flight has been submitted to the socket and the sender is idle. This does not
// drain independent NIC queues; relay data therefore always uses the reader. Going to fanout nothing is in flight:
// inline sends finish before the reader moves on.
func (p *stagedTx) gate(small bool) bool {
if small == p.gateInline {
p.gateStreak = 0
return small
}
if p.gateStreak++; p.gateStreak < p.f.staged.gateHold {
return p.gateInline
}
p.gateInline, p.gateStreak = small, 0
p.metricGateSwitches.Inc(1)
if !small {
return false
}
for p.flushOldest() {
}
return true
}
// ready takes back every slot the sender has finished with, and waits for one when all of them are in flight.
func (p *stagedTx) ready() {
p.reap()
if len(p.free) == 0 {
p.release(<-p.returned)
}
}
// reap takes back every slot the sender has finished with, without waiting.
func (p *stagedTx) reap() {
for {
select {
case s := <-p.returned:
p.release(s)
default:
return
}
}
}
// flushOldest waits for the oldest batch in flight to reach the wire and takes its slot back. It reports false when
// nothing is in flight.
func (p *stagedTx) flushOldest() bool {
if p.inFlight == 0 {
return false
}
p.release(<-p.returned)
return true
}
func (p *stagedTx) release(s *txSlot) {
p.inFlight--
p.free = append(p.free, s)
}
// sender is the ordered send stage: it puts each batch on the wire once its worker is done, in dispatch order.
func (p *stagedTx) sender() {
defer close(p.senderDone)
if p.senderCPU >= 0 {
p.f.pinThreadToCPU(p.q, p.senderCPU)
}
for s := range p.ordered {
<-s.done
if p.f.closed.Load() {
// The sockets are gone, a flush would only log errors
s.sb.Reset()
} else {
p.f.flushSendBatch(s.sb, p.q)
}
s.releaseBudget()
clear(s.items)
s.items = s.items[:0]
p.returned <- s
}
}
// close sends everything still in flight, in order, and stops the sender and the workers.
func (p *stagedTx) close() {
close(p.ordered)
<-p.senderDone
p.pool.close()
}
// small counts segments against the inline threshold.
func (p *stagedTx) small(pkts []tio.Packet) bool {
below := uint64(p.f.staged.inlineBelow)
if below == 0 {
return false
}
var n uint64
for _, pkt := range pkts {
if n += uint64(pkt.SegmentCount()); n >= below {
return false
}
}
return true
}
// inline encrypts a budgeted batch on the reader, with no worker handoff or buffer swap. The local sender is idle,
// but other readers may hold reservations on these tunnels, so keep our credit until the flush completes.
func (p *stagedTx) inline(slot *txSlot) {
p.metricInline.Inc(1)
for i := range slot.items {
p.f.encryptTxItem(&slot.items[i], p.nb, p.sb)
if p.sb.Len() >= batch.SendBatchCap {
p.f.flushSendBatch(p.sb, p.q)
}
}
p.f.flushSendBatch(p.sb, p.q)
slot.releaseBudget()
clear(slot.items)
slot.items = slot.items[:0]
}
// inlineSoleReader needs no credit: the interface has no other tun reader and this reader has drained every
// reservation before taking the inline path.
func (p *stagedTx) inlineSoleReader(pkts []tio.Packet, fwPacket *firewall.ParsedPacket, rejectBuf []byte, localCache firewall.ConntrackCache) {
p.metricInline.Inc(1)
for _, pkt := range pkts {
p.f.consumeInsidePacket(pkt, fwPacket, p.nb, p.sb, rejectBuf, p.q, localCache)
if p.sb.Len() >= batch.SendBatchCap {
p.f.flushSendBatch(p.sb, p.q)
}
}
p.f.flushSendBatch(p.sb, p.q)
}
-231
View File
@@ -1,231 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
import (
"fmt"
"net/netip"
"sync"
"time"
"os"
"os/exec"
"testing"
"github.com/slackhq/nebula/overlay/tio"
"github.com/flynn/noise"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util"
"github.com/stretchr/testify/require"
)
// queuedWire models two CPU-selected NIC queues with independent completion: the reader's
// queue is congested, while another CPU's queue drains immediately. WriteBatch returning only
// means the kernel accepted the packets; draining staged slots does not drain a NIC queue.
// The sampled affinity makes the schedule deterministic without a real socket or NIC. An
// unpinned sender alternates between allowed CPUs, even if it happened not to in this run.
func (c *affinityConn) queuedWire(readerCPU int) [][]byte {
var slow, fast [][]byte
wire := c.captured()
off := 0
for i, n := range c.sizes {
mask := c.masks[i]
if mask.IsSet(readerCPU) && (mask.Count() == 1 || i%2 == 0) {
slow = append(slow, wire[off:off+n]...)
} else {
fast = append(fast, wire[off:off+n]...)
}
off += n
}
return append(fast, slow...)
}
// TestFlowStagedTXRelayQueuedWire reproduces reorder after successful, ordered socket
// submissions. Mixed reads switch the sticky gate with an empty software pipeline; relayed
// data must stay on the reader throughout, without moving the direct sender. Each routine owns a flow.
func TestFlowStagedTXRelayQueuedWire(t *testing.T) {
if os.Getenv("NEBULA_FLOW_RING_TEST") != "1" {
cmd := exec.Command(os.Args[0], "-test.run=^TestFlowStagedTXRelayQueuedWire$")
cmd.Env = append(os.Environ(), "NEBULA_FLOW_RING_TEST=1")
out, err := cmd.CombinedOutput()
require.NoError(t, err, "%s", out)
return
}
allowed, err := util.AllowedCPUs()
require.NoError(t, err)
if len(allowed) < 2 {
t.Skip("requires two allowed CPUs")
}
for _, mode := range []string{"gate", "always-staged", "oversized"} {
for _, explicit := range []bool{false, true} {
for _, routines := range []int{1, 2} {
for q := range routines {
t.Run(fmt.Sprintf("%s/explicit=%t/routines=%d/q=%d", mode, explicit, routines, q), func(t *testing.T) {
sc := gated(stagedGateHold)
if mode != "gate" {
sc.inlineBelow = 0
}
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
f := sender.f
f.pinThreads, f.routines = true, routines
if explicit {
f.cpuAffinity = allowed[:2]
}
_, relayHI, relayPeer := addTestRelay(t, sender, ReplayWindow)
conn := &affinityConn{}
f.writers = make([]udp.Conn, routines)
for i := range f.writers {
f.writers[i] = conn
}
shape := append(repeatBool(false, 2), repeatBool(true, stagedGateHold+2)...)
shape = append(shape, repeatBool(false, stagedGateHold+2)...)
if mode != "gate" {
shape = []bool{false, true, false, true}
}
reads, want := gateReads(shape...)
if mode == "oversized" {
// A deliberately tiny outer budget alone forces the oversized fallback.
relayHI.ConnectionState.window = NewBits(16)
}
queue := &affinityQueue{flowBatchQueue: newFlowBatchQueue(reads), conn: conn}
// Each Read waits for all previous socket submissions, making software drain
// irrelevant to the independently queued packets in queuedWire.
f.listenIn(queue, q)
unwrap := func(wire [][]byte) [][]byte {
inner := make([][]byte, 0, len(wire))
for _, pkt := range wire {
inner = append(inner, pkt[header.Len:len(pkt)-relayPeer.dKey.Overhead()])
}
return inner
}
require.Equal(t, want, decryptWire(t, receiver, unwrap(conn.captured())), "socket submission order")
require.Equal(t, want, decryptWire(t, receiver, unwrap(conn.queuedWire(f.threadCPU(q)))), "NIC queue completion order")
})
}
}
}
}
}
// Mixed reads must keep direct sends on the original staged sender, including when relay data is present.
func TestFlowStagedTXMixedPlacement(t *testing.T) {
if os.Getenv("NEBULA_FLOW_MIXED_TEST") != "1" {
cmd := exec.Command(os.Args[0], "-test.run=^TestFlowStagedTXMixedPlacement$")
cmd.Env = append(os.Environ(), "NEBULA_FLOW_MIXED_TEST=1")
out, err := cmd.CombinedOutput()
require.NoError(t, err, "%s", out)
return
}
allowed, err := util.AllowedCPUs()
require.NoError(t, err)
if len(allowed) < 2 {
t.Skip("requires two allowed CPUs")
}
for _, explicit := range []bool{false, true} {
t.Run(fmt.Sprintf("explicit=%t", explicit), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
_, _, relayPeer := addTestRelay(t, sender, ReplayWindow)
direct, directReceiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
directAddr := netip.MustParseAddr("10.0.0.4")
direct.hostinfo.vpnAddrs = []netip.Addr{directAddr}
f := sender.f
f.hostMap.Hosts[directAddr] = direct.hostinfo
f.pinThreads, f.routines = true, 2
sendCPU := -1
if explicit {
f.cpuAffinity = allowed[:2]
sendCPU = allowed[1]
}
conn := &affinityConn{}
f.writers = []udp.Conn{conn, conn}
var reads [][]tio.Packet
var wantRelay, wantDirect []decodedSeg
for seq := range uint32(6) {
r := flowTCPSuperpacket(0, seq, 3)
d := flowTCPSuperpacket(1, seq, 9)
copy(d.Bytes[16:20], directAddr.AsSlice())
reads = append(reads, []tio.Packet{r, d})
for i := range 3 {
wantRelay = append(wantRelay, decodedSeg{seq: seq, idx: uint16(i)})
}
for i := range 9 {
wantDirect = append(wantDirect, decodedSeg{flow: 1, seq: seq, idx: uint16(i)})
}
}
queue := &affinityQueue{flowBatchQueue: newFlowBatchQueue(reads), conn: conn}
f.listenIn(queue, 0)
wire := conn.captured()
var directWire, relayWire [][]byte
off := 0
for i, n := range conn.sizes {
for _, pkt := range wire[off : off+n] {
var h header.H
require.NoError(t, h.Parse(pkt))
cpu := sendCPU
if h.Subtype == header.MessageRelay {
cpu = f.threadCPU(0)
require.NoError(t, relayPeer.VerifyRelay(nil, h.MessageCounter, pkt, make([]byte, 12)))
relayWire = append(relayWire, pkt[header.Len:len(pkt)-relayPeer.dKey.Overhead()])
} else {
directWire = append(directWire, pkt)
}
requireFlowCPU(t, conn.masks[i:i+1], cpu, allowed)
}
off += n
}
require.Equal(t, wantDirect, decryptWire(t, directReceiver, directWire))
require.Equal(t, wantRelay, decryptWire(t, receiver, relayWire))
})
}
}
// A relayed batch shares its outer tunnel with older staged direct data on the SAME routine. Inline completion
// must not refund that direct sender's credit while its oldest reservation is stalled.
func TestFlowStagedTXRelaySharedBudget(t *testing.T) {
sender, _ := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
_, via, _ := addTestRelay(t, sender, ReplayWindow)
// Give the relay host the same ordinary peer metadata as a direct tunnel.
direct := newFlowTestPeer(t, flowTestLocal, via.vpnAddrs[0], via.ConnectionState, 300, 400)
via.buildNetworks(sender.f.myVpnNetworksTable, direct.hostinfo.ConnectionState.peerCert.Certificate)
via.remoteIndexId = direct.hostinfo.remoteIndexId
ci := via.ConnectionState
ci.window = NewBits(32)
start := ci.messageCounter.Load()
entered, unblock := make(chan struct{}), make(chan struct{})
var release sync.Once
delayCrypto(direct, func(n uint64) {
if n == start+1 {
close(entered)
<-unblock
}
})
p := newStagedTx(sender.f, newFlowBatchQueue(nil), 0)
defer p.close()
defer release.Do(func() { close(unblock) })
d := flowTCPSuperpacket(1, 0, 9)
copy(d.Bytes[16:20], via.vpnAddrs[0].AsSlice())
p.send([]tio.Packet{d})
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("direct worker did not start")
}
for seq := range uint32(7) {
p.send([]tio.Packet{flowTCPSuperpacket(0, seq, 1)})
}
require.Equal(t, uint64(16), (ci.txBudget.state.Load()>>24)&txBudgetMask)
done := make(chan struct{})
go func() { defer close(done); p.send([]tio.Packet{flowTCPSuperpacket(0, 7, 1)}) }()
select {
case <-done:
t.Error("relay recycled credit around a stalled direct reservation")
case <-time.After(50 * time.Millisecond):
}
require.Equal(t, start+16, ci.messageCounter.Load())
release.Do(func() { close(unblock) })
<-done
p.drain()
require.Zero(t, ci.txBudget.state.Load())
}
-604
View File
@@ -1,604 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
import (
"fmt"
"net/netip"
"os"
"os/exec"
"slices"
"sync"
"testing"
"time"
"github.com/flynn/noise"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/cpupick"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
"golang.org/x/sys/unix"
)
// TestReservedEncryptionMatchesInline encrypts a batch with counters reserved up front: the receiver accepts every
// segment, in order.
func TestReservedEncryptionMatchesInline(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedConfig{})
packets, want := flowTestBatches(1, 3, 3, 9)
r := txReserver{f: sender.f}
items := r.collect(nil, packets[0], &firewall.ParsedPacket{}, make([]byte, mtu), 0, nil)
require.Equal(t, uint64(len(want)), r.tunnels[0].n)
sb := batch.NewSendBatch(sender.conn, 128, 65536)
for _, it := range r.reserve(items) {
sender.f.encryptTxItem(&it, make([]byte, 12), sb)
}
sender.f.flushSendBatch(sb, 0)
require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured()))
}
// TestSegmentCount checks tio.Packet.SegmentCount predicts exactly the segments tio.SegmentSuperpacket cuts, odd tails included.
func TestSegmentCount(t *testing.T) {
require.Equal(t, 1, tio.Packet{Bytes: make([]byte, 60)}.SegmentCount())
for _, segs := range []int{1, 2, 3, 45} {
for _, tail := range []int{0, 1, flowTestMSS - 1} {
pkt := flowTCPSuperpacket(0, 0, segs)
pkt.Bytes = pkt.Bytes[:len(pkt.Bytes)-tail]
pkt.GSO = tio.GSOInfo{Size: flowTestMSS, HdrLen: 40, CsumStart: 20, Proto: tio.GSOProtoTCP}
want, got := pkt.SegmentCount(), 0
require.NoError(t, tio.SegmentSuperpacket(pkt, func([]byte) error { got++; return nil }))
require.Equal(t, want, got, "segs %d tail %d", segs, tail)
}
}
}
// TestFlowStagedTXOrderAndCounters runs the staged transmit pipeline with shuffled workers and a slow socket, so the
// sender lags and the reader runs into backpressure, then fails the tun read with batches in flight: the wire must
// carry every segment once, in tun read order, under strictly increasing counters that were each reserved exactly
// once, and the read error is still reported.
func TestFlowStagedTXOrderAndCounters(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
t.Run(cipher.CipherName(), func(t *testing.T) {
if cipher.CipherName() == noiseutil.CipherAESGCM.CipherName() && noiseutil.EncryptLockNeeded {
t.Skip("AES-GCM requires encryption in nonce order")
}
sender, receiver := newFlowTestPair(t, cipher, stagedOn(4))
delayCrypto(sender, func(n uint64) {
if n%54 == 0 {
time.Sleep(time.Millisecond)
}
})
// Far slower than the reader, even under the race detector
sender.conn.delay = 8 * time.Millisecond
batches, want := flowTestBatches(60, 6, 3, 9)
start := sender.hostinfo.ConnectionState.messageCounter.Load()
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
tx0 := tx.Count()
sender.f.listenIn(newFlowBatchQueue(batches), 0)
require.Equal(t, int64(len(batches)), tx.Count()-tx0, "every batch uses the ordered sender")
require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured()), "wire order must match tun read order")
require.Equal(t, start+uint64(len(want)), sender.hostinfo.ConnectionState.messageCounter.Load())
require.NotNil(t, sender.f.fatalErr.Load())
})
}
}
// TestFlowStagedTXCloseInFlight closes the interface and tun while batches are in flight, 20 times over.
func TestFlowStagedTXCloseInFlight(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
for range 20 {
sender, _ := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(3))
sender.conn.delay = time.Millisecond
batches, _ := flowTestBatches(400, 4, 2, 8)
q := newFlowBatchQueue(batches)
done := make(chan struct{})
go func() { sender.f.listenIn(q, 0); close(done) }()
require.Eventually(t, func() bool { return q.reads.Load() > 5 }, 5*time.Second, 100*time.Microsecond)
sender.f.closed.Store(true)
require.NoError(t, q.Close())
<-done
}
}
// TestFlowStagedTXExhausted drops every batch of a tunnel whose counter runs out, like inline sends do.
func TestFlowStagedTXExhausted(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sender, _ := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
ci := sender.hostinfo.ConnectionState
ci.messageCounter.Store(RejectAfterMessages - 3)
sender.f.listenIn(newFlowBatchQueue([][]tio.Packet{{flowTCPSuperpacket(0, 0, 10)}, {flowTCPSuperpacket(0, 1, 2)}}), 0)
require.Empty(t, sender.conn.captured())
require.Equal(t, RejectAfterMessages, ci.messageCounter.Load())
require.Equal(t, int64(12), sender.f.messageMetrics.txExhausted.Count())
require.Zero(t, ci.txBudget.state.Load())
}
// TestFlowStagedTXRelay sends staged batches, and an empty read, through a relay: outer and inner layers both verify,
// in order, and go to the relay.
// With a tiny MSS, the middle superpacket alone exceeds the tunnels' whole credit: it must be split, not admitted as an
// unbounded lone batch, with either replay window.
func TestFlowStagedTXRelay(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
for _, window := range budgetWindows {
t.Run(fmt.Sprintf("window=%d", window), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
sender.hostinfo.ConnectionState.window = NewBits(window)
receiver.hostinfo.ConnectionState.window = NewBits(window)
relayRemote, relayHI, relayPeer := addTestRelay(t, sender, window)
batches, want := flowTestBatches(2, 3, 1, 6)
var payload []byte
large := flowTCPSuperpacket(0, 9, 45)
large.GSO.Size = 1
payload = append([]byte(nil), large.Bytes[40:]...)
batches = [][]tio.Packet{batches[0], {large}, batches[1]}
sender.f.listenIn(newFlowBatchQueue(append(batches, nil)), 0)
wire := sender.conn.captured()
require.Len(t, wire, len(want)+len(payload))
var got []decodedSeg
var tiny []byte
var lastOuter, lastInner uint64
for i, pkt := range wire {
require.Equal(t, relayRemote, sender.conn.addrs[i])
h := &header.H{}
require.NoError(t, h.Parse(pkt))
require.Greater(t, h.MessageCounter, lastOuter)
lastOuter = h.MessageCounter
require.NoError(t, relayPeer.VerifyRelay(nil, h.MessageCounter, pkt, make([]byte, 12)))
pkt = pkt[header.Len : len(pkt)-16]
require.NoError(t, h.Parse(pkt))
require.Greater(t, h.MessageCounter, lastInner)
lastInner = h.MessageCounter
plain, err := receiver.hostinfo.ConnectionState.Decrypt(receiver.f.l, h.MessageCounter, pkt, make([]byte, 12))
require.NoError(t, err)
if len(plain) == 41 {
tiny = append(tiny, plain[40])
} else {
got = append(got, decodeSeg(plain[40:]))
}
}
require.Equal(t, want, got)
require.Equal(t, payload, tiny)
require.Zero(t, sender.hostinfo.ConnectionState.txBudget.state.Load())
require.Zero(t, relayHI.ConnectionState.txBudget.state.Load())
})
}
}
// addTestRelay makes sender reach its peer only through a relay tunnel with replay windows of window. It returns the
// relay's underlay address, its hostinfo and the relay's side of the tunnel.
func addTestRelay(t testing.TB, sender *flowTestPeer, window uint64) (netip.AddrPort, *HostInfo, *ConnectionState) {
relayInit, relayResp := runTestHandshake(t)
relayCI, err := newConnectionStateFromResult(relayInit)
require.NoError(t, err)
relayPeer, err := newConnectionStateFromResult(relayResp)
require.NoError(t, err)
relayCI.window, relayPeer.window = NewBits(window), NewBits(window)
relayAddr := netip.MustParseAddr("10.0.0.3")
relayRemote := netip.MustParseAddrPort("192.0.2.3:4242")
relay := &Relay{Type: TerminalType, State: Established, LocalIndex: 5, RemoteIndex: 6, PeerAddr: flowTestRemote}
relayHI := &HostInfo{
vpnAddrs: []netip.Addr{relayAddr},
ConnectionState: relayCI,
relayState: RelayState{
relayForByAddr: map[netip.Addr]*Relay{flowTestRemote: relay},
relayForByIdx: map[uint32]*Relay{relay.LocalIndex: relay},
},
}
relayHI.remote.Store(&relayRemote)
sender.f.hostMap.Hosts[relayAddr] = relayHI
sender.hostinfo.remote.Store(&netip.AddrPort{})
sender.hostinfo.relayState.relays = []netip.Addr{relayAddr}
return relayRemote, relayHI, relayPeer
}
// affinityQueue makes every read start with an idle pipeline: before each read it waits until conn has sent what was
// read before, recording how many sends that took, then samples the reader's actual thread affinity.
type affinityQueue struct {
*flowBatchQueue
conn *affinityConn
sent int
boundaries []int
masks []unix.CPUSet
}
func (q *affinityQueue) Read() ([]tio.Packet, error) {
if n := q.reads.Load(); n > 0 {
for _, p := range q.batches[n-1] {
q.sent += p.SegmentCount()
}
}
q.boundaries = append(q.boundaries, q.conn.waitFor(q.sent))
q.masks = append(q.masks, threadMask())
return q.flowBatchQueue.Read()
}
// affinityConn captures every send, sampling the sending thread's actual affinity in each WriteBatch.
type affinityConn struct {
flowCaptureConn
masks []unix.CPUSet
sizes []int
}
func (c *affinityConn) WriteBatch(p [][]byte, addrs []netip.AddrPort) (int, error) {
mask := threadMask()
c.mu.Lock()
c.masks = append(c.masks, mask)
c.sizes = append(c.sizes, len(p))
c.mu.Unlock()
return c.flowCaptureConn.WriteBatch(p, addrs)
}
// waitFor waits until n packets were sent and returns how many WriteBatch calls that took.
func (c *affinityConn) waitFor(n int) int {
for len(c.captured()) < n {
time.Sleep(50 * time.Microsecond)
}
c.mu.Lock()
defer c.mu.Unlock()
return len(c.masks)
}
func threadMask() unix.CPUSet {
var mask unix.CPUSet
if err := unix.SchedGetaffinity(0, &mask); err != nil {
panic(err)
}
return mask
}
// requireFlowCPU checks every mask is just cpu, or with cpu -1, the whole allowed set.
func requireFlowCPU(t *testing.T, masks []unix.CPUSet, cpu int, allowed []int) {
t.Helper()
require.NotEmpty(t, masks)
for _, mask := range masks {
if cpu >= 0 {
require.Equal(t, 1, mask.Count())
require.True(t, mask.IsSet(cpu), "expected CPU %d", cpu)
continue
}
require.Equal(t, len(allowed), mask.Count())
for _, c := range allowed {
require.True(t, mask.IsSet(c))
}
}
}
// TestFlowStagedTXAffinity sends large, small and large reads, each from an idle pipeline, and checks the order on the
// wire and where the reader and each send actually ran, for every CPU placement, with one routine and with several. A
// staged sender is only pinned to an explicit entry other than its reader's CPU, and inline sends stay on the reader.
// With the threshold, only the small read goes inline.
// The reader only reads again once its earlier reads are on the wire, so this also checks the sender never waits for
// the reader. Pinning threads changes the process, so the test runs in a child.
func TestFlowStagedTXAffinity(t *testing.T) {
if os.Getenv("NEBULA_FLOW_AFFINITY_TEST") != "1" {
cmd := exec.Command(os.Args[0], "-test.run=^TestFlowStagedTXAffinity$")
cmd.Env = append(os.Environ(), "NEBULA_FLOW_AFFINITY_TEST=1")
out, err := cmd.CombinedOutput()
require.NoError(t, err, "%s", out)
return
}
allowed, err := util.AllowedCPUs()
require.NoError(t, err)
for _, placement := range []string{"fallback", "automatic", "explicit", "explicit-single"} {
for _, pin := range []bool{false, true} {
for _, sc := range sendPaths() {
for _, routines := range []int{1, 2} {
t.Run(fmt.Sprintf("%s/pin=%t/%s/routines=%d", placement, pin, stagedName(sc), routines), func(t *testing.T) {
if pin && len(allowed) < 2 {
t.Skip("requires at least two allowed CPUs")
}
if sc.inlineBelow > 0 {
// Switch on every read that wants the other mode, so the small read goes inline between
// staged ones
sc.gateHold = 1
}
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
f := sender.f
f.pinThreads, f.routines = pin, routines
switch placement {
case "automatic":
f.defaultCpuAffinity = cpupick.Default(routines, 4242, nil)
case "explicit":
f.cpuAffinity = []int{allowed[len(allowed)-1], allowed[0]}
case "explicit-single":
f.cpuAffinity = allowed[len(allowed)-1:]
}
readerCPU, sendCPU := -1, -1
if pin {
readerCPU = f.threadCPU(0)
sendCPU = readerCPU
if sc.tx {
// Never the reader's CPU, and no automatic choice: unpinned unless an explicit entry fits
sendCPU = -1
if placement == "explicit" {
sendCPU = f.cpuAffinity[1]
}
}
}
smallCPU := sendCPU
if sc.inlineBelow > 0 {
smallCPU = readerCPU
}
conn := &affinityConn{}
f.writers = []udp.Conn{conn}
batches, want := flowTestBatches(3, 8, 1, 45)
batches[1] = batches[1][:1]
want = append(want[:405], want[720:]...)
q := &affinityQueue{flowBatchQueue: newFlowBatchQueue(batches), conn: conn}
inline := metrics.GetOrRegisterCounter("flow.tx.inline", nil)
inline0 := inline.Count()
f.listenIn(q, 0)
require.Equal(t, want, decryptWire(t, receiver, conn.captured()))
if sc.inlineBelow > 0 {
require.Equal(t, int64(1), inline.Count()-inline0)
}
b := q.boundaries
requireFlowCPU(t, q.masks, readerCPU, allowed)
requireFlowCPU(t, conn.masks[:b[1]], sendCPU, allowed)
requireFlowCPU(t, conn.masks[b[1]:b[2]], smallCPU, allowed)
requireFlowCPU(t, conn.masks[b[2]:], sendCPU, allowed)
})
}
}
}
}
}
// sendPaths is inline, staged and, with the threshold, adaptive transmit.
func sendPaths() []stagedConfig {
return []stagedConfig{{}, stagedOn(2), adaptive(2)}
}
// requireAccepted checks the receiver accepts every packet of wire in the order given. With several readers the wire
// order differs from counter order, but every packet must still authenticate and fit within the replay window.
func requireAccepted(t *testing.T, receiver *flowTestPeer, wire [][]byte) {
t.Helper()
for _, pkt := range wire {
h := &header.H{}
require.NoError(t, h.Parse(pkt))
_, err := receiver.hostinfo.ConnectionState.Decrypt(receiver.f.l, h.MessageCounter, pkt, make([]byte, 12))
require.NoError(t, err, "counter %d", h.MessageCounter)
}
}
// slowConn delays every WriteBatch, then writes to the shared capture.
type slowConn struct {
*flowCaptureConn
delay time.Duration
}
func (c slowConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
time.Sleep(c.delay)
return c.flowCaptureConn.WriteBatch(bufs, addrs)
}
// TestFlowTXBudgetManyReaders saturates one tunnel from eight readers, one of them on a slow socket that lets the
// others overtake it, alongside a burst of unreserved control messages: in the order they reach the wire, every packet
// is accepted by the receiver's replay window, so none was sent twice or displaced past the window.
func TestFlowTXBudgetManyReaders(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
for _, window := range budgetWindows {
t.Run(fmt.Sprintf("window=%d", window), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(3))
ci := sender.hostinfo.ConnectionState
ci.window = NewBits(window)
receiver.hostinfo.ConnectionState.window = NewBits(window)
sender.f.routines = 8
sender.f.writers = append([]udp.Conn{slowConn{sender.conn, 4 * time.Millisecond}}, slices.Repeat(sender.f.writers, 8)...)
start := ci.messageCounter.Load()
var wg sync.WaitGroup
for q := range 8 {
wg.Go(func() {
batches, _ := flowTestBatches(28, 8, 2, 45)
p := newStagedTx(sender.f, newFlowBatchQueue(nil), q)
defer p.close()
for _, packets := range batches {
p.send(packets)
}
})
}
// A bounded burst of unreserved control messages uses the documented margin, on writer 8
wg.Go(func() {
for range 128 {
sender.f.sendNoMetrics(header.Test, header.TestRequest, ci, sender.hostinfo, flowTestUnder, []byte("x"), make([]byte, 12), make([]byte, mtu), 8)
time.Sleep(100 * time.Microsecond)
}
})
wg.Wait()
require.Zero(t, ci.txBudget.state.Load())
wire := sender.conn.captured()
require.Len(t, wire, 8*28*360+128)
require.Equal(t, start+uint64(len(wire)), ci.messageCounter.Load())
requireAccepted(t, receiver, wire)
})
}
}
// TestFlowStagedTXInlineOnlyWhenEmpty: a small read behind a staged batch still in flight must not go inline before
// that batch is on the wire, it would overtake it.
func TestFlowStagedTXInlineOnlyWhenEmpty(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sc := adaptive(2)
sc.inlineBelow, sc.gateHold = 4, 1
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
p := newStagedTx(sender.f, newFlowBatchQueue(nil), 0)
block := make(chan struct{})
delayCrypto(sender, func(uint64) { <-block })
batches, want := flowTestBatches(1, 4, 1, 3)
p.send(batches[0][:3]) // switches to fanout, its worker is blocked
require.Equal(t, 1, p.inFlight)
unblock := time.AfterFunc(20*time.Millisecond, func() { close(block) })
defer unblock.Stop()
p.send(batches[0][3:]) // switches to inline, which waits for the staged batch first
require.Zero(t, p.inFlight)
p.close()
require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured()))
}
// TestFlowStagedTXInlineBelowZero: with inline_below 0 an idle reader never sends a batch itself, in either
// direction: every read goes through the workers and the ordered sender or writer.
func TestFlowStagedTXInlineBelowZero(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
l := test.NewLogger()
c := config.NewC(l)
c.Settings = map[string]any{"tun": map[string]any{"staged": map[string]any{"workers": 2, "inline_below": 0}}}
sc := parseStagedConfig(c, l)
if !sc.tx {
t.Skip("transmit stays inline in this build")
}
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
batches, want := flowTestBatches(20, 2, 1, 4)
txInline := metrics.GetOrRegisterCounter("flow.tx.inline", nil)
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
rxInline := metrics.GetOrRegisterCounter("flow.rx.inline", nil)
txInline0, tx0, rxInline0, rx0 := txInline.Count(), tx.Count(), rxInline.Count(), rxBatches.Count()
sender.f.listenIn(newFlowBatchQueue(batches), 0)
wire := sender.conn.captured()
require.Equal(t, want, decryptWire(t, receiver, wire))
require.Zero(t, txInline.Count()-txInline0)
require.Equal(t, int64(len(batches)), tx.Count()-tx0)
receiver.receive(&rxConn{wire: wire, sizes: []int{1, 2, 4}})
require.Equal(t, want, receiver.tun.segments(t))
require.Zero(t, rxInline.Count()-rxInline0)
require.Positive(t, rxBatches.Count()-rx0)
}
// TestFlowStagedTXDisabled: with tx false the reader never builds a staged pipeline, even on a queue that could hand
// off its batches. Receive stays staged.
func TestFlowStagedTXDisabled(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
l := test.NewLogger()
c := config.NewC(l)
c.Settings = map[string]any{"tun": map[string]any{"staged": map[string]any{"workers": 2, "tx": false}}}
sc := parseStagedConfig(c, l)
require.False(t, sc.tx)
require.True(t, sc.rx)
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
batches, want := flowTestBatches(4, 8, 1, 45)
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
txInline := metrics.GetOrRegisterCounter("flow.tx.inline", nil)
tx0, txInline0, rx0 := tx.Count(), txInline.Count(), rxBatches.Count()
q := &swapCountQueue{flowBatchQueue: newFlowBatchQueue(batches)}
sender.f.listenIn(q, 0)
wire := sender.conn.captured()
require.Equal(t, want, decryptWire(t, receiver, wire))
require.Zero(t, q.swaps, "newStagedTx always takes the queue's buffer, so it never ran")
require.Equal(t, tx0, tx.Count())
require.Equal(t, txInline0, txInline.Count())
receiver.receive(&rxConn{wire: wire, batchLen: 512})
require.Equal(t, want, receiver.tun.segments(t))
require.Greater(t, rxBatches.Count(), rx0)
}
// swapCountQueue counts SwapRxBuf calls.
type swapCountQueue struct {
*flowBatchQueue
swaps int
}
func (q *swapCountQueue) SwapRxBuf(buf []byte) []byte {
q.swaps++
return q.flowBatchQueue.SwapRxBuf(buf)
}
// TestFlowStagedTXAdaptiveBudget: a fast reader's adaptive sends must stop spending credit until another reader's
// earlier reservation reaches the wire.
func TestFlowStagedTXAdaptiveBudget(t *testing.T) {
for _, shape := range []struct {
name string
packets, segments int
}{
{"single", 1, 1}, {"batch", 8, 30},
} {
t.Run(shape.name, func(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sc := adaptive(2)
// p0's large read must go to the workers at once, p1 stays inline
sc.gateHold = 1
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
sender.f.routines = 2
sender.f.writers = []udp.Conn{sender.conn, sender.conn}
ci := sender.hostinfo.ConnectionState
start := ci.messageCounter.Load()
entered, unblock := make(chan struct{}), make(chan struct{})
delayCrypto(sender, func(n uint64) {
if n == start+1 {
close(entered)
<-unblock
}
})
p0 := newStagedTx(sender.f, newFlowBatchQueue(nil), 0)
p1 := newStagedTx(sender.f, newFlowBatchQueue(nil), 1)
defer p0.close()
defer p1.close()
var release sync.Once
defer release.Do(func() { close(unblock) })
large, _ := flowTestBatches(1, 8, 1, 45)
p0.send(large[0])
<-entered
perBatch := shape.packets * shape.segments
small, _ := flowTestBatches(9600/perBatch, shape.packets, 1, shape.segments)
fits := int(ci.txBudgetLimit()-360) / perBatch
for _, packets := range small[:fits] {
p1.send(packets)
}
require.Equal(t, uint64(360+fits*perBatch), (ci.txBudget.state.Load()>>24)&txBudgetMask, "inline sends must spend shared credit")
done := make(chan struct{})
go func() {
defer close(done)
for _, packets := range small[fits:] {
p1.send(packets)
}
}()
select {
case <-done:
t.Error("adaptive reader bypassed outstanding credit")
case <-time.After(50 * time.Millisecond):
}
require.LessOrEqual(t, ci.messageCounter.Load()-start, ci.txBudgetLimit())
release.Do(func() { close(unblock) })
<-done
p0.drain()
p1.drain()
wire := sender.conn.captured()
require.Len(t, wire, 360+40*240)
requireAccepted(t, receiver, wire)
require.Zero(t, ci.txBudget.state.Load())
})
}
}
// BenchmarkFlowStagedTX runs reads of eight 45-segment superpackets through listenIn into a socket that discards
// them, inline and staged.
func BenchmarkFlowStagedTX(b *testing.B) {
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
for _, sc := range []stagedConfig{{}, stagedOn(4)} {
b.Run(cipher.CipherName()+"/"+stagedName(sc), func(b *testing.B) {
if sc.tx && cipher.CipherName() == noiseutil.CipherAESGCM.CipherName() && noiseutil.EncryptLockNeeded {
b.Skip("AES-GCM requires encryption in nonce order")
}
sender, _ := newFlowTestPair(b, cipher, sc)
sender.f.writers = []udp.Conn{udp.NoopConn{}}
batch, _ := flowTestBatches(1, 8, 4, 45)
batches := make([][]tio.Packet, b.N)
for i := range batches {
batches[i] = batch[0]
}
b.SetBytes(int64(8 * 45 * flowTestMSS))
b.ReportAllocs()
b.ResetTimer()
sender.f.listenIn(newFlowBatchQueue(batches), 0)
})
}
}
}
+1 -1
View File
@@ -29,7 +29,7 @@ require (
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
golang.zx2c4.com/wireguard v0.0.0-20260522210424-ecfc5a8d5446
golang.zx2c4.com/wireguard/windows v1.0.1
google.golang.org/protobuf v1.36.12
gopkg.in/yaml.v3 v3.0.1
+2 -2
View File
@@ -218,8 +218,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
golang.zx2c4.com/wireguard v0.0.0-20260522210424-ecfc5a8d5446 h1:cqHQ3AycTHvM2R7ikgyX57D+XvtcSnGylsLkOVhta/w=
golang.zx2c4.com/wireguard v0.0.0-20260522210424-ecfc5a8d5446/go.mod h1:rpwXGsirqLqN2L0JDJQlwOboGHmptD5ZD6T2VmcqhTw=
golang.zx2c4.com/wireguard/windows v1.0.1 h1:eOxiDVbywPC+ZQqvdCK7x+ZwWXKbYv50TtH8ysFIbw8=
golang.zx2c4.com/wireguard/windows v1.0.1/go.mod h1:+fbT3FFdX4zzYDLwJh5+HPEcNN/3HyNdzhNSVsQM+zs=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+165 -89
View File
@@ -18,20 +18,12 @@ import (
)
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
if hostinfo := f.filterInsidePacket(pkt, fwPacket, rejectBuf, q, localCache); hostinfo != nil {
f.sendInsideMessage(hostinfo, pkt, nb, sendBatch)
}
}
// filterInsidePacket parses and filters pkt, handling local traffic, destination lookup, pending handshakes and
// the outbound firewall. It returns the hostinfo to send pkt to, or nil when pkt was dropped, rejected, forwarded
// to self or cached for a pending handshake.
func (f *Interface) filterInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, rejectBuf []byte, q int, localCache firewall.ConntrackCache) *HostInfo {
// borrowed: pkt.Bytes is owned by the originating tio.Queue and is
// only valid until the next Read on that queue. Filtering and sending
// must finish before then. Self-forwarding writes synchronously and the
// handshake cache copies each segment. To retain the packet beyond the
// next Read, use pkt.Clone() to detach it from the borrow.
// only valid until the next Read on that queue. Every consumer below
// (parse, self-forward, handshake cache, sendInsideMessage) reads it
// synchronously; do not retain pkt outside this call. If a future
// caller needs to keep the packet, use pkt.Clone() to detach it from
// the borrow.
//
// pkt.Bytes is either one IP datagram (GSO zero) or a TSO/USO
// superpacket. In both cases the L3+L4 headers at the start describe
@@ -46,13 +38,13 @@ func (f *Interface) filterInsidePacket(pkt tio.Packet, fwPacket *firewall.Parsed
"error", err,
)
}
return nil
return
}
// Ignore local broadcast packets
if f.dropLocalBroadcast {
if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
return nil
return
}
}
@@ -79,12 +71,12 @@ func (f *Interface) filterInsidePacket(pkt tio.Packet, fwPacket *firewall.Parsed
}
// Otherwise, drop. On linux, we should never see these packets - Linux
// routes packets from the nebula addr to the nebula addr through the loopback device.
return nil
return
}
// Ignore multicast packets
if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() {
return nil
return
}
hostinfo, ready := f.getOrHandshakeConsiderRouting(&fwPacket.Packet, func(hh *HandshakeHostInfo) {
@@ -111,26 +103,50 @@ func (f *Interface) filterInsidePacket(pkt tio.Packet, fwPacket *firewall.Parsed
"fwPacket", fwPacket,
)
}
return nil
return
}
if !ready {
return nil
return
}
dropReason := f.firewall.Drop(fwPacket.Packet, false, hostinfo, f.pki.GetCAPool(), localCache)
if dropReason == nil {
return hostinfo
f.sendInsideMessage(hostinfo, pkt, nb, sendBatch)
} else {
f.rejectInside(packet, rejectBuf, q)
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("dropping outbound packet",
"fwPacket", fwPacket,
"reason", dropReason,
)
}
}
}
func (f *Interface) sendInsideEncrypt(hostinfo *HostInfo, ci *ConnectionState, seg, scratch, nb []byte) []byte {
if noiseutil.EncryptLockNeeded {
ci.writeLock.Lock()
}
c := ci.messageCounter.Add(1)
out := header.Encode(scratch, header.Version, header.Message, 0, hostinfo.remoteIndexId, c)
out, encErr := ci.eKey.EncryptDanger(out, out, seg, c, nb)
if noiseutil.EncryptLockNeeded {
ci.writeLock.Unlock()
}
if encErr != nil {
hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet",
"error", encErr,
"udpAddr", hostinfo.GetRemote(),
"counter", c,
)
// Skip this segment; the rest of the superpacket can still go out. TCP will retransmit anything we drop here.
return nil
}
f.rejectInside(packet, rejectBuf, q)
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("dropping outbound packet",
"fwPacket", fwPacket,
"reason", dropReason,
)
}
return nil
return out
}
// sendInsideMessage encrypts a firewall-approved inside packet (or every
@@ -140,46 +156,88 @@ func (f *Interface) filterInsidePacket(pkt tio.Packet, fwPacket *firewall.Parsed
// scratch arena: SegmentSuperpacket builds each segment's plaintext in
// segScratch[:segLen] in turn, and we encrypt directly into a fresh SendBatch slot.
func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt tio.Packet, nb []byte, sendBatch *batch.SendBatch) {
var it txItem
if f.prepareTxItem(&it, hostinfo, pkt) {
f.encryptTxItem(&it, nb, sendBatch)
ci := hostinfo.ConnectionState
if ci.eKey == nil {
return
}
}
// recordOutbound marks traffic out on hostinfo, once per packet or superpacket, and when we rebound since this tunnel
// last sent, asks the lighthouse to get the far side punching at us again.
func (f *Interface) recordOutbound(hostinfo *HostInfo) {
// One traffic-out mark covers every segment of the superpacket; doing it
// per segment in sendInsideEncrypt paid an atomic store up to ~45 extra
// times per TSO packet, inside writeLock under boring crypto.
//
// We rebound since this tunnel last sent, ask the lighthouse to get the far side punching at us again
if f.connectionManager.Out(hostinfo) {
f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Lighthouse update triggered for punch due to rebind epoch", "vpnAddrs", hostinfo.vpnAddrs)
hostinfo.logger(f.l).Debug("Lighthouse update triggered for punch due to rebind epoch",
"vpnAddrs", hostinfo.vpnAddrs,
)
}
}
}
// findRelay returns the first relay that can reach hostinfo, forgetting the ones that no longer resolve.
func (f *Interface) findRelay(hostinfo *HostInfo) (*HostInfo, *Relay) {
for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
via, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
if err == nil {
return via, relay
remote := hostinfo.GetRemote()
if !remote.IsValid() { //the relay path
//first, find our relay hostinfo:
var relayHostInfo *HostInfo
var relay *Relay
var err error
for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
relayHostInfo, relay, err = f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
if err != nil {
hostinfo.relayState.DeleteRelay(relayIP)
hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo",
"relay", relayIP,
"error", err,
)
continue
}
break
}
if relayHostInfo == nil || relay == nil {
//failure already logged
return
}
hostinfo.relayState.DeleteRelay(relayIP)
hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo", "relay", relayIP, "error", err)
}
return nil, nil
}
// encryptMessage encodes the header for counter c and encrypts p behind it into out. The caller allocates c and
// holds writeLock when the cipher requires encryption in nonce order.
func (f *Interface) encryptMessage(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, ci *ConnectionState, c uint64, p, nb, out []byte, remote netip.AddrPort) []byte {
out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
out, err := ci.eKey.EncryptDanger(out, out, p, c, nb)
if err != nil {
hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet", "error", err, "udpAddr", remote, "counter", c)
err = tio.SegmentSuperpacket(pkt, func(seg []byte) error {
//relay header + header + plaintext + AEAD tag + relay tag
scratch := sendBatch.Reserve(len(seg) + header.MaxOverhead)
innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb)
if innerPacket == nil {
return nil
}
//now we need to do a relay-encrypt:
toSend, err := f.prepareSendVia(relayHostInfo, relay, innerPacket, nb, scratch, true)
if err != nil {
//already logged
return nil
}
sendBatch.Commit(toSend, relayHostInfo.GetRemote())
return nil
})
if err != nil {
hostinfo.logger(f.l).Error("Failed to segment superpacket for relay send", "error", err)
}
return
}
err := tio.SegmentSuperpacket(pkt, func(seg []byte) error {
// header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305)
scratch := sendBatch.Reserve(header.Len + len(seg) + 16)
out := f.sendInsideEncrypt(hostinfo, ci, seg, scratch, nb)
if out == nil {
return nil
}
sendBatch.Commit(out, remote)
return nil
})
if err != nil {
hostinfo.logger(f.l).Error("Failed to segment superpacket for send", "error", err)
}
return out
}
func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
@@ -388,33 +446,27 @@ func (f *Interface) prepareSendVia(via *HostInfo,
nb,
out []byte,
nocopy bool,
c uint64,
) ([]byte, error) {
// A nonzero c was reserved by the caller, who also did the connection manager bookkeeping
reserved := c != 0
locked := !reserved && noiseutil.EncryptLockNeeded
if locked {
if noiseutil.EncryptLockNeeded {
// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
via.ConnectionState.writeLock.Lock()
}
if !reserved {
var ok bool
c, ok = via.ConnectionState.NextMessageCounter()
if !ok {
if locked {
via.ConnectionState.writeLock.Unlock()
}
f.dropExhausted(via, c, "Dropping outbound relay packets, tunnel message counter is exhausted")
return nil, fmt.Errorf("tunnel message counter is exhausted")
c, ok := via.ConnectionState.NextMessageCounter()
if !ok {
if noiseutil.EncryptLockNeeded {
via.ConnectionState.writeLock.Unlock()
}
f.connectionManager.OutNoRebind(via)
f.dropExhausted(via, c, "Dropping outbound relay packets, tunnel message counter is exhausted")
return nil, fmt.Errorf("tunnel message counter is exhausted")
}
out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
f.connectionManager.OutNoRebind(via)
// Authenticate the header and payload, but do not encrypt for this message type.
// The payload consists of the inner, unencrypted Nebula header, as well as the end-to-end encrypted payload.
if len(out)+len(ad)+via.ConnectionState.eKey.Overhead() > cap(out) {
if locked {
if noiseutil.EncryptLockNeeded {
via.ConnectionState.writeLock.Unlock()
}
via.logger(f.l).Error("SendVia out buffer not large enough for relay",
@@ -438,16 +490,14 @@ func (f *Interface) prepareSendVia(via *HostInfo,
var err error
out, err = via.ConnectionState.eKey.EncryptDanger(out, out, nil, c, nb)
if locked {
if noiseutil.EncryptLockNeeded {
via.ConnectionState.writeLock.Unlock()
}
if err != nil {
via.logger(f.l).Info("Failed to EncryptDanger in sendVia", "error", err)
return nil, err
}
if !reserved {
f.connectionManager.RelayUsed(relay.LocalIndex)
}
f.connectionManager.RelayUsed(relay.LocalIndex)
return out, nil
}
@@ -460,7 +510,7 @@ func (f *Interface) prepareSendVia(via *HostInfo,
// out is a buffer used to store the result of the Encrypt operation
// q indicates which writer to use to send the packet.
func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, nocopy bool, q int) {
toSend, err := f.prepareSendVia(via, relay, ad, nb, out, nocopy, 0)
toSend, err := f.prepareSendVia(via, relay, ad, nb, out, nocopy)
if err != nil {
// already logged by prepareSendVia
return
@@ -514,19 +564,34 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
return
}
// A closing tunnel needs neither traffic tracking nor another punch.
if t != header.CloseTunnel {
f.recordOutbound(hostinfo)
}
out = f.encryptMessage(t, st, hostinfo, ci, c, p, nb, out, remote)
if noiseutil.EncryptLockNeeded {
ci.writeLock.Unlock()
}
if out == nil {
return
//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
// A closing tunnel is torn down right after this, so skip the connection manager entirely: no point recording
// traffic or asking the lighthouse for a punch. Otherwise, if we rebound since this tunnel last sent, ask the
// lighthouse to get the far side punching at us again.
if t != header.CloseTunnel && f.connectionManager.Out(hostinfo) {
f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
if f.l.Enabled(context.Background(), slog.LevelDebug) {
f.l.Debug("Lighthouse update triggered for punch due to rebind epoch",
"vpnAddrs", hostinfo.vpnAddrs,
)
}
}
var err error
out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
if noiseutil.EncryptLockNeeded {
ci.writeLock.Unlock()
}
if err != nil {
hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet",
"error", err,
"udpAddr", remote,
"counter", c,
)
return
}
if remote.IsValid() {
err = f.writers[q].WriteTo(out, remote)
if err != nil {
@@ -548,8 +613,19 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
}
}
} else {
if via, relay := f.findRelay(hostinfo); via != nil && relay != nil {
f.SendVia(via, relay, out, nb, fullOut[:header.Len+len(out)], true, q)
// Try to send via a relay
for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
if err != nil {
hostinfo.relayState.DeleteRelay(relayIP)
hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo",
"relay", relayIP,
"error", err,
)
continue
}
f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true, q)
break
}
}
}
-180
View File
@@ -1,180 +0,0 @@
package nebula
import (
"net/netip"
"testing"
"time"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/overlaytest"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newInsideFilterTestInterface(t *testing.T, allow bool) (*Interface, *HostInfo, *capturingTun) {
t.Helper()
local := netip.MustParseAddr("192.0.2.1")
remote := netip.MustParseAddr("192.0.2.2")
f, tun := newSelfForwardInterface(local)
f.inside = overlaytest.NoopTun{}
f.myVpnNetworksTable = &bart.Lite{}
f.myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.0/24"))
f.myBroadcastAddrsTable.Insert(netip.MustParsePrefix("192.0.2.255/32"))
f.dropLocalBroadcast, f.dropMulticast = true, true
localCert := &dummyCert{version: cert.Version1, networks: []netip.Prefix{netip.PrefixFrom(local, 24)}}
peerCert := &dummyCert{version: cert.Version1, networks: []netip.Prefix{netip.PrefixFrom(remote, 24)}}
f.firewall = NewFirewall(f.l, time.Minute, time.Minute, time.Minute, localCert)
if allow {
require.NoError(t, f.firewall.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
}
f.pki = &PKI{}
f.pki.caPool.Store(cert.NewCAPool())
f.hostMap = newHostMap(f.l)
f.lightHouse = newTestLighthouse()
f.lightHouse.amLighthouse = true // Keep destination lookup from probing other remotes.
f.handshakeManager = NewHandshakeManager(f.l, f.hostMap, f.lightHouse, udp.NoopConn{}, defaultHandshakeConfig)
f.handshakeManager.f = f
f.connectionManager = &connectionManager{intf: f}
h := &HostInfo{
vpnAddrs: []netip.Addr{remote},
ConnectionState: &ConnectionState{
peerCert: &cert.CachedCertificate{Certificate: peerCert},
},
}
f.hostMap.Hosts[remote] = h
return f, h, tun
}
func TestFilterInsidePacket(t *testing.T) {
local := netip.MustParseAddr("192.0.2.1")
remote := netip.MustParseAddr("192.0.2.2")
for _, tt := range []struct {
name string
packet []byte
allow bool
reject bool
ready bool
writes int
}{
{name: "allowed", packet: buildIPv4(local, remote, tcpSyn), allow: true, ready: true},
{name: "malformed", packet: []byte{0x45}, allow: true},
{name: "broadcast", packet: buildIPv4(local, netip.MustParseAddr("192.0.2.255"), udpDatagram), allow: true},
{name: "multicast", packet: buildIPv4(local, netip.MustParseAddr("224.0.0.1"), udpDatagram), allow: true},
{name: "firewall drop", packet: buildIPv4(local, remote, tcpSyn)},
{name: "firewall reject", packet: buildIPv4(local, remote, tcpSyn), reject: true, writes: 1},
{name: "unroutable drop", packet: buildIPv4(local, netip.MustParseAddr("198.51.100.1"), tcpSyn), allow: true},
{name: "unroutable reject", packet: buildIPv4(local, netip.MustParseAddr("198.51.100.1"), tcpSyn), allow: true, reject: true, writes: 1},
} {
t.Run(tt.name, func(t *testing.T) {
f, h, tun := newInsideFilterTestInterface(t, tt.allow)
f.firewall.OutboundSendReject = tt.reject
fp := &firewall.ParsedPacket{}
got := f.filterInsidePacket(tio.Packet{Bytes: tt.packet}, fp, make([]byte, mtu), 0, nil)
if tt.ready {
require.Same(t, h, got)
assert.Equal(t, local, fp.LocalAddr)
assert.Equal(t, remote, fp.RemoteAddr)
} else {
require.Nil(t, got)
}
assert.Zero(t, h.ConnectionState.messageCounter.Load(), "filtering must not allocate encryption counters")
assert.False(t, h.sentSinceCheck(), "filtering must not mark outbound traffic")
require.Len(t, tun.writes, tt.writes)
if tt.writes > 0 {
require.GreaterOrEqual(t, len(tun.writes[0]), 40)
assert.Equal(t, tt.packet[16:20], tun.writes[0][12:16])
assert.Equal(t, tt.packet[12:16], tun.writes[0][16:20])
assert.NotZero(t, tun.writes[0][33]&0x04, "reject must carry a TCP reset")
}
})
}
}
func TestFilterInsidePacketSelf(t *testing.T) {
local := netip.MustParseAddr("192.0.2.1")
f, _, tun := newInsideFilterTestInterface(t, true)
pkt := buildIPv4(local, local, tcpSyn)
require.Nil(t, f.filterInsidePacket(tio.Packet{Bytes: pkt}, &firewall.ParsedPacket{}, nil, 0, nil))
if immediatelyForwardToSelf {
require.Len(t, tun.writes, 1)
assert.Equal(t, pkt, tun.writes[0])
} else {
assert.Empty(t, tun.writes)
}
}
func TestFilterInsidePacketPendingHandshake(t *testing.T) {
f, h, tun := newInsideFilterTestInterface(t, true)
remote := h.vpnAddrs[0]
delete(f.hostMap.Hosts, remote)
hh := &HandshakeHostInfo{hostinfo: h}
f.handshakeManager.vpnIps[remote] = hh
pkt := buildIPv4(netip.MustParseAddr("192.0.2.1"), remote, udpDatagram)
want := append([]byte(nil), pkt...)
require.Nil(t, f.filterInsidePacket(tio.Packet{Bytes: pkt}, &firewall.ParsedPacket{}, nil, 0, nil))
require.Len(t, hh.packetStore, 1)
clear(pkt)
assert.Equal(t, want, hh.packetStore[0].packet, "the handshake cache must own its packet")
assert.Equal(t, header.Message, hh.packetStore[0].messageType)
assert.Empty(t, tun.writes)
assert.Zero(t, h.ConnectionState.messageCounter.Load())
}
type insideFilterConn struct {
udp.NoopConn
packets [][]byte
}
func (c *insideFilterConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
for _, b := range bufs {
c.packets = append(c.packets, append([]byte(nil), b...))
}
return len(bufs), nil
}
func TestConsumeInsidePacketFilterAndSend(t *testing.T) {
for _, allow := range []bool{false, true} {
t.Run(map[bool]string{false: "denied", true: "allowed"}[allow], func(t *testing.T) {
f, h, _ := newInsideFilterTestInterface(t, allow)
initR, respR := runTestHandshake(t)
ci, err := newConnectionStateFromResult(initR)
require.NoError(t, err)
rci, err := newConnectionStateFromResult(respR)
require.NoError(t, err)
ci.peerCert = h.ConnectionState.peerCert
h.ConnectionState = ci
h.remoteIndexId = 100
remote := netip.MustParseAddrPort("198.51.100.2:4242")
h.remote.Store(&remote)
conn := &insideFilterConn{}
sb := batch.NewSendBatch(conn, 1, mtu)
pkt := buildIPv4(netip.MustParseAddr("192.0.2.1"), h.vpnAddrs[0], udpDatagram)
before := ci.messageCounter.Load()
f.consumeInsidePacket(tio.Packet{Bytes: pkt}, &firewall.ParsedPacket{}, make([]byte, 12), sb, nil, 0, nil)
_, err = sb.Flush()
require.NoError(t, err)
if !allow {
assert.Empty(t, conn.packets)
assert.Equal(t, before, ci.messageCounter.Load())
assert.False(t, h.sentSinceCheck())
return
}
require.Len(t, conn.packets, 1)
assert.True(t, h.sentSinceCheck())
hdr := &header.H{}
require.NoError(t, hdr.Parse(conn.packets[0]))
assert.Equal(t, header.Message, hdr.Type)
assert.Equal(t, h.remoteIndexId, hdr.RemoteIndex)
assert.Equal(t, before+1, hdr.MessageCounter)
plain, err := rci.Decrypt(f.l, hdr.MessageCounter, conn.packets[0], make([]byte, 12))
require.NoError(t, err)
assert.Equal(t, pkt, plain)
})
}
}
+22 -53
View File
@@ -57,19 +57,14 @@ type InterfaceConfig struct {
// CpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
// should pin to. Queue i pins to CpuAffinity[i % len(CpuAffinity)] —
// shorter lists than `routines` cycle. Staged sender i uses entry i+1,
// unless that is its reader's CPU. Empty uses the automatic order. Only
// consulted when PinThreads is true.
// shorter lists than `routines` cycle. Empty list keeps the default
// pin-to-(i % NumCPU) behavior. Only consulted when PinThreads is true.
CpuAffinity []int
// defaultCpuAffinity is the automatic pin order for readers, kept apart
// from CpuAffinity because staged senders are only pinned to an explicit list.
defaultCpuAffinity []int
// PinThreads controls whether each TUN reader OS thread is pinned to a
// single CPU (via tun.pin_threads, default true). Pinning keeps each
// goroutine's sendmmsg on one XPS-selected NIC TX ring so per-flow
// packets stay ordered on the wire.
PinThreads bool
staged stagedConfig
l *slog.Logger
}
@@ -99,13 +94,11 @@ type Interface struct {
// should pin to. Queue i pins to cpuAffinity[i % len(cpuAffinity)].
// Empty falls back to the default pin-to-(allowed CPU) behavior.
// Only consulted when pinThreads is true.
cpuAffinity []int
defaultCpuAffinity []int
cpuAffinity []int
// pinThreads controls whether listenIn pins each TUN reader OS thread to
// a CPU at all (tun.pin_threads, default true). When false, threads are
// left free to migrate as on stock nebula.
pinThreads bool
staged stagedConfig
relayManager *relayManager
tryPromoteEvery atomic.Uint32
@@ -254,9 +247,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
connectionManager: c.connectionManager,
conntrackCacheTimeout: c.ConntrackCacheTimeout,
cpuAffinity: c.CpuAffinity,
defaultCpuAffinity: c.defaultCpuAffinity,
pinThreads: c.PinThreads,
staged: c.staged,
udpTxErrLimiter: udpTxErrLimiter,
udpRelayTxErrLimiter: udpRelayTxErrLimiter,
txOversizeLimiter: txOversizeLimiter,
@@ -422,19 +413,15 @@ func (f *Interface) listenOut(i int) {
f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, payload, rxc)
}
flusher := func() { f.flushOutside(rxc) }
var err error
if hc, ok := li.(handoffConn); ok && f.staged.rx {
// Every stage on its own goroutine, see flow_rx.go. rxc becomes the ordered writer's context.
err = newStagedRx(f, i, rxc).listen(hc)
} else {
if f.staged.rx {
f.l.Info("udp conn cannot hand off receive batches, receive stays inline", "reader", i)
flusher := func() {
if err := f.batchers[i].Flush(); err != nil {
f.l.Error("Failed to flush tun coalescer", "error", err)
}
err = li.ListenOut(listener, flusher)
clear(rxc.hostmapCache)
}
err := li.ListenOut(listener, flusher)
// An error after teardown began is shutdown noise, the closed flag covers resources
// Close releases itself and the cancelled ctx covers ones torn down by their owners
// reacting to it, like the user device pipes
@@ -446,30 +433,32 @@ func (f *Interface) listenOut(i int) {
f.l.Debug("underlay reader is done", "reader", i)
}
// threadCPU is the CPU tun reader i pins to.
func (f *Interface) threadCPU(i int) int {
func (f *Interface) pinThisThread(i int) {
var cpu int
if n := len(f.cpuAffinity); n > 0 {
// Explicit tun.cpu_affinity list wins; parseCpuAffinity already
// validated the entries against the allowed CPU set.
return f.cpuAffinity[i%n]
} else if n := len(f.defaultCpuAffinity); n > 0 {
return f.defaultCpuAffinity[i%n]
cpu = f.cpuAffinity[i%n]
} else if allowed, err := util.AllowedCPUs(); err == nil && len(allowed) > 0 {
// Default: spread queues across the CPUs we're actually allowed to
// run on. Under a cpuset/taskset mask these aren't 0..NumCPU-1, so
// i % NumCPU would pick unrunnable IDs and every pin would fail.
return allowed[i%len(allowed)]
cpu = allowed[i%len(allowed)]
} else {
cpu = i % runtime.NumCPU()
}
return i % runtime.NumCPU()
}
func (f *Interface) pinThreadToCPU(i, cpu int) {
if err := util.PinThreadToCPU(cpu); err != nil {
f.l.Warn("failed to pin tun processing thread to CPU", "queue", i, "cpu", cpu, "err", err)
f.l.Warn("failed to pin tun reader to CPU", "queue", i, "cpu", cpu, "err", err)
}
}
func (f *Interface) listenIn(queue tio.Queue, i int) {
// Pinning this thread (and goroutine) to a single CPU keeps every sendmmsg from this goroutine going through the
// same TX ring on the nic, so the wire sees per-flow order. Skip entirely when tun.pin_threads is false.
if f.pinThreads {
f.pinThisThread(i)
}
rejectBuf := make([]byte, mtu)
arenaSize := batch.SendBatchCap * (udp.MTU + 32)
sb := batch.NewSendBatch(f.writers[i], batch.SendBatchCap, arenaSize)
@@ -478,26 +467,6 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
if sq, ok := queue.(handoffQueue); ok && f.staged.tx {
// Every stage on its own goroutine, see flow_tx.go
err := newStagedTx(f, sq, i).run(fwPacket, rejectBuf, conntrackCache)
if !f.closed.Load() && f.ctx.Err() == nil {
f.l.Error("Error while reading outbound packet, closing", "error", err, "reader", i)
f.onFatal(err)
}
f.l.Debug("overlay reader is done", "reader", i)
return
}
if f.staged.tx {
f.l.Info("tun queue cannot hand off read batches, transmit stays inline", "reader", i)
}
// Pinning this thread (and goroutine) to a single CPU keeps every sendmmsg from this goroutine going through the
// same TX ring on the nic, so the wire sees per-flow order. Skip entirely when tun.pin_threads is false.
if f.pinThreads {
f.pinThreadToCPU(i, f.threadCPU(i))
}
for {
pkts, err := queue.Read()
if err != nil {
+1 -5
View File
@@ -244,8 +244,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
pinThreads := c.GetBool("tun.pin_threads", true)
cpuAffinity := parseCpuAffinity(c, l, routines)
var defaultCpuAffinity []int
staged := parseStagedConfig(c, l)
if pinThreads && routines > 1 && len(cpuAffinity) == 0 && !configTest {
// The operator didn't choose pin CPUs, so pick a default set that
// prefers performance cores and doesn't stack co-located instances
@@ -272,7 +270,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
l.Warn("tun.pin_threads_key is invalid, using PID")
}
defaultCpuAffinity = cpupick.Default(routines, key, l)
cpuAffinity = cpupick.Default(routines, key, l)
}
ifConfig := &InterfaceConfig{
@@ -297,9 +295,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
punchy: punchy,
ConntrackCacheTimeout: conntrackCacheTimeout,
CpuAffinity: cpuAffinity,
defaultCpuAffinity: defaultCpuAffinity,
PinThreads: pinThreads,
staged: staged,
l: l,
}
-69
View File
@@ -5,14 +5,12 @@ import (
"net/netip"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/slackhq/nebula/cert"
cert_test "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/util"
"github.com/stretchr/testify/assert"
@@ -125,70 +123,3 @@ firewall:
require.NoError(t, err)
require.Nil(t, ctrl, "a config test hands back nothing to stop, so it must stop itself")
}
func TestParseStagedConfig(t *testing.T) {
l := test.NewLogger()
staged := func(workers any) stagedConfig {
c := config.NewC(l)
if workers != nil {
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": workers}}
}
return parseStagedConfig(c, l)
}
for _, workers := range []any{nil, -1, 0} {
require.Equal(t, stagedConfig{}, staged(workers))
}
if runtime.GOOS != "linux" {
require.Equal(t, stagedConfig{}, staged(2))
return
}
want := stagedConfig{workers: 2, tx: !noiseutil.EncryptLockNeeded, rx: true, inlineBelow: 256, gateHold: stagedGateHold,
txSlots: 8, rxSlots: 6}
require.Equal(t, want, staged(2))
c := config.NewC(l)
c.Settings["listen"] = map[string]any{"batch": 128}
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "tx_slots": 3, "rx_slots": 1}}
got := parseStagedConfig(c, l)
require.Equal(t, []int{3, 2}, []int{got.txSlots, got.rxSlots}, "overrides apply, down to the minimum")
c = config.NewC(l)
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "tx": false, "inline_below": 0}}
got = parseStagedConfig(c, l)
require.False(t, got.tx, "tx: false keeps transmit inline")
require.True(t, got.rx)
require.Zero(t, got.inlineBelow, "inline_below: 0 hands every batch off")
c = config.NewC(l)
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "inline_below": 32}}
require.Equal(t, 32, parseStagedConfig(c, l).inlineBelow)
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "inline_below": -1}}
require.Zero(t, parseStagedConfig(c, l).inlineBelow, "a negative threshold is 0")
for _, tt := range []struct {
hold any
want int
}{{nil, stagedGateHold}, {8, 8}, {0, 1}, {-3, 1}} {
c = config.NewC(l)
settings := map[string]any{"workers": 2}
if tt.hold != nil {
settings["gate_hold"] = tt.hold
}
c.Settings["tun"] = map[string]any{"staged": settings}
require.Equal(t, tt.want, parseStagedConfig(c, l).gateHold, "gate_hold %v", tt.hold)
}
}
func TestStagedSlots(t *testing.T) {
for _, tt := range []struct{ workers, batch, tx, rx int }{
{workers: 2, batch: 64, tx: 8, rx: 6},
{workers: 12, batch: 64, tx: 14, rx: 6},
{workers: 2, batch: 96, tx: 8, rx: 4},
{workers: 2, batch: 512, tx: 8, rx: 4},
{workers: 2, batch: 32, tx: 8, rx: 6},
{workers: 2, batch: 1, tx: 8, rx: 6},
} {
tx, rx := stagedSlots(tt.workers, tt.batch, 0, 0)
require.Equal(t, []int{tt.tx, tt.rx}, []int{tx, rx}, "workers %d batch %d", tt.workers, tt.batch)
}
}
+10 -28
View File
@@ -130,10 +130,17 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
}
out, err := hostinfo.ConnectionState.Decrypt(f.l, h.MessageCounter, packet, rxc.nb)
if !f.finishOutsideDecrypt(hostinfo, via, h.MessageCounter, err) {
if err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Failed to decrypt packet", "error", err, "from", via, "header", h)
}
return
}
// Roam before we respond
f.handleHostRoaming(hostinfo, via)
f.connectionManager.In(hostinfo)
switch h.Type {
case header.Message:
switch h.Subtype {
@@ -466,36 +473,11 @@ func parseV4(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
return nil
}
// finishOutsideDecrypt logs a failed decrypt, or roams and records the traffic for an authenticated packet before it
// is handled. It reports whether the packet authenticated.
func (f *Interface) finishOutsideDecrypt(hostinfo *HostInfo, via ViaSender, counter uint64, err error) bool {
if err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Failed to decrypt packet", "error", err, "from", via, "counter", counter)
}
return false
}
// Roam before we respond
f.handleHostRoaming(hostinfo, via)
f.connectionManager.In(hostinfo)
return true
}
// flushOutside ends a receive batch: it writes out the tun coalescer and drops the batch's hostmap cache.
func (f *Interface) flushOutside(rxc *rxContext) {
if err := f.batchers[rxc.q].Flush(); err != nil {
f.l.Error("Failed to flush tun coalescer", "error", err)
}
clear(rxc.hostmapCache)
}
func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounter uint64, out []byte, rxc *rxContext) {
err := newPacket(out, true, rxc.fwPacket)
if err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out)
return
}
hostinfo.logger(f.l).Warn("Error while validating inbound packet", "error", err, "packet", out)
return
}
dropReason := f.firewall.Drop(rxc.fwPacket.Packet, true, hostinfo, f.pki.GetCAPool(), rxc.ctCache.Get())
+4 -9
View File
@@ -43,14 +43,6 @@ func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort) {
b.dsts = append(b.dsts, dst)
}
// Reset drops every queued packet without sending it.
func (b *SendBatch) Reset() {
clear(b.bufs)
b.bufs = b.bufs[:0]
b.dsts = b.dsts[:0]
b.arena.Reset()
}
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations
// were undeliverable; the batch is drained either way.
func (b *SendBatch) Flush() (int, error) {
@@ -59,6 +51,9 @@ func (b *SendBatch) Flush() (int, error) {
if len(b.bufs) > 0 {
written, err = b.out.WriteBatch(b.bufs, b.dsts)
}
b.Reset()
clear(b.bufs)
b.bufs = b.bufs[:0]
b.dsts = b.dsts[:0]
b.arena.Reset()
return written, err
}
-9
View File
@@ -75,15 +75,6 @@ type GSOInfo struct {
// superpacket that needs segmentation before its bytes can be encrypted and sent on the wire.
func (g GSOInfo) IsSuperpacket() bool { return g.Size > 0 }
// SegmentCount reports how many segments SegmentSuperpacket will cut p into, without touching its bytes.
func (p Packet) SegmentCount() int {
if !p.GSO.IsSuperpacket() {
return 1
}
n := (len(p.Bytes) - int(p.GSO.HdrLen) + int(p.GSO.Size) - 1) / int(p.GSO.Size)
return max(n, 1)
}
// Clone returns a Packet whose Bytes is a freshly allocated copy of p.Bytes,
// safe to retain past the next Read or Close on the originating Queue.
// GSO metadata is copied verbatim.
-15
View File
@@ -219,21 +219,6 @@ func (r *Offload) Read() ([]Packet, error) {
return r.pending, nil
}
// SwapRxBuf installs buf as the receive buffer for the following Reads and returns the one backing the packets
// of the most recent Read, so the caller can keep them past the next Read. A nil buf allocates a replacement of
// the required size. Single-reader only, like Read.
func (r *Offload) SwapRxBuf(buf []byte) []byte {
if buf == nil {
buf = make([]byte, tunRxBufCap)
}
if len(buf) < tunRxBufCap {
panic(fmt.Sprintf("tio: SwapRxBuf buffer is %d bytes, need %d", len(buf), tunRxBufCap))
}
old := r.rxBuf
r.rxBuf = buf
return old
}
// logDroppedRead reports a tun packet dropped for a bad/unsupported virtio
// header. Debug-gated so the happy path never pays for attribute assembly.
func (r *Offload) logDroppedRead(err error) {
-19
View File
@@ -911,25 +911,6 @@ func TestOffloadWriteZeroLength(t *testing.T) {
}
}
// TestOffloadSwapRxBuf: SwapRxBuf hands over the buffer the last Read filled and reads into the one it is given.
func TestOffloadSwapRxBuf(t *testing.T) {
o := &Offload{rxBuf: make([]byte, tunRxBufCap)}
read := o.rxBuf
owned := o.SwapRxBuf(nil)
if &owned[0] != &read[0] || len(o.rxBuf) != tunRxBufCap || &o.rxBuf[0] == &read[0] {
t.Fatal("SwapRxBuf(nil) must hand over the last read buffer and allocate a new one")
}
if back := o.SwapRxBuf(owned); &back[0] == &owned[0] || &o.rxBuf[0] != &owned[0] {
t.Fatal("SwapRxBuf must install the buffer it is given")
}
defer func() {
if recover() == nil {
t.Fatal("SwapRxBuf must reject a short buffer")
}
}()
o.SwapRxBuf(make([]byte, 10))
}
// TestWriteGSOSuperpacketGeometry decodes the vnet header the kernel would see for a multi-segment write:
// the GSO type must match the proto and IP version
// gso_size must be the per-segment size (the kernel rejects a superpacket with gso_size == 0),
+4 -19
View File
@@ -2,9 +2,7 @@ package nebula
import (
"crypto/fips140"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -235,14 +233,12 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError {
caPool, err := loadCAPoolFromConfig(p.l, c)
if caPool != nil {
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
}
if err != nil {
return util.NewContextualError("Failed to load ca from config", nil, err)
}
p.caPool.Store(caPool)
p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
return nil
}
@@ -617,22 +613,11 @@ func loadCAPoolFromConfig(l *slog.Logger, c *config.C) (*cert.CAPool, error) {
bl := c.GetStringSlice("pki.blocklist", []string{})
if len(bl) > 0 {
var bad []string
for i, raw := range bl {
// Fingerprints are compared as lowercase hex strings, a malformed entry would silently block nothing
fp := strings.ToLower(strings.TrimSpace(raw))
if b, err := hex.DecodeString(fp); err != nil || len(b) != sha256.Size {
bad = append(bad, fmt.Sprintf("entry %d %q", i+1, raw))
continue
}
for _, fp := range bl {
caPool.BlocklistFingerprint(fp)
}
l.Info("Blocklisted certificates", "fingerprintCount", len(bl)-len(bad))
if len(bad) > 0 {
// The valid entries are still returned so a failed reload blocks them, startup refuses the error
return caPool, fmt.Errorf("pki.blocklist entries are not hex certificate fingerprints: %s", strings.Join(bad, ", "))
}
l.Info("Blocklisted certificates", "fingerprintCount", len(bl))
}
return caPool, nil
-43
View File
@@ -217,46 +217,3 @@ func TestPKIReloadChachapolyInFIPSOnlyFails(t *testing.T) {
assert.True(t, res.Failed)
assert.False(t, res.RestartRequired)
}
func TestLoadCAPoolBlocklist(t *testing.T) {
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
crt, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil)
fp, err := crt.Fingerprint()
require.NoError(t, err)
load := func(entry string) (*cert.CAPool, error) {
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(fmt.Sprintf("pki:\n ca: %q\n blocklist:\n - %q\n", caPEM, entry)))
return loadCAPoolFromConfig(test.NewLogger(), c)
}
for _, entry := range []string{fp, strings.ToUpper(fp), " " + fp + "\t"} {
pool, err := load(entry)
require.NoError(t, err, entry)
assert.True(t, pool.IsBlocklisted(fp), entry)
}
for _, entry := range []string{fp[1:], fp + "00", "sha256:" + fp, "zz" + fp[2:], ""} {
_, err := load(entry)
require.Error(t, err, entry)
}
// A bad entry on reload fails it, the valid entries are still blocked
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "me", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.1/24")}, nil, nil)
pkiConfig := func(entry string) string {
return fmt.Sprintf("pki:\n ca: %q\n cert: %q\n key: %q\n blocklist:\n - %q\n", caPEM, certPEM, keyPEM, entry)
}
c := config.NewC(test.NewLogger())
require.NoError(t, c.LoadString(pkiConfig(fp)))
p, err := NewPKIFromConfig(test.NewLogger(), c)
require.NoError(t, err)
other, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "other", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.3/24")}, nil, nil)
otherFp, err := other.Fingerprint()
require.NoError(t, err)
res := c.ReloadConfigString(pkiConfig(fp) + fmt.Sprintf(" - %q\n - %q\n", otherFp, fp[1:]))
assert.True(t, res.Failed)
assert.True(t, p.GetCAPool().IsBlocklisted(fp))
assert.True(t, p.GetCAPool().IsBlocklisted(otherFp))
}
+175 -1
View File
@@ -11,8 +11,12 @@ import (
"net"
"net/netip"
"os"
"path/filepath"
"runtime"
"runtime/pprof"
"slices"
"sort"
"strconv"
"strings"
"github.com/slackhq/nebula/config"
@@ -56,6 +60,8 @@ type sshDeviceInfoFlags struct {
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
// sshd.sandbox_dir is read once, when the commands are attached
r.RestartRequiredIfChanged("sshd.sandbox_dir")
if c.GetBool("sshd.enabled", false) {
// Restarting the server drops every open session, the one that asked for this reload included. Only restart
// when it isn't running or its config changed. The host key file can change under the same config, new
@@ -87,7 +93,8 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
})
}
// sshdServerKeys are what configSSH reads
// sshdServerKeys are what configSSH reads. sshd.sandbox_dir isn't one, it's read once at startup and restarting the
// server doesn't apply it
var sshdServerKeys = []string{"sshd.enabled", "sshd.listen", "sshd.host_key", "sshd.trusted_cas", "sshd.authorized_users"}
// loadSSHHostKey returns sshd.host_key, read from its file when it names one.
@@ -229,6 +236,12 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
}
func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
// sandboxDir defaults to a dir in temp. The intention is that end user will
// create this dir as needed. Overriding this config value to "" allows
// writing to anywhere in the system.
defaultDir := filepath.Join(os.TempDir(), "nebula-debug")
sandboxDir := c.GetString("sshd.sandbox_dir", defaultDir)
ssh.RegisterCommand(&sshd.Command{
Name: "list-hostmap",
ShortDescription: "List all known previously connected hosts",
@@ -284,6 +297,45 @@ func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interfa
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "start-cpu-profile",
ShortDescription: "Starts a cpu profile and write output to the provided file, ex: `cpu-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshStartCpuProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "stop-cpu-profile",
ShortDescription: "Stops a cpu profile and writes output to the previously provided file",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
pprof.StopCPUProfile()
return w.WriteLine("If a CPU profile was running it is now stopped")
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "save-heap-profile",
ShortDescription: "Saves a heap profile to the provided path, ex: `heap-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetHeapProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "mutex-profile-fraction",
ShortDescription: "Gets or sets runtime.SetMutexProfileFraction",
Callback: sshMutexProfileFraction,
})
ssh.RegisterCommand(&sshd.Command{
Name: "save-mutex-profile",
ShortDescription: "Saves a mutex profile to the provided path, ex: `mutex-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetMutexProfile(sandboxDir, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "log-level",
ShortDescription: "Gets or sets the current log level",
@@ -514,6 +566,58 @@ func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) er
return nil
}
// sshSanitizeFilePath validates that the given file path is within the sandbox directory.
// If sandboxDir is empty, the path is returned as-is for backwards compatibility.
func sshSanitizeFilePath(sandboxDir, filePath string) (string, error) {
if sandboxDir == "" {
return filePath, nil
}
// Clean and resolve the path relative to the sandbox directory
if !filepath.IsAbs(filePath) {
filePath = filepath.Join(sandboxDir, filePath)
}
cleaned := filepath.Clean(filePath)
// Ensure the resolved path is within the sandbox directory
cleanedSandbox := filepath.Clean(sandboxDir)
if cleaned == cleanedSandbox {
return "", fmt.Errorf("path %q resolves to the sandbox directory itself %q", filePath, sandboxDir)
}
if !strings.HasPrefix(cleaned, cleanedSandbox+string(filepath.Separator)) {
return "", fmt.Errorf("path %q is outside the sandbox directory %q", filePath, sandboxDir)
}
return cleaned, nil
}
func sshStartCpuProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
err := w.WriteLine("No path to write profile provided")
return err
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
}
err = pprof.StartCPUProfile(file)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to start cpu profile: %s", err))
return err
}
err = w.WriteLine(fmt.Sprintf("Started cpu profile, issue stop-cpu-profile to write the output to %s", a))
return err
}
func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
return w.WriteLine(fmt.Sprintf("%s", ifce.version))
}
@@ -666,6 +770,76 @@ func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) e
return w.WriteLine("Changed")
}
func sshGetHeapProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
}
err = pprof.WriteHeapProfile(file)
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
return err
}
err = w.WriteLine(fmt.Sprintf("Mem profile created at %s", a))
return err
}
func sshMutexProfileFraction(fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
rate := runtime.SetMutexProfileFraction(-1)
return w.WriteLine(fmt.Sprintf("Current value: %d", rate))
}
newRate, err := strconv.Atoi(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("Invalid argument: %s", a[0]))
}
oldRate := runtime.SetMutexProfileFraction(newRate)
return w.WriteLine(fmt.Sprintf("New value: %d. Old value: %d", newRate, oldRate))
}
func sshGetMutexProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
if err != nil {
return w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
}
defer file.Close()
mutexProfile := pprof.Lookup("mutex")
if mutexProfile == nil {
return w.WriteLine("Unable to get pprof.Lookup(\"mutex\")")
}
err = mutexProfile.WriteTo(file, 0)
if err != nil {
return w.WriteLine(fmt.Sprintf("Unable to write profile: %s", err))
}
return w.WriteLine(fmt.Sprintf("Mutex profile created at %s", a))
}
func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level
+12
View File
@@ -165,6 +165,18 @@ func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
}
}
// sandbox_dir is only read at startup, restarting sshd for it would drop every session and still not apply it
func TestSSHDReload_SandboxDirKeepsSessions(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
h.extra = " sandbox_dir: " + t.TempDir() + "\n"
res := h.c.ReloadConfigString(h.config())
require.NoError(t, res.Err)
assert.Equal(t, []string{"sshd.sandbox_dir"}, res.RestartKeys)
assert.False(t, dropped(client, 500*time.Millisecond), "sshd was restarted")
}
// The config can stay the same while the host key file it names changes. New connections get the new key, open
// sessions are left alone
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
+1 -27
View File
@@ -246,18 +246,6 @@ func getFrom(names [][]byte, i int, isV4 bool) netip.AddrPort {
}
func (u *StdConn) ListenOut(r EncReader, flush func()) error {
return u.listenOut(r, flush, nil)
}
// ListenOutHandoff is ListenOut for a reader that keeps the packets: after each batch, handoff takes the batch's
// buffers and returns replacements, of the same count and at least the same size, to receive the next one into.
func (u *StdConn) ListenOutHandoff(r EncReader, handoff func(bufs [][]byte) [][]byte) error {
return u.listenOut(r, nil, handoff)
}
// listenOut is the receive loop. After each batch it calls flush, or with handoff set, hands the batch's buffers
// over and points the recvmmsg iovecs at the ones it gets back.
func (u *StdConn) listenOut(r EncReader, flush func(), handoff func(bufs [][]byte) [][]byte) error {
bufSize := MTU
cmsgSpace := 0
if u.groSupported {
@@ -295,21 +283,7 @@ func (u *StdConn) listenOut(r EncReader, flush func(), handoff func(bufs [][]byt
deliverSegments(r, from, payload, segSize)
}
if handoff == nil {
flush()
continue
}
next := handoff(buffers)
if len(next) != len(buffers) {
panic(fmt.Sprintf("udp: handoff returned %d buffers, need %d", len(next), len(buffers)))
}
for i := range next {
if len(next[i]) < bufSize {
panic(fmt.Sprintf("udp: handoff buffer is %d bytes, need %d", len(next[i]), bufSize))
}
msgs[i].Hdr.Iov.Base = &next[i][0]
}
buffers = next
flush()
}
}
+3 -12
View File
@@ -183,24 +183,15 @@ func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, err
}
func (u *TesterConn) ListenOut(r EncReader, flush func()) error {
// The batcher borrows plaintext decrypted in place inside p.Data
// until Flush, so the packet must stay alive across flush()
return u.listenOut(r, func(p *Packet) { flush() })
}
// ListenOutHandoff hands each packet's buffer over, the replacement goes back to the pool.
func (u *TesterConn) ListenOutHandoff(r EncReader, handoff func(bufs [][]byte) [][]byte) error {
return u.listenOut(r, func(p *Packet) { p.Data = handoff([][]byte{p.Data})[0] })
}
func (u *TesterConn) listenOut(r EncReader, finish func(*Packet)) error {
for {
select {
case <-u.done:
return os.ErrClosed
case p := <-u.RxPackets:
r(p.From, p.Data[:len(p.Data):len(p.Data)])
finish(p)
// The batcher borrows plaintext decrypted in place inside p.Data
// until Flush, so the packet must stay alive across flush()
flush()
p.Release()
}
}