nebula/flow_budget.go

package nebula

import (
	"sync/atomic"
	"time"

	"github.com/slackhq/nebula/overlay/tio"
)

// txBudget is the reservation credit of one tunnel, shared by every staged reader sending on it. The low 24 bits of
// state are outstanding counters, the next 24 spent credit, and the top 16 the sole ordered reader (q+1); owner zero
// means shared. Readers beyond the owner field's range always take the conservative shared path.
//
// A cap on outstanding counters alone is not enough: a fast reader could repeatedly send and reacquire credit,
// overtaking a paused reader by arbitrarily far. So once readers overlap, or another reader is blocked, spent credit
// is only recycled when ALL reservations have drained. A sole reader instead gets each ordered completion refunded,
// its later batches cannot overtake its earlier ones. All transitions are one CAS, without locks.
//
// The credit is half the peer's replay window. The other half is left for uncredited sends and transport reordering:
// inline and control traffic, and relay forwarding through SendVia, which takes counters on the relay tunnel without
// credit. Unlimited bypass traffic or network delay can still exceed it. The window is not negotiated, so
// ci.window.length, from the build-time ReplayWindow, is also the assumed peer window.
type txBudget struct{ state atomic.Uint64 }

const txBudgetMask = uint64(1<<24 - 1)

// Fail compilation for unsupported build-time windows: a power of two, at least 8192, fitting the credit fields (up
// to 16777216).
const (
	_ = uint64(txBudgetMask) - ReplayWindow/2
	_ = uint64(ReplayWindow) - 8192
	_ = uint64(0) - (ReplayWindow & (ReplayWindow - 1))
)

func txBudgetOwner(q int) uint64 {
	if q < 0 || q >= 65535 {
		return 0
	}
	return uint64(q + 1)
}

func (b *txBudget) acquire(n, limit, owner uint64) bool {
	for {
		old := b.state.Load()
		pending, spent, who := old&txBudgetMask, (old>>24)&txBudgetMask, old>>48
		if n > limit || spent > limit-n {
			// A blocked reader ends sole ownership so ordered completions stop refunding credit. Otherwise a busy
			// owner could keep a larger waiter blocked forever without ever sharing a reservation.
			if who != 0 && who != owner && !b.state.CompareAndSwap(old, old&^(uint64(0xffff)<<48)) {
				continue
			}
			return false
		}
		if pending == 0 {
			who = owner
		} else if who != owner {
			who = 0
		}
		next := who<<48 | (spent+n)<<24 | (pending + n)
		if b.state.CompareAndSwap(old, next) {
			return true
		}
	}
}

// release runs only after the socket flush or an explicit drop. A rollback returns credit before any message counter
// was allocated, as when a later tunnel of a relay or multi-tunnel batch could not acquire its own.
func (b *txBudget) release(n uint64, rollback bool) {
	for {
		old := b.state.Load()
		pending, spent, owner := old&txBudgetMask, (old>>24)&txBudgetMask, old>>48
		pending -= n
		if rollback || owner != 0 {
			spent -= n
		}
		next := owner<<48 | spent<<24 | pending
		if pending == 0 {
			next = 0
		}
		if b.state.CompareAndSwap(old, next) {
			return
		}
	}
}

func (ci *ConnectionState) txBudgetLimit() uint64 { return ci.window.length / 2 }

func (r *txReserver) oversized() bool {
	for _, t := range r.tunnels {
		if t.n > t.ci.txBudgetLimit() {
			return true
		}
	}
	return false
}

func (r *txReserver) tryBudget() bool {
	for i, t := range r.tunnels {
		if !t.ci.txBudget.acquire(t.n, t.ci.txBudgetLimit(), r.owner) {
			for _, acquired := range r.tunnels[:i] {
				acquired.ci.txBudget.release(acquired.n, true)
			}
			return false
		}
	}
	return true
}

// waitBudget acquires credit for every tunnel of the batch, never holding a partial acquisition while it waits. Each
// reader first flushes its own oldest batch, flushes need no credit; with none, another reader holds all outstanding
// work and can flush it independently. So there is no circular wait, even with relay tunnels acquired in opposite
// orders. The sleep is only for readers with no local work, not the uncontended hot path. Acquisition is
// all-or-nothing within each direct or relay batch, so pressure on one tunnel also delays packets for idle tunnels
// in that batch. Reader fairness is not guaranteed.
func (p *stagedTx) waitBudget(r *txReserver) bool {
	if r.tryBudget() {
		return true
	}
	for {
		if p.f.closed.Load() {
			return false
		}
		if !p.flushOldest() {
			time.Sleep(50 * time.Microsecond)
		}
		if r.tryBudget() {
			return true
		}
	}
}

func (s *txSlot) releaseBudget() {
	for _, t := range s.reservations {
		t.ci.txBudget.release(t.n, false)
	}
	clear(s.reservations)
	s.reservations = s.reservations[:0]
}

// sendLarge handles the unusual reads that exceed a whole tunnel's credit, such as tiny-MSS superpackets. It drains
// the pipeline, then segments and sends them synchronously with credit for each segment, never exceeding the cap and
// never filtering or running connection manager side effects twice. This deliberately slow path avoids a lone-batch
// exception to the shared invariant.
func (p *stagedTx) sendLarge(r *txReserver, items []txItem) {
	for p.flushOldest() {
	}
	for _, it := range items {
		err := tio.SegmentSuperpacket(it.pkt, func(seg []byte) error {
			if p.f.closed.Load() {
				return nil
			}
			part := it
			part.pkt, part.segs = tio.Packet{Bytes: seg}, 1
			r.tunnels = r.tunnels[:0]
			part.counter = r.need(part.hostinfo, part.ci, 1)
			if part.relayHI != nil {
				part.relayCounter = r.need(part.relayHI, part.relayHI.ConnectionState, 1)
			}
			if !p.waitBudget(r) {
				return nil
			}
			one := [1]txItem{part}
			if reserved := r.reserve(one[:]); len(reserved) != 0 {
				p.f.encryptTxItem(&reserved[0], p.nb, p.sb)
				p.f.flushSendBatch(p.sb, p.q)
			}
			for _, t := range r.tunnels {
				t.ci.txBudget.release(t.n, false)
			}
			return nil
		})
		if err != nil {
			it.hostinfo.logger(p.f.l).Error("Failed to segment superpacket for send", "error", err)
		}
	}
	clear(items)
}