mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-15 12:36:52 +01:00
Caleb JasikandClaude Opus 5.5
3bb5918e7c
On darwin an unconnected sendto runs the address and MAC policy checks and the EndpointSecurity send check for every datagram, and while xprotectd throttles it each send takes about 185us, which caps a tunnel's upload and, through its ACKs, its download. A connected socket runs those checks once, at connect, and it is the only UDP send that reports the interface queue's flow control, as ENOBUFS. Only the current direct remote of an established tunnel is eligible. udp.EstablishedPeerConn is an optional Conn interface, implemented on darwin; establishedRemotes refcounts remotes over the main hostmap and tells each writer on the first and last reference: handshake completion adds one, deleting or retiring a hostinfo removes it, SetRemote (roaming, SetRemoteIfPreferred, control, ssh) moves it. Pending and relayed hostinfos hold none, and a rehandshake's two hostinfos share one. An eligible peer that Nebula sends NEBULA_CONNSOCK_RUN (64) datagrams to within NEBULA_CONNSOCK_WINDOW (1s) gets a connected socket: SO_REUSEPORT, bound to the source address the listener's route to the peer picks on the listener's port, carrying the listener's IP_BOUND_IF. The peer sees the same address and port as from the listener, and the socket's reader delivers through the listener's reader. The listener itself never sets SO_REUSEPORT (listen routines still do), so no other uid can bind the port; with a specific listen.host and no routines connected sockets stay off, since sharing an exact address needs SO_REUSEPORT on both. All routines listening on one address share one set of connected sockets: at most NEBULA_CONNSOCKS (8), one per peer, with opens from a bucket that refills one per NEBULA_CONNSOCK_OPEN_EVERY (1s). A socket closes when its peer stops being eligible, after NEBULA_CONNSOCK_IDLE (30s) without traffic, on Rebind (of every routine) and Close, and on an error that means the connected path is gone. Closing never runs under the hostmap's or the eligibility lock; the peer can't reopen until the close finishes. A send that gets ENOBUFS or EAGAIN waits within one 10ms budget shared by every writer since the last send that went through, then drops; nothing waits in the netpoller without a deadline. When nothing is eligible or open, a send costs one atomic load before the listener path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
91 lines
2.6 KiB
Go
91 lines
2.6 KiB
Go
package nebula
|
|
|
|
import (
|
|
"net/netip"
|
|
"sync"
|
|
|
|
"github.com/slackhq/nebula/udp"
|
|
)
|
|
|
|
// establishedRemotes tells the udp conns that implement udp.EstablishedPeerConn which addresses are the current
|
|
// direct remote of at least one established tunnel. A hostinfo holds a reference on its remote while it is in the
|
|
// main hostmap, and SetRemote moves the reference; a relayed tunnel has no direct remote and holds none.
|
|
type establishedRemotes struct {
|
|
mu sync.Mutex
|
|
conns []udp.EstablishedPeerConn
|
|
refs map[netip.AddrPort]int
|
|
}
|
|
|
|
// newEstablishedRemotes returns nil when no conn wants to know, which every method accepts.
|
|
func newEstablishedRemotes(conns []udp.Conn) *establishedRemotes {
|
|
var pc []udp.EstablishedPeerConn
|
|
for _, c := range conns {
|
|
if p, ok := c.(udp.EstablishedPeerConn); ok {
|
|
pc = append(pc, p)
|
|
}
|
|
}
|
|
if len(pc) == 0 {
|
|
return nil
|
|
}
|
|
return &establishedRemotes{conns: pc, refs: map[netip.AddrPort]int{}}
|
|
}
|
|
|
|
// add makes hostinfo hold a reference on its remote, as it joins the main hostmap. A hostinfo holds references in
|
|
// one establishedRemotes at a time, so an add while it is in another is ignored.
|
|
func (e *establishedRemotes) add(hostinfo *HostInfo) {
|
|
if e == nil {
|
|
return
|
|
}
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
if !hostinfo.establishedIn.CompareAndSwap(nil, e) && hostinfo.establishedIn.Load() != e {
|
|
return
|
|
}
|
|
e.syncLocked(hostinfo)
|
|
}
|
|
|
|
// remove drops hostinfo's reference, as it leaves the main hostmap.
|
|
func (e *establishedRemotes) remove(hostinfo *HostInfo) {
|
|
if e == nil {
|
|
return
|
|
}
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
if hostinfo.establishedIn.CompareAndSwap(e, nil) {
|
|
e.syncLocked(hostinfo)
|
|
}
|
|
}
|
|
|
|
// syncLocked moves hostinfo's reference to the remote it should hold now: its current remote while it is
|
|
// established, none otherwise. Reading both under mu makes the last call win however add, remove and SetRemote
|
|
// interleave. Only the establishedRemotes that hostinfo is in, or has just left, may call it, since
|
|
// hostinfo.establishedRemote is a reference in that one's refs.
|
|
func (e *establishedRemotes) syncLocked(hostinfo *HostInfo) {
|
|
var want netip.AddrPort
|
|
if hostinfo.establishedIn.Load() == e {
|
|
want = hostinfo.GetRemote()
|
|
}
|
|
held := hostinfo.establishedRemote
|
|
if want == held {
|
|
return
|
|
}
|
|
if held.IsValid() {
|
|
e.refs[held]--
|
|
if e.refs[held] == 0 {
|
|
delete(e.refs, held)
|
|
for _, c := range e.conns {
|
|
c.SetEstablishedPeer(held, false)
|
|
}
|
|
}
|
|
}
|
|
if want.IsValid() {
|
|
e.refs[want]++
|
|
if e.refs[want] == 1 {
|
|
for _, c := range e.conns {
|
|
c.SetEstablishedPeer(want, true)
|
|
}
|
|
}
|
|
}
|
|
hostinfo.establishedRemote = want
|
|
}
|