mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 12:36:52 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3db4c89e9e | ||
|
|
d3e21a8fc8 | ||
|
|
cda3710d5a | ||
|
|
c26871fe08 | ||
|
|
7daacc6a48 | ||
|
|
428dad94b3 |
No files matched your search
@@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- Signing a v2 certificate with an empty name, a name longer than 253 bytes, or an empty group now fails.
|
||||
These certificates were signed successfully but could not be loaded.
|
||||
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
|
||||
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
|
||||
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
|
||||
|
||||
+10
-1
@@ -389,7 +389,16 @@ func (c *certificateV2) fromTBSCertificate(t *TBSCertificate) error {
|
||||
}
|
||||
|
||||
func (c *certificateV2) validate() error {
|
||||
// Empty names are allowed
|
||||
// Mirror the name and group rules in unmarshalDetails so signing fails instead of producing an unparseable certificate
|
||||
if c.details.name == "" || len(c.details.name) > MaxNameLength {
|
||||
return NewErrInvalidCertificateProperties("name must be between 1 and %d bytes", MaxNameLength)
|
||||
}
|
||||
|
||||
for _, group := range c.details.groups {
|
||||
if group == "" {
|
||||
return NewErrInvalidCertificateProperties("groups may not be empty")
|
||||
}
|
||||
}
|
||||
|
||||
if len(c.publicKey) == 0 {
|
||||
return ErrInvalidPublicKey
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -135,3 +136,47 @@ func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
|
||||
assert.True(t, normie)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertificateV2_SignRejectsUnparseable(t *testing.T) {
|
||||
_, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
require.NoError(t, err)
|
||||
var invalid *ErrInvalidCertificateProperties
|
||||
|
||||
newTBS := func() TBSCertificate {
|
||||
return TBSCertificate{
|
||||
Version: Version2,
|
||||
Name: "testing",
|
||||
Networks: []netip.Prefix{mustParsePrefixUnmapped("10.1.1.1/24")},
|
||||
Groups: []string{"test-group1"},
|
||||
NotBefore: time.Now().Add(-time.Minute),
|
||||
NotAfter: time.Now().Add(time.Minute),
|
||||
PublicKey: []byte("1234567890abcedfghij1234567890ab"),
|
||||
Curve: Curve_CURVE25519,
|
||||
IsCA: true,
|
||||
}
|
||||
}
|
||||
|
||||
tbs := newTBS()
|
||||
tbs.Name = strings.Repeat("a", MaxNameLength)
|
||||
c, err := tbs.Sign(nil, Curve_CURVE25519, priv)
|
||||
require.NoError(t, err)
|
||||
b, err := c.Marshal()
|
||||
require.NoError(t, err)
|
||||
_, err = unmarshalCertificateV2(b, nil, Curve_CURVE25519)
|
||||
require.NoError(t, err)
|
||||
|
||||
tbs = newTBS()
|
||||
tbs.Name = strings.Repeat("a", MaxNameLength+1)
|
||||
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
|
||||
require.ErrorAs(t, err, &invalid)
|
||||
|
||||
tbs = newTBS()
|
||||
tbs.Name = ""
|
||||
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
|
||||
require.ErrorAs(t, err, &invalid)
|
||||
|
||||
tbs = newTBS()
|
||||
tbs.Groups = []string{"test-group1", ""}
|
||||
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
|
||||
require.ErrorAs(t, err, &invalid)
|
||||
}
|
||||
+17
-22
@@ -65,7 +65,7 @@ type HandshakeManager struct {
|
||||
lightHouse *LightHouse
|
||||
outside udp.Conn
|
||||
config HandshakeConfig
|
||||
OutboundHandshakeTimer *LockingTimerWheel[*HandshakeHostInfo]
|
||||
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
|
||||
messageMetrics *MessageMetrics
|
||||
metricInitiated metrics.Counter
|
||||
metricTimedOut metrics.Counter
|
||||
@@ -81,7 +81,6 @@ type HandshakeHostInfo struct {
|
||||
|
||||
startTime time.Time // Time that we first started trying with this handshake
|
||||
ready bool // Is the handshake ready
|
||||
deleted bool // Has the handshake been removed from the pending maps, deleters hold the lock
|
||||
initiatingVersionOverride cert.Version // Should we use a non-default cert version for this handshake?
|
||||
counter int64 // How many attempts have we made so far
|
||||
lastRemotes []netip.AddrPort // Remotes that we sent to during the previous attempt
|
||||
@@ -126,7 +125,7 @@ func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *Light
|
||||
outside: outside,
|
||||
config: config,
|
||||
trigger: make(chan netip.Addr, config.triggerBuffer),
|
||||
OutboundHandshakeTimer: NewLockingTimerWheel[*HandshakeHostInfo](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
|
||||
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
|
||||
messageMetrics: config.messageMetrics,
|
||||
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
|
||||
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
|
||||
@@ -143,9 +142,7 @@ func (hm *HandshakeManager) Run(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case vpnIP := <-hm.trigger:
|
||||
if hh := hm.queryVpnIp(vpnIP); hh != nil {
|
||||
hm.handleOutbound(hh, true)
|
||||
}
|
||||
hm.handleOutbound(vpnIP, true)
|
||||
case now := <-clockSource.C:
|
||||
hm.NextOutboundHandshakeTimerTick(now)
|
||||
}
|
||||
@@ -200,25 +197,23 @@ func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *head
|
||||
func (hm *HandshakeManager) NextOutboundHandshakeTimerTick(now time.Time) {
|
||||
hm.OutboundHandshakeTimer.Advance(now)
|
||||
for {
|
||||
hh, has := hm.OutboundHandshakeTimer.Purge()
|
||||
vpnIp, has := hm.OutboundHandshakeTimer.Purge()
|
||||
if !has {
|
||||
break
|
||||
}
|
||||
hm.handleOutbound(hh, false)
|
||||
hm.handleOutbound(vpnIp, false)
|
||||
}
|
||||
}
|
||||
|
||||
func (hm *HandshakeManager) handleOutbound(hh *HandshakeHostInfo, lighthouseTriggered bool) {
|
||||
func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered bool) {
|
||||
hh := hm.queryVpnIp(vpnIp)
|
||||
if hh == nil {
|
||||
return
|
||||
}
|
||||
hh.Lock()
|
||||
defer hh.Unlock()
|
||||
|
||||
// Timer entries outlive their handshake. A new handshake for the same vpn addr runs its own timer chain.
|
||||
if hh.deleted {
|
||||
return
|
||||
}
|
||||
|
||||
hostinfo := hh.hostinfo
|
||||
vpnIp := hostinfo.vpnAddrs[0]
|
||||
// If we are out of time, clean up
|
||||
if hh.counter >= hm.config.retries {
|
||||
fields := []any{
|
||||
@@ -244,14 +239,10 @@ func (hm *HandshakeManager) handleOutbound(hh *HandshakeHostInfo, lighthouseTrig
|
||||
// Increment the counter to increase our delay, linear backoff
|
||||
hh.counter++
|
||||
|
||||
// A lighthouse triggered attempt is still in the timer wheel, adding it again would start a second retry chain
|
||||
if !lighthouseTriggered {
|
||||
defer hm.OutboundHandshakeTimer.Add(hh, hm.config.tryInterval*time.Duration(hh.counter))
|
||||
}
|
||||
|
||||
// Check if we have a handshake packet to transmit yet
|
||||
if !hh.ready {
|
||||
if !hm.buildStage0Packet(hh) {
|
||||
hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval*time.Duration(hh.counter))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -332,6 +323,11 @@ func (hm *HandshakeManager) handleOutbound(hh *HandshakeHostInfo, lighthouseTrig
|
||||
}
|
||||
|
||||
hm.f.relayManager.StartRelays(hm.f, vpnIp, hh, stage0)
|
||||
|
||||
// If a lighthouse triggered this attempt then we are still in the timer wheel and do not need to re-add
|
||||
if !lighthouseTriggered {
|
||||
hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval*time.Duration(hh.counter))
|
||||
}
|
||||
}
|
||||
|
||||
// GetOrHandshake will try to find a hostinfo with a fully formed tunnel or start a new handshake if one is not present
|
||||
@@ -384,7 +380,7 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
|
||||
}
|
||||
hm.vpnIps[vpnAddr] = hh
|
||||
hm.metricInitiated.Inc(1)
|
||||
hm.OutboundHandshakeTimer.Add(hh, hm.config.tryInterval)
|
||||
hm.OutboundHandshakeTimer.Add(vpnAddr, hm.config.tryInterval)
|
||||
|
||||
if cacheCb != nil {
|
||||
cacheCb(hh)
|
||||
@@ -536,7 +532,6 @@ func (hm *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) {
|
||||
func (hm *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) {
|
||||
for _, addr := range hostinfo.vpnAddrs {
|
||||
if cur, ok := hm.vpnIps[addr]; ok && cur.hostinfo == hostinfo {
|
||||
cur.deleted = true
|
||||
delete(hm.vpnIps, addr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,69 +65,7 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
|
||||
assert.NotContains(t, blah.vpnIps, ip)
|
||||
}
|
||||
|
||||
// newTestHandshakeManager returns a manager whose handshakes never build a stage 0 packet (the cert has no key), so
|
||||
// every attempt just bumps the counter and schedules the next one. tick advances its timer wheel by one tryInterval.
|
||||
func newTestHandshakeManager() (hm *HandshakeManager, tick func()) {
|
||||
l := test.NewLogger()
|
||||
mainHM := newHostMap(l)
|
||||
mainHM.preferredRanges.Store(&[]netip.Prefix{})
|
||||
|
||||
hm = NewHandshakeManager(l, mainHM, newTestLighthouse(), &udp.NoopConn{}, defaultHandshakeConfig)
|
||||
hm.f = &Interface{handshakeManager: hm, pki: &PKI{}, l: l}
|
||||
hm.f.pki.cs.Store(&CertState{
|
||||
initiatingVersion: cert.Version1,
|
||||
privateKey: []byte{},
|
||||
v1Cert: &dummyCert{version: cert.Version1},
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
hm.NextOutboundHandshakeTimerTick(now)
|
||||
return hm, func() {
|
||||
now = now.Add(DefaultHandshakeTryInterval)
|
||||
hm.NextOutboundHandshakeTimerTick(now)
|
||||
}
|
||||
}
|
||||
|
||||
func Test_HandshakeManager_RestartDropsOldTimer(t *testing.T) {
|
||||
ip := netip.MustParseAddr("172.1.1.2")
|
||||
hm, tick := newTestHandshakeManager()
|
||||
|
||||
hm.StartHandshake(ip, nil)
|
||||
old := hm.queryVpnIp(ip)
|
||||
tick()
|
||||
tick()
|
||||
require.Equal(t, int64(1), old.counter)
|
||||
|
||||
// Restart the handshake while the old one still has a retry queued
|
||||
hm.DeleteHostInfo(old.hostinfo)
|
||||
hm.StartHandshake(ip, nil)
|
||||
hh := hm.queryVpnIp(ip)
|
||||
require.NotSame(t, old, hh)
|
||||
|
||||
// An entry for timeout d fires d/tryInterval+1 ticks after it is added, so linear backoff lands attempts on ticks
|
||||
// 2, 4 and 7
|
||||
var counters []int64
|
||||
for range 7 {
|
||||
tick()
|
||||
counters = append(counters, hh.counter)
|
||||
}
|
||||
assert.Equal(t, []int64{0, 1, 1, 2, 2, 2, 3}, counters)
|
||||
|
||||
// The old handshake's queued retry ended its chain instead of retrying a deleted handshake
|
||||
assert.Equal(t, int64(1), old.counter)
|
||||
assert.Equal(t, 1, testCountTimerWheelEntries(hm.OutboundHandshakeTimer))
|
||||
}
|
||||
|
||||
func Test_HandshakeManager_LighthouseTriggerKeepsOneTimer(t *testing.T) {
|
||||
ip := netip.MustParseAddr("172.1.1.2")
|
||||
hm, _ := newTestHandshakeManager()
|
||||
|
||||
hm.StartHandshake(ip, nil)
|
||||
hm.handleOutbound(hm.queryVpnIp(ip), true)
|
||||
assert.Equal(t, 1, testCountTimerWheelEntries(hm.OutboundHandshakeTimer))
|
||||
}
|
||||
|
||||
func testCountTimerWheelEntries(tw *LockingTimerWheel[*HandshakeHostInfo]) (c int) {
|
||||
func testCountTimerWheelEntries(tw *LockingTimerWheel[netip.Addr]) (c int) {
|
||||
for _, i := range tw.t.wheel {
|
||||
n := i.Head
|
||||
for n != nil {
|
||||
|
||||
@@ -384,6 +384,8 @@ type rxContext struct {
|
||||
hostmapCache map[uint32]*HostInfo
|
||||
lhh *LightHouseHandler
|
||||
ctCache *firewall.ConntrackCacheTicker
|
||||
// relayBatch holds forwarded relay packets, which borrow their receive buffers until flushed
|
||||
relayBatch *batch.SendBatch
|
||||
}
|
||||
|
||||
func newRxContext(f *Interface, q int) *rxContext {
|
||||
@@ -396,6 +398,7 @@ func newRxContext(f *Interface, q int) *rxContext {
|
||||
hostmapCache: map[uint32]*HostInfo{},
|
||||
lhh: f.lightHouse.NewRequestHandler(),
|
||||
ctCache: firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout),
|
||||
relayBatch: batch.NewSendBatch(f.writers[q], batch.SendBatchCap, 0),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -414,6 +417,7 @@ func (f *Interface) listenOut(i int) {
|
||||
}
|
||||
|
||||
flusher := func() {
|
||||
f.flushSendBatch(rxc.relayBatch, i)
|
||||
if err := f.batchers[i].Flush(); err != nil {
|
||||
f.l.Error("Failed to flush tun coalescer", "error", err)
|
||||
}
|
||||
|
||||
+10
-4
@@ -25,7 +25,7 @@ var ErrOutOfWindow = errors.New("out of window packet")
|
||||
|
||||
// readOutsidePackets processes one received underlay packet.
|
||||
// Message payloads are decrypted IN PLACE, so packet must stay untouched
|
||||
// by the caller until the batcher for queue q has been flushed
|
||||
// by the caller until the TUN and relay batches for queue q have been flushed.
|
||||
func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) {
|
||||
h := rxc.h
|
||||
err := h.Parse(packet)
|
||||
@@ -233,8 +233,14 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
|
||||
// Forward this packet through the relay tunnel, rebuilding it in place.
|
||||
// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
|
||||
fwdBuf := packet[:0]
|
||||
//todo it would potentially be nice to batch these
|
||||
f.SendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true, rxc.q)
|
||||
out, err := f.prepareSendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true)
|
||||
if err == nil {
|
||||
// Borrows the receive buffer, the listener's flush callback drains it before the buffer is reused
|
||||
rxc.relayBatch.Commit(out, targetHI.GetRemote())
|
||||
if rxc.relayBatch.Len() >= batch.SendBatchCap {
|
||||
f.flushSendBatch(rxc.relayBatch, rxc.q)
|
||||
}
|
||||
}
|
||||
case TerminalType:
|
||||
hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
|
||||
return
|
||||
@@ -478,8 +484,8 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounte
|
||||
if err != nil {
|
||||
if f.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||
hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
dropReason := f.firewall.Drop(rxc.fwPacket.Packet, true, hostinfo, f.pki.GetCAPool(), rxc.ctCache.Get())
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
package nebula
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/flynn/noise"
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/rcrowley/go-metrics"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/noiseutil"
|
||||
"github.com/slackhq/nebula/overlay/batch"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/slackhq/nebula/udp"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type relayBatchWrite struct {
|
||||
packets [][]byte
|
||||
remotes []netip.AddrPort
|
||||
batched bool
|
||||
}
|
||||
|
||||
type relayBatchConn struct {
|
||||
udp.NoopConn
|
||||
read func(udp.EncReader, func())
|
||||
writes []relayBatchWrite
|
||||
}
|
||||
|
||||
func (c *relayBatchConn) ListenOut(r udp.EncReader, flush func()) error {
|
||||
c.read(r, flush)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *relayBatchConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||
w := relayBatchWrite{remotes: append([]netip.AddrPort(nil), addrs...), batched: true}
|
||||
for _, b := range bufs {
|
||||
w.packets = append(w.packets, bytes.Clone(b))
|
||||
}
|
||||
c.writes = append(c.writes, w)
|
||||
return len(bufs), nil
|
||||
}
|
||||
|
||||
func (c *relayBatchConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
||||
c.writes = append(c.writes, relayBatchWrite{packets: [][]byte{bytes.Clone(b)}, remotes: []netip.AddrPort{addr}})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *relayBatchConn) packets() [][]byte {
|
||||
var packets [][]byte
|
||||
for _, w := range c.writes {
|
||||
packets = append(packets, w.packets...)
|
||||
}
|
||||
return packets
|
||||
}
|
||||
|
||||
type relayBatchPath struct {
|
||||
f *Interface
|
||||
conn *relayBatchConn
|
||||
source, target *HostInfo
|
||||
incoming, outgoing *Relay
|
||||
sourceKey noiseutil.CipherState
|
||||
cipher noise.CipherFunc
|
||||
counter uint64
|
||||
}
|
||||
|
||||
// Separate cipher instances let the reference SendVia path use the same keys and
|
||||
// counters without rewinding an AEAD instance (including FIPS nonce checks).
|
||||
func relayBatchKey(cipher noise.CipherFunc, id byte) noiseutil.CipherState {
|
||||
suite := noise.NewCipherSuite(noise.DH25519, cipher, noise.HashSHA256)
|
||||
return noiseutil.NewCipherState(noise.UnsafeNewCipherState(suite, [32]byte{id}, 0), cipher)
|
||||
}
|
||||
|
||||
func newRelayBatchPath(t *testing.T, cipher noise.CipherFunc) *relayBatchPath {
|
||||
t.Helper()
|
||||
l := test.NewLogger()
|
||||
conn := &relayBatchConn{}
|
||||
f := &Interface{
|
||||
ctx: t.Context(), l: l, hostMap: newHostMap(l), outside: conn,
|
||||
writers: []udp.Conn{conn}, lightHouse: &LightHouse{l: l},
|
||||
myVpnNetworksTable: &bart.Lite{}, metricTxDropped: metrics.NewCounter(),
|
||||
batchers: []*batch.MultiCoalescer{batch.NewMultiCoalescer(&capturingTun{}, l)},
|
||||
}
|
||||
f.connectionManager = &connectionManager{
|
||||
intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{},
|
||||
}
|
||||
newHost := func(addr string, remote string, id byte) *HostInfo {
|
||||
h := &HostInfo{
|
||||
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
|
||||
ConnectionState: &ConnectionState{
|
||||
eKey: relayBatchKey(cipher, id), dKey: relayBatchKey(cipher, id), window: NewBits(ReplayWindow),
|
||||
},
|
||||
relayState: RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}},
|
||||
}
|
||||
u := netip.MustParseAddrPort(remote)
|
||||
h.remote.Store(&u)
|
||||
f.hostMap.Hosts[h.vpnAddrs[0]] = h
|
||||
return h
|
||||
}
|
||||
source := newHost("192.0.2.1", "198.51.100.1:4242", 1)
|
||||
target := newHost("192.0.2.2", "198.51.100.2:4242", 2)
|
||||
incoming := &Relay{Type: ForwardingType, State: Established, LocalIndex: 301, RemoteIndex: 302, PeerAddr: target.vpnAddrs[0]}
|
||||
outgoing := &Relay{Type: ForwardingType, State: Established, LocalIndex: 303, RemoteIndex: 304, PeerAddr: source.vpnAddrs[0]}
|
||||
source.relayState.InsertRelay(incoming.PeerAddr, incoming.LocalIndex, incoming)
|
||||
target.relayState.InsertRelay(outgoing.PeerAddr, outgoing.LocalIndex, outgoing)
|
||||
f.hostMap.Relays[incoming.LocalIndex] = source
|
||||
f.hostMap.Indexes[100] = source
|
||||
return &relayBatchPath{f: f, conn: conn, source: source, target: target, incoming: incoming, outgoing: outgoing, sourceKey: relayBatchKey(cipher, 1), cipher: cipher}
|
||||
}
|
||||
|
||||
func (p *relayBatchPath) wrap(t *testing.T, payload []byte) []byte {
|
||||
t.Helper()
|
||||
p.counter++
|
||||
b := header.Encode(make([]byte, header.Len), header.Version, header.Message, header.MessageRelay, p.incoming.LocalIndex, p.counter)
|
||||
b = append(b, payload...)
|
||||
b, err := p.sourceKey.EncryptDanger(b, b, nil, p.counter, make([]byte, 12))
|
||||
require.NoError(t, err)
|
||||
return b
|
||||
}
|
||||
|
||||
func (p *relayBatchPath) reference(payloads [][]byte) [][]byte {
|
||||
conn := &relayBatchConn{}
|
||||
f := &Interface{writers: []udp.Conn{conn}, connectionManager: p.f.connectionManager, l: p.f.l}
|
||||
h := &HostInfo{ConnectionState: &ConnectionState{eKey: relayBatchKey(p.cipher, 2)}}
|
||||
u := p.target.GetRemote()
|
||||
h.remote.Store(&u)
|
||||
for _, payload := range payloads {
|
||||
f.SendVia(h, p.outgoing, payload, make([]byte, 12), make([]byte, 0, len(payload)+header.MaxOverhead), false, 0)
|
||||
}
|
||||
return conn.packets()
|
||||
}
|
||||
|
||||
func TestRelayReceiveBatch(t *testing.T) {
|
||||
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
|
||||
t.Run(cipher.CipherName(), func(t *testing.T) {
|
||||
p := newRelayBatchPath(t, cipher)
|
||||
const count = 8
|
||||
var payloads, wire [][]byte
|
||||
for i := range count * 2 {
|
||||
payload := bytes.Repeat([]byte{byte(i)}, 64)
|
||||
payloads = append(payloads, payload)
|
||||
wire = append(wire, p.wrap(t, payload))
|
||||
}
|
||||
want := p.reference(payloads)
|
||||
// Model a listener reusing the same slots for the next receive batch.
|
||||
buffers := make([][]byte, count)
|
||||
for i := range buffers {
|
||||
buffers[i] = make([]byte, len(wire[i]))
|
||||
}
|
||||
p.conn.read = func(r udp.EncReader, flush func()) {
|
||||
for b := range 2 {
|
||||
for i := range count {
|
||||
copy(buffers[i], wire[b*count+i])
|
||||
r(p.source.GetRemote(), buffers[i])
|
||||
require.Len(t, p.conn.writes, b, "forwarding waits for the batch boundary")
|
||||
}
|
||||
flush()
|
||||
require.Len(t, p.conn.writes, b+1, "flush must send before receive slots are reused")
|
||||
for _, buffer := range buffers {
|
||||
clear(buffer)
|
||||
}
|
||||
}
|
||||
}
|
||||
p.f.listenOut(0)
|
||||
require.Equal(t, want, p.conn.packets())
|
||||
for _, w := range p.conn.writes {
|
||||
require.True(t, w.batched)
|
||||
require.Len(t, w.packets, count)
|
||||
for _, remote := range w.remotes {
|
||||
require.Equal(t, p.target.GetRemote(), remote)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRelayBatchMixedDestinations(t *testing.T) {
|
||||
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
|
||||
first, second := p.target.GetRemote(), netip.MustParseAddrPort("198.51.100.3:4242")
|
||||
remotes := []netip.AddrPort{first, first, second, second, first}
|
||||
var payloads, wire [][]byte
|
||||
for i := range remotes {
|
||||
payload := []byte{byte(i)}
|
||||
payloads = append(payloads, payload)
|
||||
wire = append(wire, p.wrap(t, payload))
|
||||
}
|
||||
want := p.reference(payloads)
|
||||
p.conn.read = func(r udp.EncReader, flush func()) {
|
||||
for i, remote := range remotes {
|
||||
p.target.remote.Store(&remote)
|
||||
r(p.source.GetRemote(), wire[i])
|
||||
}
|
||||
flush()
|
||||
}
|
||||
p.f.listenOut(0)
|
||||
require.Equal(t, want, p.conn.packets())
|
||||
require.Len(t, p.conn.writes, 1, "WriteBatch splits runs by destination, the batch does not need to")
|
||||
require.Equal(t, remotes, p.conn.writes[0].remotes)
|
||||
}
|
||||
|
||||
func TestRelayBatchCapacity(t *testing.T) {
|
||||
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
|
||||
p.conn.read = func(r udp.EncReader, flush func()) {
|
||||
for i := range batch.SendBatchCap + 1 {
|
||||
r(p.source.GetRemote(), p.wrap(t, []byte{byte(i)}))
|
||||
}
|
||||
require.Len(t, p.conn.writes, 1)
|
||||
require.Len(t, p.conn.writes[0].packets, batch.SendBatchCap)
|
||||
flush()
|
||||
}
|
||||
p.f.listenOut(0)
|
||||
require.Len(t, p.conn.writes, 2)
|
||||
require.Len(t, p.conn.writes[1].packets, 1)
|
||||
}
|
||||
|
||||
func TestRelayBatchRemovedTarget(t *testing.T) {
|
||||
for _, remove := range []string{"host", "relay", "disestablished"} {
|
||||
t.Run(remove, func(t *testing.T) {
|
||||
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
|
||||
first, removed := p.wrap(t, []byte("first")), p.wrap(t, []byte("removed"))
|
||||
want := p.reference([][]byte{[]byte("first")})
|
||||
p.conn.read = func(r udp.EncReader, flush func()) {
|
||||
r(p.source.GetRemote(), first)
|
||||
switch remove {
|
||||
case "host":
|
||||
delete(p.f.hostMap.Hosts, p.target.vpnAddrs[0])
|
||||
case "relay":
|
||||
delete(p.target.relayState.relayForByAddr, p.source.vpnAddrs[0])
|
||||
case "disestablished":
|
||||
p.target.relayState.UpdateRelayForByIpState(p.source.vpnAddrs[0], Disestablished)
|
||||
}
|
||||
r(p.source.GetRemote(), removed)
|
||||
flush()
|
||||
}
|
||||
p.f.listenOut(0)
|
||||
require.Len(t, p.conn.writes, 1)
|
||||
require.Equal(t, want, p.conn.packets(), "only the packet accepted before removal is sent")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -10,8 +10,8 @@ type batchWriter interface {
|
||||
}
|
||||
|
||||
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
|
||||
// One SendBatch is owned by each listenIn goroutine; no locking is needed.
|
||||
// Slots are backed by an Arena (see its docs)
|
||||
// Each reader goroutine owns its own SendBatch; no locking is needed.
|
||||
// Slots can borrow receive buffers or be backed by an Arena (see its docs).
|
||||
type SendBatch struct {
|
||||
out batchWriter
|
||||
bufs [][]byte
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
//go:build !ios && !e2e_testing
|
||||
|
||||
package overlay
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestTun_WriteFraming(t *testing.T) {
|
||||
r, w, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
defer w.Close()
|
||||
tn := &tun{f: w}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
pkt []byte
|
||||
af byte
|
||||
}{
|
||||
// Literal families rather than syscall.AF_*, so a wrong constant in Write can't also fix the expectation.
|
||||
{"ipv4", []byte{0x45, 1, 2, 3, 4, 5}, 2},
|
||||
{"ipv6", []byte{0x60, 1, 2, 3, 4, 5}, 30},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
n, err := tn.Write(tc.pkt)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, len(tc.pkt), n)
|
||||
|
||||
got := make([]byte, 64)
|
||||
n, err = r.Read(got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, append([]byte{0, 0, 0, tc.af}, tc.pkt...), got[:n])
|
||||
})
|
||||
}
|
||||
|
||||
_, err = tn.Write([]byte{0x10, 1, 2, 3})
|
||||
require.Error(t, err)
|
||||
_, err = tn.Write([]byte{})
|
||||
require.ErrorIs(t, err, syscall.EIO)
|
||||
|
||||
// The rejected writes must not have put anything on the fd, so the next read sees only this packet.
|
||||
pkt := []byte{0x45, 9, 9}
|
||||
_, err = tn.Write(pkt)
|
||||
require.NoError(t, err)
|
||||
got := make([]byte, 64)
|
||||
n, err := r.Read(got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, append([]byte{0, 0, 0, 2}, pkt...), got[:n])
|
||||
}
|
||||
|
||||
func TestTun_ReadStripsHeader(t *testing.T) {
|
||||
r, w, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
defer w.Close()
|
||||
tn := &tun{f: r}
|
||||
|
||||
pkt := []byte{0x45, 1, 2, 3, 4, 5}
|
||||
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
|
||||
require.NoError(t, err)
|
||||
|
||||
got := make([]byte, 64)
|
||||
n, err := tn.Read(got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, pkt, got[:n])
|
||||
}
|
||||
|
||||
func TestTun_ReadShort(t *testing.T) {
|
||||
r, w, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
defer w.Close()
|
||||
tn := &tun{f: r}
|
||||
|
||||
// Fewer bytes than the protocol header is not a packet.
|
||||
_, err = w.Write([]byte{0, 0})
|
||||
require.NoError(t, err)
|
||||
n, err := tn.Read(make([]byte, 64))
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, n)
|
||||
|
||||
// The next packet reads cleanly only if the short read consumed exactly what was written.
|
||||
pkt := []byte{0x45, 1, 2, 3}
|
||||
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
|
||||
require.NoError(t, err)
|
||||
got := make([]byte, 64)
|
||||
n, err = tn.Read(got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, pkt, got[:n])
|
||||
}
|
||||
|
||||
// A heap allocation per packet rarely shows in a short throughput run, only across GC pauses, so
|
||||
// this pins Read and Write to zero allocations directly.
|
||||
func TestTun_ReadWriteDoNotAllocate(t *testing.T) {
|
||||
r, w, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
defer w.Close()
|
||||
tw := &tun{f: w}
|
||||
tr := &tun{f: r}
|
||||
|
||||
pkts := [][]byte{bytes.Repeat([]byte{0x45}, 1300), bytes.Repeat([]byte{0x60}, 1300)}
|
||||
buf := make([]byte, 1500)
|
||||
i := 0
|
||||
allocs := testing.AllocsPerRun(1000, func() {
|
||||
i++
|
||||
if _, err := tw.Write(pkts[i%len(pkts)]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tr.Read(buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
require.Zero(t, allocs)
|
||||
}
|
||||
+4
-2
@@ -36,11 +36,13 @@ func (l *tcpListener) Accept() (net.Conn, error) {
|
||||
|
||||
func (l *tcpListener) Close() error {
|
||||
l.closeOnce.Do(func() {
|
||||
// Close before leaving the map so a concurrent closeListeners that
|
||||
// no longer sees this listener knows Accept is already released.
|
||||
close(l.closed)
|
||||
|
||||
l.s.mu.Lock()
|
||||
delete(l.s.mu.listeners, l.port)
|
||||
l.s.mu.Unlock()
|
||||
|
||||
close(l.closed)
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+70
-7
@@ -32,13 +32,18 @@ import (
|
||||
const nicID = 1
|
||||
|
||||
type Service struct {
|
||||
eg *errgroup.Group
|
||||
control *nebula.Control
|
||||
ipstack *stack.Stack
|
||||
|
||||
// done is closed once the forwarding goroutines have exited and the
|
||||
// netstack is torn down, at which point err holds the first fatal error.
|
||||
done chan struct{}
|
||||
err error
|
||||
|
||||
mu struct {
|
||||
sync.Mutex
|
||||
|
||||
closed bool
|
||||
listeners map[uint16]*tcpListener
|
||||
}
|
||||
}
|
||||
@@ -65,7 +70,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
||||
ctx := control.Context()
|
||||
eg, ctx := errgroup.WithContext(ctx)
|
||||
s := Service{
|
||||
eg: eg,
|
||||
control: control,
|
||||
}
|
||||
s.mu.listeners = map[uint16]*tcpListener{}
|
||||
@@ -74,6 +78,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
||||
NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
|
||||
TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6},
|
||||
})
|
||||
// The teardown goroutine only owns the stack once New succeeds
|
||||
defer func() {
|
||||
if reterr != nil {
|
||||
s.ipstack.Destroy()
|
||||
}
|
||||
}()
|
||||
sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default
|
||||
tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt)
|
||||
if tcpipErr != nil {
|
||||
@@ -123,6 +133,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
||||
// this will read exactly one packet
|
||||
n, err := reader.Read(buf)
|
||||
if err != nil {
|
||||
// An error after shutdown began is noise from the device pipes
|
||||
// closing, anything else is fatal and must stop nebula too
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
s.control.Stop()
|
||||
return err
|
||||
}
|
||||
packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{
|
||||
@@ -130,8 +146,8 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
||||
})
|
||||
linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf)
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -139,13 +155,18 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
||||
for {
|
||||
packet := linkEP.ReadContext(ctx)
|
||||
if packet == nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
bufView := packet.ToView()
|
||||
if _, err := bufView.WriteTo(writer); err != nil {
|
||||
// Same shutdown noise handling as the inbound reader
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
s.control.Stop()
|
||||
return err
|
||||
}
|
||||
bufView.Release()
|
||||
@@ -158,6 +179,22 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
|
||||
return control.Wait()
|
||||
})
|
||||
|
||||
// Tear down here rather than in Close so it also happens when nebula stops
|
||||
// on its own. The forwarding goroutines must exit before the stack they feed
|
||||
// is destroyed.
|
||||
s.done = make(chan struct{})
|
||||
go func() {
|
||||
s.err = eg.Wait()
|
||||
s.closeListeners()
|
||||
// Destroy aborts the endpoints that exist when it starts, then waits on
|
||||
// every endpoint, so a Dial registering in between would hold Wait until
|
||||
// its connect timed out. Without the NIC new dials fail route lookup.
|
||||
// RemoveNIC also closes linkEP.
|
||||
_ = s.ipstack.RemoveNIC(nicID)
|
||||
s.ipstack.Destroy()
|
||||
close(s.done)
|
||||
}()
|
||||
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
@@ -237,6 +274,9 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.mu.closed {
|
||||
return nil, net.ErrClosed
|
||||
}
|
||||
if _, ok := s.mu.listeners[port]; ok {
|
||||
return nil, fmt.Errorf("already listening on port %d", port)
|
||||
}
|
||||
@@ -245,15 +285,38 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// Wait blocks until the service has fully stopped, either via Close or a fatal
|
||||
// error, and returns the first fatal error if there was one.
|
||||
func (s *Service) Wait() error {
|
||||
return s.eg.Wait()
|
||||
<-s.done
|
||||
return s.err
|
||||
}
|
||||
|
||||
// Close closes all listeners and stops nebula. Use Wait to block until
|
||||
// shutdown has completed.
|
||||
func (s *Service) Close() error {
|
||||
s.closeListeners()
|
||||
s.control.Stop()
|
||||
return nil
|
||||
}
|
||||
|
||||
// closeListeners unblocks pending Accept calls and rejects new listeners.
|
||||
func (s *Service) closeListeners() {
|
||||
// Every caller closes every listener rather than returning early once
|
||||
// closed is set, so none returns while another is still closing them.
|
||||
s.mu.Lock()
|
||||
s.mu.closed = true
|
||||
listeners := make([]*tcpListener, 0, len(s.mu.listeners))
|
||||
for _, l := range s.mu.listeners {
|
||||
listeners = append(listeners, l)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
for _, l := range listeners {
|
||||
_ = l.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) tcpHandler(r *tcp.ForwarderRequest) {
|
||||
endpointID := r.ID()
|
||||
|
||||
|
||||
@@ -327,3 +327,162 @@ func TestListenerAcceptAfterClose(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close must release blocked Accept calls and let Wait report a clean shutdown.
|
||||
func TestServiceClose(t *testing.T) {
|
||||
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
|
||||
"static_host_map": m{},
|
||||
"listen": m{
|
||||
"host": "127.0.0.1",
|
||||
"port": 0,
|
||||
},
|
||||
})
|
||||
|
||||
ln, err := a.Listen("tcp", ":1234")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
accepted := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := ln.Accept()
|
||||
accepted <- err
|
||||
}()
|
||||
|
||||
if err := a.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-accepted:
|
||||
if !errors.Is(err, net.ErrClosed) {
|
||||
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Accept still blocked after Close")
|
||||
}
|
||||
|
||||
waited := make(chan error, 1)
|
||||
go func() { waited <- a.Wait() }()
|
||||
select {
|
||||
case err := <-waited:
|
||||
if err != nil {
|
||||
t.Fatalf("expected clean shutdown from Wait, got %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Wait still blocked after Close")
|
||||
}
|
||||
|
||||
if a.ipstack.HasNIC(nicID) {
|
||||
t.Fatal("netstack not torn down after Wait")
|
||||
}
|
||||
if err := a.Close(); err != nil {
|
||||
t.Fatalf("second close: %v", err)
|
||||
}
|
||||
if _, err := a.Listen("tcp", ":1235"); !errors.Is(err, net.ErrClosed) {
|
||||
t.Fatalf("expected net.ErrClosed from Listen after Close, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// If nebula stops without Close being called, the service must still release
|
||||
// Accept and tear down the netstack.
|
||||
func TestServiceNebulaStopped(t *testing.T) {
|
||||
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
|
||||
"static_host_map": m{},
|
||||
"listen": m{
|
||||
"host": "127.0.0.1",
|
||||
"port": 0,
|
||||
},
|
||||
})
|
||||
|
||||
ln, err := a.Listen("tcp", ":1234")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
accepted := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := ln.Accept()
|
||||
accepted <- err
|
||||
}()
|
||||
|
||||
a.control.Stop()
|
||||
|
||||
waited := make(chan error, 1)
|
||||
go func() { waited <- a.Wait() }()
|
||||
select {
|
||||
case err := <-waited:
|
||||
if err != nil {
|
||||
t.Fatalf("expected clean shutdown from Wait, got %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Wait still blocked after nebula stopped")
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-accepted:
|
||||
if !errors.Is(err, net.ErrClosed) {
|
||||
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Accept still blocked after nebula stopped")
|
||||
}
|
||||
|
||||
if a.ipstack.HasNIC(nicID) {
|
||||
t.Fatal("netstack not torn down after Wait")
|
||||
}
|
||||
}
|
||||
|
||||
// Dials racing Close must not hold Wait until their connect times out.
|
||||
func TestServiceDialDuringClose(t *testing.T) {
|
||||
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
|
||||
"static_host_map": m{},
|
||||
"listen": m{
|
||||
"host": "127.0.0.1",
|
||||
"port": 0,
|
||||
},
|
||||
})
|
||||
|
||||
stop := make(chan struct{})
|
||||
var eg errgroup.Group
|
||||
for range 8 {
|
||||
eg.Go(func() error {
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
// 10.0.0.2 has no tunnel, so each dial sits in SYN-SENT until aborted
|
||||
if c, err := a.Dial("tcp", "10.0.0.2:80"); err == nil {
|
||||
_ = c.Close()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
if err := a.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waited := make(chan error, 1)
|
||||
go func() { waited <- a.Wait() }()
|
||||
select {
|
||||
case err := <-waited:
|
||||
if err != nil {
|
||||
t.Fatalf("expected clean shutdown from Wait, got %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Wait blocked behind a dial that raced Close")
|
||||
}
|
||||
|
||||
close(stop)
|
||||
dialersDone := make(chan error, 1)
|
||||
go func() { dialersDone <- eg.Wait() }()
|
||||
select {
|
||||
case <-dialersDone:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Dial still blocked after Wait")
|
||||
}
|
||||
}
|
||||
+2
-5
@@ -147,10 +147,7 @@ func (tw *TimerWheel[T]) Purge() (T, bool) {
|
||||
tw.expired.Tail = nil
|
||||
}
|
||||
|
||||
// Clear out the items references, a cached item must not keep its value alive
|
||||
v := ti.Item
|
||||
var zero T
|
||||
ti.Item = zero
|
||||
// Clear out the items references
|
||||
ti.Next = nil
|
||||
|
||||
// Maybe cache it for later
|
||||
@@ -160,7 +157,7 @@ func (tw *TimerWheel[T]) Purge() (T, bool) {
|
||||
tw.itemsCached++
|
||||
}
|
||||
|
||||
return v, true
|
||||
return ti.Item, true
|
||||
}
|
||||
|
||||
// findWheel find the next position in the wheel for the provided timeout given the current tick
|
||||
|
||||
+1
-3
@@ -7,7 +7,6 @@ import (
|
||||
|
||||
"github.com/slackhq/nebula/firewall"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewTimerWheel(t *testing.T) {
|
||||
@@ -151,8 +150,7 @@ func TestTimerWheel_Purge(t *testing.T) {
|
||||
assert.Equal(t, 4, tw.itemsCached)
|
||||
ci := tw.itemCache
|
||||
for range 4 {
|
||||
require.NotNil(t, ci)
|
||||
assert.Zero(t, ci.Item)
|
||||
assert.NotNil(t, ci)
|
||||
ci = ci.Next
|
||||
}
|
||||
assert.Nil(t, ci)
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ type Conn interface {
|
||||
//
|
||||
// Returns the number of packets successfully written. A destination the kernel rejects costs only
|
||||
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
|
||||
// Not safe for concurrent use on the same Conn.
|
||||
// Safe for concurrent use on the same Conn.
|
||||
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
|
||||
// ReloadConfig applies the listen settings that can change, r is nil at startup
|
||||
ReloadConfig(c *config.C, r *config.Reload)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"net/netip"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
@@ -20,7 +21,7 @@ import (
|
||||
// batchWriter owns the sendmmsg(2)/UDP-GSO transmit path for a StdConn: the
|
||||
// scratch WriteBatch packs mmsghdr entries into, plus the GSO capability
|
||||
// state probed at socket creation. Each queue has its own StdConn and
|
||||
// batchWriter, so no locking is needed.
|
||||
// batchWriter; mu serializes batches from the TUN reader and relay forwarding.
|
||||
//
|
||||
// Terminology, smallest to largest:
|
||||
//
|
||||
@@ -39,6 +40,7 @@ import (
|
||||
// chunk the entries packed for one sendmmsg call, at most MaxWriteBatch.
|
||||
// batch the caller's whole bufs/addrs pair, processed as one or more chunks.
|
||||
type batchWriter struct {
|
||||
mu sync.Mutex
|
||||
fd int
|
||||
isV4 bool
|
||||
|
||||
@@ -192,6 +194,9 @@ func parseRelease(r string) (major, minor int) {
|
||||
// Returns the number of packets sent. An error means the call itself failed.
|
||||
// A short count means some destinations were undeliverable.
|
||||
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
if len(bufs) != len(addrs) {
|
||||
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
//go:build linux && !android && !e2e_testing
|
||||
|
||||
package udp
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The TUN reader and relay forwarding share a socket. Exercise their concurrent
|
||||
// batches against the actual GSO planner and scratch, without relying on a NIC.
|
||||
func TestWriteBatchConcurrent(t *testing.T) {
|
||||
const batches = 64
|
||||
w := newRewindTestWriter()
|
||||
sent := make(chan [][]byte, 2*batches)
|
||||
w.sendFn = func(start, n int) (int, error) {
|
||||
runtime.Gosched()
|
||||
sent <- capturePrepared(w, start, n)
|
||||
return n, nil
|
||||
}
|
||||
conn := &StdConn{bw: w}
|
||||
remote := netip.MustParseAddrPort("192.0.2.1:4242")
|
||||
var wg sync.WaitGroup
|
||||
start := make(chan struct{})
|
||||
for producer := range 2 {
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
for i := range batches {
|
||||
var bufs [][]byte
|
||||
var addrs []netip.AddrPort
|
||||
for j := range 4 {
|
||||
bufs = append(bufs, []byte{byte(producer), byte(i), byte(j), 0})
|
||||
addrs = append(addrs, remote)
|
||||
}
|
||||
n, err := conn.WriteBatch(bufs, addrs)
|
||||
if err != nil || n != len(bufs) {
|
||||
t.Errorf("WriteBatch = %d, %v; want %d, nil", n, err, len(bufs))
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(sent)
|
||||
var counts [2]int
|
||||
for packets := range sent {
|
||||
require.Len(t, packets, 4)
|
||||
producer := packets[0][0]
|
||||
require.Less(t, int(producer), len(counts))
|
||||
for j, packet := range packets {
|
||||
require.Equal(t, []byte{producer, byte(counts[producer]), byte(j), 0}, packet)
|
||||
}
|
||||
counts[producer]++
|
||||
}
|
||||
require.Equal(t, [2]int{batches, batches}, counts)
|
||||
}
|
||||
Reference in new issue
Block a user