Compare commits

..
42 changed files with 1250 additions and 579 deletions

No files matched your search

+11
View File
@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Changed
- A tunnel whose remote becomes unreachable, such as an IPv6 remote after moving to a network without IPv6, now
moves to another of the peer's addresses within a round trip or two instead of waiting 20-35s to be torn down
and rehandshaked. When a send fails because there is no route to the remote, the tunnel sends a test packet to
every other address it knows for the peer and asks the lighthouse to have the peer punch its new address.
Whichever path answers carries the tunnel. With `punchy.punch` enabled, idle tunnels notice on their next
keepalive punch. An idle tunnel whose remote is unreachable and none of whose other addresses answer is now
torn down by the connection manager rather than left in place. Not yet effective for relayed traffic or on
Windows with the default RIO backend.
### Removed
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
+28 -12
View File
@@ -14,6 +14,7 @@ import (
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
)
type trafficDecision int
@@ -171,7 +172,7 @@ func (cm *connectionManager) Start(ctx context.Context) {
}
func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now)
decision, hostinfo, primary, punch := cm.makeTrafficDecision(localIndex, now)
switch decision {
case deleteTunnel:
@@ -196,6 +197,20 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
}
cm.resetRelayTrafficCheck(hostinfo)
if punch != nil {
cm.sendPunch(punch)
}
}
// sendPunch keeps a tunnel's NAT state alive when we are not sending on it. When we have no route to its remote any
// more, typically after moving to a network without that address family, the tunnel probes its other remotes so the
// peer roams over to us. Nothing else would notice, a tunnel we are not sending on has nothing else that could fail.
func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
remote, err := cm.punchy.SendPunch(hostinfo)
if udp.IsUnreachable(err) {
hostinfo.remoteUnreachable(cm.intf, remote)
}
}
func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
@@ -313,7 +328,9 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
}
}
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) {
// makeTrafficDecision decides what to do with a tunnel under the hostmap lock. It also returns the tunnel to punch when
// we are not sending on it, for the caller to send once the lock is released.
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo, *HostInfo) {
// Read lock the main hostmap to order decisions based on tunnels being the primary tunnel
cm.hostMap.RLock()
defer cm.hostMap.RUnlock()
@@ -321,17 +338,17 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
hostinfo := cm.hostMap.Indexes[localIndex]
if hostinfo == nil {
cm.l.Debug("Not found in hostmap", "localIndex", localIndex)
return doNothing, nil, nil
return doNothing, nil, nil, nil
}
if cm.isInvalidCertificate(now, hostinfo) {
return closeTunnel, hostinfo, nil
return closeTunnel, hostinfo, nil, nil
}
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
return deleteTunnel, hostinfo, nil
return deleteTunnel, hostinfo, nil, nil
}
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
@@ -368,10 +385,10 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
if !outTraffic {
// Send a punch packet to keep the NAT state alive
cm.punchy.SendPunch(hostinfo)
return decision, hostinfo, primary, hostinfo
}
return decision, hostinfo, primary
return decision, hostinfo, primary, nil
}
if hostinfo.isPendingDeletion() {
@@ -380,7 +397,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
"tunnelCheck", m{"state": "dead", "method": "active"},
)
return deleteTunnel, hostinfo, nil
return deleteTunnel, hostinfo, nil, nil
}
decision := doNothing
@@ -394,14 +411,13 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
"primary", mainHostInfo,
)
return closeTunnel, hostinfo, primary
return closeTunnel, hostinfo, primary, nil
}
// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
// Just maintain NAT state if configured to do so.
cm.punchy.SendPunch(hostinfo)
cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
return doNothing, nil, nil
return doNothing, nil, nil, hostinfo
}
// We aren't receiving traffic but we are sending it. The outbound
@@ -427,7 +443,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
hostinfo.setPendingDeletion(true)
cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
return decision, hostinfo, nil
return decision, hostinfo, nil, nil
}
func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) {
+247 -6
View File
@@ -3,13 +3,18 @@ package nebula
import (
"crypto/ed25519"
"crypto/rand"
"io"
"net/netip"
"testing"
"time"
"github.com/flynn/noise"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/overlaytest"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
@@ -266,12 +271,12 @@ func Test_NewConnectionManager_CounterLimits(t *testing.T) {
// Still below the reject limit, the tunnel stays up
nc.In(hostinfo)
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, tryRehandshake, decision)
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, deleteTunnel, decision)
}
@@ -333,14 +338,14 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.True(t, (hostinfo.state.Load()&stateIn != 0))
now := time.Now()
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -348,7 +353,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
// Do another traffic check tick, should still not be pending deletion
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -358,7 +363,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
// Finally advance beyond the inactivity timeout
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
assert.Equal(t, closeTunnel, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -578,3 +583,239 @@ func (d *dummyCert) MarshalJSON() ([]byte, error) {
func (d *dummyCert) Copy() cert.Certificate {
return d
}
// newPunchTestConnectionManager is a connection manager punching through conn, holding the tunnel from newPunchTarget
func newPunchTestConnectionManager(t *testing.T, conn udp.Conn) (*connectionManager, *HostInfo) {
l := test.NewLogger()
hostMap := newHostMap(l)
hostMap.preferredRanges.Store(&[]netip.Prefix{})
lh := newTestLighthouse()
ifce := &Interface{
hostMap: hostMap,
inside: &overlaytest.NoopTun{},
outside: &udp.NoopConn{},
firewall: &Firewall{},
lightHouse: lh,
pki: &PKI{},
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
l: l,
}
ifce.pki.cs.Store(&CertState{
initiatingVersion: cert.Version1,
privateKey: []byte{},
v1Cert: &dummyCert{version: cert.Version1},
})
punchy := newPunchyForTest(t, map[string]any{"punch": true}, conn)
nc := newConnectionManagerFromConfig(l, config.NewC(l), hostMap, punchy)
nc.intf = ifce
ifce.connectionManager = nc
// Tunnel sends go through conn too, though without a key from withTestKey they never get as far as writing
ifce.writers = []udp.Conn{conn}
ifce.metricTxDropped = metrics.NilCounter{}
hostinfo := newPunchTarget()
hostinfo.localIndexId = 1099
hostinfo.remoteIndexId = 9901
hostinfo.ConnectionState = &ConnectionState{myCert: &dummyCert{version: cert.Version1}}
nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
return nc, hostinfo
}
// A tunnel we are not sending on gets punched whether or not it is receiving, the punch is the only thing that
// would notice its remote has become unreachable.
func Test_makeTrafficDecision_punchesWhenNotSending(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
now := time.Now()
// Receiving but not sending
hostinfo.takeTraffic()
nc.In(hostinfo)
decision, _, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Same(t, hostinfo, punch)
// Sending and receiving, our own traffic keeps the NAT state alive
nc.Out(hostinfo)
nc.In(hostinfo)
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Nil(t, punch)
// Unused
decision, decided, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, doNothing, decision)
assert.Nil(t, decided, "an unused tunnel has no relay usage to reset")
assert.Same(t, hostinfo, punch)
// Sending but not receiving gets a test packet instead, which finds an unreachable remote on its own
nc.Out(hostinfo)
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, sendTestPacket, decision)
assert.Nil(t, punch)
}
func Test_sendPunch_probesWhenRemoteUnreachable(t *testing.T) {
t.Run("unreachable", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
"the probe should ask the lighthouse to have the peer punch us")
// Within the probe interval nothing more goes out, however many sends fail
nc.sendPunch(hostinfo)
nc.sendPunch(hostinfo)
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("delivered", func(t *testing.T) {
conn := &recordingConn{}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("other failure", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: io.ErrShortWrite}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "only a missing route says to look elsewhere")
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
}
// A packet the data path had no route for names its tunnel in its header, the peer's index for it
func Test_sendUnreachable_findsTunnelFromHeader(t *testing.T) {
head := func(st header.MessageSubType, remoteIndex uint32) []byte {
return header.Encode(make([]byte, header.Len), header.Version, header.Message, st, remoteIndex, 1)
}
t.Run("direct", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV6)
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("not its remote", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV4)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "a failure on a remote the tunnel is not using is stale")
})
t.Run("relayed", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
// A relay header's index is the relay session's, it must not be taken for a tunnel's even when they collide
nc.intf.sendUnreachable(head(header.MessageRelay, hostinfo.remoteIndexId), punchTargetV6)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("unknown tunnel", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId+1), punchTargetV6)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
})
}
// withTestKey gives a tunnel a real key so its sends get encrypted and written instead of returning early
func withTestKey(hostinfo *HostInfo) {
cs := noise.NewCipherSuite(noise.DH25519, noise.CipherAESGCM, noise.HashSHA256)
hostinfo.ConnectionState.eKey = noiseutil.NewCipherState(noise.UnsafeNewCipherState(cs, [32]byte{}, 0), noise.CipherAESGCM)
}
// The probe itself, end to end from a failed send: a test packet to every remote but the one we cannot reach, and a
// lighthouse query, through both places a send can fail.
func Test_remoteUnreachable_probesOtherRemotes(t *testing.T) {
t.Run("control send", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV4}, conn.wrote, "the failed send, then the probe")
assert.Equal(t, []header.MessageType{header.Test, header.Test}, conn.types)
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("data path batch", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
// Two of the tunnel's packets fail in one batch, they probe once
sb := batch.NewSendBatch(conn, 4, 4096)
for c := uint64(1); c <= 2; c++ {
pkt := header.Encode(sb.Reserve(64), header.Version, header.Message, header.MessageNone, hostinfo.remoteIndexId, c)
sb.Commit(append(pkt, make([]byte, 32)...), punchTargetV6)
}
nc.intf.flushSendBatch(sb, 0)
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6, punchTargetV4}, conn.wrote,
"the failed packets, then one probe")
assert.Equal(t, header.Test, conn.types[2])
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("closing", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.SendMessageToHostInfo(header.CloseTunnel, 0, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a tunnel being torn down does not probe")
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("lighthouse", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.lightHouse.amLighthouse = true
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a lighthouse does not roam its clients")
})
t.Run("nothing else to probe", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.reQueryWait.Store(int64(time.Minute))
// Forget the IPv4 remote, all the tunnel knows is the one it cannot reach
hostinfo.remotes = NewRemoteList(hostinfo.vpnAddrs, nil)
hostinfo.remotes.LearnRemote(hostinfo.vpnAddrs[0], punchTargetV6)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
"the lighthouse is the only way to learn of another remote")
// Past the probe interval, but the lighthouse is only asked again on the requery schedule
hostinfo.nextUnreachableProbe.Store(0)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Empty(t, drainQueries(nc.intf.lightHouse))
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6}, conn.wrote)
})
}
// drainQueries returns every lighthouse query waiting to go out
func drainQueries(lh *LightHouse) []netip.Addr {
var queries []netip.Addr
for {
select {
case addr := <-lh.queryChan:
queries = append(queries, addr)
default:
return queries
}
}
}
+1 -1
View File
@@ -148,7 +148,7 @@ func (c *fakeConn) Rebind() error { c.rebinds++; ret
func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil }
func (c *fakeConn) ListenOut(_ udp.EncReader, _ func()) error { return nil }
func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
return len(bufs), nil
}
func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {}
+12
View File
@@ -117,6 +117,18 @@ func (c *Control) SetUDPAddr(addr netip.AddrPort) {
c.f.outside.(*udp.TesterConn).SetAddr(addr)
}
// GetUDPAddrFor returns the underlay address this node sends to dst from, which differs by family for a dual stack node
func (c *Control) GetUDPAddrFor(dst netip.AddrPort) netip.AddrPort {
return c.f.outside.(*udp.TesterConn).GetAddrFor(dst)
}
// SetDualStackUDPAddrs gives this node one underlay address per family, sending from whichever matches the
// destination. An invalid AddrPort leaves the node without that family, sends to it fail with ENETUNREACH. Register
// any address the node did not start on with the router as well or nothing will route back.
func (c *Control) SetDualStackUDPAddrs(v4, v6 netip.AddrPort) {
c.f.outside.(*udp.TesterConn).SetDualStackAddrs(v4, v6)
}
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
+305
View File
@@ -0,0 +1,305 @@
//go:build e2e_testing
package e2e
import (
"net/netip"
"sync/atomic"
"testing"
"time"
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/e2e/router"
"github.com/slackhq/nebula/udp"
)
// TestDualStackRoamToV4Only measures how long a laptop's tunnels take to come back after it moves from a dual stack
// network, where its tunnels formed over IPv6, to one with no IPv6 service at all: the office to hotel wifi. Every
// peer still has an IPv4 address the laptop can reach, so nothing about the move should strand a tunnel.
//
// It is a measurement, not a pass/fail assertion, and it runs on the production connection manager timers.
//
// go test -tags e2e_testing -v -run TestDualStackRoamToV4Only ./e2e/
func TestDualStackRoamToV4Only(t *testing.T) {
for _, tc := range []dualStackRoamCase{
{name: "both send", meSends: true, themSends: true},
{name: "both send, rebind", meSends: true, themSends: true, rebind: true},
{name: "both send, two tunnels", meSends: true, themSends: true, simultaneous: true},
{name: "laptop sends", meSends: true},
{name: "peer sends", themSends: true},
{name: "peer sends, rebind", themSends: true, rebind: true},
{name: "peer sends, lh over v4", themSends: true, lhOverV4: true},
{name: "peer sends, punchy", themSends: true, punchy: true},
{name: "peer sends, punchy all", themSends: true, punchy: true, punchAll: true},
{name: "laptop sends, peer NAT", meSends: true, peerNAT: true},
{name: "laptop sends, peer NAT, lh v4", meSends: true, peerNAT: true, lhOverV4: true},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fwd, rev := measureDualStackRoam(t, tc)
t.Logf("RESULT %-24s laptop->peer %-9v peer->laptop %-9v", tc.name, fmtRecovery(fwd), fmtRecovery(rev))
})
}
}
// TestDualStackRoamRecovers holds the line TestDualStackRoamToV4Only measures: a tunnel whose remote became unreachable
// moves to the peer's other address instead of waiting out the connection manager to rehandshake, which takes 20s.
func TestDualStackRoamRecovers(t *testing.T) {
for _, tc := range []struct {
dualStackRoamCase
// within bounds recovery in each direction that has traffic
within time.Duration
}{
// Measures 0.4s. Under the 5s connection manager check, so it fails if the data path stops probing.
{dualStackRoamCase{name: "both send", meSends: true, themSends: true}, 2 * time.Second},
// Measures 2.6s or 3.8s, depending on whether a probe beats the peer's punch. Half the 20s it used to take.
{dualStackRoamCase{name: "laptop sends, peer NAT", meSends: true, peerNAT: true}, 10 * time.Second},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fwd, rev := measureDualStackRoam(t, tc.dualStackRoamCase)
if fwd == 0 || fwd > tc.within {
t.Errorf("laptop->peer recovered in %v, want within %v", fmtRecovery(fwd), tc.within)
}
if tc.themSends && (rev == 0 || rev > tc.within) {
t.Errorf("peer->laptop recovered in %v, want within %v", fmtRecovery(rev), tc.within)
}
})
}
}
type dualStackRoamCase struct {
name string
// meSends and themSends say who keeps traffic flowing after the move
meSends, themSends bool
// rebind fires RebindUDPServer on the move, as the darwin network change monitor would
rebind bool
// lhOverV4 forms the lighthouse tunnel over IPv4 at the office, so the move does not break it
lhOverV4 bool
// simultaneous has both sides initiate at once at the office, which can leave each side a different primary
// tunnel for the other
simultaneous bool
// punchy turns on punchy.punch everywhere, punchAll adds punchy.target_all_remotes
punchy, punchAll bool
// peerNAT puts the peer behind a NAT that, after the move, only lets in traffic from addresses the peer has sent to
peerNAT bool
}
func fmtRecovery(d time.Duration) string {
if d == 0 {
return "never"
}
return d.Round(time.Millisecond).String()
}
type dualStackNode struct {
c *nebula.Control
vpnAddr netip.Addr
v4, v6 netip.AddrPort
adv atomic.Pointer[[]netip.Addr]
}
// move puts the node on a new network, advertising only what it can be reached on there
func (n *dualStackNode) move(v4, v6 netip.AddrPort) {
n.v4, n.v6 = v4, v6
n.c.SetDualStackUDPAddrs(v4, v6)
var adv []netip.Addr
for _, a := range []netip.AddrPort{v4, v6} {
if a.IsValid() {
adv = append(adv, a.Addr())
}
}
n.adv.Store(&adv)
}
func newDualStackNode(ca cert.Certificate, caKey []byte, name, vpnNet string, v4, v6 netip.AddrPort, overrides m) *dualStackNode {
c, vpnNets, _, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, name, vpnNet, v4, overrides)
n := &dualStackNode{c: c, vpnAddr: vpnNets[0].Addr()}
n.move(v4, v6)
c.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
return *n.adv.Load()
})
return n
}
// measureDualStackRoam returns how long after the move the first packet made it laptop->peer and peer->laptop, zero
// for a direction that never recovered.
func measureDualStackRoam(t *testing.T, tc dualStackRoamCase) (time.Duration, time.Duration) {
t.Helper()
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
ap := netip.MustParseAddrPort
lhV4, lhV6 := ap("10.0.0.1:4242"), ap("[fd00::1]:4242")
lh := newDualStackNode(ca, caKey, "lh", "10.128.0.1/24", lhV4, lhV6, m{
"lighthouse": m{"am_lighthouse": true},
})
// Production timers rather than the 2s the harness defaults to, this is about how long a real laptop waits
peerCfg := m{
"lighthouse": m{
"hosts": []any{lh.vpnAddr.String()},
"interval": 10,
},
"static_host_map": m{
lh.vpnAddr.String(): []any{lhV4.String(), lhV6.String()},
},
"timers": m{
"connection_alive_interval": 5,
"pending_deletion_interval": 10,
},
"punchy": m{
"punch": tc.punchy,
"target_all_remotes": tc.punchAll,
},
}
me := newDualStackNode(ca, caKey, "me", "10.128.0.2/24", ap("10.0.0.2:4242"), ap("[fd00::2]:4242"), peerCfg)
them := newDualStackNode(ca, caKey, "them", "10.128.0.3/24", ap("10.0.0.3:4242"), ap("[fd00::3]:4242"), peerCfg)
r := router.NewR(t, lh.c, me.c, them.c)
r.CancelFlowLogs()
for _, n := range []*dualStackNode{lh, me, them} {
r.AddRoute(n.v6.Addr(), n.v6.Port(), n.c)
}
defer func() {
lh.c.Stop()
me.c.Stop()
them.c.Stop()
}()
lh.c.Start()
me.c.Start()
them.c.Start()
// At the office IPv6 wins every race, so everything forms over it. The lighthouse tunnel optionally does not,
// for a static_host_map that only lists the lighthouse's IPv4 address.
officeV6 := func(p *udp.Packet, _ *nebula.Control) router.ExitType {
lhTraffic := p.To == lhV4 || p.To == lhV6 || p.From == lhV4 || p.From == lhV6
if p.To.Addr().Is4() == (tc.lhOverV4 && lhTraffic) {
return router.KeepRouting
}
return router.Drop
}
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
// One side at a time, so both sides hold the same single tunnel, unless the case wants them racing
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("establish")))
if !tc.simultaneous {
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
}
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("establish")))
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
if tc.simultaneous && me.c.GetHostmapIndexCount() < 3 {
// A tunnel to the lighthouse and two to the peer, or the race did not happen
t.Logf("simultaneous handshakes settled on a single tunnel this run")
}
drainTun(me.c)
drainTun(them.c)
for _, want := range []struct {
name string
c *nebula.Control
vpnAddr netip.Addr
v6 bool
}{
{"laptop->peer", me.c, them.vpnAddr, true},
{"peer->laptop", them.c, me.vpnAddr, true},
{"laptop->lh", me.c, lh.vpnAddr, !tc.lhOverV4},
} {
hi := want.c.GetHostInfoByVpnAddr(want.vpnAddr, false)
if hi == nil {
t.Fatalf("%s: no tunnel at the office", want.name)
}
if hi.CurrentRemote.Addr().Is6() != want.v6 {
t.Fatalf("%s: tunnel formed over %v, the scenario wants v6=%v", want.name, hi.CurrentRemote, want.v6)
}
}
// Hotel wifi: a new NAT'd IPv4 address and no IPv6 at all. The office addresses are gone, nothing reaches them.
officeV4, officeV6Addr := me.v4, me.v6
hotel := ap("10.0.0.99:4242")
me.move(hotel, netip.AddrPort{})
r.AddRoute(hotel.Addr(), hotel.Port(), me.c)
start := time.Now()
if tc.rebind {
me.c.RebindUDPServer()
}
var fwd, rev atomic.Int64
stop := make(chan struct{})
defer close(stop)
watchTun(them.c, stop, start, &fwd)
watchTun(me.c, stop, start, &rev)
go func() {
tick := time.NewTicker(time.Millisecond * 200)
defer tick.Stop()
for {
select {
case <-stop:
return
case <-tick.C:
if tc.meSends {
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("retry")))
}
if tc.themSends {
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("retry")))
}
}
}
}()
// When only one side has traffic, the other direction recovering is a bonus we still report but do not wait for
done := func() bool {
return (!tc.meSends || fwd.Load() != 0) && (!tc.themSends || rev.Load() != 0)
}
// The peer's NAT has state for the lighthouse, which its own updates keep fresh, and for wherever it sends from here
natOpen := map[netip.AddrPort]bool{lhV4: true, lhV6: true}
isPeer := func(a netip.AddrPort) bool { return a == them.v4 || a == them.v6 }
r.RouteForAllExitFuncOrTimeout(time.Second*90, func(p *udp.Packet, _ *nebula.Control) router.ExitType {
if p.To == officeV4 || p.To == officeV6Addr {
return router.Drop
}
if tc.peerNAT {
if isPeer(p.From) {
natOpen[p.To] = true
} else if isPeer(p.To) && !natOpen[p.From] {
return router.Drop
}
}
if done() {
return router.ExitNow
}
return router.KeepRouting
})
return time.Duration(fwd.Load()), time.Duration(rev.Load())
}
func drainTun(c *nebula.Control) {
for {
select {
case <-c.GetTunTxChan():
default:
return
}
}
}
// watchTun records how long after start the first packet came out of c's tun, and keeps the tun drained after
func watchTun(c *nebula.Control, stop chan struct{}, start time.Time, first *atomic.Int64) {
go func() {
for {
select {
case <-stop:
return
case <-c.GetTunTxChan():
first.CompareAndSwap(0, int64(time.Since(start)))
}
}
}()
}
+14 -13
View File
@@ -497,11 +497,10 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []
// Nope, lets push the sender along
case p := <-udpTx:
r.Lock()
a := sender.GetUDPAddr()
c := r.getControl(a, p.To, p)
c := r.getControl(p)
if c == nil {
r.Unlock()
panic("No control for udp tx " + a.String())
panic("No control for udp tx " + p.To.String())
}
fp := r.unlockedInjectFlow(sender, c, p, false)
c.InjectUDPPacket(p) // copies internally; original is ours to release
@@ -622,8 +621,7 @@ func (r *R) stopFanInWorkers() {
func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
r.Lock()
defer r.Unlock()
a := from.GetUDPAddr()
c := r.getControl(a, p.To, p)
c := r.getControl(p)
if c == nil {
panic(fmt.Sprintf("No control for udp tx %s", p.To))
}
@@ -671,7 +669,7 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
panic(err)
}
receiver := r.getControl(sender.GetUDPAddr(), p.To, p)
receiver := r.getControl(p)
if receiver == nil {
r.Unlock()
panic("Can't RouteExitFunc for host: " + p.To.String())
@@ -767,7 +765,7 @@ func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc)
r.Lock()
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
receiver := r.getControl(p)
if receiver == nil {
r.Unlock()
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
@@ -876,7 +874,7 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
r.Lock()
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
receiver := r.getControl(p)
if receiver == nil {
r.Unlock()
panic("Can't RouteForAllExitFunc for host: " + p.To.String())
@@ -939,7 +937,7 @@ func (r *R) FlushAll() {
})
for {
x, rx, ok := reflect.Select(sc)
_, rx, ok := reflect.Select(sc)
if !ok {
return
}
@@ -947,7 +945,7 @@ func (r *R) FlushAll() {
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
receiver := r.getControl(p)
if receiver == nil {
r.Unlock()
panic("Can't FlushAll for host: " + p.To.String())
@@ -958,16 +956,19 @@ func (r *R) FlushAll() {
}
}
// getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change
// getControl performs or seeds NAT translation and returns the control for p.To, p from fields may change
// This is an internal router function, the caller must hold the lock
func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control {
func (r *R) getControl(p *udp.Packet) *nebula.Control {
// Key on the address the packet actually left from, a dual stack sender has one per family
fromAddr, toAddr := p.From, p.To
if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
p.From = newAddr
}
c, ok := r.inNat[toAddr]
if ok {
r.outNat[outNatKey{from: c.GetUDPAddr(), to: fromAddr}] = toAddr
// Key the return path on the address the receiver will answer from, which is per family for a dual stack node
r.outNat[outNatKey{from: c.GetUDPAddrFor(fromAddr), to: fromAddr}] = toAddr
return c
}
+2 -3
View File
@@ -39,9 +39,8 @@ static_host_map:
# public address. Other valid options are "ip6" and "ip" (returns both.)
#network: ip4
# lookup_timeout is the DNS query timeout. A failed lookup keeps the last known good addresses, only a not found
# answer removes them.
#lookup_timeout: 5s
# lookup_timeout is the DNS query timeout.
#lookup_timeout: 250ms
lighthouse:
# am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
-2
View File
@@ -405,8 +405,6 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
hm.Unlock()
hm.lightHouse.QueryServer(vpnAddr)
// If this host came from a static_map hostname its addresses may have moved, retries will pick up anything new
hostinfo.remotes.ResolveHostnames()
return hostinfo
}
+62
View File
@@ -264,6 +264,9 @@ type HostInfo struct {
// This is used to limit lighthouse re-queries in chatty clients
nextLHQuery atomic.Int64
// nextUnreachableProbe is the earliest we will probe our other remotes again after failing to reach the current one
nextUnreachableProbe atomic.Int64
// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
// This is used to avoid an attack where a handshake packet is replayed after some time
@@ -766,6 +769,65 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
}
}
// unreachableProbeInterval bounds how often a tunnel whose remote we cannot reach probes its other remotes
const unreachableProbeInterval = time.Second
// remoteUnreachable is called when a send to our current remote failed because we have no route to it, typically
// after moving to a network without that address family. Sends a test packet to every other remote we know for this
// host: the first one that arrives roams the peer over to wherever we are now, and its reply roams us onto that path.
//
// A peer behind a NAT drops a test from an address it has never sent to, so every probe also asks the lighthouse
// about the peer. The lighthouse tells the peer where we are now, the peer punches us, and the next probe gets in.
func (i *HostInfo) remoteUnreachable(ifce *Interface, dead netip.AddrPort) {
if i.remotes == nil || i.ConnectionState == nil || i.GetRemote() != dead {
// Nothing to probe with, or we already moved on from the remote that failed
return
}
if ifce.lightHouse == nil || ifce.lightHouse.amLighthouse {
// A lighthouse hands out the address it reaches a client on, it must not go roaming its clients itself
return
}
now := time.Now().UnixNano()
next := i.nextUnreachableProbe.Load()
if now < next || !i.nextUnreachableProbe.CompareAndSwap(next, now+int64(unreachableProbeInterval)) {
return
}
nb := make([]byte, 12)
out := make([]byte, mtu)
probed := 0
i.remotes.ForEach(ifce.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) {
if addr == dead {
return
}
probed++
ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), nb, out)
})
if ifce.l.Enabled(context.Background(), slog.LevelDebug) {
i.logger(ifce.l).Debug("Remote unreachable, probing other remotes",
"udpAddr", dead,
"probed", probed,
)
}
if probed == 0 {
// With nothing else to try, the lighthouse is our only way to learn of another remote. The peer may not have
// one, so ask on the usual requery schedule rather than every probe for as long as the tunnel sits here.
if now < i.nextLHQuery.Load() {
return
}
i.nextLHQuery.Store(now + ifce.reQueryWait.Load())
}
// With other remotes to probe we ask on every probe, without backing off. A repeat re-sends the peer its punch
// notification in case the last one or its punches were lost. If the tunnel stays up on inbound traffic while
// our sends to its remote keep failing (only one direction broken), this continues once a second for as long as
// that lasts. That is a bounded cost for a rare state, and the probe that would clear it has to keep running.
ifce.lightHouse.QueryServer(i.vpnAddrs[0])
}
// Bits within HostInfo.state, everything above stateEpochShift is the epoch
const (
stateIn uint32 = 1 << iota
+5
View File
@@ -15,6 +15,7 @@ import (
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/udp"
)
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
@@ -611,6 +612,10 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
"udpAddr", hr,
)
}
// A closing tunnel is about to be torn down, there is no point looking for another path for it
if t != header.CloseTunnel && udp.IsUnreachable(err) {
hostinfo.remoteUnreachable(f, hr)
}
}
} else {
// Try to send via a relay
+50 -59
View File
@@ -1,6 +1,7 @@
package nebula
import (
"bytes"
"context"
"crypto/fips140"
"errors"
@@ -124,8 +125,6 @@ type Interface struct {
// reused (every udp.Conn ListenOut guarantees that ordering).
batchers []*batch.MultiCoalescer
wg sync.WaitGroup
// readers covers just the listenIn and listenOut goroutines, wg also holds the construction token Close releases
readers sync.WaitGroup
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
@@ -343,18 +342,14 @@ func (f *Interface) activate() error {
func (f *Interface) run() {
// Launch n queues to read packets from udp
for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() {
defer f.readers.Done()
f.listenOut(i)
})
}
// Launch n queues to read packets from tun dev
for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() {
defer f.readers.Done()
f.listenIn(f.queues[i], i)
})
}
@@ -501,7 +496,8 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
// flushSendBatch drains sb to the underlay and accounts for anything it could not deliver. A shortfall means
// specific destinations were undeliverable (a stale remote, a reject rule), which the backend logs per peer at
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch.
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch. Tunnels whose remote
// we no longer have a route to go looking for another path.
func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
queued := sb.Len()
written, err := sb.Flush()
@@ -511,6 +507,44 @@ func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
if dropped := queued - written; dropped > 0 {
f.metricTxDropped.Inc(int64(dropped))
}
if f.lightHouse == nil || f.lightHouse.amLighthouse {
// Lighthouses never probe, see remoteUnreachable, so skip finding the tunnels
return
}
// A tunnel's packets tend to fail together, so skip straight past a repeat of the one before. The first 8 bytes
// of a header say whose packet it is, the counter after them changes with every packet.
var last batch.Failure
for i, fail := range sb.Failures() {
if i > 0 && fail.Dst == last.Dst && bytes.Equal(fail.Head[:8], last.Head[:8]) {
continue
}
last = fail
if udp.IsUnreachable(fail.Err) {
f.sendUnreachable(fail.Head[:], fail.Dst)
}
}
}
// sendUnreachable finds the tunnel a packet we had no route for was sent on, from the start of the packet, and has it
// look for another path to its peer.
func (f *Interface) sendUnreachable(head []byte, dst netip.AddrPort) {
var h header.H
if h.Parse(head) != nil {
return
}
if h.Type == header.Message && h.Subtype == header.MessageRelay {
// TODO: a relayed packet's header carries the relay session's index, not the relay tunnel's, so it does not
// lead us to the tunnel whose remote failed. The relay tunnel's own traffic finds out soon enough.
return
}
if hostinfo := f.hostMap.QueryReverseIndex(h.RemoteIndex); hostinfo != nil {
hostinfo.remoteUnreachable(f, dst)
}
}
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
@@ -689,14 +723,6 @@ func (f *Interface) GetCertState() *CertState {
return f.pki.getCertState()
}
// shutdowner is a udp.Conn or overlay.Device whose readers can be woken without releasing their fds
type shutdowner interface {
Shutdown() error
}
// readerShutdownTimeout bounds how long Close waits on woken readers before releasing their fds anyway
const readerShutdownTimeout = 5 * time.Second
// Close releases the interface's resources: the udp sockets and the tun device.
// It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated,
// calls after the first return nil without doing anything.
@@ -706,55 +732,20 @@ func (f *Interface) Close() error {
}
var errs []error
closeUdp := func(i int, u udp.Conn) {
if err := u.Close(); err != nil {
// Release the udp readers
for i, u := range f.writers {
err := u.Close()
if err != nil {
f.l.Error("Error while closing udp socket", "error", err, "writer", i)
errs = append(errs, err)
}
}
closeTun := func() {
if err := f.inside.Close(); err != nil {
errs = append(errs, err)
}
}
// Wake every reader before releasing any fd, one that loops back into a syscall on a reused fd number never wakes.
// Anything without Shutdown is closed right away, as before.
var deferred []func()
for i, u := range f.writers {
if s, ok := u.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
f.l.Error("Error while shutting down udp socket", "error", err, "writer", i)
errs = append(errs, err)
}
deferred = append(deferred, func() { closeUdp(i, u) })
} else {
closeUdp(i, u)
}
}
if s, ok := f.inside.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
errs = append(errs, err)
}
deferred = append(deferred, closeTun)
} else {
closeTun()
}
if len(deferred) > 0 {
readersDone := make(chan struct{})
go func() {
f.readers.Wait()
close(readersDone)
}()
select {
case <-readersDone:
case <-time.After(readerShutdownTimeout):
f.l.Warn("Readers did not stop, closing their fds anyway", "timeout", readerShutdownTimeout)
}
for _, closeFn := range deferred {
closeFn()
}
// Release the tun device (closing the tun also closes all readers)
closeErr := f.inside.Close()
if closeErr != nil {
errs = append(errs, closeErr)
}
// Release the construction token so waiters know the resources are gone
+3 -5
View File
@@ -452,7 +452,7 @@ func getStaticMapCadence(c *config.C) (time.Duration, error) {
}
func getStaticMapLookupTimeout(c *config.C) (time.Duration, error) {
lookupTimeout := c.GetString("static_map.lookup_timeout", "5s")
lookupTimeout := c.GetString("static_map.lookup_timeout", "250ms")
d, err := time.ParseDuration(lookupTimeout)
if err != nil {
return 0, err
@@ -630,8 +630,7 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
ctx := lh.ctx
lh.Unlock()
// On reload am.hr is the previous resolver, its last known good addresses carry over in case DNS is down right now
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, am.hr, func() {
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, func() {
// This callback runs whenever the DNS hostname resolver finds a different set of addr's
// in its resolution for hostnames.
am.Lock()
@@ -643,8 +642,7 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
}
am.unlockedSetHostnamesResults(hr)
// Only literals belong in the cache, anything from DNS has to stay in hr where a not found answer can remove it
for _, addrPort := range hr.literalAddrs() {
for _, addrPort := range hr.GetAddrs() {
if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) {
continue
}
+41 -5
View File
@@ -1,12 +1,27 @@
package batch
import "net/netip"
import (
"net/netip"
"github.com/slackhq/nebula/header"
)
const SendBatchCap = 128
// FailureHeadLen is how much of a refused packet a Failure keeps, nebula's header, which says whose packet it was
const FailureHeadLen = header.Len
// batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush.
type batchWriter interface {
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
}
// Failure is a packet the underlay would not send
type Failure struct {
// Head is the start of the packet, zero padded if the packet was shorter
Head [FailureHeadLen]byte
Dst netip.AddrPort
Err error
}
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
@@ -17,16 +32,28 @@ type SendBatch struct {
bufs [][]byte
dsts []netip.AddrPort
arena *Arena
failures []Failure
// recordFailure is bound once so handing it to WriteBatch does not allocate on every Flush
recordFailure func(i int, err error)
}
// NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots
func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch {
return &SendBatch{
b := &SendBatch{
out: out,
bufs: make([][]byte, 0, batchCap),
dsts: make([]netip.AddrPort, 0, batchCap),
arena: NewArena(arenaSize),
}
b.recordFailure = b.failed
return b
}
func (b *SendBatch) failed(i int, err error) {
f := Failure{Dst: b.dsts[i], Err: err}
copy(f.Head[:], b.bufs[i])
b.failures = append(b.failures, f)
}
func (b *SendBatch) Reserve(sz int) []byte {
@@ -44,12 +71,15 @@ func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort) {
}
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations
// were undeliverable; the batch is drained either way.
// were undeliverable, Failures says which; the batch is drained either way.
func (b *SendBatch) Flush() (int, error) {
clear(b.failures)
b.failures = b.failures[:0]
var err error
written := 0
if len(b.bufs) > 0 {
written, err = b.out.WriteBatch(b.bufs, b.dsts)
written, err = b.out.WriteBatch(b.bufs, b.dsts, b.recordFailure)
}
clear(b.bufs)
b.bufs = b.bufs[:0]
@@ -57,3 +87,9 @@ func (b *SendBatch) Flush() (int, error) {
b.arena.Reset()
return written, err
}
// Failures returns every packet the last Flush could not send, in the order they were committed. Valid until the
// next Flush.
func (b *SendBatch) Failures() []Failure {
return b.failures
}
+84 -1
View File
@@ -1,7 +1,9 @@
package batch
import (
"errors"
"net/netip"
"slices"
"testing"
)
@@ -10,7 +12,7 @@ type fakeBatchWriter struct {
addrs []netip.AddrPort
}
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, _ func(int, error)) (int, error) {
// Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch
// returns, so tests must capture data before that happens.
w.bufs = make([][]byte, len(bufs))
@@ -120,3 +122,84 @@ func TestSendBatchGrowPreservesCommitted(t *testing.T) {
t.Errorf("second packet on the wire: %x", fw.bufs[1])
}
}
// failingBatchWriter refuses every packet whose first byte is in refuse
type failingBatchWriter struct {
refuse map[byte]error
}
func (w *failingBatchWriter) WriteBatch(bufs [][]byte, _ []netip.AddrPort, failed func(int, error)) (int, error) {
written := 0
for i, b := range bufs {
if err, ok := w.refuse[b[0]]; ok {
failed(i, err)
continue
}
written++
}
return written, nil
}
func TestSendBatchReportsFailures(t *testing.T) {
errNoRoute := errors.New("no route")
fw := &failingBatchWriter{refuse: map[byte]error{2: errNoRoute, 3: errNoRoute}}
b := NewSendBatch(fw, 4, 128)
// pkt is n bytes starting with the refuse key, each later byte its offset plus 0x10 so the head is recognizable
pkt := func(key byte, n int) []byte {
p := b.Reserve(n)[:n]
p[0] = key
for i := 1; i < n; i++ {
p[i] = byte(0x10 + i)
}
return p
}
head := func(p []byte) (h [FailureHeadLen]byte) {
copy(h[:], p)
return h
}
dsts := []netip.AddrPort{
netip.MustParseAddrPort("10.0.0.1:4242"),
netip.MustParseAddrPort("[fd00::2]:4242"),
netip.MustParseAddrPort("[fd00::3]:4242"),
netip.MustParseAddrPort("10.0.0.4:4242"),
}
long := pkt(2, FailureHeadLen+8) // only its head is kept
short := pkt(3, 4) // zero padded
b.Commit(pkt(1, 20), dsts[0])
b.Commit(long, dsts[1])
b.Commit(short, dsts[2])
b.Commit(pkt(1, 20), dsts[3])
written, err := b.Flush()
if err != nil {
t.Fatalf("Flush: %v", err)
}
if written != 2 {
t.Errorf("written = %d, want 2", written)
}
want := []Failure{{head(long), dsts[1], errNoRoute}, {head(short), dsts[2], errNoRoute}}
if got := b.Failures(); !slices.Equal(got, want) {
t.Errorf("failures = %v, want %v", got, want)
}
// Failures do not leak into the next flush
again := pkt(2, 20)
b.Commit(again, dsts[0])
if _, err := b.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if want := []Failure{{head(again), dsts[0], errNoRoute}}; !slices.Equal(b.Failures(), want) {
t.Errorf("second flush failures = %v, want %v", b.Failures(), want)
}
// A clean flush forgets the last one's failures
b.Commit(pkt(1, 20), dsts[0])
if _, err := b.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if got := b.Failures(); len(got) != 0 {
t.Errorf("clean flush failures = %v, want none", got)
}
}
+3 -13
View File
@@ -20,7 +20,6 @@ type offloadQueueSet struct {
// usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel.
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
usoEnabled bool
shutdown atomic.Bool
closed atomic.Bool
// l is handed to each queue for its bad-vnet-header drop logging.
l *slog.Logger
@@ -70,17 +69,6 @@ func (c *offloadQueueSet) wakeForShutdown() error {
return err
}
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *offloadQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *offloadQueueSet) Close() error {
if c.closed.Swap(true) {
return nil
@@ -88,7 +76,9 @@ func (c *offloadQueueSet) Close() error {
errs := []error{}
if err := c.Shutdown(); err != nil {
// Signal all readers blocked in poll to wake up and exit.
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
errs = append(errs, err)
}
+3 -13
View File
@@ -16,7 +16,6 @@ type pollQueueSet struct {
// pqi is exactly the same as pq, but stored as the interface type
pqi []Queue
shutdownFd int
shutdown atomic.Bool
closed atomic.Bool
}
@@ -60,17 +59,6 @@ func (c *pollQueueSet) wakeForShutdown() error {
return err
}
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *pollQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *pollQueueSet) Close() error {
if c.closed.Swap(true) {
return nil
@@ -78,7 +66,9 @@ func (c *pollQueueSet) Close() error {
errs := []error{}
if err := c.Shutdown(); err != nil {
// Signal all readers blocked in poll to wake up and exit.
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
errs = append(errs, err)
}
-2
View File
@@ -7,8 +7,6 @@ import (
// QueueSet holds one or many Queue objects and helps close them in an orderly way.
type QueueSet interface {
io.Closer
// Shutdown wakes every reader without releasing any fds, Close only once they have exited
Shutdown() error
Queues() []Queue
// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
+1 -6
View File
@@ -66,9 +66,7 @@ type Offload struct {
// usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD,
// so writers can decide whether emitting GSO_UDP_L4 superpackets is safe.
usoEnabled bool
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool
closed atomic.Bool
// gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted
// by WriteGSO. Kept separate from the read-only package-level validVnetHdr
@@ -178,9 +176,6 @@ func (r *Offload) readPacket(block bool) (int, error) {
// This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs).
// Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue.
func (r *Offload) Read() ([]Packet, error) {
if r.shutdown.Load() {
return nil, os.ErrClosed
}
r.pending = r.pending[:0]
r.rxOff = 0
+1 -6
View File
@@ -13,9 +13,7 @@ import (
type Poll struct {
fd int
shutdownFd int
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool
closed atomic.Bool
readBuf []byte
batchRet [1]Packet
@@ -54,9 +52,6 @@ func (t *Poll) blockOnWrite() error {
// config currently lose that batching. blockOn and the EAGAIN plumbing are
// already shared; kept one-packet-per-Read for now to preserve behavior.
func (t *Poll) Read() ([]Packet, error) {
if t.shutdown.Load() {
return nil, os.ErrClosed
}
n, err := t.readOne(t.readBuf)
if err != nil {
return nil, err
-27
View File
@@ -223,30 +223,3 @@ func TestOffloadQueueSet_Close_ClosesShutdownFd(t *testing.T) {
// Second Close must not touch fds (shutdownFd is now -1) and must return nil.
require.NoError(t, qs.Close())
}
// A busy reader never polls the shutdown eventfd, so Read has to stop on its own with data still waiting
func TestQueueSet_ShutdownStopsReaderWithDataWaiting(t *testing.T) {
sets := map[string]func() (QueueSet, error){
"poll": NewPollQueueSet,
"offload": func() (QueueSet, error) { return NewOffloadQueueSet(false, slog.New(slog.DiscardHandler)) },
}
for name, newSet := range sets {
t.Run(name, func(t *testing.T) {
var fds [2]int
require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
t.Cleanup(func() { _ = unix.Close(fds[1]) })
qs, err := newSet()
require.NoError(t, err)
require.NoError(t, qs.Add(fds[0]))
t.Cleanup(func() { _ = qs.Close() })
_, err = unix.Write(fds[1], make([]byte, 64))
require.NoError(t, err)
require.NoError(t, qs.Shutdown())
_, err = qs.Queues()[0].Read()
require.ErrorIs(t, err, os.ErrClosed)
})
}
}
-5
View File
@@ -832,11 +832,6 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
t.routeTree.Store(newTree)
}
// Shutdown wakes the tun readers without releasing their fds, Close releases them
func (t *tun) Shutdown() error {
return t.readers.Shutdown()
}
func (t *tun) Close() error {
t.closeLock.Lock()
defer t.closeLock.Unlock()
+19 -10
View File
@@ -165,20 +165,24 @@ func (p *Punchy) scheduleJob(job holepunchJob, delay time.Duration) {
// SendPunch sends an immediate keepalive punch for an idle hostinfo.
// The configured punchy.target_all_remotes mode picks the targets. Gated on punchy.punch and the lighthouse-skip rule
// (lighthouses don't get keepalive punches because the regular update interval keeps their NAT state warm).
func (p *Punchy) SendPunch(hostinfo *HostInfo) {
// Returns the hostinfo's current remote and the error from punching it, an invalid remote if it was not punched.
func (p *Punchy) SendPunch(hostinfo *HostInfo) (netip.AddrPort, error) {
if !p.punch.Load() {
return
return netip.AddrPort{}, nil
}
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
return
return netip.AddrPort{}, nil
}
hr := hostinfo.GetRemote()
if p.punchEverything.Load() {
p.sendPunchToAllRemotes(hostinfo)
} else if hr := hostinfo.GetRemote(); hr.IsValid() {
p.metricPunchyTx.Inc(1)
p.punchConn.WriteTo([]byte{1}, hr)
return hr, p.sendPunchToAllRemotes(hostinfo, hr)
}
if !hr.IsValid() {
return hr, nil
}
p.metricPunchyTx.Inc(1)
return hr, p.punchConn.WriteTo([]byte{1}, hr)
}
// SendPunchToAll punches every known remote for hostinfo, but only when punchy.target_all_remotes is enabled.
@@ -195,14 +199,19 @@ func (p *Punchy) SendPunchToAll(hostinfo *HostInfo) {
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
return
}
p.sendPunchToAllRemotes(hostinfo)
p.sendPunchToAllRemotes(hostinfo, netip.AddrPort{})
}
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo) {
// sendPunchToAllRemotes punches every known remote for hostinfo and returns the error from punching current
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo, current netip.AddrPort) error {
var currentErr error
hostinfo.remotes.ForEach(p.hm.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
p.metricPunchyTx.Inc(1)
p.punchConn.WriteTo([]byte{1}, addr)
if err := p.punchConn.WriteTo([]byte{1}, addr); addr == current {
currentErr = err
}
})
return currentErr
}
// Start wires the runtime dependencies and spawns the scheduler worker.
+112
View File
@@ -3,11 +3,14 @@ package nebula
import (
"context"
"log/slog"
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -242,3 +245,112 @@ func TestPunchyReloadRestartRequired(t *testing.T) {
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
}
// errNoRoute is the unreachable error every platform classifies the same way
var errNoRoute = udp.ErrInvalidIPv6RemoteForSocket
// recordingConn records where every write went and what kind of message it was, and fails the ones to an address in
// fail
type recordingConn struct {
udp.NoopConn
fail map[netip.AddrPort]error
wrote []netip.AddrPort
// types is the message type of each write, a punch has none
types []header.MessageType
}
func (c *recordingConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.wrote = append(c.wrote, addr)
var h header.H
if h.Parse(b) == nil {
c.types = append(c.types, h.Type)
}
return c.fail[addr]
}
func (c *recordingConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(int, error)) (int, error) {
written := 0
for i := range bufs {
if err := c.WriteTo(bufs[i], addrs[i]); err != nil {
failed(i, err)
continue
}
written++
}
return written, nil
}
type noLighthouses struct{}
func (noLighthouses) IsAnyLighthouseAddr([]netip.Addr) bool { return false }
var (
punchTargetV4 = netip.MustParseAddrPort("10.0.0.3:4242")
punchTargetV6 = netip.MustParseAddrPort("[fd00::3]:4242")
)
// newPunchTarget is a tunnel on its IPv6 remote that also knows the peer's IPv4 one
func newPunchTarget() *HostInfo {
vpnAddr := netip.MustParseAddr("10.128.0.3")
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
remotes: NewRemoteList([]netip.Addr{vpnAddr}, nil),
}
hostinfo.remotes.LearnRemote(vpnAddr, punchTargetV4)
hostinfo.SetRemote(punchTargetV6)
return hostinfo
}
func newPunchyForTest(t *testing.T, settings map[string]any, conn udp.Conn) *Punchy {
c := config.NewC(test.NewLogger())
c.Settings["punchy"] = settings
p := NewPunchyFromConfig(test.NewLogger(), c, conn)
hm := newHostMap(test.NewLogger())
hm.preferredRanges.Store(&[]netip.Prefix{})
p.hm = hm
p.lh = noLighthouses{}
return p
}
func TestPunchy_SendPunchReportsCurrentRemote(t *testing.T) {
t.Run("disabled", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": false}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.False(t, remote.IsValid())
require.NoError(t, err)
assert.Empty(t, conn.wrote)
})
t.Run("current remote", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.ErrorIs(t, err, errNoRoute)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
})
t.Run("all remotes, current fails", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.ErrorIs(t, err, errNoRoute)
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
})
t.Run("all remotes, only another fails", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV4: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.NoError(t, err, "a failure on a remote we are not using says nothing about the current one")
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
})
}
-4
View File
@@ -528,10 +528,6 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
if !rm.GetAmRelay() {
return
}
if !slices.Contains(h.vpnAddrs, from) {
logMsg.Info("Discarding relay request where relayFrom is not the sending peer")
return
}
peer := rm.hostmap.QueryVpnAddr(target)
if peer == nil {
// Try to establish a connection to this host. If we get a future relay request,
-57
View File
@@ -7,10 +7,8 @@ import (
"testing"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestStartRelaysLogDedupe verifies that repeated attempts with the same relay set drop the log
@@ -97,58 +95,3 @@ func TestStartRelaysLogDedupe(t *testing.T) {
assert.NotContains(t, buf.String(), msg, "should not log when we shortcut out")
})
}
// TestHandleCreateRelayRequestRejectsSpoofedFrom verifies that a relay only forwards a CreateRelayRequest
// when relayFrom is the peer that sent it. Otherwise any peer could reset another host's established
// relay back to Requested, or allocate unbounded relay entries on the target's hostinfo.
func TestHandleCreateRelayRequestRejectsSpoofedFrom(t *testing.T) {
l := test.NewLogger()
relayAddr := netip.MustParseAddr("10.128.0.128")
malloryAddr := netip.MustParseAddr("10.128.0.3")
victimAddr := netip.MustParseAddr("10.128.0.1")
targetAddr := netip.MustParseAddr("10.128.0.2")
addrTable := new(bart.Lite)
addrTable.Insert(netip.PrefixFrom(relayAddr, relayAddr.BitLen()))
hm := newHostMap(l)
f := &Interface{myVpnAddrsTable: addrTable, hostMap: hm}
rm := &relayManager{l: l, hostmap: hm}
rm.amRelay.Store(true)
newRelayState := func() RelayState {
return RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}}
}
mallory := &HostInfo{vpnAddrs: []netip.Addr{malloryAddr}, localIndexId: 1, relayState: newRelayState()}
target := &HostInfo{
vpnAddrs: []netip.Addr{targetAddr},
localIndexId: 2,
remotes: NewRemoteList([]netip.Addr{targetAddr}, nil),
relayState: newRelayState(),
}
target.SetRemote(netip.MustParseAddrPort("192.0.2.2:4242"))
hm.unlockedAddHostInfo(mallory, f)
hm.unlockedAddHostInfo(target, f)
// The victim already has a working relay to the target through us.
_, err := AddRelay(l, target, hm, victimAddr, nil, ForwardingType, Established)
require.NoError(t, err)
relaysBefore := len(hm.Relays)
for i := range 10 {
spoofed := netip.AddrFrom4([4]byte{10, 128, 1, byte(i)})
for _, from := range []netip.Addr{victimAddr, spoofed} {
rm.handleCreateRelayRequest(cert.Version2, mallory, f, &NebulaControl{
Type: NebulaControl_CreateRelayRequest,
InitiatorRelayIndex: uint32(100 + i),
RelayFromAddr: netAddrToProtoAddr(from),
RelayToAddr: netAddrToProtoAddr(targetAddr),
})
}
}
r, ok := target.relayState.QueryRelayForByIp(victimAddr)
require.True(t, ok)
assert.Equal(t, Established, r.State, "victim's relay must not be reset by a request from another peer")
assert.Len(t, hm.Relays, relaysBefore, "spoofed requests must not allocate relay entries")
assert.Empty(t, mallory.relayState.CopyAllRelayFor(), "spoofed requests must not create state on the sender")
}
+19 -124
View File
@@ -2,7 +2,6 @@ package nebula
import (
"context"
"errors"
"log/slog"
"net"
"net/netip"
@@ -59,8 +58,6 @@ type cacheV6 struct {
type hostnamePort struct {
name string
port uint16
// addr is set when name is an ip literal, these never need a lookup
addr netip.Addr
}
type hostnamesResults struct {
@@ -68,20 +65,15 @@ type hostnamesResults struct {
network string
lookupTimeout time.Duration
cancelFn func()
resolveNow chan struct{}
l *slog.Logger
ips atomic.Pointer[map[netip.AddrPort]struct{}]
// known is the last known good addresses per hostname, kept so a reload can carry them forward
known atomic.Pointer[map[hostnamePort][]netip.AddrPort]
}
// NewHostnameResults starts resolving hostPorts. Any hostnames also in prev start with its last known good addresses.
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, prev *hostnamesResults, onUpdate func()) (*hostnamesResults, error) {
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, onUpdate func()) (*hostnamesResults, error) {
r := &hostnamesResults{
hostnames: make([]hostnamePort, len(hostPorts)),
network: network,
lookupTimeout: timeout,
resolveNow: make(chan struct{}, 1),
l: l,
}
@@ -89,13 +81,6 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
// DNS lookups for hostnames that aren't hardcoded IP's will happen in a background goroutine.
performBackgroundLookup := false
ips := map[netip.AddrPort]struct{}{}
known := make([][]netip.AddrPort, len(hostPorts))
var prevKnown map[hostnamePort][]netip.AddrPort
if prev != nil && prev.network == network {
if p := prev.known.Load(); p != nil {
prevKnown = *p
}
}
for idx, hostPort := range hostPorts {
rIp, sPort, err := net.SplitHostPort(hostPort)
@@ -113,19 +98,13 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
if err != nil {
// This address is a hostname, not an IP address
performBackgroundLookup = true
known[idx] = prevKnown[r.hostnames[idx]]
for _, a := range known[idx] {
ips[a] = struct{}{}
}
continue
}
// Save the IP address immediately
r.hostnames[idx].addr = addr
ips[netip.AddrPortFrom(addr, uint16(iPort))] = struct{}{}
}
r.ips.Store(&ips)
r.storeKnown(known)
// Time for the DNS lookup goroutine
if performBackgroundLookup {
@@ -135,12 +114,21 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
go func() {
defer ticker.Stop()
for {
r.resolve(newCtx, known)
r.storeKnown(known)
netipAddrs := map[netip.AddrPort]struct{}{}
for _, addrs := range known {
for _, hostPort := range r.hostnames {
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, r.lookupTimeout)
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, r.network, hostPort.name)
timeoutCancel()
if err != nil {
l.Error("DNS resolution failed for static_map host",
"hostname", hostPort.name,
"network", r.network,
"error", err,
)
continue
}
for _, a := range addrs {
netipAddrs[a] = struct{}{}
netipAddrs[netip.AddrPortFrom(a.Unmap(), hostPort.port)] = struct{}{}
}
}
origSet := r.ips.Load()
@@ -171,7 +159,7 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
case <-newCtx.Done():
return
case <-ticker.C:
case <-r.resolveNow:
continue
}
}
}()
@@ -180,84 +168,6 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
return r, nil
}
// resolve looks up every hostname in parallel and folds the results into known
func (hr *hostnamesResults) resolve(ctx context.Context, known [][]netip.AddrPort) {
var wg sync.WaitGroup
for i, hostPort := range hr.hostnames {
if hostPort.addr.IsValid() {
known[i] = []netip.AddrPort{netip.AddrPortFrom(hostPort.addr, hostPort.port)}
continue
}
wg.Go(func() {
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, hr.lookupTimeout)
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, hr.network, hostPort.name)
timeoutCancel()
known[i] = applyLookup(known[i], hostPort.port, addrs, err)
// A canceled ctx means we are reloading or shutting down, not that DNS failed
if err != nil && ctx.Err() == nil {
hr.l.Error("DNS resolution failed for static_map host",
"hostname", hostPort.name,
"network", hr.network,
"error", err,
"usingAddrs", known[i],
)
}
})
}
wg.Wait()
}
// applyLookup returns the addresses to use for a hostname given the previous ones and a lookup result.
// Only a not found answer clears them, a stale lighthouse address costs a handshake but an empty set costs everything.
// darwin also reports not found while offline, that's harmless since nothing is reachable and the next handshake
// triggers a fresh lookup once we're back.
func applyLookup(prev []netip.AddrPort, port uint16, addrs []netip.Addr, err error) []netip.AddrPort {
if err != nil {
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
return nil
}
return prev
}
out := make([]netip.AddrPort, 0, len(addrs))
for _, a := range addrs {
out = append(out, netip.AddrPortFrom(a.Unmap(), port))
}
return out
}
func (hr *hostnamesResults) storeKnown(known [][]netip.AddrPort) {
byName := make(map[hostnamePort][]netip.AddrPort, len(known))
for i, addrs := range known {
byName[hr.hostnames[i]] = addrs
}
hr.known.Store(&byName)
}
// literalAddrs returns the entries that were configured as ip addresses rather than hostnames
func (hr *hostnamesResults) literalAddrs() []netip.AddrPort {
var out []netip.AddrPort
for _, hostPort := range hr.hostnames {
if hostPort.addr.IsValid() {
out = append(out, netip.AddrPortFrom(hostPort.addr, hostPort.port))
}
}
return out
}
// ResolveNow asks the background goroutine to look up the hostnames again without waiting for the next tick
func (hr *hostnamesResults) ResolveNow() {
if hr == nil || hr.resolveNow == nil {
return
}
select {
case hr.resolveNow <- struct{}{}:
default:
}
}
func (hr *hostnamesResults) Cancel() {
if hr != nil && hr.cancelFn != nil {
hr.cancelFn()
@@ -346,13 +256,6 @@ func (r *RemoteList) ResetForOwner(ownerVpnAddr netip.Addr) {
r.shouldRebuild = true
}
// ResolveHostnames kicks a fresh DNS lookup of any static_map hostnames for this host
func (r *RemoteList) ResolveHostnames() {
r.RLock()
defer r.RUnlock()
r.hr.ResolveNow()
}
// ClearHostnameResults cancels the in-flight DNS resolver goroutine (if any) and drops the resolved IP cache.
func (r *RemoteList) ClearHostnameResults() {
r.Lock()
@@ -557,14 +460,10 @@ func (r *RemoteList) unlockedSetV4(ownerVpnIp, vpnIp netip.Addr, to []*V4AddrPor
// Reset the slice
c.reported = c.reported[:0]
// We can't take their array but we can take their pointers.
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
for _, v := range to {
// We can't take their array but we can take their pointers
for _, v := range to[:minInt(len(to), MaxRemotes)] {
if check(vpnIp, v) {
c.reported = append(c.reported, v)
if len(c.reported) == MaxRemotes {
break
}
}
}
}
@@ -609,14 +508,10 @@ func (r *RemoteList) unlockedSetV6(ownerVpnIp, vpnIp netip.Addr, to []*V6AddrPor
// Reset the slice
c.reported = c.reported[:0]
// We can't take their array but we can take their pointers.
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
for _, v := range to {
// We can't take their array but we can take their pointers
for _, v := range to[:minInt(len(to), MaxRemotes)] {
if check(vpnIp, v) {
c.reported = append(c.reported, v)
if len(c.reported) == MaxRemotes {
break
}
}
}
}
-89
View File
@@ -1,15 +1,10 @@
package nebula
import (
"context"
"encoding/binary"
"fmt"
"net"
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -333,55 +328,6 @@ func BenchmarkSortRebuild(b *testing.B) {
})
}
func TestApplyLookup(t *testing.T) {
prev := []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}
// A successful lookup replaces the previous addresses
got := applyLookup(prev, 4242, []netip.Addr{netip.MustParseAddr("::ffff:2.2.2.2")}, nil)
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("2.2.2.2:4242")}, got)
// A not found answer means the record is gone
assert.Empty(t, applyLookup(prev, 4242, nil, &net.DNSError{Err: "no such host", IsNotFound: true}))
// Anything else is not new information, keep what we had
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "i/o timeout", IsTimeout: true}))
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "server misbehaving", IsTemporary: true}))
assert.Equal(t, prev, applyLookup(prev, 4242, nil, context.DeadlineExceeded))
}
func TestNewHostnameResults_CarriesKnown(t *testing.T) {
// A canceled context keeps the background lookup from learning anything, as if DNS were down
ctx, cancel := context.WithCancel(context.Background())
cancel()
known := netip.MustParseAddrPort("1.1.1.1:4242")
prev := &hostnamesResults{network: "ip4"}
prev.known.Store(&map[hostnamePort][]netip.AddrPort{{name: "lh.invalid", port: 4242}: {known}})
hostPorts := []string{"lh.invalid:4242", "10.0.0.1:4242"}
literal := netip.MustParseAddrPort("10.0.0.1:4242")
hr, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, prev, func() {})
require.NoError(t, err)
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr.GetAddrs())
assert.Equal(t, []netip.AddrPort{literal}, hr.literalAddrs())
// And again from the new one, before it has finished a lookup of its own
hr2, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, hr, func() {})
require.NoError(t, err)
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr2.GetAddrs())
// A port change is a different entry
hr3, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, []string{"lh.invalid:4243"}, prev, func() {})
require.NoError(t, err)
assert.Empty(t, hr3.GetAddrs())
// So is a network change
hr4, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip6", time.Second, hostPorts, prev, func() {})
require.NoError(t, err)
assert.Equal(t, []netip.AddrPort{literal}, hr4.GetAddrs())
}
func newIp4AndPortFromString(s string) *V4AddrPort {
a := netip.MustParseAddrPort(s)
v4Addr := a.Addr().As4()
@@ -400,38 +346,3 @@ func newIp6AndPortFromString(s string) *V6AddrPort {
Port: uint32(a.Port()),
}
}
func TestRemoteList_SetFiltersBeforeCap(t *testing.T) {
owner := netip.MustParseAddr("0.0.0.0")
rl := NewRemoteList([]netip.Addr{owner}, nil)
// MaxRemotes denied addrs first, then the only allowed one, then enough allowed to exceed the cap
var v4 []*V4AddrPort
for i := 0; i < MaxRemotes; i++ {
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("172.17.0.%d:4242", i+1)))
}
for i := 0; i < MaxRemotes+1; i++ {
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("70.199.182.%d:4242", i+1)))
}
rl.unlockedSetV4(owner, owner, v4, func(_ netip.Addr, a *V4AddrPort) bool {
return protoV4AddrPortToNetAddrPort(a).Addr().As4()[0] != 172
})
var v6 []*V6AddrPort
for i := 0; i < MaxRemotes; i++ {
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[fd00::%d]:4242", i+1)))
}
for i := 0; i < MaxRemotes+1; i++ {
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[2001:db8::%d]:4242", i+1)))
}
rl.unlockedSetV6(owner, owner, v6, func(_ netip.Addr, a *V6AddrPort) bool {
return protoV6AddrPortToNetAddrPort(a).Addr().As16()[0] != 0xfd
})
rl.Rebuild([]netip.Prefix{})
assert.Len(t, rl.addrs, MaxRemotes*2)
for _, a := range rl.addrs {
assert.NotEqual(t, byte(172), a.Addr().As16()[12], "denied v4 addr %v was kept", a)
assert.NotEqual(t, byte(0xfd), a.Addr().As16()[0], "denied v6 addr %v was kept", a)
}
}
+3 -2
View File
@@ -41,8 +41,9 @@ type Conn interface {
//
// Returns the number of packets successfully written. A destination the kernel rejects costs only
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
// failed, when not nil, is called with the index of every packet a destination cost us and why.
// Not safe for concurrent use on the same Conn.
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
// ReloadConfig applies the listen settings that can change, r is nil at startup
ReloadConfig(c *config.C, r *config.Reload)
SupportsMultipleReaders() bool
@@ -66,7 +67,7 @@ func (NoopConn) SupportsMultipleReaders() bool {
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
return nil
}
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
return len(bufs), nil
}
func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) {
+4 -1
View File
@@ -139,7 +139,7 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
}
}
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
// silently drops the rest of a burst (linux blocks instead). Poll for
@@ -150,6 +150,9 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+4 -1
View File
@@ -44,7 +44,7 @@ func (u *GenericConn) WriteTo(b []byte, addr netip.AddrPort) error {
return err
}
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
written := 0
for i, b := range bufs {
@@ -52,6 +52,9 @@ func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+7 -15
View File
@@ -255,10 +255,6 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
msgs, buffers, names, _ := prepareRawMessages(u.batch, bufSize, cmsgSpace)
for {
// A shut down socket still queues and returns datagrams, a busy reader would never see an empty read
if u.closed.Load() {
return net.ErrClosed
}
if cmsgSpace > 0 {
for i := range msgs {
setMsgControllen(&msgs[i].Hdr, cmsgSpace)
@@ -362,8 +358,8 @@ func sendto(fd int, b []byte, addr netip.AddrPort, isV4 bool) error {
// WriteBatch sends bufs via sendmmsg(2), coalescing same-destination runs into UDP-GSO superpackets when supported.
// See batchWriter in udp_linux_writebatch.go for the mechanics.
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
return u.bw.WriteBatch(bufs, addrs)
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
return u.bw.WriteBatch(bufs, addrs, failed)
}
// writeSockaddr encodes addr into buf (which must be at least SizeofSockaddrInet6 bytes).
@@ -450,16 +446,12 @@ func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
return nil
}
// Shutdown wakes the reader without releasing the fd, Close only once it has exited
func (u *StdConn) Shutdown() error {
u.closed.Store(true)
// shutdown(2) on an unconnected socket returns ENOTCONN but still wakes the reader, so ignore the error
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
return nil
}
func (u *StdConn) Close() error {
_ = u.Shutdown()
u.closed.Store(true)
// Wake the reader parked in recvmmsg. shutdown(2) on an unconnected socket
// returns ENOTCONN but still wakes it, so ignore the error.
// The reader then sees closed and stops touching the fd, making the Close below safe.
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
return unix.Close(u.sysFd)
}
+38 -9
View File
@@ -3,6 +3,7 @@
package udp
import (
"errors"
"fmt"
"log/slog"
"net"
@@ -96,13 +97,22 @@ func TestWriteBatchBadFamilyDeliversOthers(t *testing.T) {
bufs := [][]byte{[]byte("AAA"), []byte("BBB"), []byte("CCC")}
addrs := []netip.AddrPort{good, bad, good}
n, err := sender.WriteBatch(bufs, addrs)
var failed []int
n, err := sender.WriteBatch(bufs, addrs, func(i int, err error) {
if !IsUnreachable(err) {
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
}
failed = append(failed, i)
})
if err != nil {
t.Fatalf("WriteBatch returned error, want nil (bad dest should be isolated): %v", err)
}
if n != 2 {
t.Errorf("WriteBatch wrote %d packets, want 2 of 3 (the bad-family dest is the only casualty)", n)
}
if want := []int{1}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
got := map[string]bool{}
rx.SetReadDeadline(time.Now().Add(2 * time.Second))
@@ -154,7 +164,7 @@ func TestWriteBatchUnreachableDestDeliversOthers(t *testing.T) {
addrs := []netip.AddrPort{good, good, bad, good, good}
// The bad destination is reported, but only after every other packet has been attempted.
if _, err := sender.WriteBatch(bufs, addrs); err == nil {
if _, err := sender.WriteBatch(bufs, addrs, nil); err == nil {
t.Log("WriteBatch returned nil; kernel accepted the reserved address, delivery assertions still apply")
}
@@ -376,7 +386,7 @@ func TestWriteBatchPartialSendRewind(t *testing.T) {
return accept, nil
}
written, err := w.WriteBatch(bufs, addrs)
written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -429,13 +439,22 @@ func TestWriteBatchSkipUnroutableRunAccounting(t *testing.T) {
return accept, nil
}
written, err := w.WriteBatch(bufs, addrs)
var failed []int
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
if !IsUnreachable(err) {
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
}
failed = append(failed, i)
})
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
if written != 4 {
t.Errorf("written = %d, want 4 (the unroutable run is the only casualty)", written)
}
if want := []int{2}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
wantTags := []byte{1, 2, 4, 5}
if len(wire) != len(wantTags) {
t.Fatalf("wire got %d packets, want %d (dup or loss around the skip)", len(wire), len(wantTags))
@@ -486,13 +505,23 @@ func TestWriteBatchMidChunkRejectResumes(t *testing.T) {
}
}
written, err := w.WriteBatch(bufs, addrs)
var failed []int
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
if !errors.Is(err, unix.EPERM) {
t.Errorf("packet %d failed with %v, want EPERM", i, err)
}
failed = append(failed, i)
})
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
if written != 3 {
t.Errorf("written = %d, want 3 (B's rejected run is the only casualty)", written)
}
// Every packet in B's run is reported, not just the entry
if want := []int{2, 3}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
wantTags := []byte{1, 2, 5}
if len(wire) != len(wantTags) {
t.Fatalf("wire got %d packets, want %d (dup or loss around the mid-chunk reject)", len(wire), len(wantTags))
@@ -543,7 +572,7 @@ func TestWriteBatchMidChunkEIODisablesGSOWithoutDup(t *testing.T) {
}
}
written, err := w.WriteBatch(bufs, addrs)
written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -571,7 +600,7 @@ func TestWriteBatchZeroProgress(t *testing.T) {
w.sendFn = func(start, n int) (int, error) { return 0, nil }
bufs := [][]byte{make([]byte, 100)}
addrs := []netip.AddrPort{netip.MustParseAddrPort("127.0.0.1:4242")}
if _, err := w.WriteBatch(bufs, addrs); err == nil {
if _, err := w.WriteBatch(bufs, addrs, nil); err == nil {
t.Fatal("WriteBatch = nil error on zero progress, want error")
}
}
@@ -597,7 +626,7 @@ func TestWriteBatchEIODisablesGSOAndReplays(t *testing.T) {
return n, nil
}
written, err := w.WriteBatch(bufs, addrs)
written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -674,7 +703,7 @@ func TestGSOEngagesOnLoopback(t *testing.T) {
addrs[i] = dst
}
written, err := sc.WriteBatch(bufs, addrs)
written, err := sc.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
+2 -2
View File
@@ -86,7 +86,7 @@ func TestWriteBatchOffloadsDisabledScripted(t *testing.T) {
addrs[i] = dst
}
written, err := w.WriteBatch(bufs, addrs)
written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -163,7 +163,7 @@ func TestOffloadsDisabledOnLoopback(t *testing.T) {
addrs[i] = dst
}
written, err := sc.WriteBatch(bufs, addrs)
written, err := sc.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
+5 -74
View File
@@ -76,7 +76,7 @@ func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
}
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
// reader must tear down cleanly on Shutdown() regardless of what the socket has
// reader must tear down cleanly on Close() regardless of what the socket has
// carried. The three cases the report called out:
//
// no traffic ever -> works (shutdown wakes recvmmsg with n==0)
@@ -162,12 +162,11 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
// empty queue with the socket already having received a packet.
time.Sleep(500 * time.Millisecond)
// Tear down the way Interface.Close does, the fd stays open until the reader is gone
start := time.Now()
if err := sc.Shutdown(); err != nil {
t.Fatalf("Shutdown: %v", err)
if err := sc.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
defer func() { _ = sc.Close() }()
close(stop)
select {
case err := <-loopDone:
@@ -180,75 +179,7 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
case <-time.After(3 * time.Second):
buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true)
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Shutdown\n%s", name, buf[:n])
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
}
close(stop)
<-trafficDone
}
// A shut down socket still returns queued datagrams, so a reader that is behind has to stop on its own within the
// batch it was working on
func TestListenOutShutdown_BusyReader(t *testing.T) {
const batch = 64
udpSettings := Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Multi: true, Batch: batch}
c, err := NewListener(testLogger(), udpSettings)
if err != nil {
t.Fatalf("NewListener: %v", err)
}
sc := c.(*StdConn)
defer sc.Close()
addr, err := sc.LocalAddr()
if err != nil {
t.Fatalf("LocalAddr: %v", err)
}
stop := make(chan struct{})
defer close(stop)
for range 8 {
send, err := net.Dial("udp", addr.String())
if err != nil {
t.Fatalf("dial: %v", err)
}
defer send.Close()
go func() {
b := make([]byte, 1200)
for {
select {
case <-stop:
return
default:
_, _ = send.Write(b)
}
}
}()
}
var afterShutdown atomic.Int64
loopDone := make(chan error, 1)
go func() {
loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
if sc.closed.Load() {
afterShutdown.Add(1)
}
// Slower than the senders so the queue never drains
for start := time.Now(); time.Since(start) < 200*time.Microsecond; {
}
}, func() {})
}()
time.Sleep(200 * time.Millisecond)
if err := sc.Shutdown(); err != nil {
t.Fatalf("Shutdown: %v", err)
}
select {
case err := <-loopDone:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("ListenOut returned %v, want net.ErrClosed", err)
}
case <-time.After(3 * time.Second):
t.Fatalf("ListenOut still running 3s after Shutdown, delivered %d packets since", afterShutdown.Load())
}
if n := afterShutdown.Load(); n > batch {
t.Fatalf("delivered %d packets after Shutdown, more than the one batch in flight", n)
}
}
+14 -2
View File
@@ -190,8 +190,8 @@ func parseRelease(r string) (major, minor int) {
// its packets are dropped and the rest of the chunk resumes in place.
//
// Returns the number of packets sent. An error means the call itself failed.
// A short count means some destinations were undeliverable.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// A short count means some destinations were undeliverable, failed hears about each of their packets.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
if len(bufs) != len(addrs) {
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
}
@@ -233,6 +233,7 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
if w.l.Enabled(context.Background(), slog.LevelDebug) {
w.l.Debug("skipping unroutable batch destination", "udpAddr", addrs[i], "packets", runLen, "error", err)
}
reportFailed(failed, i, i+runLen, err)
i += runLen
continue
}
@@ -300,12 +301,23 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
"gso", w.gsoSupported,
)
}
reportFailed(failed, w.entryEnd[done]-w.entryPkts[done], w.entryEnd[done], serr)
done++
}
}
return written, nil
}
// reportFailed tells failed, if anyone is listening, that every packet in bufs[start:end] was lost to err
func reportFailed(failed func(i int, err error), start, end int, err error) {
if failed == nil {
return
}
for i := start; i < end; i++ {
failed(i, err)
}
}
// planRun returns the length of the run starting at start and its segment
// size (len(bufs[start])). A run of length 1 carries no UDP_SEGMENT cmsg
// and is sent as a plain datagram; without GSO support planRun always returns 1.
+2 -2
View File
@@ -80,11 +80,11 @@ func TestWriteBatchNoAllocs(t *testing.T) {
var werr error
// Warm-up outside the measured runs.
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
t.Fatalf("WriteBatch warm-up: %v", err)
}
allocs := testing.AllocsPerRun(100, func() {
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
werr = err
}
})
+4 -1
View File
@@ -319,7 +319,7 @@ func (u *RIOConn) WriteTo(buf []byte, ip netip.AddrPort) error {
return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
}
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
written := 0
for i, b := range bufs {
@@ -327,6 +327,9 @@ func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+64 -7
View File
@@ -6,6 +6,7 @@ import (
"context"
"io"
"log/slog"
"net"
"net/netip"
"os"
"sync"
@@ -63,10 +64,19 @@ func acquirePacket() *Packet {
}
}
// testerAddrs is where a TesterConn sends from. A single address host sends everything from addr, whatever the
// destination. A dual stack host sends from the address matching the destination's family and has no route at all
// for a family it holds no address in.
type testerAddrs struct {
addr netip.AddrPort
v4, v6 netip.AddrPort
dual bool
}
type TesterConn struct {
// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
// addrs is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
addr atomic.Pointer[netip.AddrPort]
addrs atomic.Pointer[testerAddrs]
RxPackets chan *Packet // Packets to receive into nebula
TxPackets chan *Packet // Packets transmitted outside by nebula
@@ -94,14 +104,53 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
return c, nil
}
// GetAddr returns the underlay address this conn currently sends from.
// GetAddr returns the underlay address this conn currently sends from. For a dual stack conn that is the IPv4
// address, or the IPv6 one when there is no IPv4 service.
func (u *TesterConn) GetAddr() netip.AddrPort {
return *u.addr.Load()
return u.addrs.Load().addr
}
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
func (u *TesterConn) SetAddr(addr netip.AddrPort) {
u.addr.Store(&addr)
u.addrs.Store(&testerAddrs{addr: addr})
}
// SetDualStackAddrs gives this conn one underlay address per family, standing in for a host whose routing table picks
// the source by destination. Pass an invalid AddrPort for a family the network does not offer, sends to that family
// then fail like they would on a host with no route for it.
func (u *TesterConn) SetDualStackAddrs(v4, v6 netip.AddrPort) {
a := &testerAddrs{v4: v4, v6: v6, dual: true}
if v4.IsValid() {
a.addr = v4
} else {
a.addr = v6
}
u.addrs.Store(a)
}
// GetAddrFor returns the underlay address a send to dst leaves from, GetAddr when there is no route to dst
func (u *TesterConn) GetAddrFor(dst netip.AddrPort) netip.AddrPort {
if src, err := u.sourceFor(dst); err == nil {
return src
}
return u.GetAddr()
}
// sourceFor returns the address a send to dst leaves from
func (u *TesterConn) sourceFor(dst netip.AddrPort) (netip.AddrPort, error) {
a := u.addrs.Load()
if !a.dual {
return a.addr, nil
}
src := a.v6
if dst.Addr().Unmap().Is4() {
src = a.v4
}
if !src.IsValid() {
return netip.AddrPort{}, &net.OpError{Op: "write", Net: "udp", Addr: net.UDPAddrFromAddrPort(dst), Err: errNetUnreachable}
}
return src, nil
}
// Send will place a UdpPacket onto the receive queue for nebula to consume
@@ -153,6 +202,11 @@ func (u *TesterConn) Get(block bool) *Packet {
//********************************************************************************************************************//
func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
from, err := u.sourceFor(addr)
if err != nil {
return err
}
p := acquirePacket()
if cap(p.Data) < len(b) {
p.Data = make([]byte, len(b))
@@ -160,7 +214,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
p.Data = p.Data[:len(b)]
}
copy(p.Data, b)
p.From = u.GetAddr()
p.From = from
p.To = addr
select {
case <-u.done:
@@ -170,13 +224,16 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
return nil
}
}
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
written := 0
for i, b := range bufs {
if err := u.WriteTo(b, addrs[i]); err == nil {
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+22
View File
@@ -0,0 +1,22 @@
//go:build !windows
package udp
import (
"errors"
"syscall"
)
// errNetUnreachable is the error this platform's sockets give for a destination with no route
var errNetUnreachable error = syscall.ENETUNREACH
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
// to an IPv6 remote into one of these.
func IsUnreachable(err error) bool {
return errors.Is(err, syscall.ENETUNREACH) ||
errors.Is(err, syscall.EHOSTUNREACH) ||
errors.Is(err, syscall.EADDRNOTAVAIL) ||
errors.Is(err, syscall.EAFNOSUPPORT) ||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
}
+37
View File
@@ -0,0 +1,37 @@
//go:build !windows
package udp
import (
"net"
"syscall"
"testing"
)
func TestIsUnreachable(t *testing.T) {
wrap := func(err error) error { return &net.OpError{Op: "write", Net: "udp", Err: err} }
for _, err := range []error{
wrap(syscall.ENETUNREACH),
wrap(syscall.EHOSTUNREACH),
wrap(syscall.EADDRNOTAVAIL),
wrap(syscall.EAFNOSUPPORT),
ErrInvalidIPv6RemoteForSocket,
} {
if !IsUnreachable(err) {
t.Errorf("IsUnreachable(%v) = false, want true", err)
}
}
for _, err := range []error{
nil,
wrap(syscall.EPERM),
wrap(syscall.ENOBUFS),
wrap(syscall.EMSGSIZE),
net.ErrClosed,
} {
if IsUnreachable(err) {
t.Errorf("IsUnreachable(%v) = true, want false", err)
}
}
}
+21
View File
@@ -0,0 +1,21 @@
package udp
import (
"errors"
"golang.org/x/sys/windows"
)
// errNetUnreachable is the error this platform's sockets give for a destination with no route
var errNetUnreachable error = windows.WSAENETUNREACH
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
// to an IPv6 remote into one of these.
func IsUnreachable(err error) bool {
return errors.Is(err, windows.WSAENETUNREACH) ||
errors.Is(err, windows.WSAEHOSTUNREACH) ||
errors.Is(err, windows.WSAEADDRNOTAVAIL) ||
errors.Is(err, windows.WSAEAFNOSUPPORT) ||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
}