mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 22:46:52 +01:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c543718ee0 | ||
|
|
44e3365d4f | ||
|
|
995f22ca53 | ||
|
|
b69cf4ecb4 | ||
|
|
0f07faab0e | ||
|
|
2fd9966fef | ||
|
|
0ff077fea3 |
No files matched your search
@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- A tunnel whose remote becomes unreachable, such as an IPv6 remote after moving to a network without IPv6, now
|
||||||
|
moves to another of the peer's addresses within a round trip or two instead of waiting 20-35s to be torn down
|
||||||
|
and rehandshaked. When a send fails because there is no route to the remote, the tunnel sends a test packet to
|
||||||
|
every other address it knows for the peer and asks the lighthouse to have the peer punch its new address.
|
||||||
|
Whichever path answers carries the tunnel. With `punchy.punch` enabled, idle tunnels notice on their next
|
||||||
|
keepalive punch. An idle tunnel whose remote is unreachable and none of whose other addresses answer is now
|
||||||
|
torn down by the connection manager rather than left in place. Not yet effective for relayed traffic or on
|
||||||
|
Windows with the default RIO backend.
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
|
|
||||||
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
|
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
|
||||||
|
|||||||
+28
-12
@@ -14,6 +14,7 @@ import (
|
|||||||
"github.com/slackhq/nebula/cert"
|
"github.com/slackhq/nebula/cert"
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
"github.com/slackhq/nebula/header"
|
"github.com/slackhq/nebula/header"
|
||||||
|
"github.com/slackhq/nebula/udp"
|
||||||
)
|
)
|
||||||
|
|
||||||
type trafficDecision int
|
type trafficDecision int
|
||||||
@@ -171,7 +172,7 @@ func (cm *connectionManager) Start(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
|
func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
|
||||||
decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now)
|
decision, hostinfo, primary, punch := cm.makeTrafficDecision(localIndex, now)
|
||||||
|
|
||||||
switch decision {
|
switch decision {
|
||||||
case deleteTunnel:
|
case deleteTunnel:
|
||||||
@@ -196,6 +197,20 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
|
|||||||
}
|
}
|
||||||
|
|
||||||
cm.resetRelayTrafficCheck(hostinfo)
|
cm.resetRelayTrafficCheck(hostinfo)
|
||||||
|
|
||||||
|
if punch != nil {
|
||||||
|
cm.sendPunch(punch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendPunch keeps a tunnel's NAT state alive when we are not sending on it. When we have no route to its remote any
|
||||||
|
// more, typically after moving to a network without that address family, the tunnel probes its other remotes so the
|
||||||
|
// peer roams over to us. Nothing else would notice, a tunnel we are not sending on has nothing else that could fail.
|
||||||
|
func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
|
||||||
|
remote, err := cm.punchy.SendPunch(hostinfo)
|
||||||
|
if udp.IsUnreachable(err) {
|
||||||
|
hostinfo.remoteUnreachable(cm.intf, remote)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
|
func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
|
||||||
@@ -313,7 +328,9 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) {
|
// makeTrafficDecision decides what to do with a tunnel under the hostmap lock. It also returns the tunnel to punch when
|
||||||
|
// we are not sending on it, for the caller to send once the lock is released.
|
||||||
|
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo, *HostInfo) {
|
||||||
// Read lock the main hostmap to order decisions based on tunnels being the primary tunnel
|
// Read lock the main hostmap to order decisions based on tunnels being the primary tunnel
|
||||||
cm.hostMap.RLock()
|
cm.hostMap.RLock()
|
||||||
defer cm.hostMap.RUnlock()
|
defer cm.hostMap.RUnlock()
|
||||||
@@ -321,17 +338,17 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
|||||||
hostinfo := cm.hostMap.Indexes[localIndex]
|
hostinfo := cm.hostMap.Indexes[localIndex]
|
||||||
if hostinfo == nil {
|
if hostinfo == nil {
|
||||||
cm.l.Debug("Not found in hostmap", "localIndex", localIndex)
|
cm.l.Debug("Not found in hostmap", "localIndex", localIndex)
|
||||||
return doNothing, nil, nil
|
return doNothing, nil, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if cm.isInvalidCertificate(now, hostinfo) {
|
if cm.isInvalidCertificate(now, hostinfo) {
|
||||||
return closeTunnel, hostinfo, nil
|
return closeTunnel, hostinfo, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
|
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
|
||||||
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
|
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
|
||||||
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
|
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
|
||||||
return deleteTunnel, hostinfo, nil
|
return deleteTunnel, hostinfo, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
|
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
|
||||||
@@ -368,10 +385,10 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
|||||||
|
|
||||||
if !outTraffic {
|
if !outTraffic {
|
||||||
// Send a punch packet to keep the NAT state alive
|
// Send a punch packet to keep the NAT state alive
|
||||||
cm.punchy.SendPunch(hostinfo)
|
return decision, hostinfo, primary, hostinfo
|
||||||
}
|
}
|
||||||
|
|
||||||
return decision, hostinfo, primary
|
return decision, hostinfo, primary, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if hostinfo.isPendingDeletion() {
|
if hostinfo.isPendingDeletion() {
|
||||||
@@ -380,7 +397,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
|||||||
"tunnelCheck", m{"state": "dead", "method": "active"},
|
"tunnelCheck", m{"state": "dead", "method": "active"},
|
||||||
)
|
)
|
||||||
|
|
||||||
return deleteTunnel, hostinfo, nil
|
return deleteTunnel, hostinfo, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
decision := doNothing
|
decision := doNothing
|
||||||
@@ -394,14 +411,13 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
|||||||
"primary", mainHostInfo,
|
"primary", mainHostInfo,
|
||||||
)
|
)
|
||||||
|
|
||||||
return closeTunnel, hostinfo, primary
|
return closeTunnel, hostinfo, primary, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
|
// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
|
||||||
// Just maintain NAT state if configured to do so.
|
// Just maintain NAT state if configured to do so.
|
||||||
cm.punchy.SendPunch(hostinfo)
|
|
||||||
cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
|
cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
|
||||||
return doNothing, nil, nil
|
return doNothing, nil, nil, hostinfo
|
||||||
}
|
}
|
||||||
|
|
||||||
// We aren't receiving traffic but we are sending it. The outbound
|
// We aren't receiving traffic but we are sending it. The outbound
|
||||||
@@ -427,7 +443,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
|||||||
|
|
||||||
hostinfo.setPendingDeletion(true)
|
hostinfo.setPendingDeletion(true)
|
||||||
cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
|
cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
|
||||||
return decision, hostinfo, nil
|
return decision, hostinfo, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) {
|
func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) {
|
||||||
|
|||||||
+247
-6
@@ -3,13 +3,18 @@ package nebula
|
|||||||
import (
|
import (
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
|
"io"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/flynn/noise"
|
||||||
"github.com/rcrowley/go-metrics"
|
"github.com/rcrowley/go-metrics"
|
||||||
"github.com/slackhq/nebula/cert"
|
"github.com/slackhq/nebula/cert"
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
|
"github.com/slackhq/nebula/header"
|
||||||
|
"github.com/slackhq/nebula/noiseutil"
|
||||||
|
"github.com/slackhq/nebula/overlay/batch"
|
||||||
"github.com/slackhq/nebula/overlay/overlaytest"
|
"github.com/slackhq/nebula/overlay/overlaytest"
|
||||||
"github.com/slackhq/nebula/test"
|
"github.com/slackhq/nebula/test"
|
||||||
"github.com/slackhq/nebula/udp"
|
"github.com/slackhq/nebula/udp"
|
||||||
@@ -266,12 +271,12 @@ func Test_NewConnectionManager_CounterLimits(t *testing.T) {
|
|||||||
|
|
||||||
// Still below the reject limit, the tunnel stays up
|
// Still below the reject limit, the tunnel stays up
|
||||||
nc.In(hostinfo)
|
nc.In(hostinfo)
|
||||||
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
||||||
assert.Equal(t, tryRehandshake, decision)
|
assert.Equal(t, tryRehandshake, decision)
|
||||||
|
|
||||||
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
|
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
|
||||||
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
|
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
|
||||||
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
||||||
assert.Equal(t, deleteTunnel, decision)
|
assert.Equal(t, deleteTunnel, decision)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -333,14 +338,14 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
|
|||||||
assert.True(t, (hostinfo.state.Load()&stateIn != 0))
|
assert.True(t, (hostinfo.state.Load()&stateIn != 0))
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
||||||
assert.Equal(t, tryRehandshake, decision)
|
assert.Equal(t, tryRehandshake, decision)
|
||||||
assert.Equal(t, now, hostinfo.lastUsed)
|
assert.Equal(t, now, hostinfo.lastUsed)
|
||||||
assert.False(t, hostinfo.isPendingDeletion())
|
assert.False(t, hostinfo.isPendingDeletion())
|
||||||
assert.False(t, hostinfo.sentSinceCheck())
|
assert.False(t, hostinfo.sentSinceCheck())
|
||||||
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
|
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
|
||||||
|
|
||||||
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
|
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
|
||||||
assert.Equal(t, doNothing, decision)
|
assert.Equal(t, doNothing, decision)
|
||||||
assert.Equal(t, now, hostinfo.lastUsed)
|
assert.Equal(t, now, hostinfo.lastUsed)
|
||||||
assert.False(t, hostinfo.isPendingDeletion())
|
assert.False(t, hostinfo.isPendingDeletion())
|
||||||
@@ -348,7 +353,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
|
|||||||
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
|
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
|
||||||
|
|
||||||
// Do another traffic check tick, should still not be pending deletion
|
// Do another traffic check tick, should still not be pending deletion
|
||||||
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
|
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
|
||||||
assert.Equal(t, doNothing, decision)
|
assert.Equal(t, doNothing, decision)
|
||||||
assert.Equal(t, now, hostinfo.lastUsed)
|
assert.Equal(t, now, hostinfo.lastUsed)
|
||||||
assert.False(t, hostinfo.isPendingDeletion())
|
assert.False(t, hostinfo.isPendingDeletion())
|
||||||
@@ -358,7 +363,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
|
|||||||
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
|
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
|
||||||
|
|
||||||
// Finally advance beyond the inactivity timeout
|
// Finally advance beyond the inactivity timeout
|
||||||
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
|
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
|
||||||
assert.Equal(t, closeTunnel, decision)
|
assert.Equal(t, closeTunnel, decision)
|
||||||
assert.Equal(t, now, hostinfo.lastUsed)
|
assert.Equal(t, now, hostinfo.lastUsed)
|
||||||
assert.False(t, hostinfo.isPendingDeletion())
|
assert.False(t, hostinfo.isPendingDeletion())
|
||||||
@@ -578,3 +583,239 @@ func (d *dummyCert) MarshalJSON() ([]byte, error) {
|
|||||||
func (d *dummyCert) Copy() cert.Certificate {
|
func (d *dummyCert) Copy() cert.Certificate {
|
||||||
return d
|
return d
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// newPunchTestConnectionManager is a connection manager punching through conn, holding the tunnel from newPunchTarget
|
||||||
|
func newPunchTestConnectionManager(t *testing.T, conn udp.Conn) (*connectionManager, *HostInfo) {
|
||||||
|
l := test.NewLogger()
|
||||||
|
hostMap := newHostMap(l)
|
||||||
|
hostMap.preferredRanges.Store(&[]netip.Prefix{})
|
||||||
|
|
||||||
|
lh := newTestLighthouse()
|
||||||
|
ifce := &Interface{
|
||||||
|
hostMap: hostMap,
|
||||||
|
inside: &overlaytest.NoopTun{},
|
||||||
|
outside: &udp.NoopConn{},
|
||||||
|
firewall: &Firewall{},
|
||||||
|
lightHouse: lh,
|
||||||
|
pki: &PKI{},
|
||||||
|
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
|
||||||
|
l: l,
|
||||||
|
}
|
||||||
|
ifce.pki.cs.Store(&CertState{
|
||||||
|
initiatingVersion: cert.Version1,
|
||||||
|
privateKey: []byte{},
|
||||||
|
v1Cert: &dummyCert{version: cert.Version1},
|
||||||
|
})
|
||||||
|
|
||||||
|
punchy := newPunchyForTest(t, map[string]any{"punch": true}, conn)
|
||||||
|
nc := newConnectionManagerFromConfig(l, config.NewC(l), hostMap, punchy)
|
||||||
|
nc.intf = ifce
|
||||||
|
ifce.connectionManager = nc
|
||||||
|
// Tunnel sends go through conn too, though without a key from withTestKey they never get as far as writing
|
||||||
|
ifce.writers = []udp.Conn{conn}
|
||||||
|
ifce.metricTxDropped = metrics.NilCounter{}
|
||||||
|
|
||||||
|
hostinfo := newPunchTarget()
|
||||||
|
hostinfo.localIndexId = 1099
|
||||||
|
hostinfo.remoteIndexId = 9901
|
||||||
|
hostinfo.ConnectionState = &ConnectionState{myCert: &dummyCert{version: cert.Version1}}
|
||||||
|
nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
|
||||||
|
return nc, hostinfo
|
||||||
|
}
|
||||||
|
|
||||||
|
// A tunnel we are not sending on gets punched whether or not it is receiving, the punch is the only thing that
|
||||||
|
// would notice its remote has become unreachable.
|
||||||
|
func Test_makeTrafficDecision_punchesWhenNotSending(t *testing.T) {
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
// Receiving but not sending
|
||||||
|
hostinfo.takeTraffic()
|
||||||
|
nc.In(hostinfo)
|
||||||
|
decision, _, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
||||||
|
assert.Equal(t, tryRehandshake, decision)
|
||||||
|
assert.Same(t, hostinfo, punch)
|
||||||
|
|
||||||
|
// Sending and receiving, our own traffic keeps the NAT state alive
|
||||||
|
nc.Out(hostinfo)
|
||||||
|
nc.In(hostinfo)
|
||||||
|
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
||||||
|
assert.Equal(t, tryRehandshake, decision)
|
||||||
|
assert.Nil(t, punch)
|
||||||
|
|
||||||
|
// Unused
|
||||||
|
decision, decided, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
||||||
|
assert.Equal(t, doNothing, decision)
|
||||||
|
assert.Nil(t, decided, "an unused tunnel has no relay usage to reset")
|
||||||
|
assert.Same(t, hostinfo, punch)
|
||||||
|
|
||||||
|
// Sending but not receiving gets a test packet instead, which finds an unreachable remote on its own
|
||||||
|
nc.Out(hostinfo)
|
||||||
|
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
||||||
|
assert.Equal(t, sendTestPacket, decision)
|
||||||
|
assert.Nil(t, punch)
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_sendPunch_probesWhenRemoteUnreachable(t *testing.T) {
|
||||||
|
t.Run("unreachable", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
||||||
|
|
||||||
|
nc.sendPunch(hostinfo)
|
||||||
|
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
|
||||||
|
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
|
||||||
|
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
|
||||||
|
"the probe should ask the lighthouse to have the peer punch us")
|
||||||
|
|
||||||
|
// Within the probe interval nothing more goes out, however many sends fail
|
||||||
|
nc.sendPunch(hostinfo)
|
||||||
|
nc.sendPunch(hostinfo)
|
||||||
|
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("delivered", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{}
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
||||||
|
|
||||||
|
nc.sendPunch(hostinfo)
|
||||||
|
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
|
||||||
|
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
|
||||||
|
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("other failure", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: io.ErrShortWrite}}
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
||||||
|
|
||||||
|
nc.sendPunch(hostinfo)
|
||||||
|
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "only a missing route says to look elsewhere")
|
||||||
|
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// A packet the data path had no route for names its tunnel in its header, the peer's index for it
|
||||||
|
func Test_sendUnreachable_findsTunnelFromHeader(t *testing.T) {
|
||||||
|
head := func(st header.MessageSubType, remoteIndex uint32) []byte {
|
||||||
|
return header.Encode(make([]byte, header.Len), header.Version, header.Message, st, remoteIndex, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("direct", func(t *testing.T) {
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
|
||||||
|
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV6)
|
||||||
|
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
|
||||||
|
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("not its remote", func(t *testing.T) {
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
|
||||||
|
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV4)
|
||||||
|
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "a failure on a remote the tunnel is not using is stale")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("relayed", func(t *testing.T) {
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
|
||||||
|
// A relay header's index is the relay session's, it must not be taken for a tunnel's even when they collide
|
||||||
|
nc.intf.sendUnreachable(head(header.MessageRelay, hostinfo.remoteIndexId), punchTargetV6)
|
||||||
|
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
|
||||||
|
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unknown tunnel", func(t *testing.T) {
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
|
||||||
|
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId+1), punchTargetV6)
|
||||||
|
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// withTestKey gives a tunnel a real key so its sends get encrypted and written instead of returning early
|
||||||
|
func withTestKey(hostinfo *HostInfo) {
|
||||||
|
cs := noise.NewCipherSuite(noise.DH25519, noise.CipherAESGCM, noise.HashSHA256)
|
||||||
|
hostinfo.ConnectionState.eKey = noiseutil.NewCipherState(noise.UnsafeNewCipherState(cs, [32]byte{}, 0), noise.CipherAESGCM)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The probe itself, end to end from a failed send: a test packet to every remote but the one we cannot reach, and a
|
||||||
|
// lighthouse query, through both places a send can fail.
|
||||||
|
func Test_remoteUnreachable_probesOtherRemotes(t *testing.T) {
|
||||||
|
t.Run("control send", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
||||||
|
withTestKey(hostinfo)
|
||||||
|
|
||||||
|
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
|
||||||
|
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV4}, conn.wrote, "the failed send, then the probe")
|
||||||
|
assert.Equal(t, []header.MessageType{header.Test, header.Test}, conn.types)
|
||||||
|
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("data path batch", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
||||||
|
withTestKey(hostinfo)
|
||||||
|
|
||||||
|
// Two of the tunnel's packets fail in one batch, they probe once
|
||||||
|
sb := batch.NewSendBatch(conn, 4, 4096)
|
||||||
|
for c := uint64(1); c <= 2; c++ {
|
||||||
|
pkt := header.Encode(sb.Reserve(64), header.Version, header.Message, header.MessageNone, hostinfo.remoteIndexId, c)
|
||||||
|
sb.Commit(append(pkt, make([]byte, 32)...), punchTargetV6)
|
||||||
|
}
|
||||||
|
nc.intf.flushSendBatch(sb, 0)
|
||||||
|
|
||||||
|
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6, punchTargetV4}, conn.wrote,
|
||||||
|
"the failed packets, then one probe")
|
||||||
|
assert.Equal(t, header.Test, conn.types[2])
|
||||||
|
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("closing", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
||||||
|
withTestKey(hostinfo)
|
||||||
|
|
||||||
|
nc.intf.SendMessageToHostInfo(header.CloseTunnel, 0, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
|
||||||
|
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a tunnel being torn down does not probe")
|
||||||
|
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("lighthouse", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
||||||
|
withTestKey(hostinfo)
|
||||||
|
nc.intf.lightHouse.amLighthouse = true
|
||||||
|
|
||||||
|
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
|
||||||
|
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a lighthouse does not roam its clients")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("nothing else to probe", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
||||||
|
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
||||||
|
withTestKey(hostinfo)
|
||||||
|
nc.intf.reQueryWait.Store(int64(time.Minute))
|
||||||
|
// Forget the IPv4 remote, all the tunnel knows is the one it cannot reach
|
||||||
|
hostinfo.remotes = NewRemoteList(hostinfo.vpnAddrs, nil)
|
||||||
|
hostinfo.remotes.LearnRemote(hostinfo.vpnAddrs[0], punchTargetV6)
|
||||||
|
|
||||||
|
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
|
||||||
|
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
|
||||||
|
"the lighthouse is the only way to learn of another remote")
|
||||||
|
|
||||||
|
// Past the probe interval, but the lighthouse is only asked again on the requery schedule
|
||||||
|
hostinfo.nextUnreachableProbe.Store(0)
|
||||||
|
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
|
||||||
|
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
||||||
|
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6}, conn.wrote)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// drainQueries returns every lighthouse query waiting to go out
|
||||||
|
func drainQueries(lh *LightHouse) []netip.Addr {
|
||||||
|
var queries []netip.Addr
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case addr := <-lh.queryChan:
|
||||||
|
queries = append(queries, addr)
|
||||||
|
default:
|
||||||
|
return queries
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -148,7 +148,7 @@ func (c *fakeConn) Rebind() error { c.rebinds++; ret
|
|||||||
func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil }
|
func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil }
|
||||||
func (c *fakeConn) ListenOut(_ udp.EncReader, _ func()) error { return nil }
|
func (c *fakeConn) ListenOut(_ udp.EncReader, _ func()) error { return nil }
|
||||||
func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
|
func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
|
||||||
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
|
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
|
||||||
return len(bufs), nil
|
return len(bufs), nil
|
||||||
}
|
}
|
||||||
func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {}
|
func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {}
|
||||||
|
|||||||
@@ -117,6 +117,18 @@ func (c *Control) SetUDPAddr(addr netip.AddrPort) {
|
|||||||
c.f.outside.(*udp.TesterConn).SetAddr(addr)
|
c.f.outside.(*udp.TesterConn).SetAddr(addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetUDPAddrFor returns the underlay address this node sends to dst from, which differs by family for a dual stack node
|
||||||
|
func (c *Control) GetUDPAddrFor(dst netip.AddrPort) netip.AddrPort {
|
||||||
|
return c.f.outside.(*udp.TesterConn).GetAddrFor(dst)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetDualStackUDPAddrs gives this node one underlay address per family, sending from whichever matches the
|
||||||
|
// destination. An invalid AddrPort leaves the node without that family, sends to it fail with ENETUNREACH. Register
|
||||||
|
// any address the node did not start on with the router as well or nothing will route back.
|
||||||
|
func (c *Control) SetDualStackUDPAddrs(v4, v6 netip.AddrPort) {
|
||||||
|
c.f.outside.(*udp.TesterConn).SetDualStackAddrs(v4, v6)
|
||||||
|
}
|
||||||
|
|
||||||
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
|
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
|
||||||
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
|
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
|
||||||
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
|
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
|
||||||
|
|||||||
@@ -0,0 +1,305 @@
|
|||||||
|
//go:build e2e_testing
|
||||||
|
|
||||||
|
package e2e
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/netip"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/slackhq/nebula"
|
||||||
|
"github.com/slackhq/nebula/cert"
|
||||||
|
"github.com/slackhq/nebula/cert_test"
|
||||||
|
"github.com/slackhq/nebula/e2e/router"
|
||||||
|
"github.com/slackhq/nebula/udp"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestDualStackRoamToV4Only measures how long a laptop's tunnels take to come back after it moves from a dual stack
|
||||||
|
// network, where its tunnels formed over IPv6, to one with no IPv6 service at all: the office to hotel wifi. Every
|
||||||
|
// peer still has an IPv4 address the laptop can reach, so nothing about the move should strand a tunnel.
|
||||||
|
//
|
||||||
|
// It is a measurement, not a pass/fail assertion, and it runs on the production connection manager timers.
|
||||||
|
//
|
||||||
|
// go test -tags e2e_testing -v -run TestDualStackRoamToV4Only ./e2e/
|
||||||
|
func TestDualStackRoamToV4Only(t *testing.T) {
|
||||||
|
for _, tc := range []dualStackRoamCase{
|
||||||
|
{name: "both send", meSends: true, themSends: true},
|
||||||
|
{name: "both send, rebind", meSends: true, themSends: true, rebind: true},
|
||||||
|
{name: "both send, two tunnels", meSends: true, themSends: true, simultaneous: true},
|
||||||
|
{name: "laptop sends", meSends: true},
|
||||||
|
{name: "peer sends", themSends: true},
|
||||||
|
{name: "peer sends, rebind", themSends: true, rebind: true},
|
||||||
|
{name: "peer sends, lh over v4", themSends: true, lhOverV4: true},
|
||||||
|
{name: "peer sends, punchy", themSends: true, punchy: true},
|
||||||
|
{name: "peer sends, punchy all", themSends: true, punchy: true, punchAll: true},
|
||||||
|
{name: "laptop sends, peer NAT", meSends: true, peerNAT: true},
|
||||||
|
{name: "laptop sends, peer NAT, lh v4", meSends: true, peerNAT: true, lhOverV4: true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
fwd, rev := measureDualStackRoam(t, tc)
|
||||||
|
t.Logf("RESULT %-24s laptop->peer %-9v peer->laptop %-9v", tc.name, fmtRecovery(fwd), fmtRecovery(rev))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDualStackRoamRecovers holds the line TestDualStackRoamToV4Only measures: a tunnel whose remote became unreachable
|
||||||
|
// moves to the peer's other address instead of waiting out the connection manager to rehandshake, which takes 20s.
|
||||||
|
func TestDualStackRoamRecovers(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
dualStackRoamCase
|
||||||
|
// within bounds recovery in each direction that has traffic
|
||||||
|
within time.Duration
|
||||||
|
}{
|
||||||
|
// Measures 0.4s. Under the 5s connection manager check, so it fails if the data path stops probing.
|
||||||
|
{dualStackRoamCase{name: "both send", meSends: true, themSends: true}, 2 * time.Second},
|
||||||
|
// Measures 2.6s or 3.8s, depending on whether a probe beats the peer's punch. Half the 20s it used to take.
|
||||||
|
{dualStackRoamCase{name: "laptop sends, peer NAT", meSends: true, peerNAT: true}, 10 * time.Second},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
fwd, rev := measureDualStackRoam(t, tc.dualStackRoamCase)
|
||||||
|
if fwd == 0 || fwd > tc.within {
|
||||||
|
t.Errorf("laptop->peer recovered in %v, want within %v", fmtRecovery(fwd), tc.within)
|
||||||
|
}
|
||||||
|
if tc.themSends && (rev == 0 || rev > tc.within) {
|
||||||
|
t.Errorf("peer->laptop recovered in %v, want within %v", fmtRecovery(rev), tc.within)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type dualStackRoamCase struct {
|
||||||
|
name string
|
||||||
|
// meSends and themSends say who keeps traffic flowing after the move
|
||||||
|
meSends, themSends bool
|
||||||
|
// rebind fires RebindUDPServer on the move, as the darwin network change monitor would
|
||||||
|
rebind bool
|
||||||
|
// lhOverV4 forms the lighthouse tunnel over IPv4 at the office, so the move does not break it
|
||||||
|
lhOverV4 bool
|
||||||
|
// simultaneous has both sides initiate at once at the office, which can leave each side a different primary
|
||||||
|
// tunnel for the other
|
||||||
|
simultaneous bool
|
||||||
|
// punchy turns on punchy.punch everywhere, punchAll adds punchy.target_all_remotes
|
||||||
|
punchy, punchAll bool
|
||||||
|
// peerNAT puts the peer behind a NAT that, after the move, only lets in traffic from addresses the peer has sent to
|
||||||
|
peerNAT bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func fmtRecovery(d time.Duration) string {
|
||||||
|
if d == 0 {
|
||||||
|
return "never"
|
||||||
|
}
|
||||||
|
return d.Round(time.Millisecond).String()
|
||||||
|
}
|
||||||
|
|
||||||
|
type dualStackNode struct {
|
||||||
|
c *nebula.Control
|
||||||
|
vpnAddr netip.Addr
|
||||||
|
v4, v6 netip.AddrPort
|
||||||
|
adv atomic.Pointer[[]netip.Addr]
|
||||||
|
}
|
||||||
|
|
||||||
|
// move puts the node on a new network, advertising only what it can be reached on there
|
||||||
|
func (n *dualStackNode) move(v4, v6 netip.AddrPort) {
|
||||||
|
n.v4, n.v6 = v4, v6
|
||||||
|
n.c.SetDualStackUDPAddrs(v4, v6)
|
||||||
|
var adv []netip.Addr
|
||||||
|
for _, a := range []netip.AddrPort{v4, v6} {
|
||||||
|
if a.IsValid() {
|
||||||
|
adv = append(adv, a.Addr())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
n.adv.Store(&adv)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newDualStackNode(ca cert.Certificate, caKey []byte, name, vpnNet string, v4, v6 netip.AddrPort, overrides m) *dualStackNode {
|
||||||
|
c, vpnNets, _, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, name, vpnNet, v4, overrides)
|
||||||
|
n := &dualStackNode{c: c, vpnAddr: vpnNets[0].Addr()}
|
||||||
|
n.move(v4, v6)
|
||||||
|
c.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
|
||||||
|
return *n.adv.Load()
|
||||||
|
})
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// measureDualStackRoam returns how long after the move the first packet made it laptop->peer and peer->laptop, zero
|
||||||
|
// for a direction that never recovered.
|
||||||
|
func measureDualStackRoam(t *testing.T, tc dualStackRoamCase) (time.Duration, time.Duration) {
|
||||||
|
t.Helper()
|
||||||
|
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||||
|
|
||||||
|
ap := netip.MustParseAddrPort
|
||||||
|
lhV4, lhV6 := ap("10.0.0.1:4242"), ap("[fd00::1]:4242")
|
||||||
|
lh := newDualStackNode(ca, caKey, "lh", "10.128.0.1/24", lhV4, lhV6, m{
|
||||||
|
"lighthouse": m{"am_lighthouse": true},
|
||||||
|
})
|
||||||
|
|
||||||
|
// Production timers rather than the 2s the harness defaults to, this is about how long a real laptop waits
|
||||||
|
peerCfg := m{
|
||||||
|
"lighthouse": m{
|
||||||
|
"hosts": []any{lh.vpnAddr.String()},
|
||||||
|
"interval": 10,
|
||||||
|
},
|
||||||
|
"static_host_map": m{
|
||||||
|
lh.vpnAddr.String(): []any{lhV4.String(), lhV6.String()},
|
||||||
|
},
|
||||||
|
"timers": m{
|
||||||
|
"connection_alive_interval": 5,
|
||||||
|
"pending_deletion_interval": 10,
|
||||||
|
},
|
||||||
|
"punchy": m{
|
||||||
|
"punch": tc.punchy,
|
||||||
|
"target_all_remotes": tc.punchAll,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
me := newDualStackNode(ca, caKey, "me", "10.128.0.2/24", ap("10.0.0.2:4242"), ap("[fd00::2]:4242"), peerCfg)
|
||||||
|
them := newDualStackNode(ca, caKey, "them", "10.128.0.3/24", ap("10.0.0.3:4242"), ap("[fd00::3]:4242"), peerCfg)
|
||||||
|
|
||||||
|
r := router.NewR(t, lh.c, me.c, them.c)
|
||||||
|
r.CancelFlowLogs()
|
||||||
|
for _, n := range []*dualStackNode{lh, me, them} {
|
||||||
|
r.AddRoute(n.v6.Addr(), n.v6.Port(), n.c)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
lh.c.Stop()
|
||||||
|
me.c.Stop()
|
||||||
|
them.c.Stop()
|
||||||
|
}()
|
||||||
|
|
||||||
|
lh.c.Start()
|
||||||
|
me.c.Start()
|
||||||
|
them.c.Start()
|
||||||
|
|
||||||
|
// At the office IPv6 wins every race, so everything forms over it. The lighthouse tunnel optionally does not,
|
||||||
|
// for a static_host_map that only lists the lighthouse's IPv4 address.
|
||||||
|
officeV6 := func(p *udp.Packet, _ *nebula.Control) router.ExitType {
|
||||||
|
lhTraffic := p.To == lhV4 || p.To == lhV6 || p.From == lhV4 || p.From == lhV6
|
||||||
|
if p.To.Addr().Is4() == (tc.lhOverV4 && lhTraffic) {
|
||||||
|
return router.KeepRouting
|
||||||
|
}
|
||||||
|
return router.Drop
|
||||||
|
}
|
||||||
|
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
|
||||||
|
|
||||||
|
// One side at a time, so both sides hold the same single tunnel, unless the case wants them racing
|
||||||
|
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("establish")))
|
||||||
|
if !tc.simultaneous {
|
||||||
|
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
|
||||||
|
}
|
||||||
|
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("establish")))
|
||||||
|
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
|
||||||
|
if tc.simultaneous && me.c.GetHostmapIndexCount() < 3 {
|
||||||
|
// A tunnel to the lighthouse and two to the peer, or the race did not happen
|
||||||
|
t.Logf("simultaneous handshakes settled on a single tunnel this run")
|
||||||
|
}
|
||||||
|
drainTun(me.c)
|
||||||
|
drainTun(them.c)
|
||||||
|
|
||||||
|
for _, want := range []struct {
|
||||||
|
name string
|
||||||
|
c *nebula.Control
|
||||||
|
vpnAddr netip.Addr
|
||||||
|
v6 bool
|
||||||
|
}{
|
||||||
|
{"laptop->peer", me.c, them.vpnAddr, true},
|
||||||
|
{"peer->laptop", them.c, me.vpnAddr, true},
|
||||||
|
{"laptop->lh", me.c, lh.vpnAddr, !tc.lhOverV4},
|
||||||
|
} {
|
||||||
|
hi := want.c.GetHostInfoByVpnAddr(want.vpnAddr, false)
|
||||||
|
if hi == nil {
|
||||||
|
t.Fatalf("%s: no tunnel at the office", want.name)
|
||||||
|
}
|
||||||
|
if hi.CurrentRemote.Addr().Is6() != want.v6 {
|
||||||
|
t.Fatalf("%s: tunnel formed over %v, the scenario wants v6=%v", want.name, hi.CurrentRemote, want.v6)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hotel wifi: a new NAT'd IPv4 address and no IPv6 at all. The office addresses are gone, nothing reaches them.
|
||||||
|
officeV4, officeV6Addr := me.v4, me.v6
|
||||||
|
hotel := ap("10.0.0.99:4242")
|
||||||
|
me.move(hotel, netip.AddrPort{})
|
||||||
|
r.AddRoute(hotel.Addr(), hotel.Port(), me.c)
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
if tc.rebind {
|
||||||
|
me.c.RebindUDPServer()
|
||||||
|
}
|
||||||
|
|
||||||
|
var fwd, rev atomic.Int64
|
||||||
|
stop := make(chan struct{})
|
||||||
|
defer close(stop)
|
||||||
|
watchTun(them.c, stop, start, &fwd)
|
||||||
|
watchTun(me.c, stop, start, &rev)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
tick := time.NewTicker(time.Millisecond * 200)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
if tc.meSends {
|
||||||
|
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("retry")))
|
||||||
|
}
|
||||||
|
if tc.themSends {
|
||||||
|
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("retry")))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// When only one side has traffic, the other direction recovering is a bonus we still report but do not wait for
|
||||||
|
done := func() bool {
|
||||||
|
return (!tc.meSends || fwd.Load() != 0) && (!tc.themSends || rev.Load() != 0)
|
||||||
|
}
|
||||||
|
// The peer's NAT has state for the lighthouse, which its own updates keep fresh, and for wherever it sends from here
|
||||||
|
natOpen := map[netip.AddrPort]bool{lhV4: true, lhV6: true}
|
||||||
|
isPeer := func(a netip.AddrPort) bool { return a == them.v4 || a == them.v6 }
|
||||||
|
|
||||||
|
r.RouteForAllExitFuncOrTimeout(time.Second*90, func(p *udp.Packet, _ *nebula.Control) router.ExitType {
|
||||||
|
if p.To == officeV4 || p.To == officeV6Addr {
|
||||||
|
return router.Drop
|
||||||
|
}
|
||||||
|
if tc.peerNAT {
|
||||||
|
if isPeer(p.From) {
|
||||||
|
natOpen[p.To] = true
|
||||||
|
} else if isPeer(p.To) && !natOpen[p.From] {
|
||||||
|
return router.Drop
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if done() {
|
||||||
|
return router.ExitNow
|
||||||
|
}
|
||||||
|
return router.KeepRouting
|
||||||
|
})
|
||||||
|
|
||||||
|
return time.Duration(fwd.Load()), time.Duration(rev.Load())
|
||||||
|
}
|
||||||
|
|
||||||
|
func drainTun(c *nebula.Control) {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-c.GetTunTxChan():
|
||||||
|
default:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchTun records how long after start the first packet came out of c's tun, and keeps the tun drained after
|
||||||
|
func watchTun(c *nebula.Control, stop chan struct{}, start time.Time, first *atomic.Int64) {
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
case <-c.GetTunTxChan():
|
||||||
|
first.CompareAndSwap(0, int64(time.Since(start)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
+14
-13
@@ -497,11 +497,10 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []
|
|||||||
// Nope, lets push the sender along
|
// Nope, lets push the sender along
|
||||||
case p := <-udpTx:
|
case p := <-udpTx:
|
||||||
r.Lock()
|
r.Lock()
|
||||||
a := sender.GetUDPAddr()
|
c := r.getControl(p)
|
||||||
c := r.getControl(a, p.To, p)
|
|
||||||
if c == nil {
|
if c == nil {
|
||||||
r.Unlock()
|
r.Unlock()
|
||||||
panic("No control for udp tx " + a.String())
|
panic("No control for udp tx " + p.To.String())
|
||||||
}
|
}
|
||||||
fp := r.unlockedInjectFlow(sender, c, p, false)
|
fp := r.unlockedInjectFlow(sender, c, p, false)
|
||||||
c.InjectUDPPacket(p) // copies internally; original is ours to release
|
c.InjectUDPPacket(p) // copies internally; original is ours to release
|
||||||
@@ -622,8 +621,7 @@ func (r *R) stopFanInWorkers() {
|
|||||||
func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
|
func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
|
||||||
r.Lock()
|
r.Lock()
|
||||||
defer r.Unlock()
|
defer r.Unlock()
|
||||||
a := from.GetUDPAddr()
|
c := r.getControl(p)
|
||||||
c := r.getControl(a, p.To, p)
|
|
||||||
if c == nil {
|
if c == nil {
|
||||||
panic(fmt.Sprintf("No control for udp tx %s", p.To))
|
panic(fmt.Sprintf("No control for udp tx %s", p.To))
|
||||||
}
|
}
|
||||||
@@ -671,7 +669,7 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
receiver := r.getControl(sender.GetUDPAddr(), p.To, p)
|
receiver := r.getControl(p)
|
||||||
if receiver == nil {
|
if receiver == nil {
|
||||||
r.Unlock()
|
r.Unlock()
|
||||||
panic("Can't RouteExitFunc for host: " + p.To.String())
|
panic("Can't RouteExitFunc for host: " + p.To.String())
|
||||||
@@ -767,7 +765,7 @@ func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc)
|
|||||||
|
|
||||||
r.Lock()
|
r.Lock()
|
||||||
p := rx.Interface().(*udp.Packet)
|
p := rx.Interface().(*udp.Packet)
|
||||||
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
|
receiver := r.getControl(p)
|
||||||
if receiver == nil {
|
if receiver == nil {
|
||||||
r.Unlock()
|
r.Unlock()
|
||||||
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
|
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
|
||||||
@@ -876,7 +874,7 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
|
|||||||
r.Lock()
|
r.Lock()
|
||||||
|
|
||||||
p := rx.Interface().(*udp.Packet)
|
p := rx.Interface().(*udp.Packet)
|
||||||
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
|
receiver := r.getControl(p)
|
||||||
if receiver == nil {
|
if receiver == nil {
|
||||||
r.Unlock()
|
r.Unlock()
|
||||||
panic("Can't RouteForAllExitFunc for host: " + p.To.String())
|
panic("Can't RouteForAllExitFunc for host: " + p.To.String())
|
||||||
@@ -939,7 +937,7 @@ func (r *R) FlushAll() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
for {
|
for {
|
||||||
x, rx, ok := reflect.Select(sc)
|
_, rx, ok := reflect.Select(sc)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -947,7 +945,7 @@ func (r *R) FlushAll() {
|
|||||||
|
|
||||||
p := rx.Interface().(*udp.Packet)
|
p := rx.Interface().(*udp.Packet)
|
||||||
|
|
||||||
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
|
receiver := r.getControl(p)
|
||||||
if receiver == nil {
|
if receiver == nil {
|
||||||
r.Unlock()
|
r.Unlock()
|
||||||
panic("Can't FlushAll for host: " + p.To.String())
|
panic("Can't FlushAll for host: " + p.To.String())
|
||||||
@@ -958,16 +956,19 @@ func (r *R) FlushAll() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change
|
// getControl performs or seeds NAT translation and returns the control for p.To, p from fields may change
|
||||||
// This is an internal router function, the caller must hold the lock
|
// This is an internal router function, the caller must hold the lock
|
||||||
func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control {
|
func (r *R) getControl(p *udp.Packet) *nebula.Control {
|
||||||
|
// Key on the address the packet actually left from, a dual stack sender has one per family
|
||||||
|
fromAddr, toAddr := p.From, p.To
|
||||||
if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
|
if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
|
||||||
p.From = newAddr
|
p.From = newAddr
|
||||||
}
|
}
|
||||||
|
|
||||||
c, ok := r.inNat[toAddr]
|
c, ok := r.inNat[toAddr]
|
||||||
if ok {
|
if ok {
|
||||||
r.outNat[outNatKey{from: c.GetUDPAddr(), to: fromAddr}] = toAddr
|
// Key the return path on the address the receiver will answer from, which is per family for a dual stack node
|
||||||
|
r.outNat[outNatKey{from: c.GetUDPAddrFor(fromAddr), to: fromAddr}] = toAddr
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+62
@@ -264,6 +264,9 @@ type HostInfo struct {
|
|||||||
// This is used to limit lighthouse re-queries in chatty clients
|
// This is used to limit lighthouse re-queries in chatty clients
|
||||||
nextLHQuery atomic.Int64
|
nextLHQuery atomic.Int64
|
||||||
|
|
||||||
|
// nextUnreachableProbe is the earliest we will probe our other remotes again after failing to reach the current one
|
||||||
|
nextUnreachableProbe atomic.Int64
|
||||||
|
|
||||||
// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
|
// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
|
||||||
// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
|
// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
|
||||||
// This is used to avoid an attack where a handshake packet is replayed after some time
|
// This is used to avoid an attack where a handshake packet is replayed after some time
|
||||||
@@ -766,6 +769,65 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unreachableProbeInterval bounds how often a tunnel whose remote we cannot reach probes its other remotes
|
||||||
|
const unreachableProbeInterval = time.Second
|
||||||
|
|
||||||
|
// remoteUnreachable is called when a send to our current remote failed because we have no route to it, typically
|
||||||
|
// after moving to a network without that address family. Sends a test packet to every other remote we know for this
|
||||||
|
// host: the first one that arrives roams the peer over to wherever we are now, and its reply roams us onto that path.
|
||||||
|
//
|
||||||
|
// A peer behind a NAT drops a test from an address it has never sent to, so every probe also asks the lighthouse
|
||||||
|
// about the peer. The lighthouse tells the peer where we are now, the peer punches us, and the next probe gets in.
|
||||||
|
func (i *HostInfo) remoteUnreachable(ifce *Interface, dead netip.AddrPort) {
|
||||||
|
if i.remotes == nil || i.ConnectionState == nil || i.GetRemote() != dead {
|
||||||
|
// Nothing to probe with, or we already moved on from the remote that failed
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if ifce.lightHouse == nil || ifce.lightHouse.amLighthouse {
|
||||||
|
// A lighthouse hands out the address it reaches a client on, it must not go roaming its clients itself
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now().UnixNano()
|
||||||
|
next := i.nextUnreachableProbe.Load()
|
||||||
|
if now < next || !i.nextUnreachableProbe.CompareAndSwap(next, now+int64(unreachableProbeInterval)) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
nb := make([]byte, 12)
|
||||||
|
out := make([]byte, mtu)
|
||||||
|
probed := 0
|
||||||
|
i.remotes.ForEach(ifce.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) {
|
||||||
|
if addr == dead {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
probed++
|
||||||
|
ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), nb, out)
|
||||||
|
})
|
||||||
|
|
||||||
|
if ifce.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||||
|
i.logger(ifce.l).Debug("Remote unreachable, probing other remotes",
|
||||||
|
"udpAddr", dead,
|
||||||
|
"probed", probed,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if probed == 0 {
|
||||||
|
// With nothing else to try, the lighthouse is our only way to learn of another remote. The peer may not have
|
||||||
|
// one, so ask on the usual requery schedule rather than every probe for as long as the tunnel sits here.
|
||||||
|
if now < i.nextLHQuery.Load() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
i.nextLHQuery.Store(now + ifce.reQueryWait.Load())
|
||||||
|
}
|
||||||
|
|
||||||
|
// With other remotes to probe we ask on every probe, without backing off. A repeat re-sends the peer its punch
|
||||||
|
// notification in case the last one or its punches were lost. If the tunnel stays up on inbound traffic while
|
||||||
|
// our sends to its remote keep failing (only one direction broken), this continues once a second for as long as
|
||||||
|
// that lasts. That is a bounded cost for a rare state, and the probe that would clear it has to keep running.
|
||||||
|
ifce.lightHouse.QueryServer(i.vpnAddrs[0])
|
||||||
|
}
|
||||||
|
|
||||||
// Bits within HostInfo.state, everything above stateEpochShift is the epoch
|
// Bits within HostInfo.state, everything above stateEpochShift is the epoch
|
||||||
const (
|
const (
|
||||||
stateIn uint32 = 1 << iota
|
stateIn uint32 = 1 << iota
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"github.com/slackhq/nebula/overlay/batch"
|
"github.com/slackhq/nebula/overlay/batch"
|
||||||
"github.com/slackhq/nebula/overlay/tio"
|
"github.com/slackhq/nebula/overlay/tio"
|
||||||
"github.com/slackhq/nebula/routing"
|
"github.com/slackhq/nebula/routing"
|
||||||
|
"github.com/slackhq/nebula/udp"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
|
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
|
||||||
@@ -611,6 +612,10 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
|
|||||||
"udpAddr", hr,
|
"udpAddr", hr,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
// A closing tunnel is about to be torn down, there is no point looking for another path for it
|
||||||
|
if t != header.CloseTunnel && udp.IsUnreachable(err) {
|
||||||
|
hostinfo.remoteUnreachable(f, hr)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Try to send via a relay
|
// Try to send via a relay
|
||||||
|
|||||||
+41
-1
@@ -1,6 +1,7 @@
|
|||||||
package nebula
|
package nebula
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/fips140"
|
"crypto/fips140"
|
||||||
"errors"
|
"errors"
|
||||||
@@ -495,7 +496,8 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
|
|||||||
|
|
||||||
// flushSendBatch drains sb to the underlay and accounts for anything it could not deliver. A shortfall means
|
// flushSendBatch drains sb to the underlay and accounts for anything it could not deliver. A shortfall means
|
||||||
// specific destinations were undeliverable (a stale remote, a reject rule), which the backend logs per peer at
|
// specific destinations were undeliverable (a stale remote, a reject rule), which the backend logs per peer at
|
||||||
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch.
|
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch. Tunnels whose remote
|
||||||
|
// we no longer have a route to go looking for another path.
|
||||||
func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
|
func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
|
||||||
queued := sb.Len()
|
queued := sb.Len()
|
||||||
written, err := sb.Flush()
|
written, err := sb.Flush()
|
||||||
@@ -505,6 +507,44 @@ func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
|
|||||||
if dropped := queued - written; dropped > 0 {
|
if dropped := queued - written; dropped > 0 {
|
||||||
f.metricTxDropped.Inc(int64(dropped))
|
f.metricTxDropped.Inc(int64(dropped))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if f.lightHouse == nil || f.lightHouse.amLighthouse {
|
||||||
|
// Lighthouses never probe, see remoteUnreachable, so skip finding the tunnels
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// A tunnel's packets tend to fail together, so skip straight past a repeat of the one before. The first 8 bytes
|
||||||
|
// of a header say whose packet it is, the counter after them changes with every packet.
|
||||||
|
var last batch.Failure
|
||||||
|
for i, fail := range sb.Failures() {
|
||||||
|
if i > 0 && fail.Dst == last.Dst && bytes.Equal(fail.Head[:8], last.Head[:8]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
last = fail
|
||||||
|
|
||||||
|
if udp.IsUnreachable(fail.Err) {
|
||||||
|
f.sendUnreachable(fail.Head[:], fail.Dst)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendUnreachable finds the tunnel a packet we had no route for was sent on, from the start of the packet, and has it
|
||||||
|
// look for another path to its peer.
|
||||||
|
func (f *Interface) sendUnreachable(head []byte, dst netip.AddrPort) {
|
||||||
|
var h header.H
|
||||||
|
if h.Parse(head) != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if h.Type == header.Message && h.Subtype == header.MessageRelay {
|
||||||
|
// TODO: a relayed packet's header carries the relay session's index, not the relay tunnel's, so it does not
|
||||||
|
// lead us to the tunnel whose remote failed. The relay tunnel's own traffic finds out soon enough.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if hostinfo := f.hostMap.QueryReverseIndex(h.RemoteIndex); hostinfo != nil {
|
||||||
|
hostinfo.remoteUnreachable(f, dst)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
|
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
|
||||||
|
|||||||
@@ -1,12 +1,27 @@
|
|||||||
package batch
|
package batch
|
||||||
|
|
||||||
import "net/netip"
|
import (
|
||||||
|
"net/netip"
|
||||||
|
|
||||||
|
"github.com/slackhq/nebula/header"
|
||||||
|
)
|
||||||
|
|
||||||
const SendBatchCap = 128
|
const SendBatchCap = 128
|
||||||
|
|
||||||
|
// FailureHeadLen is how much of a refused packet a Failure keeps, nebula's header, which says whose packet it was
|
||||||
|
const FailureHeadLen = header.Len
|
||||||
|
|
||||||
// batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush.
|
// batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush.
|
||||||
type batchWriter interface {
|
type batchWriter interface {
|
||||||
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
|
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Failure is a packet the underlay would not send
|
||||||
|
type Failure struct {
|
||||||
|
// Head is the start of the packet, zero padded if the packet was shorter
|
||||||
|
Head [FailureHeadLen]byte
|
||||||
|
Dst netip.AddrPort
|
||||||
|
Err error
|
||||||
}
|
}
|
||||||
|
|
||||||
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
|
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
|
||||||
@@ -17,16 +32,28 @@ type SendBatch struct {
|
|||||||
bufs [][]byte
|
bufs [][]byte
|
||||||
dsts []netip.AddrPort
|
dsts []netip.AddrPort
|
||||||
arena *Arena
|
arena *Arena
|
||||||
|
|
||||||
|
failures []Failure
|
||||||
|
// recordFailure is bound once so handing it to WriteBatch does not allocate on every Flush
|
||||||
|
recordFailure func(i int, err error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots
|
// NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots
|
||||||
func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch {
|
func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch {
|
||||||
return &SendBatch{
|
b := &SendBatch{
|
||||||
out: out,
|
out: out,
|
||||||
bufs: make([][]byte, 0, batchCap),
|
bufs: make([][]byte, 0, batchCap),
|
||||||
dsts: make([]netip.AddrPort, 0, batchCap),
|
dsts: make([]netip.AddrPort, 0, batchCap),
|
||||||
arena: NewArena(arenaSize),
|
arena: NewArena(arenaSize),
|
||||||
}
|
}
|
||||||
|
b.recordFailure = b.failed
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *SendBatch) failed(i int, err error) {
|
||||||
|
f := Failure{Dst: b.dsts[i], Err: err}
|
||||||
|
copy(f.Head[:], b.bufs[i])
|
||||||
|
b.failures = append(b.failures, f)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *SendBatch) Reserve(sz int) []byte {
|
func (b *SendBatch) Reserve(sz int) []byte {
|
||||||
@@ -44,12 +71,15 @@ func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations
|
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations
|
||||||
// were undeliverable; the batch is drained either way.
|
// were undeliverable, Failures says which; the batch is drained either way.
|
||||||
func (b *SendBatch) Flush() (int, error) {
|
func (b *SendBatch) Flush() (int, error) {
|
||||||
|
clear(b.failures)
|
||||||
|
b.failures = b.failures[:0]
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
written := 0
|
written := 0
|
||||||
if len(b.bufs) > 0 {
|
if len(b.bufs) > 0 {
|
||||||
written, err = b.out.WriteBatch(b.bufs, b.dsts)
|
written, err = b.out.WriteBatch(b.bufs, b.dsts, b.recordFailure)
|
||||||
}
|
}
|
||||||
clear(b.bufs)
|
clear(b.bufs)
|
||||||
b.bufs = b.bufs[:0]
|
b.bufs = b.bufs[:0]
|
||||||
@@ -57,3 +87,9 @@ func (b *SendBatch) Flush() (int, error) {
|
|||||||
b.arena.Reset()
|
b.arena.Reset()
|
||||||
return written, err
|
return written, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Failures returns every packet the last Flush could not send, in the order they were committed. Valid until the
|
||||||
|
// next Flush.
|
||||||
|
func (b *SendBatch) Failures() []Failure {
|
||||||
|
return b.failures
|
||||||
|
}
|
||||||
@@ -1,7 +1,9 @@
|
|||||||
package batch
|
package batch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -10,7 +12,7 @@ type fakeBatchWriter struct {
|
|||||||
addrs []netip.AddrPort
|
addrs []netip.AddrPort
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, _ func(int, error)) (int, error) {
|
||||||
// Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch
|
// Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch
|
||||||
// returns, so tests must capture data before that happens.
|
// returns, so tests must capture data before that happens.
|
||||||
w.bufs = make([][]byte, len(bufs))
|
w.bufs = make([][]byte, len(bufs))
|
||||||
@@ -120,3 +122,84 @@ func TestSendBatchGrowPreservesCommitted(t *testing.T) {
|
|||||||
t.Errorf("second packet on the wire: %x", fw.bufs[1])
|
t.Errorf("second packet on the wire: %x", fw.bufs[1])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// failingBatchWriter refuses every packet whose first byte is in refuse
|
||||||
|
type failingBatchWriter struct {
|
||||||
|
refuse map[byte]error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *failingBatchWriter) WriteBatch(bufs [][]byte, _ []netip.AddrPort, failed func(int, error)) (int, error) {
|
||||||
|
written := 0
|
||||||
|
for i, b := range bufs {
|
||||||
|
if err, ok := w.refuse[b[0]]; ok {
|
||||||
|
failed(i, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
written++
|
||||||
|
}
|
||||||
|
return written, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSendBatchReportsFailures(t *testing.T) {
|
||||||
|
errNoRoute := errors.New("no route")
|
||||||
|
fw := &failingBatchWriter{refuse: map[byte]error{2: errNoRoute, 3: errNoRoute}}
|
||||||
|
b := NewSendBatch(fw, 4, 128)
|
||||||
|
|
||||||
|
// pkt is n bytes starting with the refuse key, each later byte its offset plus 0x10 so the head is recognizable
|
||||||
|
pkt := func(key byte, n int) []byte {
|
||||||
|
p := b.Reserve(n)[:n]
|
||||||
|
p[0] = key
|
||||||
|
for i := 1; i < n; i++ {
|
||||||
|
p[i] = byte(0x10 + i)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
head := func(p []byte) (h [FailureHeadLen]byte) {
|
||||||
|
copy(h[:], p)
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
dsts := []netip.AddrPort{
|
||||||
|
netip.MustParseAddrPort("10.0.0.1:4242"),
|
||||||
|
netip.MustParseAddrPort("[fd00::2]:4242"),
|
||||||
|
netip.MustParseAddrPort("[fd00::3]:4242"),
|
||||||
|
netip.MustParseAddrPort("10.0.0.4:4242"),
|
||||||
|
}
|
||||||
|
long := pkt(2, FailureHeadLen+8) // only its head is kept
|
||||||
|
short := pkt(3, 4) // zero padded
|
||||||
|
b.Commit(pkt(1, 20), dsts[0])
|
||||||
|
b.Commit(long, dsts[1])
|
||||||
|
b.Commit(short, dsts[2])
|
||||||
|
b.Commit(pkt(1, 20), dsts[3])
|
||||||
|
|
||||||
|
written, err := b.Flush()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if written != 2 {
|
||||||
|
t.Errorf("written = %d, want 2", written)
|
||||||
|
}
|
||||||
|
want := []Failure{{head(long), dsts[1], errNoRoute}, {head(short), dsts[2], errNoRoute}}
|
||||||
|
if got := b.Failures(); !slices.Equal(got, want) {
|
||||||
|
t.Errorf("failures = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Failures do not leak into the next flush
|
||||||
|
again := pkt(2, 20)
|
||||||
|
b.Commit(again, dsts[0])
|
||||||
|
if _, err := b.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if want := []Failure{{head(again), dsts[0], errNoRoute}}; !slices.Equal(b.Failures(), want) {
|
||||||
|
t.Errorf("second flush failures = %v, want %v", b.Failures(), want)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A clean flush forgets the last one's failures
|
||||||
|
b.Commit(pkt(1, 20), dsts[0])
|
||||||
|
if _, err := b.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if got := b.Failures(); len(got) != 0 {
|
||||||
|
t.Errorf("clean flush failures = %v, want none", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -165,20 +165,24 @@ func (p *Punchy) scheduleJob(job holepunchJob, delay time.Duration) {
|
|||||||
// SendPunch sends an immediate keepalive punch for an idle hostinfo.
|
// SendPunch sends an immediate keepalive punch for an idle hostinfo.
|
||||||
// The configured punchy.target_all_remotes mode picks the targets. Gated on punchy.punch and the lighthouse-skip rule
|
// The configured punchy.target_all_remotes mode picks the targets. Gated on punchy.punch and the lighthouse-skip rule
|
||||||
// (lighthouses don't get keepalive punches because the regular update interval keeps their NAT state warm).
|
// (lighthouses don't get keepalive punches because the regular update interval keeps their NAT state warm).
|
||||||
func (p *Punchy) SendPunch(hostinfo *HostInfo) {
|
// Returns the hostinfo's current remote and the error from punching it, an invalid remote if it was not punched.
|
||||||
|
func (p *Punchy) SendPunch(hostinfo *HostInfo) (netip.AddrPort, error) {
|
||||||
if !p.punch.Load() {
|
if !p.punch.Load() {
|
||||||
return
|
return netip.AddrPort{}, nil
|
||||||
}
|
}
|
||||||
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
|
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
|
||||||
return
|
return netip.AddrPort{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
hr := hostinfo.GetRemote()
|
||||||
if p.punchEverything.Load() {
|
if p.punchEverything.Load() {
|
||||||
p.sendPunchToAllRemotes(hostinfo)
|
return hr, p.sendPunchToAllRemotes(hostinfo, hr)
|
||||||
} else if hr := hostinfo.GetRemote(); hr.IsValid() {
|
|
||||||
p.metricPunchyTx.Inc(1)
|
|
||||||
p.punchConn.WriteTo([]byte{1}, hr)
|
|
||||||
}
|
}
|
||||||
|
if !hr.IsValid() {
|
||||||
|
return hr, nil
|
||||||
|
}
|
||||||
|
p.metricPunchyTx.Inc(1)
|
||||||
|
return hr, p.punchConn.WriteTo([]byte{1}, hr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SendPunchToAll punches every known remote for hostinfo, but only when punchy.target_all_remotes is enabled.
|
// SendPunchToAll punches every known remote for hostinfo, but only when punchy.target_all_remotes is enabled.
|
||||||
@@ -195,14 +199,19 @@ func (p *Punchy) SendPunchToAll(hostinfo *HostInfo) {
|
|||||||
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
|
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
p.sendPunchToAllRemotes(hostinfo)
|
p.sendPunchToAllRemotes(hostinfo, netip.AddrPort{})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo) {
|
// sendPunchToAllRemotes punches every known remote for hostinfo and returns the error from punching current
|
||||||
|
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo, current netip.AddrPort) error {
|
||||||
|
var currentErr error
|
||||||
hostinfo.remotes.ForEach(p.hm.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
|
hostinfo.remotes.ForEach(p.hm.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
|
||||||
p.metricPunchyTx.Inc(1)
|
p.metricPunchyTx.Inc(1)
|
||||||
p.punchConn.WriteTo([]byte{1}, addr)
|
if err := p.punchConn.WriteTo([]byte{1}, addr); addr == current {
|
||||||
|
currentErr = err
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
return currentErr
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start wires the runtime dependencies and spawns the scheduler worker.
|
// Start wires the runtime dependencies and spawns the scheduler worker.
|
||||||
|
|||||||
+112
@@ -3,11 +3,14 @@ package nebula
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"net/netip"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
|
"github.com/slackhq/nebula/header"
|
||||||
"github.com/slackhq/nebula/test"
|
"github.com/slackhq/nebula/test"
|
||||||
|
"github.com/slackhq/nebula/udp"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
@@ -242,3 +245,112 @@ func TestPunchyReloadRestartRequired(t *testing.T) {
|
|||||||
require.NoError(t, res.Err)
|
require.NoError(t, res.Err)
|
||||||
assert.True(t, res.RestartRequired)
|
assert.True(t, res.RestartRequired)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// errNoRoute is the unreachable error every platform classifies the same way
|
||||||
|
var errNoRoute = udp.ErrInvalidIPv6RemoteForSocket
|
||||||
|
|
||||||
|
// recordingConn records where every write went and what kind of message it was, and fails the ones to an address in
|
||||||
|
// fail
|
||||||
|
type recordingConn struct {
|
||||||
|
udp.NoopConn
|
||||||
|
fail map[netip.AddrPort]error
|
||||||
|
wrote []netip.AddrPort
|
||||||
|
// types is the message type of each write, a punch has none
|
||||||
|
types []header.MessageType
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *recordingConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
||||||
|
c.wrote = append(c.wrote, addr)
|
||||||
|
var h header.H
|
||||||
|
if h.Parse(b) == nil {
|
||||||
|
c.types = append(c.types, h.Type)
|
||||||
|
}
|
||||||
|
return c.fail[addr]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *recordingConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(int, error)) (int, error) {
|
||||||
|
written := 0
|
||||||
|
for i := range bufs {
|
||||||
|
if err := c.WriteTo(bufs[i], addrs[i]); err != nil {
|
||||||
|
failed(i, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
written++
|
||||||
|
}
|
||||||
|
return written, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type noLighthouses struct{}
|
||||||
|
|
||||||
|
func (noLighthouses) IsAnyLighthouseAddr([]netip.Addr) bool { return false }
|
||||||
|
|
||||||
|
var (
|
||||||
|
punchTargetV4 = netip.MustParseAddrPort("10.0.0.3:4242")
|
||||||
|
punchTargetV6 = netip.MustParseAddrPort("[fd00::3]:4242")
|
||||||
|
)
|
||||||
|
|
||||||
|
// newPunchTarget is a tunnel on its IPv6 remote that also knows the peer's IPv4 one
|
||||||
|
func newPunchTarget() *HostInfo {
|
||||||
|
vpnAddr := netip.MustParseAddr("10.128.0.3")
|
||||||
|
hostinfo := &HostInfo{
|
||||||
|
vpnAddrs: []netip.Addr{vpnAddr},
|
||||||
|
remotes: NewRemoteList([]netip.Addr{vpnAddr}, nil),
|
||||||
|
}
|
||||||
|
hostinfo.remotes.LearnRemote(vpnAddr, punchTargetV4)
|
||||||
|
hostinfo.SetRemote(punchTargetV6)
|
||||||
|
return hostinfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPunchyForTest(t *testing.T, settings map[string]any, conn udp.Conn) *Punchy {
|
||||||
|
c := config.NewC(test.NewLogger())
|
||||||
|
c.Settings["punchy"] = settings
|
||||||
|
p := NewPunchyFromConfig(test.NewLogger(), c, conn)
|
||||||
|
|
||||||
|
hm := newHostMap(test.NewLogger())
|
||||||
|
hm.preferredRanges.Store(&[]netip.Prefix{})
|
||||||
|
p.hm = hm
|
||||||
|
p.lh = noLighthouses{}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPunchy_SendPunchReportsCurrentRemote(t *testing.T) {
|
||||||
|
t.Run("disabled", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
||||||
|
p := newPunchyForTest(t, map[string]any{"punch": false}, conn)
|
||||||
|
|
||||||
|
remote, err := p.SendPunch(newPunchTarget())
|
||||||
|
assert.False(t, remote.IsValid())
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, conn.wrote)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("current remote", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
||||||
|
p := newPunchyForTest(t, map[string]any{"punch": true}, conn)
|
||||||
|
|
||||||
|
remote, err := p.SendPunch(newPunchTarget())
|
||||||
|
assert.Equal(t, punchTargetV6, remote)
|
||||||
|
require.ErrorIs(t, err, errNoRoute)
|
||||||
|
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("all remotes, current fails", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
||||||
|
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
|
||||||
|
|
||||||
|
remote, err := p.SendPunch(newPunchTarget())
|
||||||
|
assert.Equal(t, punchTargetV6, remote)
|
||||||
|
require.ErrorIs(t, err, errNoRoute)
|
||||||
|
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("all remotes, only another fails", func(t *testing.T) {
|
||||||
|
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV4: errNoRoute}}
|
||||||
|
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
|
||||||
|
|
||||||
|
remote, err := p.SendPunch(newPunchTarget())
|
||||||
|
assert.Equal(t, punchTargetV6, remote)
|
||||||
|
require.NoError(t, err, "a failure on a remote we are not using says nothing about the current one")
|
||||||
|
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
|
||||||
|
})
|
||||||
|
}
|
||||||
+3
-2
@@ -41,8 +41,9 @@ type Conn interface {
|
|||||||
//
|
//
|
||||||
// Returns the number of packets successfully written. A destination the kernel rejects costs only
|
// Returns the number of packets successfully written. A destination the kernel rejects costs only
|
||||||
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
|
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
|
||||||
|
// failed, when not nil, is called with the index of every packet a destination cost us and why.
|
||||||
// Not safe for concurrent use on the same Conn.
|
// Not safe for concurrent use on the same Conn.
|
||||||
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
|
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
|
||||||
// ReloadConfig applies the listen settings that can change, r is nil at startup
|
// ReloadConfig applies the listen settings that can change, r is nil at startup
|
||||||
ReloadConfig(c *config.C, r *config.Reload)
|
ReloadConfig(c *config.C, r *config.Reload)
|
||||||
SupportsMultipleReaders() bool
|
SupportsMultipleReaders() bool
|
||||||
@@ -66,7 +67,7 @@ func (NoopConn) SupportsMultipleReaders() bool {
|
|||||||
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
|
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
|
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
|
||||||
return len(bufs), nil
|
return len(bufs), nil
|
||||||
}
|
}
|
||||||
func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) {
|
func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) {
|
||||||
|
|||||||
+4
-1
@@ -139,7 +139,7 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
||||||
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
||||||
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
|
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
|
||||||
// silently drops the rest of a burst (linux blocks instead). Poll for
|
// silently drops the rest of a burst (linux blocks instead). Poll for
|
||||||
@@ -150,6 +150,9 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
|
|||||||
written++
|
written++
|
||||||
} else {
|
} else {
|
||||||
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
||||||
|
if failed != nil {
|
||||||
|
failed(i, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
|
|||||||
+4
-1
@@ -44,7 +44,7 @@ func (u *GenericConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
||||||
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
||||||
written := 0
|
written := 0
|
||||||
for i, b := range bufs {
|
for i, b := range bufs {
|
||||||
@@ -52,6 +52,9 @@ func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
|
|||||||
written++
|
written++
|
||||||
} else {
|
} else {
|
||||||
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
||||||
|
if failed != nil {
|
||||||
|
failed(i, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
|
|||||||
+2
-2
@@ -358,8 +358,8 @@ func sendto(fd int, b []byte, addr netip.AddrPort, isV4 bool) error {
|
|||||||
|
|
||||||
// WriteBatch sends bufs via sendmmsg(2), coalescing same-destination runs into UDP-GSO superpackets when supported.
|
// WriteBatch sends bufs via sendmmsg(2), coalescing same-destination runs into UDP-GSO superpackets when supported.
|
||||||
// See batchWriter in udp_linux_writebatch.go for the mechanics.
|
// See batchWriter in udp_linux_writebatch.go for the mechanics.
|
||||||
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
||||||
return u.bw.WriteBatch(bufs, addrs)
|
return u.bw.WriteBatch(bufs, addrs, failed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeSockaddr encodes addr into buf (which must be at least SizeofSockaddrInet6 bytes).
|
// writeSockaddr encodes addr into buf (which must be at least SizeofSockaddrInet6 bytes).
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
package udp
|
package udp
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
@@ -96,13 +97,22 @@ func TestWriteBatchBadFamilyDeliversOthers(t *testing.T) {
|
|||||||
bufs := [][]byte{[]byte("AAA"), []byte("BBB"), []byte("CCC")}
|
bufs := [][]byte{[]byte("AAA"), []byte("BBB"), []byte("CCC")}
|
||||||
addrs := []netip.AddrPort{good, bad, good}
|
addrs := []netip.AddrPort{good, bad, good}
|
||||||
|
|
||||||
n, err := sender.WriteBatch(bufs, addrs)
|
var failed []int
|
||||||
|
n, err := sender.WriteBatch(bufs, addrs, func(i int, err error) {
|
||||||
|
if !IsUnreachable(err) {
|
||||||
|
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
|
||||||
|
}
|
||||||
|
failed = append(failed, i)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch returned error, want nil (bad dest should be isolated): %v", err)
|
t.Fatalf("WriteBatch returned error, want nil (bad dest should be isolated): %v", err)
|
||||||
}
|
}
|
||||||
if n != 2 {
|
if n != 2 {
|
||||||
t.Errorf("WriteBatch wrote %d packets, want 2 of 3 (the bad-family dest is the only casualty)", n)
|
t.Errorf("WriteBatch wrote %d packets, want 2 of 3 (the bad-family dest is the only casualty)", n)
|
||||||
}
|
}
|
||||||
|
if want := []int{1}; !slices.Equal(failed, want) {
|
||||||
|
t.Errorf("failed reported %v, want %v", failed, want)
|
||||||
|
}
|
||||||
|
|
||||||
got := map[string]bool{}
|
got := map[string]bool{}
|
||||||
rx.SetReadDeadline(time.Now().Add(2 * time.Second))
|
rx.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||||
@@ -154,7 +164,7 @@ func TestWriteBatchUnreachableDestDeliversOthers(t *testing.T) {
|
|||||||
addrs := []netip.AddrPort{good, good, bad, good, good}
|
addrs := []netip.AddrPort{good, good, bad, good, good}
|
||||||
|
|
||||||
// The bad destination is reported, but only after every other packet has been attempted.
|
// The bad destination is reported, but only after every other packet has been attempted.
|
||||||
if _, err := sender.WriteBatch(bufs, addrs); err == nil {
|
if _, err := sender.WriteBatch(bufs, addrs, nil); err == nil {
|
||||||
t.Log("WriteBatch returned nil; kernel accepted the reserved address, delivery assertions still apply")
|
t.Log("WriteBatch returned nil; kernel accepted the reserved address, delivery assertions still apply")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -376,7 +386,7 @@ func TestWriteBatchPartialSendRewind(t *testing.T) {
|
|||||||
return accept, nil
|
return accept, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := w.WriteBatch(bufs, addrs)
|
written, err := w.WriteBatch(bufs, addrs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
@@ -429,13 +439,22 @@ func TestWriteBatchSkipUnroutableRunAccounting(t *testing.T) {
|
|||||||
return accept, nil
|
return accept, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := w.WriteBatch(bufs, addrs)
|
var failed []int
|
||||||
|
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
|
||||||
|
if !IsUnreachable(err) {
|
||||||
|
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
|
||||||
|
}
|
||||||
|
failed = append(failed, i)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
if written != 4 {
|
if written != 4 {
|
||||||
t.Errorf("written = %d, want 4 (the unroutable run is the only casualty)", written)
|
t.Errorf("written = %d, want 4 (the unroutable run is the only casualty)", written)
|
||||||
}
|
}
|
||||||
|
if want := []int{2}; !slices.Equal(failed, want) {
|
||||||
|
t.Errorf("failed reported %v, want %v", failed, want)
|
||||||
|
}
|
||||||
wantTags := []byte{1, 2, 4, 5}
|
wantTags := []byte{1, 2, 4, 5}
|
||||||
if len(wire) != len(wantTags) {
|
if len(wire) != len(wantTags) {
|
||||||
t.Fatalf("wire got %d packets, want %d (dup or loss around the skip)", len(wire), len(wantTags))
|
t.Fatalf("wire got %d packets, want %d (dup or loss around the skip)", len(wire), len(wantTags))
|
||||||
@@ -486,13 +505,23 @@ func TestWriteBatchMidChunkRejectResumes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := w.WriteBatch(bufs, addrs)
|
var failed []int
|
||||||
|
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
|
||||||
|
if !errors.Is(err, unix.EPERM) {
|
||||||
|
t.Errorf("packet %d failed with %v, want EPERM", i, err)
|
||||||
|
}
|
||||||
|
failed = append(failed, i)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
if written != 3 {
|
if written != 3 {
|
||||||
t.Errorf("written = %d, want 3 (B's rejected run is the only casualty)", written)
|
t.Errorf("written = %d, want 3 (B's rejected run is the only casualty)", written)
|
||||||
}
|
}
|
||||||
|
// Every packet in B's run is reported, not just the entry
|
||||||
|
if want := []int{2, 3}; !slices.Equal(failed, want) {
|
||||||
|
t.Errorf("failed reported %v, want %v", failed, want)
|
||||||
|
}
|
||||||
wantTags := []byte{1, 2, 5}
|
wantTags := []byte{1, 2, 5}
|
||||||
if len(wire) != len(wantTags) {
|
if len(wire) != len(wantTags) {
|
||||||
t.Fatalf("wire got %d packets, want %d (dup or loss around the mid-chunk reject)", len(wire), len(wantTags))
|
t.Fatalf("wire got %d packets, want %d (dup or loss around the mid-chunk reject)", len(wire), len(wantTags))
|
||||||
@@ -543,7 +572,7 @@ func TestWriteBatchMidChunkEIODisablesGSOWithoutDup(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := w.WriteBatch(bufs, addrs)
|
written, err := w.WriteBatch(bufs, addrs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
@@ -571,7 +600,7 @@ func TestWriteBatchZeroProgress(t *testing.T) {
|
|||||||
w.sendFn = func(start, n int) (int, error) { return 0, nil }
|
w.sendFn = func(start, n int) (int, error) { return 0, nil }
|
||||||
bufs := [][]byte{make([]byte, 100)}
|
bufs := [][]byte{make([]byte, 100)}
|
||||||
addrs := []netip.AddrPort{netip.MustParseAddrPort("127.0.0.1:4242")}
|
addrs := []netip.AddrPort{netip.MustParseAddrPort("127.0.0.1:4242")}
|
||||||
if _, err := w.WriteBatch(bufs, addrs); err == nil {
|
if _, err := w.WriteBatch(bufs, addrs, nil); err == nil {
|
||||||
t.Fatal("WriteBatch = nil error on zero progress, want error")
|
t.Fatal("WriteBatch = nil error on zero progress, want error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -597,7 +626,7 @@ func TestWriteBatchEIODisablesGSOAndReplays(t *testing.T) {
|
|||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := w.WriteBatch(bufs, addrs)
|
written, err := w.WriteBatch(bufs, addrs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
@@ -674,7 +703,7 @@ func TestGSOEngagesOnLoopback(t *testing.T) {
|
|||||||
addrs[i] = dst
|
addrs[i] = dst
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := sc.WriteBatch(bufs, addrs)
|
written, err := sc.WriteBatch(bufs, addrs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,7 +86,7 @@ func TestWriteBatchOffloadsDisabledScripted(t *testing.T) {
|
|||||||
addrs[i] = dst
|
addrs[i] = dst
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := w.WriteBatch(bufs, addrs)
|
written, err := w.WriteBatch(bufs, addrs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
@@ -163,7 +163,7 @@ func TestOffloadsDisabledOnLoopback(t *testing.T) {
|
|||||||
addrs[i] = dst
|
addrs[i] = dst
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := sc.WriteBatch(bufs, addrs)
|
written, err := sc.WriteBatch(bufs, addrs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -190,8 +190,8 @@ func parseRelease(r string) (major, minor int) {
|
|||||||
// its packets are dropped and the rest of the chunk resumes in place.
|
// its packets are dropped and the rest of the chunk resumes in place.
|
||||||
//
|
//
|
||||||
// Returns the number of packets sent. An error means the call itself failed.
|
// Returns the number of packets sent. An error means the call itself failed.
|
||||||
// A short count means some destinations were undeliverable.
|
// A short count means some destinations were undeliverable, failed hears about each of their packets.
|
||||||
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
||||||
if len(bufs) != len(addrs) {
|
if len(bufs) != len(addrs) {
|
||||||
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
|
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
|
||||||
}
|
}
|
||||||
@@ -233,6 +233,7 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
|
|||||||
if w.l.Enabled(context.Background(), slog.LevelDebug) {
|
if w.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||||
w.l.Debug("skipping unroutable batch destination", "udpAddr", addrs[i], "packets", runLen, "error", err)
|
w.l.Debug("skipping unroutable batch destination", "udpAddr", addrs[i], "packets", runLen, "error", err)
|
||||||
}
|
}
|
||||||
|
reportFailed(failed, i, i+runLen, err)
|
||||||
i += runLen
|
i += runLen
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -300,12 +301,23 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
|
|||||||
"gso", w.gsoSupported,
|
"gso", w.gsoSupported,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
reportFailed(failed, w.entryEnd[done]-w.entryPkts[done], w.entryEnd[done], serr)
|
||||||
done++
|
done++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reportFailed tells failed, if anyone is listening, that every packet in bufs[start:end] was lost to err
|
||||||
|
func reportFailed(failed func(i int, err error), start, end int, err error) {
|
||||||
|
if failed == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for i := start; i < end; i++ {
|
||||||
|
failed(i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// planRun returns the length of the run starting at start and its segment
|
// planRun returns the length of the run starting at start and its segment
|
||||||
// size (len(bufs[start])). A run of length 1 carries no UDP_SEGMENT cmsg
|
// size (len(bufs[start])). A run of length 1 carries no UDP_SEGMENT cmsg
|
||||||
// and is sent as a plain datagram; without GSO support planRun always returns 1.
|
// and is sent as a plain datagram; without GSO support planRun always returns 1.
|
||||||
|
|||||||
@@ -80,11 +80,11 @@ func TestWriteBatchNoAllocs(t *testing.T) {
|
|||||||
|
|
||||||
var werr error
|
var werr error
|
||||||
// Warm-up outside the measured runs.
|
// Warm-up outside the measured runs.
|
||||||
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
|
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
|
||||||
t.Fatalf("WriteBatch warm-up: %v", err)
|
t.Fatalf("WriteBatch warm-up: %v", err)
|
||||||
}
|
}
|
||||||
allocs := testing.AllocsPerRun(100, func() {
|
allocs := testing.AllocsPerRun(100, func() {
|
||||||
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
|
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
|
||||||
werr = err
|
werr = err
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -319,7 +319,7 @@ func (u *RIOConn) WriteTo(buf []byte, ip netip.AddrPort) error {
|
|||||||
return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
|
return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
||||||
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
||||||
written := 0
|
written := 0
|
||||||
for i, b := range bufs {
|
for i, b := range bufs {
|
||||||
@@ -327,6 +327,9 @@ func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
|
|||||||
written++
|
written++
|
||||||
} else {
|
} else {
|
||||||
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
||||||
|
if failed != nil {
|
||||||
|
failed(i, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
|
|||||||
+64
-7
@@ -6,6 +6,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -63,10 +64,19 @@ func acquirePacket() *Packet {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// testerAddrs is where a TesterConn sends from. A single address host sends everything from addr, whatever the
|
||||||
|
// destination. A dual stack host sends from the address matching the destination's family and has no route at all
|
||||||
|
// for a family it holds no address in.
|
||||||
|
type testerAddrs struct {
|
||||||
|
addr netip.AddrPort
|
||||||
|
v4, v6 netip.AddrPort
|
||||||
|
dual bool
|
||||||
|
}
|
||||||
|
|
||||||
type TesterConn struct {
|
type TesterConn struct {
|
||||||
// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
|
// addrs is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
|
||||||
// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
|
// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
|
||||||
addr atomic.Pointer[netip.AddrPort]
|
addrs atomic.Pointer[testerAddrs]
|
||||||
|
|
||||||
RxPackets chan *Packet // Packets to receive into nebula
|
RxPackets chan *Packet // Packets to receive into nebula
|
||||||
TxPackets chan *Packet // Packets transmitted outside by nebula
|
TxPackets chan *Packet // Packets transmitted outside by nebula
|
||||||
@@ -94,14 +104,53 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
|
|||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAddr returns the underlay address this conn currently sends from.
|
// GetAddr returns the underlay address this conn currently sends from. For a dual stack conn that is the IPv4
|
||||||
|
// address, or the IPv6 one when there is no IPv4 service.
|
||||||
func (u *TesterConn) GetAddr() netip.AddrPort {
|
func (u *TesterConn) GetAddr() netip.AddrPort {
|
||||||
return *u.addr.Load()
|
return u.addrs.Load().addr
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
|
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
|
||||||
func (u *TesterConn) SetAddr(addr netip.AddrPort) {
|
func (u *TesterConn) SetAddr(addr netip.AddrPort) {
|
||||||
u.addr.Store(&addr)
|
u.addrs.Store(&testerAddrs{addr: addr})
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetDualStackAddrs gives this conn one underlay address per family, standing in for a host whose routing table picks
|
||||||
|
// the source by destination. Pass an invalid AddrPort for a family the network does not offer, sends to that family
|
||||||
|
// then fail like they would on a host with no route for it.
|
||||||
|
func (u *TesterConn) SetDualStackAddrs(v4, v6 netip.AddrPort) {
|
||||||
|
a := &testerAddrs{v4: v4, v6: v6, dual: true}
|
||||||
|
if v4.IsValid() {
|
||||||
|
a.addr = v4
|
||||||
|
} else {
|
||||||
|
a.addr = v6
|
||||||
|
}
|
||||||
|
u.addrs.Store(a)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAddrFor returns the underlay address a send to dst leaves from, GetAddr when there is no route to dst
|
||||||
|
func (u *TesterConn) GetAddrFor(dst netip.AddrPort) netip.AddrPort {
|
||||||
|
if src, err := u.sourceFor(dst); err == nil {
|
||||||
|
return src
|
||||||
|
}
|
||||||
|
return u.GetAddr()
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceFor returns the address a send to dst leaves from
|
||||||
|
func (u *TesterConn) sourceFor(dst netip.AddrPort) (netip.AddrPort, error) {
|
||||||
|
a := u.addrs.Load()
|
||||||
|
if !a.dual {
|
||||||
|
return a.addr, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
src := a.v6
|
||||||
|
if dst.Addr().Unmap().Is4() {
|
||||||
|
src = a.v4
|
||||||
|
}
|
||||||
|
if !src.IsValid() {
|
||||||
|
return netip.AddrPort{}, &net.OpError{Op: "write", Net: "udp", Addr: net.UDPAddrFromAddrPort(dst), Err: errNetUnreachable}
|
||||||
|
}
|
||||||
|
return src, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send will place a UdpPacket onto the receive queue for nebula to consume
|
// Send will place a UdpPacket onto the receive queue for nebula to consume
|
||||||
@@ -153,6 +202,11 @@ func (u *TesterConn) Get(block bool) *Packet {
|
|||||||
//********************************************************************************************************************//
|
//********************************************************************************************************************//
|
||||||
|
|
||||||
func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
||||||
|
from, err := u.sourceFor(addr)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
p := acquirePacket()
|
p := acquirePacket()
|
||||||
if cap(p.Data) < len(b) {
|
if cap(p.Data) < len(b) {
|
||||||
p.Data = make([]byte, len(b))
|
p.Data = make([]byte, len(b))
|
||||||
@@ -160,7 +214,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
|||||||
p.Data = p.Data[:len(b)]
|
p.Data = p.Data[:len(b)]
|
||||||
}
|
}
|
||||||
copy(p.Data, b)
|
copy(p.Data, b)
|
||||||
p.From = u.GetAddr()
|
p.From = from
|
||||||
p.To = addr
|
p.To = addr
|
||||||
select {
|
select {
|
||||||
case <-u.done:
|
case <-u.done:
|
||||||
@@ -170,13 +224,16 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
||||||
written := 0
|
written := 0
|
||||||
for i, b := range bufs {
|
for i, b := range bufs {
|
||||||
if err := u.WriteTo(b, addrs[i]); err == nil {
|
if err := u.WriteTo(b, addrs[i]); err == nil {
|
||||||
written++
|
written++
|
||||||
} else {
|
} else {
|
||||||
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
||||||
|
if failed != nil {
|
||||||
|
failed(i, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package udp
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// errNetUnreachable is the error this platform's sockets give for a destination with no route
|
||||||
|
var errNetUnreachable error = syscall.ENETUNREACH
|
||||||
|
|
||||||
|
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
|
||||||
|
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
|
||||||
|
// to an IPv6 remote into one of these.
|
||||||
|
func IsUnreachable(err error) bool {
|
||||||
|
return errors.Is(err, syscall.ENETUNREACH) ||
|
||||||
|
errors.Is(err, syscall.EHOSTUNREACH) ||
|
||||||
|
errors.Is(err, syscall.EADDRNOTAVAIL) ||
|
||||||
|
errors.Is(err, syscall.EAFNOSUPPORT) ||
|
||||||
|
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package udp
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIsUnreachable(t *testing.T) {
|
||||||
|
wrap := func(err error) error { return &net.OpError{Op: "write", Net: "udp", Err: err} }
|
||||||
|
|
||||||
|
for _, err := range []error{
|
||||||
|
wrap(syscall.ENETUNREACH),
|
||||||
|
wrap(syscall.EHOSTUNREACH),
|
||||||
|
wrap(syscall.EADDRNOTAVAIL),
|
||||||
|
wrap(syscall.EAFNOSUPPORT),
|
||||||
|
ErrInvalidIPv6RemoteForSocket,
|
||||||
|
} {
|
||||||
|
if !IsUnreachable(err) {
|
||||||
|
t.Errorf("IsUnreachable(%v) = false, want true", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, err := range []error{
|
||||||
|
nil,
|
||||||
|
wrap(syscall.EPERM),
|
||||||
|
wrap(syscall.ENOBUFS),
|
||||||
|
wrap(syscall.EMSGSIZE),
|
||||||
|
net.ErrClosed,
|
||||||
|
} {
|
||||||
|
if IsUnreachable(err) {
|
||||||
|
t.Errorf("IsUnreachable(%v) = true, want false", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
package udp
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"golang.org/x/sys/windows"
|
||||||
|
)
|
||||||
|
|
||||||
|
// errNetUnreachable is the error this platform's sockets give for a destination with no route
|
||||||
|
var errNetUnreachable error = windows.WSAENETUNREACH
|
||||||
|
|
||||||
|
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
|
||||||
|
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
|
||||||
|
// to an IPv6 remote into one of these.
|
||||||
|
func IsUnreachable(err error) bool {
|
||||||
|
return errors.Is(err, windows.WSAENETUNREACH) ||
|
||||||
|
errors.Is(err, windows.WSAEHOSTUNREACH) ||
|
||||||
|
errors.Is(err, windows.WSAEADDRNOTAVAIL) ||
|
||||||
|
errors.Is(err, windows.WSAEAFNOSUPPORT) ||
|
||||||
|
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
|
||||||
|
}
|
||||||
Reference in new issue
Block a user