mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 17:16:52 +01:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
357b129749 |
No files matched your search
@@ -7,17 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
### Changed
|
|
||||||
|
|
||||||
- A tunnel whose remote becomes unreachable, such as an IPv6 remote after moving to a network without IPv6, now
|
|
||||||
moves to another of the peer's addresses within a round trip or two instead of waiting 20-35s to be torn down
|
|
||||||
and rehandshaked. When a send fails because there is no route to the remote, the tunnel sends a test packet to
|
|
||||||
every other address it knows for the peer and asks the lighthouse to have the peer punch its new address.
|
|
||||||
Whichever path answers carries the tunnel. With `punchy.punch` enabled, idle tunnels notice on their next
|
|
||||||
keepalive punch. An idle tunnel whose remote is unreachable and none of whose other addresses answer is now
|
|
||||||
torn down by the connection manager rather than left in place. Not yet effective for relayed traffic or on
|
|
||||||
Windows with the default RIO backend.
|
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
|
|
||||||
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
|
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
|
||||||
|
|||||||
@@ -86,21 +86,6 @@ func (c *C) InitialLoad() bool {
|
|||||||
return c.oldSettings == nil
|
return c.oldSettings == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Render returns keys and their values as yaml, for a subsystem to remember what it applied. Empty means they couldn't
|
|
||||||
// be rendered and should never match.
|
|
||||||
func (c *C) Render(keys ...string) string {
|
|
||||||
m := make(map[string]any, len(keys))
|
|
||||||
for _, k := range keys {
|
|
||||||
m[k] = c.Get(k)
|
|
||||||
}
|
|
||||||
b, err := yaml.Marshal(m)
|
|
||||||
if err != nil {
|
|
||||||
c.l.Debug("Failed to render config", "keys", keys, "error", err)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return string(b)
|
|
||||||
}
|
|
||||||
|
|
||||||
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
|
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
|
||||||
// k in both the old and new settings will be serialized, the result of the string comparison is returned.
|
// k in both the old and new settings will be serialized, the result of the string comparison is returned.
|
||||||
// If k is an empty string the entire config is tested.
|
// If k is an empty string the entire config is tested.
|
||||||
|
|||||||
+12
-28
@@ -14,7 +14,6 @@ import (
|
|||||||
"github.com/slackhq/nebula/cert"
|
"github.com/slackhq/nebula/cert"
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
"github.com/slackhq/nebula/header"
|
"github.com/slackhq/nebula/header"
|
||||||
"github.com/slackhq/nebula/udp"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type trafficDecision int
|
type trafficDecision int
|
||||||
@@ -172,7 +171,7 @@ func (cm *connectionManager) Start(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
|
func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
|
||||||
decision, hostinfo, primary, punch := cm.makeTrafficDecision(localIndex, now)
|
decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now)
|
||||||
|
|
||||||
switch decision {
|
switch decision {
|
||||||
case deleteTunnel:
|
case deleteTunnel:
|
||||||
@@ -197,20 +196,6 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
|
|||||||
}
|
}
|
||||||
|
|
||||||
cm.resetRelayTrafficCheck(hostinfo)
|
cm.resetRelayTrafficCheck(hostinfo)
|
||||||
|
|
||||||
if punch != nil {
|
|
||||||
cm.sendPunch(punch)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendPunch keeps a tunnel's NAT state alive when we are not sending on it. When we have no route to its remote any
|
|
||||||
// more, typically after moving to a network without that address family, the tunnel probes its other remotes so the
|
|
||||||
// peer roams over to us. Nothing else would notice, a tunnel we are not sending on has nothing else that could fail.
|
|
||||||
func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
|
|
||||||
remote, err := cm.punchy.SendPunch(hostinfo)
|
|
||||||
if udp.IsUnreachable(err) {
|
|
||||||
hostinfo.remoteUnreachable(cm.intf, remote)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
|
func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
|
||||||
@@ -328,9 +313,7 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// makeTrafficDecision decides what to do with a tunnel under the hostmap lock. It also returns the tunnel to punch when
|
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) {
|
||||||
// we are not sending on it, for the caller to send once the lock is released.
|
|
||||||
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo, *HostInfo) {
|
|
||||||
// Read lock the main hostmap to order decisions based on tunnels being the primary tunnel
|
// Read lock the main hostmap to order decisions based on tunnels being the primary tunnel
|
||||||
cm.hostMap.RLock()
|
cm.hostMap.RLock()
|
||||||
defer cm.hostMap.RUnlock()
|
defer cm.hostMap.RUnlock()
|
||||||
@@ -338,17 +321,17 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
|||||||
hostinfo := cm.hostMap.Indexes[localIndex]
|
hostinfo := cm.hostMap.Indexes[localIndex]
|
||||||
if hostinfo == nil {
|
if hostinfo == nil {
|
||||||
cm.l.Debug("Not found in hostmap", "localIndex", localIndex)
|
cm.l.Debug("Not found in hostmap", "localIndex", localIndex)
|
||||||
return doNothing, nil, nil, nil
|
return doNothing, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if cm.isInvalidCertificate(now, hostinfo) {
|
if cm.isInvalidCertificate(now, hostinfo) {
|
||||||
return closeTunnel, hostinfo, nil, nil
|
return closeTunnel, hostinfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
|
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
|
||||||
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
|
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
|
||||||
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
|
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
|
||||||
return deleteTunnel, hostinfo, nil, nil
|
return deleteTunnel, hostinfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
|
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
|
||||||
@@ -385,10 +368,10 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
|||||||
|
|
||||||
if !outTraffic {
|
if !outTraffic {
|
||||||
// Send a punch packet to keep the NAT state alive
|
// Send a punch packet to keep the NAT state alive
|
||||||
return decision, hostinfo, primary, hostinfo
|
cm.punchy.SendPunch(hostinfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
return decision, hostinfo, primary, nil
|
return decision, hostinfo, primary
|
||||||
}
|
}
|
||||||
|
|
||||||
if hostinfo.isPendingDeletion() {
|
if hostinfo.isPendingDeletion() {
|
||||||
@@ -397,7 +380,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
|||||||
"tunnelCheck", m{"state": "dead", "method": "active"},
|
"tunnelCheck", m{"state": "dead", "method": "active"},
|
||||||
)
|
)
|
||||||
|
|
||||||
return deleteTunnel, hostinfo, nil, nil
|
return deleteTunnel, hostinfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
decision := doNothing
|
decision := doNothing
|
||||||
@@ -411,13 +394,14 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
|||||||
"primary", mainHostInfo,
|
"primary", mainHostInfo,
|
||||||
)
|
)
|
||||||
|
|
||||||
return closeTunnel, hostinfo, primary, nil
|
return closeTunnel, hostinfo, primary
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
|
// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
|
||||||
// Just maintain NAT state if configured to do so.
|
// Just maintain NAT state if configured to do so.
|
||||||
|
cm.punchy.SendPunch(hostinfo)
|
||||||
cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
|
cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
|
||||||
return doNothing, nil, nil, hostinfo
|
return doNothing, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// We aren't receiving traffic but we are sending it. The outbound
|
// We aren't receiving traffic but we are sending it. The outbound
|
||||||
@@ -443,7 +427,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
|||||||
|
|
||||||
hostinfo.setPendingDeletion(true)
|
hostinfo.setPendingDeletion(true)
|
||||||
cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
|
cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
|
||||||
return decision, hostinfo, nil, nil
|
return decision, hostinfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) {
|
func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) {
|
||||||
|
|||||||
+6
-247
@@ -3,18 +3,13 @@ package nebula
|
|||||||
import (
|
import (
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"io"
|
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/flynn/noise"
|
|
||||||
"github.com/rcrowley/go-metrics"
|
"github.com/rcrowley/go-metrics"
|
||||||
"github.com/slackhq/nebula/cert"
|
"github.com/slackhq/nebula/cert"
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
"github.com/slackhq/nebula/header"
|
|
||||||
"github.com/slackhq/nebula/noiseutil"
|
|
||||||
"github.com/slackhq/nebula/overlay/batch"
|
|
||||||
"github.com/slackhq/nebula/overlay/overlaytest"
|
"github.com/slackhq/nebula/overlay/overlaytest"
|
||||||
"github.com/slackhq/nebula/test"
|
"github.com/slackhq/nebula/test"
|
||||||
"github.com/slackhq/nebula/udp"
|
"github.com/slackhq/nebula/udp"
|
||||||
@@ -271,12 +266,12 @@ func Test_NewConnectionManager_CounterLimits(t *testing.T) {
|
|||||||
|
|
||||||
// Still below the reject limit, the tunnel stays up
|
// Still below the reject limit, the tunnel stays up
|
||||||
nc.In(hostinfo)
|
nc.In(hostinfo)
|
||||||
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
||||||
assert.Equal(t, tryRehandshake, decision)
|
assert.Equal(t, tryRehandshake, decision)
|
||||||
|
|
||||||
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
|
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
|
||||||
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
|
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
|
||||||
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
||||||
assert.Equal(t, deleteTunnel, decision)
|
assert.Equal(t, deleteTunnel, decision)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -338,14 +333,14 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
|
|||||||
assert.True(t, (hostinfo.state.Load()&stateIn != 0))
|
assert.True(t, (hostinfo.state.Load()&stateIn != 0))
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
||||||
assert.Equal(t, tryRehandshake, decision)
|
assert.Equal(t, tryRehandshake, decision)
|
||||||
assert.Equal(t, now, hostinfo.lastUsed)
|
assert.Equal(t, now, hostinfo.lastUsed)
|
||||||
assert.False(t, hostinfo.isPendingDeletion())
|
assert.False(t, hostinfo.isPendingDeletion())
|
||||||
assert.False(t, hostinfo.sentSinceCheck())
|
assert.False(t, hostinfo.sentSinceCheck())
|
||||||
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
|
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
|
||||||
|
|
||||||
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
|
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
|
||||||
assert.Equal(t, doNothing, decision)
|
assert.Equal(t, doNothing, decision)
|
||||||
assert.Equal(t, now, hostinfo.lastUsed)
|
assert.Equal(t, now, hostinfo.lastUsed)
|
||||||
assert.False(t, hostinfo.isPendingDeletion())
|
assert.False(t, hostinfo.isPendingDeletion())
|
||||||
@@ -353,7 +348,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
|
|||||||
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
|
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
|
||||||
|
|
||||||
// Do another traffic check tick, should still not be pending deletion
|
// Do another traffic check tick, should still not be pending deletion
|
||||||
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
|
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
|
||||||
assert.Equal(t, doNothing, decision)
|
assert.Equal(t, doNothing, decision)
|
||||||
assert.Equal(t, now, hostinfo.lastUsed)
|
assert.Equal(t, now, hostinfo.lastUsed)
|
||||||
assert.False(t, hostinfo.isPendingDeletion())
|
assert.False(t, hostinfo.isPendingDeletion())
|
||||||
@@ -363,7 +358,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
|
|||||||
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
|
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
|
||||||
|
|
||||||
// Finally advance beyond the inactivity timeout
|
// Finally advance beyond the inactivity timeout
|
||||||
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
|
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
|
||||||
assert.Equal(t, closeTunnel, decision)
|
assert.Equal(t, closeTunnel, decision)
|
||||||
assert.Equal(t, now, hostinfo.lastUsed)
|
assert.Equal(t, now, hostinfo.lastUsed)
|
||||||
assert.False(t, hostinfo.isPendingDeletion())
|
assert.False(t, hostinfo.isPendingDeletion())
|
||||||
@@ -583,239 +578,3 @@ func (d *dummyCert) MarshalJSON() ([]byte, error) {
|
|||||||
func (d *dummyCert) Copy() cert.Certificate {
|
func (d *dummyCert) Copy() cert.Certificate {
|
||||||
return d
|
return d
|
||||||
}
|
}
|
||||||
|
|
||||||
// newPunchTestConnectionManager is a connection manager punching through conn, holding the tunnel from newPunchTarget
|
|
||||||
func newPunchTestConnectionManager(t *testing.T, conn udp.Conn) (*connectionManager, *HostInfo) {
|
|
||||||
l := test.NewLogger()
|
|
||||||
hostMap := newHostMap(l)
|
|
||||||
hostMap.preferredRanges.Store(&[]netip.Prefix{})
|
|
||||||
|
|
||||||
lh := newTestLighthouse()
|
|
||||||
ifce := &Interface{
|
|
||||||
hostMap: hostMap,
|
|
||||||
inside: &overlaytest.NoopTun{},
|
|
||||||
outside: &udp.NoopConn{},
|
|
||||||
firewall: &Firewall{},
|
|
||||||
lightHouse: lh,
|
|
||||||
pki: &PKI{},
|
|
||||||
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
|
|
||||||
l: l,
|
|
||||||
}
|
|
||||||
ifce.pki.cs.Store(&CertState{
|
|
||||||
initiatingVersion: cert.Version1,
|
|
||||||
privateKey: []byte{},
|
|
||||||
v1Cert: &dummyCert{version: cert.Version1},
|
|
||||||
})
|
|
||||||
|
|
||||||
punchy := newPunchyForTest(t, map[string]any{"punch": true}, conn)
|
|
||||||
nc := newConnectionManagerFromConfig(l, config.NewC(l), hostMap, punchy)
|
|
||||||
nc.intf = ifce
|
|
||||||
ifce.connectionManager = nc
|
|
||||||
// Tunnel sends go through conn too, though without a key from withTestKey they never get as far as writing
|
|
||||||
ifce.writers = []udp.Conn{conn}
|
|
||||||
ifce.metricTxDropped = metrics.NilCounter{}
|
|
||||||
|
|
||||||
hostinfo := newPunchTarget()
|
|
||||||
hostinfo.localIndexId = 1099
|
|
||||||
hostinfo.remoteIndexId = 9901
|
|
||||||
hostinfo.ConnectionState = &ConnectionState{myCert: &dummyCert{version: cert.Version1}}
|
|
||||||
nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
|
|
||||||
return nc, hostinfo
|
|
||||||
}
|
|
||||||
|
|
||||||
// A tunnel we are not sending on gets punched whether or not it is receiving, the punch is the only thing that
|
|
||||||
// would notice its remote has become unreachable.
|
|
||||||
func Test_makeTrafficDecision_punchesWhenNotSending(t *testing.T) {
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
|
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
// Receiving but not sending
|
|
||||||
hostinfo.takeTraffic()
|
|
||||||
nc.In(hostinfo)
|
|
||||||
decision, _, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
|
||||||
assert.Equal(t, tryRehandshake, decision)
|
|
||||||
assert.Same(t, hostinfo, punch)
|
|
||||||
|
|
||||||
// Sending and receiving, our own traffic keeps the NAT state alive
|
|
||||||
nc.Out(hostinfo)
|
|
||||||
nc.In(hostinfo)
|
|
||||||
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
|
||||||
assert.Equal(t, tryRehandshake, decision)
|
|
||||||
assert.Nil(t, punch)
|
|
||||||
|
|
||||||
// Unused
|
|
||||||
decision, decided, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
|
||||||
assert.Equal(t, doNothing, decision)
|
|
||||||
assert.Nil(t, decided, "an unused tunnel has no relay usage to reset")
|
|
||||||
assert.Same(t, hostinfo, punch)
|
|
||||||
|
|
||||||
// Sending but not receiving gets a test packet instead, which finds an unreachable remote on its own
|
|
||||||
nc.Out(hostinfo)
|
|
||||||
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
|
|
||||||
assert.Equal(t, sendTestPacket, decision)
|
|
||||||
assert.Nil(t, punch)
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_sendPunch_probesWhenRemoteUnreachable(t *testing.T) {
|
|
||||||
t.Run("unreachable", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
|
||||||
|
|
||||||
nc.sendPunch(hostinfo)
|
|
||||||
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
|
|
||||||
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
|
|
||||||
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
|
|
||||||
"the probe should ask the lighthouse to have the peer punch us")
|
|
||||||
|
|
||||||
// Within the probe interval nothing more goes out, however many sends fail
|
|
||||||
nc.sendPunch(hostinfo)
|
|
||||||
nc.sendPunch(hostinfo)
|
|
||||||
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("delivered", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{}
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
|
||||||
|
|
||||||
nc.sendPunch(hostinfo)
|
|
||||||
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
|
|
||||||
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
|
|
||||||
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("other failure", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: io.ErrShortWrite}}
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
|
||||||
|
|
||||||
nc.sendPunch(hostinfo)
|
|
||||||
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "only a missing route says to look elsewhere")
|
|
||||||
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// A packet the data path had no route for names its tunnel in its header, the peer's index for it
|
|
||||||
func Test_sendUnreachable_findsTunnelFromHeader(t *testing.T) {
|
|
||||||
head := func(st header.MessageSubType, remoteIndex uint32) []byte {
|
|
||||||
return header.Encode(make([]byte, header.Len), header.Version, header.Message, st, remoteIndex, 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("direct", func(t *testing.T) {
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
|
|
||||||
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV6)
|
|
||||||
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
|
|
||||||
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("not its remote", func(t *testing.T) {
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
|
|
||||||
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV4)
|
|
||||||
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "a failure on a remote the tunnel is not using is stale")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("relayed", func(t *testing.T) {
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
|
|
||||||
// A relay header's index is the relay session's, it must not be taken for a tunnel's even when they collide
|
|
||||||
nc.intf.sendUnreachable(head(header.MessageRelay, hostinfo.remoteIndexId), punchTargetV6)
|
|
||||||
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
|
|
||||||
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("unknown tunnel", func(t *testing.T) {
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
|
|
||||||
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId+1), punchTargetV6)
|
|
||||||
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// withTestKey gives a tunnel a real key so its sends get encrypted and written instead of returning early
|
|
||||||
func withTestKey(hostinfo *HostInfo) {
|
|
||||||
cs := noise.NewCipherSuite(noise.DH25519, noise.CipherAESGCM, noise.HashSHA256)
|
|
||||||
hostinfo.ConnectionState.eKey = noiseutil.NewCipherState(noise.UnsafeNewCipherState(cs, [32]byte{}, 0), noise.CipherAESGCM)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The probe itself, end to end from a failed send: a test packet to every remote but the one we cannot reach, and a
|
|
||||||
// lighthouse query, through both places a send can fail.
|
|
||||||
func Test_remoteUnreachable_probesOtherRemotes(t *testing.T) {
|
|
||||||
t.Run("control send", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
|
||||||
withTestKey(hostinfo)
|
|
||||||
|
|
||||||
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
|
|
||||||
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV4}, conn.wrote, "the failed send, then the probe")
|
|
||||||
assert.Equal(t, []header.MessageType{header.Test, header.Test}, conn.types)
|
|
||||||
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("data path batch", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
|
||||||
withTestKey(hostinfo)
|
|
||||||
|
|
||||||
// Two of the tunnel's packets fail in one batch, they probe once
|
|
||||||
sb := batch.NewSendBatch(conn, 4, 4096)
|
|
||||||
for c := uint64(1); c <= 2; c++ {
|
|
||||||
pkt := header.Encode(sb.Reserve(64), header.Version, header.Message, header.MessageNone, hostinfo.remoteIndexId, c)
|
|
||||||
sb.Commit(append(pkt, make([]byte, 32)...), punchTargetV6)
|
|
||||||
}
|
|
||||||
nc.intf.flushSendBatch(sb, 0)
|
|
||||||
|
|
||||||
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6, punchTargetV4}, conn.wrote,
|
|
||||||
"the failed packets, then one probe")
|
|
||||||
assert.Equal(t, header.Test, conn.types[2])
|
|
||||||
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("closing", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
|
||||||
withTestKey(hostinfo)
|
|
||||||
|
|
||||||
nc.intf.SendMessageToHostInfo(header.CloseTunnel, 0, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
|
|
||||||
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a tunnel being torn down does not probe")
|
|
||||||
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("lighthouse", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
|
||||||
withTestKey(hostinfo)
|
|
||||||
nc.intf.lightHouse.amLighthouse = true
|
|
||||||
|
|
||||||
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
|
|
||||||
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a lighthouse does not roam its clients")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("nothing else to probe", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
|
||||||
nc, hostinfo := newPunchTestConnectionManager(t, conn)
|
|
||||||
withTestKey(hostinfo)
|
|
||||||
nc.intf.reQueryWait.Store(int64(time.Minute))
|
|
||||||
// Forget the IPv4 remote, all the tunnel knows is the one it cannot reach
|
|
||||||
hostinfo.remotes = NewRemoteList(hostinfo.vpnAddrs, nil)
|
|
||||||
hostinfo.remotes.LearnRemote(hostinfo.vpnAddrs[0], punchTargetV6)
|
|
||||||
|
|
||||||
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
|
|
||||||
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
|
|
||||||
"the lighthouse is the only way to learn of another remote")
|
|
||||||
|
|
||||||
// Past the probe interval, but the lighthouse is only asked again on the requery schedule
|
|
||||||
hostinfo.nextUnreachableProbe.Store(0)
|
|
||||||
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
|
|
||||||
assert.Empty(t, drainQueries(nc.intf.lightHouse))
|
|
||||||
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6}, conn.wrote)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// drainQueries returns every lighthouse query waiting to go out
|
|
||||||
func drainQueries(lh *LightHouse) []netip.Addr {
|
|
||||||
var queries []netip.Addr
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case addr := <-lh.queryChan:
|
|
||||||
queries = append(queries, addr)
|
|
||||||
default:
|
|
||||||
return queries
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -148,7 +148,7 @@ func (c *fakeConn) Rebind() error { c.rebinds++; ret
|
|||||||
func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil }
|
func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil }
|
||||||
func (c *fakeConn) ListenOut(_ udp.EncReader, _ func()) error { return nil }
|
func (c *fakeConn) ListenOut(_ udp.EncReader, _ func()) error { return nil }
|
||||||
func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
|
func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
|
||||||
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
|
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
|
||||||
return len(bufs), nil
|
return len(bufs), nil
|
||||||
}
|
}
|
||||||
func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {}
|
func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {}
|
||||||
|
|||||||
@@ -117,18 +117,6 @@ func (c *Control) SetUDPAddr(addr netip.AddrPort) {
|
|||||||
c.f.outside.(*udp.TesterConn).SetAddr(addr)
|
c.f.outside.(*udp.TesterConn).SetAddr(addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetUDPAddrFor returns the underlay address this node sends to dst from, which differs by family for a dual stack node
|
|
||||||
func (c *Control) GetUDPAddrFor(dst netip.AddrPort) netip.AddrPort {
|
|
||||||
return c.f.outside.(*udp.TesterConn).GetAddrFor(dst)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetDualStackUDPAddrs gives this node one underlay address per family, sending from whichever matches the
|
|
||||||
// destination. An invalid AddrPort leaves the node without that family, sends to it fail with ENETUNREACH. Register
|
|
||||||
// any address the node did not start on with the router as well or nothing will route back.
|
|
||||||
func (c *Control) SetDualStackUDPAddrs(v4, v6 netip.AddrPort) {
|
|
||||||
c.f.outside.(*udp.TesterConn).SetDualStackAddrs(v4, v6)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
|
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
|
||||||
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
|
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
|
||||||
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
|
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
|
||||||
|
|||||||
+49
-22
@@ -38,8 +38,8 @@ type dnsServer struct {
|
|||||||
|
|
||||||
serverMu sync.Mutex
|
serverMu sync.Mutex
|
||||||
server *dns.Server
|
server *dns.Server
|
||||||
// started is closed once `server` has finished binding (or after
|
// started is closed once `server` is serving (or after serve returns
|
||||||
// ListenAndServe returns on a bind failure). Stop waits on it before
|
// without ever starting). Stop waits on it before
|
||||||
// calling Shutdown to avoid the miekg/dns "server not started" race
|
// calling Shutdown to avoid the miekg/dns "server not started" race
|
||||||
// where a Shutdown that arrives before bind completes is silently
|
// where a Shutdown that arrives before bind completes is silently
|
||||||
// ignored, leaving the listener running forever.
|
// ignored, leaving the listener running forever.
|
||||||
@@ -84,7 +84,7 @@ func newDnsServerFromConfig(ctx context.Context, l *slog.Logger, pki *PKI, hostM
|
|||||||
}
|
}
|
||||||
|
|
||||||
// reload applies the latest config and reconciles the running state with it:
|
// reload applies the latest config and reconciles the running state with it:
|
||||||
// - enabled toggled on -> spawn a runner
|
// - enabled toggled on -> bind, then serve in the background
|
||||||
// - enabled toggled off -> stop the runner
|
// - enabled toggled off -> stop the runner
|
||||||
// - listen address changed (while running) -> restart on the new address
|
// - listen address changed (while running) -> restart on the new address
|
||||||
// - everything else -> no-op
|
// - everything else -> no-op
|
||||||
@@ -121,13 +121,20 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if !running {
|
if running && !sameAddr {
|
||||||
// Was disabled (or never started); bring it up now.
|
// Stop clears the slot before shutting down, otherwise the listen below can find the dying server and refuse
|
||||||
go d.Start()
|
|
||||||
} else if !sameAddr {
|
|
||||||
// Stop clears the slot before shutting down, otherwise the Start below can find the dying server and refuse
|
|
||||||
d.Stop()
|
d.Stop()
|
||||||
go d.Start()
|
running = false
|
||||||
|
}
|
||||||
|
if !running {
|
||||||
|
// Bind here rather than in the background, so a reload that can't listen says so
|
||||||
|
server, started, err := d.listen()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if server != nil {
|
||||||
|
go d.serve(server, started)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Refresh the self entry every enabled reload so cert renewals that change our name or VPN addresses are picked up.
|
// Refresh the self entry every enabled reload so cert renewals that change our name or VPN addresses are picked up.
|
||||||
@@ -156,30 +163,50 @@ func (d *dnsServer) shutdownServer(srv *dns.Server, started chan struct{}, reaso
|
|||||||
// Must be invoked after the tun device is active so that lighthouse.dns.host
|
// Must be invoked after the tun device is active so that lighthouse.dns.host
|
||||||
// may bind to a nebula IP.
|
// may bind to a nebula IP.
|
||||||
func (d *dnsServer) Start() {
|
func (d *dnsServer) Start() {
|
||||||
if !d.enabled.Load() {
|
server, started, err := d.listen()
|
||||||
|
if err != nil {
|
||||||
|
d.l.Warn("Failed to run the DNS responder", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if server != nil {
|
||||||
|
d.serve(server, started)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// listen binds the responder and claims the server slot for serve. It returns a nil server, and no error, when DNS is
|
||||||
|
// disabled or a server is already running.
|
||||||
|
func (d *dnsServer) listen() (*dns.Server, chan struct{}, error) {
|
||||||
|
if !d.enabled.Load() {
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
started := make(chan struct{})
|
started := make(chan struct{})
|
||||||
d.serverMu.Lock()
|
d.serverMu.Lock()
|
||||||
|
defer d.serverMu.Unlock()
|
||||||
// Re-check enabled under the lock, a disable that raced our check above snapshots the slot under it too.
|
// Re-check enabled under the lock, a disable that raced our check above snapshots the slot under it too.
|
||||||
// Two reloads in quick succession can both spawn a Start, the loser would orphan the live listener past Stop
|
// A reload and Control.Start can both get here, the loser would orphan the live listener past Stop
|
||||||
if d.ctx.Err() != nil || d.server != nil || !d.enabled.Load() {
|
if d.ctx.Err() != nil || d.server != nil || !d.enabled.Load() {
|
||||||
d.serverMu.Unlock()
|
return nil, nil, nil
|
||||||
return
|
}
|
||||||
|
pc, err := net.ListenPacket("udp", d.addr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
addr := d.addr
|
|
||||||
server := &dns.Server{
|
server := &dns.Server{
|
||||||
Addr: addr,
|
Addr: d.addr,
|
||||||
Net: "udp",
|
PacketConn: pc,
|
||||||
Handler: d.mux,
|
Handler: d.mux,
|
||||||
NotifyStartedFunc: func() { close(started) },
|
NotifyStartedFunc: func() { close(started) },
|
||||||
}
|
}
|
||||||
d.server = server
|
d.server = server
|
||||||
d.started = started
|
d.started = started
|
||||||
d.serverMu.Unlock()
|
d.l.Info("Starting DNS responder", "dnsListener", pc.LocalAddr())
|
||||||
|
return server, started, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Per-invocation ctx watcher. Exits when Start does, so we don't leak a
|
// serve answers queries on a server from listen until Stop or the context ends it.
|
||||||
|
func (d *dnsServer) serve(server *dns.Server, started chan struct{}) {
|
||||||
|
// Per-invocation ctx watcher. Exits when serve does, so we don't leak a
|
||||||
// watcher per reload-driven restart.
|
// watcher per reload-driven restart.
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
@@ -190,16 +217,16 @@ func (d *dnsServer) Start() {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
d.l.Info("Starting DNS responder", "dnsListener", addr)
|
err := server.ActivateAndServe()
|
||||||
err := server.ListenAndServe()
|
|
||||||
close(done)
|
close(done)
|
||||||
|
|
||||||
// If the listener never bound (bind error) NotifyStartedFunc never fires,
|
// If serving failed before it started NotifyStartedFunc never fires, so close started here to release any Stop
|
||||||
// so close started here to release any Stop caller waiting on it.
|
// caller waiting on it. ActivateAndServe doesn't close the socket on that path either
|
||||||
select {
|
select {
|
||||||
case <-started:
|
case <-started:
|
||||||
default:
|
default:
|
||||||
close(started)
|
close(started)
|
||||||
|
_ = server.PacketConn.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Release our slot, unless a reload already replaced us, so a dead listener can't block a future Start
|
// Release our slot, unless a reload already replaced us, so a dead listener can't block a future Start
|
||||||
|
|||||||
@@ -629,3 +629,22 @@ func TestDnsServer_Start_refusesWhenDisabledUnderLock(t *testing.T) {
|
|||||||
require.NoError(t, err, "an orphaned listener is still holding the port")
|
require.NoError(t, err, "an orphaned listener is still holding the port")
|
||||||
_ = pc.Close()
|
_ = pc.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A reload that can't bind says so, rather than finding out in the background
|
||||||
|
func TestDnsServer_reload_busyPortFails(t *testing.T) {
|
||||||
|
busy, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _ = busy.Close() })
|
||||||
|
_, port, err := net.SplitHostPort(busy.LocalAddr().String())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ds, c := newTestDnsServer(t)
|
||||||
|
setDnsConfig(c, "127.0.0.1", port, false, true)
|
||||||
|
require.NoError(t, ds.reload(c, true))
|
||||||
|
|
||||||
|
setDnsConfig(c, "127.0.0.1", port, true, true)
|
||||||
|
require.Error(t, ds.reload(c, false))
|
||||||
|
ds.serverMu.Lock()
|
||||||
|
assert.Nil(t, ds.server, "a failed bind kept the slot")
|
||||||
|
ds.serverMu.Unlock()
|
||||||
|
}
|
||||||
@@ -1,305 +0,0 @@
|
|||||||
//go:build e2e_testing
|
|
||||||
|
|
||||||
package e2e
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/slackhq/nebula"
|
|
||||||
"github.com/slackhq/nebula/cert"
|
|
||||||
"github.com/slackhq/nebula/cert_test"
|
|
||||||
"github.com/slackhq/nebula/e2e/router"
|
|
||||||
"github.com/slackhq/nebula/udp"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestDualStackRoamToV4Only measures how long a laptop's tunnels take to come back after it moves from a dual stack
|
|
||||||
// network, where its tunnels formed over IPv6, to one with no IPv6 service at all: the office to hotel wifi. Every
|
|
||||||
// peer still has an IPv4 address the laptop can reach, so nothing about the move should strand a tunnel.
|
|
||||||
//
|
|
||||||
// It is a measurement, not a pass/fail assertion, and it runs on the production connection manager timers.
|
|
||||||
//
|
|
||||||
// go test -tags e2e_testing -v -run TestDualStackRoamToV4Only ./e2e/
|
|
||||||
func TestDualStackRoamToV4Only(t *testing.T) {
|
|
||||||
for _, tc := range []dualStackRoamCase{
|
|
||||||
{name: "both send", meSends: true, themSends: true},
|
|
||||||
{name: "both send, rebind", meSends: true, themSends: true, rebind: true},
|
|
||||||
{name: "both send, two tunnels", meSends: true, themSends: true, simultaneous: true},
|
|
||||||
{name: "laptop sends", meSends: true},
|
|
||||||
{name: "peer sends", themSends: true},
|
|
||||||
{name: "peer sends, rebind", themSends: true, rebind: true},
|
|
||||||
{name: "peer sends, lh over v4", themSends: true, lhOverV4: true},
|
|
||||||
{name: "peer sends, punchy", themSends: true, punchy: true},
|
|
||||||
{name: "peer sends, punchy all", themSends: true, punchy: true, punchAll: true},
|
|
||||||
{name: "laptop sends, peer NAT", meSends: true, peerNAT: true},
|
|
||||||
{name: "laptop sends, peer NAT, lh v4", meSends: true, peerNAT: true, lhOverV4: true},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
fwd, rev := measureDualStackRoam(t, tc)
|
|
||||||
t.Logf("RESULT %-24s laptop->peer %-9v peer->laptop %-9v", tc.name, fmtRecovery(fwd), fmtRecovery(rev))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDualStackRoamRecovers holds the line TestDualStackRoamToV4Only measures: a tunnel whose remote became unreachable
|
|
||||||
// moves to the peer's other address instead of waiting out the connection manager to rehandshake, which takes 20s.
|
|
||||||
func TestDualStackRoamRecovers(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
dualStackRoamCase
|
|
||||||
// within bounds recovery in each direction that has traffic
|
|
||||||
within time.Duration
|
|
||||||
}{
|
|
||||||
// Measures 0.4s. Under the 5s connection manager check, so it fails if the data path stops probing.
|
|
||||||
{dualStackRoamCase{name: "both send", meSends: true, themSends: true}, 2 * time.Second},
|
|
||||||
// Measures 2.6s or 3.8s, depending on whether a probe beats the peer's punch. Half the 20s it used to take.
|
|
||||||
{dualStackRoamCase{name: "laptop sends, peer NAT", meSends: true, peerNAT: true}, 10 * time.Second},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
fwd, rev := measureDualStackRoam(t, tc.dualStackRoamCase)
|
|
||||||
if fwd == 0 || fwd > tc.within {
|
|
||||||
t.Errorf("laptop->peer recovered in %v, want within %v", fmtRecovery(fwd), tc.within)
|
|
||||||
}
|
|
||||||
if tc.themSends && (rev == 0 || rev > tc.within) {
|
|
||||||
t.Errorf("peer->laptop recovered in %v, want within %v", fmtRecovery(rev), tc.within)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type dualStackRoamCase struct {
|
|
||||||
name string
|
|
||||||
// meSends and themSends say who keeps traffic flowing after the move
|
|
||||||
meSends, themSends bool
|
|
||||||
// rebind fires RebindUDPServer on the move, as the darwin network change monitor would
|
|
||||||
rebind bool
|
|
||||||
// lhOverV4 forms the lighthouse tunnel over IPv4 at the office, so the move does not break it
|
|
||||||
lhOverV4 bool
|
|
||||||
// simultaneous has both sides initiate at once at the office, which can leave each side a different primary
|
|
||||||
// tunnel for the other
|
|
||||||
simultaneous bool
|
|
||||||
// punchy turns on punchy.punch everywhere, punchAll adds punchy.target_all_remotes
|
|
||||||
punchy, punchAll bool
|
|
||||||
// peerNAT puts the peer behind a NAT that, after the move, only lets in traffic from addresses the peer has sent to
|
|
||||||
peerNAT bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func fmtRecovery(d time.Duration) string {
|
|
||||||
if d == 0 {
|
|
||||||
return "never"
|
|
||||||
}
|
|
||||||
return d.Round(time.Millisecond).String()
|
|
||||||
}
|
|
||||||
|
|
||||||
type dualStackNode struct {
|
|
||||||
c *nebula.Control
|
|
||||||
vpnAddr netip.Addr
|
|
||||||
v4, v6 netip.AddrPort
|
|
||||||
adv atomic.Pointer[[]netip.Addr]
|
|
||||||
}
|
|
||||||
|
|
||||||
// move puts the node on a new network, advertising only what it can be reached on there
|
|
||||||
func (n *dualStackNode) move(v4, v6 netip.AddrPort) {
|
|
||||||
n.v4, n.v6 = v4, v6
|
|
||||||
n.c.SetDualStackUDPAddrs(v4, v6)
|
|
||||||
var adv []netip.Addr
|
|
||||||
for _, a := range []netip.AddrPort{v4, v6} {
|
|
||||||
if a.IsValid() {
|
|
||||||
adv = append(adv, a.Addr())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
n.adv.Store(&adv)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newDualStackNode(ca cert.Certificate, caKey []byte, name, vpnNet string, v4, v6 netip.AddrPort, overrides m) *dualStackNode {
|
|
||||||
c, vpnNets, _, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, name, vpnNet, v4, overrides)
|
|
||||||
n := &dualStackNode{c: c, vpnAddr: vpnNets[0].Addr()}
|
|
||||||
n.move(v4, v6)
|
|
||||||
c.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
|
|
||||||
return *n.adv.Load()
|
|
||||||
})
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
// measureDualStackRoam returns how long after the move the first packet made it laptop->peer and peer->laptop, zero
|
|
||||||
// for a direction that never recovered.
|
|
||||||
func measureDualStackRoam(t *testing.T, tc dualStackRoamCase) (time.Duration, time.Duration) {
|
|
||||||
t.Helper()
|
|
||||||
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
|
||||||
|
|
||||||
ap := netip.MustParseAddrPort
|
|
||||||
lhV4, lhV6 := ap("10.0.0.1:4242"), ap("[fd00::1]:4242")
|
|
||||||
lh := newDualStackNode(ca, caKey, "lh", "10.128.0.1/24", lhV4, lhV6, m{
|
|
||||||
"lighthouse": m{"am_lighthouse": true},
|
|
||||||
})
|
|
||||||
|
|
||||||
// Production timers rather than the 2s the harness defaults to, this is about how long a real laptop waits
|
|
||||||
peerCfg := m{
|
|
||||||
"lighthouse": m{
|
|
||||||
"hosts": []any{lh.vpnAddr.String()},
|
|
||||||
"interval": 10,
|
|
||||||
},
|
|
||||||
"static_host_map": m{
|
|
||||||
lh.vpnAddr.String(): []any{lhV4.String(), lhV6.String()},
|
|
||||||
},
|
|
||||||
"timers": m{
|
|
||||||
"connection_alive_interval": 5,
|
|
||||||
"pending_deletion_interval": 10,
|
|
||||||
},
|
|
||||||
"punchy": m{
|
|
||||||
"punch": tc.punchy,
|
|
||||||
"target_all_remotes": tc.punchAll,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
me := newDualStackNode(ca, caKey, "me", "10.128.0.2/24", ap("10.0.0.2:4242"), ap("[fd00::2]:4242"), peerCfg)
|
|
||||||
them := newDualStackNode(ca, caKey, "them", "10.128.0.3/24", ap("10.0.0.3:4242"), ap("[fd00::3]:4242"), peerCfg)
|
|
||||||
|
|
||||||
r := router.NewR(t, lh.c, me.c, them.c)
|
|
||||||
r.CancelFlowLogs()
|
|
||||||
for _, n := range []*dualStackNode{lh, me, them} {
|
|
||||||
r.AddRoute(n.v6.Addr(), n.v6.Port(), n.c)
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
lh.c.Stop()
|
|
||||||
me.c.Stop()
|
|
||||||
them.c.Stop()
|
|
||||||
}()
|
|
||||||
|
|
||||||
lh.c.Start()
|
|
||||||
me.c.Start()
|
|
||||||
them.c.Start()
|
|
||||||
|
|
||||||
// At the office IPv6 wins every race, so everything forms over it. The lighthouse tunnel optionally does not,
|
|
||||||
// for a static_host_map that only lists the lighthouse's IPv4 address.
|
|
||||||
officeV6 := func(p *udp.Packet, _ *nebula.Control) router.ExitType {
|
|
||||||
lhTraffic := p.To == lhV4 || p.To == lhV6 || p.From == lhV4 || p.From == lhV6
|
|
||||||
if p.To.Addr().Is4() == (tc.lhOverV4 && lhTraffic) {
|
|
||||||
return router.KeepRouting
|
|
||||||
}
|
|
||||||
return router.Drop
|
|
||||||
}
|
|
||||||
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
|
|
||||||
|
|
||||||
// One side at a time, so both sides hold the same single tunnel, unless the case wants them racing
|
|
||||||
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("establish")))
|
|
||||||
if !tc.simultaneous {
|
|
||||||
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
|
|
||||||
}
|
|
||||||
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("establish")))
|
|
||||||
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
|
|
||||||
if tc.simultaneous && me.c.GetHostmapIndexCount() < 3 {
|
|
||||||
// A tunnel to the lighthouse and two to the peer, or the race did not happen
|
|
||||||
t.Logf("simultaneous handshakes settled on a single tunnel this run")
|
|
||||||
}
|
|
||||||
drainTun(me.c)
|
|
||||||
drainTun(them.c)
|
|
||||||
|
|
||||||
for _, want := range []struct {
|
|
||||||
name string
|
|
||||||
c *nebula.Control
|
|
||||||
vpnAddr netip.Addr
|
|
||||||
v6 bool
|
|
||||||
}{
|
|
||||||
{"laptop->peer", me.c, them.vpnAddr, true},
|
|
||||||
{"peer->laptop", them.c, me.vpnAddr, true},
|
|
||||||
{"laptop->lh", me.c, lh.vpnAddr, !tc.lhOverV4},
|
|
||||||
} {
|
|
||||||
hi := want.c.GetHostInfoByVpnAddr(want.vpnAddr, false)
|
|
||||||
if hi == nil {
|
|
||||||
t.Fatalf("%s: no tunnel at the office", want.name)
|
|
||||||
}
|
|
||||||
if hi.CurrentRemote.Addr().Is6() != want.v6 {
|
|
||||||
t.Fatalf("%s: tunnel formed over %v, the scenario wants v6=%v", want.name, hi.CurrentRemote, want.v6)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Hotel wifi: a new NAT'd IPv4 address and no IPv6 at all. The office addresses are gone, nothing reaches them.
|
|
||||||
officeV4, officeV6Addr := me.v4, me.v6
|
|
||||||
hotel := ap("10.0.0.99:4242")
|
|
||||||
me.move(hotel, netip.AddrPort{})
|
|
||||||
r.AddRoute(hotel.Addr(), hotel.Port(), me.c)
|
|
||||||
|
|
||||||
start := time.Now()
|
|
||||||
if tc.rebind {
|
|
||||||
me.c.RebindUDPServer()
|
|
||||||
}
|
|
||||||
|
|
||||||
var fwd, rev atomic.Int64
|
|
||||||
stop := make(chan struct{})
|
|
||||||
defer close(stop)
|
|
||||||
watchTun(them.c, stop, start, &fwd)
|
|
||||||
watchTun(me.c, stop, start, &rev)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
tick := time.NewTicker(time.Millisecond * 200)
|
|
||||||
defer tick.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-stop:
|
|
||||||
return
|
|
||||||
case <-tick.C:
|
|
||||||
if tc.meSends {
|
|
||||||
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("retry")))
|
|
||||||
}
|
|
||||||
if tc.themSends {
|
|
||||||
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("retry")))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
// When only one side has traffic, the other direction recovering is a bonus we still report but do not wait for
|
|
||||||
done := func() bool {
|
|
||||||
return (!tc.meSends || fwd.Load() != 0) && (!tc.themSends || rev.Load() != 0)
|
|
||||||
}
|
|
||||||
// The peer's NAT has state for the lighthouse, which its own updates keep fresh, and for wherever it sends from here
|
|
||||||
natOpen := map[netip.AddrPort]bool{lhV4: true, lhV6: true}
|
|
||||||
isPeer := func(a netip.AddrPort) bool { return a == them.v4 || a == them.v6 }
|
|
||||||
|
|
||||||
r.RouteForAllExitFuncOrTimeout(time.Second*90, func(p *udp.Packet, _ *nebula.Control) router.ExitType {
|
|
||||||
if p.To == officeV4 || p.To == officeV6Addr {
|
|
||||||
return router.Drop
|
|
||||||
}
|
|
||||||
if tc.peerNAT {
|
|
||||||
if isPeer(p.From) {
|
|
||||||
natOpen[p.To] = true
|
|
||||||
} else if isPeer(p.To) && !natOpen[p.From] {
|
|
||||||
return router.Drop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if done() {
|
|
||||||
return router.ExitNow
|
|
||||||
}
|
|
||||||
return router.KeepRouting
|
|
||||||
})
|
|
||||||
|
|
||||||
return time.Duration(fwd.Load()), time.Duration(rev.Load())
|
|
||||||
}
|
|
||||||
|
|
||||||
func drainTun(c *nebula.Control) {
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-c.GetTunTxChan():
|
|
||||||
default:
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// watchTun records how long after start the first packet came out of c's tun, and keeps the tun drained after
|
|
||||||
func watchTun(c *nebula.Control, stop chan struct{}, start time.Time, first *atomic.Int64) {
|
|
||||||
go func() {
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-stop:
|
|
||||||
return
|
|
||||||
case <-c.GetTunTxChan():
|
|
||||||
first.CompareAndSwap(0, int64(time.Since(start)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
+13
-14
@@ -497,10 +497,11 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []
|
|||||||
// Nope, lets push the sender along
|
// Nope, lets push the sender along
|
||||||
case p := <-udpTx:
|
case p := <-udpTx:
|
||||||
r.Lock()
|
r.Lock()
|
||||||
c := r.getControl(p)
|
a := sender.GetUDPAddr()
|
||||||
|
c := r.getControl(a, p.To, p)
|
||||||
if c == nil {
|
if c == nil {
|
||||||
r.Unlock()
|
r.Unlock()
|
||||||
panic("No control for udp tx " + p.To.String())
|
panic("No control for udp tx " + a.String())
|
||||||
}
|
}
|
||||||
fp := r.unlockedInjectFlow(sender, c, p, false)
|
fp := r.unlockedInjectFlow(sender, c, p, false)
|
||||||
c.InjectUDPPacket(p) // copies internally; original is ours to release
|
c.InjectUDPPacket(p) // copies internally; original is ours to release
|
||||||
@@ -621,7 +622,8 @@ func (r *R) stopFanInWorkers() {
|
|||||||
func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
|
func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
|
||||||
r.Lock()
|
r.Lock()
|
||||||
defer r.Unlock()
|
defer r.Unlock()
|
||||||
c := r.getControl(p)
|
a := from.GetUDPAddr()
|
||||||
|
c := r.getControl(a, p.To, p)
|
||||||
if c == nil {
|
if c == nil {
|
||||||
panic(fmt.Sprintf("No control for udp tx %s", p.To))
|
panic(fmt.Sprintf("No control for udp tx %s", p.To))
|
||||||
}
|
}
|
||||||
@@ -669,7 +671,7 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
receiver := r.getControl(p)
|
receiver := r.getControl(sender.GetUDPAddr(), p.To, p)
|
||||||
if receiver == nil {
|
if receiver == nil {
|
||||||
r.Unlock()
|
r.Unlock()
|
||||||
panic("Can't RouteExitFunc for host: " + p.To.String())
|
panic("Can't RouteExitFunc for host: " + p.To.String())
|
||||||
@@ -765,7 +767,7 @@ func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc)
|
|||||||
|
|
||||||
r.Lock()
|
r.Lock()
|
||||||
p := rx.Interface().(*udp.Packet)
|
p := rx.Interface().(*udp.Packet)
|
||||||
receiver := r.getControl(p)
|
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
|
||||||
if receiver == nil {
|
if receiver == nil {
|
||||||
r.Unlock()
|
r.Unlock()
|
||||||
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
|
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
|
||||||
@@ -874,7 +876,7 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
|
|||||||
r.Lock()
|
r.Lock()
|
||||||
|
|
||||||
p := rx.Interface().(*udp.Packet)
|
p := rx.Interface().(*udp.Packet)
|
||||||
receiver := r.getControl(p)
|
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
|
||||||
if receiver == nil {
|
if receiver == nil {
|
||||||
r.Unlock()
|
r.Unlock()
|
||||||
panic("Can't RouteForAllExitFunc for host: " + p.To.String())
|
panic("Can't RouteForAllExitFunc for host: " + p.To.String())
|
||||||
@@ -937,7 +939,7 @@ func (r *R) FlushAll() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
for {
|
for {
|
||||||
_, rx, ok := reflect.Select(sc)
|
x, rx, ok := reflect.Select(sc)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -945,7 +947,7 @@ func (r *R) FlushAll() {
|
|||||||
|
|
||||||
p := rx.Interface().(*udp.Packet)
|
p := rx.Interface().(*udp.Packet)
|
||||||
|
|
||||||
receiver := r.getControl(p)
|
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
|
||||||
if receiver == nil {
|
if receiver == nil {
|
||||||
r.Unlock()
|
r.Unlock()
|
||||||
panic("Can't FlushAll for host: " + p.To.String())
|
panic("Can't FlushAll for host: " + p.To.String())
|
||||||
@@ -956,19 +958,16 @@ func (r *R) FlushAll() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// getControl performs or seeds NAT translation and returns the control for p.To, p from fields may change
|
// getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change
|
||||||
// This is an internal router function, the caller must hold the lock
|
// This is an internal router function, the caller must hold the lock
|
||||||
func (r *R) getControl(p *udp.Packet) *nebula.Control {
|
func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control {
|
||||||
// Key on the address the packet actually left from, a dual stack sender has one per family
|
|
||||||
fromAddr, toAddr := p.From, p.To
|
|
||||||
if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
|
if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
|
||||||
p.From = newAddr
|
p.From = newAddr
|
||||||
}
|
}
|
||||||
|
|
||||||
c, ok := r.inNat[toAddr]
|
c, ok := r.inNat[toAddr]
|
||||||
if ok {
|
if ok {
|
||||||
// Key the return path on the address the receiver will answer from, which is per family for a dual stack node
|
r.outNat[outNatKey{from: c.GetUDPAddr(), to: fromAddr}] = toAddr
|
||||||
r.outNat[outNatKey{from: c.GetUDPAddrFor(fromAddr), to: fromAddr}] = toAddr
|
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -331,9 +331,6 @@ tun:
|
|||||||
# `mtu`: will default to tun mtu if this option is not specified
|
# `mtu`: will default to tun mtu if this option is not specified
|
||||||
# `metric`: will default to 0 if this option is not specified
|
# `metric`: will default to 0 if this option is not specified
|
||||||
# `install`: will default to true, controls whether this route is installed in the systems routing table.
|
# `install`: will default to true, controls whether this route is installed in the systems routing table.
|
||||||
# On macOS, a route whose destination another interface already routes, like the local network, is installed once
|
|
||||||
# that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another
|
|
||||||
# interface routes its destination and lets it go.
|
|
||||||
# This setting is reloadable.
|
# This setting is reloadable.
|
||||||
unsafe_routes:
|
unsafe_routes:
|
||||||
#- route: 172.16.1.0/24
|
#- route: 172.16.1.0/24
|
||||||
|
|||||||
@@ -66,9 +66,6 @@ type Firewall struct {
|
|||||||
rules string
|
rules string
|
||||||
rulesVersion uint16
|
rulesVersion uint16
|
||||||
|
|
||||||
// source is the rendered firewall config this firewall was built from, the cert networks are kept above
|
|
||||||
source string
|
|
||||||
|
|
||||||
defaultLocalCIDRAny bool
|
defaultLocalCIDRAny bool
|
||||||
incomingMetrics firewallMetrics
|
incomingMetrics firewallMetrics
|
||||||
outgoingMetrics firewallMetrics
|
outgoingMetrics firewallMetrics
|
||||||
@@ -249,7 +246,6 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
fw.source = c.Render("firewall")
|
|
||||||
return fw, nil
|
return fw, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+1
-11
@@ -69,7 +69,6 @@ type HandshakeManager struct {
|
|||||||
messageMetrics *MessageMetrics
|
messageMetrics *MessageMetrics
|
||||||
metricInitiated metrics.Counter
|
metricInitiated metrics.Counter
|
||||||
metricTimedOut metrics.Counter
|
metricTimedOut metrics.Counter
|
||||||
metricRejectedBlocklisted metrics.Counter
|
|
||||||
f *Interface
|
f *Interface
|
||||||
l *slog.Logger
|
l *slog.Logger
|
||||||
|
|
||||||
@@ -130,7 +129,6 @@ func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *Light
|
|||||||
messageMetrics: config.messageMetrics,
|
messageMetrics: config.messageMetrics,
|
||||||
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
|
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
|
||||||
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
|
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
|
||||||
metricRejectedBlocklisted: metrics.GetOrRegisterCounter("handshake_manager.rejected.blocklisted", nil),
|
|
||||||
l: l,
|
l: l,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -719,12 +717,7 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
|
|||||||
|
|
||||||
response, result, err := machine.ProcessPacket(nil, packet)
|
response, result, err := machine.ProcessPacket(nil, packet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
level := slog.LevelError
|
f.l.Error("Failed to process handshake packet", "from", via, "error", err)
|
||||||
if errors.Is(err, cert.ErrBlockListed) {
|
|
||||||
level = slog.LevelDebug
|
|
||||||
hm.metricRejectedBlocklisted.Inc(1)
|
|
||||||
}
|
|
||||||
f.l.Log(context.Background(), level, "Failed to process handshake packet", "from", via, "error", err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -856,9 +849,6 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
|
|||||||
|
|
||||||
response, result, err := machine.ProcessPacket(nil, packet)
|
response, result, err := machine.ProcessPacket(nil, packet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, cert.ErrBlockListed) {
|
|
||||||
hm.metricRejectedBlocklisted.Inc(1)
|
|
||||||
}
|
|
||||||
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
|
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
|
||||||
if machine.Failed() {
|
if machine.Failed() {
|
||||||
f.l.Warn("Failed to process handshake packet, abandoning",
|
f.l.Warn("Failed to process handshake packet, abandoning",
|
||||||
|
|||||||
@@ -1,17 +1,12 @@
|
|||||||
package nebula
|
package nebula
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"log/slog"
|
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/flynn/noise"
|
|
||||||
"github.com/gaissmai/bart"
|
"github.com/gaissmai/bart"
|
||||||
"github.com/rcrowley/go-metrics"
|
|
||||||
"github.com/slackhq/nebula/cert"
|
"github.com/slackhq/nebula/cert"
|
||||||
ct "github.com/slackhq/nebula/cert_test"
|
|
||||||
"github.com/slackhq/nebula/handshake"
|
|
||||||
"github.com/slackhq/nebula/header"
|
"github.com/slackhq/nebula/header"
|
||||||
"github.com/slackhq/nebula/test"
|
"github.com/slackhq/nebula/test"
|
||||||
"github.com/slackhq/nebula/udp"
|
"github.com/slackhq/nebula/udp"
|
||||||
@@ -240,89 +235,6 @@ func TestHandleIncomingDispatch(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBlocklistedHandshakeLogLevelAndCounter(t *testing.T) {
|
|
||||||
ca, _, caKey, _ := ct.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
|
|
||||||
pool := ct.NewTestCAPool(ca)
|
|
||||||
|
|
||||||
newCredential := func(name, network string) (cert.Certificate, *handshake.Credential) {
|
|
||||||
c, _, rawKey, _ := ct.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey,
|
|
||||||
name, ca.NotBefore(), ca.NotAfter(), []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
|
|
||||||
privateKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey)
|
|
||||||
require.NoError(t, err)
|
|
||||||
certBytes, err := c.MarshalForHandshakes()
|
|
||||||
require.NoError(t, err)
|
|
||||||
cipherSuite := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
|
|
||||||
return c, handshake.NewCredential(c, certBytes, privateKey, cipherSuite)
|
|
||||||
}
|
|
||||||
|
|
||||||
initiatorCert, initiatorCredential := newCredential("initiator", "10.0.0.1/24")
|
|
||||||
responderCert, responderCredential := newCredential("responder", "10.0.0.2/24")
|
|
||||||
fingerprint, err := initiatorCert.Fingerprint()
|
|
||||||
require.NoError(t, err)
|
|
||||||
pool.BlocklistFingerprint(fingerprint)
|
|
||||||
|
|
||||||
initiator, err := handshake.NewMachine(cert.Version2,
|
|
||||||
func(cert.Version) *handshake.Credential { return initiatorCredential },
|
|
||||||
func(c cert.Certificate) (*cert.CachedCertificate, error) {
|
|
||||||
return pool.VerifyCertificate(time.Now(), c)
|
|
||||||
},
|
|
||||||
func() (uint32, error) { return 1, nil }, true, header.HandshakeIXPSK0)
|
|
||||||
require.NoError(t, err)
|
|
||||||
packet, err := initiator.Initiate(nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
handler := &capturingHandler{}
|
|
||||||
logger := slog.New(handler)
|
|
||||||
pki := &PKI{}
|
|
||||||
pki.cs.Store(&CertState{initiatingVersion: cert.Version2, v2Credential: responderCredential})
|
|
||||||
pki.caPool.Store(pool)
|
|
||||||
hm := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
|
|
||||||
hm.metricRejectedBlocklisted = metrics.NewCounter()
|
|
||||||
hm.f = &Interface{pki: pki, l: logger}
|
|
||||||
via := ViaSender{UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242")}
|
|
||||||
|
|
||||||
hm.beginHandshake(via, packet, nil)
|
|
||||||
entry := findEntry(t, handler.entries, "Failed to process handshake packet")
|
|
||||||
assert.Equal(t, slog.LevelDebug, entry.Level)
|
|
||||||
require.ErrorIs(t, entry.Attrs["error"].(error), cert.ErrBlockListed)
|
|
||||||
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
|
|
||||||
|
|
||||||
handler.entries = nil
|
|
||||||
hm.beginHandshake(via, nil, nil)
|
|
||||||
entry = findEntry(t, handler.entries, "Failed to process handshake packet")
|
|
||||||
assert.Equal(t, slog.LevelError, entry.Level)
|
|
||||||
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
|
|
||||||
|
|
||||||
// The initiator sees the same rejection in continueHandshake, after a
|
|
||||||
// responder has accepted its first packet and sent a valid reply.
|
|
||||||
cleanPool := ct.NewTestCAPool(ca)
|
|
||||||
responder, err := handshake.NewMachine(cert.Version2,
|
|
||||||
func(cert.Version) *handshake.Credential { return responderCredential },
|
|
||||||
func(c cert.Certificate) (*cert.CachedCertificate, error) {
|
|
||||||
return cleanPool.VerifyCertificate(time.Now(), c)
|
|
||||||
},
|
|
||||||
func() (uint32, error) { return 2, nil }, false, header.HandshakeIXPSK0)
|
|
||||||
require.NoError(t, err)
|
|
||||||
response, _, err := responder.ProcessPacket(nil, packet)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
responderFingerprint, err := responderCert.Fingerprint()
|
|
||||||
require.NoError(t, err)
|
|
||||||
pool.BlocklistFingerprint(responderFingerprint)
|
|
||||||
|
|
||||||
initiatingHM := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
|
|
||||||
initiatingHM.metricRejectedBlocklisted = metrics.NewCounter()
|
|
||||||
initiatingHM.f = &Interface{l: logger}
|
|
||||||
hh := &HandshakeHostInfo{
|
|
||||||
machine: initiator,
|
|
||||||
hostinfo: &HostInfo{localIndexId: 1, vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}},
|
|
||||||
}
|
|
||||||
initiatingHM.indexes[1] = hh
|
|
||||||
via.IsRelayed = true // skip the remote allow list before certificate validation
|
|
||||||
initiatingHM.continueHandshake(via, hh, response)
|
|
||||||
assert.Equal(t, int64(1), initiatingHM.metricRejectedBlocklisted.Count())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
|
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
|
||||||
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
|
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
|
||||||
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
|
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
|
||||||
|
|||||||
-62
@@ -264,9 +264,6 @@ type HostInfo struct {
|
|||||||
// This is used to limit lighthouse re-queries in chatty clients
|
// This is used to limit lighthouse re-queries in chatty clients
|
||||||
nextLHQuery atomic.Int64
|
nextLHQuery atomic.Int64
|
||||||
|
|
||||||
// nextUnreachableProbe is the earliest we will probe our other remotes again after failing to reach the current one
|
|
||||||
nextUnreachableProbe atomic.Int64
|
|
||||||
|
|
||||||
// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
|
// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
|
||||||
// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
|
// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
|
||||||
// This is used to avoid an attack where a handshake packet is replayed after some time
|
// This is used to avoid an attack where a handshake packet is replayed after some time
|
||||||
@@ -769,65 +766,6 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// unreachableProbeInterval bounds how often a tunnel whose remote we cannot reach probes its other remotes
|
|
||||||
const unreachableProbeInterval = time.Second
|
|
||||||
|
|
||||||
// remoteUnreachable is called when a send to our current remote failed because we have no route to it, typically
|
|
||||||
// after moving to a network without that address family. Sends a test packet to every other remote we know for this
|
|
||||||
// host: the first one that arrives roams the peer over to wherever we are now, and its reply roams us onto that path.
|
|
||||||
//
|
|
||||||
// A peer behind a NAT drops a test from an address it has never sent to, so every probe also asks the lighthouse
|
|
||||||
// about the peer. The lighthouse tells the peer where we are now, the peer punches us, and the next probe gets in.
|
|
||||||
func (i *HostInfo) remoteUnreachable(ifce *Interface, dead netip.AddrPort) {
|
|
||||||
if i.remotes == nil || i.ConnectionState == nil || i.GetRemote() != dead {
|
|
||||||
// Nothing to probe with, or we already moved on from the remote that failed
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if ifce.lightHouse == nil || ifce.lightHouse.amLighthouse {
|
|
||||||
// A lighthouse hands out the address it reaches a client on, it must not go roaming its clients itself
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now().UnixNano()
|
|
||||||
next := i.nextUnreachableProbe.Load()
|
|
||||||
if now < next || !i.nextUnreachableProbe.CompareAndSwap(next, now+int64(unreachableProbeInterval)) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
nb := make([]byte, 12)
|
|
||||||
out := make([]byte, mtu)
|
|
||||||
probed := 0
|
|
||||||
i.remotes.ForEach(ifce.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) {
|
|
||||||
if addr == dead {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
probed++
|
|
||||||
ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), nb, out)
|
|
||||||
})
|
|
||||||
|
|
||||||
if ifce.l.Enabled(context.Background(), slog.LevelDebug) {
|
|
||||||
i.logger(ifce.l).Debug("Remote unreachable, probing other remotes",
|
|
||||||
"udpAddr", dead,
|
|
||||||
"probed", probed,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if probed == 0 {
|
|
||||||
// With nothing else to try, the lighthouse is our only way to learn of another remote. The peer may not have
|
|
||||||
// one, so ask on the usual requery schedule rather than every probe for as long as the tunnel sits here.
|
|
||||||
if now < i.nextLHQuery.Load() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
i.nextLHQuery.Store(now + ifce.reQueryWait.Load())
|
|
||||||
}
|
|
||||||
|
|
||||||
// With other remotes to probe we ask on every probe, without backing off. A repeat re-sends the peer its punch
|
|
||||||
// notification in case the last one or its punches were lost. If the tunnel stays up on inbound traffic while
|
|
||||||
// our sends to its remote keep failing (only one direction broken), this continues once a second for as long as
|
|
||||||
// that lasts. That is a bounded cost for a rare state, and the probe that would clear it has to keep running.
|
|
||||||
ifce.lightHouse.QueryServer(i.vpnAddrs[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
// Bits within HostInfo.state, everything above stateEpochShift is the epoch
|
// Bits within HostInfo.state, everything above stateEpochShift is the epoch
|
||||||
const (
|
const (
|
||||||
stateIn uint32 = 1 << iota
|
stateIn uint32 = 1 << iota
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import (
|
|||||||
"github.com/slackhq/nebula/overlay/batch"
|
"github.com/slackhq/nebula/overlay/batch"
|
||||||
"github.com/slackhq/nebula/overlay/tio"
|
"github.com/slackhq/nebula/overlay/tio"
|
||||||
"github.com/slackhq/nebula/routing"
|
"github.com/slackhq/nebula/routing"
|
||||||
"github.com/slackhq/nebula/udp"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
|
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
|
||||||
@@ -612,10 +611,6 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
|
|||||||
"udpAddr", hr,
|
"udpAddr", hr,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
// A closing tunnel is about to be torn down, there is no point looking for another path for it
|
|
||||||
if t != header.CloseTunnel && udp.IsUnreachable(err) {
|
|
||||||
hostinfo.remoteUnreachable(f, hr)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Try to send via a relay
|
// Try to send via a relay
|
||||||
|
|||||||
+6
-47
@@ -1,7 +1,6 @@
|
|||||||
package nebula
|
package nebula
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"crypto/fips140"
|
"crypto/fips140"
|
||||||
"errors"
|
"errors"
|
||||||
@@ -496,8 +495,7 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
|
|||||||
|
|
||||||
// flushSendBatch drains sb to the underlay and accounts for anything it could not deliver. A shortfall means
|
// flushSendBatch drains sb to the underlay and accounts for anything it could not deliver. A shortfall means
|
||||||
// specific destinations were undeliverable (a stale remote, a reject rule), which the backend logs per peer at
|
// specific destinations were undeliverable (a stale remote, a reject rule), which the backend logs per peer at
|
||||||
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch. Tunnels whose remote
|
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch.
|
||||||
// we no longer have a route to go looking for another path.
|
|
||||||
func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
|
func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
|
||||||
queued := sb.Len()
|
queued := sb.Len()
|
||||||
written, err := sb.Flush()
|
written, err := sb.Flush()
|
||||||
@@ -507,44 +505,6 @@ func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
|
|||||||
if dropped := queued - written; dropped > 0 {
|
if dropped := queued - written; dropped > 0 {
|
||||||
f.metricTxDropped.Inc(int64(dropped))
|
f.metricTxDropped.Inc(int64(dropped))
|
||||||
}
|
}
|
||||||
|
|
||||||
if f.lightHouse == nil || f.lightHouse.amLighthouse {
|
|
||||||
// Lighthouses never probe, see remoteUnreachable, so skip finding the tunnels
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// A tunnel's packets tend to fail together, so skip straight past a repeat of the one before. The first 8 bytes
|
|
||||||
// of a header say whose packet it is, the counter after them changes with every packet.
|
|
||||||
var last batch.Failure
|
|
||||||
for i, fail := range sb.Failures() {
|
|
||||||
if i > 0 && fail.Dst == last.Dst && bytes.Equal(fail.Head[:8], last.Head[:8]) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
last = fail
|
|
||||||
|
|
||||||
if udp.IsUnreachable(fail.Err) {
|
|
||||||
f.sendUnreachable(fail.Head[:], fail.Dst)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendUnreachable finds the tunnel a packet we had no route for was sent on, from the start of the packet, and has it
|
|
||||||
// look for another path to its peer.
|
|
||||||
func (f *Interface) sendUnreachable(head []byte, dst netip.AddrPort) {
|
|
||||||
var h header.H
|
|
||||||
if h.Parse(head) != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if h.Type == header.Message && h.Subtype == header.MessageRelay {
|
|
||||||
// TODO: a relayed packet's header carries the relay session's index, not the relay tunnel's, so it does not
|
|
||||||
// lead us to the tunnel whose remote failed. The relay tunnel's own traffic finds out soon enough.
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if hostinfo := f.hostMap.QueryReverseIndex(h.RemoteIndex); hostinfo != nil {
|
|
||||||
hostinfo.remoteUnreachable(f, dst)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
|
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
|
||||||
@@ -577,12 +537,11 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
|
|||||||
curCert = cs.getCertificate(cert.Version1)
|
curCert = cs.getCertificate(cert.Version1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compare with what the running firewall was built from, not the last config, so a firewall that failed to build
|
// The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction.
|
||||||
// is tried again. The cert is the one this reload brought, PKI registers its reload callback before this one
|
// Check to see if that set has changed, and if so, rebuild the firewall.
|
||||||
src := c.Render("firewall")
|
certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks)
|
||||||
if src != "" && src == f.firewall.source && curCert != nil &&
|
|
||||||
slices.Equal(curCert.Networks(), f.firewall.assignedNetworks) &&
|
if !c.HasChanged("firewall") && !certUnsafeChanged {
|
||||||
slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) {
|
|
||||||
f.l.Debug("No firewall config change detected")
|
f.l.Debug("No firewall config change detected")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -135,63 +135,8 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
|
|||||||
f := &Interface{pki: pki, firewall: fw, l: l}
|
f := &Interface{pki: pki, firewall: fw, l: l}
|
||||||
cfg.RegisterReloadCallback(f.reloadFirewall)
|
cfg.RegisterReloadCallback(f.reloadFirewall)
|
||||||
|
|
||||||
// The running firewall still isn't built from what the config asks for, so the same reload fails again
|
|
||||||
for range 2 {
|
|
||||||
res := cfg.ReloadConfigString(rules("nope"))
|
res := cfg.ReloadConfigString(rules("nope"))
|
||||||
require.NoError(t, res.Err)
|
require.NoError(t, res.Err)
|
||||||
assert.True(t, res.Failed)
|
assert.True(t, res.Failed)
|
||||||
assert.Same(t, fw, f.firewall)
|
assert.Same(t, fw, f.firewall)
|
||||||
}
|
|
||||||
|
|
||||||
res := cfg.ReloadConfigString(rules("tcp"))
|
|
||||||
require.NoError(t, res.Err)
|
|
||||||
assert.False(t, res.Failed)
|
|
||||||
assert.NotSame(t, fw, f.firewall)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Every input the firewall is built from rebuilds it when it changes, and nothing else does
|
|
||||||
func TestReloadFirewall_RebuildsOnAnyInput(t *testing.T) {
|
|
||||||
l := test.NewLogger()
|
|
||||||
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
|
|
||||||
base := "firewall:\n inbound:\n - port: any\n proto: any\n host: any\n"
|
|
||||||
setup := func() (*config.C, *Interface, *PKI) {
|
|
||||||
pki := &PKI{}
|
|
||||||
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks}, initiatingVersion: cert.Version2})
|
|
||||||
cfg := config.NewC(l)
|
|
||||||
require.NoError(t, cfg.LoadString(base))
|
|
||||||
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
|
|
||||||
require.NoError(t, err)
|
|
||||||
f := &Interface{pki: pki, firewall: fw, l: l}
|
|
||||||
cfg.RegisterReloadCallback(f.reloadFirewall)
|
|
||||||
return cfg, f, pki
|
|
||||||
}
|
|
||||||
setCert := func(pki *PKI, networks, unsafe []netip.Prefix) {
|
|
||||||
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks, unsafeNetworks: unsafe}, initiatingVersion: cert.Version2})
|
|
||||||
}
|
|
||||||
|
|
||||||
cases := map[string]struct {
|
|
||||||
cfg string
|
|
||||||
cert func(*PKI)
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
"nothing": {base + "punchy:\n punch: true\n", nil, false},
|
|
||||||
"rules": {base + " outbound:\n - port: any\n proto: any\n host: any\n", nil, true},
|
|
||||||
"inbound_action": {base + " inbound_action: reject\n", nil, true},
|
|
||||||
"conntrack": {base + " conntrack:\n tcp_timeout: 1m\n", nil, true},
|
|
||||||
"networks": {base, func(p *PKI) { setCert(p, []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil) }, true},
|
|
||||||
"unsafe_networks": {base, func(p *PKI) { setCert(p, networks, []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}) }, true},
|
|
||||||
}
|
|
||||||
for name, tc := range cases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
cfg, f, pki := setup()
|
|
||||||
old := f.firewall
|
|
||||||
if tc.cert != nil {
|
|
||||||
tc.cert(pki)
|
|
||||||
}
|
|
||||||
res := cfg.ReloadConfigString(tc.cfg)
|
|
||||||
require.NoError(t, res.Err)
|
|
||||||
assert.False(t, res.Failed)
|
|
||||||
assert.Equal(t, tc.want, f.firewall != old)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -1,27 +1,12 @@
|
|||||||
package batch
|
package batch
|
||||||
|
|
||||||
import (
|
import "net/netip"
|
||||||
"net/netip"
|
|
||||||
|
|
||||||
"github.com/slackhq/nebula/header"
|
|
||||||
)
|
|
||||||
|
|
||||||
const SendBatchCap = 128
|
const SendBatchCap = 128
|
||||||
|
|
||||||
// FailureHeadLen is how much of a refused packet a Failure keeps, nebula's header, which says whose packet it was
|
|
||||||
const FailureHeadLen = header.Len
|
|
||||||
|
|
||||||
// batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush.
|
// batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush.
|
||||||
type batchWriter interface {
|
type batchWriter interface {
|
||||||
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
|
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
|
||||||
}
|
|
||||||
|
|
||||||
// Failure is a packet the underlay would not send
|
|
||||||
type Failure struct {
|
|
||||||
// Head is the start of the packet, zero padded if the packet was shorter
|
|
||||||
Head [FailureHeadLen]byte
|
|
||||||
Dst netip.AddrPort
|
|
||||||
Err error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
|
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
|
||||||
@@ -32,28 +17,16 @@ type SendBatch struct {
|
|||||||
bufs [][]byte
|
bufs [][]byte
|
||||||
dsts []netip.AddrPort
|
dsts []netip.AddrPort
|
||||||
arena *Arena
|
arena *Arena
|
||||||
|
|
||||||
failures []Failure
|
|
||||||
// recordFailure is bound once so handing it to WriteBatch does not allocate on every Flush
|
|
||||||
recordFailure func(i int, err error)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots
|
// NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots
|
||||||
func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch {
|
func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch {
|
||||||
b := &SendBatch{
|
return &SendBatch{
|
||||||
out: out,
|
out: out,
|
||||||
bufs: make([][]byte, 0, batchCap),
|
bufs: make([][]byte, 0, batchCap),
|
||||||
dsts: make([]netip.AddrPort, 0, batchCap),
|
dsts: make([]netip.AddrPort, 0, batchCap),
|
||||||
arena: NewArena(arenaSize),
|
arena: NewArena(arenaSize),
|
||||||
}
|
}
|
||||||
b.recordFailure = b.failed
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *SendBatch) failed(i int, err error) {
|
|
||||||
f := Failure{Dst: b.dsts[i], Err: err}
|
|
||||||
copy(f.Head[:], b.bufs[i])
|
|
||||||
b.failures = append(b.failures, f)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *SendBatch) Reserve(sz int) []byte {
|
func (b *SendBatch) Reserve(sz int) []byte {
|
||||||
@@ -71,15 +44,12 @@ func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations
|
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations
|
||||||
// were undeliverable, Failures says which; the batch is drained either way.
|
// were undeliverable; the batch is drained either way.
|
||||||
func (b *SendBatch) Flush() (int, error) {
|
func (b *SendBatch) Flush() (int, error) {
|
||||||
clear(b.failures)
|
|
||||||
b.failures = b.failures[:0]
|
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
written := 0
|
written := 0
|
||||||
if len(b.bufs) > 0 {
|
if len(b.bufs) > 0 {
|
||||||
written, err = b.out.WriteBatch(b.bufs, b.dsts, b.recordFailure)
|
written, err = b.out.WriteBatch(b.bufs, b.dsts)
|
||||||
}
|
}
|
||||||
clear(b.bufs)
|
clear(b.bufs)
|
||||||
b.bufs = b.bufs[:0]
|
b.bufs = b.bufs[:0]
|
||||||
@@ -87,9 +57,3 @@ func (b *SendBatch) Flush() (int, error) {
|
|||||||
b.arena.Reset()
|
b.arena.Reset()
|
||||||
return written, err
|
return written, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Failures returns every packet the last Flush could not send, in the order they were committed. Valid until the
|
|
||||||
// next Flush.
|
|
||||||
func (b *SendBatch) Failures() []Failure {
|
|
||||||
return b.failures
|
|
||||||
}
|
|
||||||
@@ -1,9 +1,7 @@
|
|||||||
package batch
|
package batch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -12,7 +10,7 @@ type fakeBatchWriter struct {
|
|||||||
addrs []netip.AddrPort
|
addrs []netip.AddrPort
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, _ func(int, error)) (int, error) {
|
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||||
// Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch
|
// Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch
|
||||||
// returns, so tests must capture data before that happens.
|
// returns, so tests must capture data before that happens.
|
||||||
w.bufs = make([][]byte, len(bufs))
|
w.bufs = make([][]byte, len(bufs))
|
||||||
@@ -122,84 +120,3 @@ func TestSendBatchGrowPreservesCommitted(t *testing.T) {
|
|||||||
t.Errorf("second packet on the wire: %x", fw.bufs[1])
|
t.Errorf("second packet on the wire: %x", fw.bufs[1])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// failingBatchWriter refuses every packet whose first byte is in refuse
|
|
||||||
type failingBatchWriter struct {
|
|
||||||
refuse map[byte]error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *failingBatchWriter) WriteBatch(bufs [][]byte, _ []netip.AddrPort, failed func(int, error)) (int, error) {
|
|
||||||
written := 0
|
|
||||||
for i, b := range bufs {
|
|
||||||
if err, ok := w.refuse[b[0]]; ok {
|
|
||||||
failed(i, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
written++
|
|
||||||
}
|
|
||||||
return written, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSendBatchReportsFailures(t *testing.T) {
|
|
||||||
errNoRoute := errors.New("no route")
|
|
||||||
fw := &failingBatchWriter{refuse: map[byte]error{2: errNoRoute, 3: errNoRoute}}
|
|
||||||
b := NewSendBatch(fw, 4, 128)
|
|
||||||
|
|
||||||
// pkt is n bytes starting with the refuse key, each later byte its offset plus 0x10 so the head is recognizable
|
|
||||||
pkt := func(key byte, n int) []byte {
|
|
||||||
p := b.Reserve(n)[:n]
|
|
||||||
p[0] = key
|
|
||||||
for i := 1; i < n; i++ {
|
|
||||||
p[i] = byte(0x10 + i)
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
head := func(p []byte) (h [FailureHeadLen]byte) {
|
|
||||||
copy(h[:], p)
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
|
|
||||||
dsts := []netip.AddrPort{
|
|
||||||
netip.MustParseAddrPort("10.0.0.1:4242"),
|
|
||||||
netip.MustParseAddrPort("[fd00::2]:4242"),
|
|
||||||
netip.MustParseAddrPort("[fd00::3]:4242"),
|
|
||||||
netip.MustParseAddrPort("10.0.0.4:4242"),
|
|
||||||
}
|
|
||||||
long := pkt(2, FailureHeadLen+8) // only its head is kept
|
|
||||||
short := pkt(3, 4) // zero padded
|
|
||||||
b.Commit(pkt(1, 20), dsts[0])
|
|
||||||
b.Commit(long, dsts[1])
|
|
||||||
b.Commit(short, dsts[2])
|
|
||||||
b.Commit(pkt(1, 20), dsts[3])
|
|
||||||
|
|
||||||
written, err := b.Flush()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Flush: %v", err)
|
|
||||||
}
|
|
||||||
if written != 2 {
|
|
||||||
t.Errorf("written = %d, want 2", written)
|
|
||||||
}
|
|
||||||
want := []Failure{{head(long), dsts[1], errNoRoute}, {head(short), dsts[2], errNoRoute}}
|
|
||||||
if got := b.Failures(); !slices.Equal(got, want) {
|
|
||||||
t.Errorf("failures = %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Failures do not leak into the next flush
|
|
||||||
again := pkt(2, 20)
|
|
||||||
b.Commit(again, dsts[0])
|
|
||||||
if _, err := b.Flush(); err != nil {
|
|
||||||
t.Fatalf("Flush: %v", err)
|
|
||||||
}
|
|
||||||
if want := []Failure{{head(again), dsts[0], errNoRoute}}; !slices.Equal(b.Failures(), want) {
|
|
||||||
t.Errorf("second flush failures = %v, want %v", b.Failures(), want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A clean flush forgets the last one's failures
|
|
||||||
b.Commit(pkt(1, 20), dsts[0])
|
|
||||||
if _, err := b.Flush(); err != nil {
|
|
||||||
t.Fatalf("Flush: %v", err)
|
|
||||||
}
|
|
||||||
if got := b.Failures(); len(got) != 0 {
|
|
||||||
t.Errorf("clean flush failures = %v, want none", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+79
-49
@@ -3,6 +3,7 @@
|
|||||||
package overlay
|
package overlay
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"math/bits"
|
"math/bits"
|
||||||
@@ -11,7 +12,6 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
"github.com/gaissmai/bart"
|
"github.com/gaissmai/bart"
|
||||||
@@ -35,26 +35,6 @@ type tun struct {
|
|||||||
// and routes for it. NEPacketTunnelProvider on darwin does this.
|
// and routes for it. NEPacketTunnelProvider on darwin does this.
|
||||||
hostOwned bool
|
hostOwned bool
|
||||||
l *slog.Logger
|
l *slog.Logger
|
||||||
|
|
||||||
// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
|
|
||||||
// fields below it
|
|
||||||
routesMu sync.Mutex
|
|
||||||
// routeStates is who held each installable unsafe_route prefix when we last looked
|
|
||||||
routeStates map[netip.Prefix]routeState
|
|
||||||
// routeWatch is the routing socket watchRoutes reads, Close closes it
|
|
||||||
routeWatch *os.File
|
|
||||||
// routeSweep reconciles the unsafe_routes every routeSweepEvery
|
|
||||||
routeSweep *time.Timer
|
|
||||||
routeSweepEvery time.Duration
|
|
||||||
closed bool
|
|
||||||
|
|
||||||
// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
|
|
||||||
// lock so the route watcher never waits on a reconcile
|
|
||||||
routeMoversMu sync.Mutex
|
|
||||||
routeMovers map[netip.Prefix]routeMover
|
|
||||||
|
|
||||||
// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
|
|
||||||
lastRouteEvent atomic.Int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type ifReq struct {
|
type ifReq struct {
|
||||||
@@ -163,7 +143,6 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
|
|||||||
Device: name,
|
Device: name,
|
||||||
vpnNetworks: vpnNetworks,
|
vpnNetworks: vpnNetworks,
|
||||||
DefaultMTU: getMTU(l, c),
|
DefaultMTU: getMTU(l, c),
|
||||||
routeSweepEvery: routeSweepInterval,
|
|
||||||
l: l,
|
l: l,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -240,16 +219,6 @@ func utunNameFromFd(fd int) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) Close() error {
|
func (t *tun) Close() error {
|
||||||
t.routesMu.Lock()
|
|
||||||
t.closed = true
|
|
||||||
if t.routeWatch != nil {
|
|
||||||
_ = t.routeWatch.Close()
|
|
||||||
}
|
|
||||||
if t.routeSweep != nil {
|
|
||||||
t.routeSweep.Stop()
|
|
||||||
}
|
|
||||||
t.routesMu.Unlock()
|
|
||||||
|
|
||||||
if t.f != nil {
|
if t.f != nil {
|
||||||
return t.f.Close()
|
return t.f.Close()
|
||||||
}
|
}
|
||||||
@@ -318,22 +287,8 @@ func (t *tun) Activate() error {
|
|||||||
return fmt.Errorf("failed to run tun device: %s", err)
|
return fmt.Errorf("failed to run tun device: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Watch before adding so a route removed in between is still noticed
|
|
||||||
watch, err := openRouteSocket()
|
|
||||||
if err != nil {
|
|
||||||
t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Unsafe path routes
|
// Unsafe path routes
|
||||||
if err = t.addRoutes(false, nil); err != nil {
|
return t.addRoutes(false, nil)
|
||||||
if watch != nil {
|
|
||||||
_ = watch.Close()
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
t.startRouteWatch(watch)
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
|
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
|
||||||
@@ -438,8 +393,7 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !initial && !change {
|
if !initial && !change {
|
||||||
// Any reload puts back the unsafe_routes someone deleted
|
return nil
|
||||||
return t.addRoutes(true, r)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
routeTree, err := makeRouteTree(t.l, routes, false)
|
routeTree, err := makeRouteTree(t.l, routes, false)
|
||||||
@@ -557,6 +511,82 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
|
|||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
|
||||||
|
// The route tree is still ours, the system routing table is not
|
||||||
|
if t.hostOwned {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Activate hasn't run yet, it will install the routes
|
||||||
|
linkAddr := t.linkAddr.Load()
|
||||||
|
if linkAddr == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
routes := *t.Routes.Load()
|
||||||
|
systemRoutes := sync.OnceValues(fetchRoutes)
|
||||||
|
|
||||||
|
for _, r := range routes {
|
||||||
|
if len(r.Via) == 0 || !r.Install {
|
||||||
|
// We don't allow route MTUs so only install routes with a via
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
err := addRoute(r.Cidr, linkAddr)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, unix.EEXIST) {
|
||||||
|
// Every route we added already exists on a reload, only warn about someone else's
|
||||||
|
msgs, err := systemRoutes()
|
||||||
|
if err != nil {
|
||||||
|
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
|
||||||
|
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
|
||||||
|
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
||||||
|
if logErrors {
|
||||||
|
retErr.Log(t.l)
|
||||||
|
rl.Failed()
|
||||||
|
} else {
|
||||||
|
return retErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
t.l.Info("Added route", "route", r)
|
||||||
|
// A cidr listed twice would otherwise find our own route missing from an older fetch
|
||||||
|
systemRoutes = sync.OnceValues(fetchRoutes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
|
||||||
|
linkAddr := t.linkAddr.Load()
|
||||||
|
if t.hostOwned || linkAddr == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, r := range routes {
|
||||||
|
if len(r.Via) == 0 || !r.Install {
|
||||||
|
// addRoutes never installs these
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
err := delRoute(r.Cidr, linkAddr)
|
||||||
|
if errors.Is(err, unix.ESRCH) {
|
||||||
|
// Already gone, which is what we wanted
|
||||||
|
t.l.Info("Route was already removed", "route", r)
|
||||||
|
} else if err != nil {
|
||||||
|
t.l.Error("Failed to remove route", "error", err, "route", r)
|
||||||
|
rl.Failed()
|
||||||
|
} else {
|
||||||
|
t.l.Info("Removed route", "route", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
|
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
|
||||||
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -3,17 +3,12 @@
|
|||||||
package overlay
|
package overlay
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/binary"
|
|
||||||
"encoding/hex"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
"github.com/slackhq/nebula/routing"
|
"github.com/slackhq/nebula/routing"
|
||||||
@@ -109,367 +104,6 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
|
|||||||
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
|
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
|
|
||||||
const (
|
|
||||||
// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
|
|
||||||
kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
|
|
||||||
addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
|
|
||||||
// With no interface address, which x/net/route skips
|
|
||||||
interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRoutesMayHaveMoved(t *testing.T) {
|
|
||||||
cidr := netip.MustParsePrefix("192.168.1.0/24")
|
|
||||||
via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
|
|
||||||
routes := []Route{{Cidr: cidr, Via: via, Install: true}}
|
|
||||||
// A request like route(8) sends, the kernel echoes it to every routing socket
|
|
||||||
request := func(typ, flags int, dst, mask [4]byte) []byte {
|
|
||||||
m := &netroute.RouteMessage{
|
|
||||||
Version: unix.RTM_VERSION,
|
|
||||||
Type: typ,
|
|
||||||
Flags: flags,
|
|
||||||
ID: 1234,
|
|
||||||
Addrs: []netroute.Addr{
|
|
||||||
unix.RTAX_DST: &netroute.Inet4Addr{IP: dst},
|
|
||||||
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
|
|
||||||
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
b, err := m.Marshal()
|
|
||||||
require.NoError(t, err)
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
raw := func(s string) []byte {
|
|
||||||
b, err := hex.DecodeString(s)
|
|
||||||
require.NoError(t, err)
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
|
|
||||||
routesMayHaveMoved := func(b []byte, routes []Route) bool {
|
|
||||||
moved, _, _ := routesMayHaveMoved(b, routes)
|
|
||||||
return moved
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
|
|
||||||
assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
|
|
||||||
assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
|
|
||||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
|
|
||||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
|
|
||||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
|
|
||||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
|
|
||||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
|
|
||||||
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
|
|
||||||
|
|
||||||
failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
|
|
||||||
binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
|
|
||||||
assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
|
|
||||||
|
|
||||||
_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
|
|
||||||
require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
|
|
||||||
assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
|
|
||||||
neighbor := raw(kernelSubnetDelete)
|
|
||||||
binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
|
|
||||||
assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
|
|
||||||
|
|
||||||
assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
|
|
||||||
assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
|
|
||||||
}
|
|
||||||
|
|
||||||
// A request names its prefix and who made it, the kernel's own and interface changes name neither
|
|
||||||
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
|
|
||||||
cidr := netip.MustParsePrefix("192.168.1.0/24")
|
|
||||||
routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
|
|
||||||
m := &netroute.RouteMessage{
|
|
||||||
Version: unix.RTM_VERSION,
|
|
||||||
Type: unix.RTM_CHANGE,
|
|
||||||
Flags: unix.RTF_UP,
|
|
||||||
ID: 1234,
|
|
||||||
Addrs: []netroute.Addr{
|
|
||||||
unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
|
|
||||||
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
|
|
||||||
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
b, err := m.Marshal()
|
|
||||||
require.NoError(t, err)
|
|
||||||
moved, p, pid := routesMayHaveMoved(b, routes)
|
|
||||||
assert.True(t, moved)
|
|
||||||
assert.Equal(t, cidr, p)
|
|
||||||
assert.Equal(t, int32(1234), pid)
|
|
||||||
|
|
||||||
for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
|
|
||||||
b, err := hex.DecodeString(s)
|
|
||||||
require.NoError(t, err)
|
|
||||||
moved, p, pid := routesMayHaveMoved(b, routes)
|
|
||||||
assert.True(t, moved, name)
|
|
||||||
assert.False(t, p.IsValid(), name)
|
|
||||||
assert.Zero(t, pid, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessName(t *testing.T) {
|
|
||||||
assert.Equal(t, "launchd", processName(1))
|
|
||||||
self := filepath.Base(os.Args[0])
|
|
||||||
assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
|
|
||||||
assert.Empty(t, processName(-1))
|
|
||||||
}
|
|
||||||
|
|
||||||
// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
|
|
||||||
// the minute between sweeps when it isn't zero.
|
|
||||||
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
|
|
||||||
requireRoot(t)
|
|
||||||
|
|
||||||
cfg := "tun:\n unsafe_routes:\n"
|
|
||||||
for _, cidr := range cidrs {
|
|
||||||
via := "10.250.0.2"
|
|
||||||
if netip.MustParsePrefix(cidr).Addr().Is6() {
|
|
||||||
via = "fd00:250::2"
|
|
||||||
}
|
|
||||||
cfg += " - route: " + cidr + "\n via: " + via + "\n"
|
|
||||||
}
|
|
||||||
|
|
||||||
l := test.NewLogger()
|
|
||||||
c := config.NewC(l)
|
|
||||||
require.NoError(t, c.LoadString(cfg))
|
|
||||||
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
|
|
||||||
require.NoError(t, err)
|
|
||||||
if sweep != 0 {
|
|
||||||
tn.routeSweepEvery = sweep
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { _ = tn.Close() })
|
|
||||||
require.NoError(t, tn.Activate())
|
|
||||||
return tn, c, cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
func requireRoot(t *testing.T) {
|
|
||||||
if os.Geteuid() != 0 {
|
|
||||||
t.Skip("creating interfaces and adding routes needs root")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// routeVia is the index of the interface that routes cidr, 0 if none does
|
|
||||||
func routeVia(t *testing.T, cidr string) int {
|
|
||||||
msgs, err := fetchRoutes()
|
|
||||||
require.NoError(t, err)
|
|
||||||
return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
|
|
||||||
}
|
|
||||||
|
|
||||||
// routeStateOf is what tn made of cidr when it last looked
|
|
||||||
func routeStateOf(tn *tun, cidr string) routeState {
|
|
||||||
tn.routesMu.Lock()
|
|
||||||
defer tn.routesMu.Unlock()
|
|
||||||
return tn.routeStates[netip.MustParsePrefix(cidr)]
|
|
||||||
}
|
|
||||||
|
|
||||||
func routeArgs(action, cidr string, extra ...string) []string {
|
|
||||||
args := []string{"-q", "-n", action}
|
|
||||||
if netip.MustParsePrefix(cidr).Addr().Is6() {
|
|
||||||
args = append(args, "-inet6")
|
|
||||||
}
|
|
||||||
return append(append(args, "-net", cidr), extra...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func routeCmd(t *testing.T, action, cidr string, extra ...string) {
|
|
||||||
requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
|
|
||||||
}
|
|
||||||
|
|
||||||
// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
|
|
||||||
// says so
|
|
||||||
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
|
|
||||||
out, err := cmd.CombinedOutput()
|
|
||||||
require.NoError(t, err, string(out))
|
|
||||||
require.NotContains(t, string(out), "writing to routing socket")
|
|
||||||
}
|
|
||||||
|
|
||||||
// cleanupRoute deletes cidr's route when the test ends, whoever holds it
|
|
||||||
func cleanupRoute(t *testing.T, cidr string) {
|
|
||||||
t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
|
|
||||||
}
|
|
||||||
|
|
||||||
// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
|
|
||||||
func routeThroughLo0(t *testing.T, cidr string) int {
|
|
||||||
requireRoot(t)
|
|
||||||
routeCmd(t, "add", cidr, "-interface", "lo0")
|
|
||||||
cleanupRoute(t, cidr)
|
|
||||||
lo0, err := net.InterfaceByName("lo0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
return lo0.Index
|
|
||||||
}
|
|
||||||
|
|
||||||
// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
|
|
||||||
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
|
|
||||||
for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
|
|
||||||
t.Run(cidr, func(t *testing.T) {
|
|
||||||
lo0 := routeThroughLo0(t, cidr)
|
|
||||||
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
|
||||||
require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
|
|
||||||
|
|
||||||
routeCmd(t, "delete", cidr)
|
|
||||||
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
|
|
||||||
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
|
|
||||||
cidr := "10.251.1.0/24"
|
|
||||||
lo0 := routeThroughLo0(t, cidr)
|
|
||||||
_, c, _ := newRouteTestTun(t, 0, cidr)
|
|
||||||
|
|
||||||
res := c.ReloadConfigString("tun: {}\n")
|
|
||||||
require.NoError(t, res.Err)
|
|
||||||
assert.False(t, res.Failed)
|
|
||||||
assert.Equal(t, lo0, routeVia(t, cidr))
|
|
||||||
}
|
|
||||||
|
|
||||||
// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
|
|
||||||
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
|
|
||||||
cidr := "10.251.3.0/24"
|
|
||||||
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
|
||||||
ours := tn.linkAddr.Load().Index
|
|
||||||
require.Equal(t, ours, routeVia(t, cidr))
|
|
||||||
|
|
||||||
cleanupRoute(t, cidr)
|
|
||||||
requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
|
|
||||||
lo0, err := net.InterfaceByName("lo0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
time.Sleep(2 * routeRetryDelay)
|
|
||||||
require.Equal(t, lo0.Index, routeVia(t, cidr))
|
|
||||||
|
|
||||||
routeCmd(t, "delete", cidr)
|
|
||||||
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
|
|
||||||
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
|
|
||||||
cidr := "10.251.4.0/24"
|
|
||||||
tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
|
|
||||||
ours := tn.linkAddr.Load().Index
|
|
||||||
require.Equal(t, ours, routeVia(t, cidr))
|
|
||||||
|
|
||||||
routeCmd(t, "delete", cidr)
|
|
||||||
assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
|
|
||||||
|
|
||||||
// A reload with nothing changed is how to ask for it back
|
|
||||||
res := c.ReloadConfigString(cfg)
|
|
||||||
require.NoError(t, res.Err)
|
|
||||||
assert.False(t, res.Failed)
|
|
||||||
assert.Equal(t, ours, routeVia(t, cidr))
|
|
||||||
}
|
|
||||||
|
|
||||||
// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
|
|
||||||
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
|
|
||||||
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
|
|
||||||
for name, tc := range map[string]struct {
|
|
||||||
cidr string
|
|
||||||
sweep time.Duration
|
|
||||||
misfile func(t *testing.T, tn *tun, cidr string)
|
|
||||||
}{
|
|
||||||
// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
|
|
||||||
"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
|
|
||||||
require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
|
|
||||||
}},
|
|
||||||
// The sweep that caps a table that never settles
|
|
||||||
"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
|
|
||||||
tn.reconcileRoutes()
|
|
||||||
require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
|
|
||||||
}},
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
|
|
||||||
ours := tn.linkAddr.Load().Index
|
|
||||||
require.Equal(t, ours, routeVia(t, tc.cidr))
|
|
||||||
|
|
||||||
routeCmd(t, "delete", tc.cidr)
|
|
||||||
tc.misfile(t, tn, tc.cidr)
|
|
||||||
|
|
||||||
lo0 := routeThroughLo0(t, tc.cidr)
|
|
||||||
require.Eventually(t, func() bool {
|
|
||||||
s := routeStateOf(tn, tc.cidr)
|
|
||||||
return s.kind == routeYielded && s.holder == lo0
|
|
||||||
}, 5*time.Second, 50*time.Millisecond)
|
|
||||||
|
|
||||||
routeCmd(t, "delete", tc.cidr)
|
|
||||||
assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
|
|
||||||
// taken for a hand delete
|
|
||||||
func TestTunYieldsToARouteChangedAway(t *testing.T) {
|
|
||||||
for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
|
|
||||||
t.Run(cidr, func(t *testing.T) {
|
|
||||||
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
|
||||||
ours := tn.linkAddr.Load().Index
|
|
||||||
require.Equal(t, ours, routeVia(t, cidr))
|
|
||||||
|
|
||||||
cleanupRoute(t, cidr)
|
|
||||||
routeCmd(t, "change", cidr, "-interface", "lo0")
|
|
||||||
lo0, err := net.InterfaceByName("lo0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, lo0.Index, routeVia(t, cidr))
|
|
||||||
require.Eventually(t, func() bool {
|
|
||||||
s := routeStateOf(tn, cidr)
|
|
||||||
return s.kind == routeYielded && s.holder == lo0.Index
|
|
||||||
}, 5*time.Second, 50*time.Millisecond)
|
|
||||||
|
|
||||||
routeCmd(t, "delete", cidr)
|
|
||||||
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
|
|
||||||
// ethernet configd routed the lan through
|
|
||||||
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
|
|
||||||
requireRoot(t)
|
|
||||||
var feth string
|
|
||||||
for i := 100; i < 200 && feth == ""; i++ {
|
|
||||||
if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
|
|
||||||
feth = fmt.Sprintf("feth%d", i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
require.NotEmpty(t, feth)
|
|
||||||
require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
|
|
||||||
t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
|
|
||||||
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
|
|
||||||
|
|
||||||
// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
|
|
||||||
// refuses one named by the feth itself
|
|
||||||
cidr := "10.251.2.0/24"
|
|
||||||
routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
|
|
||||||
cleanupRoute(t, cidr)
|
|
||||||
ifc, err := net.InterfaceByName(feth)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tn, _, _ := newRouteTestTun(t, 0, cidr)
|
|
||||||
require.Equal(t, ifc.Index, routeVia(t, cidr))
|
|
||||||
|
|
||||||
// Losing the address first leaves the route, the reconcile finds it still held
|
|
||||||
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
|
|
||||||
time.Sleep(2 * routeRetryDelay)
|
|
||||||
require.Equal(t, ifc.Index, routeVia(t, cidr))
|
|
||||||
|
|
||||||
// Only RTM_IFINFO says the interface went, its route goes with no message of its own
|
|
||||||
require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
|
|
||||||
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
|
|
||||||
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
|
|
||||||
cidr := "10.251.5.0/24"
|
|
||||||
routeThroughLo0(t, cidr)
|
|
||||||
tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
|
|
||||||
|
|
||||||
tn.routesMu.Lock()
|
|
||||||
require.NoError(t, tn.routeWatch.Close())
|
|
||||||
tn.routesMu.Unlock()
|
|
||||||
|
|
||||||
routeCmd(t, "delete", cidr)
|
|
||||||
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
|
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
|
||||||
func TestTunDevNeedsRestart(t *testing.T) {
|
func TestTunDevNeedsRestart(t *testing.T) {
|
||||||
for name, want := range map[string]bool{
|
for name, want := range map[string]bool{
|
||||||
|
|||||||
@@ -1,451 +0,0 @@
|
|||||||
//go:build !ios && !e2e_testing
|
|
||||||
|
|
||||||
package overlay
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"net/netip"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
"sync/atomic"
|
|
||||||
"time"
|
|
||||||
"unsafe"
|
|
||||||
|
|
||||||
"github.com/slackhq/nebula/config"
|
|
||||||
"github.com/slackhq/nebula/util"
|
|
||||||
netroute "golang.org/x/net/route"
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
|
|
||||||
// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
|
|
||||||
// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
|
|
||||||
// or until another interface takes the prefix, after which ours follows theirs like any other.
|
|
||||||
|
|
||||||
const (
|
|
||||||
// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
|
|
||||||
// replaces a route by deleting and adding, looking in between would find the prefix free
|
|
||||||
routeRetryDelay = time.Second
|
|
||||||
|
|
||||||
// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
|
|
||||||
// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
|
|
||||||
// a message for each
|
|
||||||
routeSweepInterval = time.Minute
|
|
||||||
|
|
||||||
// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
|
|
||||||
routeSocketRcvBuf = 1 << 20
|
|
||||||
)
|
|
||||||
|
|
||||||
type routeStateKind int
|
|
||||||
|
|
||||||
const (
|
|
||||||
routeUnknown routeStateKind = iota
|
|
||||||
// routeInstalled is our route holding the prefix
|
|
||||||
routeInstalled
|
|
||||||
// routeYielded is another interface holding the prefix, ours is added once it lets go
|
|
||||||
routeYielded
|
|
||||||
// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
|
|
||||||
// another interface takes the prefix
|
|
||||||
routeRemoved
|
|
||||||
// routeFailed is adding our route failing for some other reason, it is retried
|
|
||||||
routeFailed
|
|
||||||
)
|
|
||||||
|
|
||||||
type routeState struct {
|
|
||||||
kind routeStateKind
|
|
||||||
// holder is the index of the interface holding the prefix when kind is routeYielded
|
|
||||||
holder int
|
|
||||||
}
|
|
||||||
|
|
||||||
// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
|
|
||||||
// is looked up as the message arrives, route(8) has exited by the time the table settles
|
|
||||||
type routeMover struct {
|
|
||||||
pid int32
|
|
||||||
name string
|
|
||||||
}
|
|
||||||
|
|
||||||
// logArgs appends m to a log line's args, when we know who it was
|
|
||||||
func (m routeMover) logArgs(args ...any) []any {
|
|
||||||
if m.pid == 0 {
|
|
||||||
return args
|
|
||||||
}
|
|
||||||
args = append(args, "pid", m.pid)
|
|
||||||
if m.name != "" {
|
|
||||||
args = append(args, "process", m.name)
|
|
||||||
}
|
|
||||||
return args
|
|
||||||
}
|
|
||||||
|
|
||||||
// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
|
|
||||||
func processName(pid int32) string {
|
|
||||||
kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
|
|
||||||
if err != nil || kp.Proc.P_pid != pid {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return unix.ByteSliceToString(kp.Proc.P_comm[:])
|
|
||||||
}
|
|
||||||
|
|
||||||
// installRoutes returns the routes we put in the system routing table, one per prefix
|
|
||||||
func installRoutes(routes []Route) []Route {
|
|
||||||
seen := make(map[netip.Prefix]struct{}, len(routes))
|
|
||||||
var out []Route
|
|
||||||
for _, r := range routes {
|
|
||||||
if len(r.Via) == 0 || !r.Install {
|
|
||||||
// We don't allow route MTUs so only install routes with a via
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
p := r.Cidr.Masked()
|
|
||||||
if _, ok := seen[p]; ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[p] = struct{}{}
|
|
||||||
out = append(out, r)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
|
|
||||||
// interface holds its prefix. It runs on start and on every reload.
|
|
||||||
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
|
|
||||||
// The route tree is still ours, the system routing table is not
|
|
||||||
if t.hostOwned {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
t.routesMu.Lock()
|
|
||||||
defer t.routesMu.Unlock()
|
|
||||||
|
|
||||||
// Activate hasn't run yet, it will install the routes
|
|
||||||
linkAddr := t.linkAddr.Load()
|
|
||||||
if linkAddr == nil || t.closed {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var msgs []netroute.Message
|
|
||||||
var msgsErr error
|
|
||||||
fetched := false
|
|
||||||
states := make(map[netip.Prefix]routeState)
|
|
||||||
for _, r := range installRoutes(*t.Routes.Load()) {
|
|
||||||
p := r.Cidr.Masked()
|
|
||||||
prev := t.routeStates[p]
|
|
||||||
|
|
||||||
err := addRoute(p, linkAddr)
|
|
||||||
if err == nil {
|
|
||||||
t.l.Info("Added route", "route", r)
|
|
||||||
states[p] = routeState{kind: routeInstalled}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if !errors.Is(err, unix.EEXIST) {
|
|
||||||
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
|
||||||
if !logErrors {
|
|
||||||
return retErr
|
|
||||||
}
|
|
||||||
retErr.Log(t.l)
|
|
||||||
rl.Failed()
|
|
||||||
states[p] = routeState{kind: routeFailed}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Every route we added already exists on a reload, only warn about someone else's
|
|
||||||
if !fetched {
|
|
||||||
msgs, msgsErr = fetchRoutes()
|
|
||||||
fetched = true
|
|
||||||
}
|
|
||||||
if msgsErr != nil {
|
|
||||||
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
|
|
||||||
states[p] = prev
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
switch holder := routeInterfaceIn(msgs, p); holder {
|
|
||||||
case linkAddr.Index:
|
|
||||||
states[p] = routeState{kind: routeInstalled}
|
|
||||||
case 0:
|
|
||||||
// Removed since the add, the watcher will see to it
|
|
||||||
states[p] = prev
|
|
||||||
default:
|
|
||||||
if prev.kind != routeYielded || prev.holder != holder {
|
|
||||||
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
|
|
||||||
}
|
|
||||||
states[p] = routeState{kind: routeYielded, holder: holder}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
t.routeStates = states
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
|
|
||||||
if t.hostOwned {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
t.routesMu.Lock()
|
|
||||||
defer t.routesMu.Unlock()
|
|
||||||
|
|
||||||
linkAddr := t.linkAddr.Load()
|
|
||||||
if linkAddr == nil || t.closed {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
|
|
||||||
msgs, err := fetchRoutes()
|
|
||||||
if err != nil {
|
|
||||||
var left []netip.Prefix
|
|
||||||
for _, r := range installRoutes(routes) {
|
|
||||||
left = append(left, r.Cidr)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, r := range installRoutes(routes) {
|
|
||||||
p := r.Cidr.Masked()
|
|
||||||
// Whatever happens next, the prefix is no longer one we know about
|
|
||||||
delete(t.routeStates, p)
|
|
||||||
if routeInterfaceIn(msgs, p) != linkAddr.Index {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err = delRoute(p, linkAddr)
|
|
||||||
if errors.Is(err, unix.ESRCH) {
|
|
||||||
// Already gone, which is what we wanted
|
|
||||||
t.l.Info("Route was already removed", "route", r)
|
|
||||||
} else if err != nil {
|
|
||||||
t.l.Error("Failed to remove route", "error", err, "route", r)
|
|
||||||
rl.Failed()
|
|
||||||
} else {
|
|
||||||
t.l.Info("Removed route", "route", r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
|
|
||||||
// changed since it last looked, it runs every routeSweepEvery.
|
|
||||||
func (t *tun) reconcileRoutes() {
|
|
||||||
t.routesMu.Lock()
|
|
||||||
defer t.routesMu.Unlock()
|
|
||||||
|
|
||||||
linkAddr := t.linkAddr.Load()
|
|
||||||
if linkAddr == nil || t.closed {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
t.routeMoversMu.Lock()
|
|
||||||
movers := t.routeMovers
|
|
||||||
t.routeMovers = nil
|
|
||||||
t.routeMoversMu.Unlock()
|
|
||||||
|
|
||||||
msgs, err := fetchRoutes()
|
|
||||||
if err != nil {
|
|
||||||
t.l.Error("Failed to read the system routing table", "error", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
states := make(map[netip.Prefix]routeState)
|
|
||||||
for _, r := range installRoutes(*t.Routes.Load()) {
|
|
||||||
p := r.Cidr.Masked()
|
|
||||||
prev := t.routeStates[p]
|
|
||||||
by := movers[p]
|
|
||||||
|
|
||||||
holder := routeInterfaceIn(msgs, p)
|
|
||||||
switch {
|
|
||||||
case holder == linkAddr.Index:
|
|
||||||
states[p] = routeState{kind: routeInstalled}
|
|
||||||
|
|
||||||
case holder != 0:
|
|
||||||
// Another interface taking a prefix we left alone puts ours back in line behind theirs
|
|
||||||
if prev.kind != routeYielded || prev.holder != holder {
|
|
||||||
t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
|
|
||||||
}
|
|
||||||
states[p] = routeState{kind: routeYielded, holder: holder}
|
|
||||||
|
|
||||||
case prev.kind == routeInstalled:
|
|
||||||
t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
|
|
||||||
states[p] = routeState{kind: routeRemoved}
|
|
||||||
|
|
||||||
case prev.kind == routeRemoved:
|
|
||||||
states[p] = prev
|
|
||||||
|
|
||||||
default:
|
|
||||||
err := addRoute(p, linkAddr)
|
|
||||||
switch {
|
|
||||||
case err == nil:
|
|
||||||
if prev.kind == routeYielded {
|
|
||||||
t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
|
|
||||||
} else {
|
|
||||||
t.l.Info("Added route", "route", r)
|
|
||||||
}
|
|
||||||
states[p] = routeState{kind: routeInstalled}
|
|
||||||
case errors.Is(err, unix.EEXIST):
|
|
||||||
// Taken since the fetch, the next look will see by whom
|
|
||||||
states[p] = prev
|
|
||||||
default:
|
|
||||||
if prev.kind != routeFailed {
|
|
||||||
util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
|
|
||||||
}
|
|
||||||
states[p] = routeState{kind: routeFailed}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.routeStates = states
|
|
||||||
}
|
|
||||||
|
|
||||||
// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
|
|
||||||
// routing table settles after anything that may have moved a prefix
|
|
||||||
func (t *tun) startRouteWatch(watch *os.File) {
|
|
||||||
t.routesMu.Lock()
|
|
||||||
defer t.routesMu.Unlock()
|
|
||||||
if t.closed {
|
|
||||||
if watch != nil {
|
|
||||||
_ = watch.Close()
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
|
|
||||||
if watch != nil {
|
|
||||||
t.routeWatch = watch
|
|
||||||
go t.watchRoutes(watch)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) sweepRoutes() {
|
|
||||||
// A table still changing gets reconciled once it settles
|
|
||||||
if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
|
|
||||||
t.reconcileRoutes()
|
|
||||||
}
|
|
||||||
|
|
||||||
t.routesMu.Lock()
|
|
||||||
defer t.routesMu.Unlock()
|
|
||||||
if !t.closed {
|
|
||||||
t.routeSweep.Reset(t.routeSweepEvery)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func openRouteSocket() (*os.File, error) {
|
|
||||||
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
|
|
||||||
}
|
|
||||||
// Fewer drops, the sweep catches the rest
|
|
||||||
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
|
|
||||||
// Non-blocking puts it on the poller, so Close interrupts a Read
|
|
||||||
if err = unix.SetNonblock(fd, true); err != nil {
|
|
||||||
_ = unix.Close(fd)
|
|
||||||
return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
|
|
||||||
}
|
|
||||||
return os.NewFile(uintptr(fd), "route"), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
|
|
||||||
// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
|
|
||||||
func (t *tun) watchRoutes(f *os.File) {
|
|
||||||
var reconciled atomic.Int64
|
|
||||||
settle := time.AfterFunc(time.Hour, func() {
|
|
||||||
reconciled.Store(time.Now().UnixNano())
|
|
||||||
t.reconcileRoutes()
|
|
||||||
})
|
|
||||||
settle.Stop()
|
|
||||||
defer settle.Stop()
|
|
||||||
|
|
||||||
var burst time.Time
|
|
||||||
self := int32(os.Getpid())
|
|
||||||
b := make([]byte, os.Getpagesize())
|
|
||||||
for {
|
|
||||||
n, err := f.Read(b)
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, os.ErrClosed) {
|
|
||||||
t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
|
|
||||||
if !moved {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Our own are already in routeStates
|
|
||||||
if p.IsValid() && pid != self {
|
|
||||||
t.routeMoversMu.Lock()
|
|
||||||
if t.routeMovers == nil {
|
|
||||||
t.routeMovers = make(map[netip.Prefix]routeMover)
|
|
||||||
}
|
|
||||||
t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
|
|
||||||
t.routeMoversMu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
t.lastRouteEvent.Store(now.UnixNano())
|
|
||||||
if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
|
|
||||||
burst = now
|
|
||||||
}
|
|
||||||
settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
|
|
||||||
// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
|
|
||||||
// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
|
|
||||||
// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
|
|
||||||
// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
|
|
||||||
// and the pid of the process that made it.
|
|
||||||
func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
|
|
||||||
if len(b) < 4 {
|
|
||||||
return false, p, 0
|
|
||||||
}
|
|
||||||
switch int(b[3]) {
|
|
||||||
case unix.RTM_IFINFO, unix.RTM_DELADDR:
|
|
||||||
return true, p, 0
|
|
||||||
case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
|
|
||||||
default:
|
|
||||||
return false, p, 0
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(b) < unix.SizeofRtMsghdr {
|
|
||||||
return false, p, 0
|
|
||||||
}
|
|
||||||
hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
|
|
||||||
// A failed request is reported too, and a scoped route never held the prefix
|
|
||||||
if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
|
|
||||||
return false, p, 0
|
|
||||||
}
|
|
||||||
|
|
||||||
if hdr.Pid == 0 {
|
|
||||||
// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
|
|
||||||
// could be holding a prefix
|
|
||||||
return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// Replies to a request carry the requester's padded sockaddrs
|
|
||||||
msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
|
|
||||||
if err != nil || len(msgs) != 1 {
|
|
||||||
return false, p, 0
|
|
||||||
}
|
|
||||||
m, ok := msgs[0].(*netroute.RouteMessage)
|
|
||||||
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
|
|
||||||
return false, p, 0
|
|
||||||
}
|
|
||||||
p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
|
|
||||||
if !ok {
|
|
||||||
return false, netip.Prefix{}, 0
|
|
||||||
}
|
|
||||||
p = p.Masked()
|
|
||||||
for _, r := range installRoutes(routes) {
|
|
||||||
if r.Cidr.Masked() == p {
|
|
||||||
return true, p, hdr.Pid
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false, netip.Prefix{}, 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// interfaceName is the name of the interface with index i, for logs
|
|
||||||
func interfaceName(i int) string {
|
|
||||||
if ifc, err := net.InterfaceByIndex(i); err == nil {
|
|
||||||
return ifc.Name
|
|
||||||
}
|
|
||||||
return strconv.Itoa(i)
|
|
||||||
}
|
|
||||||
@@ -165,24 +165,20 @@ func (p *Punchy) scheduleJob(job holepunchJob, delay time.Duration) {
|
|||||||
// SendPunch sends an immediate keepalive punch for an idle hostinfo.
|
// SendPunch sends an immediate keepalive punch for an idle hostinfo.
|
||||||
// The configured punchy.target_all_remotes mode picks the targets. Gated on punchy.punch and the lighthouse-skip rule
|
// The configured punchy.target_all_remotes mode picks the targets. Gated on punchy.punch and the lighthouse-skip rule
|
||||||
// (lighthouses don't get keepalive punches because the regular update interval keeps their NAT state warm).
|
// (lighthouses don't get keepalive punches because the regular update interval keeps their NAT state warm).
|
||||||
// Returns the hostinfo's current remote and the error from punching it, an invalid remote if it was not punched.
|
func (p *Punchy) SendPunch(hostinfo *HostInfo) {
|
||||||
func (p *Punchy) SendPunch(hostinfo *HostInfo) (netip.AddrPort, error) {
|
|
||||||
if !p.punch.Load() {
|
if !p.punch.Load() {
|
||||||
return netip.AddrPort{}, nil
|
return
|
||||||
}
|
}
|
||||||
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
|
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
|
||||||
return netip.AddrPort{}, nil
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
hr := hostinfo.GetRemote()
|
|
||||||
if p.punchEverything.Load() {
|
if p.punchEverything.Load() {
|
||||||
return hr, p.sendPunchToAllRemotes(hostinfo, hr)
|
p.sendPunchToAllRemotes(hostinfo)
|
||||||
}
|
} else if hr := hostinfo.GetRemote(); hr.IsValid() {
|
||||||
if !hr.IsValid() {
|
|
||||||
return hr, nil
|
|
||||||
}
|
|
||||||
p.metricPunchyTx.Inc(1)
|
p.metricPunchyTx.Inc(1)
|
||||||
return hr, p.punchConn.WriteTo([]byte{1}, hr)
|
p.punchConn.WriteTo([]byte{1}, hr)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SendPunchToAll punches every known remote for hostinfo, but only when punchy.target_all_remotes is enabled.
|
// SendPunchToAll punches every known remote for hostinfo, but only when punchy.target_all_remotes is enabled.
|
||||||
@@ -199,19 +195,14 @@ func (p *Punchy) SendPunchToAll(hostinfo *HostInfo) {
|
|||||||
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
|
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
p.sendPunchToAllRemotes(hostinfo, netip.AddrPort{})
|
p.sendPunchToAllRemotes(hostinfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
// sendPunchToAllRemotes punches every known remote for hostinfo and returns the error from punching current
|
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo) {
|
||||||
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo, current netip.AddrPort) error {
|
|
||||||
var currentErr error
|
|
||||||
hostinfo.remotes.ForEach(p.hm.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
|
hostinfo.remotes.ForEach(p.hm.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
|
||||||
p.metricPunchyTx.Inc(1)
|
p.metricPunchyTx.Inc(1)
|
||||||
if err := p.punchConn.WriteTo([]byte{1}, addr); addr == current {
|
p.punchConn.WriteTo([]byte{1}, addr)
|
||||||
currentErr = err
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
return currentErr
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start wires the runtime dependencies and spawns the scheduler worker.
|
// Start wires the runtime dependencies and spawns the scheduler worker.
|
||||||
|
|||||||
-112
@@ -3,14 +3,11 @@ package nebula
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/netip"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
"github.com/slackhq/nebula/header"
|
|
||||||
"github.com/slackhq/nebula/test"
|
"github.com/slackhq/nebula/test"
|
||||||
"github.com/slackhq/nebula/udp"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
@@ -245,112 +242,3 @@ func TestPunchyReloadRestartRequired(t *testing.T) {
|
|||||||
require.NoError(t, res.Err)
|
require.NoError(t, res.Err)
|
||||||
assert.True(t, res.RestartRequired)
|
assert.True(t, res.RestartRequired)
|
||||||
}
|
}
|
||||||
|
|
||||||
// errNoRoute is the unreachable error every platform classifies the same way
|
|
||||||
var errNoRoute = udp.ErrInvalidIPv6RemoteForSocket
|
|
||||||
|
|
||||||
// recordingConn records where every write went and what kind of message it was, and fails the ones to an address in
|
|
||||||
// fail
|
|
||||||
type recordingConn struct {
|
|
||||||
udp.NoopConn
|
|
||||||
fail map[netip.AddrPort]error
|
|
||||||
wrote []netip.AddrPort
|
|
||||||
// types is the message type of each write, a punch has none
|
|
||||||
types []header.MessageType
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *recordingConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
|
||||||
c.wrote = append(c.wrote, addr)
|
|
||||||
var h header.H
|
|
||||||
if h.Parse(b) == nil {
|
|
||||||
c.types = append(c.types, h.Type)
|
|
||||||
}
|
|
||||||
return c.fail[addr]
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *recordingConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(int, error)) (int, error) {
|
|
||||||
written := 0
|
|
||||||
for i := range bufs {
|
|
||||||
if err := c.WriteTo(bufs[i], addrs[i]); err != nil {
|
|
||||||
failed(i, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
written++
|
|
||||||
}
|
|
||||||
return written, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type noLighthouses struct{}
|
|
||||||
|
|
||||||
func (noLighthouses) IsAnyLighthouseAddr([]netip.Addr) bool { return false }
|
|
||||||
|
|
||||||
var (
|
|
||||||
punchTargetV4 = netip.MustParseAddrPort("10.0.0.3:4242")
|
|
||||||
punchTargetV6 = netip.MustParseAddrPort("[fd00::3]:4242")
|
|
||||||
)
|
|
||||||
|
|
||||||
// newPunchTarget is a tunnel on its IPv6 remote that also knows the peer's IPv4 one
|
|
||||||
func newPunchTarget() *HostInfo {
|
|
||||||
vpnAddr := netip.MustParseAddr("10.128.0.3")
|
|
||||||
hostinfo := &HostInfo{
|
|
||||||
vpnAddrs: []netip.Addr{vpnAddr},
|
|
||||||
remotes: NewRemoteList([]netip.Addr{vpnAddr}, nil),
|
|
||||||
}
|
|
||||||
hostinfo.remotes.LearnRemote(vpnAddr, punchTargetV4)
|
|
||||||
hostinfo.SetRemote(punchTargetV6)
|
|
||||||
return hostinfo
|
|
||||||
}
|
|
||||||
|
|
||||||
func newPunchyForTest(t *testing.T, settings map[string]any, conn udp.Conn) *Punchy {
|
|
||||||
c := config.NewC(test.NewLogger())
|
|
||||||
c.Settings["punchy"] = settings
|
|
||||||
p := NewPunchyFromConfig(test.NewLogger(), c, conn)
|
|
||||||
|
|
||||||
hm := newHostMap(test.NewLogger())
|
|
||||||
hm.preferredRanges.Store(&[]netip.Prefix{})
|
|
||||||
p.hm = hm
|
|
||||||
p.lh = noLighthouses{}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPunchy_SendPunchReportsCurrentRemote(t *testing.T) {
|
|
||||||
t.Run("disabled", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
|
||||||
p := newPunchyForTest(t, map[string]any{"punch": false}, conn)
|
|
||||||
|
|
||||||
remote, err := p.SendPunch(newPunchTarget())
|
|
||||||
assert.False(t, remote.IsValid())
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Empty(t, conn.wrote)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("current remote", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
|
||||||
p := newPunchyForTest(t, map[string]any{"punch": true}, conn)
|
|
||||||
|
|
||||||
remote, err := p.SendPunch(newPunchTarget())
|
|
||||||
assert.Equal(t, punchTargetV6, remote)
|
|
||||||
require.ErrorIs(t, err, errNoRoute)
|
|
||||||
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("all remotes, current fails", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
|
|
||||||
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
|
|
||||||
|
|
||||||
remote, err := p.SendPunch(newPunchTarget())
|
|
||||||
assert.Equal(t, punchTargetV6, remote)
|
|
||||||
require.ErrorIs(t, err, errNoRoute)
|
|
||||||
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("all remotes, only another fails", func(t *testing.T) {
|
|
||||||
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV4: errNoRoute}}
|
|
||||||
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
|
|
||||||
|
|
||||||
remote, err := p.SendPunch(newPunchTarget())
|
|
||||||
assert.Equal(t, punchTargetV6, remote)
|
|
||||||
require.NoError(t, err, "a failure on a remote we are not using says nothing about the current one")
|
|
||||||
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -30,15 +30,20 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
|
|||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
sshRun, err := configSSH(l, ssh, c, r)
|
if _, err := configSSH(l, ssh, c, r); err != nil {
|
||||||
if err != nil {
|
|
||||||
l.Error("Failed to reconfigure the sshd", "error", err)
|
l.Error("Failed to reconfigure the sshd", "error", err)
|
||||||
ssh.Stop()
|
ssh.Stop()
|
||||||
r.Failed()
|
r.Failed()
|
||||||
|
return
|
||||||
}
|
}
|
||||||
if sshRun != nil {
|
// Bind here rather than in the background, so a reload that can't listen says so
|
||||||
go sshRun()
|
listener, err := ssh.Listen(c.GetString("sshd.listen", ""))
|
||||||
|
if err != nil {
|
||||||
|
l.Error("Failed to run the SSH server", "error", err)
|
||||||
|
r.Failed()
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
go ssh.Serve(listener)
|
||||||
} else {
|
} else {
|
||||||
ssh.Stop()
|
ssh.Stop()
|
||||||
}
|
}
|
||||||
|
|||||||
+20
-7
@@ -184,19 +184,34 @@ func (s *SSHServer) AddAuthorizedKey(user, pubKey string) error {
|
|||||||
// from the constructor-supplied ctx so a Stop+Run sequence (used by config reload) starts clean
|
// from the constructor-supplied ctx so a Stop+Run sequence (used by config reload) starts clean
|
||||||
// rather than carrying a permanently-cancelled context across runs.
|
// rather than carrying a permanently-cancelled context across runs.
|
||||||
func (s *SSHServer) Run(addr string) error {
|
func (s *SSHServer) Run(addr string) error {
|
||||||
if s.ctx.Err() != nil {
|
listener, err := s.Listen(addr)
|
||||||
return s.ctx.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
listener, err := net.Listen("tcp", addr)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
s.Serve(listener)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Listen binds addr for Serve, so a caller can tell the bind failed before serving in the background. Stop can close
|
||||||
|
// the listener from here on, Serve then returns as soon as it starts.
|
||||||
|
func (s *SSHServer) Listen(addr string) (net.Listener, error) {
|
||||||
|
if s.ctx.Err() != nil {
|
||||||
|
return nil, s.ctx.Err()
|
||||||
|
}
|
||||||
|
listener, err := net.Listen("tcp", addr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
// s.listener is the public handle Stop uses to interrupt the active run; listener (the local) is what
|
// s.listener is the public handle Stop uses to interrupt the active run; listener (the local) is what
|
||||||
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
|
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
|
||||||
// listener before this run's watcher fires, so each run must close its own listener via the local
|
// listener before this run's watcher fires, so each run must close its own listener via the local
|
||||||
// reference.
|
// reference.
|
||||||
s.listener.Store(&listener)
|
s.listener.Store(&listener)
|
||||||
|
return listener, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Serve accepts connections on a listener from Listen until Stop, or the context passed to NewSSHServer, ends it.
|
||||||
|
func (s *SSHServer) Serve(listener net.Listener) {
|
||||||
s.runs.Add(1)
|
s.runs.Add(1)
|
||||||
bound := listener.Addr()
|
bound := listener.Addr()
|
||||||
s.addr.Store(&bound)
|
s.addr.Store(&bound)
|
||||||
@@ -224,8 +239,6 @@ func (s *SSHServer) Run(addr string) error {
|
|||||||
s.run(runCtx, listener)
|
s.run(runCtx, listener)
|
||||||
|
|
||||||
s.l.Info("SSH server stopped listening")
|
s.l.Info("SSH server stopped listening")
|
||||||
// We don't return an error because run logs for us
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
|
func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
|
||||||
|
|||||||
@@ -279,3 +279,15 @@ func TestSSHExecExitStatusReflectsTheCommand(t *testing.T) {
|
|||||||
assert.Equal(t, 1, exit.ExitStatus())
|
assert.Equal(t, 1, exit.ExitStatus())
|
||||||
assert.True(t, strings.HasPrefix(string(out), "did not understand: nope\nAvailable commands:\n"), string(out))
|
assert.True(t, strings.HasPrefix(string(out), "did not understand: nope\nAvailable commands:\n"), string(out))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A reload that can't bind the new address says so, rather than finding out in the background
|
||||||
|
func TestSSHDReload_BusyPortFails(t *testing.T) {
|
||||||
|
h := newSSHDReloadHarness(t)
|
||||||
|
busy, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _ = busy.Close() })
|
||||||
|
|
||||||
|
res := h.c.ReloadConfigString(strings.Replace(h.config(), "127.0.0.1:0", busy.Addr().String(), 1))
|
||||||
|
require.NoError(t, res.Err)
|
||||||
|
assert.True(t, res.Failed)
|
||||||
|
}
|
||||||
@@ -140,8 +140,13 @@ func (s *statsServer) reload(c *config.C, initial bool) error {
|
|||||||
|
|
||||||
// Even with nothing running, a runtime whose listener failed can still be finishing its last pass
|
// Even with nothing running, a runtime whose listener failed can still be finishing its last pass
|
||||||
s.Stop()
|
s.Stop()
|
||||||
if enabled && !s.configTest {
|
// Bind here rather than in the background, so a reload that can't listen says so
|
||||||
go s.Start()
|
run, err := s.start()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if run != nil {
|
||||||
|
go run()
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -151,36 +156,64 @@ func (s *statsServer) reload(c *config.C, initial bool) error {
|
|||||||
// the HTTP listener. For graphite it blocks on the capture loop's context.
|
// the HTTP listener. For graphite it blocks on the capture loop's context.
|
||||||
// Safe to call when stats are disabled or already running (both no-op).
|
// Safe to call when stats are disabled or already running (both no-op).
|
||||||
func (s *statsServer) Start() {
|
func (s *statsServer) Start() {
|
||||||
if !s.enabled.Load() || s.configTest {
|
run, err := s.start()
|
||||||
|
if err != nil {
|
||||||
|
s.l.Error("Failed to start the prometheus stats listener", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if run != nil {
|
||||||
|
run()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// start claims the runtime and binds the prometheus listener. The returned run serves it and blocks until Stop or ctx
|
||||||
|
// ends it. A nil run and no error means stats are off or already running.
|
||||||
|
func (s *statsServer) start() (func(), error) {
|
||||||
|
if !s.enabled.Load() || s.configTest {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
s.runMu.Lock()
|
s.runMu.Lock()
|
||||||
|
defer s.runMu.Unlock()
|
||||||
if s.ctx.Err() != nil || s.run != nil || s.runCfg == nil {
|
if s.ctx.Err() != nil || s.run != nil || s.runCfg == nil {
|
||||||
s.runMu.Unlock()
|
return nil, nil
|
||||||
return
|
|
||||||
}
|
}
|
||||||
cfg := *s.runCfg
|
cfg := *s.runCfg
|
||||||
runCtx, cancel := context.WithCancel(s.ctx)
|
runCtx, cancel := context.WithCancel(s.ctx)
|
||||||
captureFns, listener := s.buildRuntime(runCtx, cfg)
|
captureFns, listener := s.buildRuntime(runCtx, cfg)
|
||||||
|
var ln net.Listener
|
||||||
|
if listener != nil {
|
||||||
|
var err error
|
||||||
|
ln, err = net.Listen("tcp", listener.Addr)
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
// Drop the config so a SIGHUP with the same one tries again once the problem is fixed
|
||||||
|
s.runCfg = nil
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
rt := &statsRuntime{cancel: cancel, listener: listener}
|
rt := &statsRuntime{cancel: cancel, listener: listener}
|
||||||
s.run = rt
|
s.run = rt
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
s.loopDone = done
|
s.loopDone = done
|
||||||
s.runMu.Unlock()
|
|
||||||
|
|
||||||
|
return func() { s.serve(runCtx, rt, captureFns, cfg.interval, ln, done) }, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// serve runs the capture loop, and the prometheus listener on ln, until Stop or ctx ends them.
|
||||||
|
func (s *statsServer) serve(runCtx context.Context, rt *statsRuntime, captureFns []func(), interval time.Duration, ln net.Listener, done chan struct{}) {
|
||||||
// Prime before serving, so a scrape that lands before the first tick sees real values instead of zeros (issue #907)
|
// Prime before serving, so a scrape that lands before the first tick sees real values instead of zeros (issue #907)
|
||||||
for _, fn := range captureFns {
|
for _, fn := range captureFns {
|
||||||
fn()
|
fn()
|
||||||
}
|
}
|
||||||
go captureStatsLoop(runCtx, cfg.interval, captureFns, done)
|
go captureStatsLoop(runCtx, interval, captureFns, done)
|
||||||
|
|
||||||
cleanExit := true
|
cleanExit := true
|
||||||
if listener == nil {
|
if rt.listener == nil {
|
||||||
// Graphite: no HTTP listener to serve; block until teardown.
|
// Graphite: no HTTP listener to serve; block until teardown.
|
||||||
<-runCtx.Done()
|
<-runCtx.Done()
|
||||||
} else {
|
} else {
|
||||||
cleanExit = s.serveListener(listener)
|
cleanExit = s.serveListener(rt.listener, ln)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clear our runtime only if nothing has replaced it. Stop races through
|
// Clear our runtime only if nothing has replaced it. Stop races through
|
||||||
@@ -189,10 +222,9 @@ func (s *statsServer) Start() {
|
|||||||
if s.run == rt {
|
if s.run == rt {
|
||||||
rt.cancel()
|
rt.cancel()
|
||||||
s.run = nil
|
s.run = nil
|
||||||
// A listener that exited with an error (e.g., bind conflict) leaves
|
// A listener that exited with an error leaves runCfg cached as if it
|
||||||
// runCfg cached as if it were applied. Drop it so a SIGHUP with the
|
// were applied. Drop it so a SIGHUP with the same config re-triggers
|
||||||
// same config re-triggers Start once the user fixes the underlying
|
// Start once the user fixes the underlying problem.
|
||||||
// problem.
|
|
||||||
if !cleanExit {
|
if !cleanExit {
|
||||||
s.runCfg = nil
|
s.runCfg = nil
|
||||||
}
|
}
|
||||||
@@ -200,12 +232,12 @@ func (s *statsServer) Start() {
|
|||||||
s.runMu.Unlock()
|
s.runMu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// serveListener runs ListenAndServe and ensures ctx cancellation unblocks it.
|
// serveListener serves listener on ln and ensures ctx cancellation unblocks it.
|
||||||
// Returns true if the listener exited cleanly (Stop, ctx cancellation, or any
|
// Returns true if the listener exited cleanly (Stop, ctx cancellation, or any
|
||||||
// other http.ErrServerClosed path), false on an unexpected error.
|
// other http.ErrServerClosed path), false on an unexpected error.
|
||||||
func (s *statsServer) serveListener(listener *http.Server) bool {
|
func (s *statsServer) serveListener(listener *http.Server, ln net.Listener) bool {
|
||||||
// Per-invocation watcher: ctx cancellation triggers a listener shutdown
|
// Per-invocation watcher: ctx cancellation triggers a listener shutdown
|
||||||
// which in turn unblocks ListenAndServe. Closing `done` on exit keeps
|
// which in turn unblocks Serve. Closing `done` on exit keeps
|
||||||
// the watcher from outliving this call.
|
// the watcher from outliving this call.
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
@@ -221,8 +253,8 @@ func (s *statsServer) serveListener(listener *http.Server) bool {
|
|||||||
}()
|
}()
|
||||||
defer close(done)
|
defer close(done)
|
||||||
|
|
||||||
s.l.Info("Starting prometheus stats listener", "addr", listener.Addr)
|
s.l.Info("Starting prometheus stats listener", "addr", ln.Addr())
|
||||||
err := listener.ListenAndServe()
|
err := listener.Serve(ln)
|
||||||
if err == nil || errors.Is(err, http.ErrServerClosed) {
|
if err == nil || errors.Is(err, http.ErrServerClosed) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -673,3 +673,28 @@ func TestStatsServer_Start_duringStopWaitsForThePass(t *testing.T) {
|
|||||||
waitFor(t, func() bool { return len(entered) == 1 })
|
waitFor(t, func() bool { return len(entered) == 1 })
|
||||||
assert.Equal(t, int64(1), most.Load(), "a Start captured alongside the pass Stop was waiting out")
|
assert.Equal(t, int64(1), most.Load(), "a Start captured alongside the pass Stop was waiting out")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A reload that can't bind says so rather than finding out in the background, and the same config tries again once
|
||||||
|
// the port is free
|
||||||
|
func TestStatsServer_reload_bindFailureFails(t *testing.T) {
|
||||||
|
blocker, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
port := strconv.Itoa(blocker.Addr().(*net.TCPAddr).Port)
|
||||||
|
|
||||||
|
s, c := newTestStatsServer(t)
|
||||||
|
stopAtCleanup(t, &s)
|
||||||
|
require.NoError(t, s.reload(c, true))
|
||||||
|
|
||||||
|
setStatsConfig(c, map[string]any{
|
||||||
|
"type": "prometheus",
|
||||||
|
"interval": "1s",
|
||||||
|
"listen": "127.0.0.1:" + port,
|
||||||
|
"path": "/metrics",
|
||||||
|
})
|
||||||
|
require.Error(t, s.reload(c, false))
|
||||||
|
assert.Nil(t, currentRuntime(s))
|
||||||
|
|
||||||
|
require.NoError(t, blocker.Close())
|
||||||
|
require.NoError(t, s.reload(c, false))
|
||||||
|
waitForListening(t, "127.0.0.1:"+port)
|
||||||
|
}
|
||||||
+2
-3
@@ -41,9 +41,8 @@ type Conn interface {
|
|||||||
//
|
//
|
||||||
// Returns the number of packets successfully written. A destination the kernel rejects costs only
|
// Returns the number of packets successfully written. A destination the kernel rejects costs only
|
||||||
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
|
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
|
||||||
// failed, when not nil, is called with the index of every packet a destination cost us and why.
|
|
||||||
// Not safe for concurrent use on the same Conn.
|
// Not safe for concurrent use on the same Conn.
|
||||||
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
|
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
|
||||||
// ReloadConfig applies the listen settings that can change, r is nil at startup
|
// ReloadConfig applies the listen settings that can change, r is nil at startup
|
||||||
ReloadConfig(c *config.C, r *config.Reload)
|
ReloadConfig(c *config.C, r *config.Reload)
|
||||||
SupportsMultipleReaders() bool
|
SupportsMultipleReaders() bool
|
||||||
@@ -67,7 +66,7 @@ func (NoopConn) SupportsMultipleReaders() bool {
|
|||||||
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
|
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
|
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
|
||||||
return len(bufs), nil
|
return len(bufs), nil
|
||||||
}
|
}
|
||||||
func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) {
|
func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) {
|
||||||
|
|||||||
+1
-4
@@ -139,7 +139,7 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||||
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
||||||
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
|
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
|
||||||
// silently drops the rest of a burst (linux blocks instead). Poll for
|
// silently drops the rest of a burst (linux blocks instead). Poll for
|
||||||
@@ -150,9 +150,6 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(
|
|||||||
written++
|
written++
|
||||||
} else {
|
} else {
|
||||||
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
||||||
if failed != nil {
|
|
||||||
failed(i, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
|
|||||||
+1
-4
@@ -44,7 +44,7 @@ func (u *GenericConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||||
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
||||||
written := 0
|
written := 0
|
||||||
for i, b := range bufs {
|
for i, b := range bufs {
|
||||||
@@ -52,9 +52,6 @@ func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed f
|
|||||||
written++
|
written++
|
||||||
} else {
|
} else {
|
||||||
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
||||||
if failed != nil {
|
|
||||||
failed(i, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
|
|||||||
+2
-2
@@ -358,8 +358,8 @@ func sendto(fd int, b []byte, addr netip.AddrPort, isV4 bool) error {
|
|||||||
|
|
||||||
// WriteBatch sends bufs via sendmmsg(2), coalescing same-destination runs into UDP-GSO superpackets when supported.
|
// WriteBatch sends bufs via sendmmsg(2), coalescing same-destination runs into UDP-GSO superpackets when supported.
|
||||||
// See batchWriter in udp_linux_writebatch.go for the mechanics.
|
// See batchWriter in udp_linux_writebatch.go for the mechanics.
|
||||||
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||||
return u.bw.WriteBatch(bufs, addrs, failed)
|
return u.bw.WriteBatch(bufs, addrs)
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeSockaddr encodes addr into buf (which must be at least SizeofSockaddrInet6 bytes).
|
// writeSockaddr encodes addr into buf (which must be at least SizeofSockaddrInet6 bytes).
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
package udp
|
package udp
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
@@ -97,22 +96,13 @@ func TestWriteBatchBadFamilyDeliversOthers(t *testing.T) {
|
|||||||
bufs := [][]byte{[]byte("AAA"), []byte("BBB"), []byte("CCC")}
|
bufs := [][]byte{[]byte("AAA"), []byte("BBB"), []byte("CCC")}
|
||||||
addrs := []netip.AddrPort{good, bad, good}
|
addrs := []netip.AddrPort{good, bad, good}
|
||||||
|
|
||||||
var failed []int
|
n, err := sender.WriteBatch(bufs, addrs)
|
||||||
n, err := sender.WriteBatch(bufs, addrs, func(i int, err error) {
|
|
||||||
if !IsUnreachable(err) {
|
|
||||||
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
|
|
||||||
}
|
|
||||||
failed = append(failed, i)
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch returned error, want nil (bad dest should be isolated): %v", err)
|
t.Fatalf("WriteBatch returned error, want nil (bad dest should be isolated): %v", err)
|
||||||
}
|
}
|
||||||
if n != 2 {
|
if n != 2 {
|
||||||
t.Errorf("WriteBatch wrote %d packets, want 2 of 3 (the bad-family dest is the only casualty)", n)
|
t.Errorf("WriteBatch wrote %d packets, want 2 of 3 (the bad-family dest is the only casualty)", n)
|
||||||
}
|
}
|
||||||
if want := []int{1}; !slices.Equal(failed, want) {
|
|
||||||
t.Errorf("failed reported %v, want %v", failed, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := map[string]bool{}
|
got := map[string]bool{}
|
||||||
rx.SetReadDeadline(time.Now().Add(2 * time.Second))
|
rx.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||||
@@ -164,7 +154,7 @@ func TestWriteBatchUnreachableDestDeliversOthers(t *testing.T) {
|
|||||||
addrs := []netip.AddrPort{good, good, bad, good, good}
|
addrs := []netip.AddrPort{good, good, bad, good, good}
|
||||||
|
|
||||||
// The bad destination is reported, but only after every other packet has been attempted.
|
// The bad destination is reported, but only after every other packet has been attempted.
|
||||||
if _, err := sender.WriteBatch(bufs, addrs, nil); err == nil {
|
if _, err := sender.WriteBatch(bufs, addrs); err == nil {
|
||||||
t.Log("WriteBatch returned nil; kernel accepted the reserved address, delivery assertions still apply")
|
t.Log("WriteBatch returned nil; kernel accepted the reserved address, delivery assertions still apply")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -386,7 +376,7 @@ func TestWriteBatchPartialSendRewind(t *testing.T) {
|
|||||||
return accept, nil
|
return accept, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := w.WriteBatch(bufs, addrs, nil)
|
written, err := w.WriteBatch(bufs, addrs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
@@ -439,22 +429,13 @@ func TestWriteBatchSkipUnroutableRunAccounting(t *testing.T) {
|
|||||||
return accept, nil
|
return accept, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var failed []int
|
written, err := w.WriteBatch(bufs, addrs)
|
||||||
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
|
|
||||||
if !IsUnreachable(err) {
|
|
||||||
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
|
|
||||||
}
|
|
||||||
failed = append(failed, i)
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
if written != 4 {
|
if written != 4 {
|
||||||
t.Errorf("written = %d, want 4 (the unroutable run is the only casualty)", written)
|
t.Errorf("written = %d, want 4 (the unroutable run is the only casualty)", written)
|
||||||
}
|
}
|
||||||
if want := []int{2}; !slices.Equal(failed, want) {
|
|
||||||
t.Errorf("failed reported %v, want %v", failed, want)
|
|
||||||
}
|
|
||||||
wantTags := []byte{1, 2, 4, 5}
|
wantTags := []byte{1, 2, 4, 5}
|
||||||
if len(wire) != len(wantTags) {
|
if len(wire) != len(wantTags) {
|
||||||
t.Fatalf("wire got %d packets, want %d (dup or loss around the skip)", len(wire), len(wantTags))
|
t.Fatalf("wire got %d packets, want %d (dup or loss around the skip)", len(wire), len(wantTags))
|
||||||
@@ -505,23 +486,13 @@ func TestWriteBatchMidChunkRejectResumes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var failed []int
|
written, err := w.WriteBatch(bufs, addrs)
|
||||||
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
|
|
||||||
if !errors.Is(err, unix.EPERM) {
|
|
||||||
t.Errorf("packet %d failed with %v, want EPERM", i, err)
|
|
||||||
}
|
|
||||||
failed = append(failed, i)
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
if written != 3 {
|
if written != 3 {
|
||||||
t.Errorf("written = %d, want 3 (B's rejected run is the only casualty)", written)
|
t.Errorf("written = %d, want 3 (B's rejected run is the only casualty)", written)
|
||||||
}
|
}
|
||||||
// Every packet in B's run is reported, not just the entry
|
|
||||||
if want := []int{2, 3}; !slices.Equal(failed, want) {
|
|
||||||
t.Errorf("failed reported %v, want %v", failed, want)
|
|
||||||
}
|
|
||||||
wantTags := []byte{1, 2, 5}
|
wantTags := []byte{1, 2, 5}
|
||||||
if len(wire) != len(wantTags) {
|
if len(wire) != len(wantTags) {
|
||||||
t.Fatalf("wire got %d packets, want %d (dup or loss around the mid-chunk reject)", len(wire), len(wantTags))
|
t.Fatalf("wire got %d packets, want %d (dup or loss around the mid-chunk reject)", len(wire), len(wantTags))
|
||||||
@@ -572,7 +543,7 @@ func TestWriteBatchMidChunkEIODisablesGSOWithoutDup(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := w.WriteBatch(bufs, addrs, nil)
|
written, err := w.WriteBatch(bufs, addrs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
@@ -600,7 +571,7 @@ func TestWriteBatchZeroProgress(t *testing.T) {
|
|||||||
w.sendFn = func(start, n int) (int, error) { return 0, nil }
|
w.sendFn = func(start, n int) (int, error) { return 0, nil }
|
||||||
bufs := [][]byte{make([]byte, 100)}
|
bufs := [][]byte{make([]byte, 100)}
|
||||||
addrs := []netip.AddrPort{netip.MustParseAddrPort("127.0.0.1:4242")}
|
addrs := []netip.AddrPort{netip.MustParseAddrPort("127.0.0.1:4242")}
|
||||||
if _, err := w.WriteBatch(bufs, addrs, nil); err == nil {
|
if _, err := w.WriteBatch(bufs, addrs); err == nil {
|
||||||
t.Fatal("WriteBatch = nil error on zero progress, want error")
|
t.Fatal("WriteBatch = nil error on zero progress, want error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -626,7 +597,7 @@ func TestWriteBatchEIODisablesGSOAndReplays(t *testing.T) {
|
|||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := w.WriteBatch(bufs, addrs, nil)
|
written, err := w.WriteBatch(bufs, addrs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
@@ -703,7 +674,7 @@ func TestGSOEngagesOnLoopback(t *testing.T) {
|
|||||||
addrs[i] = dst
|
addrs[i] = dst
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := sc.WriteBatch(bufs, addrs, nil)
|
written, err := sc.WriteBatch(bufs, addrs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,7 +86,7 @@ func TestWriteBatchOffloadsDisabledScripted(t *testing.T) {
|
|||||||
addrs[i] = dst
|
addrs[i] = dst
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := w.WriteBatch(bufs, addrs, nil)
|
written, err := w.WriteBatch(bufs, addrs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
@@ -163,7 +163,7 @@ func TestOffloadsDisabledOnLoopback(t *testing.T) {
|
|||||||
addrs[i] = dst
|
addrs[i] = dst
|
||||||
}
|
}
|
||||||
|
|
||||||
written, err := sc.WriteBatch(bufs, addrs, nil)
|
written, err := sc.WriteBatch(bufs, addrs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("WriteBatch: %v", err)
|
t.Fatalf("WriteBatch: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -190,8 +190,8 @@ func parseRelease(r string) (major, minor int) {
|
|||||||
// its packets are dropped and the rest of the chunk resumes in place.
|
// its packets are dropped and the rest of the chunk resumes in place.
|
||||||
//
|
//
|
||||||
// Returns the number of packets sent. An error means the call itself failed.
|
// Returns the number of packets sent. An error means the call itself failed.
|
||||||
// A short count means some destinations were undeliverable, failed hears about each of their packets.
|
// A short count means some destinations were undeliverable.
|
||||||
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||||
if len(bufs) != len(addrs) {
|
if len(bufs) != len(addrs) {
|
||||||
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
|
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
|
||||||
}
|
}
|
||||||
@@ -233,7 +233,6 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed f
|
|||||||
if w.l.Enabled(context.Background(), slog.LevelDebug) {
|
if w.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||||
w.l.Debug("skipping unroutable batch destination", "udpAddr", addrs[i], "packets", runLen, "error", err)
|
w.l.Debug("skipping unroutable batch destination", "udpAddr", addrs[i], "packets", runLen, "error", err)
|
||||||
}
|
}
|
||||||
reportFailed(failed, i, i+runLen, err)
|
|
||||||
i += runLen
|
i += runLen
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -301,23 +300,12 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed f
|
|||||||
"gso", w.gsoSupported,
|
"gso", w.gsoSupported,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
reportFailed(failed, w.entryEnd[done]-w.entryPkts[done], w.entryEnd[done], serr)
|
|
||||||
done++
|
done++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// reportFailed tells failed, if anyone is listening, that every packet in bufs[start:end] was lost to err
|
|
||||||
func reportFailed(failed func(i int, err error), start, end int, err error) {
|
|
||||||
if failed == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for i := start; i < end; i++ {
|
|
||||||
failed(i, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// planRun returns the length of the run starting at start and its segment
|
// planRun returns the length of the run starting at start and its segment
|
||||||
// size (len(bufs[start])). A run of length 1 carries no UDP_SEGMENT cmsg
|
// size (len(bufs[start])). A run of length 1 carries no UDP_SEGMENT cmsg
|
||||||
// and is sent as a plain datagram; without GSO support planRun always returns 1.
|
// and is sent as a plain datagram; without GSO support planRun always returns 1.
|
||||||
|
|||||||
@@ -80,11 +80,11 @@ func TestWriteBatchNoAllocs(t *testing.T) {
|
|||||||
|
|
||||||
var werr error
|
var werr error
|
||||||
// Warm-up outside the measured runs.
|
// Warm-up outside the measured runs.
|
||||||
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
|
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
|
||||||
t.Fatalf("WriteBatch warm-up: %v", err)
|
t.Fatalf("WriteBatch warm-up: %v", err)
|
||||||
}
|
}
|
||||||
allocs := testing.AllocsPerRun(100, func() {
|
allocs := testing.AllocsPerRun(100, func() {
|
||||||
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
|
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
|
||||||
werr = err
|
werr = err
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -319,7 +319,7 @@ func (u *RIOConn) WriteTo(buf []byte, ip netip.AddrPort) error {
|
|||||||
return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
|
return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||||
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
|
||||||
written := 0
|
written := 0
|
||||||
for i, b := range bufs {
|
for i, b := range bufs {
|
||||||
@@ -327,9 +327,6 @@ func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(
|
|||||||
written++
|
written++
|
||||||
} else {
|
} else {
|
||||||
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
||||||
if failed != nil {
|
|
||||||
failed(i, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
|
|||||||
+7
-64
@@ -6,7 +6,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -64,19 +63,10 @@ func acquirePacket() *Packet {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// testerAddrs is where a TesterConn sends from. A single address host sends everything from addr, whatever the
|
|
||||||
// destination. A dual stack host sends from the address matching the destination's family and has no route at all
|
|
||||||
// for a family it holds no address in.
|
|
||||||
type testerAddrs struct {
|
|
||||||
addr netip.AddrPort
|
|
||||||
v4, v6 netip.AddrPort
|
|
||||||
dual bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type TesterConn struct {
|
type TesterConn struct {
|
||||||
// addrs is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
|
// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
|
||||||
// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
|
// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
|
||||||
addrs atomic.Pointer[testerAddrs]
|
addr atomic.Pointer[netip.AddrPort]
|
||||||
|
|
||||||
RxPackets chan *Packet // Packets to receive into nebula
|
RxPackets chan *Packet // Packets to receive into nebula
|
||||||
TxPackets chan *Packet // Packets transmitted outside by nebula
|
TxPackets chan *Packet // Packets transmitted outside by nebula
|
||||||
@@ -104,53 +94,14 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
|
|||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAddr returns the underlay address this conn currently sends from. For a dual stack conn that is the IPv4
|
// GetAddr returns the underlay address this conn currently sends from.
|
||||||
// address, or the IPv6 one when there is no IPv4 service.
|
|
||||||
func (u *TesterConn) GetAddr() netip.AddrPort {
|
func (u *TesterConn) GetAddr() netip.AddrPort {
|
||||||
return u.addrs.Load().addr
|
return *u.addr.Load()
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
|
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
|
||||||
func (u *TesterConn) SetAddr(addr netip.AddrPort) {
|
func (u *TesterConn) SetAddr(addr netip.AddrPort) {
|
||||||
u.addrs.Store(&testerAddrs{addr: addr})
|
u.addr.Store(&addr)
|
||||||
}
|
|
||||||
|
|
||||||
// SetDualStackAddrs gives this conn one underlay address per family, standing in for a host whose routing table picks
|
|
||||||
// the source by destination. Pass an invalid AddrPort for a family the network does not offer, sends to that family
|
|
||||||
// then fail like they would on a host with no route for it.
|
|
||||||
func (u *TesterConn) SetDualStackAddrs(v4, v6 netip.AddrPort) {
|
|
||||||
a := &testerAddrs{v4: v4, v6: v6, dual: true}
|
|
||||||
if v4.IsValid() {
|
|
||||||
a.addr = v4
|
|
||||||
} else {
|
|
||||||
a.addr = v6
|
|
||||||
}
|
|
||||||
u.addrs.Store(a)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetAddrFor returns the underlay address a send to dst leaves from, GetAddr when there is no route to dst
|
|
||||||
func (u *TesterConn) GetAddrFor(dst netip.AddrPort) netip.AddrPort {
|
|
||||||
if src, err := u.sourceFor(dst); err == nil {
|
|
||||||
return src
|
|
||||||
}
|
|
||||||
return u.GetAddr()
|
|
||||||
}
|
|
||||||
|
|
||||||
// sourceFor returns the address a send to dst leaves from
|
|
||||||
func (u *TesterConn) sourceFor(dst netip.AddrPort) (netip.AddrPort, error) {
|
|
||||||
a := u.addrs.Load()
|
|
||||||
if !a.dual {
|
|
||||||
return a.addr, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
src := a.v6
|
|
||||||
if dst.Addr().Unmap().Is4() {
|
|
||||||
src = a.v4
|
|
||||||
}
|
|
||||||
if !src.IsValid() {
|
|
||||||
return netip.AddrPort{}, &net.OpError{Op: "write", Net: "udp", Addr: net.UDPAddrFromAddrPort(dst), Err: errNetUnreachable}
|
|
||||||
}
|
|
||||||
return src, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send will place a UdpPacket onto the receive queue for nebula to consume
|
// Send will place a UdpPacket onto the receive queue for nebula to consume
|
||||||
@@ -202,11 +153,6 @@ func (u *TesterConn) Get(block bool) *Packet {
|
|||||||
//********************************************************************************************************************//
|
//********************************************************************************************************************//
|
||||||
|
|
||||||
func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
||||||
from, err := u.sourceFor(addr)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
p := acquirePacket()
|
p := acquirePacket()
|
||||||
if cap(p.Data) < len(b) {
|
if cap(p.Data) < len(b) {
|
||||||
p.Data = make([]byte, len(b))
|
p.Data = make([]byte, len(b))
|
||||||
@@ -214,7 +160,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
|||||||
p.Data = p.Data[:len(b)]
|
p.Data = p.Data[:len(b)]
|
||||||
}
|
}
|
||||||
copy(p.Data, b)
|
copy(p.Data, b)
|
||||||
p.From = from
|
p.From = u.GetAddr()
|
||||||
p.To = addr
|
p.To = addr
|
||||||
select {
|
select {
|
||||||
case <-u.done:
|
case <-u.done:
|
||||||
@@ -224,16 +170,13 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
|
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
|
||||||
written := 0
|
written := 0
|
||||||
for i, b := range bufs {
|
for i, b := range bufs {
|
||||||
if err := u.WriteTo(b, addrs[i]); err == nil {
|
if err := u.WriteTo(b, addrs[i]); err == nil {
|
||||||
written++
|
written++
|
||||||
} else {
|
} else {
|
||||||
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
|
||||||
if failed != nil {
|
|
||||||
failed(i, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
|
|||||||
@@ -1,22 +0,0 @@
|
|||||||
//go:build !windows
|
|
||||||
|
|
||||||
package udp
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"syscall"
|
|
||||||
)
|
|
||||||
|
|
||||||
// errNetUnreachable is the error this platform's sockets give for a destination with no route
|
|
||||||
var errNetUnreachable error = syscall.ENETUNREACH
|
|
||||||
|
|
||||||
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
|
|
||||||
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
|
|
||||||
// to an IPv6 remote into one of these.
|
|
||||||
func IsUnreachable(err error) bool {
|
|
||||||
return errors.Is(err, syscall.ENETUNREACH) ||
|
|
||||||
errors.Is(err, syscall.EHOSTUNREACH) ||
|
|
||||||
errors.Is(err, syscall.EADDRNOTAVAIL) ||
|
|
||||||
errors.Is(err, syscall.EAFNOSUPPORT) ||
|
|
||||||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
|
|
||||||
}
|
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
//go:build !windows
|
|
||||||
|
|
||||||
package udp
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net"
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestIsUnreachable(t *testing.T) {
|
|
||||||
wrap := func(err error) error { return &net.OpError{Op: "write", Net: "udp", Err: err} }
|
|
||||||
|
|
||||||
for _, err := range []error{
|
|
||||||
wrap(syscall.ENETUNREACH),
|
|
||||||
wrap(syscall.EHOSTUNREACH),
|
|
||||||
wrap(syscall.EADDRNOTAVAIL),
|
|
||||||
wrap(syscall.EAFNOSUPPORT),
|
|
||||||
ErrInvalidIPv6RemoteForSocket,
|
|
||||||
} {
|
|
||||||
if !IsUnreachable(err) {
|
|
||||||
t.Errorf("IsUnreachable(%v) = false, want true", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, err := range []error{
|
|
||||||
nil,
|
|
||||||
wrap(syscall.EPERM),
|
|
||||||
wrap(syscall.ENOBUFS),
|
|
||||||
wrap(syscall.EMSGSIZE),
|
|
||||||
net.ErrClosed,
|
|
||||||
} {
|
|
||||||
if IsUnreachable(err) {
|
|
||||||
t.Errorf("IsUnreachable(%v) = true, want false", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
package udp
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
|
|
||||||
"golang.org/x/sys/windows"
|
|
||||||
)
|
|
||||||
|
|
||||||
// errNetUnreachable is the error this platform's sockets give for a destination with no route
|
|
||||||
var errNetUnreachable error = windows.WSAENETUNREACH
|
|
||||||
|
|
||||||
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
|
|
||||||
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
|
|
||||||
// to an IPv6 remote into one of these.
|
|
||||||
func IsUnreachable(err error) bool {
|
|
||||||
return errors.Is(err, windows.WSAENETUNREACH) ||
|
|
||||||
errors.Is(err, windows.WSAEHOSTUNREACH) ||
|
|
||||||
errors.Is(err, windows.WSAEADDRNOTAVAIL) ||
|
|
||||||
errors.Is(err, windows.WSAEAFNOSUPPORT) ||
|
|
||||||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
|
|
||||||
}
|
|
||||||
Reference in new issue
Block a user