Compare commits

...
Author SHA1 Message Date
JackDoan c543718ee0 Log probes at debug, skip tunnel lookups on lighthouses, per-case recovery bounds, changelog 2026-10-20 16:18:54 -05:00
JackDoan 44e3365d4f Size send failure heads from the header length and tolerate a missing lighthouse 2026-10-20 15:47:43 -05:00
JackDoan 995f22ca53 Fail dual stack test sends with the platform's own unreachable error 2026-10-20 14:23:36 -05:00
JackDoan b69cf4ecb4 Find a refused packet's tunnel from its header, ask the lighthouse when probing, and address review 2026-10-20 14:17:13 -05:00
JackDoan 0f07faab0e Probe for another path when a keepalive punch finds a tunnel's remote unreachable 2026-10-20 13:46:26 -05:00
JackDoan 2fd9966fef Probe for another path when the data path finds a tunnel's remote unreachable 2026-10-20 13:34:01 -05:00
JackDoan 0ff077fea3 Probe a tunnel's other remotes when its current one is unreachable 2026-10-20 13:10:31 -05:00
omrzk f552cebc51 handshake: lower log level for blocklisted certificates (#1969)
smoke-extra / freebsd-amd64 (push) Failing after 23s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 14s
smoke-extra / netbsd-amd64 (push) Failing after 12s
smoke-extra / openbsd-amd64 (push) Failing after 11s
smoke-extra / linux-386 (push) Failing after 13s
smoke / Run multi node smoke test (push) Failing after 1m42s
Build and test / Static checks (push) Successful in 2m15s
Build and test / Test linux (push) Failing after 2m17s
Build and test / Test linux-pkcs11 (push) Failing after 2m10s
Build and test / Test linux-boringcrypto (push) Failing after 2m58s
Build and test / Test linux-fips140 (push) Failing after 2m54s
Build and test / Cross-build linux-arm (push) Successful in 3m6s
Build and test / Cross-build linux-mips (push) Successful in 3m46s
Build and test / Cross-build linux-other (push) Successful in 3m10s
Build and test / Cross-build windows (push) Successful in 1m3s
Build and test / Cross-build freebsd (push) Successful in 1m33s
Build and test / Cross-build netbsd (push) Successful in 1m34s
Build and test / Cross-build openbsd (push) Successful in 1m33s
Build and test / Cross-build mobile (push) Successful in 3m28s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-20 11:32:03 -04:00
Nate Brown b04d71d96a Compare a firewall reload with what the running firewall was built from (#1955) 2026-10-20 10:37:48 -04:00
Jack Doan cacfa8fac4 darwin: Add an unsafe_route once the route holding its prefix is removed (#1942) 2026-10-20 09:37:20 -05:00
36 changed files with 2277 additions and 193 deletions

No files matched your search

+11
View File
@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
### Changed
- A tunnel whose remote becomes unreachable, such as an IPv6 remote after moving to a network without IPv6, now
moves to another of the peer's addresses within a round trip or two instead of waiting 20-35s to be torn down
and rehandshaked. When a send fails because there is no route to the remote, the tunnel sends a test packet to
every other address it knows for the peer and asks the lighthouse to have the peer punch its new address.
Whichever path answers carries the tunnel. With `punchy.punch` enabled, idle tunnels notice on their next
keepalive punch. An idle tunnel whose remote is unreachable and none of whose other addresses answer is now
torn down by the connection manager rather than left in place. Not yet effective for relayed traffic or on
Windows with the default RIO backend.
### Removed ### Removed
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and - The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
+15
View File
@@ -86,6 +86,21 @@ func (c *C) InitialLoad() bool {
return c.oldSettings == nil return c.oldSettings == nil
} }
// Render returns keys and their values as yaml, for a subsystem to remember what it applied. Empty means they couldn't
// be rendered and should never match.
func (c *C) Render(keys ...string) string {
m := make(map[string]any, len(keys))
for _, k := range keys {
m[k] = c.Get(k)
}
b, err := yaml.Marshal(m)
if err != nil {
c.l.Debug("Failed to render config", "keys", keys, "error", err)
return ""
}
return string(b)
}
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of // HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
// k in both the old and new settings will be serialized, the result of the string comparison is returned. // k in both the old and new settings will be serialized, the result of the string comparison is returned.
// If k is an empty string the entire config is tested. // If k is an empty string the entire config is tested.
+28 -12
View File
@@ -14,6 +14,7 @@ import (
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
) )
type trafficDecision int type trafficDecision int
@@ -171,7 +172,7 @@ func (cm *connectionManager) Start(ctx context.Context) {
} }
func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) { func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now) decision, hostinfo, primary, punch := cm.makeTrafficDecision(localIndex, now)
switch decision { switch decision {
case deleteTunnel: case deleteTunnel:
@@ -196,6 +197,20 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
} }
cm.resetRelayTrafficCheck(hostinfo) cm.resetRelayTrafficCheck(hostinfo)
if punch != nil {
cm.sendPunch(punch)
}
}
// sendPunch keeps a tunnel's NAT state alive when we are not sending on it. When we have no route to its remote any
// more, typically after moving to a network without that address family, the tunnel probes its other remotes so the
// peer roams over to us. Nothing else would notice, a tunnel we are not sending on has nothing else that could fail.
func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
remote, err := cm.punchy.SendPunch(hostinfo)
if udp.IsUnreachable(err) {
hostinfo.remoteUnreachable(cm.intf, remote)
}
} }
func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) { func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
@@ -313,7 +328,9 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
} }
} }
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) { // makeTrafficDecision decides what to do with a tunnel under the hostmap lock. It also returns the tunnel to punch when
// we are not sending on it, for the caller to send once the lock is released.
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo, *HostInfo) {
// Read lock the main hostmap to order decisions based on tunnels being the primary tunnel // Read lock the main hostmap to order decisions based on tunnels being the primary tunnel
cm.hostMap.RLock() cm.hostMap.RLock()
defer cm.hostMap.RUnlock() defer cm.hostMap.RUnlock()
@@ -321,17 +338,17 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
hostinfo := cm.hostMap.Indexes[localIndex] hostinfo := cm.hostMap.Indexes[localIndex]
if hostinfo == nil { if hostinfo == nil {
cm.l.Debug("Not found in hostmap", "localIndex", localIndex) cm.l.Debug("Not found in hostmap", "localIndex", localIndex)
return doNothing, nil, nil return doNothing, nil, nil, nil
} }
if cm.isInvalidCertificate(now, hostinfo) { if cm.isInvalidCertificate(now, hostinfo) {
return closeTunnel, hostinfo, nil return closeTunnel, hostinfo, nil, nil
} }
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages { if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error. // Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted") hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
return deleteTunnel, hostinfo, nil return deleteTunnel, hostinfo, nil, nil
} }
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]] primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
@@ -368,10 +385,10 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
if !outTraffic { if !outTraffic {
// Send a punch packet to keep the NAT state alive // Send a punch packet to keep the NAT state alive
cm.punchy.SendPunch(hostinfo) return decision, hostinfo, primary, hostinfo
} }
return decision, hostinfo, primary return decision, hostinfo, primary, nil
} }
if hostinfo.isPendingDeletion() { if hostinfo.isPendingDeletion() {
@@ -380,7 +397,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
"tunnelCheck", m{"state": "dead", "method": "active"}, "tunnelCheck", m{"state": "dead", "method": "active"},
) )
return deleteTunnel, hostinfo, nil return deleteTunnel, hostinfo, nil, nil
} }
decision := doNothing decision := doNothing
@@ -394,14 +411,13 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
"primary", mainHostInfo, "primary", mainHostInfo,
) )
return closeTunnel, hostinfo, primary return closeTunnel, hostinfo, primary, nil
} }
// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel. // If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
// Just maintain NAT state if configured to do so. // Just maintain NAT state if configured to do so.
cm.punchy.SendPunch(hostinfo)
cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval) cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
return doNothing, nil, nil return doNothing, nil, nil, hostinfo
} }
// We aren't receiving traffic but we are sending it. The outbound // We aren't receiving traffic but we are sending it. The outbound
@@ -427,7 +443,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
hostinfo.setPendingDeletion(true) hostinfo.setPendingDeletion(true)
cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval) cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
return decision, hostinfo, nil return decision, hostinfo, nil, nil
} }
func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) { func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) {
+247 -6
View File
@@ -3,13 +3,18 @@ package nebula
import ( import (
"crypto/ed25519" "crypto/ed25519"
"crypto/rand" "crypto/rand"
"io"
"net/netip" "net/netip"
"testing" "testing"
"time" "time"
"github.com/flynn/noise"
"github.com/rcrowley/go-metrics" "github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/overlaytest" "github.com/slackhq/nebula/overlay/overlaytest"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
@@ -266,12 +271,12 @@ func Test_NewConnectionManager_CounterLimits(t *testing.T) {
// Still below the reject limit, the tunnel stays up // Still below the reject limit, the tunnel stays up
nc.In(hostinfo) nc.In(hostinfo)
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now()) decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, tryRehandshake, decision) assert.Equal(t, tryRehandshake, decision)
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify // At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages) hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now()) decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, deleteTunnel, decision) assert.Equal(t, deleteTunnel, decision)
} }
@@ -333,14 +338,14 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.True(t, (hostinfo.state.Load()&stateIn != 0)) assert.True(t, (hostinfo.state.Load()&stateIn != 0))
now := time.Now() now := time.Now()
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now) decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision) assert.Equal(t, tryRehandshake, decision)
assert.Equal(t, now, hostinfo.lastUsed) assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion()) assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.sentSinceCheck()) assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, (hostinfo.state.Load()&stateIn != 0)) assert.False(t, (hostinfo.state.Load()&stateIn != 0))
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5)) decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
assert.Equal(t, doNothing, decision) assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed) assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion()) assert.False(t, hostinfo.isPendingDeletion())
@@ -348,7 +353,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.False(t, (hostinfo.state.Load()&stateIn != 0)) assert.False(t, (hostinfo.state.Load()&stateIn != 0))
// Do another traffic check tick, should still not be pending deletion // Do another traffic check tick, should still not be pending deletion
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10)) decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
assert.Equal(t, doNothing, decision) assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed) assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion()) assert.False(t, hostinfo.isPendingDeletion())
@@ -358,7 +363,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
// Finally advance beyond the inactivity timeout // Finally advance beyond the inactivity timeout
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10)) decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
assert.Equal(t, closeTunnel, decision) assert.Equal(t, closeTunnel, decision)
assert.Equal(t, now, hostinfo.lastUsed) assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion()) assert.False(t, hostinfo.isPendingDeletion())
@@ -578,3 +583,239 @@ func (d *dummyCert) MarshalJSON() ([]byte, error) {
func (d *dummyCert) Copy() cert.Certificate { func (d *dummyCert) Copy() cert.Certificate {
return d return d
} }
// newPunchTestConnectionManager is a connection manager punching through conn, holding the tunnel from newPunchTarget
func newPunchTestConnectionManager(t *testing.T, conn udp.Conn) (*connectionManager, *HostInfo) {
l := test.NewLogger()
hostMap := newHostMap(l)
hostMap.preferredRanges.Store(&[]netip.Prefix{})
lh := newTestLighthouse()
ifce := &Interface{
hostMap: hostMap,
inside: &overlaytest.NoopTun{},
outside: &udp.NoopConn{},
firewall: &Firewall{},
lightHouse: lh,
pki: &PKI{},
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
l: l,
}
ifce.pki.cs.Store(&CertState{
initiatingVersion: cert.Version1,
privateKey: []byte{},
v1Cert: &dummyCert{version: cert.Version1},
})
punchy := newPunchyForTest(t, map[string]any{"punch": true}, conn)
nc := newConnectionManagerFromConfig(l, config.NewC(l), hostMap, punchy)
nc.intf = ifce
ifce.connectionManager = nc
// Tunnel sends go through conn too, though without a key from withTestKey they never get as far as writing
ifce.writers = []udp.Conn{conn}
ifce.metricTxDropped = metrics.NilCounter{}
hostinfo := newPunchTarget()
hostinfo.localIndexId = 1099
hostinfo.remoteIndexId = 9901
hostinfo.ConnectionState = &ConnectionState{myCert: &dummyCert{version: cert.Version1}}
nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
return nc, hostinfo
}
// A tunnel we are not sending on gets punched whether or not it is receiving, the punch is the only thing that
// would notice its remote has become unreachable.
func Test_makeTrafficDecision_punchesWhenNotSending(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
now := time.Now()
// Receiving but not sending
hostinfo.takeTraffic()
nc.In(hostinfo)
decision, _, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Same(t, hostinfo, punch)
// Sending and receiving, our own traffic keeps the NAT state alive
nc.Out(hostinfo)
nc.In(hostinfo)
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Nil(t, punch)
// Unused
decision, decided, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, doNothing, decision)
assert.Nil(t, decided, "an unused tunnel has no relay usage to reset")
assert.Same(t, hostinfo, punch)
// Sending but not receiving gets a test packet instead, which finds an unreachable remote on its own
nc.Out(hostinfo)
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, sendTestPacket, decision)
assert.Nil(t, punch)
}
func Test_sendPunch_probesWhenRemoteUnreachable(t *testing.T) {
t.Run("unreachable", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
"the probe should ask the lighthouse to have the peer punch us")
// Within the probe interval nothing more goes out, however many sends fail
nc.sendPunch(hostinfo)
nc.sendPunch(hostinfo)
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("delivered", func(t *testing.T) {
conn := &recordingConn{}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("other failure", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: io.ErrShortWrite}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "only a missing route says to look elsewhere")
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
}
// A packet the data path had no route for names its tunnel in its header, the peer's index for it
func Test_sendUnreachable_findsTunnelFromHeader(t *testing.T) {
head := func(st header.MessageSubType, remoteIndex uint32) []byte {
return header.Encode(make([]byte, header.Len), header.Version, header.Message, st, remoteIndex, 1)
}
t.Run("direct", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV6)
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("not its remote", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV4)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "a failure on a remote the tunnel is not using is stale")
})
t.Run("relayed", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
// A relay header's index is the relay session's, it must not be taken for a tunnel's even when they collide
nc.intf.sendUnreachable(head(header.MessageRelay, hostinfo.remoteIndexId), punchTargetV6)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("unknown tunnel", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId+1), punchTargetV6)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
})
}
// withTestKey gives a tunnel a real key so its sends get encrypted and written instead of returning early
func withTestKey(hostinfo *HostInfo) {
cs := noise.NewCipherSuite(noise.DH25519, noise.CipherAESGCM, noise.HashSHA256)
hostinfo.ConnectionState.eKey = noiseutil.NewCipherState(noise.UnsafeNewCipherState(cs, [32]byte{}, 0), noise.CipherAESGCM)
}
// The probe itself, end to end from a failed send: a test packet to every remote but the one we cannot reach, and a
// lighthouse query, through both places a send can fail.
func Test_remoteUnreachable_probesOtherRemotes(t *testing.T) {
t.Run("control send", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV4}, conn.wrote, "the failed send, then the probe")
assert.Equal(t, []header.MessageType{header.Test, header.Test}, conn.types)
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("data path batch", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
// Two of the tunnel's packets fail in one batch, they probe once
sb := batch.NewSendBatch(conn, 4, 4096)
for c := uint64(1); c <= 2; c++ {
pkt := header.Encode(sb.Reserve(64), header.Version, header.Message, header.MessageNone, hostinfo.remoteIndexId, c)
sb.Commit(append(pkt, make([]byte, 32)...), punchTargetV6)
}
nc.intf.flushSendBatch(sb, 0)
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6, punchTargetV4}, conn.wrote,
"the failed packets, then one probe")
assert.Equal(t, header.Test, conn.types[2])
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("closing", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.SendMessageToHostInfo(header.CloseTunnel, 0, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a tunnel being torn down does not probe")
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("lighthouse", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.lightHouse.amLighthouse = true
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a lighthouse does not roam its clients")
})
t.Run("nothing else to probe", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.reQueryWait.Store(int64(time.Minute))
// Forget the IPv4 remote, all the tunnel knows is the one it cannot reach
hostinfo.remotes = NewRemoteList(hostinfo.vpnAddrs, nil)
hostinfo.remotes.LearnRemote(hostinfo.vpnAddrs[0], punchTargetV6)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
"the lighthouse is the only way to learn of another remote")
// Past the probe interval, but the lighthouse is only asked again on the requery schedule
hostinfo.nextUnreachableProbe.Store(0)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Empty(t, drainQueries(nc.intf.lightHouse))
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6}, conn.wrote)
})
}
// drainQueries returns every lighthouse query waiting to go out
func drainQueries(lh *LightHouse) []netip.Addr {
var queries []netip.Addr
for {
select {
case addr := <-lh.queryChan:
queries = append(queries, addr)
default:
return queries
}
}
}
+1 -1
View File
@@ -148,7 +148,7 @@ func (c *fakeConn) Rebind() error { c.rebinds++; ret
func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil } func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil }
func (c *fakeConn) ListenOut(_ udp.EncReader, _ func()) error { return nil } func (c *fakeConn) ListenOut(_ udp.EncReader, _ func()) error { return nil }
func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil } func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) { func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
return len(bufs), nil return len(bufs), nil
} }
func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {} func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {}
+12
View File
@@ -117,6 +117,18 @@ func (c *Control) SetUDPAddr(addr netip.AddrPort) {
c.f.outside.(*udp.TesterConn).SetAddr(addr) c.f.outside.(*udp.TesterConn).SetAddr(addr)
} }
// GetUDPAddrFor returns the underlay address this node sends to dst from, which differs by family for a dual stack node
func (c *Control) GetUDPAddrFor(dst netip.AddrPort) netip.AddrPort {
return c.f.outside.(*udp.TesterConn).GetAddrFor(dst)
}
// SetDualStackUDPAddrs gives this node one underlay address per family, sending from whichever matches the
// destination. An invalid AddrPort leaves the node without that family, sends to it fail with ENETUNREACH. Register
// any address the node did not start on with the router as well or nothing will route back.
func (c *Control) SetDualStackUDPAddrs(v4, v6 netip.AddrPort) {
c.f.outside.(*udp.TesterConn).SetDualStackAddrs(v4, v6)
}
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of // SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker. // whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) { func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
+305
View File
@@ -0,0 +1,305 @@
//go:build e2e_testing
package e2e
import (
"net/netip"
"sync/atomic"
"testing"
"time"
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/e2e/router"
"github.com/slackhq/nebula/udp"
)
// TestDualStackRoamToV4Only measures how long a laptop's tunnels take to come back after it moves from a dual stack
// network, where its tunnels formed over IPv6, to one with no IPv6 service at all: the office to hotel wifi. Every
// peer still has an IPv4 address the laptop can reach, so nothing about the move should strand a tunnel.
//
// It is a measurement, not a pass/fail assertion, and it runs on the production connection manager timers.
//
// go test -tags e2e_testing -v -run TestDualStackRoamToV4Only ./e2e/
func TestDualStackRoamToV4Only(t *testing.T) {
for _, tc := range []dualStackRoamCase{
{name: "both send", meSends: true, themSends: true},
{name: "both send, rebind", meSends: true, themSends: true, rebind: true},
{name: "both send, two tunnels", meSends: true, themSends: true, simultaneous: true},
{name: "laptop sends", meSends: true},
{name: "peer sends", themSends: true},
{name: "peer sends, rebind", themSends: true, rebind: true},
{name: "peer sends, lh over v4", themSends: true, lhOverV4: true},
{name: "peer sends, punchy", themSends: true, punchy: true},
{name: "peer sends, punchy all", themSends: true, punchy: true, punchAll: true},
{name: "laptop sends, peer NAT", meSends: true, peerNAT: true},
{name: "laptop sends, peer NAT, lh v4", meSends: true, peerNAT: true, lhOverV4: true},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fwd, rev := measureDualStackRoam(t, tc)
t.Logf("RESULT %-24s laptop->peer %-9v peer->laptop %-9v", tc.name, fmtRecovery(fwd), fmtRecovery(rev))
})
}
}
// TestDualStackRoamRecovers holds the line TestDualStackRoamToV4Only measures: a tunnel whose remote became unreachable
// moves to the peer's other address instead of waiting out the connection manager to rehandshake, which takes 20s.
func TestDualStackRoamRecovers(t *testing.T) {
for _, tc := range []struct {
dualStackRoamCase
// within bounds recovery in each direction that has traffic
within time.Duration
}{
// Measures 0.4s. Under the 5s connection manager check, so it fails if the data path stops probing.
{dualStackRoamCase{name: "both send", meSends: true, themSends: true}, 2 * time.Second},
// Measures 2.6s or 3.8s, depending on whether a probe beats the peer's punch. Half the 20s it used to take.
{dualStackRoamCase{name: "laptop sends, peer NAT", meSends: true, peerNAT: true}, 10 * time.Second},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fwd, rev := measureDualStackRoam(t, tc.dualStackRoamCase)
if fwd == 0 || fwd > tc.within {
t.Errorf("laptop->peer recovered in %v, want within %v", fmtRecovery(fwd), tc.within)
}
if tc.themSends && (rev == 0 || rev > tc.within) {
t.Errorf("peer->laptop recovered in %v, want within %v", fmtRecovery(rev), tc.within)
}
})
}
}
type dualStackRoamCase struct {
name string
// meSends and themSends say who keeps traffic flowing after the move
meSends, themSends bool
// rebind fires RebindUDPServer on the move, as the darwin network change monitor would
rebind bool
// lhOverV4 forms the lighthouse tunnel over IPv4 at the office, so the move does not break it
lhOverV4 bool
// simultaneous has both sides initiate at once at the office, which can leave each side a different primary
// tunnel for the other
simultaneous bool
// punchy turns on punchy.punch everywhere, punchAll adds punchy.target_all_remotes
punchy, punchAll bool
// peerNAT puts the peer behind a NAT that, after the move, only lets in traffic from addresses the peer has sent to
peerNAT bool
}
func fmtRecovery(d time.Duration) string {
if d == 0 {
return "never"
}
return d.Round(time.Millisecond).String()
}
type dualStackNode struct {
c *nebula.Control
vpnAddr netip.Addr
v4, v6 netip.AddrPort
adv atomic.Pointer[[]netip.Addr]
}
// move puts the node on a new network, advertising only what it can be reached on there
func (n *dualStackNode) move(v4, v6 netip.AddrPort) {
n.v4, n.v6 = v4, v6
n.c.SetDualStackUDPAddrs(v4, v6)
var adv []netip.Addr
for _, a := range []netip.AddrPort{v4, v6} {
if a.IsValid() {
adv = append(adv, a.Addr())
}
}
n.adv.Store(&adv)
}
func newDualStackNode(ca cert.Certificate, caKey []byte, name, vpnNet string, v4, v6 netip.AddrPort, overrides m) *dualStackNode {
c, vpnNets, _, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, name, vpnNet, v4, overrides)
n := &dualStackNode{c: c, vpnAddr: vpnNets[0].Addr()}
n.move(v4, v6)
c.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
return *n.adv.Load()
})
return n
}
// measureDualStackRoam returns how long after the move the first packet made it laptop->peer and peer->laptop, zero
// for a direction that never recovered.
func measureDualStackRoam(t *testing.T, tc dualStackRoamCase) (time.Duration, time.Duration) {
t.Helper()
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
ap := netip.MustParseAddrPort
lhV4, lhV6 := ap("10.0.0.1:4242"), ap("[fd00::1]:4242")
lh := newDualStackNode(ca, caKey, "lh", "10.128.0.1/24", lhV4, lhV6, m{
"lighthouse": m{"am_lighthouse": true},
})
// Production timers rather than the 2s the harness defaults to, this is about how long a real laptop waits
peerCfg := m{
"lighthouse": m{
"hosts": []any{lh.vpnAddr.String()},
"interval": 10,
},
"static_host_map": m{
lh.vpnAddr.String(): []any{lhV4.String(), lhV6.String()},
},
"timers": m{
"connection_alive_interval": 5,
"pending_deletion_interval": 10,
},
"punchy": m{
"punch": tc.punchy,
"target_all_remotes": tc.punchAll,
},
}
me := newDualStackNode(ca, caKey, "me", "10.128.0.2/24", ap("10.0.0.2:4242"), ap("[fd00::2]:4242"), peerCfg)
them := newDualStackNode(ca, caKey, "them", "10.128.0.3/24", ap("10.0.0.3:4242"), ap("[fd00::3]:4242"), peerCfg)
r := router.NewR(t, lh.c, me.c, them.c)
r.CancelFlowLogs()
for _, n := range []*dualStackNode{lh, me, them} {
r.AddRoute(n.v6.Addr(), n.v6.Port(), n.c)
}
defer func() {
lh.c.Stop()
me.c.Stop()
them.c.Stop()
}()
lh.c.Start()
me.c.Start()
them.c.Start()
// At the office IPv6 wins every race, so everything forms over it. The lighthouse tunnel optionally does not,
// for a static_host_map that only lists the lighthouse's IPv4 address.
officeV6 := func(p *udp.Packet, _ *nebula.Control) router.ExitType {
lhTraffic := p.To == lhV4 || p.To == lhV6 || p.From == lhV4 || p.From == lhV6
if p.To.Addr().Is4() == (tc.lhOverV4 && lhTraffic) {
return router.KeepRouting
}
return router.Drop
}
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
// One side at a time, so both sides hold the same single tunnel, unless the case wants them racing
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("establish")))
if !tc.simultaneous {
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
}
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("establish")))
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
if tc.simultaneous && me.c.GetHostmapIndexCount() < 3 {
// A tunnel to the lighthouse and two to the peer, or the race did not happen
t.Logf("simultaneous handshakes settled on a single tunnel this run")
}
drainTun(me.c)
drainTun(them.c)
for _, want := range []struct {
name string
c *nebula.Control
vpnAddr netip.Addr
v6 bool
}{
{"laptop->peer", me.c, them.vpnAddr, true},
{"peer->laptop", them.c, me.vpnAddr, true},
{"laptop->lh", me.c, lh.vpnAddr, !tc.lhOverV4},
} {
hi := want.c.GetHostInfoByVpnAddr(want.vpnAddr, false)
if hi == nil {
t.Fatalf("%s: no tunnel at the office", want.name)
}
if hi.CurrentRemote.Addr().Is6() != want.v6 {
t.Fatalf("%s: tunnel formed over %v, the scenario wants v6=%v", want.name, hi.CurrentRemote, want.v6)
}
}
// Hotel wifi: a new NAT'd IPv4 address and no IPv6 at all. The office addresses are gone, nothing reaches them.
officeV4, officeV6Addr := me.v4, me.v6
hotel := ap("10.0.0.99:4242")
me.move(hotel, netip.AddrPort{})
r.AddRoute(hotel.Addr(), hotel.Port(), me.c)
start := time.Now()
if tc.rebind {
me.c.RebindUDPServer()
}
var fwd, rev atomic.Int64
stop := make(chan struct{})
defer close(stop)
watchTun(them.c, stop, start, &fwd)
watchTun(me.c, stop, start, &rev)
go func() {
tick := time.NewTicker(time.Millisecond * 200)
defer tick.Stop()
for {
select {
case <-stop:
return
case <-tick.C:
if tc.meSends {
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("retry")))
}
if tc.themSends {
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("retry")))
}
}
}
}()
// When only one side has traffic, the other direction recovering is a bonus we still report but do not wait for
done := func() bool {
return (!tc.meSends || fwd.Load() != 0) && (!tc.themSends || rev.Load() != 0)
}
// The peer's NAT has state for the lighthouse, which its own updates keep fresh, and for wherever it sends from here
natOpen := map[netip.AddrPort]bool{lhV4: true, lhV6: true}
isPeer := func(a netip.AddrPort) bool { return a == them.v4 || a == them.v6 }
r.RouteForAllExitFuncOrTimeout(time.Second*90, func(p *udp.Packet, _ *nebula.Control) router.ExitType {
if p.To == officeV4 || p.To == officeV6Addr {
return router.Drop
}
if tc.peerNAT {
if isPeer(p.From) {
natOpen[p.To] = true
} else if isPeer(p.To) && !natOpen[p.From] {
return router.Drop
}
}
if done() {
return router.ExitNow
}
return router.KeepRouting
})
return time.Duration(fwd.Load()), time.Duration(rev.Load())
}
func drainTun(c *nebula.Control) {
for {
select {
case <-c.GetTunTxChan():
default:
return
}
}
}
// watchTun records how long after start the first packet came out of c's tun, and keeps the tun drained after
func watchTun(c *nebula.Control, stop chan struct{}, start time.Time, first *atomic.Int64) {
go func() {
for {
select {
case <-stop:
return
case <-c.GetTunTxChan():
first.CompareAndSwap(0, int64(time.Since(start)))
}
}
}()
}
+14 -13
View File
@@ -497,11 +497,10 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []
// Nope, lets push the sender along // Nope, lets push the sender along
case p := <-udpTx: case p := <-udpTx:
r.Lock() r.Lock()
a := sender.GetUDPAddr() c := r.getControl(p)
c := r.getControl(a, p.To, p)
if c == nil { if c == nil {
r.Unlock() r.Unlock()
panic("No control for udp tx " + a.String()) panic("No control for udp tx " + p.To.String())
} }
fp := r.unlockedInjectFlow(sender, c, p, false) fp := r.unlockedInjectFlow(sender, c, p, false)
c.InjectUDPPacket(p) // copies internally; original is ours to release c.InjectUDPPacket(p) // copies internally; original is ours to release
@@ -622,8 +621,7 @@ func (r *R) stopFanInWorkers() {
func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) { func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()
a := from.GetUDPAddr() c := r.getControl(p)
c := r.getControl(a, p.To, p)
if c == nil { if c == nil {
panic(fmt.Sprintf("No control for udp tx %s", p.To)) panic(fmt.Sprintf("No control for udp tx %s", p.To))
} }
@@ -671,7 +669,7 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
panic(err) panic(err)
} }
receiver := r.getControl(sender.GetUDPAddr(), p.To, p) receiver := r.getControl(p)
if receiver == nil { if receiver == nil {
r.Unlock() r.Unlock()
panic("Can't RouteExitFunc for host: " + p.To.String()) panic("Can't RouteExitFunc for host: " + p.To.String())
@@ -767,7 +765,7 @@ func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc)
r.Lock() r.Lock()
p := rx.Interface().(*udp.Packet) p := rx.Interface().(*udp.Packet)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p) receiver := r.getControl(p)
if receiver == nil { if receiver == nil {
r.Unlock() r.Unlock()
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String()) panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
@@ -876,7 +874,7 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
r.Lock() r.Lock()
p := rx.Interface().(*udp.Packet) p := rx.Interface().(*udp.Packet)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p) receiver := r.getControl(p)
if receiver == nil { if receiver == nil {
r.Unlock() r.Unlock()
panic("Can't RouteForAllExitFunc for host: " + p.To.String()) panic("Can't RouteForAllExitFunc for host: " + p.To.String())
@@ -939,7 +937,7 @@ func (r *R) FlushAll() {
}) })
for { for {
x, rx, ok := reflect.Select(sc) _, rx, ok := reflect.Select(sc)
if !ok { if !ok {
return return
} }
@@ -947,7 +945,7 @@ func (r *R) FlushAll() {
p := rx.Interface().(*udp.Packet) p := rx.Interface().(*udp.Packet)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p) receiver := r.getControl(p)
if receiver == nil { if receiver == nil {
r.Unlock() r.Unlock()
panic("Can't FlushAll for host: " + p.To.String()) panic("Can't FlushAll for host: " + p.To.String())
@@ -958,16 +956,19 @@ func (r *R) FlushAll() {
} }
} }
// getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change // getControl performs or seeds NAT translation and returns the control for p.To, p from fields may change
// This is an internal router function, the caller must hold the lock // This is an internal router function, the caller must hold the lock
func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control { func (r *R) getControl(p *udp.Packet) *nebula.Control {
// Key on the address the packet actually left from, a dual stack sender has one per family
fromAddr, toAddr := p.From, p.To
if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok { if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
p.From = newAddr p.From = newAddr
} }
c, ok := r.inNat[toAddr] c, ok := r.inNat[toAddr]
if ok { if ok {
r.outNat[outNatKey{from: c.GetUDPAddr(), to: fromAddr}] = toAddr // Key the return path on the address the receiver will answer from, which is per family for a dual stack node
r.outNat[outNatKey{from: c.GetUDPAddrFor(fromAddr), to: fromAddr}] = toAddr
return c return c
} }
+3
View File
@@ -331,6 +331,9 @@ tun:
# `mtu`: will default to tun mtu if this option is not specified # `mtu`: will default to tun mtu if this option is not specified
# `metric`: will default to 0 if this option is not specified # `metric`: will default to 0 if this option is not specified
# `install`: will default to true, controls whether this route is installed in the systems routing table. # `install`: will default to true, controls whether this route is installed in the systems routing table.
# On macOS, a route whose destination another interface already routes, like the local network, is installed once
# that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another
# interface routes its destination and lets it go.
# This setting is reloadable. # This setting is reloadable.
unsafe_routes: unsafe_routes:
#- route: 172.16.1.0/24 #- route: 172.16.1.0/24
+4
View File
@@ -66,6 +66,9 @@ type Firewall struct {
rules string rules string
rulesVersion uint16 rulesVersion uint16
// source is the rendered firewall config this firewall was built from, the cert networks are kept above
source string
defaultLocalCIDRAny bool defaultLocalCIDRAny bool
incomingMetrics firewallMetrics incomingMetrics firewallMetrics
outgoingMetrics firewallMetrics outgoingMetrics firewallMetrics
@@ -246,6 +249,7 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
return nil, err return nil, err
} }
fw.source = c.Render("firewall")
return fw, nil return fw, nil
} }
+33 -23
View File
@@ -61,16 +61,17 @@ type HandshakeManager struct {
vpnIps map[netip.Addr]*HandshakeHostInfo vpnIps map[netip.Addr]*HandshakeHostInfo
indexes map[uint32]*HandshakeHostInfo indexes map[uint32]*HandshakeHostInfo
mainHostMap *HostMap mainHostMap *HostMap
lightHouse *LightHouse lightHouse *LightHouse
outside udp.Conn outside udp.Conn
config HandshakeConfig config HandshakeConfig
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr] OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
messageMetrics *MessageMetrics messageMetrics *MessageMetrics
metricInitiated metrics.Counter metricInitiated metrics.Counter
metricTimedOut metrics.Counter metricTimedOut metrics.Counter
f *Interface metricRejectedBlocklisted metrics.Counter
l *slog.Logger f *Interface
l *slog.Logger
// can be used to trigger outbound handshake for the given vpnIp // can be used to trigger outbound handshake for the given vpnIp
trigger chan netip.Addr trigger chan netip.Addr
@@ -118,18 +119,19 @@ func (hh *HandshakeHostInfo) cachePacket(l *slog.Logger, t header.MessageType, s
func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager { func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager {
return &HandshakeManager{ return &HandshakeManager{
vpnIps: map[netip.Addr]*HandshakeHostInfo{}, vpnIps: map[netip.Addr]*HandshakeHostInfo{},
indexes: map[uint32]*HandshakeHostInfo{}, indexes: map[uint32]*HandshakeHostInfo{},
mainHostMap: mainHostMap, mainHostMap: mainHostMap,
lightHouse: lightHouse, lightHouse: lightHouse,
outside: outside, outside: outside,
config: config, config: config,
trigger: make(chan netip.Addr, config.triggerBuffer), trigger: make(chan netip.Addr, config.triggerBuffer),
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)), OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
messageMetrics: config.messageMetrics, messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil), metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil), metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
l: l, metricRejectedBlocklisted: metrics.GetOrRegisterCounter("handshake_manager.rejected.blocklisted", nil),
l: l,
} }
} }
@@ -717,7 +719,12 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
response, result, err := machine.ProcessPacket(nil, packet) response, result, err := machine.ProcessPacket(nil, packet)
if err != nil { if err != nil {
f.l.Error("Failed to process handshake packet", "from", via, "error", err) level := slog.LevelError
if errors.Is(err, cert.ErrBlockListed) {
level = slog.LevelDebug
hm.metricRejectedBlocklisted.Inc(1)
}
f.l.Log(context.Background(), level, "Failed to process handshake packet", "from", via, "error", err)
return return
} }
@@ -849,6 +856,9 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
response, result, err := machine.ProcessPacket(nil, packet) response, result, err := machine.ProcessPacket(nil, packet)
if err != nil { if err != nil {
if errors.Is(err, cert.ErrBlockListed) {
hm.metricRejectedBlocklisted.Inc(1)
}
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn. // Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
if machine.Failed() { if machine.Failed() {
f.l.Warn("Failed to process handshake packet, abandoning", f.l.Warn("Failed to process handshake packet, abandoning",
+88
View File
@@ -1,12 +1,17 @@
package nebula package nebula
import ( import (
"log/slog"
"net/netip" "net/netip"
"testing" "testing"
"time" "time"
"github.com/flynn/noise"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
@@ -235,6 +240,89 @@ func TestHandleIncomingDispatch(t *testing.T) {
}) })
} }
func TestBlocklistedHandshakeLogLevelAndCounter(t *testing.T) {
ca, _, caKey, _ := ct.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
pool := ct.NewTestCAPool(ca)
newCredential := func(name, network string) (cert.Certificate, *handshake.Credential) {
c, _, rawKey, _ := ct.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey,
name, ca.NotBefore(), ca.NotAfter(), []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
privateKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey)
require.NoError(t, err)
certBytes, err := c.MarshalForHandshakes()
require.NoError(t, err)
cipherSuite := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
return c, handshake.NewCredential(c, certBytes, privateKey, cipherSuite)
}
initiatorCert, initiatorCredential := newCredential("initiator", "10.0.0.1/24")
responderCert, responderCredential := newCredential("responder", "10.0.0.2/24")
fingerprint, err := initiatorCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(fingerprint)
initiator, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return initiatorCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return pool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 1, nil }, true, header.HandshakeIXPSK0)
require.NoError(t, err)
packet, err := initiator.Initiate(nil)
require.NoError(t, err)
handler := &capturingHandler{}
logger := slog.New(handler)
pki := &PKI{}
pki.cs.Store(&CertState{initiatingVersion: cert.Version2, v2Credential: responderCredential})
pki.caPool.Store(pool)
hm := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
hm.metricRejectedBlocklisted = metrics.NewCounter()
hm.f = &Interface{pki: pki, l: logger}
via := ViaSender{UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242")}
hm.beginHandshake(via, packet, nil)
entry := findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelDebug, entry.Level)
require.ErrorIs(t, entry.Attrs["error"].(error), cert.ErrBlockListed)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
handler.entries = nil
hm.beginHandshake(via, nil, nil)
entry = findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelError, entry.Level)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
// The initiator sees the same rejection in continueHandshake, after a
// responder has accepted its first packet and sent a valid reply.
cleanPool := ct.NewTestCAPool(ca)
responder, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return responderCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return cleanPool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 2, nil }, false, header.HandshakeIXPSK0)
require.NoError(t, err)
response, _, err := responder.ProcessPacket(nil, packet)
require.NoError(t, err)
responderFingerprint, err := responderCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(responderFingerprint)
initiatingHM := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
initiatingHM.metricRejectedBlocklisted = metrics.NewCounter()
initiatingHM.f = &Interface{l: logger}
hh := &HandshakeHostInfo{
machine: initiator,
hostinfo: &HostInfo{localIndexId: 1, vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}},
}
initiatingHM.indexes[1] = hh
via.IsRelayed = true // skip the remote allow list before certificate validation
initiatingHM.continueHandshake(via, hh, response)
assert.Equal(t, int64(1), initiatingHM.metricRejectedBlocklisted.Count())
}
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already // Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the // carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
// handshake loop has run, is safe and the address becomes one the handshake is sent to. // handshake loop has run, is safe and the address becomes one the handshake is sent to.
+62
View File
@@ -264,6 +264,9 @@ type HostInfo struct {
// This is used to limit lighthouse re-queries in chatty clients // This is used to limit lighthouse re-queries in chatty clients
nextLHQuery atomic.Int64 nextLHQuery atomic.Int64
// nextUnreachableProbe is the earliest we will probe our other remotes again after failing to reach the current one
nextUnreachableProbe atomic.Int64
// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally // lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator // Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
// This is used to avoid an attack where a handshake packet is replayed after some time // This is used to avoid an attack where a handshake packet is replayed after some time
@@ -766,6 +769,65 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
} }
} }
// unreachableProbeInterval bounds how often a tunnel whose remote we cannot reach probes its other remotes
const unreachableProbeInterval = time.Second
// remoteUnreachable is called when a send to our current remote failed because we have no route to it, typically
// after moving to a network without that address family. Sends a test packet to every other remote we know for this
// host: the first one that arrives roams the peer over to wherever we are now, and its reply roams us onto that path.
//
// A peer behind a NAT drops a test from an address it has never sent to, so every probe also asks the lighthouse
// about the peer. The lighthouse tells the peer where we are now, the peer punches us, and the next probe gets in.
func (i *HostInfo) remoteUnreachable(ifce *Interface, dead netip.AddrPort) {
if i.remotes == nil || i.ConnectionState == nil || i.GetRemote() != dead {
// Nothing to probe with, or we already moved on from the remote that failed
return
}
if ifce.lightHouse == nil || ifce.lightHouse.amLighthouse {
// A lighthouse hands out the address it reaches a client on, it must not go roaming its clients itself
return
}
now := time.Now().UnixNano()
next := i.nextUnreachableProbe.Load()
if now < next || !i.nextUnreachableProbe.CompareAndSwap(next, now+int64(unreachableProbeInterval)) {
return
}
nb := make([]byte, 12)
out := make([]byte, mtu)
probed := 0
i.remotes.ForEach(ifce.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) {
if addr == dead {
return
}
probed++
ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), nb, out)
})
if ifce.l.Enabled(context.Background(), slog.LevelDebug) {
i.logger(ifce.l).Debug("Remote unreachable, probing other remotes",
"udpAddr", dead,
"probed", probed,
)
}
if probed == 0 {
// With nothing else to try, the lighthouse is our only way to learn of another remote. The peer may not have
// one, so ask on the usual requery schedule rather than every probe for as long as the tunnel sits here.
if now < i.nextLHQuery.Load() {
return
}
i.nextLHQuery.Store(now + ifce.reQueryWait.Load())
}
// With other remotes to probe we ask on every probe, without backing off. A repeat re-sends the peer its punch
// notification in case the last one or its punches were lost. If the tunnel stays up on inbound traffic while
// our sends to its remote keep failing (only one direction broken), this continues once a second for as long as
// that lasts. That is a bounded cost for a rare state, and the probe that would clear it has to keep running.
ifce.lightHouse.QueryServer(i.vpnAddrs[0])
}
// Bits within HostInfo.state, everything above stateEpochShift is the epoch // Bits within HostInfo.state, everything above stateEpochShift is the epoch
const ( const (
stateIn uint32 = 1 << iota stateIn uint32 = 1 << iota
+5
View File
@@ -15,6 +15,7 @@ import (
"github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/udp"
) )
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) { func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
@@ -611,6 +612,10 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
"udpAddr", hr, "udpAddr", hr,
) )
} }
// A closing tunnel is about to be torn down, there is no point looking for another path for it
if t != header.CloseTunnel && udp.IsUnreachable(err) {
hostinfo.remoteUnreachable(f, hr)
}
} }
} else { } else {
// Try to send via a relay // Try to send via a relay
+47 -6
View File
@@ -1,6 +1,7 @@
package nebula package nebula
import ( import (
"bytes"
"context" "context"
"crypto/fips140" "crypto/fips140"
"errors" "errors"
@@ -495,7 +496,8 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
// flushSendBatch drains sb to the underlay and accounts for anything it could not deliver. A shortfall means // flushSendBatch drains sb to the underlay and accounts for anything it could not deliver. A shortfall means
// specific destinations were undeliverable (a stale remote, a reject rule), which the backend logs per peer at // specific destinations were undeliverable (a stale remote, a reject rule), which the backend logs per peer at
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch. // debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch. Tunnels whose remote
// we no longer have a route to go looking for another path.
func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) { func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
queued := sb.Len() queued := sb.Len()
written, err := sb.Flush() written, err := sb.Flush()
@@ -505,6 +507,44 @@ func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
if dropped := queued - written; dropped > 0 { if dropped := queued - written; dropped > 0 {
f.metricTxDropped.Inc(int64(dropped)) f.metricTxDropped.Inc(int64(dropped))
} }
if f.lightHouse == nil || f.lightHouse.amLighthouse {
// Lighthouses never probe, see remoteUnreachable, so skip finding the tunnels
return
}
// A tunnel's packets tend to fail together, so skip straight past a repeat of the one before. The first 8 bytes
// of a header say whose packet it is, the counter after them changes with every packet.
var last batch.Failure
for i, fail := range sb.Failures() {
if i > 0 && fail.Dst == last.Dst && bytes.Equal(fail.Head[:8], last.Head[:8]) {
continue
}
last = fail
if udp.IsUnreachable(fail.Err) {
f.sendUnreachable(fail.Head[:], fail.Dst)
}
}
}
// sendUnreachable finds the tunnel a packet we had no route for was sent on, from the start of the packet, and has it
// look for another path to its peer.
func (f *Interface) sendUnreachable(head []byte, dst netip.AddrPort) {
var h header.H
if h.Parse(head) != nil {
return
}
if h.Type == header.Message && h.Subtype == header.MessageRelay {
// TODO: a relayed packet's header carries the relay session's index, not the relay tunnel's, so it does not
// lead us to the tunnel whose remote failed. The relay tunnel's own traffic finds out soon enough.
return
}
if hostinfo := f.hostMap.QueryReverseIndex(h.RemoteIndex); hostinfo != nil {
hostinfo.remoteUnreachable(f, dst)
}
} }
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) { func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
@@ -537,11 +577,12 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
curCert = cs.getCertificate(cert.Version1) curCert = cs.getCertificate(cert.Version1)
} }
// The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction. // Compare with what the running firewall was built from, not the last config, so a firewall that failed to build
// Check to see if that set has changed, and if so, rebuild the firewall. // is tried again. The cert is the one this reload brought, PKI registers its reload callback before this one
certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) src := c.Render("firewall")
if src != "" && src == f.firewall.source && curCert != nil &&
if !c.HasChanged("firewall") && !certUnsafeChanged { slices.Equal(curCert.Networks(), f.firewall.assignedNetworks) &&
slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) {
f.l.Debug("No firewall config change detected") f.l.Debug("No firewall config change detected")
return return
} }
+58 -3
View File
@@ -135,8 +135,63 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
f := &Interface{pki: pki, firewall: fw, l: l} f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall) cfg.RegisterReloadCallback(f.reloadFirewall)
res := cfg.ReloadConfigString(rules("nope")) // The running firewall still isn't built from what the config asks for, so the same reload fails again
for range 2 {
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
res := cfg.ReloadConfigString(rules("tcp"))
require.NoError(t, res.Err) require.NoError(t, res.Err)
assert.True(t, res.Failed) assert.False(t, res.Failed)
assert.Same(t, fw, f.firewall) assert.NotSame(t, fw, f.firewall)
}
// Every input the firewall is built from rebuilds it when it changes, and nothing else does
func TestReloadFirewall_RebuildsOnAnyInput(t *testing.T) {
l := test.NewLogger()
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
base := "firewall:\n inbound:\n - port: any\n proto: any\n host: any\n"
setup := func() (*config.C, *Interface, *PKI) {
pki := &PKI{}
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks}, initiatingVersion: cert.Version2})
cfg := config.NewC(l)
require.NoError(t, cfg.LoadString(base))
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
require.NoError(t, err)
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
return cfg, f, pki
}
setCert := func(pki *PKI, networks, unsafe []netip.Prefix) {
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks, unsafeNetworks: unsafe}, initiatingVersion: cert.Version2})
}
cases := map[string]struct {
cfg string
cert func(*PKI)
want bool
}{
"nothing": {base + "punchy:\n punch: true\n", nil, false},
"rules": {base + " outbound:\n - port: any\n proto: any\n host: any\n", nil, true},
"inbound_action": {base + " inbound_action: reject\n", nil, true},
"conntrack": {base + " conntrack:\n tcp_timeout: 1m\n", nil, true},
"networks": {base, func(p *PKI) { setCert(p, []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil) }, true},
"unsafe_networks": {base, func(p *PKI) { setCert(p, networks, []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}) }, true},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
cfg, f, pki := setup()
old := f.firewall
if tc.cert != nil {
tc.cert(pki)
}
res := cfg.ReloadConfigString(tc.cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, tc.want, f.firewall != old)
})
}
} }
+41 -5
View File
@@ -1,12 +1,27 @@
package batch package batch
import "net/netip" import (
"net/netip"
"github.com/slackhq/nebula/header"
)
const SendBatchCap = 128 const SendBatchCap = 128
// FailureHeadLen is how much of a refused packet a Failure keeps, nebula's header, which says whose packet it was
const FailureHeadLen = header.Len
// batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush. // batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush.
type batchWriter interface { type batchWriter interface {
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
}
// Failure is a packet the underlay would not send
type Failure struct {
// Head is the start of the packet, zero padded if the packet was shorter
Head [FailureHeadLen]byte
Dst netip.AddrPort
Err error
} }
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch. // SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
@@ -17,16 +32,28 @@ type SendBatch struct {
bufs [][]byte bufs [][]byte
dsts []netip.AddrPort dsts []netip.AddrPort
arena *Arena arena *Arena
failures []Failure
// recordFailure is bound once so handing it to WriteBatch does not allocate on every Flush
recordFailure func(i int, err error)
} }
// NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots // NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots
func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch { func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch {
return &SendBatch{ b := &SendBatch{
out: out, out: out,
bufs: make([][]byte, 0, batchCap), bufs: make([][]byte, 0, batchCap),
dsts: make([]netip.AddrPort, 0, batchCap), dsts: make([]netip.AddrPort, 0, batchCap),
arena: NewArena(arenaSize), arena: NewArena(arenaSize),
} }
b.recordFailure = b.failed
return b
}
func (b *SendBatch) failed(i int, err error) {
f := Failure{Dst: b.dsts[i], Err: err}
copy(f.Head[:], b.bufs[i])
b.failures = append(b.failures, f)
} }
func (b *SendBatch) Reserve(sz int) []byte { func (b *SendBatch) Reserve(sz int) []byte {
@@ -44,12 +71,15 @@ func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort) {
} }
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations // Flush writes every queued packet and reports how many actually went out. A short count means some destinations
// were undeliverable; the batch is drained either way. // were undeliverable, Failures says which; the batch is drained either way.
func (b *SendBatch) Flush() (int, error) { func (b *SendBatch) Flush() (int, error) {
clear(b.failures)
b.failures = b.failures[:0]
var err error var err error
written := 0 written := 0
if len(b.bufs) > 0 { if len(b.bufs) > 0 {
written, err = b.out.WriteBatch(b.bufs, b.dsts) written, err = b.out.WriteBatch(b.bufs, b.dsts, b.recordFailure)
} }
clear(b.bufs) clear(b.bufs)
b.bufs = b.bufs[:0] b.bufs = b.bufs[:0]
@@ -57,3 +87,9 @@ func (b *SendBatch) Flush() (int, error) {
b.arena.Reset() b.arena.Reset()
return written, err return written, err
} }
// Failures returns every packet the last Flush could not send, in the order they were committed. Valid until the
// next Flush.
func (b *SendBatch) Failures() []Failure {
return b.failures
}
+84 -1
View File
@@ -1,7 +1,9 @@
package batch package batch
import ( import (
"errors"
"net/netip" "net/netip"
"slices"
"testing" "testing"
) )
@@ -10,7 +12,7 @@ type fakeBatchWriter struct {
addrs []netip.AddrPort addrs []netip.AddrPort
} }
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, _ func(int, error)) (int, error) {
// Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch // Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch
// returns, so tests must capture data before that happens. // returns, so tests must capture data before that happens.
w.bufs = make([][]byte, len(bufs)) w.bufs = make([][]byte, len(bufs))
@@ -120,3 +122,84 @@ func TestSendBatchGrowPreservesCommitted(t *testing.T) {
t.Errorf("second packet on the wire: %x", fw.bufs[1]) t.Errorf("second packet on the wire: %x", fw.bufs[1])
} }
} }
// failingBatchWriter refuses every packet whose first byte is in refuse
type failingBatchWriter struct {
refuse map[byte]error
}
func (w *failingBatchWriter) WriteBatch(bufs [][]byte, _ []netip.AddrPort, failed func(int, error)) (int, error) {
written := 0
for i, b := range bufs {
if err, ok := w.refuse[b[0]]; ok {
failed(i, err)
continue
}
written++
}
return written, nil
}
func TestSendBatchReportsFailures(t *testing.T) {
errNoRoute := errors.New("no route")
fw := &failingBatchWriter{refuse: map[byte]error{2: errNoRoute, 3: errNoRoute}}
b := NewSendBatch(fw, 4, 128)
// pkt is n bytes starting with the refuse key, each later byte its offset plus 0x10 so the head is recognizable
pkt := func(key byte, n int) []byte {
p := b.Reserve(n)[:n]
p[0] = key
for i := 1; i < n; i++ {
p[i] = byte(0x10 + i)
}
return p
}
head := func(p []byte) (h [FailureHeadLen]byte) {
copy(h[:], p)
return h
}
dsts := []netip.AddrPort{
netip.MustParseAddrPort("10.0.0.1:4242"),
netip.MustParseAddrPort("[fd00::2]:4242"),
netip.MustParseAddrPort("[fd00::3]:4242"),
netip.MustParseAddrPort("10.0.0.4:4242"),
}
long := pkt(2, FailureHeadLen+8) // only its head is kept
short := pkt(3, 4) // zero padded
b.Commit(pkt(1, 20), dsts[0])
b.Commit(long, dsts[1])
b.Commit(short, dsts[2])
b.Commit(pkt(1, 20), dsts[3])
written, err := b.Flush()
if err != nil {
t.Fatalf("Flush: %v", err)
}
if written != 2 {
t.Errorf("written = %d, want 2", written)
}
want := []Failure{{head(long), dsts[1], errNoRoute}, {head(short), dsts[2], errNoRoute}}
if got := b.Failures(); !slices.Equal(got, want) {
t.Errorf("failures = %v, want %v", got, want)
}
// Failures do not leak into the next flush
again := pkt(2, 20)
b.Commit(again, dsts[0])
if _, err := b.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if want := []Failure{{head(again), dsts[0], errNoRoute}}; !slices.Equal(b.Failures(), want) {
t.Errorf("second flush failures = %v, want %v", b.Failures(), want)
}
// A clean flush forgets the last one's failures
b.Commit(pkt(1, 20), dsts[0])
if _, err := b.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if got := b.Failures(); len(got) != 0 {
t.Errorf("clean flush failures = %v, want none", got)
}
}
+54 -84
View File
@@ -3,7 +3,6 @@
package overlay package overlay
import ( import (
"errors"
"fmt" "fmt"
"log/slog" "log/slog"
"math/bits" "math/bits"
@@ -12,6 +11,7 @@ import (
"sync" "sync"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"time"
"unsafe" "unsafe"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
@@ -35,6 +35,26 @@ type tun struct {
// and routes for it. NEPacketTunnelProvider on darwin does this. // and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool hostOwned bool
l *slog.Logger l *slog.Logger
// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
// fields below it
routesMu sync.Mutex
// routeStates is who held each installable unsafe_route prefix when we last looked
routeStates map[netip.Prefix]routeState
// routeWatch is the routing socket watchRoutes reads, Close closes it
routeWatch *os.File
// routeSweep reconciles the unsafe_routes every routeSweepEvery
routeSweep *time.Timer
routeSweepEvery time.Duration
closed bool
// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
// lock so the route watcher never waits on a reconcile
routeMoversMu sync.Mutex
routeMovers map[netip.Prefix]routeMover
// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
lastRouteEvent atomic.Int64
} }
type ifReq struct { type ifReq struct {
@@ -139,11 +159,12 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
} }
t := &tun{ t := &tun{
f: os.NewFile(uintptr(fd), ""), f: os.NewFile(uintptr(fd), ""),
Device: name, Device: name,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c), DefaultMTU: getMTU(l, c),
l: l, routeSweepEvery: routeSweepInterval,
l: l,
} }
err = t.reload(c, true, nil) err = t.reload(c, true, nil)
@@ -219,6 +240,16 @@ func utunNameFromFd(fd int) string {
} }
func (t *tun) Close() error { func (t *tun) Close() error {
t.routesMu.Lock()
t.closed = true
if t.routeWatch != nil {
_ = t.routeWatch.Close()
}
if t.routeSweep != nil {
t.routeSweep.Stop()
}
t.routesMu.Unlock()
if t.f != nil { if t.f != nil {
return t.f.Close() return t.f.Close()
} }
@@ -287,8 +318,22 @@ func (t *tun) Activate() error {
return fmt.Errorf("failed to run tun device: %s", err) return fmt.Errorf("failed to run tun device: %s", err)
} }
// Watch before adding so a route removed in between is still noticed
watch, err := openRouteSocket()
if err != nil {
t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
// Unsafe path routes // Unsafe path routes
return t.addRoutes(false, nil) if err = t.addRoutes(false, nil); err != nil {
if watch != nil {
_ = watch.Close()
}
return err
}
t.startRouteWatch(watch)
return nil
} }
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly. // activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
@@ -393,7 +438,8 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
} }
if !initial && !change { if !initial && !change {
return nil // Any reload puts back the unsafe_routes someone deleted
return t.addRoutes(true, r)
} }
routeTree, err := makeRouteTree(t.l, routes, false) routeTree, err := makeRouteTree(t.l, routes, false)
@@ -511,82 +557,6 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
return nil, nil return nil, nil
} }
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
err := addRoute(r.Cidr, linkAddr)
if err != nil {
if errors.Is(err, unix.EEXIST) {
// Every route we added already exists on a reload, only warn about someone else's
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
retErr.Log(t.l)
rl.Failed()
} else {
return retErr
}
}
} else {
t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
}
}
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
linkAddr := t.linkAddr.Load()
if t.hostOwned || linkAddr == nil {
return nil
}
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue
}
err := delRoute(r.Cidr, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error { func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil { if err != nil {
+366
View File
@@ -3,12 +3,17 @@
package overlay package overlay
import ( import (
"encoding/binary"
"encoding/hex"
"fmt"
"net" "net"
"net/netip" "net/netip"
"os" "os"
"os/exec" "os/exec"
"path/filepath"
"strings" "strings"
"testing" "testing"
"time"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/routing"
@@ -104,6 +109,367 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))) assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
} }
// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
const (
// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
// With no interface address, which x/net/route skips
interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
)
func TestRoutesMayHaveMoved(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
routes := []Route{{Cidr: cidr, Via: via, Install: true}}
// A request like route(8) sends, the kernel echoes it to every routing socket
request := func(typ, flags int, dst, mask [4]byte) []byte {
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: typ,
Flags: flags,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: dst},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
},
}
b, err := m.Marshal()
require.NoError(t, err)
return b
}
raw := func(s string) []byte {
b, err := hex.DecodeString(s)
require.NoError(t, err)
return b
}
lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
routesMayHaveMoved := func(b []byte, routes []Route) bool {
moved, _, _ := routesMayHaveMoved(b, routes)
return moved
}
assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
neighbor := raw(kernelSubnetDelete)
binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
}
// A request names its prefix and who made it, the kernel's own and interface changes name neither
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: unix.RTM_CHANGE,
Flags: unix.RTF_UP,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
},
}
b, err := m.Marshal()
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved)
assert.Equal(t, cidr, p)
assert.Equal(t, int32(1234), pid)
for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
b, err := hex.DecodeString(s)
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved, name)
assert.False(t, p.IsValid(), name)
assert.Zero(t, pid, name)
}
}
func TestProcessName(t *testing.T) {
assert.Equal(t, "launchd", processName(1))
self := filepath.Base(os.Args[0])
assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
assert.Empty(t, processName(-1))
}
// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
// the minute between sweeps when it isn't zero.
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
requireRoot(t)
cfg := "tun:\n unsafe_routes:\n"
for _, cidr := range cidrs {
via := "10.250.0.2"
if netip.MustParsePrefix(cidr).Addr().Is6() {
via = "fd00:250::2"
}
cfg += " - route: " + cidr + "\n via: " + via + "\n"
}
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString(cfg))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
require.NoError(t, err)
if sweep != 0 {
tn.routeSweepEvery = sweep
}
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
return tn, c, cfg
}
func requireRoot(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating interfaces and adding routes needs root")
}
}
// routeVia is the index of the interface that routes cidr, 0 if none does
func routeVia(t *testing.T, cidr string) int {
msgs, err := fetchRoutes()
require.NoError(t, err)
return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
}
// routeStateOf is what tn made of cidr when it last looked
func routeStateOf(tn *tun, cidr string) routeState {
tn.routesMu.Lock()
defer tn.routesMu.Unlock()
return tn.routeStates[netip.MustParsePrefix(cidr)]
}
func routeArgs(action, cidr string, extra ...string) []string {
args := []string{"-q", "-n", action}
if netip.MustParsePrefix(cidr).Addr().Is6() {
args = append(args, "-inet6")
}
return append(append(args, "-net", cidr), extra...)
}
func routeCmd(t *testing.T, action, cidr string, extra ...string) {
requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
}
// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
// says so
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
require.NotContains(t, string(out), "writing to routing socket")
}
// cleanupRoute deletes cidr's route when the test ends, whoever holds it
func cleanupRoute(t *testing.T, cidr string) {
t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
}
// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
func routeThroughLo0(t *testing.T, cidr string) int {
requireRoot(t)
routeCmd(t, "add", cidr, "-interface", "lo0")
cleanupRoute(t, cidr)
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
return lo0.Index
}
// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
t.Run(cidr, func(t *testing.T) {
lo0 := routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
})
}
}
// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
cidr := "10.251.1.0/24"
lo0 := routeThroughLo0(t, cidr)
_, c, _ := newRouteTestTun(t, 0, cidr)
res := c.ReloadConfigString("tun: {}\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, lo0, routeVia(t, cidr))
}
// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
cidr := "10.251.3.0/24"
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
time.Sleep(2 * routeRetryDelay)
require.Equal(t, lo0.Index, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
}
// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
cidr := "10.251.4.0/24"
tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
// A reload with nothing changed is how to ask for it back
res := c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, ours, routeVia(t, cidr))
}
// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
for name, tc := range map[string]struct {
cidr string
sweep time.Duration
misfile func(t *testing.T, tn *tun, cidr string)
}{
// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
}},
// The sweep that caps a table that never settles
"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
tn.reconcileRoutes()
require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
}},
} {
t.Run(name, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, tc.cidr))
routeCmd(t, "delete", tc.cidr)
tc.misfile(t, tn, tc.cidr)
lo0 := routeThroughLo0(t, tc.cidr)
require.Eventually(t, func() bool {
s := routeStateOf(tn, tc.cidr)
return s.kind == routeYielded && s.holder == lo0
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", tc.cidr)
assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
// taken for a hand delete
func TestTunYieldsToARouteChangedAway(t *testing.T) {
for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
t.Run(cidr, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
routeCmd(t, "change", cidr, "-interface", "lo0")
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
require.Equal(t, lo0.Index, routeVia(t, cidr))
require.Eventually(t, func() bool {
s := routeStateOf(tn, cidr)
return s.kind == routeYielded && s.holder == lo0.Index
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
// ethernet configd routed the lan through
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
requireRoot(t)
var feth string
for i := 100; i < 200 && feth == ""; i++ {
if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
feth = fmt.Sprintf("feth%d", i)
}
}
require.NotEmpty(t, feth)
require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
// refuses one named by the feth itself
cidr := "10.251.2.0/24"
routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
cleanupRoute(t, cidr)
ifc, err := net.InterfaceByName(feth)
require.NoError(t, err)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Losing the address first leaves the route, the reconcile finds it still held
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
time.Sleep(2 * routeRetryDelay)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Only RTM_IFINFO says the interface went, its route goes with no message of its own
require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
cidr := "10.251.5.0/24"
routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
tn.routesMu.Lock()
require.NoError(t, tn.routeWatch.Close())
tn.routesMu.Unlock()
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free // Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
func TestTunDevNeedsRestart(t *testing.T) { func TestTunDevNeedsRestart(t *testing.T) {
for name, want := range map[string]bool{ for name, want := range map[string]bool{
+451
View File
@@ -0,0 +1,451 @@
//go:build !ios && !e2e_testing
package overlay
import (
"errors"
"fmt"
"net"
"net/netip"
"os"
"strconv"
"sync/atomic"
"time"
"unsafe"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/util"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
// or until another interface takes the prefix, after which ours follows theirs like any other.
const (
// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
// replaces a route by deleting and adding, looking in between would find the prefix free
routeRetryDelay = time.Second
// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
// a message for each
routeSweepInterval = time.Minute
// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
routeSocketRcvBuf = 1 << 20
)
type routeStateKind int
const (
routeUnknown routeStateKind = iota
// routeInstalled is our route holding the prefix
routeInstalled
// routeYielded is another interface holding the prefix, ours is added once it lets go
routeYielded
// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
// another interface takes the prefix
routeRemoved
// routeFailed is adding our route failing for some other reason, it is retried
routeFailed
)
type routeState struct {
kind routeStateKind
// holder is the index of the interface holding the prefix when kind is routeYielded
holder int
}
// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
// is looked up as the message arrives, route(8) has exited by the time the table settles
type routeMover struct {
pid int32
name string
}
// logArgs appends m to a log line's args, when we know who it was
func (m routeMover) logArgs(args ...any) []any {
if m.pid == 0 {
return args
}
args = append(args, "pid", m.pid)
if m.name != "" {
args = append(args, "process", m.name)
}
return args
}
// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
func processName(pid int32) string {
kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
if err != nil || kp.Proc.P_pid != pid {
return ""
}
return unix.ByteSliceToString(kp.Proc.P_comm[:])
}
// installRoutes returns the routes we put in the system routing table, one per prefix
func installRoutes(routes []Route) []Route {
seen := make(map[netip.Prefix]struct{}, len(routes))
var out []Route
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
p := r.Cidr.Masked()
if _, ok := seen[p]; ok {
continue
}
seen[p] = struct{}{}
out = append(out, r)
}
return out
}
// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
// interface holds its prefix. It runs on start and on every reload.
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
var msgs []netroute.Message
var msgsErr error
fetched := false
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
err := addRoute(p, linkAddr)
if err == nil {
t.l.Info("Added route", "route", r)
states[p] = routeState{kind: routeInstalled}
continue
}
if !errors.Is(err, unix.EEXIST) {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if !logErrors {
return retErr
}
retErr.Log(t.l)
rl.Failed()
states[p] = routeState{kind: routeFailed}
continue
}
// Every route we added already exists on a reload, only warn about someone else's
if !fetched {
msgs, msgsErr = fetchRoutes()
fetched = true
}
if msgsErr != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
states[p] = prev
continue
}
switch holder := routeInterfaceIn(msgs, p); holder {
case linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case 0:
// Removed since the add, the watcher will see to it
states[p] = prev
default:
if prev.kind != routeYielded || prev.holder != holder {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
}
states[p] = routeState{kind: routeYielded, holder: holder}
}
}
t.routeStates = states
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
msgs, err := fetchRoutes()
if err != nil {
var left []netip.Prefix
for _, r := range installRoutes(routes) {
left = append(left, r.Cidr)
}
return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
}
for _, r := range installRoutes(routes) {
p := r.Cidr.Masked()
// Whatever happens next, the prefix is no longer one we know about
delete(t.routeStates, p)
if routeInterfaceIn(msgs, p) != linkAddr.Index {
continue
}
err = delRoute(p, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
// changed since it last looked, it runs every routeSweepEvery.
func (t *tun) reconcileRoutes() {
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return
}
t.routeMoversMu.Lock()
movers := t.routeMovers
t.routeMovers = nil
t.routeMoversMu.Unlock()
msgs, err := fetchRoutes()
if err != nil {
t.l.Error("Failed to read the system routing table", "error", err)
return
}
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
by := movers[p]
holder := routeInterfaceIn(msgs, p)
switch {
case holder == linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case holder != 0:
// Another interface taking a prefix we left alone puts ours back in line behind theirs
if prev.kind != routeYielded || prev.holder != holder {
t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
}
states[p] = routeState{kind: routeYielded, holder: holder}
case prev.kind == routeInstalled:
t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
states[p] = routeState{kind: routeRemoved}
case prev.kind == routeRemoved:
states[p] = prev
default:
err := addRoute(p, linkAddr)
switch {
case err == nil:
if prev.kind == routeYielded {
t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
} else {
t.l.Info("Added route", "route", r)
}
states[p] = routeState{kind: routeInstalled}
case errors.Is(err, unix.EEXIST):
// Taken since the fetch, the next look will see by whom
states[p] = prev
default:
if prev.kind != routeFailed {
util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
}
states[p] = routeState{kind: routeFailed}
}
}
}
t.routeStates = states
}
// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
// routing table settles after anything that may have moved a prefix
func (t *tun) startRouteWatch(watch *os.File) {
t.routesMu.Lock()
defer t.routesMu.Unlock()
if t.closed {
if watch != nil {
_ = watch.Close()
}
return
}
t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
if watch != nil {
t.routeWatch = watch
go t.watchRoutes(watch)
}
}
func (t *tun) sweepRoutes() {
// A table still changing gets reconciled once it settles
if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
t.reconcileRoutes()
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
if !t.closed {
t.routeSweep.Reset(t.routeSweepEvery)
}
}
func openRouteSocket() (*os.File, error) {
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
}
// Fewer drops, the sweep catches the rest
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
// Non-blocking puts it on the poller, so Close interrupts a Read
if err = unix.SetNonblock(fd, true); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
}
return os.NewFile(uintptr(fd), "route"), nil
}
// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
func (t *tun) watchRoutes(f *os.File) {
var reconciled atomic.Int64
settle := time.AfterFunc(time.Hour, func() {
reconciled.Store(time.Now().UnixNano())
t.reconcileRoutes()
})
settle.Stop()
defer settle.Stop()
var burst time.Time
self := int32(os.Getpid())
b := make([]byte, os.Getpagesize())
for {
n, err := f.Read(b)
if err != nil {
if !errors.Is(err, os.ErrClosed) {
t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
return
}
moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
if !moved {
continue
}
// Our own are already in routeStates
if p.IsValid() && pid != self {
t.routeMoversMu.Lock()
if t.routeMovers == nil {
t.routeMovers = make(map[netip.Prefix]routeMover)
}
t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
t.routeMoversMu.Unlock()
}
now := time.Now()
t.lastRouteEvent.Store(now.UnixNano())
if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
burst = now
}
settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
}
}
// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
// and the pid of the process that made it.
func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
if len(b) < 4 {
return false, p, 0
}
switch int(b[3]) {
case unix.RTM_IFINFO, unix.RTM_DELADDR:
return true, p, 0
case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
default:
return false, p, 0
}
if len(b) < unix.SizeofRtMsghdr {
return false, p, 0
}
hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
// A failed request is reported too, and a scoped route never held the prefix
if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
return false, p, 0
}
if hdr.Pid == 0 {
// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
// could be holding a prefix
return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
}
// Replies to a request carry the requester's padded sockaddrs
msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
if err != nil || len(msgs) != 1 {
return false, p, 0
}
m, ok := msgs[0].(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
return false, p, 0
}
p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
if !ok {
return false, netip.Prefix{}, 0
}
p = p.Masked()
for _, r := range installRoutes(routes) {
if r.Cidr.Masked() == p {
return true, p, hdr.Pid
}
}
return false, netip.Prefix{}, 0
}
// interfaceName is the name of the interface with index i, for logs
func interfaceName(i int) string {
if ifc, err := net.InterfaceByIndex(i); err == nil {
return ifc.Name
}
return strconv.Itoa(i)
}
+19 -10
View File
@@ -165,20 +165,24 @@ func (p *Punchy) scheduleJob(job holepunchJob, delay time.Duration) {
// SendPunch sends an immediate keepalive punch for an idle hostinfo. // SendPunch sends an immediate keepalive punch for an idle hostinfo.
// The configured punchy.target_all_remotes mode picks the targets. Gated on punchy.punch and the lighthouse-skip rule // The configured punchy.target_all_remotes mode picks the targets. Gated on punchy.punch and the lighthouse-skip rule
// (lighthouses don't get keepalive punches because the regular update interval keeps their NAT state warm). // (lighthouses don't get keepalive punches because the regular update interval keeps their NAT state warm).
func (p *Punchy) SendPunch(hostinfo *HostInfo) { // Returns the hostinfo's current remote and the error from punching it, an invalid remote if it was not punched.
func (p *Punchy) SendPunch(hostinfo *HostInfo) (netip.AddrPort, error) {
if !p.punch.Load() { if !p.punch.Load() {
return return netip.AddrPort{}, nil
} }
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) { if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
return return netip.AddrPort{}, nil
} }
hr := hostinfo.GetRemote()
if p.punchEverything.Load() { if p.punchEverything.Load() {
p.sendPunchToAllRemotes(hostinfo) return hr, p.sendPunchToAllRemotes(hostinfo, hr)
} else if hr := hostinfo.GetRemote(); hr.IsValid() {
p.metricPunchyTx.Inc(1)
p.punchConn.WriteTo([]byte{1}, hr)
} }
if !hr.IsValid() {
return hr, nil
}
p.metricPunchyTx.Inc(1)
return hr, p.punchConn.WriteTo([]byte{1}, hr)
} }
// SendPunchToAll punches every known remote for hostinfo, but only when punchy.target_all_remotes is enabled. // SendPunchToAll punches every known remote for hostinfo, but only when punchy.target_all_remotes is enabled.
@@ -195,14 +199,19 @@ func (p *Punchy) SendPunchToAll(hostinfo *HostInfo) {
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) { if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
return return
} }
p.sendPunchToAllRemotes(hostinfo) p.sendPunchToAllRemotes(hostinfo, netip.AddrPort{})
} }
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo) { // sendPunchToAllRemotes punches every known remote for hostinfo and returns the error from punching current
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo, current netip.AddrPort) error {
var currentErr error
hostinfo.remotes.ForEach(p.hm.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) { hostinfo.remotes.ForEach(p.hm.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
p.metricPunchyTx.Inc(1) p.metricPunchyTx.Inc(1)
p.punchConn.WriteTo([]byte{1}, addr) if err := p.punchConn.WriteTo([]byte{1}, addr); addr == current {
currentErr = err
}
}) })
return currentErr
} }
// Start wires the runtime dependencies and spawns the scheduler worker. // Start wires the runtime dependencies and spawns the scheduler worker.
+112
View File
@@ -3,11 +3,14 @@ package nebula
import ( import (
"context" "context"
"log/slog" "log/slog"
"net/netip"
"testing" "testing"
"time" "time"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test" "github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -242,3 +245,112 @@ func TestPunchyReloadRestartRequired(t *testing.T) {
require.NoError(t, res.Err) require.NoError(t, res.Err)
assert.True(t, res.RestartRequired) assert.True(t, res.RestartRequired)
} }
// errNoRoute is the unreachable error every platform classifies the same way
var errNoRoute = udp.ErrInvalidIPv6RemoteForSocket
// recordingConn records where every write went and what kind of message it was, and fails the ones to an address in
// fail
type recordingConn struct {
udp.NoopConn
fail map[netip.AddrPort]error
wrote []netip.AddrPort
// types is the message type of each write, a punch has none
types []header.MessageType
}
func (c *recordingConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.wrote = append(c.wrote, addr)
var h header.H
if h.Parse(b) == nil {
c.types = append(c.types, h.Type)
}
return c.fail[addr]
}
func (c *recordingConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(int, error)) (int, error) {
written := 0
for i := range bufs {
if err := c.WriteTo(bufs[i], addrs[i]); err != nil {
failed(i, err)
continue
}
written++
}
return written, nil
}
type noLighthouses struct{}
func (noLighthouses) IsAnyLighthouseAddr([]netip.Addr) bool { return false }
var (
punchTargetV4 = netip.MustParseAddrPort("10.0.0.3:4242")
punchTargetV6 = netip.MustParseAddrPort("[fd00::3]:4242")
)
// newPunchTarget is a tunnel on its IPv6 remote that also knows the peer's IPv4 one
func newPunchTarget() *HostInfo {
vpnAddr := netip.MustParseAddr("10.128.0.3")
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
remotes: NewRemoteList([]netip.Addr{vpnAddr}, nil),
}
hostinfo.remotes.LearnRemote(vpnAddr, punchTargetV4)
hostinfo.SetRemote(punchTargetV6)
return hostinfo
}
func newPunchyForTest(t *testing.T, settings map[string]any, conn udp.Conn) *Punchy {
c := config.NewC(test.NewLogger())
c.Settings["punchy"] = settings
p := NewPunchyFromConfig(test.NewLogger(), c, conn)
hm := newHostMap(test.NewLogger())
hm.preferredRanges.Store(&[]netip.Prefix{})
p.hm = hm
p.lh = noLighthouses{}
return p
}
func TestPunchy_SendPunchReportsCurrentRemote(t *testing.T) {
t.Run("disabled", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": false}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.False(t, remote.IsValid())
require.NoError(t, err)
assert.Empty(t, conn.wrote)
})
t.Run("current remote", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.ErrorIs(t, err, errNoRoute)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
})
t.Run("all remotes, current fails", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.ErrorIs(t, err, errNoRoute)
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
})
t.Run("all remotes, only another fails", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV4: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.NoError(t, err, "a failure on a remote we are not using says nothing about the current one")
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
})
}
+3 -2
View File
@@ -41,8 +41,9 @@ type Conn interface {
// //
// Returns the number of packets successfully written. A destination the kernel rejects costs only // Returns the number of packets successfully written. A destination the kernel rejects costs only
// its own packet, so a short count means some peers were undeliverable, not that the batch failed. // its own packet, so a short count means some peers were undeliverable, not that the batch failed.
// failed, when not nil, is called with the index of every packet a destination cost us and why.
// Not safe for concurrent use on the same Conn. // Not safe for concurrent use on the same Conn.
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
// ReloadConfig applies the listen settings that can change, r is nil at startup // ReloadConfig applies the listen settings that can change, r is nil at startup
ReloadConfig(c *config.C, r *config.Reload) ReloadConfig(c *config.C, r *config.Reload)
SupportsMultipleReaders() bool SupportsMultipleReaders() bool
@@ -66,7 +67,7 @@ func (NoopConn) SupportsMultipleReaders() bool {
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error { func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
return nil return nil
} }
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) { func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
return len(bufs), nil return len(bufs), nil
} }
func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) { func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) {
+4 -1
View File
@@ -139,7 +139,7 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
} }
} }
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch. // An un-sendable destination costs its own packet, never the ones behind it in the batch.
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer // TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
// silently drops the rest of a burst (linux blocks instead). Poll for // silently drops the rest of a burst (linux blocks instead). Poll for
@@ -150,6 +150,9 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
written++ written++
} else { } else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err) u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
} }
} }
return written, nil return written, nil
+4 -1
View File
@@ -44,7 +44,7 @@ func (u *GenericConn) WriteTo(b []byte, addr netip.AddrPort) error {
return err return err
} }
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch. // An un-sendable destination costs its own packet, never the ones behind it in the batch.
written := 0 written := 0
for i, b := range bufs { for i, b := range bufs {
@@ -52,6 +52,9 @@ func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
written++ written++
} else { } else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err) u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
} }
} }
return written, nil return written, nil
+2 -2
View File
@@ -358,8 +358,8 @@ func sendto(fd int, b []byte, addr netip.AddrPort, isV4 bool) error {
// WriteBatch sends bufs via sendmmsg(2), coalescing same-destination runs into UDP-GSO superpackets when supported. // WriteBatch sends bufs via sendmmsg(2), coalescing same-destination runs into UDP-GSO superpackets when supported.
// See batchWriter in udp_linux_writebatch.go for the mechanics. // See batchWriter in udp_linux_writebatch.go for the mechanics.
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
return u.bw.WriteBatch(bufs, addrs) return u.bw.WriteBatch(bufs, addrs, failed)
} }
// writeSockaddr encodes addr into buf (which must be at least SizeofSockaddrInet6 bytes). // writeSockaddr encodes addr into buf (which must be at least SizeofSockaddrInet6 bytes).
+38 -9
View File
@@ -3,6 +3,7 @@
package udp package udp
import ( import (
"errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net" "net"
@@ -96,13 +97,22 @@ func TestWriteBatchBadFamilyDeliversOthers(t *testing.T) {
bufs := [][]byte{[]byte("AAA"), []byte("BBB"), []byte("CCC")} bufs := [][]byte{[]byte("AAA"), []byte("BBB"), []byte("CCC")}
addrs := []netip.AddrPort{good, bad, good} addrs := []netip.AddrPort{good, bad, good}
n, err := sender.WriteBatch(bufs, addrs) var failed []int
n, err := sender.WriteBatch(bufs, addrs, func(i int, err error) {
if !IsUnreachable(err) {
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
}
failed = append(failed, i)
})
if err != nil { if err != nil {
t.Fatalf("WriteBatch returned error, want nil (bad dest should be isolated): %v", err) t.Fatalf("WriteBatch returned error, want nil (bad dest should be isolated): %v", err)
} }
if n != 2 { if n != 2 {
t.Errorf("WriteBatch wrote %d packets, want 2 of 3 (the bad-family dest is the only casualty)", n) t.Errorf("WriteBatch wrote %d packets, want 2 of 3 (the bad-family dest is the only casualty)", n)
} }
if want := []int{1}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
got := map[string]bool{} got := map[string]bool{}
rx.SetReadDeadline(time.Now().Add(2 * time.Second)) rx.SetReadDeadline(time.Now().Add(2 * time.Second))
@@ -154,7 +164,7 @@ func TestWriteBatchUnreachableDestDeliversOthers(t *testing.T) {
addrs := []netip.AddrPort{good, good, bad, good, good} addrs := []netip.AddrPort{good, good, bad, good, good}
// The bad destination is reported, but only after every other packet has been attempted. // The bad destination is reported, but only after every other packet has been attempted.
if _, err := sender.WriteBatch(bufs, addrs); err == nil { if _, err := sender.WriteBatch(bufs, addrs, nil); err == nil {
t.Log("WriteBatch returned nil; kernel accepted the reserved address, delivery assertions still apply") t.Log("WriteBatch returned nil; kernel accepted the reserved address, delivery assertions still apply")
} }
@@ -376,7 +386,7 @@ func TestWriteBatchPartialSendRewind(t *testing.T) {
return accept, nil return accept, nil
} }
written, err := w.WriteBatch(bufs, addrs) written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil { if err != nil {
t.Fatalf("WriteBatch: %v", err) t.Fatalf("WriteBatch: %v", err)
} }
@@ -429,13 +439,22 @@ func TestWriteBatchSkipUnroutableRunAccounting(t *testing.T) {
return accept, nil return accept, nil
} }
written, err := w.WriteBatch(bufs, addrs) var failed []int
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
if !IsUnreachable(err) {
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
}
failed = append(failed, i)
})
if err != nil { if err != nil {
t.Fatalf("WriteBatch: %v", err) t.Fatalf("WriteBatch: %v", err)
} }
if written != 4 { if written != 4 {
t.Errorf("written = %d, want 4 (the unroutable run is the only casualty)", written) t.Errorf("written = %d, want 4 (the unroutable run is the only casualty)", written)
} }
if want := []int{2}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
wantTags := []byte{1, 2, 4, 5} wantTags := []byte{1, 2, 4, 5}
if len(wire) != len(wantTags) { if len(wire) != len(wantTags) {
t.Fatalf("wire got %d packets, want %d (dup or loss around the skip)", len(wire), len(wantTags)) t.Fatalf("wire got %d packets, want %d (dup or loss around the skip)", len(wire), len(wantTags))
@@ -486,13 +505,23 @@ func TestWriteBatchMidChunkRejectResumes(t *testing.T) {
} }
} }
written, err := w.WriteBatch(bufs, addrs) var failed []int
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
if !errors.Is(err, unix.EPERM) {
t.Errorf("packet %d failed with %v, want EPERM", i, err)
}
failed = append(failed, i)
})
if err != nil { if err != nil {
t.Fatalf("WriteBatch: %v", err) t.Fatalf("WriteBatch: %v", err)
} }
if written != 3 { if written != 3 {
t.Errorf("written = %d, want 3 (B's rejected run is the only casualty)", written) t.Errorf("written = %d, want 3 (B's rejected run is the only casualty)", written)
} }
// Every packet in B's run is reported, not just the entry
if want := []int{2, 3}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
wantTags := []byte{1, 2, 5} wantTags := []byte{1, 2, 5}
if len(wire) != len(wantTags) { if len(wire) != len(wantTags) {
t.Fatalf("wire got %d packets, want %d (dup or loss around the mid-chunk reject)", len(wire), len(wantTags)) t.Fatalf("wire got %d packets, want %d (dup or loss around the mid-chunk reject)", len(wire), len(wantTags))
@@ -543,7 +572,7 @@ func TestWriteBatchMidChunkEIODisablesGSOWithoutDup(t *testing.T) {
} }
} }
written, err := w.WriteBatch(bufs, addrs) written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil { if err != nil {
t.Fatalf("WriteBatch: %v", err) t.Fatalf("WriteBatch: %v", err)
} }
@@ -571,7 +600,7 @@ func TestWriteBatchZeroProgress(t *testing.T) {
w.sendFn = func(start, n int) (int, error) { return 0, nil } w.sendFn = func(start, n int) (int, error) { return 0, nil }
bufs := [][]byte{make([]byte, 100)} bufs := [][]byte{make([]byte, 100)}
addrs := []netip.AddrPort{netip.MustParseAddrPort("127.0.0.1:4242")} addrs := []netip.AddrPort{netip.MustParseAddrPort("127.0.0.1:4242")}
if _, err := w.WriteBatch(bufs, addrs); err == nil { if _, err := w.WriteBatch(bufs, addrs, nil); err == nil {
t.Fatal("WriteBatch = nil error on zero progress, want error") t.Fatal("WriteBatch = nil error on zero progress, want error")
} }
} }
@@ -597,7 +626,7 @@ func TestWriteBatchEIODisablesGSOAndReplays(t *testing.T) {
return n, nil return n, nil
} }
written, err := w.WriteBatch(bufs, addrs) written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil { if err != nil {
t.Fatalf("WriteBatch: %v", err) t.Fatalf("WriteBatch: %v", err)
} }
@@ -674,7 +703,7 @@ func TestGSOEngagesOnLoopback(t *testing.T) {
addrs[i] = dst addrs[i] = dst
} }
written, err := sc.WriteBatch(bufs, addrs) written, err := sc.WriteBatch(bufs, addrs, nil)
if err != nil { if err != nil {
t.Fatalf("WriteBatch: %v", err) t.Fatalf("WriteBatch: %v", err)
} }
+2 -2
View File
@@ -86,7 +86,7 @@ func TestWriteBatchOffloadsDisabledScripted(t *testing.T) {
addrs[i] = dst addrs[i] = dst
} }
written, err := w.WriteBatch(bufs, addrs) written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil { if err != nil {
t.Fatalf("WriteBatch: %v", err) t.Fatalf("WriteBatch: %v", err)
} }
@@ -163,7 +163,7 @@ func TestOffloadsDisabledOnLoopback(t *testing.T) {
addrs[i] = dst addrs[i] = dst
} }
written, err := sc.WriteBatch(bufs, addrs) written, err := sc.WriteBatch(bufs, addrs, nil)
if err != nil { if err != nil {
t.Fatalf("WriteBatch: %v", err) t.Fatalf("WriteBatch: %v", err)
} }
+14 -2
View File
@@ -190,8 +190,8 @@ func parseRelease(r string) (major, minor int) {
// its packets are dropped and the rest of the chunk resumes in place. // its packets are dropped and the rest of the chunk resumes in place.
// //
// Returns the number of packets sent. An error means the call itself failed. // Returns the number of packets sent. An error means the call itself failed.
// A short count means some destinations were undeliverable. // A short count means some destinations were undeliverable, failed hears about each of their packets.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
if len(bufs) != len(addrs) { if len(bufs) != len(addrs) {
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs)) return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
} }
@@ -233,6 +233,7 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
if w.l.Enabled(context.Background(), slog.LevelDebug) { if w.l.Enabled(context.Background(), slog.LevelDebug) {
w.l.Debug("skipping unroutable batch destination", "udpAddr", addrs[i], "packets", runLen, "error", err) w.l.Debug("skipping unroutable batch destination", "udpAddr", addrs[i], "packets", runLen, "error", err)
} }
reportFailed(failed, i, i+runLen, err)
i += runLen i += runLen
continue continue
} }
@@ -300,12 +301,23 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
"gso", w.gsoSupported, "gso", w.gsoSupported,
) )
} }
reportFailed(failed, w.entryEnd[done]-w.entryPkts[done], w.entryEnd[done], serr)
done++ done++
} }
} }
return written, nil return written, nil
} }
// reportFailed tells failed, if anyone is listening, that every packet in bufs[start:end] was lost to err
func reportFailed(failed func(i int, err error), start, end int, err error) {
if failed == nil {
return
}
for i := start; i < end; i++ {
failed(i, err)
}
}
// planRun returns the length of the run starting at start and its segment // planRun returns the length of the run starting at start and its segment
// size (len(bufs[start])). A run of length 1 carries no UDP_SEGMENT cmsg // size (len(bufs[start])). A run of length 1 carries no UDP_SEGMENT cmsg
// and is sent as a plain datagram; without GSO support planRun always returns 1. // and is sent as a plain datagram; without GSO support planRun always returns 1.
+2 -2
View File
@@ -80,11 +80,11 @@ func TestWriteBatchNoAllocs(t *testing.T) {
var werr error var werr error
// Warm-up outside the measured runs. // Warm-up outside the measured runs.
if _, err := tx.WriteBatch(bufs, addrs); err != nil { if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
t.Fatalf("WriteBatch warm-up: %v", err) t.Fatalf("WriteBatch warm-up: %v", err)
} }
allocs := testing.AllocsPerRun(100, func() { allocs := testing.AllocsPerRun(100, func() {
if _, err := tx.WriteBatch(bufs, addrs); err != nil { if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
werr = err werr = err
} }
}) })
+4 -1
View File
@@ -319,7 +319,7 @@ func (u *RIOConn) WriteTo(buf []byte, ip netip.AddrPort) error {
return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0) return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
} }
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch. // An un-sendable destination costs its own packet, never the ones behind it in the batch.
written := 0 written := 0
for i, b := range bufs { for i, b := range bufs {
@@ -327,6 +327,9 @@ func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
written++ written++
} else { } else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err) u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
} }
} }
return written, nil return written, nil
+64 -7
View File
@@ -6,6 +6,7 @@ import (
"context" "context"
"io" "io"
"log/slog" "log/slog"
"net"
"net/netip" "net/netip"
"os" "os"
"sync" "sync"
@@ -63,10 +64,19 @@ func acquirePacket() *Packet {
} }
} }
// testerAddrs is where a TesterConn sends from. A single address host sends everything from addr, whatever the
// destination. A dual stack host sends from the address matching the destination's family and has no route at all
// for a family it holds no address in.
type testerAddrs struct {
addr netip.AddrPort
v4, v6 netip.AddrPort
dual bool
}
type TesterConn struct { type TesterConn struct {
// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can // addrs is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
// move it mid-run to simulate roaming, so it is atomic rather than a plain field. // move it mid-run to simulate roaming, so it is atomic rather than a plain field.
addr atomic.Pointer[netip.AddrPort] addrs atomic.Pointer[testerAddrs]
RxPackets chan *Packet // Packets to receive into nebula RxPackets chan *Packet // Packets to receive into nebula
TxPackets chan *Packet // Packets transmitted outside by nebula TxPackets chan *Packet // Packets transmitted outside by nebula
@@ -94,14 +104,53 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
return c, nil return c, nil
} }
// GetAddr returns the underlay address this conn currently sends from. // GetAddr returns the underlay address this conn currently sends from. For a dual stack conn that is the IPv4
// address, or the IPv6 one when there is no IPv4 service.
func (u *TesterConn) GetAddr() netip.AddrPort { func (u *TesterConn) GetAddr() netip.AddrPort {
return *u.addr.Load() return u.addrs.Load().addr
} }
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network. // SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
func (u *TesterConn) SetAddr(addr netip.AddrPort) { func (u *TesterConn) SetAddr(addr netip.AddrPort) {
u.addr.Store(&addr) u.addrs.Store(&testerAddrs{addr: addr})
}
// SetDualStackAddrs gives this conn one underlay address per family, standing in for a host whose routing table picks
// the source by destination. Pass an invalid AddrPort for a family the network does not offer, sends to that family
// then fail like they would on a host with no route for it.
func (u *TesterConn) SetDualStackAddrs(v4, v6 netip.AddrPort) {
a := &testerAddrs{v4: v4, v6: v6, dual: true}
if v4.IsValid() {
a.addr = v4
} else {
a.addr = v6
}
u.addrs.Store(a)
}
// GetAddrFor returns the underlay address a send to dst leaves from, GetAddr when there is no route to dst
func (u *TesterConn) GetAddrFor(dst netip.AddrPort) netip.AddrPort {
if src, err := u.sourceFor(dst); err == nil {
return src
}
return u.GetAddr()
}
// sourceFor returns the address a send to dst leaves from
func (u *TesterConn) sourceFor(dst netip.AddrPort) (netip.AddrPort, error) {
a := u.addrs.Load()
if !a.dual {
return a.addr, nil
}
src := a.v6
if dst.Addr().Unmap().Is4() {
src = a.v4
}
if !src.IsValid() {
return netip.AddrPort{}, &net.OpError{Op: "write", Net: "udp", Addr: net.UDPAddrFromAddrPort(dst), Err: errNetUnreachable}
}
return src, nil
} }
// Send will place a UdpPacket onto the receive queue for nebula to consume // Send will place a UdpPacket onto the receive queue for nebula to consume
@@ -153,6 +202,11 @@ func (u *TesterConn) Get(block bool) *Packet {
//********************************************************************************************************************// //********************************************************************************************************************//
func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error { func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
from, err := u.sourceFor(addr)
if err != nil {
return err
}
p := acquirePacket() p := acquirePacket()
if cap(p.Data) < len(b) { if cap(p.Data) < len(b) {
p.Data = make([]byte, len(b)) p.Data = make([]byte, len(b))
@@ -160,7 +214,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
p.Data = p.Data[:len(b)] p.Data = p.Data[:len(b)]
} }
copy(p.Data, b) copy(p.Data, b)
p.From = u.GetAddr() p.From = from
p.To = addr p.To = addr
select { select {
case <-u.done: case <-u.done:
@@ -170,13 +224,16 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
return nil return nil
} }
} }
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
written := 0 written := 0
for i, b := range bufs { for i, b := range bufs {
if err := u.WriteTo(b, addrs[i]); err == nil { if err := u.WriteTo(b, addrs[i]); err == nil {
written++ written++
} else { } else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err) u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
} }
} }
return written, nil return written, nil
+22
View File
@@ -0,0 +1,22 @@
//go:build !windows
package udp
import (
"errors"
"syscall"
)
// errNetUnreachable is the error this platform's sockets give for a destination with no route
var errNetUnreachable error = syscall.ENETUNREACH
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
// to an IPv6 remote into one of these.
func IsUnreachable(err error) bool {
return errors.Is(err, syscall.ENETUNREACH) ||
errors.Is(err, syscall.EHOSTUNREACH) ||
errors.Is(err, syscall.EADDRNOTAVAIL) ||
errors.Is(err, syscall.EAFNOSUPPORT) ||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
}
+37
View File
@@ -0,0 +1,37 @@
//go:build !windows
package udp
import (
"net"
"syscall"
"testing"
)
func TestIsUnreachable(t *testing.T) {
wrap := func(err error) error { return &net.OpError{Op: "write", Net: "udp", Err: err} }
for _, err := range []error{
wrap(syscall.ENETUNREACH),
wrap(syscall.EHOSTUNREACH),
wrap(syscall.EADDRNOTAVAIL),
wrap(syscall.EAFNOSUPPORT),
ErrInvalidIPv6RemoteForSocket,
} {
if !IsUnreachable(err) {
t.Errorf("IsUnreachable(%v) = false, want true", err)
}
}
for _, err := range []error{
nil,
wrap(syscall.EPERM),
wrap(syscall.ENOBUFS),
wrap(syscall.EMSGSIZE),
net.ErrClosed,
} {
if IsUnreachable(err) {
t.Errorf("IsUnreachable(%v) = true, want false", err)
}
}
}
+21
View File
@@ -0,0 +1,21 @@
package udp
import (
"errors"
"golang.org/x/sys/windows"
)
// errNetUnreachable is the error this platform's sockets give for a destination with no route
var errNetUnreachable error = windows.WSAENETUNREACH
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
// to an IPv6 remote into one of these.
func IsUnreachable(err error) bool {
return errors.Is(err, windows.WSAENETUNREACH) ||
errors.Is(err, windows.WSAEHOSTUNREACH) ||
errors.Is(err, windows.WSAEADDRNOTAVAIL) ||
errors.Is(err, windows.WSAEAFNOSUPPORT) ||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
}