Compare commits

...
7 Commits
27 changed files with 1199 additions and 78 deletions

No files matched your search

+11
View File
@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Changed
- A tunnel whose remote becomes unreachable, such as an IPv6 remote after moving to a network without IPv6, now
moves to another of the peer's addresses within a round trip or two instead of waiting 20-35s to be torn down
and rehandshaked. When a send fails because there is no route to the remote, the tunnel sends a test packet to
every other address it knows for the peer and asks the lighthouse to have the peer punch its new address.
Whichever path answers carries the tunnel. With `punchy.punch` enabled, idle tunnels notice on their next
keepalive punch. An idle tunnel whose remote is unreachable and none of whose other addresses answer is now
torn down by the connection manager rather than left in place. Not yet effective for relayed traffic or on
Windows with the default RIO backend.
### Removed
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
+28 -12
View File
@@ -14,6 +14,7 @@ import (
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
)
type trafficDecision int
@@ -171,7 +172,7 @@ func (cm *connectionManager) Start(ctx context.Context) {
}
func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now)
decision, hostinfo, primary, punch := cm.makeTrafficDecision(localIndex, now)
switch decision {
case deleteTunnel:
@@ -196,6 +197,20 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
}
cm.resetRelayTrafficCheck(hostinfo)
if punch != nil {
cm.sendPunch(punch)
}
}
// sendPunch keeps a tunnel's NAT state alive when we are not sending on it. When we have no route to its remote any
// more, typically after moving to a network without that address family, the tunnel probes its other remotes so the
// peer roams over to us. Nothing else would notice, a tunnel we are not sending on has nothing else that could fail.
func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
remote, err := cm.punchy.SendPunch(hostinfo)
if udp.IsUnreachable(err) {
hostinfo.remoteUnreachable(cm.intf, remote)
}
}
func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
@@ -313,7 +328,9 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
}
}
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) {
// makeTrafficDecision decides what to do with a tunnel under the hostmap lock. It also returns the tunnel to punch when
// we are not sending on it, for the caller to send once the lock is released.
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo, *HostInfo) {
// Read lock the main hostmap to order decisions based on tunnels being the primary tunnel
cm.hostMap.RLock()
defer cm.hostMap.RUnlock()
@@ -321,17 +338,17 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
hostinfo := cm.hostMap.Indexes[localIndex]
if hostinfo == nil {
cm.l.Debug("Not found in hostmap", "localIndex", localIndex)
return doNothing, nil, nil
return doNothing, nil, nil, nil
}
if cm.isInvalidCertificate(now, hostinfo) {
return closeTunnel, hostinfo, nil
return closeTunnel, hostinfo, nil, nil
}
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
return deleteTunnel, hostinfo, nil
return deleteTunnel, hostinfo, nil, nil
}
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
@@ -368,10 +385,10 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
if !outTraffic {
// Send a punch packet to keep the NAT state alive
cm.punchy.SendPunch(hostinfo)
return decision, hostinfo, primary, hostinfo
}
return decision, hostinfo, primary
return decision, hostinfo, primary, nil
}
if hostinfo.isPendingDeletion() {
@@ -380,7 +397,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
"tunnelCheck", m{"state": "dead", "method": "active"},
)
return deleteTunnel, hostinfo, nil
return deleteTunnel, hostinfo, nil, nil
}
decision := doNothing
@@ -394,14 +411,13 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
"primary", mainHostInfo,
)
return closeTunnel, hostinfo, primary
return closeTunnel, hostinfo, primary, nil
}
// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
// Just maintain NAT state if configured to do so.
cm.punchy.SendPunch(hostinfo)
cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
return doNothing, nil, nil
return doNothing, nil, nil, hostinfo
}
// We aren't receiving traffic but we are sending it. The outbound
@@ -427,7 +443,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
hostinfo.setPendingDeletion(true)
cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
return decision, hostinfo, nil
return decision, hostinfo, nil, nil
}
func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) {
+247 -6
View File
@@ -3,13 +3,18 @@ package nebula
import (
"crypto/ed25519"
"crypto/rand"
"io"
"net/netip"
"testing"
"time"
"github.com/flynn/noise"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/overlaytest"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
@@ -266,12 +271,12 @@ func Test_NewConnectionManager_CounterLimits(t *testing.T) {
// Still below the reject limit, the tunnel stays up
nc.In(hostinfo)
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, tryRehandshake, decision)
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, deleteTunnel, decision)
}
@@ -333,14 +338,14 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.True(t, (hostinfo.state.Load()&stateIn != 0))
now := time.Now()
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -348,7 +353,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
// Do another traffic check tick, should still not be pending deletion
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -358,7 +363,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
// Finally advance beyond the inactivity timeout
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
assert.Equal(t, closeTunnel, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -578,3 +583,239 @@ func (d *dummyCert) MarshalJSON() ([]byte, error) {
func (d *dummyCert) Copy() cert.Certificate {
return d
}
// newPunchTestConnectionManager is a connection manager punching through conn, holding the tunnel from newPunchTarget
func newPunchTestConnectionManager(t *testing.T, conn udp.Conn) (*connectionManager, *HostInfo) {
l := test.NewLogger()
hostMap := newHostMap(l)
hostMap.preferredRanges.Store(&[]netip.Prefix{})
lh := newTestLighthouse()
ifce := &Interface{
hostMap: hostMap,
inside: &overlaytest.NoopTun{},
outside: &udp.NoopConn{},
firewall: &Firewall{},
lightHouse: lh,
pki: &PKI{},
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
l: l,
}
ifce.pki.cs.Store(&CertState{
initiatingVersion: cert.Version1,
privateKey: []byte{},
v1Cert: &dummyCert{version: cert.Version1},
})
punchy := newPunchyForTest(t, map[string]any{"punch": true}, conn)
nc := newConnectionManagerFromConfig(l, config.NewC(l), hostMap, punchy)
nc.intf = ifce
ifce.connectionManager = nc
// Tunnel sends go through conn too, though without a key from withTestKey they never get as far as writing
ifce.writers = []udp.Conn{conn}
ifce.metricTxDropped = metrics.NilCounter{}
hostinfo := newPunchTarget()
hostinfo.localIndexId = 1099
hostinfo.remoteIndexId = 9901
hostinfo.ConnectionState = &ConnectionState{myCert: &dummyCert{version: cert.Version1}}
nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
return nc, hostinfo
}
// A tunnel we are not sending on gets punched whether or not it is receiving, the punch is the only thing that
// would notice its remote has become unreachable.
func Test_makeTrafficDecision_punchesWhenNotSending(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
now := time.Now()
// Receiving but not sending
hostinfo.takeTraffic()
nc.In(hostinfo)
decision, _, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Same(t, hostinfo, punch)
// Sending and receiving, our own traffic keeps the NAT state alive
nc.Out(hostinfo)
nc.In(hostinfo)
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Nil(t, punch)
// Unused
decision, decided, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, doNothing, decision)
assert.Nil(t, decided, "an unused tunnel has no relay usage to reset")
assert.Same(t, hostinfo, punch)
// Sending but not receiving gets a test packet instead, which finds an unreachable remote on its own
nc.Out(hostinfo)
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, sendTestPacket, decision)
assert.Nil(t, punch)
}
func Test_sendPunch_probesWhenRemoteUnreachable(t *testing.T) {
t.Run("unreachable", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
"the probe should ask the lighthouse to have the peer punch us")
// Within the probe interval nothing more goes out, however many sends fail
nc.sendPunch(hostinfo)
nc.sendPunch(hostinfo)
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("delivered", func(t *testing.T) {
conn := &recordingConn{}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("other failure", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: io.ErrShortWrite}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "only a missing route says to look elsewhere")
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
}
// A packet the data path had no route for names its tunnel in its header, the peer's index for it
func Test_sendUnreachable_findsTunnelFromHeader(t *testing.T) {
head := func(st header.MessageSubType, remoteIndex uint32) []byte {
return header.Encode(make([]byte, header.Len), header.Version, header.Message, st, remoteIndex, 1)
}
t.Run("direct", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV6)
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("not its remote", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV4)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "a failure on a remote the tunnel is not using is stale")
})
t.Run("relayed", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
// A relay header's index is the relay session's, it must not be taken for a tunnel's even when they collide
nc.intf.sendUnreachable(head(header.MessageRelay, hostinfo.remoteIndexId), punchTargetV6)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("unknown tunnel", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId+1), punchTargetV6)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
})
}
// withTestKey gives a tunnel a real key so its sends get encrypted and written instead of returning early
func withTestKey(hostinfo *HostInfo) {
cs := noise.NewCipherSuite(noise.DH25519, noise.CipherAESGCM, noise.HashSHA256)
hostinfo.ConnectionState.eKey = noiseutil.NewCipherState(noise.UnsafeNewCipherState(cs, [32]byte{}, 0), noise.CipherAESGCM)
}
// The probe itself, end to end from a failed send: a test packet to every remote but the one we cannot reach, and a
// lighthouse query, through both places a send can fail.
func Test_remoteUnreachable_probesOtherRemotes(t *testing.T) {
t.Run("control send", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV4}, conn.wrote, "the failed send, then the probe")
assert.Equal(t, []header.MessageType{header.Test, header.Test}, conn.types)
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("data path batch", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
// Two of the tunnel's packets fail in one batch, they probe once
sb := batch.NewSendBatch(conn, 4, 4096)
for c := uint64(1); c <= 2; c++ {
pkt := header.Encode(sb.Reserve(64), header.Version, header.Message, header.MessageNone, hostinfo.remoteIndexId, c)
sb.Commit(append(pkt, make([]byte, 32)...), punchTargetV6)
}
nc.intf.flushSendBatch(sb, 0)
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6, punchTargetV4}, conn.wrote,
"the failed packets, then one probe")
assert.Equal(t, header.Test, conn.types[2])
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("closing", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.SendMessageToHostInfo(header.CloseTunnel, 0, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a tunnel being torn down does not probe")
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("lighthouse", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.lightHouse.amLighthouse = true
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a lighthouse does not roam its clients")
})
t.Run("nothing else to probe", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.reQueryWait.Store(int64(time.Minute))
// Forget the IPv4 remote, all the tunnel knows is the one it cannot reach
hostinfo.remotes = NewRemoteList(hostinfo.vpnAddrs, nil)
hostinfo.remotes.LearnRemote(hostinfo.vpnAddrs[0], punchTargetV6)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
"the lighthouse is the only way to learn of another remote")
// Past the probe interval, but the lighthouse is only asked again on the requery schedule
hostinfo.nextUnreachableProbe.Store(0)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Empty(t, drainQueries(nc.intf.lightHouse))
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6}, conn.wrote)
})
}
// drainQueries returns every lighthouse query waiting to go out
func drainQueries(lh *LightHouse) []netip.Addr {
var queries []netip.Addr
for {
select {
case addr := <-lh.queryChan:
queries = append(queries, addr)
default:
return queries
}
}
}
+1 -1
View File
@@ -148,7 +148,7 @@ func (c *fakeConn) Rebind() error { c.rebinds++; ret
func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil }
func (c *fakeConn) ListenOut(_ udp.EncReader, _ func()) error { return nil }
func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
return len(bufs), nil
}
func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {}
+12
View File
@@ -117,6 +117,18 @@ func (c *Control) SetUDPAddr(addr netip.AddrPort) {
c.f.outside.(*udp.TesterConn).SetAddr(addr)
}
// GetUDPAddrFor returns the underlay address this node sends to dst from, which differs by family for a dual stack node
func (c *Control) GetUDPAddrFor(dst netip.AddrPort) netip.AddrPort {
return c.f.outside.(*udp.TesterConn).GetAddrFor(dst)
}
// SetDualStackUDPAddrs gives this node one underlay address per family, sending from whichever matches the
// destination. An invalid AddrPort leaves the node without that family, sends to it fail with ENETUNREACH. Register
// any address the node did not start on with the router as well or nothing will route back.
func (c *Control) SetDualStackUDPAddrs(v4, v6 netip.AddrPort) {
c.f.outside.(*udp.TesterConn).SetDualStackAddrs(v4, v6)
}
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
+305
View File
@@ -0,0 +1,305 @@
//go:build e2e_testing
package e2e
import (
"net/netip"
"sync/atomic"
"testing"
"time"
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/e2e/router"
"github.com/slackhq/nebula/udp"
)
// TestDualStackRoamToV4Only measures how long a laptop's tunnels take to come back after it moves from a dual stack
// network, where its tunnels formed over IPv6, to one with no IPv6 service at all: the office to hotel wifi. Every
// peer still has an IPv4 address the laptop can reach, so nothing about the move should strand a tunnel.
//
// It is a measurement, not a pass/fail assertion, and it runs on the production connection manager timers.
//
// go test -tags e2e_testing -v -run TestDualStackRoamToV4Only ./e2e/
func TestDualStackRoamToV4Only(t *testing.T) {
for _, tc := range []dualStackRoamCase{
{name: "both send", meSends: true, themSends: true},
{name: "both send, rebind", meSends: true, themSends: true, rebind: true},
{name: "both send, two tunnels", meSends: true, themSends: true, simultaneous: true},
{name: "laptop sends", meSends: true},
{name: "peer sends", themSends: true},
{name: "peer sends, rebind", themSends: true, rebind: true},
{name: "peer sends, lh over v4", themSends: true, lhOverV4: true},
{name: "peer sends, punchy", themSends: true, punchy: true},
{name: "peer sends, punchy all", themSends: true, punchy: true, punchAll: true},
{name: "laptop sends, peer NAT", meSends: true, peerNAT: true},
{name: "laptop sends, peer NAT, lh v4", meSends: true, peerNAT: true, lhOverV4: true},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fwd, rev := measureDualStackRoam(t, tc)
t.Logf("RESULT %-24s laptop->peer %-9v peer->laptop %-9v", tc.name, fmtRecovery(fwd), fmtRecovery(rev))
})
}
}
// TestDualStackRoamRecovers holds the line TestDualStackRoamToV4Only measures: a tunnel whose remote became unreachable
// moves to the peer's other address instead of waiting out the connection manager to rehandshake, which takes 20s.
func TestDualStackRoamRecovers(t *testing.T) {
for _, tc := range []struct {
dualStackRoamCase
// within bounds recovery in each direction that has traffic
within time.Duration
}{
// Measures 0.4s. Under the 5s connection manager check, so it fails if the data path stops probing.
{dualStackRoamCase{name: "both send", meSends: true, themSends: true}, 2 * time.Second},
// Measures 2.6s or 3.8s, depending on whether a probe beats the peer's punch. Half the 20s it used to take.
{dualStackRoamCase{name: "laptop sends, peer NAT", meSends: true, peerNAT: true}, 10 * time.Second},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fwd, rev := measureDualStackRoam(t, tc.dualStackRoamCase)
if fwd == 0 || fwd > tc.within {
t.Errorf("laptop->peer recovered in %v, want within %v", fmtRecovery(fwd), tc.within)
}
if tc.themSends && (rev == 0 || rev > tc.within) {
t.Errorf("peer->laptop recovered in %v, want within %v", fmtRecovery(rev), tc.within)
}
})
}
}
type dualStackRoamCase struct {
name string
// meSends and themSends say who keeps traffic flowing after the move
meSends, themSends bool
// rebind fires RebindUDPServer on the move, as the darwin network change monitor would
rebind bool
// lhOverV4 forms the lighthouse tunnel over IPv4 at the office, so the move does not break it
lhOverV4 bool
// simultaneous has both sides initiate at once at the office, which can leave each side a different primary
// tunnel for the other
simultaneous bool
// punchy turns on punchy.punch everywhere, punchAll adds punchy.target_all_remotes
punchy, punchAll bool
// peerNAT puts the peer behind a NAT that, after the move, only lets in traffic from addresses the peer has sent to
peerNAT bool
}
func fmtRecovery(d time.Duration) string {
if d == 0 {
return "never"
}
return d.Round(time.Millisecond).String()
}
type dualStackNode struct {
c *nebula.Control
vpnAddr netip.Addr
v4, v6 netip.AddrPort
adv atomic.Pointer[[]netip.Addr]
}
// move puts the node on a new network, advertising only what it can be reached on there
func (n *dualStackNode) move(v4, v6 netip.AddrPort) {
n.v4, n.v6 = v4, v6
n.c.SetDualStackUDPAddrs(v4, v6)
var adv []netip.Addr
for _, a := range []netip.AddrPort{v4, v6} {
if a.IsValid() {
adv = append(adv, a.Addr())
}
}
n.adv.Store(&adv)
}
func newDualStackNode(ca cert.Certificate, caKey []byte, name, vpnNet string, v4, v6 netip.AddrPort, overrides m) *dualStackNode {
c, vpnNets, _, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, name, vpnNet, v4, overrides)
n := &dualStackNode{c: c, vpnAddr: vpnNets[0].Addr()}
n.move(v4, v6)
c.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
return *n.adv.Load()
})
return n
}
// measureDualStackRoam returns how long after the move the first packet made it laptop->peer and peer->laptop, zero
// for a direction that never recovered.
func measureDualStackRoam(t *testing.T, tc dualStackRoamCase) (time.Duration, time.Duration) {
t.Helper()
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
ap := netip.MustParseAddrPort
lhV4, lhV6 := ap("10.0.0.1:4242"), ap("[fd00::1]:4242")
lh := newDualStackNode(ca, caKey, "lh", "10.128.0.1/24", lhV4, lhV6, m{
"lighthouse": m{"am_lighthouse": true},
})
// Production timers rather than the 2s the harness defaults to, this is about how long a real laptop waits
peerCfg := m{
"lighthouse": m{
"hosts": []any{lh.vpnAddr.String()},
"interval": 10,
},
"static_host_map": m{
lh.vpnAddr.String(): []any{lhV4.String(), lhV6.String()},
},
"timers": m{
"connection_alive_interval": 5,
"pending_deletion_interval": 10,
},
"punchy": m{
"punch": tc.punchy,
"target_all_remotes": tc.punchAll,
},
}
me := newDualStackNode(ca, caKey, "me", "10.128.0.2/24", ap("10.0.0.2:4242"), ap("[fd00::2]:4242"), peerCfg)
them := newDualStackNode(ca, caKey, "them", "10.128.0.3/24", ap("10.0.0.3:4242"), ap("[fd00::3]:4242"), peerCfg)
r := router.NewR(t, lh.c, me.c, them.c)
r.CancelFlowLogs()
for _, n := range []*dualStackNode{lh, me, them} {
r.AddRoute(n.v6.Addr(), n.v6.Port(), n.c)
}
defer func() {
lh.c.Stop()
me.c.Stop()
them.c.Stop()
}()
lh.c.Start()
me.c.Start()
them.c.Start()
// At the office IPv6 wins every race, so everything forms over it. The lighthouse tunnel optionally does not,
// for a static_host_map that only lists the lighthouse's IPv4 address.
officeV6 := func(p *udp.Packet, _ *nebula.Control) router.ExitType {
lhTraffic := p.To == lhV4 || p.To == lhV6 || p.From == lhV4 || p.From == lhV6
if p.To.Addr().Is4() == (tc.lhOverV4 && lhTraffic) {
return router.KeepRouting
}
return router.Drop
}
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
// One side at a time, so both sides hold the same single tunnel, unless the case wants them racing
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("establish")))
if !tc.simultaneous {
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
}
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("establish")))
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
if tc.simultaneous && me.c.GetHostmapIndexCount() < 3 {
// A tunnel to the lighthouse and two to the peer, or the race did not happen
t.Logf("simultaneous handshakes settled on a single tunnel this run")
}
drainTun(me.c)
drainTun(them.c)
for _, want := range []struct {
name string
c *nebula.Control
vpnAddr netip.Addr
v6 bool
}{
{"laptop->peer", me.c, them.vpnAddr, true},
{"peer->laptop", them.c, me.vpnAddr, true},
{"laptop->lh", me.c, lh.vpnAddr, !tc.lhOverV4},
} {
hi := want.c.GetHostInfoByVpnAddr(want.vpnAddr, false)
if hi == nil {
t.Fatalf("%s: no tunnel at the office", want.name)
}
if hi.CurrentRemote.Addr().Is6() != want.v6 {
t.Fatalf("%s: tunnel formed over %v, the scenario wants v6=%v", want.name, hi.CurrentRemote, want.v6)
}
}
// Hotel wifi: a new NAT'd IPv4 address and no IPv6 at all. The office addresses are gone, nothing reaches them.
officeV4, officeV6Addr := me.v4, me.v6
hotel := ap("10.0.0.99:4242")
me.move(hotel, netip.AddrPort{})
r.AddRoute(hotel.Addr(), hotel.Port(), me.c)
start := time.Now()
if tc.rebind {
me.c.RebindUDPServer()
}
var fwd, rev atomic.Int64
stop := make(chan struct{})
defer close(stop)
watchTun(them.c, stop, start, &fwd)
watchTun(me.c, stop, start, &rev)
go func() {
tick := time.NewTicker(time.Millisecond * 200)
defer tick.Stop()
for {
select {
case <-stop:
return
case <-tick.C:
if tc.meSends {
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("retry")))
}
if tc.themSends {
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("retry")))
}
}
}
}()
// When only one side has traffic, the other direction recovering is a bonus we still report but do not wait for
done := func() bool {
return (!tc.meSends || fwd.Load() != 0) && (!tc.themSends || rev.Load() != 0)
}
// The peer's NAT has state for the lighthouse, which its own updates keep fresh, and for wherever it sends from here
natOpen := map[netip.AddrPort]bool{lhV4: true, lhV6: true}
isPeer := func(a netip.AddrPort) bool { return a == them.v4 || a == them.v6 }
r.RouteForAllExitFuncOrTimeout(time.Second*90, func(p *udp.Packet, _ *nebula.Control) router.ExitType {
if p.To == officeV4 || p.To == officeV6Addr {
return router.Drop
}
if tc.peerNAT {
if isPeer(p.From) {
natOpen[p.To] = true
} else if isPeer(p.To) && !natOpen[p.From] {
return router.Drop
}
}
if done() {
return router.ExitNow
}
return router.KeepRouting
})
return time.Duration(fwd.Load()), time.Duration(rev.Load())
}
func drainTun(c *nebula.Control) {
for {
select {
case <-c.GetTunTxChan():
default:
return
}
}
}
// watchTun records how long after start the first packet came out of c's tun, and keeps the tun drained after
func watchTun(c *nebula.Control, stop chan struct{}, start time.Time, first *atomic.Int64) {
go func() {
for {
select {
case <-stop:
return
case <-c.GetTunTxChan():
first.CompareAndSwap(0, int64(time.Since(start)))
}
}
}()
}
+14 -13
View File
@@ -497,11 +497,10 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []
// Nope, lets push the sender along
case p := <-udpTx:
r.Lock()
a := sender.GetUDPAddr()
c := r.getControl(a, p.To, p)
c := r.getControl(p)
if c == nil {
r.Unlock()
panic("No control for udp tx " + a.String())
panic("No control for udp tx " + p.To.String())
}
fp := r.unlockedInjectFlow(sender, c, p, false)
c.InjectUDPPacket(p) // copies internally; original is ours to release
@@ -622,8 +621,7 @@ func (r *R) stopFanInWorkers() {
func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
r.Lock()
defer r.Unlock()
a := from.GetUDPAddr()
c := r.getControl(a, p.To, p)
c := r.getControl(p)
if c == nil {
panic(fmt.Sprintf("No control for udp tx %s", p.To))
}
@@ -671,7 +669,7 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
panic(err)
}
receiver := r.getControl(sender.GetUDPAddr(), p.To, p)
receiver := r.getControl(p)
if receiver == nil {
r.Unlock()
panic("Can't RouteExitFunc for host: " + p.To.String())
@@ -767,7 +765,7 @@ func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc)
r.Lock()
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
receiver := r.getControl(p)
if receiver == nil {
r.Unlock()
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
@@ -876,7 +874,7 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
r.Lock()
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
receiver := r.getControl(p)
if receiver == nil {
r.Unlock()
panic("Can't RouteForAllExitFunc for host: " + p.To.String())
@@ -939,7 +937,7 @@ func (r *R) FlushAll() {
})
for {
x, rx, ok := reflect.Select(sc)
_, rx, ok := reflect.Select(sc)
if !ok {
return
}
@@ -947,7 +945,7 @@ func (r *R) FlushAll() {
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
receiver := r.getControl(p)
if receiver == nil {
r.Unlock()
panic("Can't FlushAll for host: " + p.To.String())
@@ -958,16 +956,19 @@ func (r *R) FlushAll() {
}
}
// getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change
// getControl performs or seeds NAT translation and returns the control for p.To, p from fields may change
// This is an internal router function, the caller must hold the lock
func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control {
func (r *R) getControl(p *udp.Packet) *nebula.Control {
// Key on the address the packet actually left from, a dual stack sender has one per family
fromAddr, toAddr := p.From, p.To
if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
p.From = newAddr
}
c, ok := r.inNat[toAddr]
if ok {
r.outNat[outNatKey{from: c.GetUDPAddr(), to: fromAddr}] = toAddr
// Key the return path on the address the receiver will answer from, which is per family for a dual stack node
r.outNat[outNatKey{from: c.GetUDPAddrFor(fromAddr), to: fromAddr}] = toAddr
return c
}
+62
View File
@@ -264,6 +264,9 @@ type HostInfo struct {
// This is used to limit lighthouse re-queries in chatty clients
nextLHQuery atomic.Int64
// nextUnreachableProbe is the earliest we will probe our other remotes again after failing to reach the current one
nextUnreachableProbe atomic.Int64
// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
// This is used to avoid an attack where a handshake packet is replayed after some time
@@ -766,6 +769,65 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
}
}
// unreachableProbeInterval bounds how often a tunnel whose remote we cannot reach probes its other remotes
const unreachableProbeInterval = time.Second
// remoteUnreachable is called when a send to our current remote failed because we have no route to it, typically
// after moving to a network without that address family. Sends a test packet to every other remote we know for this
// host: the first one that arrives roams the peer over to wherever we are now, and its reply roams us onto that path.
//
// A peer behind a NAT drops a test from an address it has never sent to, so every probe also asks the lighthouse
// about the peer. The lighthouse tells the peer where we are now, the peer punches us, and the next probe gets in.
func (i *HostInfo) remoteUnreachable(ifce *Interface, dead netip.AddrPort) {
if i.remotes == nil || i.ConnectionState == nil || i.GetRemote() != dead {
// Nothing to probe with, or we already moved on from the remote that failed
return
}
if ifce.lightHouse == nil || ifce.lightHouse.amLighthouse {
// A lighthouse hands out the address it reaches a client on, it must not go roaming its clients itself
return
}
now := time.Now().UnixNano()
next := i.nextUnreachableProbe.Load()
if now < next || !i.nextUnreachableProbe.CompareAndSwap(next, now+int64(unreachableProbeInterval)) {
return
}
nb := make([]byte, 12)
out := make([]byte, mtu)
probed := 0
i.remotes.ForEach(ifce.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) {
if addr == dead {
return
}
probed++
ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), nb, out)
})
if ifce.l.Enabled(context.Background(), slog.LevelDebug) {
i.logger(ifce.l).Debug("Remote unreachable, probing other remotes",
"udpAddr", dead,
"probed", probed,
)
}
if probed == 0 {
// With nothing else to try, the lighthouse is our only way to learn of another remote. The peer may not have
// one, so ask on the usual requery schedule rather than every probe for as long as the tunnel sits here.
if now < i.nextLHQuery.Load() {
return
}
i.nextLHQuery.Store(now + ifce.reQueryWait.Load())
}
// With other remotes to probe we ask on every probe, without backing off. A repeat re-sends the peer its punch
// notification in case the last one or its punches were lost. If the tunnel stays up on inbound traffic while
// our sends to its remote keep failing (only one direction broken), this continues once a second for as long as
// that lasts. That is a bounded cost for a rare state, and the probe that would clear it has to keep running.
ifce.lightHouse.QueryServer(i.vpnAddrs[0])
}
// Bits within HostInfo.state, everything above stateEpochShift is the epoch
const (
stateIn uint32 = 1 << iota
+5
View File
@@ -15,6 +15,7 @@ import (
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/udp"
)
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
@@ -611,6 +612,10 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
"udpAddr", hr,
)
}
// A closing tunnel is about to be torn down, there is no point looking for another path for it
if t != header.CloseTunnel && udp.IsUnreachable(err) {
hostinfo.remoteUnreachable(f, hr)
}
}
} else {
// Try to send via a relay
+41 -1
View File
@@ -1,6 +1,7 @@
package nebula
import (
"bytes"
"context"
"crypto/fips140"
"errors"
@@ -495,7 +496,8 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
// flushSendBatch drains sb to the underlay and accounts for anything it could not deliver. A shortfall means
// specific destinations were undeliverable (a stale remote, a reject rule), which the backend logs per peer at
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch.
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch. Tunnels whose remote
// we no longer have a route to go looking for another path.
func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
queued := sb.Len()
written, err := sb.Flush()
@@ -505,6 +507,44 @@ func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
if dropped := queued - written; dropped > 0 {
f.metricTxDropped.Inc(int64(dropped))
}
if f.lightHouse == nil || f.lightHouse.amLighthouse {
// Lighthouses never probe, see remoteUnreachable, so skip finding the tunnels
return
}
// A tunnel's packets tend to fail together, so skip straight past a repeat of the one before. The first 8 bytes
// of a header say whose packet it is, the counter after them changes with every packet.
var last batch.Failure
for i, fail := range sb.Failures() {
if i > 0 && fail.Dst == last.Dst && bytes.Equal(fail.Head[:8], last.Head[:8]) {
continue
}
last = fail
if udp.IsUnreachable(fail.Err) {
f.sendUnreachable(fail.Head[:], fail.Dst)
}
}
}
// sendUnreachable finds the tunnel a packet we had no route for was sent on, from the start of the packet, and has it
// look for another path to its peer.
func (f *Interface) sendUnreachable(head []byte, dst netip.AddrPort) {
var h header.H
if h.Parse(head) != nil {
return
}
if h.Type == header.Message && h.Subtype == header.MessageRelay {
// TODO: a relayed packet's header carries the relay session's index, not the relay tunnel's, so it does not
// lead us to the tunnel whose remote failed. The relay tunnel's own traffic finds out soon enough.
return
}
if hostinfo := f.hostMap.QueryReverseIndex(h.RemoteIndex); hostinfo != nil {
hostinfo.remoteUnreachable(f, dst)
}
}
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
+41 -5
View File
@@ -1,12 +1,27 @@
package batch
import "net/netip"
import (
"net/netip"
"github.com/slackhq/nebula/header"
)
const SendBatchCap = 128
// FailureHeadLen is how much of a refused packet a Failure keeps, nebula's header, which says whose packet it was
const FailureHeadLen = header.Len
// batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush.
type batchWriter interface {
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
}
// Failure is a packet the underlay would not send
type Failure struct {
// Head is the start of the packet, zero padded if the packet was shorter
Head [FailureHeadLen]byte
Dst netip.AddrPort
Err error
}
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
@@ -17,16 +32,28 @@ type SendBatch struct {
bufs [][]byte
dsts []netip.AddrPort
arena *Arena
failures []Failure
// recordFailure is bound once so handing it to WriteBatch does not allocate on every Flush
recordFailure func(i int, err error)
}
// NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots
func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch {
return &SendBatch{
b := &SendBatch{
out: out,
bufs: make([][]byte, 0, batchCap),
dsts: make([]netip.AddrPort, 0, batchCap),
arena: NewArena(arenaSize),
}
b.recordFailure = b.failed
return b
}
func (b *SendBatch) failed(i int, err error) {
f := Failure{Dst: b.dsts[i], Err: err}
copy(f.Head[:], b.bufs[i])
b.failures = append(b.failures, f)
}
func (b *SendBatch) Reserve(sz int) []byte {
@@ -44,12 +71,15 @@ func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort) {
}
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations
// were undeliverable; the batch is drained either way.
// were undeliverable, Failures says which; the batch is drained either way.
func (b *SendBatch) Flush() (int, error) {
clear(b.failures)
b.failures = b.failures[:0]
var err error
written := 0
if len(b.bufs) > 0 {
written, err = b.out.WriteBatch(b.bufs, b.dsts)
written, err = b.out.WriteBatch(b.bufs, b.dsts, b.recordFailure)
}
clear(b.bufs)
b.bufs = b.bufs[:0]
@@ -57,3 +87,9 @@ func (b *SendBatch) Flush() (int, error) {
b.arena.Reset()
return written, err
}
// Failures returns every packet the last Flush could not send, in the order they were committed. Valid until the
// next Flush.
func (b *SendBatch) Failures() []Failure {
return b.failures
}
+84 -1
View File
@@ -1,7 +1,9 @@
package batch
import (
"errors"
"net/netip"
"slices"
"testing"
)
@@ -10,7 +12,7 @@ type fakeBatchWriter struct {
addrs []netip.AddrPort
}
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, _ func(int, error)) (int, error) {
// Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch
// returns, so tests must capture data before that happens.
w.bufs = make([][]byte, len(bufs))
@@ -120,3 +122,84 @@ func TestSendBatchGrowPreservesCommitted(t *testing.T) {
t.Errorf("second packet on the wire: %x", fw.bufs[1])
}
}
// failingBatchWriter refuses every packet whose first byte is in refuse
type failingBatchWriter struct {
refuse map[byte]error
}
func (w *failingBatchWriter) WriteBatch(bufs [][]byte, _ []netip.AddrPort, failed func(int, error)) (int, error) {
written := 0
for i, b := range bufs {
if err, ok := w.refuse[b[0]]; ok {
failed(i, err)
continue
}
written++
}
return written, nil
}
func TestSendBatchReportsFailures(t *testing.T) {
errNoRoute := errors.New("no route")
fw := &failingBatchWriter{refuse: map[byte]error{2: errNoRoute, 3: errNoRoute}}
b := NewSendBatch(fw, 4, 128)
// pkt is n bytes starting with the refuse key, each later byte its offset plus 0x10 so the head is recognizable
pkt := func(key byte, n int) []byte {
p := b.Reserve(n)[:n]
p[0] = key
for i := 1; i < n; i++ {
p[i] = byte(0x10 + i)
}
return p
}
head := func(p []byte) (h [FailureHeadLen]byte) {
copy(h[:], p)
return h
}
dsts := []netip.AddrPort{
netip.MustParseAddrPort("10.0.0.1:4242"),
netip.MustParseAddrPort("[fd00::2]:4242"),
netip.MustParseAddrPort("[fd00::3]:4242"),
netip.MustParseAddrPort("10.0.0.4:4242"),
}
long := pkt(2, FailureHeadLen+8) // only its head is kept
short := pkt(3, 4) // zero padded
b.Commit(pkt(1, 20), dsts[0])
b.Commit(long, dsts[1])
b.Commit(short, dsts[2])
b.Commit(pkt(1, 20), dsts[3])
written, err := b.Flush()
if err != nil {
t.Fatalf("Flush: %v", err)
}
if written != 2 {
t.Errorf("written = %d, want 2", written)
}
want := []Failure{{head(long), dsts[1], errNoRoute}, {head(short), dsts[2], errNoRoute}}
if got := b.Failures(); !slices.Equal(got, want) {
t.Errorf("failures = %v, want %v", got, want)
}
// Failures do not leak into the next flush
again := pkt(2, 20)
b.Commit(again, dsts[0])
if _, err := b.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if want := []Failure{{head(again), dsts[0], errNoRoute}}; !slices.Equal(b.Failures(), want) {
t.Errorf("second flush failures = %v, want %v", b.Failures(), want)
}
// A clean flush forgets the last one's failures
b.Commit(pkt(1, 20), dsts[0])
if _, err := b.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if got := b.Failures(); len(got) != 0 {
t.Errorf("clean flush failures = %v, want none", got)
}
}
+19 -10
View File
@@ -165,20 +165,24 @@ func (p *Punchy) scheduleJob(job holepunchJob, delay time.Duration) {
// SendPunch sends an immediate keepalive punch for an idle hostinfo.
// The configured punchy.target_all_remotes mode picks the targets. Gated on punchy.punch and the lighthouse-skip rule
// (lighthouses don't get keepalive punches because the regular update interval keeps their NAT state warm).
func (p *Punchy) SendPunch(hostinfo *HostInfo) {
// Returns the hostinfo's current remote and the error from punching it, an invalid remote if it was not punched.
func (p *Punchy) SendPunch(hostinfo *HostInfo) (netip.AddrPort, error) {
if !p.punch.Load() {
return
return netip.AddrPort{}, nil
}
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
return
return netip.AddrPort{}, nil
}
hr := hostinfo.GetRemote()
if p.punchEverything.Load() {
p.sendPunchToAllRemotes(hostinfo)
} else if hr := hostinfo.GetRemote(); hr.IsValid() {
p.metricPunchyTx.Inc(1)
p.punchConn.WriteTo([]byte{1}, hr)
return hr, p.sendPunchToAllRemotes(hostinfo, hr)
}
if !hr.IsValid() {
return hr, nil
}
p.metricPunchyTx.Inc(1)
return hr, p.punchConn.WriteTo([]byte{1}, hr)
}
// SendPunchToAll punches every known remote for hostinfo, but only when punchy.target_all_remotes is enabled.
@@ -195,14 +199,19 @@ func (p *Punchy) SendPunchToAll(hostinfo *HostInfo) {
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
return
}
p.sendPunchToAllRemotes(hostinfo)
p.sendPunchToAllRemotes(hostinfo, netip.AddrPort{})
}
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo) {
// sendPunchToAllRemotes punches every known remote for hostinfo and returns the error from punching current
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo, current netip.AddrPort) error {
var currentErr error
hostinfo.remotes.ForEach(p.hm.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
p.metricPunchyTx.Inc(1)
p.punchConn.WriteTo([]byte{1}, addr)
if err := p.punchConn.WriteTo([]byte{1}, addr); addr == current {
currentErr = err
}
})
return currentErr
}
// Start wires the runtime dependencies and spawns the scheduler worker.
+112
View File
@@ -3,11 +3,14 @@ package nebula
import (
"context"
"log/slog"
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -242,3 +245,112 @@ func TestPunchyReloadRestartRequired(t *testing.T) {
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
}
// errNoRoute is the unreachable error every platform classifies the same way
var errNoRoute = udp.ErrInvalidIPv6RemoteForSocket
// recordingConn records where every write went and what kind of message it was, and fails the ones to an address in
// fail
type recordingConn struct {
udp.NoopConn
fail map[netip.AddrPort]error
wrote []netip.AddrPort
// types is the message type of each write, a punch has none
types []header.MessageType
}
func (c *recordingConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.wrote = append(c.wrote, addr)
var h header.H
if h.Parse(b) == nil {
c.types = append(c.types, h.Type)
}
return c.fail[addr]
}
func (c *recordingConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(int, error)) (int, error) {
written := 0
for i := range bufs {
if err := c.WriteTo(bufs[i], addrs[i]); err != nil {
failed(i, err)
continue
}
written++
}
return written, nil
}
type noLighthouses struct{}
func (noLighthouses) IsAnyLighthouseAddr([]netip.Addr) bool { return false }
var (
punchTargetV4 = netip.MustParseAddrPort("10.0.0.3:4242")
punchTargetV6 = netip.MustParseAddrPort("[fd00::3]:4242")
)
// newPunchTarget is a tunnel on its IPv6 remote that also knows the peer's IPv4 one
func newPunchTarget() *HostInfo {
vpnAddr := netip.MustParseAddr("10.128.0.3")
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
remotes: NewRemoteList([]netip.Addr{vpnAddr}, nil),
}
hostinfo.remotes.LearnRemote(vpnAddr, punchTargetV4)
hostinfo.SetRemote(punchTargetV6)
return hostinfo
}
func newPunchyForTest(t *testing.T, settings map[string]any, conn udp.Conn) *Punchy {
c := config.NewC(test.NewLogger())
c.Settings["punchy"] = settings
p := NewPunchyFromConfig(test.NewLogger(), c, conn)
hm := newHostMap(test.NewLogger())
hm.preferredRanges.Store(&[]netip.Prefix{})
p.hm = hm
p.lh = noLighthouses{}
return p
}
func TestPunchy_SendPunchReportsCurrentRemote(t *testing.T) {
t.Run("disabled", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": false}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.False(t, remote.IsValid())
require.NoError(t, err)
assert.Empty(t, conn.wrote)
})
t.Run("current remote", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.ErrorIs(t, err, errNoRoute)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
})
t.Run("all remotes, current fails", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.ErrorIs(t, err, errNoRoute)
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
})
t.Run("all remotes, only another fails", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV4: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.NoError(t, err, "a failure on a remote we are not using says nothing about the current one")
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
})
}
+3 -2
View File
@@ -41,8 +41,9 @@ type Conn interface {
//
// Returns the number of packets successfully written. A destination the kernel rejects costs only
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
// failed, when not nil, is called with the index of every packet a destination cost us and why.
// Not safe for concurrent use on the same Conn.
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
// ReloadConfig applies the listen settings that can change, r is nil at startup
ReloadConfig(c *config.C, r *config.Reload)
SupportsMultipleReaders() bool
@@ -66,7 +67,7 @@ func (NoopConn) SupportsMultipleReaders() bool {
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
return nil
}
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
return len(bufs), nil
}
func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) {
+4 -1
View File
@@ -139,7 +139,7 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
}
}
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
// silently drops the rest of a burst (linux blocks instead). Poll for
@@ -150,6 +150,9 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+4 -1
View File
@@ -44,7 +44,7 @@ func (u *GenericConn) WriteTo(b []byte, addr netip.AddrPort) error {
return err
}
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
written := 0
for i, b := range bufs {
@@ -52,6 +52,9 @@ func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+2 -2
View File
@@ -358,8 +358,8 @@ func sendto(fd int, b []byte, addr netip.AddrPort, isV4 bool) error {
// WriteBatch sends bufs via sendmmsg(2), coalescing same-destination runs into UDP-GSO superpackets when supported.
// See batchWriter in udp_linux_writebatch.go for the mechanics.
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
return u.bw.WriteBatch(bufs, addrs)
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
return u.bw.WriteBatch(bufs, addrs, failed)
}
// writeSockaddr encodes addr into buf (which must be at least SizeofSockaddrInet6 bytes).
+38 -9
View File
@@ -3,6 +3,7 @@
package udp
import (
"errors"
"fmt"
"log/slog"
"net"
@@ -96,13 +97,22 @@ func TestWriteBatchBadFamilyDeliversOthers(t *testing.T) {
bufs := [][]byte{[]byte("AAA"), []byte("BBB"), []byte("CCC")}
addrs := []netip.AddrPort{good, bad, good}
n, err := sender.WriteBatch(bufs, addrs)
var failed []int
n, err := sender.WriteBatch(bufs, addrs, func(i int, err error) {
if !IsUnreachable(err) {
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
}
failed = append(failed, i)
})
if err != nil {
t.Fatalf("WriteBatch returned error, want nil (bad dest should be isolated): %v", err)
}
if n != 2 {
t.Errorf("WriteBatch wrote %d packets, want 2 of 3 (the bad-family dest is the only casualty)", n)
}
if want := []int{1}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
got := map[string]bool{}
rx.SetReadDeadline(time.Now().Add(2 * time.Second))
@@ -154,7 +164,7 @@ func TestWriteBatchUnreachableDestDeliversOthers(t *testing.T) {
addrs := []netip.AddrPort{good, good, bad, good, good}
// The bad destination is reported, but only after every other packet has been attempted.
if _, err := sender.WriteBatch(bufs, addrs); err == nil {
if _, err := sender.WriteBatch(bufs, addrs, nil); err == nil {
t.Log("WriteBatch returned nil; kernel accepted the reserved address, delivery assertions still apply")
}
@@ -376,7 +386,7 @@ func TestWriteBatchPartialSendRewind(t *testing.T) {
return accept, nil
}
written, err := w.WriteBatch(bufs, addrs)
written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -429,13 +439,22 @@ func TestWriteBatchSkipUnroutableRunAccounting(t *testing.T) {
return accept, nil
}
written, err := w.WriteBatch(bufs, addrs)
var failed []int
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
if !IsUnreachable(err) {
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
}
failed = append(failed, i)
})
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
if written != 4 {
t.Errorf("written = %d, want 4 (the unroutable run is the only casualty)", written)
}
if want := []int{2}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
wantTags := []byte{1, 2, 4, 5}
if len(wire) != len(wantTags) {
t.Fatalf("wire got %d packets, want %d (dup or loss around the skip)", len(wire), len(wantTags))
@@ -486,13 +505,23 @@ func TestWriteBatchMidChunkRejectResumes(t *testing.T) {
}
}
written, err := w.WriteBatch(bufs, addrs)
var failed []int
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
if !errors.Is(err, unix.EPERM) {
t.Errorf("packet %d failed with %v, want EPERM", i, err)
}
failed = append(failed, i)
})
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
if written != 3 {
t.Errorf("written = %d, want 3 (B's rejected run is the only casualty)", written)
}
// Every packet in B's run is reported, not just the entry
if want := []int{2, 3}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
wantTags := []byte{1, 2, 5}
if len(wire) != len(wantTags) {
t.Fatalf("wire got %d packets, want %d (dup or loss around the mid-chunk reject)", len(wire), len(wantTags))
@@ -543,7 +572,7 @@ func TestWriteBatchMidChunkEIODisablesGSOWithoutDup(t *testing.T) {
}
}
written, err := w.WriteBatch(bufs, addrs)
written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -571,7 +600,7 @@ func TestWriteBatchZeroProgress(t *testing.T) {
w.sendFn = func(start, n int) (int, error) { return 0, nil }
bufs := [][]byte{make([]byte, 100)}
addrs := []netip.AddrPort{netip.MustParseAddrPort("127.0.0.1:4242")}
if _, err := w.WriteBatch(bufs, addrs); err == nil {
if _, err := w.WriteBatch(bufs, addrs, nil); err == nil {
t.Fatal("WriteBatch = nil error on zero progress, want error")
}
}
@@ -597,7 +626,7 @@ func TestWriteBatchEIODisablesGSOAndReplays(t *testing.T) {
return n, nil
}
written, err := w.WriteBatch(bufs, addrs)
written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -674,7 +703,7 @@ func TestGSOEngagesOnLoopback(t *testing.T) {
addrs[i] = dst
}
written, err := sc.WriteBatch(bufs, addrs)
written, err := sc.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
+2 -2
View File
@@ -86,7 +86,7 @@ func TestWriteBatchOffloadsDisabledScripted(t *testing.T) {
addrs[i] = dst
}
written, err := w.WriteBatch(bufs, addrs)
written, err := w.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -163,7 +163,7 @@ func TestOffloadsDisabledOnLoopback(t *testing.T) {
addrs[i] = dst
}
written, err := sc.WriteBatch(bufs, addrs)
written, err := sc.WriteBatch(bufs, addrs, nil)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
+14 -2
View File
@@ -190,8 +190,8 @@ func parseRelease(r string) (major, minor int) {
// its packets are dropped and the rest of the chunk resumes in place.
//
// Returns the number of packets sent. An error means the call itself failed.
// A short count means some destinations were undeliverable.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// A short count means some destinations were undeliverable, failed hears about each of their packets.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
if len(bufs) != len(addrs) {
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
}
@@ -233,6 +233,7 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
if w.l.Enabled(context.Background(), slog.LevelDebug) {
w.l.Debug("skipping unroutable batch destination", "udpAddr", addrs[i], "packets", runLen, "error", err)
}
reportFailed(failed, i, i+runLen, err)
i += runLen
continue
}
@@ -300,12 +301,23 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, er
"gso", w.gsoSupported,
)
}
reportFailed(failed, w.entryEnd[done]-w.entryPkts[done], w.entryEnd[done], serr)
done++
}
}
return written, nil
}
// reportFailed tells failed, if anyone is listening, that every packet in bufs[start:end] was lost to err
func reportFailed(failed func(i int, err error), start, end int, err error) {
if failed == nil {
return
}
for i := start; i < end; i++ {
failed(i, err)
}
}
// planRun returns the length of the run starting at start and its segment
// size (len(bufs[start])). A run of length 1 carries no UDP_SEGMENT cmsg
// and is sent as a plain datagram; without GSO support planRun always returns 1.
+2 -2
View File
@@ -80,11 +80,11 @@ func TestWriteBatchNoAllocs(t *testing.T) {
var werr error
// Warm-up outside the measured runs.
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
t.Fatalf("WriteBatch warm-up: %v", err)
}
allocs := testing.AllocsPerRun(100, func() {
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
werr = err
}
})
+4 -1
View File
@@ -319,7 +319,7 @@ func (u *RIOConn) WriteTo(buf []byte, ip netip.AddrPort) error {
return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
}
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
written := 0
for i, b := range bufs {
@@ -327,6 +327,9 @@ func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+64 -7
View File
@@ -6,6 +6,7 @@ import (
"context"
"io"
"log/slog"
"net"
"net/netip"
"os"
"sync"
@@ -63,10 +64,19 @@ func acquirePacket() *Packet {
}
}
// testerAddrs is where a TesterConn sends from. A single address host sends everything from addr, whatever the
// destination. A dual stack host sends from the address matching the destination's family and has no route at all
// for a family it holds no address in.
type testerAddrs struct {
addr netip.AddrPort
v4, v6 netip.AddrPort
dual bool
}
type TesterConn struct {
// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
// addrs is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
addr atomic.Pointer[netip.AddrPort]
addrs atomic.Pointer[testerAddrs]
RxPackets chan *Packet // Packets to receive into nebula
TxPackets chan *Packet // Packets transmitted outside by nebula
@@ -94,14 +104,53 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
return c, nil
}
// GetAddr returns the underlay address this conn currently sends from.
// GetAddr returns the underlay address this conn currently sends from. For a dual stack conn that is the IPv4
// address, or the IPv6 one when there is no IPv4 service.
func (u *TesterConn) GetAddr() netip.AddrPort {
return *u.addr.Load()
return u.addrs.Load().addr
}
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
func (u *TesterConn) SetAddr(addr netip.AddrPort) {
u.addr.Store(&addr)
u.addrs.Store(&testerAddrs{addr: addr})
}
// SetDualStackAddrs gives this conn one underlay address per family, standing in for a host whose routing table picks
// the source by destination. Pass an invalid AddrPort for a family the network does not offer, sends to that family
// then fail like they would on a host with no route for it.
func (u *TesterConn) SetDualStackAddrs(v4, v6 netip.AddrPort) {
a := &testerAddrs{v4: v4, v6: v6, dual: true}
if v4.IsValid() {
a.addr = v4
} else {
a.addr = v6
}
u.addrs.Store(a)
}
// GetAddrFor returns the underlay address a send to dst leaves from, GetAddr when there is no route to dst
func (u *TesterConn) GetAddrFor(dst netip.AddrPort) netip.AddrPort {
if src, err := u.sourceFor(dst); err == nil {
return src
}
return u.GetAddr()
}
// sourceFor returns the address a send to dst leaves from
func (u *TesterConn) sourceFor(dst netip.AddrPort) (netip.AddrPort, error) {
a := u.addrs.Load()
if !a.dual {
return a.addr, nil
}
src := a.v6
if dst.Addr().Unmap().Is4() {
src = a.v4
}
if !src.IsValid() {
return netip.AddrPort{}, &net.OpError{Op: "write", Net: "udp", Addr: net.UDPAddrFromAddrPort(dst), Err: errNetUnreachable}
}
return src, nil
}
// Send will place a UdpPacket onto the receive queue for nebula to consume
@@ -153,6 +202,11 @@ func (u *TesterConn) Get(block bool) *Packet {
//********************************************************************************************************************//
func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
from, err := u.sourceFor(addr)
if err != nil {
return err
}
p := acquirePacket()
if cap(p.Data) < len(b) {
p.Data = make([]byte, len(b))
@@ -160,7 +214,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
p.Data = p.Data[:len(b)]
}
copy(p.Data, b)
p.From = u.GetAddr()
p.From = from
p.To = addr
select {
case <-u.done:
@@ -170,13 +224,16 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
return nil
}
}
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
written := 0
for i, b := range bufs {
if err := u.WriteTo(b, addrs[i]); err == nil {
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+22
View File
@@ -0,0 +1,22 @@
//go:build !windows
package udp
import (
"errors"
"syscall"
)
// errNetUnreachable is the error this platform's sockets give for a destination with no route
var errNetUnreachable error = syscall.ENETUNREACH
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
// to an IPv6 remote into one of these.
func IsUnreachable(err error) bool {
return errors.Is(err, syscall.ENETUNREACH) ||
errors.Is(err, syscall.EHOSTUNREACH) ||
errors.Is(err, syscall.EADDRNOTAVAIL) ||
errors.Is(err, syscall.EAFNOSUPPORT) ||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
}
+37
View File
@@ -0,0 +1,37 @@
//go:build !windows
package udp
import (
"net"
"syscall"
"testing"
)
func TestIsUnreachable(t *testing.T) {
wrap := func(err error) error { return &net.OpError{Op: "write", Net: "udp", Err: err} }
for _, err := range []error{
wrap(syscall.ENETUNREACH),
wrap(syscall.EHOSTUNREACH),
wrap(syscall.EADDRNOTAVAIL),
wrap(syscall.EAFNOSUPPORT),
ErrInvalidIPv6RemoteForSocket,
} {
if !IsUnreachable(err) {
t.Errorf("IsUnreachable(%v) = false, want true", err)
}
}
for _, err := range []error{
nil,
wrap(syscall.EPERM),
wrap(syscall.ENOBUFS),
wrap(syscall.EMSGSIZE),
net.ErrClosed,
} {
if IsUnreachable(err) {
t.Errorf("IsUnreachable(%v) = true, want false", err)
}
}
}
+21
View File
@@ -0,0 +1,21 @@
package udp
import (
"errors"
"golang.org/x/sys/windows"
)
// errNetUnreachable is the error this platform's sockets give for a destination with no route
var errNetUnreachable error = windows.WSAENETUNREACH
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
// to an IPv6 remote into one of these.
func IsUnreachable(err error) bool {
return errors.Is(err, windows.WSAENETUNREACH) ||
errors.Is(err, windows.WSAEHOSTUNREACH) ||
errors.Is(err, windows.WSAEADDRNOTAVAIL) ||
errors.Is(err, windows.WSAEAFNOSUPPORT) ||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
}