Compare commits

..
Author SHA1 Message Date
dependabot[bot] ca1e6178a1 Bump golang.zx2c4.com/wireguard/windows
Bumps the zx2c4-dependencies group with 1 update in the / directory: golang.zx2c4.com/wireguard/windows.


Updates `golang.zx2c4.com/wireguard/windows` from 1.0.1 to 1.1.1

---
updated-dependencies:
- dependency-name: golang.zx2c4.com/wireguard/windows
  dependency-version: 1.1.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: zx2c4-dependencies
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-10-19 19:53:12 +00:00
38 changed files with 196 additions and 2280 deletions

No files matched your search

-11
View File
@@ -7,17 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Changed
- A tunnel whose remote becomes unreachable, such as an IPv6 remote after moving to a network without IPv6, now
moves to another of the peer's addresses within a round trip or two instead of waiting 20-35s to be torn down
and rehandshaked. When a send fails because there is no route to the remote, the tunnel sends a test packet to
every other address it knows for the peer and asks the lighthouse to have the peer punch its new address.
Whichever path answers carries the tunnel. With `punchy.punch` enabled, idle tunnels notice on their next
keepalive punch. An idle tunnel whose remote is unreachable and none of whose other addresses answer is now
torn down by the connection manager rather than left in place. Not yet effective for relayed traffic or on
Windows with the default RIO backend.
### Removed
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
-15
View File
@@ -86,21 +86,6 @@ func (c *C) InitialLoad() bool {
return c.oldSettings == nil
}
// Render returns keys and their values as yaml, for a subsystem to remember what it applied. Empty means they couldn't
// be rendered and should never match.
func (c *C) Render(keys ...string) string {
m := make(map[string]any, len(keys))
for _, k := range keys {
m[k] = c.Get(k)
}
b, err := yaml.Marshal(m)
if err != nil {
c.l.Debug("Failed to render config", "keys", keys, "error", err)
return ""
}
return string(b)
}
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
// k in both the old and new settings will be serialized, the result of the string comparison is returned.
// If k is an empty string the entire config is tested.
+12 -28
View File
@@ -14,7 +14,6 @@ import (
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
)
type trafficDecision int
@@ -172,7 +171,7 @@ func (cm *connectionManager) Start(ctx context.Context) {
}
func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
decision, hostinfo, primary, punch := cm.makeTrafficDecision(localIndex, now)
decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now)
switch decision {
case deleteTunnel:
@@ -197,20 +196,6 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
}
cm.resetRelayTrafficCheck(hostinfo)
if punch != nil {
cm.sendPunch(punch)
}
}
// sendPunch keeps a tunnel's NAT state alive when we are not sending on it. When we have no route to its remote any
// more, typically after moving to a network without that address family, the tunnel probes its other remotes so the
// peer roams over to us. Nothing else would notice, a tunnel we are not sending on has nothing else that could fail.
func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
remote, err := cm.punchy.SendPunch(hostinfo)
if udp.IsUnreachable(err) {
hostinfo.remoteUnreachable(cm.intf, remote)
}
}
func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
@@ -328,9 +313,7 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
}
}
// makeTrafficDecision decides what to do with a tunnel under the hostmap lock. It also returns the tunnel to punch when
// we are not sending on it, for the caller to send once the lock is released.
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo, *HostInfo) {
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) {
// Read lock the main hostmap to order decisions based on tunnels being the primary tunnel
cm.hostMap.RLock()
defer cm.hostMap.RUnlock()
@@ -338,17 +321,17 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
hostinfo := cm.hostMap.Indexes[localIndex]
if hostinfo == nil {
cm.l.Debug("Not found in hostmap", "localIndex", localIndex)
return doNothing, nil, nil, nil
return doNothing, nil, nil
}
if cm.isInvalidCertificate(now, hostinfo) {
return closeTunnel, hostinfo, nil, nil
return closeTunnel, hostinfo, nil
}
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
return deleteTunnel, hostinfo, nil, nil
return deleteTunnel, hostinfo, nil
}
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
@@ -385,10 +368,10 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
if !outTraffic {
// Send a punch packet to keep the NAT state alive
return decision, hostinfo, primary, hostinfo
cm.punchy.SendPunch(hostinfo)
}
return decision, hostinfo, primary, nil
return decision, hostinfo, primary
}
if hostinfo.isPendingDeletion() {
@@ -397,7 +380,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
"tunnelCheck", m{"state": "dead", "method": "active"},
)
return deleteTunnel, hostinfo, nil, nil
return deleteTunnel, hostinfo, nil
}
decision := doNothing
@@ -411,13 +394,14 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
"primary", mainHostInfo,
)
return closeTunnel, hostinfo, primary, nil
return closeTunnel, hostinfo, primary
}
// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
// Just maintain NAT state if configured to do so.
cm.punchy.SendPunch(hostinfo)
cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
return doNothing, nil, nil, hostinfo
return doNothing, nil, nil
}
// We aren't receiving traffic but we are sending it. The outbound
@@ -443,7 +427,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
hostinfo.setPendingDeletion(true)
cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
return decision, hostinfo, nil, nil
return decision, hostinfo, nil
}
func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) {
+6 -247
View File
@@ -3,18 +3,13 @@ package nebula
import (
"crypto/ed25519"
"crypto/rand"
"io"
"net/netip"
"testing"
"time"
"github.com/flynn/noise"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/overlaytest"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
@@ -271,12 +266,12 @@ func Test_NewConnectionManager_CounterLimits(t *testing.T) {
// Still below the reject limit, the tunnel stays up
nc.In(hostinfo)
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, tryRehandshake, decision)
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, deleteTunnel, decision)
}
@@ -338,14 +333,14 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.True(t, (hostinfo.state.Load()&stateIn != 0))
now := time.Now()
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -353,7 +348,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
// Do another traffic check tick, should still not be pending deletion
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -363,7 +358,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
// Finally advance beyond the inactivity timeout
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
assert.Equal(t, closeTunnel, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -583,239 +578,3 @@ func (d *dummyCert) MarshalJSON() ([]byte, error) {
func (d *dummyCert) Copy() cert.Certificate {
return d
}
// newPunchTestConnectionManager is a connection manager punching through conn, holding the tunnel from newPunchTarget
func newPunchTestConnectionManager(t *testing.T, conn udp.Conn) (*connectionManager, *HostInfo) {
l := test.NewLogger()
hostMap := newHostMap(l)
hostMap.preferredRanges.Store(&[]netip.Prefix{})
lh := newTestLighthouse()
ifce := &Interface{
hostMap: hostMap,
inside: &overlaytest.NoopTun{},
outside: &udp.NoopConn{},
firewall: &Firewall{},
lightHouse: lh,
pki: &PKI{},
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
l: l,
}
ifce.pki.cs.Store(&CertState{
initiatingVersion: cert.Version1,
privateKey: []byte{},
v1Cert: &dummyCert{version: cert.Version1},
})
punchy := newPunchyForTest(t, map[string]any{"punch": true}, conn)
nc := newConnectionManagerFromConfig(l, config.NewC(l), hostMap, punchy)
nc.intf = ifce
ifce.connectionManager = nc
// Tunnel sends go through conn too, though without a key from withTestKey they never get as far as writing
ifce.writers = []udp.Conn{conn}
ifce.metricTxDropped = metrics.NilCounter{}
hostinfo := newPunchTarget()
hostinfo.localIndexId = 1099
hostinfo.remoteIndexId = 9901
hostinfo.ConnectionState = &ConnectionState{myCert: &dummyCert{version: cert.Version1}}
nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
return nc, hostinfo
}
// A tunnel we are not sending on gets punched whether or not it is receiving, the punch is the only thing that
// would notice its remote has become unreachable.
func Test_makeTrafficDecision_punchesWhenNotSending(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
now := time.Now()
// Receiving but not sending
hostinfo.takeTraffic()
nc.In(hostinfo)
decision, _, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Same(t, hostinfo, punch)
// Sending and receiving, our own traffic keeps the NAT state alive
nc.Out(hostinfo)
nc.In(hostinfo)
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Nil(t, punch)
// Unused
decision, decided, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, doNothing, decision)
assert.Nil(t, decided, "an unused tunnel has no relay usage to reset")
assert.Same(t, hostinfo, punch)
// Sending but not receiving gets a test packet instead, which finds an unreachable remote on its own
nc.Out(hostinfo)
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, sendTestPacket, decision)
assert.Nil(t, punch)
}
func Test_sendPunch_probesWhenRemoteUnreachable(t *testing.T) {
t.Run("unreachable", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
"the probe should ask the lighthouse to have the peer punch us")
// Within the probe interval nothing more goes out, however many sends fail
nc.sendPunch(hostinfo)
nc.sendPunch(hostinfo)
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("delivered", func(t *testing.T) {
conn := &recordingConn{}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("other failure", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: io.ErrShortWrite}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "only a missing route says to look elsewhere")
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
}
// A packet the data path had no route for names its tunnel in its header, the peer's index for it
func Test_sendUnreachable_findsTunnelFromHeader(t *testing.T) {
head := func(st header.MessageSubType, remoteIndex uint32) []byte {
return header.Encode(make([]byte, header.Len), header.Version, header.Message, st, remoteIndex, 1)
}
t.Run("direct", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV6)
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("not its remote", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV4)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "a failure on a remote the tunnel is not using is stale")
})
t.Run("relayed", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
// A relay header's index is the relay session's, it must not be taken for a tunnel's even when they collide
nc.intf.sendUnreachable(head(header.MessageRelay, hostinfo.remoteIndexId), punchTargetV6)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("unknown tunnel", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId+1), punchTargetV6)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
})
}
// withTestKey gives a tunnel a real key so its sends get encrypted and written instead of returning early
func withTestKey(hostinfo *HostInfo) {
cs := noise.NewCipherSuite(noise.DH25519, noise.CipherAESGCM, noise.HashSHA256)
hostinfo.ConnectionState.eKey = noiseutil.NewCipherState(noise.UnsafeNewCipherState(cs, [32]byte{}, 0), noise.CipherAESGCM)
}
// The probe itself, end to end from a failed send: a test packet to every remote but the one we cannot reach, and a
// lighthouse query, through both places a send can fail.
func Test_remoteUnreachable_probesOtherRemotes(t *testing.T) {
t.Run("control send", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV4}, conn.wrote, "the failed send, then the probe")
assert.Equal(t, []header.MessageType{header.Test, header.Test}, conn.types)
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("data path batch", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
// Two of the tunnel's packets fail in one batch, they probe once
sb := batch.NewSendBatch(conn, 4, 4096)
for c := uint64(1); c <= 2; c++ {
pkt := header.Encode(sb.Reserve(64), header.Version, header.Message, header.MessageNone, hostinfo.remoteIndexId, c)
sb.Commit(append(pkt, make([]byte, 32)...), punchTargetV6)
}
nc.intf.flushSendBatch(sb, 0)
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6, punchTargetV4}, conn.wrote,
"the failed packets, then one probe")
assert.Equal(t, header.Test, conn.types[2])
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("closing", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.SendMessageToHostInfo(header.CloseTunnel, 0, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a tunnel being torn down does not probe")
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("lighthouse", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.lightHouse.amLighthouse = true
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a lighthouse does not roam its clients")
})
t.Run("nothing else to probe", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.reQueryWait.Store(int64(time.Minute))
// Forget the IPv4 remote, all the tunnel knows is the one it cannot reach
hostinfo.remotes = NewRemoteList(hostinfo.vpnAddrs, nil)
hostinfo.remotes.LearnRemote(hostinfo.vpnAddrs[0], punchTargetV6)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
"the lighthouse is the only way to learn of another remote")
// Past the probe interval, but the lighthouse is only asked again on the requery schedule
hostinfo.nextUnreachableProbe.Store(0)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Empty(t, drainQueries(nc.intf.lightHouse))
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6}, conn.wrote)
})
}
// drainQueries returns every lighthouse query waiting to go out
func drainQueries(lh *LightHouse) []netip.Addr {
var queries []netip.Addr
for {
select {
case addr := <-lh.queryChan:
queries = append(queries, addr)
default:
return queries
}
}
}
+1 -1
View File
@@ -148,7 +148,7 @@ func (c *fakeConn) Rebind() error { c.rebinds++; ret
func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil }
func (c *fakeConn) ListenOut(_ udp.EncReader, _ func()) error { return nil }
func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
return len(bufs), nil
}
func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {}
-12
View File
@@ -117,18 +117,6 @@ func (c *Control) SetUDPAddr(addr netip.AddrPort) {
c.f.outside.(*udp.TesterConn).SetAddr(addr)
}
// GetUDPAddrFor returns the underlay address this node sends to dst from, which differs by family for a dual stack node
func (c *Control) GetUDPAddrFor(dst netip.AddrPort) netip.AddrPort {
return c.f.outside.(*udp.TesterConn).GetAddrFor(dst)
}
// SetDualStackUDPAddrs gives this node one underlay address per family, sending from whichever matches the
// destination. An invalid AddrPort leaves the node without that family, sends to it fail with ENETUNREACH. Register
// any address the node did not start on with the router as well or nothing will route back.
func (c *Control) SetDualStackUDPAddrs(v4, v6 netip.AddrPort) {
c.f.outside.(*udp.TesterConn).SetDualStackAddrs(v4, v6)
}
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
-305
View File
@@ -1,305 +0,0 @@
//go:build e2e_testing
package e2e
import (
"net/netip"
"sync/atomic"
"testing"
"time"
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/e2e/router"
"github.com/slackhq/nebula/udp"
)
// TestDualStackRoamToV4Only measures how long a laptop's tunnels take to come back after it moves from a dual stack
// network, where its tunnels formed over IPv6, to one with no IPv6 service at all: the office to hotel wifi. Every
// peer still has an IPv4 address the laptop can reach, so nothing about the move should strand a tunnel.
//
// It is a measurement, not a pass/fail assertion, and it runs on the production connection manager timers.
//
// go test -tags e2e_testing -v -run TestDualStackRoamToV4Only ./e2e/
func TestDualStackRoamToV4Only(t *testing.T) {
for _, tc := range []dualStackRoamCase{
{name: "both send", meSends: true, themSends: true},
{name: "both send, rebind", meSends: true, themSends: true, rebind: true},
{name: "both send, two tunnels", meSends: true, themSends: true, simultaneous: true},
{name: "laptop sends", meSends: true},
{name: "peer sends", themSends: true},
{name: "peer sends, rebind", themSends: true, rebind: true},
{name: "peer sends, lh over v4", themSends: true, lhOverV4: true},
{name: "peer sends, punchy", themSends: true, punchy: true},
{name: "peer sends, punchy all", themSends: true, punchy: true, punchAll: true},
{name: "laptop sends, peer NAT", meSends: true, peerNAT: true},
{name: "laptop sends, peer NAT, lh v4", meSends: true, peerNAT: true, lhOverV4: true},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fwd, rev := measureDualStackRoam(t, tc)
t.Logf("RESULT %-24s laptop->peer %-9v peer->laptop %-9v", tc.name, fmtRecovery(fwd), fmtRecovery(rev))
})
}
}
// TestDualStackRoamRecovers holds the line TestDualStackRoamToV4Only measures: a tunnel whose remote became unreachable
// moves to the peer's other address instead of waiting out the connection manager to rehandshake, which takes 20s.
func TestDualStackRoamRecovers(t *testing.T) {
for _, tc := range []struct {
dualStackRoamCase
// within bounds recovery in each direction that has traffic
within time.Duration
}{
// Measures 0.4s. Under the 5s connection manager check, so it fails if the data path stops probing.
{dualStackRoamCase{name: "both send", meSends: true, themSends: true}, 2 * time.Second},
// Measures 2.6s or 3.8s, depending on whether a probe beats the peer's punch. Half the 20s it used to take.
{dualStackRoamCase{name: "laptop sends, peer NAT", meSends: true, peerNAT: true}, 10 * time.Second},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fwd, rev := measureDualStackRoam(t, tc.dualStackRoamCase)
if fwd == 0 || fwd > tc.within {
t.Errorf("laptop->peer recovered in %v, want within %v", fmtRecovery(fwd), tc.within)
}
if tc.themSends && (rev == 0 || rev > tc.within) {
t.Errorf("peer->laptop recovered in %v, want within %v", fmtRecovery(rev), tc.within)
}
})
}
}
type dualStackRoamCase struct {
name string
// meSends and themSends say who keeps traffic flowing after the move
meSends, themSends bool
// rebind fires RebindUDPServer on the move, as the darwin network change monitor would
rebind bool
// lhOverV4 forms the lighthouse tunnel over IPv4 at the office, so the move does not break it
lhOverV4 bool
// simultaneous has both sides initiate at once at the office, which can leave each side a different primary
// tunnel for the other
simultaneous bool
// punchy turns on punchy.punch everywhere, punchAll adds punchy.target_all_remotes
punchy, punchAll bool
// peerNAT puts the peer behind a NAT that, after the move, only lets in traffic from addresses the peer has sent to
peerNAT bool
}
func fmtRecovery(d time.Duration) string {
if d == 0 {
return "never"
}
return d.Round(time.Millisecond).String()
}
type dualStackNode struct {
c *nebula.Control
vpnAddr netip.Addr
v4, v6 netip.AddrPort
adv atomic.Pointer[[]netip.Addr]
}
// move puts the node on a new network, advertising only what it can be reached on there
func (n *dualStackNode) move(v4, v6 netip.AddrPort) {
n.v4, n.v6 = v4, v6
n.c.SetDualStackUDPAddrs(v4, v6)
var adv []netip.Addr
for _, a := range []netip.AddrPort{v4, v6} {
if a.IsValid() {
adv = append(adv, a.Addr())
}
}
n.adv.Store(&adv)
}
func newDualStackNode(ca cert.Certificate, caKey []byte, name, vpnNet string, v4, v6 netip.AddrPort, overrides m) *dualStackNode {
c, vpnNets, _, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, name, vpnNet, v4, overrides)
n := &dualStackNode{c: c, vpnAddr: vpnNets[0].Addr()}
n.move(v4, v6)
c.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
return *n.adv.Load()
})
return n
}
// measureDualStackRoam returns how long after the move the first packet made it laptop->peer and peer->laptop, zero
// for a direction that never recovered.
func measureDualStackRoam(t *testing.T, tc dualStackRoamCase) (time.Duration, time.Duration) {
t.Helper()
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
ap := netip.MustParseAddrPort
lhV4, lhV6 := ap("10.0.0.1:4242"), ap("[fd00::1]:4242")
lh := newDualStackNode(ca, caKey, "lh", "10.128.0.1/24", lhV4, lhV6, m{
"lighthouse": m{"am_lighthouse": true},
})
// Production timers rather than the 2s the harness defaults to, this is about how long a real laptop waits
peerCfg := m{
"lighthouse": m{
"hosts": []any{lh.vpnAddr.String()},
"interval": 10,
},
"static_host_map": m{
lh.vpnAddr.String(): []any{lhV4.String(), lhV6.String()},
},
"timers": m{
"connection_alive_interval": 5,
"pending_deletion_interval": 10,
},
"punchy": m{
"punch": tc.punchy,
"target_all_remotes": tc.punchAll,
},
}
me := newDualStackNode(ca, caKey, "me", "10.128.0.2/24", ap("10.0.0.2:4242"), ap("[fd00::2]:4242"), peerCfg)
them := newDualStackNode(ca, caKey, "them", "10.128.0.3/24", ap("10.0.0.3:4242"), ap("[fd00::3]:4242"), peerCfg)
r := router.NewR(t, lh.c, me.c, them.c)
r.CancelFlowLogs()
for _, n := range []*dualStackNode{lh, me, them} {
r.AddRoute(n.v6.Addr(), n.v6.Port(), n.c)
}
defer func() {
lh.c.Stop()
me.c.Stop()
them.c.Stop()
}()
lh.c.Start()
me.c.Start()
them.c.Start()
// At the office IPv6 wins every race, so everything forms over it. The lighthouse tunnel optionally does not,
// for a static_host_map that only lists the lighthouse's IPv4 address.
officeV6 := func(p *udp.Packet, _ *nebula.Control) router.ExitType {
lhTraffic := p.To == lhV4 || p.To == lhV6 || p.From == lhV4 || p.From == lhV6
if p.To.Addr().Is4() == (tc.lhOverV4 && lhTraffic) {
return router.KeepRouting
}
return router.Drop
}
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
// One side at a time, so both sides hold the same single tunnel, unless the case wants them racing
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("establish")))
if !tc.simultaneous {
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
}
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("establish")))
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
if tc.simultaneous && me.c.GetHostmapIndexCount() < 3 {
// A tunnel to the lighthouse and two to the peer, or the race did not happen
t.Logf("simultaneous handshakes settled on a single tunnel this run")
}
drainTun(me.c)
drainTun(them.c)
for _, want := range []struct {
name string
c *nebula.Control
vpnAddr netip.Addr
v6 bool
}{
{"laptop->peer", me.c, them.vpnAddr, true},
{"peer->laptop", them.c, me.vpnAddr, true},
{"laptop->lh", me.c, lh.vpnAddr, !tc.lhOverV4},
} {
hi := want.c.GetHostInfoByVpnAddr(want.vpnAddr, false)
if hi == nil {
t.Fatalf("%s: no tunnel at the office", want.name)
}
if hi.CurrentRemote.Addr().Is6() != want.v6 {
t.Fatalf("%s: tunnel formed over %v, the scenario wants v6=%v", want.name, hi.CurrentRemote, want.v6)
}
}
// Hotel wifi: a new NAT'd IPv4 address and no IPv6 at all. The office addresses are gone, nothing reaches them.
officeV4, officeV6Addr := me.v4, me.v6
hotel := ap("10.0.0.99:4242")
me.move(hotel, netip.AddrPort{})
r.AddRoute(hotel.Addr(), hotel.Port(), me.c)
start := time.Now()
if tc.rebind {
me.c.RebindUDPServer()
}
var fwd, rev atomic.Int64
stop := make(chan struct{})
defer close(stop)
watchTun(them.c, stop, start, &fwd)
watchTun(me.c, stop, start, &rev)
go func() {
tick := time.NewTicker(time.Millisecond * 200)
defer tick.Stop()
for {
select {
case <-stop:
return
case <-tick.C:
if tc.meSends {
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("retry")))
}
if tc.themSends {
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("retry")))
}
}
}
}()
// When only one side has traffic, the other direction recovering is a bonus we still report but do not wait for
done := func() bool {
return (!tc.meSends || fwd.Load() != 0) && (!tc.themSends || rev.Load() != 0)
}
// The peer's NAT has state for the lighthouse, which its own updates keep fresh, and for wherever it sends from here
natOpen := map[netip.AddrPort]bool{lhV4: true, lhV6: true}
isPeer := func(a netip.AddrPort) bool { return a == them.v4 || a == them.v6 }
r.RouteForAllExitFuncOrTimeout(time.Second*90, func(p *udp.Packet, _ *nebula.Control) router.ExitType {
if p.To == officeV4 || p.To == officeV6Addr {
return router.Drop
}
if tc.peerNAT {
if isPeer(p.From) {
natOpen[p.To] = true
} else if isPeer(p.To) && !natOpen[p.From] {
return router.Drop
}
}
if done() {
return router.ExitNow
}
return router.KeepRouting
})
return time.Duration(fwd.Load()), time.Duration(rev.Load())
}
func drainTun(c *nebula.Control) {
for {
select {
case <-c.GetTunTxChan():
default:
return
}
}
}
// watchTun records how long after start the first packet came out of c's tun, and keeps the tun drained after
func watchTun(c *nebula.Control, stop chan struct{}, start time.Time, first *atomic.Int64) {
go func() {
for {
select {
case <-stop:
return
case <-c.GetTunTxChan():
first.CompareAndSwap(0, int64(time.Since(start)))
}
}
}()
}
+13 -14
View File
@@ -497,10 +497,11 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []
// Nope, lets push the sender along
case p := <-udpTx:
r.Lock()
c := r.getControl(p)
a := sender.GetUDPAddr()
c := r.getControl(a, p.To, p)
if c == nil {
r.Unlock()
panic("No control for udp tx " + p.To.String())
panic("No control for udp tx " + a.String())
}
fp := r.unlockedInjectFlow(sender, c, p, false)
c.InjectUDPPacket(p) // copies internally; original is ours to release
@@ -621,7 +622,8 @@ func (r *R) stopFanInWorkers() {
func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
r.Lock()
defer r.Unlock()
c := r.getControl(p)
a := from.GetUDPAddr()
c := r.getControl(a, p.To, p)
if c == nil {
panic(fmt.Sprintf("No control for udp tx %s", p.To))
}
@@ -669,7 +671,7 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
panic(err)
}
receiver := r.getControl(p)
receiver := r.getControl(sender.GetUDPAddr(), p.To, p)
if receiver == nil {
r.Unlock()
panic("Can't RouteExitFunc for host: " + p.To.String())
@@ -765,7 +767,7 @@ func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc)
r.Lock()
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(p)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
if receiver == nil {
r.Unlock()
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
@@ -874,7 +876,7 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
r.Lock()
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(p)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
if receiver == nil {
r.Unlock()
panic("Can't RouteForAllExitFunc for host: " + p.To.String())
@@ -937,7 +939,7 @@ func (r *R) FlushAll() {
})
for {
_, rx, ok := reflect.Select(sc)
x, rx, ok := reflect.Select(sc)
if !ok {
return
}
@@ -945,7 +947,7 @@ func (r *R) FlushAll() {
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(p)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
if receiver == nil {
r.Unlock()
panic("Can't FlushAll for host: " + p.To.String())
@@ -956,19 +958,16 @@ func (r *R) FlushAll() {
}
}
// getControl performs or seeds NAT translation and returns the control for p.To, p from fields may change
// getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change
// This is an internal router function, the caller must hold the lock
func (r *R) getControl(p *udp.Packet) *nebula.Control {
// Key on the address the packet actually left from, a dual stack sender has one per family
fromAddr, toAddr := p.From, p.To
func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control {
if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
p.From = newAddr
}
c, ok := r.inNat[toAddr]
if ok {
// Key the return path on the address the receiver will answer from, which is per family for a dual stack node
r.outNat[outNatKey{from: c.GetUDPAddrFor(fromAddr), to: fromAddr}] = toAddr
r.outNat[outNatKey{from: c.GetUDPAddr(), to: fromAddr}] = toAddr
return c
}
-3
View File
@@ -331,9 +331,6 @@ tun:
# `mtu`: will default to tun mtu if this option is not specified
# `metric`: will default to 0 if this option is not specified
# `install`: will default to true, controls whether this route is installed in the systems routing table.
# On macOS, a route whose destination another interface already routes, like the local network, is installed once
# that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another
# interface routes its destination and lets it go.
# This setting is reloadable.
unsafe_routes:
#- route: 172.16.1.0/24
-4
View File
@@ -66,9 +66,6 @@ type Firewall struct {
rules string
rulesVersion uint16
// source is the rendered firewall config this firewall was built from, the cert networks are kept above
source string
defaultLocalCIDRAny bool
incomingMetrics firewallMetrics
outgoingMetrics firewallMetrics
@@ -249,7 +246,6 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
return nil, err
}
fw.source = c.Render("firewall")
return fw, nil
}
+1 -1
View File
@@ -29,7 +29,7 @@ require (
golang.org/x/term v0.46.0
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
golang.zx2c4.com/wireguard/windows v1.0.1
golang.zx2c4.com/wireguard/windows v1.1.1
google.golang.org/protobuf v1.36.12
gopkg.in/yaml.v3 v3.0.1
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980
+2 -2
View File
@@ -218,8 +218,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
golang.zx2c4.com/wireguard/windows v1.0.1 h1:eOxiDVbywPC+ZQqvdCK7x+ZwWXKbYv50TtH8ysFIbw8=
golang.zx2c4.com/wireguard/windows v1.0.1/go.mod h1:+fbT3FFdX4zzYDLwJh5+HPEcNN/3HyNdzhNSVsQM+zs=
golang.zx2c4.com/wireguard/windows v1.1.1 h1:8/H97U1v1PNDNcBsMZgU3KFuND9MQdTsU2NOwmCXArE=
golang.zx2c4.com/wireguard/windows v1.1.1/go.mod h1:+fbT3FFdX4zzYDLwJh5+HPEcNN/3HyNdzhNSVsQM+zs=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+23 -33
View File
@@ -61,17 +61,16 @@ type HandshakeManager struct {
vpnIps map[netip.Addr]*HandshakeHostInfo
indexes map[uint32]*HandshakeHostInfo
mainHostMap *HostMap
lightHouse *LightHouse
outside udp.Conn
config HandshakeConfig
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
messageMetrics *MessageMetrics
metricInitiated metrics.Counter
metricTimedOut metrics.Counter
metricRejectedBlocklisted metrics.Counter
f *Interface
l *slog.Logger
mainHostMap *HostMap
lightHouse *LightHouse
outside udp.Conn
config HandshakeConfig
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
messageMetrics *MessageMetrics
metricInitiated metrics.Counter
metricTimedOut metrics.Counter
f *Interface
l *slog.Logger
// can be used to trigger outbound handshake for the given vpnIp
trigger chan netip.Addr
@@ -119,19 +118,18 @@ func (hh *HandshakeHostInfo) cachePacket(l *slog.Logger, t header.MessageType, s
func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager {
return &HandshakeManager{
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
indexes: map[uint32]*HandshakeHostInfo{},
mainHostMap: mainHostMap,
lightHouse: lightHouse,
outside: outside,
config: config,
trigger: make(chan netip.Addr, config.triggerBuffer),
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
metricRejectedBlocklisted: metrics.GetOrRegisterCounter("handshake_manager.rejected.blocklisted", nil),
l: l,
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
indexes: map[uint32]*HandshakeHostInfo{},
mainHostMap: mainHostMap,
lightHouse: lightHouse,
outside: outside,
config: config,
trigger: make(chan netip.Addr, config.triggerBuffer),
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
l: l,
}
}
@@ -719,12 +717,7 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
response, result, err := machine.ProcessPacket(nil, packet)
if err != nil {
level := slog.LevelError
if errors.Is(err, cert.ErrBlockListed) {
level = slog.LevelDebug
hm.metricRejectedBlocklisted.Inc(1)
}
f.l.Log(context.Background(), level, "Failed to process handshake packet", "from", via, "error", err)
f.l.Error("Failed to process handshake packet", "from", via, "error", err)
return
}
@@ -856,9 +849,6 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
response, result, err := machine.ProcessPacket(nil, packet)
if err != nil {
if errors.Is(err, cert.ErrBlockListed) {
hm.metricRejectedBlocklisted.Inc(1)
}
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
if machine.Failed() {
f.l.Warn("Failed to process handshake packet, abandoning",
-88
View File
@@ -1,17 +1,12 @@
package nebula
import (
"log/slog"
"net/netip"
"testing"
"time"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
@@ -240,89 +235,6 @@ func TestHandleIncomingDispatch(t *testing.T) {
})
}
func TestBlocklistedHandshakeLogLevelAndCounter(t *testing.T) {
ca, _, caKey, _ := ct.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
pool := ct.NewTestCAPool(ca)
newCredential := func(name, network string) (cert.Certificate, *handshake.Credential) {
c, _, rawKey, _ := ct.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey,
name, ca.NotBefore(), ca.NotAfter(), []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
privateKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey)
require.NoError(t, err)
certBytes, err := c.MarshalForHandshakes()
require.NoError(t, err)
cipherSuite := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
return c, handshake.NewCredential(c, certBytes, privateKey, cipherSuite)
}
initiatorCert, initiatorCredential := newCredential("initiator", "10.0.0.1/24")
responderCert, responderCredential := newCredential("responder", "10.0.0.2/24")
fingerprint, err := initiatorCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(fingerprint)
initiator, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return initiatorCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return pool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 1, nil }, true, header.HandshakeIXPSK0)
require.NoError(t, err)
packet, err := initiator.Initiate(nil)
require.NoError(t, err)
handler := &capturingHandler{}
logger := slog.New(handler)
pki := &PKI{}
pki.cs.Store(&CertState{initiatingVersion: cert.Version2, v2Credential: responderCredential})
pki.caPool.Store(pool)
hm := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
hm.metricRejectedBlocklisted = metrics.NewCounter()
hm.f = &Interface{pki: pki, l: logger}
via := ViaSender{UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242")}
hm.beginHandshake(via, packet, nil)
entry := findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelDebug, entry.Level)
require.ErrorIs(t, entry.Attrs["error"].(error), cert.ErrBlockListed)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
handler.entries = nil
hm.beginHandshake(via, nil, nil)
entry = findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelError, entry.Level)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
// The initiator sees the same rejection in continueHandshake, after a
// responder has accepted its first packet and sent a valid reply.
cleanPool := ct.NewTestCAPool(ca)
responder, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return responderCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return cleanPool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 2, nil }, false, header.HandshakeIXPSK0)
require.NoError(t, err)
response, _, err := responder.ProcessPacket(nil, packet)
require.NoError(t, err)
responderFingerprint, err := responderCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(responderFingerprint)
initiatingHM := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
initiatingHM.metricRejectedBlocklisted = metrics.NewCounter()
initiatingHM.f = &Interface{l: logger}
hh := &HandshakeHostInfo{
machine: initiator,
hostinfo: &HostInfo{localIndexId: 1, vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}},
}
initiatingHM.indexes[1] = hh
via.IsRelayed = true // skip the remote allow list before certificate validation
initiatingHM.continueHandshake(via, hh, response)
assert.Equal(t, int64(1), initiatingHM.metricRejectedBlocklisted.Count())
}
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
-62
View File
@@ -264,9 +264,6 @@ type HostInfo struct {
// This is used to limit lighthouse re-queries in chatty clients
nextLHQuery atomic.Int64
// nextUnreachableProbe is the earliest we will probe our other remotes again after failing to reach the current one
nextUnreachableProbe atomic.Int64
// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
// This is used to avoid an attack where a handshake packet is replayed after some time
@@ -769,65 +766,6 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
}
}
// unreachableProbeInterval bounds how often a tunnel whose remote we cannot reach probes its other remotes
const unreachableProbeInterval = time.Second
// remoteUnreachable is called when a send to our current remote failed because we have no route to it, typically
// after moving to a network without that address family. Sends a test packet to every other remote we know for this
// host: the first one that arrives roams the peer over to wherever we are now, and its reply roams us onto that path.
//
// A peer behind a NAT drops a test from an address it has never sent to, so every probe also asks the lighthouse
// about the peer. The lighthouse tells the peer where we are now, the peer punches us, and the next probe gets in.
func (i *HostInfo) remoteUnreachable(ifce *Interface, dead netip.AddrPort) {
if i.remotes == nil || i.ConnectionState == nil || i.GetRemote() != dead {
// Nothing to probe with, or we already moved on from the remote that failed
return
}
if ifce.lightHouse == nil || ifce.lightHouse.amLighthouse {
// A lighthouse hands out the address it reaches a client on, it must not go roaming its clients itself
return
}
now := time.Now().UnixNano()
next := i.nextUnreachableProbe.Load()
if now < next || !i.nextUnreachableProbe.CompareAndSwap(next, now+int64(unreachableProbeInterval)) {
return
}
nb := make([]byte, 12)
out := make([]byte, mtu)
probed := 0
i.remotes.ForEach(ifce.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) {
if addr == dead {
return
}
probed++
ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), nb, out)
})
if ifce.l.Enabled(context.Background(), slog.LevelDebug) {
i.logger(ifce.l).Debug("Remote unreachable, probing other remotes",
"udpAddr", dead,
"probed", probed,
)
}
if probed == 0 {
// With nothing else to try, the lighthouse is our only way to learn of another remote. The peer may not have
// one, so ask on the usual requery schedule rather than every probe for as long as the tunnel sits here.
if now < i.nextLHQuery.Load() {
return
}
i.nextLHQuery.Store(now + ifce.reQueryWait.Load())
}
// With other remotes to probe we ask on every probe, without backing off. A repeat re-sends the peer its punch
// notification in case the last one or its punches were lost. If the tunnel stays up on inbound traffic while
// our sends to its remote keep failing (only one direction broken), this continues once a second for as long as
// that lasts. That is a bounded cost for a rare state, and the probe that would clear it has to keep running.
ifce.lightHouse.QueryServer(i.vpnAddrs[0])
}
// Bits within HostInfo.state, everything above stateEpochShift is the epoch
const (
stateIn uint32 = 1 << iota
-5
View File
@@ -15,7 +15,6 @@ import (
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/udp"
)
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
@@ -612,10 +611,6 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
"udpAddr", hr,
)
}
// A closing tunnel is about to be torn down, there is no point looking for another path for it
if t != header.CloseTunnel && udp.IsUnreachable(err) {
hostinfo.remoteUnreachable(f, hr)
}
}
} else {
// Try to send via a relay
+6 -47
View File
@@ -1,7 +1,6 @@
package nebula
import (
"bytes"
"context"
"crypto/fips140"
"errors"
@@ -496,8 +495,7 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
// flushSendBatch drains sb to the underlay and accounts for anything it could not deliver. A shortfall means
// specific destinations were undeliverable (a stale remote, a reject rule), which the backend logs per peer at
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch. Tunnels whose remote
// we no longer have a route to go looking for another path.
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch.
func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
queued := sb.Len()
written, err := sb.Flush()
@@ -507,44 +505,6 @@ func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
if dropped := queued - written; dropped > 0 {
f.metricTxDropped.Inc(int64(dropped))
}
if f.lightHouse == nil || f.lightHouse.amLighthouse {
// Lighthouses never probe, see remoteUnreachable, so skip finding the tunnels
return
}
// A tunnel's packets tend to fail together, so skip straight past a repeat of the one before. The first 8 bytes
// of a header say whose packet it is, the counter after them changes with every packet.
var last batch.Failure
for i, fail := range sb.Failures() {
if i > 0 && fail.Dst == last.Dst && bytes.Equal(fail.Head[:8], last.Head[:8]) {
continue
}
last = fail
if udp.IsUnreachable(fail.Err) {
f.sendUnreachable(fail.Head[:], fail.Dst)
}
}
}
// sendUnreachable finds the tunnel a packet we had no route for was sent on, from the start of the packet, and has it
// look for another path to its peer.
func (f *Interface) sendUnreachable(head []byte, dst netip.AddrPort) {
var h header.H
if h.Parse(head) != nil {
return
}
if h.Type == header.Message && h.Subtype == header.MessageRelay {
// TODO: a relayed packet's header carries the relay session's index, not the relay tunnel's, so it does not
// lead us to the tunnel whose remote failed. The relay tunnel's own traffic finds out soon enough.
return
}
if hostinfo := f.hostMap.QueryReverseIndex(h.RemoteIndex); hostinfo != nil {
hostinfo.remoteUnreachable(f, dst)
}
}
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
@@ -577,12 +537,11 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
curCert = cs.getCertificate(cert.Version1)
}
// Compare with what the running firewall was built from, not the last config, so a firewall that failed to build
// is tried again. The cert is the one this reload brought, PKI registers its reload callback before this one
src := c.Render("firewall")
if src != "" && src == f.firewall.source && curCert != nil &&
slices.Equal(curCert.Networks(), f.firewall.assignedNetworks) &&
slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) {
// The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction.
// Check to see if that set has changed, and if so, rebuild the firewall.
certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks)
if !c.HasChanged("firewall") && !certUnsafeChanged {
f.l.Debug("No firewall config change detected")
return
}
+3 -58
View File
@@ -135,63 +135,8 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
// The running firewall still isn't built from what the config asks for, so the same reload fails again
for range 2 {
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
res := cfg.ReloadConfigString(rules("tcp"))
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.NotSame(t, fw, f.firewall)
}
// Every input the firewall is built from rebuilds it when it changes, and nothing else does
func TestReloadFirewall_RebuildsOnAnyInput(t *testing.T) {
l := test.NewLogger()
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
base := "firewall:\n inbound:\n - port: any\n proto: any\n host: any\n"
setup := func() (*config.C, *Interface, *PKI) {
pki := &PKI{}
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks}, initiatingVersion: cert.Version2})
cfg := config.NewC(l)
require.NoError(t, cfg.LoadString(base))
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
require.NoError(t, err)
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
return cfg, f, pki
}
setCert := func(pki *PKI, networks, unsafe []netip.Prefix) {
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks, unsafeNetworks: unsafe}, initiatingVersion: cert.Version2})
}
cases := map[string]struct {
cfg string
cert func(*PKI)
want bool
}{
"nothing": {base + "punchy:\n punch: true\n", nil, false},
"rules": {base + " outbound:\n - port: any\n proto: any\n host: any\n", nil, true},
"inbound_action": {base + " inbound_action: reject\n", nil, true},
"conntrack": {base + " conntrack:\n tcp_timeout: 1m\n", nil, true},
"networks": {base, func(p *PKI) { setCert(p, []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil) }, true},
"unsafe_networks": {base, func(p *PKI) { setCert(p, networks, []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}) }, true},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
cfg, f, pki := setup()
old := f.firewall
if tc.cert != nil {
tc.cert(pki)
}
res := cfg.ReloadConfigString(tc.cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, tc.want, f.firewall != old)
})
}
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
+5 -41
View File
@@ -1,27 +1,12 @@
package batch
import (
"net/netip"
"github.com/slackhq/nebula/header"
)
import "net/netip"
const SendBatchCap = 128
// FailureHeadLen is how much of a refused packet a Failure keeps, nebula's header, which says whose packet it was
const FailureHeadLen = header.Len
// batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush.
type batchWriter interface {
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
}
// Failure is a packet the underlay would not send
type Failure struct {
// Head is the start of the packet, zero padded if the packet was shorter
Head [FailureHeadLen]byte
Dst netip.AddrPort
Err error
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
}
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
@@ -32,28 +17,16 @@ type SendBatch struct {
bufs [][]byte
dsts []netip.AddrPort
arena *Arena
failures []Failure
// recordFailure is bound once so handing it to WriteBatch does not allocate on every Flush
recordFailure func(i int, err error)
}
// NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots
func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch {
b := &SendBatch{
return &SendBatch{
out: out,
bufs: make([][]byte, 0, batchCap),
dsts: make([]netip.AddrPort, 0, batchCap),
arena: NewArena(arenaSize),
}
b.recordFailure = b.failed
return b
}
func (b *SendBatch) failed(i int, err error) {
f := Failure{Dst: b.dsts[i], Err: err}
copy(f.Head[:], b.bufs[i])
b.failures = append(b.failures, f)
}
func (b *SendBatch) Reserve(sz int) []byte {
@@ -71,15 +44,12 @@ func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort) {
}
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations
// were undeliverable, Failures says which; the batch is drained either way.
// were undeliverable; the batch is drained either way.
func (b *SendBatch) Flush() (int, error) {
clear(b.failures)
b.failures = b.failures[:0]
var err error
written := 0
if len(b.bufs) > 0 {
written, err = b.out.WriteBatch(b.bufs, b.dsts, b.recordFailure)
written, err = b.out.WriteBatch(b.bufs, b.dsts)
}
clear(b.bufs)
b.bufs = b.bufs[:0]
@@ -87,9 +57,3 @@ func (b *SendBatch) Flush() (int, error) {
b.arena.Reset()
return written, err
}
// Failures returns every packet the last Flush could not send, in the order they were committed. Valid until the
// next Flush.
func (b *SendBatch) Failures() []Failure {
return b.failures
}
+1 -84
View File
@@ -1,9 +1,7 @@
package batch
import (
"errors"
"net/netip"
"slices"
"testing"
)
@@ -12,7 +10,7 @@ type fakeBatchWriter struct {
addrs []netip.AddrPort
}
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, _ func(int, error)) (int, error) {
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch
// returns, so tests must capture data before that happens.
w.bufs = make([][]byte, len(bufs))
@@ -122,84 +120,3 @@ func TestSendBatchGrowPreservesCommitted(t *testing.T) {
t.Errorf("second packet on the wire: %x", fw.bufs[1])
}
}
// failingBatchWriter refuses every packet whose first byte is in refuse
type failingBatchWriter struct {
refuse map[byte]error
}
func (w *failingBatchWriter) WriteBatch(bufs [][]byte, _ []netip.AddrPort, failed func(int, error)) (int, error) {
written := 0
for i, b := range bufs {
if err, ok := w.refuse[b[0]]; ok {
failed(i, err)
continue
}
written++
}
return written, nil
}
func TestSendBatchReportsFailures(t *testing.T) {
errNoRoute := errors.New("no route")
fw := &failingBatchWriter{refuse: map[byte]error{2: errNoRoute, 3: errNoRoute}}
b := NewSendBatch(fw, 4, 128)
// pkt is n bytes starting with the refuse key, each later byte its offset plus 0x10 so the head is recognizable
pkt := func(key byte, n int) []byte {
p := b.Reserve(n)[:n]
p[0] = key
for i := 1; i < n; i++ {
p[i] = byte(0x10 + i)
}
return p
}
head := func(p []byte) (h [FailureHeadLen]byte) {
copy(h[:], p)
return h
}
dsts := []netip.AddrPort{
netip.MustParseAddrPort("10.0.0.1:4242"),
netip.MustParseAddrPort("[fd00::2]:4242"),
netip.MustParseAddrPort("[fd00::3]:4242"),
netip.MustParseAddrPort("10.0.0.4:4242"),
}
long := pkt(2, FailureHeadLen+8) // only its head is kept
short := pkt(3, 4) // zero padded
b.Commit(pkt(1, 20), dsts[0])
b.Commit(long, dsts[1])
b.Commit(short, dsts[2])
b.Commit(pkt(1, 20), dsts[3])
written, err := b.Flush()
if err != nil {
t.Fatalf("Flush: %v", err)
}
if written != 2 {
t.Errorf("written = %d, want 2", written)
}
want := []Failure{{head(long), dsts[1], errNoRoute}, {head(short), dsts[2], errNoRoute}}
if got := b.Failures(); !slices.Equal(got, want) {
t.Errorf("failures = %v, want %v", got, want)
}
// Failures do not leak into the next flush
again := pkt(2, 20)
b.Commit(again, dsts[0])
if _, err := b.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if want := []Failure{{head(again), dsts[0], errNoRoute}}; !slices.Equal(b.Failures(), want) {
t.Errorf("second flush failures = %v, want %v", b.Failures(), want)
}
// A clean flush forgets the last one's failures
b.Commit(pkt(1, 20), dsts[0])
if _, err := b.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if got := b.Failures(); len(got) != 0 {
t.Errorf("clean flush failures = %v, want none", got)
}
}
+84 -54
View File
@@ -3,6 +3,7 @@
package overlay
import (
"errors"
"fmt"
"log/slog"
"math/bits"
@@ -11,7 +12,6 @@ import (
"sync"
"sync/atomic"
"syscall"
"time"
"unsafe"
"github.com/gaissmai/bart"
@@ -35,26 +35,6 @@ type tun struct {
// and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool
l *slog.Logger
// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
// fields below it
routesMu sync.Mutex
// routeStates is who held each installable unsafe_route prefix when we last looked
routeStates map[netip.Prefix]routeState
// routeWatch is the routing socket watchRoutes reads, Close closes it
routeWatch *os.File
// routeSweep reconciles the unsafe_routes every routeSweepEvery
routeSweep *time.Timer
routeSweepEvery time.Duration
closed bool
// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
// lock so the route watcher never waits on a reconcile
routeMoversMu sync.Mutex
routeMovers map[netip.Prefix]routeMover
// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
lastRouteEvent atomic.Int64
}
type ifReq struct {
@@ -159,12 +139,11 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
}
t := &tun{
f: os.NewFile(uintptr(fd), ""),
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c),
routeSweepEvery: routeSweepInterval,
l: l,
f: os.NewFile(uintptr(fd), ""),
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c),
l: l,
}
err = t.reload(c, true, nil)
@@ -240,16 +219,6 @@ func utunNameFromFd(fd int) string {
}
func (t *tun) Close() error {
t.routesMu.Lock()
t.closed = true
if t.routeWatch != nil {
_ = t.routeWatch.Close()
}
if t.routeSweep != nil {
t.routeSweep.Stop()
}
t.routesMu.Unlock()
if t.f != nil {
return t.f.Close()
}
@@ -318,22 +287,8 @@ func (t *tun) Activate() error {
return fmt.Errorf("failed to run tun device: %s", err)
}
// Watch before adding so a route removed in between is still noticed
watch, err := openRouteSocket()
if err != nil {
t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
// Unsafe path routes
if err = t.addRoutes(false, nil); err != nil {
if watch != nil {
_ = watch.Close()
}
return err
}
t.startRouteWatch(watch)
return nil
return t.addRoutes(false, nil)
}
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
@@ -438,8 +393,7 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
}
if !initial && !change {
// Any reload puts back the unsafe_routes someone deleted
return t.addRoutes(true, r)
return nil
}
routeTree, err := makeRouteTree(t.l, routes, false)
@@ -557,6 +511,82 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
return nil, nil
}
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
err := addRoute(r.Cidr, linkAddr)
if err != nil {
if errors.Is(err, unix.EEXIST) {
// Every route we added already exists on a reload, only warn about someone else's
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
retErr.Log(t.l)
rl.Failed()
} else {
return retErr
}
}
} else {
t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
}
}
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
linkAddr := t.linkAddr.Load()
if t.hostOwned || linkAddr == nil {
return nil
}
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue
}
err := delRoute(r.Cidr, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
-366
View File
@@ -3,17 +3,12 @@
package overlay
import (
"encoding/binary"
"encoding/hex"
"fmt"
"net"
"net/netip"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
@@ -109,367 +104,6 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
const (
// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
// With no interface address, which x/net/route skips
interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
)
func TestRoutesMayHaveMoved(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
routes := []Route{{Cidr: cidr, Via: via, Install: true}}
// A request like route(8) sends, the kernel echoes it to every routing socket
request := func(typ, flags int, dst, mask [4]byte) []byte {
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: typ,
Flags: flags,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: dst},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
},
}
b, err := m.Marshal()
require.NoError(t, err)
return b
}
raw := func(s string) []byte {
b, err := hex.DecodeString(s)
require.NoError(t, err)
return b
}
lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
routesMayHaveMoved := func(b []byte, routes []Route) bool {
moved, _, _ := routesMayHaveMoved(b, routes)
return moved
}
assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
neighbor := raw(kernelSubnetDelete)
binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
}
// A request names its prefix and who made it, the kernel's own and interface changes name neither
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: unix.RTM_CHANGE,
Flags: unix.RTF_UP,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
},
}
b, err := m.Marshal()
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved)
assert.Equal(t, cidr, p)
assert.Equal(t, int32(1234), pid)
for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
b, err := hex.DecodeString(s)
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved, name)
assert.False(t, p.IsValid(), name)
assert.Zero(t, pid, name)
}
}
func TestProcessName(t *testing.T) {
assert.Equal(t, "launchd", processName(1))
self := filepath.Base(os.Args[0])
assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
assert.Empty(t, processName(-1))
}
// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
// the minute between sweeps when it isn't zero.
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
requireRoot(t)
cfg := "tun:\n unsafe_routes:\n"
for _, cidr := range cidrs {
via := "10.250.0.2"
if netip.MustParsePrefix(cidr).Addr().Is6() {
via = "fd00:250::2"
}
cfg += " - route: " + cidr + "\n via: " + via + "\n"
}
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString(cfg))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
require.NoError(t, err)
if sweep != 0 {
tn.routeSweepEvery = sweep
}
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
return tn, c, cfg
}
func requireRoot(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating interfaces and adding routes needs root")
}
}
// routeVia is the index of the interface that routes cidr, 0 if none does
func routeVia(t *testing.T, cidr string) int {
msgs, err := fetchRoutes()
require.NoError(t, err)
return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
}
// routeStateOf is what tn made of cidr when it last looked
func routeStateOf(tn *tun, cidr string) routeState {
tn.routesMu.Lock()
defer tn.routesMu.Unlock()
return tn.routeStates[netip.MustParsePrefix(cidr)]
}
func routeArgs(action, cidr string, extra ...string) []string {
args := []string{"-q", "-n", action}
if netip.MustParsePrefix(cidr).Addr().Is6() {
args = append(args, "-inet6")
}
return append(append(args, "-net", cidr), extra...)
}
func routeCmd(t *testing.T, action, cidr string, extra ...string) {
requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
}
// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
// says so
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
require.NotContains(t, string(out), "writing to routing socket")
}
// cleanupRoute deletes cidr's route when the test ends, whoever holds it
func cleanupRoute(t *testing.T, cidr string) {
t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
}
// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
func routeThroughLo0(t *testing.T, cidr string) int {
requireRoot(t)
routeCmd(t, "add", cidr, "-interface", "lo0")
cleanupRoute(t, cidr)
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
return lo0.Index
}
// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
t.Run(cidr, func(t *testing.T) {
lo0 := routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
})
}
}
// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
cidr := "10.251.1.0/24"
lo0 := routeThroughLo0(t, cidr)
_, c, _ := newRouteTestTun(t, 0, cidr)
res := c.ReloadConfigString("tun: {}\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, lo0, routeVia(t, cidr))
}
// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
cidr := "10.251.3.0/24"
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
time.Sleep(2 * routeRetryDelay)
require.Equal(t, lo0.Index, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
}
// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
cidr := "10.251.4.0/24"
tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
// A reload with nothing changed is how to ask for it back
res := c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, ours, routeVia(t, cidr))
}
// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
for name, tc := range map[string]struct {
cidr string
sweep time.Duration
misfile func(t *testing.T, tn *tun, cidr string)
}{
// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
}},
// The sweep that caps a table that never settles
"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
tn.reconcileRoutes()
require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
}},
} {
t.Run(name, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, tc.cidr))
routeCmd(t, "delete", tc.cidr)
tc.misfile(t, tn, tc.cidr)
lo0 := routeThroughLo0(t, tc.cidr)
require.Eventually(t, func() bool {
s := routeStateOf(tn, tc.cidr)
return s.kind == routeYielded && s.holder == lo0
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", tc.cidr)
assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
// taken for a hand delete
func TestTunYieldsToARouteChangedAway(t *testing.T) {
for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
t.Run(cidr, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
routeCmd(t, "change", cidr, "-interface", "lo0")
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
require.Equal(t, lo0.Index, routeVia(t, cidr))
require.Eventually(t, func() bool {
s := routeStateOf(tn, cidr)
return s.kind == routeYielded && s.holder == lo0.Index
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
// ethernet configd routed the lan through
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
requireRoot(t)
var feth string
for i := 100; i < 200 && feth == ""; i++ {
if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
feth = fmt.Sprintf("feth%d", i)
}
}
require.NotEmpty(t, feth)
require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
// refuses one named by the feth itself
cidr := "10.251.2.0/24"
routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
cleanupRoute(t, cidr)
ifc, err := net.InterfaceByName(feth)
require.NoError(t, err)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Losing the address first leaves the route, the reconcile finds it still held
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
time.Sleep(2 * routeRetryDelay)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Only RTM_IFINFO says the interface went, its route goes with no message of its own
require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
cidr := "10.251.5.0/24"
routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
tn.routesMu.Lock()
require.NoError(t, tn.routeWatch.Close())
tn.routesMu.Unlock()
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
func TestTunDevNeedsRestart(t *testing.T) {
for name, want := range map[string]bool{
-451
View File
@@ -1,451 +0,0 @@
//go:build !ios && !e2e_testing
package overlay
import (
"errors"
"fmt"
"net"
"net/netip"
"os"
"strconv"
"sync/atomic"
"time"
"unsafe"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/util"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
// or until another interface takes the prefix, after which ours follows theirs like any other.
const (
// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
// replaces a route by deleting and adding, looking in between would find the prefix free
routeRetryDelay = time.Second
// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
// a message for each
routeSweepInterval = time.Minute
// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
routeSocketRcvBuf = 1 << 20
)
type routeStateKind int
const (
routeUnknown routeStateKind = iota
// routeInstalled is our route holding the prefix
routeInstalled
// routeYielded is another interface holding the prefix, ours is added once it lets go
routeYielded
// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
// another interface takes the prefix
routeRemoved
// routeFailed is adding our route failing for some other reason, it is retried
routeFailed
)
type routeState struct {
kind routeStateKind
// holder is the index of the interface holding the prefix when kind is routeYielded
holder int
}
// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
// is looked up as the message arrives, route(8) has exited by the time the table settles
type routeMover struct {
pid int32
name string
}
// logArgs appends m to a log line's args, when we know who it was
func (m routeMover) logArgs(args ...any) []any {
if m.pid == 0 {
return args
}
args = append(args, "pid", m.pid)
if m.name != "" {
args = append(args, "process", m.name)
}
return args
}
// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
func processName(pid int32) string {
kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
if err != nil || kp.Proc.P_pid != pid {
return ""
}
return unix.ByteSliceToString(kp.Proc.P_comm[:])
}
// installRoutes returns the routes we put in the system routing table, one per prefix
func installRoutes(routes []Route) []Route {
seen := make(map[netip.Prefix]struct{}, len(routes))
var out []Route
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
p := r.Cidr.Masked()
if _, ok := seen[p]; ok {
continue
}
seen[p] = struct{}{}
out = append(out, r)
}
return out
}
// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
// interface holds its prefix. It runs on start and on every reload.
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
var msgs []netroute.Message
var msgsErr error
fetched := false
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
err := addRoute(p, linkAddr)
if err == nil {
t.l.Info("Added route", "route", r)
states[p] = routeState{kind: routeInstalled}
continue
}
if !errors.Is(err, unix.EEXIST) {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if !logErrors {
return retErr
}
retErr.Log(t.l)
rl.Failed()
states[p] = routeState{kind: routeFailed}
continue
}
// Every route we added already exists on a reload, only warn about someone else's
if !fetched {
msgs, msgsErr = fetchRoutes()
fetched = true
}
if msgsErr != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
states[p] = prev
continue
}
switch holder := routeInterfaceIn(msgs, p); holder {
case linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case 0:
// Removed since the add, the watcher will see to it
states[p] = prev
default:
if prev.kind != routeYielded || prev.holder != holder {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
}
states[p] = routeState{kind: routeYielded, holder: holder}
}
}
t.routeStates = states
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
msgs, err := fetchRoutes()
if err != nil {
var left []netip.Prefix
for _, r := range installRoutes(routes) {
left = append(left, r.Cidr)
}
return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
}
for _, r := range installRoutes(routes) {
p := r.Cidr.Masked()
// Whatever happens next, the prefix is no longer one we know about
delete(t.routeStates, p)
if routeInterfaceIn(msgs, p) != linkAddr.Index {
continue
}
err = delRoute(p, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
// changed since it last looked, it runs every routeSweepEvery.
func (t *tun) reconcileRoutes() {
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return
}
t.routeMoversMu.Lock()
movers := t.routeMovers
t.routeMovers = nil
t.routeMoversMu.Unlock()
msgs, err := fetchRoutes()
if err != nil {
t.l.Error("Failed to read the system routing table", "error", err)
return
}
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
by := movers[p]
holder := routeInterfaceIn(msgs, p)
switch {
case holder == linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case holder != 0:
// Another interface taking a prefix we left alone puts ours back in line behind theirs
if prev.kind != routeYielded || prev.holder != holder {
t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
}
states[p] = routeState{kind: routeYielded, holder: holder}
case prev.kind == routeInstalled:
t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
states[p] = routeState{kind: routeRemoved}
case prev.kind == routeRemoved:
states[p] = prev
default:
err := addRoute(p, linkAddr)
switch {
case err == nil:
if prev.kind == routeYielded {
t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
} else {
t.l.Info("Added route", "route", r)
}
states[p] = routeState{kind: routeInstalled}
case errors.Is(err, unix.EEXIST):
// Taken since the fetch, the next look will see by whom
states[p] = prev
default:
if prev.kind != routeFailed {
util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
}
states[p] = routeState{kind: routeFailed}
}
}
}
t.routeStates = states
}
// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
// routing table settles after anything that may have moved a prefix
func (t *tun) startRouteWatch(watch *os.File) {
t.routesMu.Lock()
defer t.routesMu.Unlock()
if t.closed {
if watch != nil {
_ = watch.Close()
}
return
}
t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
if watch != nil {
t.routeWatch = watch
go t.watchRoutes(watch)
}
}
func (t *tun) sweepRoutes() {
// A table still changing gets reconciled once it settles
if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
t.reconcileRoutes()
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
if !t.closed {
t.routeSweep.Reset(t.routeSweepEvery)
}
}
func openRouteSocket() (*os.File, error) {
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
}
// Fewer drops, the sweep catches the rest
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
// Non-blocking puts it on the poller, so Close interrupts a Read
if err = unix.SetNonblock(fd, true); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
}
return os.NewFile(uintptr(fd), "route"), nil
}
// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
func (t *tun) watchRoutes(f *os.File) {
var reconciled atomic.Int64
settle := time.AfterFunc(time.Hour, func() {
reconciled.Store(time.Now().UnixNano())
t.reconcileRoutes()
})
settle.Stop()
defer settle.Stop()
var burst time.Time
self := int32(os.Getpid())
b := make([]byte, os.Getpagesize())
for {
n, err := f.Read(b)
if err != nil {
if !errors.Is(err, os.ErrClosed) {
t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
return
}
moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
if !moved {
continue
}
// Our own are already in routeStates
if p.IsValid() && pid != self {
t.routeMoversMu.Lock()
if t.routeMovers == nil {
t.routeMovers = make(map[netip.Prefix]routeMover)
}
t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
t.routeMoversMu.Unlock()
}
now := time.Now()
t.lastRouteEvent.Store(now.UnixNano())
if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
burst = now
}
settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
}
}
// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
// and the pid of the process that made it.
func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
if len(b) < 4 {
return false, p, 0
}
switch int(b[3]) {
case unix.RTM_IFINFO, unix.RTM_DELADDR:
return true, p, 0
case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
default:
return false, p, 0
}
if len(b) < unix.SizeofRtMsghdr {
return false, p, 0
}
hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
// A failed request is reported too, and a scoped route never held the prefix
if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
return false, p, 0
}
if hdr.Pid == 0 {
// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
// could be holding a prefix
return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
}
// Replies to a request carry the requester's padded sockaddrs
msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
if err != nil || len(msgs) != 1 {
return false, p, 0
}
m, ok := msgs[0].(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
return false, p, 0
}
p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
if !ok {
return false, netip.Prefix{}, 0
}
p = p.Masked()
for _, r := range installRoutes(routes) {
if r.Cidr.Masked() == p {
return true, p, hdr.Pid
}
}
return false, netip.Prefix{}, 0
}
// interfaceName is the name of the interface with index i, for logs
func interfaceName(i int) string {
if ifc, err := net.InterfaceByIndex(i); err == nil {
return ifc.Name
}
return strconv.Itoa(i)
}
+10 -19
View File
@@ -165,24 +165,20 @@ func (p *Punchy) scheduleJob(job holepunchJob, delay time.Duration) {
// SendPunch sends an immediate keepalive punch for an idle hostinfo.
// The configured punchy.target_all_remotes mode picks the targets. Gated on punchy.punch and the lighthouse-skip rule
// (lighthouses don't get keepalive punches because the regular update interval keeps their NAT state warm).
// Returns the hostinfo's current remote and the error from punching it, an invalid remote if it was not punched.
func (p *Punchy) SendPunch(hostinfo *HostInfo) (netip.AddrPort, error) {
func (p *Punchy) SendPunch(hostinfo *HostInfo) {
if !p.punch.Load() {
return netip.AddrPort{}, nil
return
}
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
return netip.AddrPort{}, nil
return
}
hr := hostinfo.GetRemote()
if p.punchEverything.Load() {
return hr, p.sendPunchToAllRemotes(hostinfo, hr)
p.sendPunchToAllRemotes(hostinfo)
} else if hr := hostinfo.GetRemote(); hr.IsValid() {
p.metricPunchyTx.Inc(1)
p.punchConn.WriteTo([]byte{1}, hr)
}
if !hr.IsValid() {
return hr, nil
}
p.metricPunchyTx.Inc(1)
return hr, p.punchConn.WriteTo([]byte{1}, hr)
}
// SendPunchToAll punches every known remote for hostinfo, but only when punchy.target_all_remotes is enabled.
@@ -199,19 +195,14 @@ func (p *Punchy) SendPunchToAll(hostinfo *HostInfo) {
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
return
}
p.sendPunchToAllRemotes(hostinfo, netip.AddrPort{})
p.sendPunchToAllRemotes(hostinfo)
}
// sendPunchToAllRemotes punches every known remote for hostinfo and returns the error from punching current
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo, current netip.AddrPort) error {
var currentErr error
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo) {
hostinfo.remotes.ForEach(p.hm.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
p.metricPunchyTx.Inc(1)
if err := p.punchConn.WriteTo([]byte{1}, addr); addr == current {
currentErr = err
}
p.punchConn.WriteTo([]byte{1}, addr)
})
return currentErr
}
// Start wires the runtime dependencies and spawns the scheduler worker.
-112
View File
@@ -3,14 +3,11 @@ package nebula
import (
"context"
"log/slog"
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -245,112 +242,3 @@ func TestPunchyReloadRestartRequired(t *testing.T) {
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
}
// errNoRoute is the unreachable error every platform classifies the same way
var errNoRoute = udp.ErrInvalidIPv6RemoteForSocket
// recordingConn records where every write went and what kind of message it was, and fails the ones to an address in
// fail
type recordingConn struct {
udp.NoopConn
fail map[netip.AddrPort]error
wrote []netip.AddrPort
// types is the message type of each write, a punch has none
types []header.MessageType
}
func (c *recordingConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.wrote = append(c.wrote, addr)
var h header.H
if h.Parse(b) == nil {
c.types = append(c.types, h.Type)
}
return c.fail[addr]
}
func (c *recordingConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(int, error)) (int, error) {
written := 0
for i := range bufs {
if err := c.WriteTo(bufs[i], addrs[i]); err != nil {
failed(i, err)
continue
}
written++
}
return written, nil
}
type noLighthouses struct{}
func (noLighthouses) IsAnyLighthouseAddr([]netip.Addr) bool { return false }
var (
punchTargetV4 = netip.MustParseAddrPort("10.0.0.3:4242")
punchTargetV6 = netip.MustParseAddrPort("[fd00::3]:4242")
)
// newPunchTarget is a tunnel on its IPv6 remote that also knows the peer's IPv4 one
func newPunchTarget() *HostInfo {
vpnAddr := netip.MustParseAddr("10.128.0.3")
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
remotes: NewRemoteList([]netip.Addr{vpnAddr}, nil),
}
hostinfo.remotes.LearnRemote(vpnAddr, punchTargetV4)
hostinfo.SetRemote(punchTargetV6)
return hostinfo
}
func newPunchyForTest(t *testing.T, settings map[string]any, conn udp.Conn) *Punchy {
c := config.NewC(test.NewLogger())
c.Settings["punchy"] = settings
p := NewPunchyFromConfig(test.NewLogger(), c, conn)
hm := newHostMap(test.NewLogger())
hm.preferredRanges.Store(&[]netip.Prefix{})
p.hm = hm
p.lh = noLighthouses{}
return p
}
func TestPunchy_SendPunchReportsCurrentRemote(t *testing.T) {
t.Run("disabled", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": false}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.False(t, remote.IsValid())
require.NoError(t, err)
assert.Empty(t, conn.wrote)
})
t.Run("current remote", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.ErrorIs(t, err, errNoRoute)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
})
t.Run("all remotes, current fails", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.ErrorIs(t, err, errNoRoute)
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
})
t.Run("all remotes, only another fails", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV4: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.NoError(t, err, "a failure on a remote we are not using says nothing about the current one")
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
})
}
+2 -3
View File
@@ -41,9 +41,8 @@ type Conn interface {
//
// Returns the number of packets successfully written. A destination the kernel rejects costs only
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
// failed, when not nil, is called with the index of every packet a destination cost us and why.
// Not safe for concurrent use on the same Conn.
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
// ReloadConfig applies the listen settings that can change, r is nil at startup
ReloadConfig(c *config.C, r *config.Reload)
SupportsMultipleReaders() bool
@@ -67,7 +66,7 @@ func (NoopConn) SupportsMultipleReaders() bool {
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
return nil
}
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
return len(bufs), nil
}
func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) {
+1 -4
View File
@@ -139,7 +139,7 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
}
}
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
// silently drops the rest of a burst (linux blocks instead). Poll for
@@ -150,9 +150,6 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+1 -4
View File
@@ -44,7 +44,7 @@ func (u *GenericConn) WriteTo(b []byte, addr netip.AddrPort) error {
return err
}
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
written := 0
for i, b := range bufs {
@@ -52,9 +52,6 @@ func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed f
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+2 -2
View File
@@ -358,8 +358,8 @@ func sendto(fd int, b []byte, addr netip.AddrPort, isV4 bool) error {
// WriteBatch sends bufs via sendmmsg(2), coalescing same-destination runs into UDP-GSO superpackets when supported.
// See batchWriter in udp_linux_writebatch.go for the mechanics.
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
return u.bw.WriteBatch(bufs, addrs, failed)
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
return u.bw.WriteBatch(bufs, addrs)
}
// writeSockaddr encodes addr into buf (which must be at least SizeofSockaddrInet6 bytes).
+9 -38
View File
@@ -3,7 +3,6 @@
package udp
import (
"errors"
"fmt"
"log/slog"
"net"
@@ -97,22 +96,13 @@ func TestWriteBatchBadFamilyDeliversOthers(t *testing.T) {
bufs := [][]byte{[]byte("AAA"), []byte("BBB"), []byte("CCC")}
addrs := []netip.AddrPort{good, bad, good}
var failed []int
n, err := sender.WriteBatch(bufs, addrs, func(i int, err error) {
if !IsUnreachable(err) {
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
}
failed = append(failed, i)
})
n, err := sender.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch returned error, want nil (bad dest should be isolated): %v", err)
}
if n != 2 {
t.Errorf("WriteBatch wrote %d packets, want 2 of 3 (the bad-family dest is the only casualty)", n)
}
if want := []int{1}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
got := map[string]bool{}
rx.SetReadDeadline(time.Now().Add(2 * time.Second))
@@ -164,7 +154,7 @@ func TestWriteBatchUnreachableDestDeliversOthers(t *testing.T) {
addrs := []netip.AddrPort{good, good, bad, good, good}
// The bad destination is reported, but only after every other packet has been attempted.
if _, err := sender.WriteBatch(bufs, addrs, nil); err == nil {
if _, err := sender.WriteBatch(bufs, addrs); err == nil {
t.Log("WriteBatch returned nil; kernel accepted the reserved address, delivery assertions still apply")
}
@@ -386,7 +376,7 @@ func TestWriteBatchPartialSendRewind(t *testing.T) {
return accept, nil
}
written, err := w.WriteBatch(bufs, addrs, nil)
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -439,22 +429,13 @@ func TestWriteBatchSkipUnroutableRunAccounting(t *testing.T) {
return accept, nil
}
var failed []int
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
if !IsUnreachable(err) {
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
}
failed = append(failed, i)
})
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
if written != 4 {
t.Errorf("written = %d, want 4 (the unroutable run is the only casualty)", written)
}
if want := []int{2}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
wantTags := []byte{1, 2, 4, 5}
if len(wire) != len(wantTags) {
t.Fatalf("wire got %d packets, want %d (dup or loss around the skip)", len(wire), len(wantTags))
@@ -505,23 +486,13 @@ func TestWriteBatchMidChunkRejectResumes(t *testing.T) {
}
}
var failed []int
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
if !errors.Is(err, unix.EPERM) {
t.Errorf("packet %d failed with %v, want EPERM", i, err)
}
failed = append(failed, i)
})
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
if written != 3 {
t.Errorf("written = %d, want 3 (B's rejected run is the only casualty)", written)
}
// Every packet in B's run is reported, not just the entry
if want := []int{2, 3}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
wantTags := []byte{1, 2, 5}
if len(wire) != len(wantTags) {
t.Fatalf("wire got %d packets, want %d (dup or loss around the mid-chunk reject)", len(wire), len(wantTags))
@@ -572,7 +543,7 @@ func TestWriteBatchMidChunkEIODisablesGSOWithoutDup(t *testing.T) {
}
}
written, err := w.WriteBatch(bufs, addrs, nil)
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -600,7 +571,7 @@ func TestWriteBatchZeroProgress(t *testing.T) {
w.sendFn = func(start, n int) (int, error) { return 0, nil }
bufs := [][]byte{make([]byte, 100)}
addrs := []netip.AddrPort{netip.MustParseAddrPort("127.0.0.1:4242")}
if _, err := w.WriteBatch(bufs, addrs, nil); err == nil {
if _, err := w.WriteBatch(bufs, addrs); err == nil {
t.Fatal("WriteBatch = nil error on zero progress, want error")
}
}
@@ -626,7 +597,7 @@ func TestWriteBatchEIODisablesGSOAndReplays(t *testing.T) {
return n, nil
}
written, err := w.WriteBatch(bufs, addrs, nil)
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -703,7 +674,7 @@ func TestGSOEngagesOnLoopback(t *testing.T) {
addrs[i] = dst
}
written, err := sc.WriteBatch(bufs, addrs, nil)
written, err := sc.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
+2 -2
View File
@@ -86,7 +86,7 @@ func TestWriteBatchOffloadsDisabledScripted(t *testing.T) {
addrs[i] = dst
}
written, err := w.WriteBatch(bufs, addrs, nil)
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -163,7 +163,7 @@ func TestOffloadsDisabledOnLoopback(t *testing.T) {
addrs[i] = dst
}
written, err := sc.WriteBatch(bufs, addrs, nil)
written, err := sc.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
+2 -14
View File
@@ -190,8 +190,8 @@ func parseRelease(r string) (major, minor int) {
// its packets are dropped and the rest of the chunk resumes in place.
//
// Returns the number of packets sent. An error means the call itself failed.
// A short count means some destinations were undeliverable, failed hears about each of their packets.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
// A short count means some destinations were undeliverable.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
if len(bufs) != len(addrs) {
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
}
@@ -233,7 +233,6 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed f
if w.l.Enabled(context.Background(), slog.LevelDebug) {
w.l.Debug("skipping unroutable batch destination", "udpAddr", addrs[i], "packets", runLen, "error", err)
}
reportFailed(failed, i, i+runLen, err)
i += runLen
continue
}
@@ -301,23 +300,12 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed f
"gso", w.gsoSupported,
)
}
reportFailed(failed, w.entryEnd[done]-w.entryPkts[done], w.entryEnd[done], serr)
done++
}
}
return written, nil
}
// reportFailed tells failed, if anyone is listening, that every packet in bufs[start:end] was lost to err
func reportFailed(failed func(i int, err error), start, end int, err error) {
if failed == nil {
return
}
for i := start; i < end; i++ {
failed(i, err)
}
}
// planRun returns the length of the run starting at start and its segment
// size (len(bufs[start])). A run of length 1 carries no UDP_SEGMENT cmsg
// and is sent as a plain datagram; without GSO support planRun always returns 1.
+2 -2
View File
@@ -80,11 +80,11 @@ func TestWriteBatchNoAllocs(t *testing.T) {
var werr error
// Warm-up outside the measured runs.
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
t.Fatalf("WriteBatch warm-up: %v", err)
}
allocs := testing.AllocsPerRun(100, func() {
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
werr = err
}
})
+1 -4
View File
@@ -319,7 +319,7 @@ func (u *RIOConn) WriteTo(buf []byte, ip netip.AddrPort) error {
return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
}
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
written := 0
for i, b := range bufs {
@@ -327,9 +327,6 @@ func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+7 -64
View File
@@ -6,7 +6,6 @@ import (
"context"
"io"
"log/slog"
"net"
"net/netip"
"os"
"sync"
@@ -64,19 +63,10 @@ func acquirePacket() *Packet {
}
}
// testerAddrs is where a TesterConn sends from. A single address host sends everything from addr, whatever the
// destination. A dual stack host sends from the address matching the destination's family and has no route at all
// for a family it holds no address in.
type testerAddrs struct {
addr netip.AddrPort
v4, v6 netip.AddrPort
dual bool
}
type TesterConn struct {
// addrs is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
addrs atomic.Pointer[testerAddrs]
addr atomic.Pointer[netip.AddrPort]
RxPackets chan *Packet // Packets to receive into nebula
TxPackets chan *Packet // Packets transmitted outside by nebula
@@ -104,53 +94,14 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
return c, nil
}
// GetAddr returns the underlay address this conn currently sends from. For a dual stack conn that is the IPv4
// address, or the IPv6 one when there is no IPv4 service.
// GetAddr returns the underlay address this conn currently sends from.
func (u *TesterConn) GetAddr() netip.AddrPort {
return u.addrs.Load().addr
return *u.addr.Load()
}
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
func (u *TesterConn) SetAddr(addr netip.AddrPort) {
u.addrs.Store(&testerAddrs{addr: addr})
}
// SetDualStackAddrs gives this conn one underlay address per family, standing in for a host whose routing table picks
// the source by destination. Pass an invalid AddrPort for a family the network does not offer, sends to that family
// then fail like they would on a host with no route for it.
func (u *TesterConn) SetDualStackAddrs(v4, v6 netip.AddrPort) {
a := &testerAddrs{v4: v4, v6: v6, dual: true}
if v4.IsValid() {
a.addr = v4
} else {
a.addr = v6
}
u.addrs.Store(a)
}
// GetAddrFor returns the underlay address a send to dst leaves from, GetAddr when there is no route to dst
func (u *TesterConn) GetAddrFor(dst netip.AddrPort) netip.AddrPort {
if src, err := u.sourceFor(dst); err == nil {
return src
}
return u.GetAddr()
}
// sourceFor returns the address a send to dst leaves from
func (u *TesterConn) sourceFor(dst netip.AddrPort) (netip.AddrPort, error) {
a := u.addrs.Load()
if !a.dual {
return a.addr, nil
}
src := a.v6
if dst.Addr().Unmap().Is4() {
src = a.v4
}
if !src.IsValid() {
return netip.AddrPort{}, &net.OpError{Op: "write", Net: "udp", Addr: net.UDPAddrFromAddrPort(dst), Err: errNetUnreachable}
}
return src, nil
u.addr.Store(&addr)
}
// Send will place a UdpPacket onto the receive queue for nebula to consume
@@ -202,11 +153,6 @@ func (u *TesterConn) Get(block bool) *Packet {
//********************************************************************************************************************//
func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
from, err := u.sourceFor(addr)
if err != nil {
return err
}
p := acquirePacket()
if cap(p.Data) < len(b) {
p.Data = make([]byte, len(b))
@@ -214,7 +160,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
p.Data = p.Data[:len(b)]
}
copy(p.Data, b)
p.From = from
p.From = u.GetAddr()
p.To = addr
select {
case <-u.done:
@@ -224,16 +170,13 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
return nil
}
}
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
written := 0
for i, b := range bufs {
if err := u.WriteTo(b, addrs[i]); err == nil {
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
-22
View File
@@ -1,22 +0,0 @@
//go:build !windows
package udp
import (
"errors"
"syscall"
)
// errNetUnreachable is the error this platform's sockets give for a destination with no route
var errNetUnreachable error = syscall.ENETUNREACH
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
// to an IPv6 remote into one of these.
func IsUnreachable(err error) bool {
return errors.Is(err, syscall.ENETUNREACH) ||
errors.Is(err, syscall.EHOSTUNREACH) ||
errors.Is(err, syscall.EADDRNOTAVAIL) ||
errors.Is(err, syscall.EAFNOSUPPORT) ||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
}
-37
View File
@@ -1,37 +0,0 @@
//go:build !windows
package udp
import (
"net"
"syscall"
"testing"
)
func TestIsUnreachable(t *testing.T) {
wrap := func(err error) error { return &net.OpError{Op: "write", Net: "udp", Err: err} }
for _, err := range []error{
wrap(syscall.ENETUNREACH),
wrap(syscall.EHOSTUNREACH),
wrap(syscall.EADDRNOTAVAIL),
wrap(syscall.EAFNOSUPPORT),
ErrInvalidIPv6RemoteForSocket,
} {
if !IsUnreachable(err) {
t.Errorf("IsUnreachable(%v) = false, want true", err)
}
}
for _, err := range []error{
nil,
wrap(syscall.EPERM),
wrap(syscall.ENOBUFS),
wrap(syscall.EMSGSIZE),
net.ErrClosed,
} {
if IsUnreachable(err) {
t.Errorf("IsUnreachable(%v) = true, want false", err)
}
}
}
-21
View File
@@ -1,21 +0,0 @@
package udp
import (
"errors"
"golang.org/x/sys/windows"
)
// errNetUnreachable is the error this platform's sockets give for a destination with no route
var errNetUnreachable error = windows.WSAENETUNREACH
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
// to an IPv6 remote into one of these.
func IsUnreachable(err error) bool {
return errors.Is(err, windows.WSAENETUNREACH) ||
errors.Is(err, windows.WSAEHOSTUNREACH) ||
errors.Is(err, windows.WSAEADDRNOTAVAIL) ||
errors.Is(err, windows.WSAEAFNOSUPPORT) ||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
}