Compare commits

..
12 changed files with 133 additions and 488 deletions

No files matched your search

-3
View File
@@ -222,14 +222,11 @@ test-cov-html:
go test -coverprofile=coverage.out go test -coverprofile=coverage.out
go tool cover -html=coverage.out go tool cover -html=coverage.out
# The package builds only compile. The final line links an android binary so a linker-only failure
# cannot pass CI.
build-test-mobile: build-test-mobile:
GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build -o /dev/null ${NEBULA_CMD_PATH}
bench: bench:
go test -bench=. go test -bench=.
+8 -8
View File
@@ -105,11 +105,12 @@ func (cm *connectionManager) getInactivityTimeout() time.Duration {
} }
func (cm *connectionManager) In(h *HostInfo) { func (cm *connectionManager) In(h *HostInfo) {
h.in.Store(true) h.markIn()
} }
func (cm *connectionManager) Out(h *HostInfo) { // Out records outbound traffic and reports whether the local network changed since this tunnel last sent.
h.out.Store(true) func (cm *connectionManager) Out(h *HostInfo) bool {
return h.markOut(cm.intf.rebindEpoch.Load())
} }
func (cm *connectionManager) RelayUsed(localIndex uint32) { func (cm *connectionManager) RelayUsed(localIndex uint32) {
@@ -128,8 +129,7 @@ func (cm *connectionManager) RelayUsed(localIndex uint32) {
// getAndResetTrafficCheck returns if there was any inbound or outbound traffic within the last tick and // getAndResetTrafficCheck returns if there was any inbound or outbound traffic within the last tick and
// resets the state for this local index // resets the state for this local index
func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time) (bool, bool) { func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time) (bool, bool) {
in := h.in.Swap(false) in, out := h.takeTraffic()
out := h.out.Swap(false)
if in || out { if in || out {
h.lastUsed = now h.lastUsed = now
} }
@@ -340,7 +340,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
"tunnelCheck", m{"state": "alive", "method": "passive"}, "tunnelCheck", m{"state": "alive", "method": "passive"},
) )
} }
hostinfo.pendingDeletion.Store(false) hostinfo.setPendingDeletion(false)
if mainHostInfo { if mainHostInfo {
decision = tryRehandshake decision = tryRehandshake
@@ -363,7 +363,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
return decision, hostinfo, primary return decision, hostinfo, primary
} }
if hostinfo.pendingDeletion.Load() { if hostinfo.isPendingDeletion() {
// We have already sent a test packet and nothing was returned, this hostinfo is dead // We have already sent a test packet and nothing was returned, this hostinfo is dead
hostinfo.logger(cm.l).Info("Tunnel status", hostinfo.logger(cm.l).Info("Tunnel status",
"tunnelCheck", m{"state": "dead", "method": "active"}, "tunnelCheck", m{"state": "dead", "method": "active"},
@@ -414,7 +414,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
} }
} }
hostinfo.pendingDeletion.Store(true) hostinfo.setPendingDeletion(true)
cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval) cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
return decision, hostinfo, nil return decision, hostinfo, nil
} }
+36 -36
View File
@@ -86,25 +86,25 @@ func Test_NewConnectionManagerTest(t *testing.T) {
// We saw traffic out to vpnIp // We saw traffic out to vpnIp
nc.Out(hostinfo) nc.Out(hostinfo)
nc.In(hostinfo) nc.In(hostinfo)
assert.False(t, hostinfo.pendingDeletion.Load()) assert.False(t, hostinfo.isPendingDeletion())
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
assert.True(t, hostinfo.out.Load()) assert.True(t, hostinfo.sentSinceCheck())
assert.True(t, hostinfo.in.Load()) assert.True(t, (hostinfo.state.Load()&stateIn != 0))
// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded // Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now()) nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
assert.False(t, hostinfo.pendingDeletion.Load()) assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.out.Load()) assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, hostinfo.in.Load()) assert.False(t, (hostinfo.state.Load()&stateIn != 0))
// Do another traffic check tick, this host should be pending deletion now // Do another traffic check tick, this host should be pending deletion now
nc.Out(hostinfo) nc.Out(hostinfo)
assert.True(t, hostinfo.out.Load()) assert.True(t, hostinfo.sentSinceCheck())
nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now()) nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
assert.True(t, hostinfo.pendingDeletion.Load()) assert.True(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.out.Load()) assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, hostinfo.in.Load()) assert.False(t, (hostinfo.state.Load()&stateIn != 0))
assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
@@ -168,33 +168,33 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
// We saw traffic out to vpnIp // We saw traffic out to vpnIp
nc.Out(hostinfo) nc.Out(hostinfo)
nc.In(hostinfo) nc.In(hostinfo)
assert.True(t, hostinfo.in.Load()) assert.True(t, (hostinfo.state.Load()&stateIn != 0))
assert.True(t, hostinfo.out.Load()) assert.True(t, hostinfo.sentSinceCheck())
assert.False(t, hostinfo.pendingDeletion.Load()) assert.False(t, hostinfo.isPendingDeletion())
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded // Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now()) nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
assert.False(t, hostinfo.pendingDeletion.Load()) assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.out.Load()) assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, hostinfo.in.Load()) assert.False(t, (hostinfo.state.Load()&stateIn != 0))
// Do another traffic check tick, this host should be pending deletion now // Do another traffic check tick, this host should be pending deletion now
nc.Out(hostinfo) nc.Out(hostinfo)
nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now()) nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
assert.True(t, hostinfo.pendingDeletion.Load()) assert.True(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.out.Load()) assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, hostinfo.in.Load()) assert.False(t, (hostinfo.state.Load()&stateIn != 0))
assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
// We saw traffic, should no longer be pending deletion // We saw traffic, should no longer be pending deletion
nc.In(hostinfo) nc.In(hostinfo)
nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now()) nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
assert.False(t, hostinfo.pendingDeletion.Load()) assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.out.Load()) assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, hostinfo.in.Load()) assert.False(t, (hostinfo.state.Load()&stateIn != 0))
assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
} }
@@ -253,31 +253,31 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
// Do a traffic check tick, in and out should be cleared but should not be pending deletion // Do a traffic check tick, in and out should be cleared but should not be pending deletion
nc.Out(hostinfo) nc.Out(hostinfo)
nc.In(hostinfo) nc.In(hostinfo)
assert.True(t, hostinfo.out.Load()) assert.True(t, hostinfo.sentSinceCheck())
assert.True(t, hostinfo.in.Load()) assert.True(t, (hostinfo.state.Load()&stateIn != 0))
now := time.Now() now := time.Now()
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now) decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision) assert.Equal(t, tryRehandshake, decision)
assert.Equal(t, now, hostinfo.lastUsed) assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.pendingDeletion.Load()) assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.out.Load()) assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, hostinfo.in.Load()) assert.False(t, (hostinfo.state.Load()&stateIn != 0))
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5)) decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
assert.Equal(t, doNothing, decision) assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed) assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.pendingDeletion.Load()) assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.out.Load()) assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, hostinfo.in.Load()) assert.False(t, (hostinfo.state.Load()&stateIn != 0))
// Do another traffic check tick, should still not be pending deletion // Do another traffic check tick, should still not be pending deletion
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10)) decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
assert.Equal(t, doNothing, decision) assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed) assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.pendingDeletion.Load()) assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.out.Load()) assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, hostinfo.in.Load()) assert.False(t, (hostinfo.state.Load()&stateIn != 0))
assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
@@ -285,9 +285,9 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10)) decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
assert.Equal(t, closeTunnel, decision) assert.Equal(t, closeTunnel, decision)
assert.Equal(t, now, hostinfo.lastUsed) assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.pendingDeletion.Load()) assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.out.Load()) assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, hostinfo.in.Load()) assert.False(t, (hostinfo.state.Load()&stateIn != 0))
assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
} }
+1 -1
View File
@@ -212,7 +212,7 @@ func (c *Control) RebindUDPServer() {
c.f.lightHouse.SendUpdate() c.f.lightHouse.SendUpdate()
// Let the main interface know that we rebound so that underlying tunnels know to trigger punches from their remotes // Let the main interface know that we rebound so that underlying tunnels know to trigger punches from their remotes
c.f.rebindCount++ c.f.rebindEpoch.Add(1)
} }
// ListHostmapHosts returns details about the actual or pending (handshaking) hostmap by vpn ip // ListHostmapHosts returns details about the actual or pending (handshaking) hostmap by vpn ip
+79 -47
View File
@@ -238,18 +238,30 @@ const (
) )
type HostInfo struct { type HostInfo struct {
// The first cache line is everything the packet paths touch. Grouping them here means a send or receive
// pulls in one line instead of two, which is what the layout looked like when state lived at the end.
remote atomic.Pointer[netip.AddrPort] remote atomic.Pointer[netip.AddrPort]
remotes *RemoteList
promoteCounter atomic.Uint32
ConnectionState *ConnectionState ConnectionState *ConnectionState
remoteIndexId uint32
localIndexId uint32 // state holds everything the hot paths need to touch per packet, in one word: whether we have seen traffic
// each way since the connection manager last looked, whether it has given up on us, and the
// Interface.rebindEpoch this tunnel last sent under. Keeping the epoch here means it survives the traffic
// bits being cleared, so a tunnel that has not sent since a rebind still notices when it does.
state atomic.Uint32
promoteCounter atomic.Uint32
remoteIndexId uint32
localIndexId uint32
remotes *RemoteList
// vpnAddrs is a list of vpn addresses assigned to this host that are within our own vpn networks // vpnAddrs is a list of vpn addresses assigned to this host that are within our own vpn networks
// The host may have other vpn addresses that are outside our // The host may have other vpn addresses that are outside our
// vpn networks but were removed because they are not usable // vpn networks but were removed because they are not usable
vpnAddrs []netip.Addr vpnAddrs []netip.Addr
// Everything below is off the packet path: handshakes, relays, roaming and the connection manager.
// networks is a combination of specific vpn addresses (not prefixes!) and full unsafe networks assigned to this host. // networks is a combination of specific vpn addresses (not prefixes!) and full unsafe networks assigned to this host.
networks *bart.Table[NetworkType] networks *bart.Table[NetworkType]
relayState RelayState relayState RelayState
@@ -262,11 +274,6 @@ type HostInfo struct {
// This is used to limit lighthouse re-queries in chatty clients // This is used to limit lighthouse re-queries in chatty clients
nextLHQuery atomic.Int64 nextLHQuery atomic.Int64
// lastRebindCount is the other side of Interface.rebindCount, if these values don't match then we need to ask LH
// for a punch from the remote end of this tunnel. The goal being to prime their conntrack for our traffic just like
// with a handshake
lastRebindCount int8
// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally // lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator // Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
// This is used to avoid an attack where a handshake packet is replayed after some time // This is used to avoid an attack where a handshake packet is replayed after some time
@@ -275,9 +282,6 @@ type HostInfo struct {
lastRoam time.Time lastRoam time.Time
lastRoamRemote netip.AddrPort lastRoamRemote netip.AddrPort
//TODO: in, out, and others might benefit from being an atomic.Int32. We could collapse connectionManager pendingDeletion, relayUsed, and in/out into this 1 thing
in, out, pendingDeletion atomic.Bool
// lastUsed tracks the last time ConnectionManager checked the tunnel and it was in use. // lastUsed tracks the last time ConnectionManager checked the tunnel and it was in use.
// This value will be behind against actual tunnel utilization in the hot path. // This value will be behind against actual tunnel utilization in the hot path.
// This should only be used by the ConnectionManagers ticker routine. // This should only be used by the ConnectionManagers ticker routine.
@@ -658,7 +662,7 @@ func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
hm.Indexes[hostinfo.localIndexId] = hostinfo hm.Indexes[hostinfo.localIndexId] = hostinfo
hm.RemoteIndexes[hostinfo.remoteIndexId] = hostinfo hm.RemoteIndexes[hostinfo.remoteIndexId] = hostinfo
hostinfo.out.Store(true) hostinfo.markOut(f.rebindEpoch.Load())
if f.connectionManager != nil { // f.connectionManager is only nil in some unit tests if f.connectionManager != nil { // f.connectionManager is only nil in some unit tests
f.connectionManager.trafficTimer.Add(hostinfo.localIndexId, f.connectionManager.checkInterval) f.connectionManager.trafficTimer.Add(hostinfo.localIndexId, f.connectionManager.checkInterval)
} }
@@ -759,6 +763,64 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
} }
} }
// Bits within HostInfo.state. Everything above stateEpochShift is the rebind epoch.
const (
stateIn uint32 = 1 << iota
stateOut
statePendingDeletion
stateFlags = stateIn | stateOut | statePendingDeletion
stateEpochShift = 3
)
// markIn records inbound traffic. Reading first keeps the cache line shared on the common path, where the bit
// is already set.
func (i *HostInfo) markIn() {
if i.state.Load()&stateIn == 0 {
i.state.Or(stateIn)
}
}
// markOut records that we sent on this tunnel under the given rebind epoch. It reports whether the epoch moved
// since our last send, which means the local network changed and we want the far side to punch at us again.
// The common path is a single load that matches and returns.
func (i *HostInfo) markOut(epoch uint32) bool {
e := epoch << stateEpochShift
for {
old := i.state.Load()
if old&stateOut != 0 && old&^stateFlags == e {
return false
}
if i.state.CompareAndSwap(old, old&stateFlags|stateOut|e) {
return old&^stateFlags != e
}
}
}
// sentSinceCheck reports whether anything has been sent since the connection manager last looked.
func (i *HostInfo) sentSinceCheck() bool {
return i.state.Load()&stateOut != 0
}
// takeTraffic clears both traffic bits, leaving the epoch alone, and reports what they were.
func (i *HostInfo) takeTraffic() (in bool, out bool) {
old := i.state.And(^(stateIn | stateOut))
return old&stateIn != 0, old&stateOut != 0
}
func (i *HostInfo) setPendingDeletion(v bool) {
if v {
i.state.Or(statePendingDeletion)
} else {
i.state.And(^statePendingDeletion)
}
}
func (i *HostInfo) isPendingDeletion() bool {
return i.state.Load()&statePendingDeletion != 0
}
func (i *HostInfo) GetCert() *cert.CachedCertificate { func (i *HostInfo) GetCert() *cert.CachedCertificate {
if i.ConnectionState != nil { if i.ConnectionState != nil {
return i.ConnectionState.peerCert return i.ConnectionState.peerCert
@@ -868,35 +930,10 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
// Utility functions // Utility functions
// localInterface is what localAddrs needs from a net.Interface, provided per platform by func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
// localInterfaces. Addrs is deferred so platforms where address lookup is a separate kernel round
// trip only pay for interfaces the allow list accepts.
type localInterface struct {
Name string
Addrs func() ([]net.Addr, error)
}
func localAddrs(l *slog.Logger, allowList *LocalAllowList) ([]netip.Addr, error) {
return collectLocalAddrs(l, allowList, localInterfaces)
}
// collectLocalAddrs takes its enumerator as an argument so tests can drive the filtering and the
// failure branches without depending on the addresses of whatever host they run on. It reports
// failures to the caller rather than logging them, because it runs on every lighthouse update and
// only the caller can tell a new failure from a repeat of the same one.
func collectLocalAddrs(
l *slog.Logger,
allowList *LocalAllowList,
interfaces func() ([]localInterface, error),
) ([]netip.Addr, error) {
//FIXME: This function is pretty garbage //FIXME: This function is pretty garbage
var finalAddrs []netip.Addr var finalAddrs []netip.Addr
var errs []error ifaces, _ := net.Interfaces()
ifaces, err := interfaces()
if err != nil {
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
}
for _, i := range ifaces { for _, i := range ifaces {
allow := allowList.AllowName(i.Name) allow := allowList.AllowName(i.Name)
if l.Enabled(context.Background(), logging.LevelTrace) { if l.Enabled(context.Background(), logging.LevelTrace) {
@@ -909,12 +946,7 @@ func collectLocalAddrs(
if !allow { if !allow {
continue continue
} }
addrs, err := i.Addrs() addrs, _ := i.Addrs()
if err != nil {
errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err))
continue
}
for _, rawAddr := range addrs { for _, rawAddr := range addrs {
var addr netip.Addr var addr netip.Addr
switch v := rawAddr.(type) { switch v := rawAddr.(type) {
@@ -949,5 +981,5 @@ func collectLocalAddrs(
} }
} }
} }
return finalAddrs, errors.Join(errs...) return finalAddrs
} }
-93
View File
@@ -1,8 +1,6 @@
package nebula package nebula
import ( import (
"errors"
"net"
"net/netip" "net/netip"
"slices" "slices"
"testing" "testing"
@@ -403,94 +401,3 @@ func TestHostMap_RelayState(t *testing.T) {
assert.Equal(t, []netip.Addr{}, h1.relayState.relays) assert.Equal(t, []netip.Addr{}, h1.relayState.relays)
} }
func TestCollectLocalAddrs(t *testing.T) {
ifaces := []net.Interface{
{Index: 1, Name: "lo"},
{Index: 2, Name: "eth0"},
{Index: 3, Name: "docker0"},
}
addrs := map[string][]net.Addr{
"lo": {
&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)},
&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
},
"eth0": {
&net.IPNet{IP: net.ParseIP("10.0.0.5"), Mask: net.CIDRMask(24, 32)},
&net.IPNet{IP: net.ParseIP("fe80::1"), Mask: net.CIDRMask(64, 128)},
&net.IPAddr{IP: net.ParseIP("fd00::5")},
},
"docker0": {
&net.IPNet{IP: net.ParseIP("172.17.0.1"), Mask: net.CIDRMask(16, 32)},
},
}
// enumerate wraps ifaces with the given address lookup.
enumerate := func(addrsFor func(name string) ([]net.Addr, error)) func() ([]localInterface, error) {
return func() ([]localInterface, error) {
out := make([]localInterface, 0, len(ifaces))
for _, i := range ifaces {
name := i.Name
out = append(out, localInterface{
Name: name,
Addrs: func() ([]net.Addr, error) { return addrsFor(name) },
})
}
return out, nil
}
}
addrsFor := func(name string) ([]net.Addr, error) { return addrs[name], nil }
// Loopback and link local are dropped, everything else on every interface is kept.
out, err := collectLocalAddrs(test.NewLogger(), nil, enumerate(addrsFor))
require.NoError(t, err)
assert.Equal(t, []netip.Addr{
netip.MustParseAddr("10.0.0.5"),
netip.MustParseAddr("fd00::5"),
netip.MustParseAddr("172.17.0.1"),
}, out)
// An interface the allow list rejects by name is never asked for its addresses.
c := config.NewC(test.NewLogger())
c.Settings["allowlist"] = map[string]any{
"interfaces": map[string]any{`docker.*`: false},
}
al, err := NewLocalAllowListFromConfig(c, "allowlist")
require.NoError(t, err)
asked := make(map[string]struct{})
countingAddrsFor := func(name string) ([]net.Addr, error) {
asked[name] = struct{}{}
return addrs[name], nil
}
out, err = collectLocalAddrs(test.NewLogger(), al, enumerate(countingAddrsFor))
require.NoError(t, err)
assert.Equal(t, []netip.Addr{
netip.MustParseAddr("10.0.0.5"),
netip.MustParseAddr("fd00::5"),
}, out)
assert.NotContains(t, asked, "docker0")
// A failure to enumerate interfaces at all is reported rather than silently advertising nothing.
out, err = collectLocalAddrs(
test.NewLogger(),
nil,
func() ([]localInterface, error) { return nil, errors.New("netlinkrib: permission denied") },
)
assert.Nil(t, out)
require.EqualError(t, err, "failed to enumerate local interfaces: netlinkrib: permission denied")
// One interface failing is reported and skipped, the rest are still collected.
out, err = collectLocalAddrs(
test.NewLogger(),
nil,
enumerate(func(name string) ([]net.Addr, error) {
if name == "eth0" {
return nil, errors.New("nope")
}
return addrs[name], nil
}),
)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("172.17.0.1")}, out)
require.EqualError(t, err, "failed to get addresses for eth0: nope")
}
+4 -9
View File
@@ -365,17 +365,12 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p) //l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c) out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
f.connectionManager.Out(hostinfo) // We rebound since this tunnel last sent, so the local network moved. Ask the lighthouse to have the far side
// punch at where we are now, which primes their conntrack the same way a handshake would.
// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against if f.connectionManager.Out(hostinfo) && t != header.CloseTunnel {
// all our addrs and enable a faster roaming.
if t != header.CloseTunnel && hostinfo.lastRebindCount != f.rebindCount {
//NOTE: there is an update hole if a tunnel isn't used and exactly 256 rebinds occur before the tunnel is
// finally used again. This tunnel would eventually be torn down and recreated if this action didn't help.
f.lightHouse.QueryServer(hostinfo.vpnAddrs[0]) f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
hostinfo.lastRebindCount = f.rebindCount
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
f.l.Debug("Lighthouse update triggered for punch due to rebind counter", f.l.Debug("Lighthouse update triggered for punch due to rebind epoch",
"vpnAddrs", hostinfo.vpnAddrs, "vpnAddrs", hostinfo.vpnAddrs,
) )
} }
+4 -2
View File
@@ -82,8 +82,10 @@ type Interface struct {
sendRecvErrorConfig recvErrorConfig sendRecvErrorConfig recvErrorConfig
acceptRecvErrorConfig recvErrorConfig acceptRecvErrorConfig recvErrorConfig
// rebindCount is used to decide if an active tunnel should trigger a punch notification through a lighthouse // rebindEpoch bumps every time the udp listener is rebound, which means the local network moved. Tunnels
rebindCount int8 // compare it against their own copy to decide they need a punch from the far side. Read on every send, only
// written on a rebind, so the cache line stays shared across the routines.
rebindEpoch atomic.Uint32
version string version string
conntrackCacheTimeout time.Duration conntrackCacheTimeout time.Duration
+1 -27
View File
@@ -40,10 +40,6 @@ type LightHouse struct {
// addresses rather than whatever this machine's NICs happen to be. Set it before Start. // addresses rather than whatever this machine's NICs happen to be. Set it before Start.
localAddrsFn func(*LocalAllowList) []netip.Addr localAddrsFn func(*LocalAllowList) []netip.Addr
// lastLocalAddrsErr is the previous localAddrsFn failure. Enumeration runs on every update, so an
// unchanged failure is demoted to Debug rather than warning every lighthouse.interval forever.
lastLocalAddrsErr atomic.Pointer[string]
// Local cache of answers from light houses // Local cache of answers from light houses
// map of vpn addr to answers // map of vpn addr to answers
addrMap map[netip.Addr]*RemoteList addrMap map[netip.Addr]*RemoteList
@@ -116,9 +112,7 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
l: l, l: l,
} }
h.localAddrsFn = func(al *LocalAllowList) []netip.Addr { h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
addrs, err := localAddrs(h.l, al) return localAddrs(h.l, al)
h.logLocalAddrsErr(err)
return addrs
} }
lighthouses := make([]netip.Addr, 0) lighthouses := make([]netip.Addr, 0)
@@ -919,26 +913,6 @@ func (lh *LightHouse) TriggerUpdate() {
} }
} }
// logLocalAddrsErr reports a localAddrs failure at Warn the first time it is seen and at Debug while
// it persists unchanged, so a permanent failure does not warn on every update forever.
func (lh *LightHouse) logLocalAddrsErr(err error) {
if err == nil {
lh.lastLocalAddrsErr.Store(nil)
return
}
msg := err.Error()
prev := lh.lastLocalAddrsErr.Swap(&msg)
if prev != nil && *prev == msg {
if lh.l.Enabled(context.Background(), slog.LevelDebug) {
lh.l.Debug("Failed to collect local addresses to advertise", "error", err)
}
return
}
lh.l.Warn("Failed to collect local addresses to advertise", "error", err)
}
func (lh *LightHouse) SendUpdate() { func (lh *LightHouse) SendUpdate() {
var v4 []*V4AddrPort var v4 []*V4AddrPort
var v6 []*V6AddrPort var v6 []*V6AddrPort
-31
View File
@@ -1,9 +1,7 @@
package nebula package nebula
import ( import (
"bytes"
"encoding/binary" "encoding/binary"
"errors"
"fmt" "fmt"
"net/netip" "net/netip"
"testing" "testing"
@@ -740,32 +738,3 @@ func TestLighthouse_DeletesWork(t *testing.T) {
out = lh.Query(testHost) out = lh.Query(testHost)
assert.Nil(t, out) assert.Nil(t, out)
} }
func TestLightHouse_logLocalAddrsErr(t *testing.T) {
out := &bytes.Buffer{}
lh := &LightHouse{l: test.NewLoggerWithOutput(out)}
// The first sighting of a failure warns.
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "permission denied")
// Repeating unchanged does not warn again, which is what keeps a permanent failure from warning
// on every lighthouse.interval for the life of the process.
out.Reset()
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.NotContains(t, out.String(), "level=WARN")
// A different failure is a new event and warns.
out.Reset()
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "something else")
// Recovering resets, so the same failure returning later warns again.
out.Reset()
lh.logLocalAddrsErr(nil)
assert.Empty(t, out.String())
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
}
-18
View File
@@ -1,18 +0,0 @@
//go:build !android
package nebula
import "net"
func localInterfaces() ([]localInterface, error) {
ifaces, err := net.Interfaces()
if err != nil {
return nil, err
}
out := make([]localInterface, len(ifaces))
for n, i := range ifaces {
out[n] = localInterface{Name: i.Name, Addrs: i.Addrs}
}
return out, nil
}
-213
View File
@@ -1,213 +0,0 @@
//go:build android
package nebula
import (
"errors"
"fmt"
"net"
"os"
"slices"
"syscall"
"unsafe"
"golang.org/x/sys/unix"
)
// The Android app sandbox denies bind() on netlink route sockets and RTM_GETLINK, so
// net.Interfaces fails with EACCES. Enumerate with an RTM_GETADDR dump on an unbound socket instead.
func localInterfaces() ([]localInterface, error) {
return netlinkInterfaceAddrs()
}
// netlinkInterfaceAddrs returns every interface that has an address, with its addresses. It dumps
// RTM_GETADDR on a netlink socket that is never bound and names each index with SIOCGIFNAME, the
// same operations Bionic's getifaddrs uses for app UIDs, which the sandbox permits.
func netlinkInterfaceAddrs() ([]localInterface, error) {
dump, err := netlinkAddrDump()
if err != nil {
return nil, err
}
msgs, err := syscall.ParseNetlinkMessage(dump)
if err != nil {
return nil, err
}
// Any socket will do for SIOCGIFNAME.
sfd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
defer unix.Close(sfd)
type iface struct {
index uint32
name string
addrs []net.Addr
}
var ifaces []*iface
byIndex := make(map[uint32]*iface) // nil marks an index that vanished before we could name it
for _, m := range msgs {
if m.Header.Type != unix.RTM_NEWADDR {
continue
}
if len(m.Data) < unix.SizeofIfAddrmsg {
return nil, fmt.Errorf("short RTM_NEWADDR message: %d bytes", len(m.Data))
}
ifam := (*unix.IfAddrmsg)(unsafe.Pointer(&m.Data[0]))
i, seen := byIndex[ifam.Index]
if !seen {
name, err := interfaceNameByIndex(sfd, ifam.Index)
if err != nil && !errors.Is(err, unix.ENODEV) {
return nil, fmt.Errorf("failed to resolve name of interface %d: %w", ifam.Index, err)
}
if err == nil {
i = &iface{index: ifam.Index, name: name}
ifaces = append(ifaces, i)
}
byIndex[ifam.Index] = i
}
if i == nil {
// The interface went away between the dump and the ioctl.
continue
}
attrs, err := syscall.ParseNetlinkRouteAttr(&m)
if err != nil {
return nil, err
}
if a := newNetlinkAddr(ifam, attrs); a != nil {
i.addrs = append(i.addrs, a)
}
}
// The dump is grouped by address family. Sort by index to match net.Interfaces.
slices.SortFunc(ifaces, func(a, b *iface) int { return int(a.index) - int(b.index) })
out := make([]localInterface, 0, len(ifaces))
for _, i := range ifaces {
addrs := i.addrs
out = append(out, localInterface{
Name: i.name,
Addrs: func() ([]net.Addr, error) { return addrs, nil },
})
}
return out, nil
}
// netlinkAddrDump is syscall.NetlinkRIB(RTM_GETADDR, AF_UNSPEC) without the bind() call. The
// kernel autobinds the socket on the first send, which the sandbox permits.
func netlinkAddrDump() ([]byte, error) {
fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
defer unix.Close(fd)
const seq = 1
req := make([]byte, unix.SizeofNlMsghdr+unix.SizeofRtGenmsg)
h := (*unix.NlMsghdr)(unsafe.Pointer(&req[0]))
h.Len = uint32(len(req))
h.Type = unix.RTM_GETADDR
h.Flags = unix.NLM_F_DUMP | unix.NLM_F_REQUEST
h.Seq = seq
req[unix.SizeofNlMsghdr] = unix.AF_UNSPEC // rtgenmsg.rtgen_family
if err := unix.Sendto(fd, req, 0, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil {
return nil, os.NewSyscallError("sendto", err)
}
// Read back the port id the kernel assigned on send so replies can be matched.
lsa, err := unix.Getsockname(fd)
if err != nil {
return nil, os.NewSyscallError("getsockname", err)
}
nl, ok := lsa.(*unix.SockaddrNetlink)
if !ok {
return nil, errors.New("netlink socket returned a non-netlink local address")
}
var out []byte
buf := make([]byte, os.Getpagesize())
for {
n, _, err := unix.Recvfrom(fd, buf, 0)
if err != nil {
return nil, os.NewSyscallError("recvfrom", err)
}
if n < unix.NLMSG_HDRLEN {
return nil, fmt.Errorf("short netlink read: %d bytes", n)
}
msgs, err := syscall.ParseNetlinkMessage(buf[:n])
if err != nil {
return nil, err
}
for _, m := range msgs {
if m.Header.Seq != seq || m.Header.Pid != nl.Pid {
return nil, fmt.Errorf("netlink reply for another request: seq %d pid %d", m.Header.Seq, m.Header.Pid)
}
switch m.Header.Type {
case unix.NLMSG_DONE:
return out, nil
case unix.NLMSG_ERROR:
if len(m.Data) < 4 {
return nil, errors.New("short NLMSG_ERROR")
}
errno := *(*int32)(unsafe.Pointer(&m.Data[0]))
return nil, os.NewSyscallError("RTM_GETADDR", unix.Errno(-errno))
}
}
out = append(out, buf[:n]...)
}
}
// interfaceNameByIndex resolves an interface index with SIOCGIFNAME.
func interfaceNameByIndex(fd int, index uint32) (string, error) {
var ifr unix.Ifreq
ifr.SetUint32(index) // ifr_ifindex is in the ifr_ifru union
if err := unix.IoctlIfreq(fd, unix.SIOCGIFNAME, &ifr); err != nil {
return "", err
}
return ifr.Name(), nil
}
// newNetlinkAddr builds the address for one RTM_NEWADDR message. On a point-to-point link
// IFA_LOCAL is our side and IFA_ADDRESS the peer, so IFA_LOCAL wins when both are present.
func newNetlinkAddr(ifam *unix.IfAddrmsg, attrs []syscall.NetlinkRouteAttr) net.Addr {
var pointToPoint bool
for _, a := range attrs {
if a.Attr.Type == unix.IFA_LOCAL {
pointToPoint = true
break
}
}
for _, a := range attrs {
if pointToPoint && a.Attr.Type == unix.IFA_ADDRESS {
continue
}
switch ifam.Family {
case unix.AF_INET:
if a.Attr.Type != unix.IFA_LOCAL && a.Attr.Type != unix.IFA_ADDRESS {
continue
}
if len(a.Value) < net.IPv4len {
return nil
}
return &net.IPNet{
IP: net.IPv4(a.Value[0], a.Value[1], a.Value[2], a.Value[3]),
Mask: net.CIDRMask(int(ifam.Prefixlen), 8*net.IPv4len),
}
case unix.AF_INET6:
if a.Attr.Type != unix.IFA_LOCAL && a.Attr.Type != unix.IFA_ADDRESS {
continue
}
if len(a.Value) < net.IPv6len {
return nil
}
ifa := &net.IPNet{IP: make(net.IP, net.IPv6len), Mask: net.CIDRMask(int(ifam.Prefixlen), 8*net.IPv6len)}
copy(ifa.IP, a.Value[:net.IPv6len])
return ifa
}
}
return nil
}