Compare commits

...
Author SHA1 Message Date
dependabot[bot] 3c3f96fb18 Bump github.com/prometheus/client_golang from 1.24.1 to 1.25.0
Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.24.1 to 1.25.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/v1.25.0/CHANGELOG.md)
- [Commits](https://github.com/prometheus/client_golang/compare/v1.24.1...v1.25.0)

---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
  dependency-version: 1.25.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-10-14 02:20:42 +00:00
Nate Brown 94372f4f51 Don't delete routes we never installed on darwin and the BSDs (#1933) 2026-10-13 16:34:38 -05:00
John Maguire 42280f04ce darwin: Assign host address to the utun and route vpn explicitly (#1931)
Tailscale with an exit node treats every interface address prefix as an
attached LAN and either claims it or excludes it, and configd then rewrites
the matching route away from nebula's utun. Interfaces holding a single
address are skipped, so assign /32 and /128 and add the network route
ourselves for both families.
2026-10-13 13:50:37 -04:00
Nate Brown 11c1917387 Warn when an unsafe_route is already routed through another interface on darwin (#1925) 2026-10-12 16:57:14 -05:00
Nate Brown 5ba7aff4b3 ssh: Only restart sshd on reload when its config or host key changed (#1923) 2026-10-12 16:56:51 -05:00
Ryan Huber 8a2de9b1b5 examples: set socket buffers and raise tun tx_queue (#1928)
* examples: set socket buffers and raise tun tx_queue

* examples: use the default tcp max window for socket buffers

* examples: note buffers are linux only, clarify tx_queue default

* tun: default tx_queue to 5000
2026-10-11 00:30:07 -04:00
Nate Brown a8c52f5c13 overlay: Handle routes Windows already has, and parse quoted route numbers (#1924) 2026-10-08 18:52:53 -05:00
Jack Doan 13af8dd6f3 Cap configured MTUs at 8937 so relayed sends never outgrow their buffers (#1899) 2026-10-08 15:41:08 -05:00
26 changed files with 1083 additions and 102 deletions

No files matched your search

+119
View File
@@ -2,6 +2,8 @@ package nebula
import (
"net/netip"
"strconv"
"sync"
"testing"
"time"
@@ -11,6 +13,7 @@ import (
ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
@@ -147,6 +150,122 @@ func TestSendNoMetricsCloseTunnelKeepsRebindEpoch(t *testing.T) {
assert.True(t, hostinfo.markOut(1), "a CloseTunnel send must not consume the rebind epoch")
}
// capturingConn is a udp.Conn that records every WriteTo.
type capturingConn struct {
udp.NoopConn
writes [][]byte
addrs []netip.AddrPort
}
func (c *capturingConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.writes = append(c.writes, append([]byte(nil), b...))
c.addrs = append(c.addrs, addr)
return nil
}
// TestSendNoMetricsViaRelay sends through a relay from the mtu sized buffer the cached packet flush uses. Payloads up
// to overlay.MaxMTU go out intact, anything bigger is dropped rather than outgrowing the buffer, which once panicked.
func TestSendNoMetricsViaRelay(t *testing.T) {
tests := []struct {
n int
sent bool
}{
{0, true},
{1300, true},
{overlay.MaxMTU, true},
{overlay.MaxMTU + 1, false}, // too big for the relay's tag
{mtu - 47, false}, // too big for the inner tag, this panicked
{9000, false}, // a full packet at tun.mtu 9000
}
for _, tt := range tests {
t.Run(strconv.Itoa(tt.n), func(t *testing.T) {
// One tunnel to the target, carried end to end, and one to the relay that wraps it
targetInit, targetResp := runTestHandshake(t)
relayInit, relayResp := runTestHandshake(t)
ci, err := newConnectionStateFromResult(targetInit)
require.NoError(t, err)
relayCI, err := newConnectionStateFromResult(relayInit)
require.NoError(t, err)
targetAddr := netip.MustParseAddr("10.0.0.2")
relayAddr := netip.MustParseAddr("10.0.0.3")
relayRemote := netip.MustParseAddrPort("192.0.2.1:4242")
relay := &Relay{Type: TerminalType, State: Established, LocalIndex: 5, RemoteIndex: 6, PeerAddr: targetAddr}
relayHI := &HostInfo{
vpnAddrs: []netip.Addr{relayAddr},
ConnectionState: relayCI,
relayState: RelayState{
relayForByAddr: map[netip.Addr]*Relay{targetAddr: relay},
relayForByIdx: map[uint32]*Relay{relay.LocalIndex: relay},
},
}
relayHI.remote.Store(&relayRemote)
// No remote of its own, so the send has to go via the relay
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{targetAddr},
ConnectionState: ci,
remoteIndexId: 7,
relayState: RelayState{
relays: []netip.Addr{relayAddr},
relayForByAddr: map[netip.Addr]*Relay{},
relayForByIdx: map[uint32]*Relay{},
},
}
l := test.NewLogger()
hm := newHostMap(l)
hm.Hosts[relayAddr] = relayHI
conn := &capturingConn{}
f := &Interface{
l: l,
hostMap: hm,
messageMetrics: &MessageMetrics{txExhausted: metrics.NewCounter()},
writers: []udp.Conn{conn},
}
f.connectionManager = &connectionManager{intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{}}
payload := make([]byte, tt.n)
for i := range payload {
payload[i] = byte(i)
}
counter := ci.messageCounter.Load()
f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, payload, make([]byte, 12), make([]byte, mtu), 0)
if !tt.sent {
assert.Empty(t, conn.writes)
assert.Equal(t, counter, ci.messageCounter.Load(), "a dropped packet must not spend a counter")
return
}
require.Len(t, conn.writes, 1)
assert.Equal(t, relayRemote, conn.addrs[0])
pkt := conn.writes[0]
// The relay authenticates the whole packet under its own header
outer := &header.H{}
require.NoError(t, outer.Parse(pkt))
assert.Equal(t, header.MessageRelay, outer.Subtype)
assert.Equal(t, relay.RemoteIndex, outer.RemoteIndex)
relayPeer, err := newConnectionStateFromResult(relayResp)
require.NoError(t, err)
require.NoError(t, relayPeer.VerifyRelay(l, outer.MessageCounter, pkt, make([]byte, 12)))
// Inside it, the target decrypts the end to end packet back to the payload
inner := pkt[header.Len : len(pkt)-relayPeer.dKey.Overhead()]
h := &header.H{}
require.NoError(t, h.Parse(inner))
assert.Equal(t, header.Message, h.Type)
assert.Equal(t, hostinfo.remoteIndexId, h.RemoteIndex)
targetPeer, err := newConnectionStateFromResult(targetResp)
require.NoError(t, err)
got, err := targetPeer.Decrypt(l, h.MessageCounter, inner, make([]byte, 12))
require.NoError(t, err)
assert.Equal(t, payload, got)
})
}
}
func TestNewConnectionStateFromResult(t *testing.T) {
initR, respR := runTestHandshake(t)
+6 -5
View File
@@ -135,12 +135,12 @@ listen:
# superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the
# default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts.
#batch: 64
# Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
# Linux only. Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
# Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
# Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide
# max, net.core.rmem_max and net.core.wmem_max
#read_buffer: 10485760
#write_buffer: 10485760
read_buffer: 3412512
write_buffer: 3412512
# On Windows only
# When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port.
@@ -262,9 +262,10 @@ tun:
drop_local_broadcast: false
# Toggles forwarding of multicast packets
drop_multicast: false
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500
tx_queue: 500
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 5000
tx_queue: 5000
# Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
# Anything above 8937, here or on a route below, is capped to 8937 with a warning
mtu: 1300
# the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable.
+4 -4
View File
@@ -16,7 +16,7 @@ require (
github.com/miekg/dns v1.1.73
github.com/miekg/pkcs11 v1.1.2
github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
github.com/prometheus/client_golang v1.24.1
github.com/prometheus/client_golang v1.25.0
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
@@ -41,9 +41,9 @@ require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/prometheus/client_model v0.6.3 // indirect
github.com/prometheus/common v0.72.0 // indirect
github.com/prometheus/procfs v0.22.0 // indirect
github.com/vishvananda/netns v0.0.5 // indirect
golang.org/x/time v0.5.0 // indirect
)
+10 -10
View File
@@ -69,8 +69,8 @@ github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFd
github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -105,24 +105,24 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0=
github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4=
github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+5
View File
@@ -24,6 +24,11 @@ type m = map[string]any
const (
Version uint8 = 1
Len = 16
// MaxOverhead is the most a payload grows on its way to the underlay.
// A relayed packet has two headers and two AEAD tags, one for the inner tunnel and another for the relay tunnel.
// Every supported cipher (today) has a 16 byte tag.
MaxOverhead = 2 * (Len + 16)
)
type MessageType uint8
+18 -8
View File
@@ -199,8 +199,8 @@ func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt tio.Packet, nb []b
}
err = tio.SegmentSuperpacket(pkt, func(seg []byte) error {
//relay header + header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305) + relay tag
scratch := sendBatch.Reserve(header.Len + header.Len + len(seg) + 16 + 16)
//relay header + header + plaintext + AEAD tag + relay tag
scratch := sendBatch.Reserve(len(seg) + header.MaxOverhead)
innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb)
if innerPacket == nil {
@@ -529,16 +529,26 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
return
}
useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid()
fullOut := out
// The encrypt appends to out in place and a relayed packet is then wrapped in place,
// so out has to fit the whole packet up front or the encrypt reallocates out from under SendVia.
// We cap the MTU of tun devices and routes we create ourselves, so only a host that set a bigger MTU than we allow gets here.
need := header.Len + len(p) + ci.eKey.Overhead()
if useRelay {
if len(out) < header.Len {
// out always has a capacity of mtu, but not always a length greater than the header.Len.
// Grow it to make sure the next operation works.
out = out[:header.Len]
// The relay tunnel runs the same cipher, and adds its own header and tag
need += header.Len + ci.eKey.Overhead()
}
if cap(out) < need {
if l, ok := logging.RateLimited(f.l, f.txOversizeLimiter); ok {
hostinfo.logger(l).Error("Dropping outbound packet, out buffer not large enough", "outCap", cap(out), "payloadLen", len(p), "relayed", useRelay)
}
return
}
fullOut := out
if useRelay {
// Save a header's worth of data at the front of the 'out' buffer.
out = out[header.Len:]
out = out[header.Len:header.Len]
}
if noiseutil.EncryptLockNeeded {
+6
View File
@@ -127,6 +127,7 @@ type Interface struct {
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
txOversizeLimiter *ratelimit.Limiter // oversized outbound packet drop log limiter
// fatalErr holds the first unexpected reader error that caused shutdown.
// nil means "no fatal error" (yet)
@@ -210,6 +211,10 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
if err != nil {
return nil, err
}
txOversizeLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
if c.routines <= 1 {
c.PinThreads = false //pinning is not useful unless there's more than one tun reader
@@ -245,6 +250,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
pinThreads: c.PinThreads,
udpTxErrLimiter: udpTxErrLimiter,
udpRelayTxErrLimiter: udpRelayTxErrLimiter,
txOversizeLimiter: txOversizeLimiter,
metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
metricTxDropped: metrics.GetOrRegisterCounter("udp.tx.dropped", nil),
+1 -3
View File
@@ -160,9 +160,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
case header.TestReply:
// No-op, useful for the Roaming and connectionManager side-effects above
case header.TestRequest:
const maxCipherOverhead = 16 //todo we use this too often, needs a real importable const
const maxOverhead = header.Len + header.Len + maxCipherOverhead + maxCipherOverhead
if maxOverhead+len(out) > len(rxc.scratch) {
if header.MaxOverhead+len(out) > len(rxc.scratch) {
// A reply that cannot fit in scratch is dropped no matter the log level.
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("dropping oversized test request", "payloadLen", len(out), "from", via)
+28 -4
View File
@@ -50,7 +50,13 @@ func (r Route) String() string {
func makeRouteTree(l *slog.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) {
routeTree := new(bart.Table[routing.Gateways])
seen := make(map[netip.Prefix]bool, len(routes))
for _, r := range routes {
if seen[r.Cidr] {
l.Warn("route is listed more than once, only the last entry's via is used", "route", r)
}
seen[r.Cidr] = true
if !allowMTU && r.MTU > 0 {
l.Warn("route MTU is not supported on this platform",
"goos", runtime.GOOS,
@@ -98,7 +104,11 @@ func parseRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
mtu, ok := rMtu.(int)
if !ok {
mtu, err = strconv.Atoi(rMtu.(string))
s, ok := rMtu.(string)
if !ok {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, rMtu)
}
mtu, err = strconv.Atoi(s)
if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, err)
}
@@ -174,7 +184,11 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
if rMtu, ok := m["mtu"]; ok {
mtu, ok = rMtu.(int)
if !ok {
mtu, err = strconv.Atoi(rMtu.(string))
s, ok := rMtu.(string)
if !ok {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, rMtu)
}
mtu, err = strconv.Atoi(s)
if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err)
}
@@ -192,10 +206,15 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
metric, ok := rMetric.(int)
if !ok {
_, err = strconv.ParseInt(rMetric.(string), 10, 32)
s, ok := rMetric.(string)
if !ok {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, rMetric)
}
n, err := strconv.ParseInt(s, 10, 32)
if err != nil {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, err)
}
metric = int(n)
}
if metric < 0 || metric > math.MaxInt32 {
@@ -248,10 +267,15 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
gatewayWeight, ok := rGatewayWeight.(int)
if !ok {
_, err = strconv.ParseInt(rGatewayWeight.(string), 10, 32)
s, ok := rGatewayWeight.(string)
if !ok {
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
}
w, err := strconv.ParseInt(s, 10, 32)
if err != nil {
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
}
gatewayWeight = int(w)
}
if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 {
+116
View File
@@ -1,13 +1,17 @@
package overlay
import (
"bytes"
"fmt"
"net/netip"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -402,3 +406,115 @@ func Test_makeMultipathUnsafeRouteTree(t *testing.T) {
routing.CalculateBucketsForGateways(expectedGateways)
assert.ElementsMatch(t, expectedGateways, r)
}
func Test_getMTU(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
// unset
assert.Equal(t, DefaultMTU, getMTU(l, c))
// the largest a relayed packet can carry in one underlay buffer
require.Equal(t, udp.MTU, MaxMTU+header.MaxOverhead)
c.Settings["tun"] = map[string]any{"mtu": MaxMTU}
assert.Equal(t, MaxMTU, getMTU(l, c))
// too big is capped
c.Settings["tun"] = map[string]any{"mtu": 9000}
assert.Equal(t, MaxMTU, getMTU(l, c))
}
func Test_getAllRoutesFromConfigCapsMTU(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
c.Settings["tun"] = map[string]any{
"routes": []any{
map[string]any{"mtu": "9000", "route": "10.0.0.0/29"},
map[string]any{"mtu": "1300", "route": "10.0.0.8/29"},
},
"unsafe_routes": []any{
map[string]any{"via": "127.0.0.1", "mtu": "9000", "route": "1.0.0.0/29"},
},
}
changed, routes, err := getAllRoutesFromConfig(l, c, []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")}, true)
require.NoError(t, err)
assert.True(t, changed)
mtus := map[string]int{}
for _, r := range routes {
mtus[r.Cidr.String()] = r.MTU
}
assert.Equal(t, map[string]int{"10.0.0.0/29": MaxMTU, "10.0.0.8/29": 1300, "1.0.0.0/29": MaxMTU}, mtus)
}
// The route tree keeps one entry per CIDR, so a CIDR listed twice only routes by the last one. Say so instead of
// letting the earlier via, or on Windows the earlier metric, quietly lose
func Test_makeRouteTree_WarnsOnDuplicate(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewLoggerWithOutput(buf)
via := func(s string) routing.Gateways {
return routing.Gateways{routing.NewGateway(netip.MustParseAddr(s), 1)}
}
cidr := netip.MustParsePrefix("10.251.1.0/24")
_, err := makeRouteTree(l, []Route{
{Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100},
{Cidr: netip.MustParsePrefix("10.251.2.0/24"), Via: via("10.250.0.2"), Install: true},
}, false)
require.NoError(t, err)
assert.Empty(t, buf.String())
tree, err := makeRouteTree(l, []Route{
{Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100},
{Cidr: cidr, Via: via("10.250.0.3"), Install: true, Metric: 200},
}, false)
require.NoError(t, err)
assert.Equal(t, 1, strings.Count(buf.String(), "route is listed more than once, only the last entry's via is used"))
gw, ok := tree.Lookup(netip.MustParseAddr("10.251.1.1"))
require.True(t, ok)
assert.Equal(t, netip.MustParseAddr("10.250.0.3"), gw[0].Addr(), "the last entry wins")
}
// A quoted metric or weight is a number written as a string, it means that number. Anything else is an error, not a
// panic
func Test_parseUnsafeRoutes_QuotedNumbers(t *testing.T) {
c := config.NewC(test.NewLogger())
n := netip.MustParsePrefix("10.0.0.1/24")
route := func(metric, weight any) {
c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{
"route": "1.0.0.0/8",
"metric": metric,
"via": []any{map[string]any{"gateway": "10.0.0.2", "weight": weight}},
}}}
}
route("100", "5")
routes, err := parseUnsafeRoutes(c, []netip.Prefix{n})
require.NoError(t, err)
require.Len(t, routes, 1)
assert.Equal(t, 100, routes[0].Metric)
assert.Equal(t, "{addr: 10.0.0.2, weight: 5}", routes[0].Via[0].String())
route(1.5, 1)
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.metric in tun.unsafe_routes is not an integer: 1.5")
route(1, true)
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry .weight in tun.unsafe_routes[1].via[1] is not an integer")
}
// An mtu that is neither a number nor a quoted one is an error, not a panic
func Test_parseRoutes_MTUNotANumber(t *testing.T) {
c := config.NewC(test.NewLogger())
n := netip.MustParsePrefix("10.0.0.1/24")
c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"route": "1.0.0.0/8", "mtu": 1500.5}}}
_, err := parseRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.mtu in tun.routes is not an integer: 1500.5")
c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"route": "1.0.0.0/8", "via": "10.0.0.2", "mtu": 1500.5}}}
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is not an integer: 1500.5")
}
+24 -2
View File
@@ -7,11 +7,27 @@ import (
"net/netip"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util"
)
const DefaultMTU = 1300
// MaxMTU is the largest tun.mtu, or route mtu, the underlay can carry.
// Every underlay buffer is udp.MTU bytes, and a relayed packet adds header.MaxOverhead to its plaintext.
const MaxMTU = udp.MTU - header.MaxOverhead
// getMTU reads tun.mtu, capping a value too big for the underlay to carry.
func getMTU(l *slog.Logger, c *config.C) int {
mtu := c.GetInt("tun.mtu", DefaultMTU)
if mtu > MaxMTU {
l.Warn("tun.mtu is too big for the underlay to carry, capping it", "mtu", mtu, "maxMTU", MaxMTU)
return MaxMTU
}
return mtu
}
type NameError struct {
Name string
Underlying error
@@ -27,7 +43,7 @@ type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix,
func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
switch {
case c.GetBool("tun.disabled", false):
tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 5000), c.GetBool("stats.message_metrics", false), l)
return tun, nil
default:
@@ -41,7 +57,7 @@ func NewFdDeviceFromConfig(fd *int) DeviceFactory {
}
}
func getAllRoutesFromConfig(c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) {
func getAllRoutesFromConfig(l *slog.Logger, c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) {
if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") {
return false, nil, nil
}
@@ -57,6 +73,12 @@ func getAllRoutesFromConfig(c *config.C, vpnNetworks []netip.Prefix, initial boo
}
routes = append(routes, unsafeRoutes...)
for i, r := range routes {
if r.MTU > MaxMTU {
l.Warn("route mtu is too big for the underlay to carry, capping it", "route", r.Cidr, "mtu", r.MTU, "maxMTU", MaxMTU)
routes[i].MTU = MaxMTU
}
}
return true, routes, nil
}
+1 -1
View File
@@ -68,7 +68,7 @@ func (t *tun) Activate() error {
}
func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil {
return err
}
+101 -17
View File
@@ -6,8 +6,10 @@ import (
"errors"
"fmt"
"log/slog"
"math/bits"
"net/netip"
"os"
"sync"
"sync/atomic"
"syscall"
"unsafe"
@@ -28,7 +30,7 @@ type tun struct {
DefaultMTU int
Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr *netroute.LinkAddr
linkAddr atomic.Pointer[netroute.LinkAddr]
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
// and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool
@@ -126,7 +128,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
f: os.NewFile(uintptr(fd), ""),
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
DefaultMTU: getMTU(l, c),
l: l,
}
@@ -167,7 +169,7 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
f: file,
Device: utunNameFromFd(deviceFd),
vpnNetworks: vpnNetworks,
DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
DefaultMTU: getMTU(l, c),
hostOwned: true,
l: l,
}
@@ -243,7 +245,7 @@ func (t *tun) Activate() error {
if linkAddr == nil {
return fmt.Errorf("unable to discover link_addr for tun interface")
}
t.linkAddr = linkAddr
t.linkAddr.Store(linkAddr)
for _, network := range t.vpnNetworks {
if network.Addr().Is4() {
@@ -269,6 +271,7 @@ func (t *tun) Activate() error {
return t.addRoutes(false)
}
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
func (t *tun) activate4(network netip.Prefix) error {
s, err := unix.Socket(
unix.AF_INET,
@@ -295,22 +298,32 @@ func (t *tun) activate4(network netip.Prefix) error {
MaskAddr: unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET,
Addr: prefixToMask(network).As4(),
Addr: [4]byte{255, 255, 255, 255}, //always make the address a /32
},
}
// On Darwin, VPNs don't edit the routing table directly.
// A network extension hands its routes to the system, and configd installs them.
// configd replaces any route already holding the same prefix, including ours.
// As an example, Tailscale with an exit node treats any interface prefix wider than one address as a LAN.
// With LAN access off, it routes each LAN into Tailscale's utun.
// With LAN access on, it excludes each LAN from its tunnel.
// configd routes an excluded prefix like its own routes would, ignoring ours, so it goes out the default route.
// Either way our route is replaced, and traffic for the vpn network no longer reaches nebula.
// A single address isn't a LAN to Tailscale, so our route is left alone.
// We route the vpn network to the tun explicitly instead.
if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun v4 address: %s", err)
}
err = addRoute(network, t.linkAddr)
if err != nil {
return err
if err := addRoute(network, t.linkAddr.Load()); err != nil {
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
}
return nil
}
// activate6 is activate4 for IPv6
func (t *tun) activate6(network netip.Prefix) error {
s, err := unix.Socket(
unix.AF_INET6,
@@ -332,25 +345,29 @@ func (t *tun) activate6(network netip.Prefix) error {
PrefixMask: unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6,
Addr: prefixToMask(network).As16(),
// Always set the netmask to be a /128. See activate4 for why we do this.
Addr: [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
},
Lifetime: addrLifetime{
// never expires
Vltime: 0xffffffff,
Pltime: 0xffffffff,
},
Flags: _IN6_IFF_NODAD,
Flags: _IN6_IFF_NODAD, // No DAD, a utun has no neighbors.
}
if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun address: %s", err)
}
if err := addRoute(network, t.linkAddr.Load()); err != nil {
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
}
return nil
}
func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil {
return err
}
@@ -394,6 +411,56 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
return routing.Gateways{}
}
func fetchRoutes() ([]netroute.Message, error) {
rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_DUMP, 0)
if err != nil {
return nil, err
}
return netroute.ParseRIB(unix.NET_RT_DUMP, rib)
}
// routeInterfaceIn returns the index of the interface prefix is routed through, or 0 if msgs has no route for exactly
// that prefix. Routes scoped to an interface are skipped, they don't stop us adding ours.
func routeInterfaceIn(msgs []netroute.Message, prefix netip.Prefix) int {
for _, msg := range msgs {
m, ok := msg.(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK || m.Flags&unix.RTF_IFSCOPE != 0 {
continue
}
if p, ok := routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK]); ok && p == prefix.Masked() {
return m.Index
}
}
return 0
}
// routePrefix returns the prefix a route covers. Host routes have no netmask.
func routePrefix(dst, mask netroute.Addr) (netip.Prefix, bool) {
switch d := dst.(type) {
case *netroute.Inet4Addr:
bits := 32
if m, ok := mask.(*netroute.Inet4Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom4(d.IP), bits), true
case *netroute.Inet6Addr:
bits := 128
if m, ok := mask.(*netroute.Inet6Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom16(d.IP), bits), true
}
return netip.Prefix{}, false
}
func maskBits(mask []byte) int {
n := 0
for _, b := range mask {
n += bits.OnesCount8(b)
}
return n
}
// Get the LinkAddr for the interface of the given name
// Is there an easier way to fetch this when we create the interface?
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
@@ -428,7 +495,14 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil
}
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
@@ -436,10 +510,16 @@ func (t *tun) addRoutes(logErrors bool) error {
continue
}
err := addRoute(r.Cidr, t.linkAddr)
err := addRoute(r.Cidr, linkAddr)
if err != nil {
if errors.Is(err, unix.EEXIST) {
t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr)
// Every route we added already exists on a reload, only warn about someone else's
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
@@ -450,6 +530,8 @@ func (t *tun) addRoutes(logErrors bool) error {
}
} else {
t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
}
}
@@ -457,16 +539,18 @@ func (t *tun) addRoutes(logErrors bool) error {
}
func (t *tun) removeRoutes(routes []Route) error {
if t.hostOwned {
linkAddr := t.linkAddr.Load()
if t.hostOwned || linkAddr == nil {
return nil
}
for _, r := range routes {
if !r.Install {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue
}
err := delRoute(r.Cidr, t.linkAddr)
err := delRoute(r.Cidr, linkAddr)
if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
} else {
+103
View File
@@ -0,0 +1,103 @@
//go:build darwin && !ios && !e2e_testing
package overlay
import (
"net"
"net/netip"
"os"
"os/exec"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Every mac routes loopback through lo0
func TestRouteInterfaceIn(t *testing.T) {
msgs, err := fetchRoutes()
require.NoError(t, err)
routeInterface := func(s string) int { return routeInterfaceIn(msgs, netip.MustParsePrefix(s)) }
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterface("127.0.0.0/8"))
assert.Equal(t, lo0.Index, routeInterface("127.0.0.1/32"), "a host route has no netmask")
assert.Equal(t, lo0.Index, routeInterface("::1/128"))
assert.Equal(t, 0, routeInterface("127.0.0.0/16"), "only an exact prefix counts")
// Most macs also list the default route scoped to their other interfaces
out, err := exec.Command("route", "-n", "get", "default").Output()
require.NoError(t, err)
var name string
for _, line := range strings.Split(string(out), "\n") {
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "interface: "); ok {
name = v
}
}
if name == "" {
t.Skip("no default route")
}
def, err := net.InterfaceByName(name)
require.NoError(t, err)
assert.Equal(t, def.Index, routeInterface("0.0.0.0/0"))
}
func TestRouteInterfaceIn_SkipsScopedCopies(t *testing.T) {
route := func(index, flags int) netroute.Message {
addrs := make([]netroute.Addr, unix.RTAX_MAX)
addrs[unix.RTAX_DST] = &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}}
addrs[unix.RTAX_NETMASK] = &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}}
return &netroute.RouteMessage{Index: index, Flags: flags, Addrs: addrs}
}
p := netip.MustParsePrefix("192.168.1.0/24")
assert.Equal(t, 7, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE), route(7, unix.RTF_UP)}, p))
assert.Equal(t, 0, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE)}, p))
}
// A reload can happen before Activate
func TestTunRoutesBeforeActivate(t *testing.T) {
tn := &tun{l: test.NewLogger()}
routes := []Route{{
Cidr: netip.MustParsePrefix("10.253.1.0/24"),
Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)},
Install: true,
}}
tn.Routes.Store(&routes)
assert.NoError(t, tn.addRoutes(true))
assert.NoError(t, tn.removeRoutes(routes))
}
// A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that
// belongs to someone else
func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating a utun and adding routes needs root")
}
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
cidr := "10.250.0.128/25"
require.NoError(t, exec.Command("route", "-q", "-n", "add", "-net", cidr, "-interface", "lo0").Run())
t.Cleanup(func() { _ = exec.Command("route", "-q", "-n", "delete", "-net", cidr).Run() })
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("tun:\n routes:\n - route: "+cidr+"\n mtu: 1300\n"))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
require.NoError(t, c.ReloadConfigString("tun: {}\n"))
msgs, err := fetchRoutes()
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
+4 -3
View File
@@ -370,7 +370,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
t := &tun{
Device: deviceName,
vpnNetworks: vpnNetworks,
MTU: c.GetInt("tun.mtu", DefaultMTU),
MTU: getMTU(l, c),
l: l,
fd: fd,
shutdownR: shutdownR,
@@ -510,7 +510,7 @@ func (t *tun) setMTU() error {
}
func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil {
return err
}
@@ -589,7 +589,8 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes {
if !r.Install {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue
}
+1 -1
View File
@@ -68,7 +68,7 @@ func (t *tun) Activate() error {
}
func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil {
return err
}
+4 -4
View File
@@ -221,7 +221,7 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
vnetHdr: vnetHdr,
offloadFlags: offloadFlags,
vpnNetworks: vpnNetworks,
TXQueueLen: c.GetInt("tun.tx_queue", 500),
TXQueueLen: c.GetInt("tun.tx_queue", 5000),
useSystemRoutes: c.GetBool("tun.use_system_route_table", false),
useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
routesFromSystem: map[netip.Prefix]routing.Gateways{},
@@ -244,7 +244,7 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
}
func (t *tun) reload(c *config.C, initial bool) error {
routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
routeChange, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil {
return err
}
@@ -256,7 +256,7 @@ func (t *tun) reload(c *config.C, initial bool) error {
// Only the mtu changed. The routes are the same, parse them again so the ones without their own mtu pick up the
// new default. Without this an mtu change would remove them all
if !routeChange {
_, routes, err = getAllRoutesFromConfig(c, t.vpnNetworks, true)
_, routes, err = getAllRoutesFromConfig(t.l, c, t.vpnNetworks, true)
if err != nil {
return err
}
@@ -276,7 +276,7 @@ func (t *tun) reload(c *config.C, initial bool) error {
oldDefaultMTU := t.DefaultMTU
oldMaxMTU := t.MaxMTU
newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU)
newDefaultMTU := getMTU(t.l, c)
newMaxMTU := newDefaultMTU
for i, r := range routes {
if r.MTU == 0 {
+4 -3
View File
@@ -99,7 +99,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
fd: fd,
Device: deviceName,
vpnNetworks: vpnNetworks,
MTU: c.GetInt("tun.mtu", DefaultMTU),
MTU: getMTU(l, c),
l: l,
}
@@ -340,7 +340,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
}
func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil {
return err
}
@@ -420,7 +420,8 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes {
if !r.Install {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue
}
+4 -3
View File
@@ -90,7 +90,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
fd: fd,
Device: deviceName,
vpnNetworks: vpnNetworks,
MTU: c.GetInt("tun.mtu", DefaultMTU),
MTU: getMTU(l, c),
l: l,
}
@@ -319,7 +319,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error {
}
func (t *tun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil {
return err
}
@@ -399,7 +399,8 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes {
if !r.Install {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue
}
+1 -1
View File
@@ -31,7 +31,7 @@ type TestTun struct {
}
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) {
_, routes, err := getAllRoutesFromConfig(c, vpnNetworks, true)
_, routes, err := getAllRoutesFromConfig(l, c, vpnNetworks, true)
if err != nil {
return nil, err
}
+77 -6
View File
@@ -4,6 +4,7 @@ package overlay
import (
"crypto"
"errors"
"fmt"
"log/slog"
"net/netip"
@@ -75,7 +76,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
t := &winTun{
Device: deviceName,
vpnNetworks: vpnNetworks,
MTU: c.GetInt("tun.mtu", DefaultMTU),
MTU: getMTU(l, c),
guid: *guid,
networkCategory: cat,
setCategory: setCat,
@@ -118,7 +119,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
}
func (t *winTun) reload(c *config.C, initial bool) error {
change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
if err != nil {
return err
}
@@ -138,7 +139,7 @@ func (t *winTun) reload(c *config.C, initial bool) error {
if !initial {
// Remove first, if the system removes a wanted route hopefully it will be re-added next
err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes))
if err != nil {
util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
}
@@ -187,17 +188,27 @@ func (t *winTun) addRoutes(logErrors bool) error {
foundDefault4 := false
carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() })
for _, r := range routes {
// A CIDR listed twice routes by its last entry, install only that one or each reload would flip between them. The
// same rule as systemRoutes
last := make(map[netip.Prefix]int, len(routes))
for i, r := range routes {
last[r.Cidr] = i
}
for i, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
if last[r.Cidr] != i {
continue
}
// A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter.
carriesV6 = carriesV6 || r.Cidr.Addr().Is6()
// Add our unsafe route as an on-link route to the nebula tun device.
err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric))
result, err := addRoute(luid, r)
if err != nil {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
@@ -207,7 +218,12 @@ func (t *winTun) addRoutes(logErrors bool) error {
return retErr
}
} else {
t.l.Info("Added route", "route", r)
switch result {
case routeAdded:
t.l.Info("Added route", "route", r)
case routeMetricUpdated:
t.l.Info("Updated route metric", "route", r)
}
}
if !foundDefault4 {
@@ -319,6 +335,61 @@ func (t *winTun) Close() error {
return t.tun.Close()
}
// systemRoutes maps each CIDR to the route that puts it in the system table. A CIDR's last entry decides, including
// one that says not to install it
func systemRoutes(routes []Route) map[netip.Prefix]Route {
out := make(map[netip.Prefix]Route, len(routes))
for _, r := range routes {
if len(r.Via) > 0 && r.Install {
out[r.Cidr] = r
} else {
delete(out, r.Cidr)
}
}
return out
}
// findRemovedSystemRoutes is the routes to delete from the system table. Windows deletes by destination and next hop,
// so only a CIDR nothing installs anymore goes, a changed metric is set on the route in place by addRoute
func findRemovedSystemRoutes(newRoutes, oldRoutes []Route) []Route {
keep := systemRoutes(newRoutes)
var removed []Route
for _, r := range systemRoutes(oldRoutes) {
if _, ok := keep[r.Cidr]; !ok {
removed = append(removed, r)
}
}
return removed
}
type routeResult int
const (
routeAdded routeResult = iota
routeMetricUpdated
routeAlreadyInstalled
)
// addRoute installs r on the interface. A reload re-adds every route to catch any the system dropped, and Windows
// refuses one it has with the same destination and next hop whatever its metric. One that is there with another
// metric, ours from before a metric change or someone else's, is brought to ours in place
func addRoute(luid winipcfg.LUID, r Route) (routeResult, error) {
nextHop := unspecifiedNextHop(r.Cidr)
err := luid.AddRoute(r.Cidr, nextHop, uint32(r.Metric))
if !errors.Is(err, windows.ERROR_OBJECT_ALREADY_EXISTS) {
return routeAdded, err
}
row, err := luid.Route(r.Cidr, nextHop)
if err != nil {
return routeAlreadyInstalled, err
}
if row.Metric != uint32(r.Metric) {
row.Metric = uint32(r.Metric)
return routeMetricUpdated, row.Set()
}
return routeAlreadyInstalled, nil
}
func unspecifiedNextHop(p netip.Prefix) netip.Addr {
if p.Addr().Is4() {
return netip.IPv4Unspecified()
+168
View File
@@ -0,0 +1,168 @@
//go:build windows && !e2e_testing
package overlay
import (
"bytes"
"fmt"
"net/netip"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/windows"
"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
)
type routeTest struct {
c *config.C
tun *winTun
luid winipcfg.LUID
log *bytes.Buffer
}
// newRouteTest makes a wintun device of its own, before installing tun.unsafe_routes from cfg
func newRouteTest(t *testing.T, cfg string) *routeTest {
t.Helper()
if !windows.GetCurrentProcessToken().IsElevated() {
t.Skip("creating a wintun device needs an elevated process")
}
if err := checkWinTunExists(); err != nil {
t.Skipf("wintun.dll is not where nebula looks for it: %v", err)
}
rt := &routeTest{log: &bytes.Buffer{}}
l := test.NewLoggerWithOutput(rt.log)
rt.c = config.NewC(l)
require.NoError(t, rt.c.LoadString(cfg))
var err error
rt.tun, err = newTun(rt.c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = rt.tun.Close() })
rt.luid = winipcfg.LUID(rt.tun.tun.LUID())
return rt
}
func (rt *routeTest) metric(t *testing.T, cidr string) uint32 {
t.Helper()
p := netip.MustParsePrefix(cidr)
row, err := rt.luid.Route(p, unspecifiedNextHop(p))
require.NoError(t, err)
return row.Metric
}
func unsafeRoutes(routes ...string) string {
var b strings.Builder
b.WriteString("tun:\n dev: nebroutetest\n unsafe_routes:\n")
// cidr[@metric][!], the ! for install: false
for _, r := range routes {
r, noInstall := strings.CutSuffix(r, "!")
cidr, metric, _ := strings.Cut(r, "@")
fmt.Fprintf(&b, " - route: %s\n via: 10.250.0.2\n", cidr)
if metric != "" {
fmt.Fprintf(&b, " metric: %s\n", metric)
}
if noInstall {
b.WriteString(" install: false\n")
}
}
return b.String()
}
// A reload that adds a route re-adds the ones already installed, Windows refusing those as already there isn't a failure
func TestWinTun_ReloadKeepsInstalledRoutes(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a))
require.NoError(t, rt.tun.Activate())
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a, b)))
assert.NotContains(t, rt.log.String(), "Failed to add route")
rt.metric(t, a)
rt.metric(t, b)
})
}
// Windows calls a route the same by destination and next hop, so a changed metric is set on the installed route in
// place rather than deleted and added, which would leave a moment with no route
func TestWinTun_ReloadMetricChangeApplies(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
require.NoError(t, rt.tun.Activate())
require.Equal(t, uint32(100), rt.metric(t, a))
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200")))
assert.NotContains(t, rt.log.String(), "Failed to add route")
assert.NotContains(t, rt.log.String(), "Removed route")
assert.Equal(t, uint32(200), rt.metric(t, a))
})
}
// A route someone else put on the device with another metric is brought to ours, at startup too
func TestWinTun_ForeignRouteGetsOurMetric(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
p := netip.MustParsePrefix(a)
require.NoError(t, rt.luid.AddRoute(p, unspecifiedNextHop(p), 50))
require.NoError(t, rt.tun.Activate())
assert.Equal(t, uint32(100), rt.metric(t, a))
assert.Contains(t, rt.log.String(), "level=INFO msg=\"Updated route metric\"")
})
}
// A CIDR listed twice is installed once, by its last entry, and reloads leave it there instead of flipping between the
// two metrics
func TestWinTun_DuplicateCidrInstallsTheLastEntry(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200"))
require.NoError(t, rt.tun.Activate())
assert.Equal(t, uint32(200), rt.metric(t, a))
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@100", a+"@200", b)))
assert.Equal(t, uint32(200), rt.metric(t, a))
assert.NotContains(t, rt.log.String(), "Updated route metric")
assert.NotContains(t, rt.log.String(), "Failed to add route")
})
}
// Dropping the duplicate that wasn't installed leaves the installed route alone. Windows deletes by destination and
// next hop, deleting the dropped one would take the live route with it
func TestWinTun_DroppingAnUninstalledDuplicateKeepsTheRoute(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200"))
require.NoError(t, rt.tun.Activate())
require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200")))
assert.NotContains(t, rt.log.String(), "Removed route")
assert.Equal(t, uint32(200), rt.metric(t, a))
})
}
// A reload ends where a fresh start with the same config would, a CIDR whose last entry says not to install it is
// removed
func TestWinTun_ReloadMatchesAFreshStart(t *testing.T) {
forFamilies(t, func(t *testing.T, a, b string) {
cidr := netip.MustParsePrefix(a)
routes := unsafeRoutes(a+"@100", a+"@100!")
fresh := newRouteTest(t, routes)
require.NoError(t, fresh.tun.Activate())
_, err := fresh.luid.Route(cidr, unspecifiedNextHop(cidr))
require.Error(t, err, "a fresh start installed it")
require.NoError(t, fresh.tun.Close())
rt := newRouteTest(t, unsafeRoutes(a+"@100"))
require.NoError(t, rt.tun.Activate())
require.NoError(t, rt.c.ReloadConfigString(routes))
_, err = rt.luid.Route(cidr, unspecifiedNextHop(cidr))
assert.Error(t, err, "the reload left it installed")
})
}
// forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4
func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) {
t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") })
t.Run("ipv6", func(t *testing.T) { f(t, "fd00:251:1::/64", "fd00:251:2::/64") })
}
+33 -13
View File
@@ -60,6 +60,20 @@ type sshDeviceInfoFlags struct {
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C) {
if c.GetBool("sshd.enabled", false) {
// Restarting the server drops every open session, the one that asked for this reload included. Only restart
// when it isn't running or its config changed. The host key file can change under the same config, new
// handshakes use the new key without a restart
if ssh.Running() && !c.HasChanged("sshd") {
key, err := loadSSHHostKey(c)
if err == nil {
err = ssh.SetHostKey(key)
}
if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err)
ssh.Stop()
}
return
}
sshRun, err := configSSH(l, ssh, c)
if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err)
@@ -74,6 +88,22 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
})
}
// loadSSHHostKey returns sshd.host_key, read from its file when it names one.
func loadSSHHostKey(c *config.C) ([]byte, error) {
hostKeyPathOrKey := c.GetString("sshd.host_key", "")
if hostKeyPathOrKey == "" {
return nil, fmt.Errorf("sshd.host_key must be provided")
}
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
return []byte(hostKeyPathOrKey), nil
}
b, err := os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
return b, nil
}
// configSSH reads the ssh info out of the passed-in Config and
// updates the passed-in SSHServer. On success, it returns a function
// that callers may invoke to run the configured ssh server. On
@@ -92,19 +122,9 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
return nil, fmt.Errorf("sshd.listen can not use port 22")
}
hostKeyPathOrKey := c.GetString("sshd.host_key", "")
if hostKeyPathOrKey == "" {
return nil, fmt.Errorf("sshd.host_key must be provided")
}
var hostKeyBytes []byte
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
hostKeyBytes = []byte(hostKeyPathOrKey)
} else {
hostKeyBytes, err = os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
hostKeyBytes, err := loadSSHHostKey(c)
if err != nil {
return nil, err
}
err = ssh.SetHostKey(hostKeyBytes)
+48 -13
View File
@@ -8,14 +8,18 @@ import (
"log/slog"
"net"
"sync"
"sync/atomic"
"github.com/armon/go-radix"
"golang.org/x/crypto/ssh"
)
type SSHServer struct {
config *ssh.ServerConfig
l *slog.Logger
// config is replaced whole when the host key changes, ssh.ServerConfig.AddHostKey only replaces a key of the same
// algorithm and appends any other
config atomic.Pointer[ssh.ServerConfig]
authenticate func(ssh.ConnMetadata, ssh.PublicKey) (*ssh.Permissions, error)
l *slog.Logger
certChecker *ssh.CertChecker
@@ -28,7 +32,12 @@ type SSHServer struct {
// List of available commands
helpCommand *Command
commands *radix.Tree
listener net.Listener
listener atomic.Pointer[net.Listener]
// runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned
runs atomic.Int32
// addr is where the newest Run listens, nil when none is
addr atomic.Pointer[net.Addr]
// ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even
// across reloads, since each Run derives a fresh child rather than reusing this one directly.
@@ -85,10 +94,8 @@ func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
},
}
s.config = &ssh.ServerConfig{
PublicKeyCallback: cc.Authenticate,
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
s.authenticate = cc.Authenticate
s.config.Store(s.newConfig())
s.RegisterCommand(&Command{
Name: "help",
@@ -107,10 +114,32 @@ func (s *SSHServer) SetHostKey(hostPrivateKey []byte) error {
return fmt.Errorf("failed to parse private key: %s", err)
}
s.config.AddHostKey(private)
config := s.newConfig()
config.AddHostKey(private)
s.config.Store(config)
return nil
}
func (s *SSHServer) newConfig() *ssh.ServerConfig {
return &ssh.ServerConfig{
PublicKeyCallback: s.authenticate,
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
}
// Addr is the address the newest Run listens on, nil when none is.
func (s *SSHServer) Addr() net.Addr {
if a := s.addr.Load(); a != nil {
return *a
}
return nil
}
// Running is true while a Run is listening.
func (s *SSHServer) Running() bool {
return s.runs.Load() > 0
}
func (s *SSHServer) ClearTrustedCAs() {
s.authLock.Lock()
s.trustedCAs = []ssh.PublicKey{}
@@ -181,10 +210,16 @@ func (s *SSHServer) Run(addr string) error {
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
// listener before this run's watcher fires, so each run must close its own listener via the local
// reference.
s.listener = listener
s.listener.Store(&listener)
s.runs.Add(1)
bound := listener.Addr()
s.addr.Store(&bound)
runCtx, cancel := context.WithCancel(s.ctx)
defer cancel()
// Deferred after cancel so they run first, a run on its way out no longer counts as listening
defer s.runs.Add(-1)
defer s.addr.CompareAndSwap(&bound, nil)
// Close the listener when this run's context is cancelled. That can come from the parent
// (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling
@@ -197,7 +232,7 @@ func (s *SSHServer) Run(addr string) error {
}
}()
s.l.Info("SSH server is listening", "sshListener", addr)
s.l.Info("SSH server is listening", "sshListener", bound)
// Run loops until there is an error
s.run(runCtx, listener)
@@ -226,7 +261,7 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
<-sessionContext.Done()
c.Close()
}()
conn, chans, reqs, err := ssh.NewServerConn(c, s.config)
conn, chans, reqs, err := ssh.NewServerConn(c, s.config.Load())
fp := ""
if conn != nil {
fp = conn.Permissions.Extensions["fp"]
@@ -264,8 +299,8 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
}
func (s *SSHServer) Stop() {
if s.listener != nil {
if err := s.listener.Close(); err != nil {
if l := s.listener.Load(); l != nil {
if err := (*l).Close(); err != nil && !errors.Is(err, net.ErrClosed) {
s.l.Warn("Failed to close the sshd listener", "error", err)
}
}
+196
View File
@@ -0,0 +1,196 @@
package nebula
import (
"crypto"
"crypto/ed25519"
"crypto/rand"
"crypto/rsa"
"encoding/pem"
"fmt"
"net"
"os"
"path/filepath"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
type sshdReloadHarness struct {
c *config.C
server *sshd.SSHServer
keyFile string
signer ssh.Signer
extra string
}
func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
t.Helper()
l := test.NewLogger()
h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key")}
h.writeHostKey(t, newEd25519Key(t))
_, userKey, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
h.signer, err = ssh.NewSignerFromKey(userKey)
require.NoError(t, err)
h.c = config.NewC(l)
require.NoError(t, h.c.LoadString(h.config()))
h.server, err = sshd.NewSSHServer(t.Context(), l)
require.NoError(t, err)
wireSSHReload(l, h.server, h.c)
run, err := configSSH(l, h.server, h.c)
require.NoError(t, err)
go run()
t.Cleanup(h.server.Stop)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
return h
}
func newEd25519Key(t *testing.T) crypto.Signer {
_, key, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
return key
}
// writeHostKey replaces the host key file and returns the public key a client should now see
func (h *sshdReloadHarness) writeHostKey(t *testing.T, key crypto.Signer) ssh.PublicKey {
t.Helper()
block, err := ssh.MarshalPrivateKey(key, "")
require.NoError(t, err)
require.NoError(t, os.WriteFile(h.keyFile, pem.EncodeToMemory(block), 0o600))
pub, err := ssh.NewPublicKey(key.Public())
require.NoError(t, err)
return pub
}
func (h *sshdReloadHarness) config() string {
return fmt.Sprintf("sshd:\n enabled: true\n listen: 127.0.0.1:0\n host_key: %s\n authorized_users:\n - user: test\n keys: [%q]\n%s",
h.keyFile, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())), h.extra)
}
// dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys
// the client accepts, none means any
func (h *sshdReloadHarness) dial(hostKeyAlgorithms ...string) (*ssh.Client, ssh.PublicKey, error) {
addr := h.server.Addr()
if addr == nil {
return nil, nil, fmt.Errorf("not listening")
}
var hostKey ssh.PublicKey
c, err := ssh.Dial("tcp", addr.String(), &ssh.ClientConfig{
User: "test",
Auth: []ssh.AuthMethod{ssh.PublicKeys(h.signer)},
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
hostKey = key
return nil
},
HostKeyAlgorithms: hostKeyAlgorithms,
Timeout: time.Second,
})
return c, hostKey, err
}
func (h *sshdReloadHarness) connect(t *testing.T) (*ssh.Client, ssh.PublicKey) {
t.Helper()
var client *ssh.Client
var hostKey ssh.PublicKey
require.Eventually(t, func() bool {
var err error
client, hostKey, err = h.dial()
return err == nil
}, 5*time.Second, 20*time.Millisecond)
t.Cleanup(func() { _ = client.Close() })
return client, hostKey
}
// dropped is true when the server closed the client's connection within wait
func dropped(client *ssh.Client, wait time.Duration) bool {
done := make(chan struct{})
go func() {
_ = client.Wait()
close(done)
}()
select {
case <-done:
return true
case <-time.After(wait):
return false
}
}
// Restarting sshd drops every open session, the one that asked for the reload included. A reload that touches nothing
// sshd reads leaves it alone
func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
require.NoError(t, h.c.ReloadConfigString(h.config()+"punchy:\n punch: true\n"))
assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted")
}
func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
h.extra = " trusted_cas: []\n"
require.NoError(t, h.c.ReloadConfigString(h.config()))
assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied")
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// The config can stay the same while the host key file it names changes. New connections get the new key, open
// sessions are left alone
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
want := h.writeHostKey(t, newEd25519Key(t))
require.NoError(t, h.c.ReloadConfigString(h.config()))
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal(), "a new host key was not applied")
assert.False(t, dropped(client, 500*time.Millisecond), "a new host key restarted sshd")
}
// A host key of another algorithm replaces the old one rather than being offered next to it
func TestSSHDReload_HostKeyAlgorithmChange(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
want := h.writeHostKey(t, rsaKey)
require.NoError(t, h.c.ReloadConfigString(h.config()))
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal())
_, _, err = h.dial(ssh.KeyAlgoED25519)
assert.Error(t, err, "the old ed25519 host key is still offered")
}
// A server that isn't running, say its port was busy, is tried again on every reload as before
func TestSSHDReload_NotRunningRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
h.server.Stop()
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
require.NoError(t, h.c.ReloadConfigString(h.config()))
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// A host key that no longer loads stops sshd, the same as a config that doesn't
func TestSSHDReload_BadHostKeyFileStops(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
require.NoError(t, os.WriteFile(h.keyFile, []byte("not a key"), 0o600))
require.NoError(t, h.c.ReloadConfigString(h.config()))
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
}
+1 -1
View File
@@ -30,7 +30,7 @@ func procyield(cycles uint32)
const (
packetsPerRing = 1024
// Caps tun.mtu at MTU-32 direct, MTU-64 relayed, unenforced anywhere else. 17.6MB page locked per socket.
// Fits a relayed packet at overlay.MaxMTU, which tun.mtu and route mtus are capped to. 17.6MB page locked per socket.
bytesPerPacket = MTU
receiveSpins = 15
)