Compare commits

...
Author SHA1 Message Date
JackDoan 5bcceac53c linux: Don't replace another interface's route with the same destination and metric 2026-10-15 10:01:05 -05:00
John Maguire afbdc87ed9 Update gvisor and bump to go1.27 (#1940)
gvisor's arm64 race-build compare-and-swap (pkg/sync/race_arm64.s)
loaded the new value into the register meant for the old one (R1
instead of R2), so it never succeeded. Sleepers could not commit to
parking and spun: under -race on arm64 every idle service netstack
busied about 14 cores, and TestService's static host lookup of
localhost missed its 250ms timeout and stalled for the 30s static_map
cadence. Fixed in gvisor 1b2946c4fa.

Go 1.27.2 is now the minimum because gvisor's go branch requires it.
golangci-lint is bumped to v2.14, the first release that can read
go1.27.2 export data.
2026-10-14 21:30:59 -04:00
8 changed files with 251 additions and 22 deletions

No files matched your search

+3 -3
View File
@@ -14,7 +14,7 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: '1.26'
go-version: '1.27'
check-latest: true
- name: Build
@@ -40,7 +40,7 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: '1.26'
go-version: '1.27'
check-latest: true
- name: Build
@@ -83,7 +83,7 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: '1.26'
go-version: '1.27'
check-latest: true
# GitHub holds ARNs, not credentials, and ARNs outlive a rotation
+3 -3
View File
@@ -34,7 +34,7 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: '1.26'
go-version: '1.27'
check-latest: true
- name: add hashicorp source
@@ -66,7 +66,7 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: '1.26'
go-version: '1.27'
check-latest: true
- name: add hashicorp source
@@ -92,7 +92,7 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: '1.26'
go-version: '1.27'
check-latest: true
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own
+2 -2
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: '1.26'
go-version: '1.27'
check-latest: true
- name: Smoke Docker
@@ -51,7 +51,7 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: '1.26'
go-version: '1.27'
check-latest: true
- name: build
+4 -4
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: '1.26'
go-version: '1.27'
check-latest: true
- name: Install goimports
@@ -42,7 +42,7 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v9
with:
version: v2.12
version: v2.14
test:
name: Test ${{ matrix.name }}
@@ -87,7 +87,7 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: '1.26'
go-version: '1.27'
check-latest: true
- name: Build
@@ -132,7 +132,7 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: '1.26'
go-version: '1.27'
check-latest: true
- name: Build ${{ matrix.name }}
+4 -3
View File
@@ -1,6 +1,6 @@
module github.com/slackhq/nebula
go 1.26.0
go 1.27.2
require (
dario.cat/mergo v1.0.2
@@ -33,7 +33,7 @@ require (
golang.zx2c4.com/wireguard/windows v1.0.1
google.golang.org/protobuf v1.36.12
gopkg.in/yaml.v3 v3.0.1
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980
)
require (
@@ -45,5 +45,6 @@ require (
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/vishvananda/netns v0.0.5 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect
golang.org/x/time v0.15.0 // indirect
)
+6 -4
View File
@@ -157,6 +157,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -203,8 +205,8 @@ golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -243,5 +245,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g=
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 h1:ribyrTYQ/QYeUIs8x8bVLYwGyepa7RnHLqJzegT5YYk=
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980/go.mod h1:pK/dapF5Bf3KGW7/kd5CI5IC4lYh9FMFY1qETAIGLXk=
+96 -3
View File
@@ -552,13 +552,17 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
if cidr.Addr().Is6() {
nr.Priority = 256
}
err := netlink.RouteReplace(&nr)
held, err := t.otherLinkRoutes()
if err != nil {
return err
}
err = t.replaceRoute(&nr, held)
if err != nil {
t.l.Warn("Failed to set default route MTU, retrying", "error", err, "cidr", cidr)
//retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument`
for range 2 {
time.Sleep(100 * time.Millisecond)
err = netlink.RouteReplace(&nr)
err = t.replaceRoute(&nr, held)
if err == nil {
break
} else {
@@ -578,6 +582,15 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
}
func (t *tun) addRoutes(logErrors bool) error {
held, err := t.otherLinkRoutes()
if err != nil {
if logErrors {
t.l.Error("Failed to add routes", "error", err)
return nil
}
return err
}
// Path routes
routes := *t.Routes.Load()
for _, r := range routes {
@@ -602,7 +615,7 @@ func (t *tun) addRoutes(logErrors bool) error {
nr.Priority = r.Metric
}
err := netlink.RouteReplace(&nr)
err := t.replaceRoute(&nr, held)
if err != nil {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
@@ -618,6 +631,86 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil
}
// ip6RoutePrioUser is the metric the kernel gives an IPv6 route added without one
const ip6RoutePrioUser = 1024
// routeKey is where the main table files a route. Replacing a route replaces the first one with the same key, whatever
// device it is on
type routeKey struct {
dst netip.Prefix
priority int
}
func routeKeyOf(dst *net.IPNet, family, priority int) (routeKey, bool) {
var p netip.Prefix
if dst == nil {
// A default route
if family == unix.AF_INET6 {
p = netip.PrefixFrom(netip.IPv6Unspecified(), 0)
} else {
p = netip.PrefixFrom(netip.IPv4Unspecified(), 0)
}
} else {
addr, ok := netip.AddrFromSlice(dst.IP)
if !ok {
return routeKey{}, false
}
ones, _ := dst.Mask.Size()
p = netip.PrefixFrom(addr.Unmap(), ones).Masked()
}
if priority == 0 && p.Addr().Is6() {
priority = ip6RoutePrioUser
}
return routeKey{dst: p, priority: priority}, true
}
// otherLinkRoutes returns the keys of the main table routes that are not on our device
func (t *tun) otherLinkRoutes() (map[routeKey]struct{}, error) {
routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Table: unix.RT_TABLE_MAIN}, netlink.RT_FILTER_TABLE)
// An interrupted dump may miss a route, which then gets replaced like it always used to be
if err != nil && !errors.Is(err, netlink.ErrDumpInterrupted) {
return nil, fmt.Errorf("failed to list the system routes: %w", err)
}
held := make(map[routeKey]struct{}, len(routes))
for _, r := range routes {
if r.LinkIndex == t.deviceIndex {
continue
}
if k, ok := routeKeyOf(r.Dst, r.Family, r.Priority); ok {
held[k] = struct{}{}
}
}
return held, nil
}
// replaceRoute puts nr in the main table in place of our own route with its key. Replacing would take the route of
// another device holding that key, so ours goes in behind it instead. Theirs keeps routing and the kernel falls back
// to ours once theirs is removed, like when leaving the lan that had it. A different metric decides which one wins.
func (t *tun) replaceRoute(nr *netlink.Route, held map[routeKey]struct{}) error {
family := unix.AF_INET6
if nr.Dst.IP.To4() != nil {
family = unix.AF_INET
}
k, ok := routeKeyOf(nr.Dst, family, nr.Priority)
if _, taken := held[k]; !ok || !taken {
return netlink.RouteReplace(nr)
}
// Appending doesn't update a route that is already ours, it adds a second one
ours := netlink.Route{
LinkIndex: nr.LinkIndex,
Dst: nr.Dst,
Scope: nr.Scope,
Table: nr.Table,
Priority: k.priority,
}
if err := netlink.RouteDel(&ours); err != nil && !errors.Is(err, unix.ESRCH) {
return err
}
t.l.Warn("Another interface already routes this destination at the same metric, our route is used once theirs is removed", "destination", k.dst, "metric", k.priority)
return netlink.RouteAppend(nr)
}
func (t *tun) removeRoutes(routes []Route) {
for _, r := range routes {
if !r.Install {
+133
View File
@@ -0,0 +1,133 @@
//go:build linux && !android && !e2e_testing
package overlay
import (
"net"
"net/netip"
"os"
"testing"
"github.com/DefinedNet/netlink"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
// An unsafe_route with the same destination and metric as a lan's route must not replace it. Ours waits behind theirs
// and takes over when the lan goes away. Run as root, or unprivileged with `unshare -rn`.
func TestTunRouteLeavesAnotherDevicesRoute(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating links and adding routes needs root")
}
lan := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "nebrt0"}, PeerName: "nebrt1"}
require.NoError(t, netlink.LinkAdd(lan))
t.Cleanup(func() { _ = netlink.LinkDel(lan) })
peer, err := netlink.LinkByName("nebrt1")
require.NoError(t, err)
require.NoError(t, netlink.LinkSetUp(lan))
require.NoError(t, netlink.LinkSetUp(peer))
lanAddrs := []*netlink.Addr{
{IPNet: &net.IPNet{IP: net.ParseIP("198.51.100.5"), Mask: net.CIDRMask(24, 32)}},
{IPNet: &net.IPNet{IP: net.ParseIP("2001:db8:1::5"), Mask: net.CIDRMask(64, 128)}, Flags: unix.IFA_F_NODAD},
}
for _, a := range lanAddrs {
require.NoError(t, netlink.AddrAdd(lan, a))
}
l := test.NewLogger()
c := config.NewC(l)
// The kernel puts a v4 lan route at metric 0, our default, and a v6 one at 256
require.NoError(t, c.LoadString(`
tun:
dev: nebrt2
unsafe_routes:
- route: 198.51.100.0/24
via: 10.250.0.2
- route: 2001:db8:1::/64
via: fd00:250::2
metric: 256
`))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
via := func(dst string) int {
routes, err := netlink.RouteGet(net.ParseIP(dst))
require.NoError(t, err)
require.NotEmpty(t, routes)
return routes[0].LinkIndex
}
devices := func(dst string) []int {
_, n, err := net.ParseCIDR(dst)
require.NoError(t, err)
routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Dst: n}, netlink.RT_FILTER_DST)
require.NoError(t, err)
var out []int
for _, r := range routes {
out = append(out, r.LinkIndex)
}
return out
}
for _, dst := range []string{"198.51.100.0/24", "2001:db8:1::/64"} {
assert.ElementsMatch(t, []int{lan.Index, tn.deviceIndex}, devices(dst), dst)
}
assert.Equal(t, lan.Index, via("198.51.100.9"))
assert.Equal(t, lan.Index, via("2001:db8:1::9"))
// Leaving the lan
for _, a := range lanAddrs {
require.NoError(t, netlink.AddrDel(lan, a))
}
assert.Equal(t, tn.deviceIndex, via("198.51.100.9"))
assert.Equal(t, tn.deviceIndex, via("2001:db8:1::9"))
// Dropping the unsafe_routes removes only ours
for _, a := range lanAddrs {
require.NoError(t, netlink.AddrAdd(lan, a))
}
require.NoError(t, c.ReloadConfigString("tun:\n dev: nebrt2\n"))
for _, dst := range []string{"198.51.100.0/24", "2001:db8:1::/64"} {
assert.Equal(t, []int{lan.Index}, devices(dst), dst)
}
}
// The route for our own vpn network is replaced to set its mtu, which must not take a lan's route for the same network
func TestTunVpnNetworkLeavesAnotherDevicesRoute(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating links and adding routes needs root")
}
lan := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "nebrt0"}, PeerName: "nebrt1"}
require.NoError(t, netlink.LinkAdd(lan))
t.Cleanup(func() { _ = netlink.LinkDel(lan) })
require.NoError(t, netlink.LinkSetUp(lan))
require.NoError(t, netlink.AddrAdd(lan, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP("10.250.0.5"), Mask: net.CIDRMask(24, 32)}}))
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("tun:\n dev: nebrt2\n"))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
_, n, err := net.ParseCIDR("10.250.0.0/24")
require.NoError(t, err)
routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{Dst: n}, netlink.RT_FILTER_DST)
require.NoError(t, err)
var devices []int
for _, r := range routes {
devices = append(devices, r.LinkIndex)
if r.LinkIndex == tn.deviceIndex {
assert.Equal(t, tn.DefaultMTU, r.MTU, "ours still carries the mtu")
}
}
assert.ElementsMatch(t, []int{lan.Index, tn.deviceIndex}, devices)
}