Compare commits

..
Author SHA1 Message Date
Nate Brown a274c7b9bd Keep static_host_map DNS results when a lookup fails 2026-10-21 09:28:03 -05:00
John Maguire 9a5bdb508e Apply remote_allow_list before capping addrs at MaxRemotes (#1985)
smoke-extra / freebsd-amd64 (push) Failing after 16s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 17s
smoke-extra / netbsd-amd64 (push) Failing after 17s
smoke-extra / openbsd-amd64 (push) Failing after 17s
smoke-extra / linux-386 (push) Failing after 18s
smoke / Run multi node smoke test (push) Failing after 1m43s
Build and test / Static checks (push) Successful in 48s
Build and test / Test linux (push) Failing after 1m28s
Build and test / Test linux-pkcs11 (push) Failing after 2m18s
Build and test / Test linux-boringcrypto (push) Failing after 2m59s
Build and test / Test linux-fips140 (push) Failing after 3m3s
Build and test / Cross-build linux-arm (push) Successful in 3m9s
Build and test / Cross-build linux-mips (push) Successful in 3m56s
Build and test / Cross-build linux-other (push) Successful in 3m15s
Build and test / Cross-build windows (push) Successful in 1m5s
Build and test / Cross-build freebsd (push) Successful in 1m36s
Build and test / Cross-build netbsd (push) Successful in 1m34s
Build and test / Cross-build openbsd (push) Successful in 1m33s
Build and test / Cross-build mobile (push) Successful in 3m34s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
smoke-extra / Run windows smoke test (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
unlockedSetV4/V6 truncated the reported list to MaxRemotes and only then
ran the allow list check. A host reporting more than MaxRemotes addrs
(docker bridges, temporary v6 addrs, etc) could have every allowed addr
discarded if the denied ones sorted first. Filter the full list and stop
once MaxRemotes allowed addrs are collected. This affects lighthouses
processing HostUpdateNotification and clients processing HostQueryReply.
2026-10-20 22:08:55 -04:00
Nate Brown 48ba879198 Wait for readers to exit before closing their fds on linux (#1979)
smoke-extra / freebsd-amd64 (push) Failing after 16s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 17s
smoke-extra / netbsd-amd64 (push) Failing after 17s
smoke-extra / openbsd-amd64 (push) Failing after 16s
smoke-extra / linux-386 (push) Failing after 15s
smoke / Run multi node smoke test (push) Failing after 1m40s
Build and test / Static checks (push) Successful in 47s
Build and test / Test linux (push) Failing after 1m52s
Build and test / Test linux-pkcs11 (push) Failing after 2m7s
Build and test / Test linux-boringcrypto (push) Failing after 3m3s
Build and test / Test linux-fips140 (push) Failing after 2m57s
Build and test / Cross-build linux-arm (push) Successful in 3m2s
Build and test / Cross-build linux-mips (push) Successful in 3m42s
Build and test / Cross-build linux-other (push) Successful in 3m12s
Build and test / Cross-build windows (push) Successful in 1m4s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m32s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m24s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-20 14:27:31 -04:00
brad-defined fb2c153dfa Relays always speak directly with peers; enforced RelayRequest msgs (#1980)
* Relays always speak directly with peers; enforce this in RelayRequest messages
2026-10-20 14:00:46 -04:00
omrzk f552cebc51 handshake: lower log level for blocklisted certificates (#1969)
smoke-extra / freebsd-amd64 (push) Failing after 23s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 14s
smoke-extra / netbsd-amd64 (push) Failing after 12s
smoke-extra / openbsd-amd64 (push) Failing after 11s
smoke-extra / linux-386 (push) Failing after 13s
smoke / Run multi node smoke test (push) Failing after 1m42s
Build and test / Static checks (push) Successful in 2m15s
Build and test / Test linux (push) Failing after 2m17s
Build and test / Test linux-pkcs11 (push) Failing after 2m10s
Build and test / Test linux-boringcrypto (push) Failing after 2m58s
Build and test / Test linux-fips140 (push) Failing after 2m54s
Build and test / Cross-build linux-arm (push) Successful in 3m6s
Build and test / Cross-build linux-mips (push) Successful in 3m46s
Build and test / Cross-build linux-other (push) Successful in 3m10s
Build and test / Cross-build windows (push) Successful in 1m3s
Build and test / Cross-build freebsd (push) Successful in 1m33s
Build and test / Cross-build netbsd (push) Successful in 1m34s
Build and test / Cross-build openbsd (push) Successful in 1m33s
Build and test / Cross-build mobile (push) Successful in 3m28s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-20 11:32:03 -04:00
Nate Brown b04d71d96a Compare a firewall reload with what the running firewall was built from (#1955) 2026-10-20 10:37:48 -04:00
Jack Doan cacfa8fac4 darwin: Add an unsafe_route once the route holding its prefix is removed (#1942) 2026-10-20 09:37:20 -05:00
Matt RichardsonandClaude Fable 5.1 8a4bd8b10f Report ssh command failures to the surface (#1959)
smoke-extra / freebsd-amd64 (push) Failing after 14s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 13s
smoke-extra / netbsd-amd64 (push) Failing after 14s
smoke-extra / openbsd-amd64 (push) Failing after 15s
smoke-extra / linux-386 (push) Failing after 14s
smoke / Run multi node smoke test (push) Failing after 1m46s
Build and test / Static checks (push) Successful in 2m38s
Build and test / Test linux (push) Failing after 1m12s
Build and test / Test linux-pkcs11 (push) Failing after 2m6s
Build and test / Test linux-boringcrypto (push) Failing after 3m0s
Build and test / Test linux-fips140 (push) Failing after 2m56s
Build and test / Cross-build linux-arm (push) Successful in 3m10s
Build and test / Cross-build linux-mips (push) Successful in 3m55s
Build and test / Cross-build linux-other (push) Successful in 3m15s
Build and test / Cross-build windows (push) Successful in 1m5s
Build and test / Cross-build freebsd (push) Successful in 1m37s
Build and test / Cross-build netbsd (push) Successful in 1m36s
Build and test / Cross-build openbsd (push) Successful in 1m35s
Build and test / Cross-build mobile (push) Successful in 3m32s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
Report ssh command failures to the surface (#1959)

The ssh debug commands told the user when something went wrong but
never told the surface that ran them, so `ssh host cmd` always exited 0
and a few failures were not reported at all. A failed command now
returns its error, a failure the user has already heard about is a
commands.Failure, and the ssh exec path exits 1 for either. The command
bodies also lose their ssh prefixes and the duplication they carried
over from ssh.go; both goldens pin that nothing else changed.

- Dispatch returns the callback's error, or shlex's; the ssh session
  logs it at debug, and not at all for a Failure the user already saw
- commands.Fail writes a message and returns it as a Failure in one step;
  an unknown command, `help` for a command that is not there, a failed
  reload and an unbalanced quote are failures
- ssh exec exit-status is 1 for a command, flag or failure error, 0
  otherwise; the e2e golden records the exit status for every case
- Clone rebinds help to the copy, so an ssh session's help lists logout
  and `help logout` and `logout -h` answer; sshd.LogoutCommand is the
  session's logout, exported for tests
- "did not understand" names the first word only, the one looked up;
  DispatchArgs takes just the tokens
- log-format writes its refusal to the user, as log-level does
- print-relays emits the relay error as a string and omits the key when
  there is none, where it printed {} or "Error":null
- list-hostmap and list-lighthouse-addrmap -json return the encode
  error; print-cert reports a certificate that will not marshal
- a flags type that is not the one registered is an internal error, not
  a silent success
- command functions and flag types are <name>Command and <name>Flags;
  parseVpnAddr and listHostMapFlagSet replace the repeated parse block
  and flag set; close-tunnel sends through sendCloseTunnel

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-19 10:48:26 -04:00
Matt RichardsonandClaude Fable 5.1 b426843f88 Extract the ssh debug commands into a transport neutral commands package (#1958)
Migrate commands from ssh into a transport neutral package

 - add golden tests validating that output remains unchanged
 - move commands to commands.go

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-19 10:25:28 -04:00
Wade Simmons cda3710d5a fix error return in handleOutsideMessagePacket (#1970)
smoke-extra / freebsd-amd64 (push) Failing after 17s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 16s
smoke-extra / netbsd-amd64 (push) Failing after 16s
smoke-extra / openbsd-amd64 (push) Failing after 16s
smoke-extra / linux-386 (push) Failing after 17s
smoke / Run multi node smoke test (push) Failing after 1m39s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m18s
Build and test / Test linux-pkcs11 (push) Failing after 2m5s
Build and test / Test linux-boringcrypto (push) Failing after 3m1s
Build and test / Test linux-fips140 (push) Failing after 2m55s
Build and test / Cross-build linux-arm (push) Successful in 3m0s
Build and test / Cross-build linux-mips (push) Successful in 3m42s
Build and test / Cross-build linux-other (push) Successful in 3m5s
Build and test / Cross-build windows (push) Successful in 1m2s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m31s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m21s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
Introduced by #1961, we should still return early for invalid packets
2026-10-17 11:09:25 -04:00
John Maguire c26871fe08 service: Close listeners and tear down the netstack on shutdown (#1954) 2026-10-17 11:08:29 -04:00
Caleb Jasik 7daacc6a48 overlay/darwin: test tun framing and zero-alloc Read/Write (#1906)
smoke-extra / freebsd-amd64 (push) Failing after 16s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 16s
smoke-extra / netbsd-amd64 (push) Failing after 17s
smoke-extra / openbsd-amd64 (push) Failing after 17s
smoke-extra / linux-386 (push) Failing after 16s
smoke / Run multi node smoke test (push) Failing after 1m41s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m47s
Build and test / Test linux-pkcs11 (push) Failing after 2m6s
Build and test / Test linux-boringcrypto (push) Failing after 2m57s
Build and test / Test linux-fips140 (push) Failing after 2m53s
Build and test / Cross-build linux-arm (push) Successful in 3m7s
Build and test / Cross-build linux-mips (push) Successful in 3m55s
Build and test / Cross-build linux-other (push) Successful in 3m13s
Build and test / Cross-build windows (push) Successful in 1m4s
Build and test / Cross-build freebsd (push) Successful in 1m35s
Build and test / Cross-build netbsd (push) Successful in 1m35s
Build and test / Cross-build openbsd (push) Successful in 1m34s
Build and test / Cross-build mobile (push) Successful in 3m26s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-16 18:12:58 -05:00
John Maguire 428dad94b3 Reject v2 cert names and groups at signing the parser rejects (#1967)
smoke-extra / freebsd-amd64 (push) Failing after 18s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 18s
smoke-extra / netbsd-amd64 (push) Failing after 16s
smoke-extra / openbsd-amd64 (push) Failing after 17s
smoke-extra / linux-386 (push) Failing after 17s
smoke / Run multi node smoke test (push) Failing after 1m41s
Build and test / Static checks (push) Successful in 45s
Build and test / Test linux (push) Failing after 1m19s
Build and test / Test linux-pkcs11 (push) Failing after 2m5s
Build and test / Test linux-boringcrypto (push) Failing after 2m56s
Build and test / Test linux-fips140 (push) Failing after 2m51s
Build and test / Cross-build linux-arm (push) Successful in 3m0s
Build and test / Cross-build linux-mips (push) Successful in 3m43s
Build and test / Cross-build linux-other (push) Successful in 3m7s
Build and test / Cross-build windows (push) Successful in 1m2s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m33s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m23s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
2026-10-16 18:05:51 -04:00
74 changed files with 4643 additions and 4864 deletions

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
# Goldens are compared byte for byte, so they must come out of git the way they went in.
*.golden -text
+2
View File
@@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- Signing a v2 certificate with an empty name, a name longer than 253 bytes, or an empty group now fails.
These certificates were signed successfully but could not be loaded.
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
+10 -1
View File
@@ -389,7 +389,16 @@ func (c *certificateV2) fromTBSCertificate(t *TBSCertificate) error {
}
func (c *certificateV2) validate() error {
// Empty names are allowed
// Mirror the name and group rules in unmarshalDetails so signing fails instead of producing an unparseable certificate
if c.details.name == "" || len(c.details.name) > MaxNameLength {
return NewErrInvalidCertificateProperties("name must be between 1 and %d bytes", MaxNameLength)
}
for _, group := range c.details.groups {
if group == "" {
return NewErrInvalidCertificateProperties("groups may not be empty")
}
}
if len(c.publicKey) == 0 {
return ErrInvalidPublicKey
+45
View File
@@ -6,6 +6,7 @@ import (
"crypto/elliptic"
"crypto/rand"
"net/netip"
"strings"
"testing"
"time"
@@ -135,3 +136,47 @@ func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
assert.True(t, normie)
}
}
func TestCertificateV2_SignRejectsUnparseable(t *testing.T) {
_, priv, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
var invalid *ErrInvalidCertificateProperties
newTBS := func() TBSCertificate {
return TBSCertificate{
Version: Version2,
Name: "testing",
Networks: []netip.Prefix{mustParsePrefixUnmapped("10.1.1.1/24")},
Groups: []string{"test-group1"},
NotBefore: time.Now().Add(-time.Minute),
NotAfter: time.Now().Add(time.Minute),
PublicKey: []byte("1234567890abcedfghij1234567890ab"),
Curve: Curve_CURVE25519,
IsCA: true,
}
}
tbs := newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength)
c, err := tbs.Sign(nil, Curve_CURVE25519, priv)
require.NoError(t, err)
b, err := c.Marshal()
require.NoError(t, err)
_, err = unmarshalCertificateV2(b, nil, Curve_CURVE25519)
require.NoError(t, err)
tbs = newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength+1)
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Name = ""
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Groups = []string{"test-group1", ""}
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
}
+710
View File
@@ -0,0 +1,710 @@
package nebula
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"log/slog"
"maps"
"net/netip"
"sort"
"strings"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/logging"
)
// The commands nebula serves. attachCommands builds the one registry every surface
// reads. See the commands package for what a Command is.
type listHostMapFlags struct {
Json bool
Pretty bool
ByIndex bool
}
// listHostMapFlagSet is the flag set list-hostmap and list-pending-hostmap share.
func listHostMapFlagSet() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := listHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table")
return fl, &s
}
type printCertFlags struct {
Json bool
Pretty bool
Raw bool
}
type printTunnelFlags struct {
Pretty bool
}
type changeRemoteFlags struct {
Address string
}
type closeTunnelFlags struct {
LocalOnly bool
}
type createTunnelFlags struct {
Address string
}
type deviceInfoFlags struct {
Json bool
Pretty bool
}
func attachCommands(l *slog.Logger, c *config.C, reg *commands.Registry, f *Interface) {
reg.Register(&commands.Command{
Name: commands.ListHostmap,
ShortDescription: "List all known previously connected hosts",
Flags: listHostMapFlagSet,
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listHostMapCommand(f.hostMap, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.ListPendingHostmap,
ShortDescription: "List all handshaking hosts",
Flags: listHostMapFlagSet,
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listHostMapCommand(f.handshakeManager, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.ListLighthouseAddrmap,
ShortDescription: "List all lighthouse map entries",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := listHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listLighthouseAddrmapCommand(f.lightHouse, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.Reload,
ShortDescription: "Reloads configuration from disk, same as sending HUP to the process",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return reloadCommand(l, c, w)
},
})
reg.Register(&commands.Command{
Name: commands.LogLevel,
ShortDescription: "Gets or sets the current log level",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return logLevelCommand(l, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.LogFormat,
ShortDescription: "Gets or sets the current log format",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return logFormatCommand(l, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.Version,
ShortDescription: "Prints the currently running version of nebula",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return versionCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.DeviceInfo,
ShortDescription: "Prints information about the network device.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := deviceInfoFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return deviceInfoCommand(f, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintCert,
ShortDescription: "Prints the current certificate being used or the certificate for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printCertFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.Raw, "raw", false, "raw prints the PEM encoded certificate, not compatible with -json or -pretty")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printCertCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintTunnel,
ShortDescription: "Prints json details about a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintRelays,
ShortDescription: "Prints json details about all relay info",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printRelaysCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.ChangeRemote,
ShortDescription: "Changes the remote address used in the tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := changeRemoteFlags{}
fl.StringVar(&s.Address, "address", "", "The new remote address, ip:port")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return changeRemoteCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.CloseTunnel,
ShortDescription: "Closes a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := closeTunnelFlags{}
fl.BoolVar(&s.LocalOnly, "local-only", false, "Disables notifying the remote that the tunnel is shutting down")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return closeTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.CreateTunnel,
ShortDescription: "Creates a tunnel for the provided vpn address",
Help: "The lighthouses will be queried for real addresses but you can provide one as well.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := createTunnelFlags{}
fl.StringVar(&s.Address, "address", "", "Optionally provide a real remote address, ip:port ")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return createTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.QueryLighthouse,
ShortDescription: "Query the lighthouses for the provided vpn address",
Help: "This command is asynchronous. Only currently known udp addresses will be printed.",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return queryLighthouseCommand(f, fs, a, w)
},
})
}
func listHostMapCommand(hl controlHostLister, a any, w commands.StringWriter) error {
fs, ok := a.(*listHostMapFlags)
if !ok {
return fmt.Errorf("internal error: expected *listHostMapFlags but was %T", a)
}
var hm []ControlHostInfo
if fs.ByIndex {
hm = listHostMapIndexes(hl)
} else {
hm = listHostMapHosts(hl)
}
sort.Slice(hm, func(i, j int) bool {
return hm[i].VpnAddrs[0].Compare(hm[j].VpnAddrs[0]) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
if err := js.Encode(hm); err != nil {
return err
}
} else {
for _, v := range hm {
err := w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddrs, v.RemoteAddrs))
if err != nil {
return err
}
}
}
return nil
}
func listLighthouseAddrmapCommand(lightHouse *LightHouse, a any, w commands.StringWriter) error {
fs, ok := a.(*listHostMapFlags)
if !ok {
return fmt.Errorf("internal error: expected *listHostMapFlags but was %T", a)
}
type lighthouseInfo struct {
VpnAddr string `json:"vpnAddr"`
Addrs *CacheMap `json:"addrs"`
}
lightHouse.RLock()
addrMap := make([]lighthouseInfo, len(lightHouse.addrMap))
x := 0
for k, v := range lightHouse.addrMap {
addrMap[x] = lighthouseInfo{
VpnAddr: k.String(),
Addrs: v.CopyCache(),
}
x++
}
lightHouse.RUnlock()
sort.Slice(addrMap, func(i, j int) bool {
return strings.Compare(addrMap[i].VpnAddr, addrMap[j].VpnAddr) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
if err := js.Encode(addrMap); err != nil {
return err
}
} else {
for _, v := range addrMap {
b, err := json.Marshal(v.Addrs)
if err != nil {
return err
}
err = w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddr, string(b)))
if err != nil {
return err
}
}
}
return nil
}
// parseVpnAddr parses the vpn address a command takes as its first operand, failing to w
// when there is none or it does not parse. noun is how the command's messages name it,
// "vpn address" or "vpn addr", which the output golden pins per command.
func parseVpnAddr(a []string, noun string, w commands.StringWriter) (netip.Addr, error) {
if len(a) == 0 {
return netip.Addr{}, commands.Fail(w, "No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return netip.Addr{}, commands.Fail(w, fmt.Sprintf("The provided %s could not be parsed: %s", noun, a[0]))
}
return vpnAddr, nil
}
func versionCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
return w.WriteLine(ifce.version)
}
func queryLighthouseCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
var cm *CacheMap
rl := ifce.lightHouse.Query(vpnAddr)
if rl != nil {
cm = rl.CopyCache()
}
return json.NewEncoder(w.GetWriter()).Encode(cm)
}
func closeTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*closeTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *closeTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
if !flags.LocalOnly {
ifce.sendCloseTunnel(hostInfo)
}
ifce.closeTunnel(hostInfo)
return w.WriteLine("Closed")
}
func createTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*createTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *createTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return commands.Fail(w, "Tunnel already exists")
}
hostInfo = ifce.handshakeManager.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return commands.Fail(w, "Tunnel already handshaking")
}
var addr netip.AddrPort
if flags.Address != "" {
addr, err = netip.ParseAddrPort(flags.Address)
if err != nil {
return commands.Fail(w, "Address could not be parsed")
}
}
// Set the remote from the callback, which StartHandshake runs under the handshake manager lock
// before the hostinfo reaches the handshake loop or the static-host trigger, so the address is
// in the RemoteList before the first handshake is sent.
ifce.handshakeManager.StartHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
if addr.IsValid() {
hh.hostinfo.SetRemote(addr)
}
})
return w.WriteLine("Created")
}
func changeRemoteCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*changeRemoteFlags)
if !ok {
return fmt.Errorf("internal error: expected *changeRemoteFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
if flags.Address == "" {
return commands.Fail(w, "No address was provided")
}
addr, err := netip.ParseAddrPort(flags.Address)
if err != nil {
return commands.Fail(w, "Address could not be parsed")
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
hostInfo.SetRemote(addr)
return w.WriteLine("Changed")
}
func logLevelCommand(l *slog.Logger, fs any, a []string, w commands.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level
SetLevel(slog.Level)
})
if !ok {
return commands.Fail(w, "Log level is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
level, err := logging.ParseLevel(strings.ToLower(a[0]))
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unknown log level %s. Possible log levels: trace, debug, info, warn, error", a))
}
ctrl.SetLevel(level)
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
func logFormatCommand(l *slog.Logger, fs any, a []string, w commands.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetFormat() string
SetFormat(string) error
})
if !ok {
return commands.Fail(w, "Log format is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
if err := ctrl.SetFormat(strings.ToLower(a[0])); err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to set log format: %s", err))
}
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
func printCertCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printCertFlags)
if !ok {
return fmt.Errorf("internal error: expected *printCertFlags but was %T", fs)
}
cert := ifce.pki.getCertState().GetDefaultCertificate()
if len(a) > 0 {
vpnAddr, err := parseVpnAddr(a, "vpn addr", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
cert = hostInfo.GetCert().Certificate
}
if args.Json || args.Pretty {
b, err := cert.MarshalJSON()
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
if args.Pretty {
buf := new(bytes.Buffer)
if err := json.Indent(buf, b, "", " "); err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
b = buf.Bytes()
}
return w.WriteBytes(b)
}
if args.Raw {
b, err := cert.MarshalPEM()
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
return w.WriteBytes(b)
}
return w.WriteLine(cert.String())
}
func printRelaysCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *printTunnelFlags but was %T", fs)
}
ifce.hostMap.RLock()
relays := maps.Clone(ifce.hostMap.Relays)
ifce.hostMap.RUnlock()
type RelayFor struct {
Error string `json:",omitempty"`
Type string
State string
PeerAddr netip.Addr
LocalIndex uint32
RemoteIndex uint32
RelayedThrough []netip.Addr
}
type RelayOutput struct {
NebulaAddr netip.Addr
RelayForAddrs []RelayFor
}
type CmdOutput struct {
Relays []*RelayOutput
}
co := CmdOutput{}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
for k, v := range relays {
ro := RelayOutput{NebulaAddr: v.vpnAddrs[0]}
co.Relays = append(co.Relays, &ro)
relayHI := ifce.hostMap.QueryVpnAddr(v.vpnAddrs[0])
if relayHI == nil {
ro.RelayForAddrs = append(ro.RelayForAddrs, RelayFor{Error: "could not find hostinfo"})
continue
}
for _, vpnAddr := range relayHI.relayState.CopyRelayForIps() {
rf := RelayFor{}
r, ok := relayHI.relayState.GetRelayForByAddr(vpnAddr)
if ok {
t := ""
switch r.Type {
case ForwardingType:
t = "forwarding"
case TerminalType:
t = "terminal"
default:
t = "unknown"
}
s := ""
switch r.State {
case Requested:
s = "requested"
case Established:
s = "established"
default:
s = "unknown"
}
rf.LocalIndex = r.LocalIndex
rf.RemoteIndex = r.RemoteIndex
rf.PeerAddr = r.PeerAddr
rf.Type = t
rf.State = s
if rf.LocalIndex != k {
rf.Error = fmt.Sprintf("hostmap LocalIndex '%v' does not match RelayState LocalIndex", k)
}
}
relayedHI := ifce.hostMap.QueryVpnAddr(vpnAddr)
if relayedHI != nil {
rf.RelayedThrough = append(rf.RelayedThrough, relayedHI.relayState.CopyRelayIps()...)
}
ro.RelayForAddrs = append(ro.RelayForAddrs, rf)
}
}
return enc.Encode(co)
}
func printTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *printTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn addr", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.GetPreferredRanges()))
}
func deviceInfoCommand(ifce *Interface, fs any, w commands.StringWriter) error {
flags, ok := fs.(*deviceInfoFlags)
if !ok {
return fmt.Errorf("internal error: expected flags to be deviceInfoFlags but was %+v", fs)
}
data := struct {
Name string `json:"name"`
Cidr []netip.Prefix `json:"cidr"`
}{
Name: ifce.inside.Name(),
Cidr: make([]netip.Prefix, len(ifce.inside.Networks())),
}
copy(data.Cidr, ifce.inside.Networks())
if flags.Json || flags.Pretty {
js := json.NewEncoder(w.GetWriter())
if flags.Pretty {
js.SetIndent("", " ")
}
return js.Encode(data)
} else {
return w.WriteLine(fmt.Sprintf("name=%v cidr=%v", data.Name, data.Cidr))
}
}
// reloadCommand reloads the config and writes the result to the client. It reloads even if the client has gone away
func reloadCommand(l *slog.Logger, c *config.C, w commands.StringWriter) error {
writeErr := w.WriteLine("Reloading config")
res := c.ReloadConfig()
res.Log(l)
if err := w.WriteLine(res.String()); writeErr == nil {
writeErr = err
}
if writeErr != nil {
return writeErr
}
if res.Err != nil || res.Failed {
return &commands.Failure{Msg: res.String()}
}
return nil
}
+83
View File
@@ -0,0 +1,83 @@
package commands
import (
"flag"
)
// CommandFlags is called before help or command execution to parse command line flags.
// It should return a flag.FlagSet instance and a pointer to the struct that will contain parsed flags
type CommandFlags func() (*flag.FlagSet, any)
// CommandCallback is called when your command should execute.
// fs will be a pointer to the struct provided by Command.Flags callback, if there was one. -h and -help are reserved
// and handled automatically for you.
// a will be any unconsumed arguments, if no Command.Flags was available this will be all the flags passed in.
// w is the writer to use when sending messages back to the client.
// An error returned by the callback is handed back to whoever dispatched the command, which reports a failure: exit
// status 1 over ssh. The error is not shown to the user. For a failure the user caused or can act on, write the
// message and return it as a *Failure, which Fail does in one step; the surface then reports the failure without
// logging what the user was already told. Any other error is logged by the surface as well.
type CommandCallback func(fs any, a []string, w StringWriter) error
type Command struct {
Name Name
ShortDescription string
Help string
Flags CommandFlags
Callback CommandCallback
}
// Failure is the error a command returns for a failure it has already explained to the user:
// the message it wrote is the whole story. A surface reports it as a failure and does not
// log it.
type Failure struct {
Msg string
}
func (f *Failure) Error() string {
return f.Msg
}
// Fail writes msg to w and returns it as a *Failure, so a command reports a failure to the
// user and to its surface in one line. If the write itself fails, that error is returned
// instead: the user was not told anything.
func Fail(w StringWriter, msg string) error {
if err := w.WriteLine(msg); err != nil {
return err
}
return &Failure{Msg: msg}
}
func execCommand(c *Command, args []string, w StringWriter) error {
var (
fl *flag.FlagSet
fs any
)
if c.Flags != nil {
fl, fs = c.Flags()
if fl != nil {
// SetOutput() here in case fl.Parse dumps usage.
fl.SetOutput(w.GetWriter())
err := fl.Parse(args)
if err != nil {
// fl.Parse has dumped error information to the user via the w writer.
return err
}
args = fl.Args()
}
}
return c.Callback(fs, args, w)
}
func checkHelpArgs(args []string) bool {
for _, a := range args {
if a == "-h" || a == "-help" {
return true
}
}
return false
}
+28
View File
@@ -0,0 +1,28 @@
package commands
// Name identifies a command. Every command nebula serves is a constant here, so a
// surface, a client and a test spell it the same way and are told at compile time when
// they do not. It is a string rather than an integer so a test can register a throwaway
// command without adding it here, and so the wire carries the name as the string it has
// always been.
type Name string
const (
ChangeRemote Name = "change-remote"
CloseTunnel Name = "close-tunnel"
CreateTunnel Name = "create-tunnel"
DeviceInfo Name = "device-info"
Help Name = "help"
ListHostmap Name = "list-hostmap"
ListLighthouseAddrmap Name = "list-lighthouse-addrmap"
ListPendingHostmap Name = "list-pending-hostmap"
LogFormat Name = "log-format"
LogLevel Name = "log-level"
Logout Name = "logout"
PrintCert Name = "print-cert"
PrintRelays Name = "print-relays"
PrintTunnel Name = "print-tunnel"
QueryLighthouse Name = "query-lighthouse"
Reload Name = "reload"
Version Name = "version"
)
+181
View File
@@ -0,0 +1,181 @@
package commands
import (
"cmp"
"fmt"
"maps"
"slices"
"strings"
"sync"
"github.com/anmitsu/go-shlex"
)
// Registry holds a set of commands. New is the only way to make one.
//
// It is safe for concurrent use. Commands may be registered while a surface is serving
// from the registry: the ssh server clones it per session, and a clone may overlap a
// Register.
type Registry struct {
mu sync.RWMutex
commands map[Name]*Command
}
// New returns an empty Registry with only `help` available.
func New() *Registry {
r := &Registry{commands: map[Name]*Command{}}
r.registerHelp()
return r
}
func (r *Registry) registerHelp() {
r.Register(&Command{
Name: Help,
ShortDescription: "prints available commands or help <command> for specific usage info",
Callback: func(a any, args []string, w StringWriter) error {
return r.help(args, w)
},
})
}
// Register adds a command that can be run by a user. A command registered under a name
// already held replaces it; see Clone for what that means for Help.
func (r *Registry) Register(c *Command) {
r.mu.Lock()
defer r.mu.Unlock()
r.commands[c.Name] = c
}
// Get returns the command registered under name, or nil if there is no such command.
func (r *Registry) Get(name Name) *Command {
r.mu.RLock()
defer r.mu.RUnlock()
return r.commands[name]
}
// Match returns the names of every command starting with prefix, sorted, as strings for
// a prompt to complete and print.
func (r *Registry) Match(prefix string) []string {
cmds := make([]string, 0)
r.mu.RLock()
for n := range r.commands {
if strings.HasPrefix(string(n), prefix) {
cmds = append(cmds, string(n))
}
}
r.mu.RUnlock()
slices.Sort(cmds)
return cmds
}
// All returns every registered command sorted by name, which is the order help lists them.
func (r *Registry) All() []*Command {
r.mu.RLock()
defer r.mu.RUnlock()
cmds := slices.Collect(maps.Values(r.commands))
slices.SortFunc(cmds, func(a, b *Command) int { return cmp.Compare(a.Name, b.Name) })
return cmds
}
// Clone returns a copy holding the same commands, with help bound to the copy: a
// command registered on the clone alone is listed and described there, and one the
// clone withholds is not. That rebinding replaces whatever was registered under Help on
// the source, built-in or not.
func (r *Registry) Clone() *Registry {
r.mu.RLock()
c := &Registry{commands: maps.Clone(r.commands)}
r.mu.RUnlock()
c.registerHelp()
return c
}
// Dispatch parses a command line and runs it, reporting problems to w. The error is for
// the caller to log, not to show; see CommandCallback.
func (r *Registry) Dispatch(line string, w StringWriter) error {
// A line shlex cannot split, an unbalanced quote.
args, err := shlex.Split(line, true)
if err != nil {
return Fail(w, "could not parse the command line: "+err.Error())
}
return r.DispatchArgs(args, w)
}
// DispatchArgs runs an already tokenized command line. The error is as for Dispatch.
func (r *Registry) DispatchArgs(args []string, w StringWriter) error {
if len(args) == 0 {
r.dumpCommands(w)
return nil
}
c := r.Get(Name(args[0]))
if c == nil {
// Only the first word was looked up, so only it is named.
msg := fmt.Sprintf("did not understand: %s", args[0])
if err := w.WriteLine(msg); err != nil {
return err
}
r.dumpCommands(w)
return &Failure{Msg: msg}
}
if checkHelpArgs(args) {
// Through the registered help command, so -h and `help <name>` are one path.
return execCommand(r.Get(Help), []string{string(c.Name)}, w)
}
return execCommand(c, args[1:], w)
}
// dumpCommands writes the list of available commands to w.
func (r *Registry) dumpCommands(w StringWriter) {
err := w.WriteLine("Available commands:")
if err != nil {
return
}
// In the order All gives, which is the display order.
cmds := make([]string, 0)
for _, l := range r.All() {
cmds = append(cmds, fmt.Sprintf("%s - %s", l.Name, l.ShortDescription))
}
_ = w.Write(strings.Join(cmds, "\n") + "\n\n")
}
// help writes the command list, or usage for the single command named in a.
func (r *Registry) help(a []string, w StringWriter) error {
// Just typed help
if len(a) == 0 {
r.dumpCommands(w)
return nil
}
// We are printing a specific commands help text
cmd := r.Get(Name(a[0]))
if cmd == nil {
return Fail(w, "Command not available "+a[0])
}
if err := w.WriteLine(fmt.Sprintf("%s - %s", cmd.Name, cmd.ShortDescription)); err != nil {
return err
}
if cmd.Help != "" {
if err := w.WriteLine(fmt.Sprintf(" %s", cmd.Help)); err != nil {
return err
}
}
if cmd.Flags != nil {
fs, _ := cmd.Flags()
if fs != nil {
fs.SetOutput(w.GetWriter())
fs.PrintDefaults()
}
}
return nil
}
+176
View File
@@ -0,0 +1,176 @@
package commands
import (
"bytes"
"errors"
"flag"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testRegistry() *Registry {
r := New()
r.Register(&Command{
Name: "served",
ShortDescription: "a command this surface serves",
Callback: func(fs any, a []string, w StringWriter) error {
return w.WriteLine("read " + strings.Join(a, ","))
},
})
r.Register(&Command{
Name: "withheld",
ShortDescription: "a command this surface withholds",
Callback: func(fs any, a []string, w StringWriter) error {
return w.WriteLine("wrote")
},
})
return r
}
// TestFlagUsageGoesToTheWriter pins that a parse failure's usage lands on the command's
// writer.
func TestFlagUsageGoesToTheWriter(t *testing.T) {
r := New()
r.Register(&Command{
Name: "emit",
ShortDescription: "emits",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
fl.Bool("json", false, "emit json")
return fl, nil
},
Callback: func(fs any, a []string, w StringWriter) error { return nil },
})
merged := &bytes.Buffer{}
require.Error(t, r.DispatchArgs([]string{"emit", "-nope"}, NewStringWriter(merged)))
assert.Contains(t, merged.String(), "flag provided but not defined")
}
// TestDispatchReportsUnparseableLine: a line shlex rejects, an unbalanced quote, is a
// failure the user is told about.
func TestDispatchReportsUnparseableLine(t *testing.T) {
out := &bytes.Buffer{}
err := testRegistry().Dispatch(`served "unterminated`, NewStringWriter(out))
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, f.Msg+"\n", out.String())
assert.Contains(t, out.String(), "could not parse the command line: ")
}
// TestFailTellsTheUserAndTheSurface: Fail writes the message and returns it as a Failure;
// when the write fails, the user heard nothing, so the write error is what comes back.
func TestFailTellsTheUserAndTheSurface(t *testing.T) {
out := &bytes.Buffer{}
err := Fail(NewStringWriter(out), "no such tunnel")
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, "no such tunnel", f.Msg)
assert.Equal(t, "no such tunnel\n", out.String())
err = Fail(NewStringWriter(failingWriter{}), "no such tunnel")
require.EqualError(t, err, "gone")
assert.NotErrorAs(t, err, &f)
}
type failingWriter struct{}
func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("gone") }
// TestAllSortsByName: registration order does not matter.
func TestAllSortsByName(t *testing.T) {
r := New()
for _, n := range []Name{"zz", Version, "aa", ChangeRemote} {
r.Register(&Command{Name: n})
}
var got []Name
for _, c := range r.All() {
got = append(got, c.Name)
}
assert.Equal(t, []Name{"aa", ChangeRemote, Help, Version, "zz"}, got)
}
// TestHelpFlagAgreesWithHelpCommand pins that -h answers through the registered help
// command, and that help on a clone describes the clone: a command registered only there
// is known to both -h and `help <name>`, and one withheld from it is known to neither.
func TestHelpFlagAgreesWithHelpCommand(t *testing.T) {
r := testRegistry()
c := r.Clone()
c.Register(&Command{
Name: Logout,
ShortDescription: "ends the session",
Callback: func(fs any, a []string, w StringWriter) error { return nil },
})
flag := &bytes.Buffer{}
c.Dispatch("logout -h", NewStringWriter(flag))
cmd := &bytes.Buffer{}
c.Dispatch("help logout", NewStringWriter(cmd))
assert.Equal(t, cmd.String(), flag.String())
assert.Equal(t, "logout - ends the session\n", flag.String())
// The source never learns of it.
src := &bytes.Buffer{}
r.Dispatch("help logout", NewStringWriter(src))
assert.Equal(t, "Command not available logout\n", src.String())
// And a command both hold is described either way.
flag.Reset()
c.Dispatch("served -h", NewStringWriter(flag))
assert.Equal(t, "served - a command this surface serves\n", flag.String())
}
// TestCloneHelpListsTheClone: the command list help prints is the clone's, so a clone
// that withholds a command does not name it, and the source still does.
func TestCloneHelpListsTheClone(t *testing.T) {
r := testRegistry()
c := New()
c.Register(r.Get("served"))
c = c.Clone()
out := &bytes.Buffer{}
c.Dispatch("help", NewStringWriter(out))
assert.Equal(t, "Available commands:\n"+
"help - prints available commands or help <command> for specific usage info\n"+
"served - a command this surface serves\n\n", out.String())
out.Reset()
c.Dispatch("help withheld", NewStringWriter(out))
assert.Equal(t, "Command not available withheld\n", out.String())
out.Reset()
r.Dispatch("help withheld", NewStringWriter(out))
assert.Equal(t, "withheld - a command this surface withholds\n", out.String())
}
// TestDispatchReturnsTheCommandsError: what the callback returns reaches the caller, so
// a surface can log it.
func TestDispatchReturnsTheCommandsError(t *testing.T) {
r := New()
r.Register(&Command{
Name: "fails",
Callback: func(fs any, a []string, w StringWriter) error {
return errors.New("boom")
},
})
out := &bytes.Buffer{}
require.EqualError(t, r.Dispatch("fails", NewStringWriter(out)), "boom")
assert.Empty(t, out.String(), "a command's own error is not shown; the command speaks for itself")
}
// TestUnknownCommandNamesTheWord: only the first word is looked up, so only it is
// reported, followed by the command list, and the line is a failure to the surface.
func TestUnknownCommandNamesTheWord(t *testing.T) {
out := &bytes.Buffer{}
err := testRegistry().Dispatch(`nope "with args"`, NewStringWriter(out))
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, "did not understand: nope", f.Msg)
assert.True(t, strings.HasPrefix(out.String(), "did not understand: nope\nAvailable commands:\n"), out.String())
}
+8 -1
View File
@@ -1,7 +1,9 @@
package sshd
package commands
import "io"
// StringWriter is how a command talks back to whoever invoked it: one stream, which the
// ssh surface points at its channel.
type StringWriter interface {
WriteLine(string) error
Write(string) error
@@ -13,6 +15,11 @@ type stringWriter struct {
w io.Writer
}
// NewStringWriter returns a StringWriter that writes to w.
func NewStringWriter(w io.Writer) StringWriter {
return &stringWriter{w: w}
}
func (w *stringWriter) WriteLine(s string) error {
return w.Write(s + "\n")
}
+67
View File
@@ -0,0 +1,67 @@
package nebula
import (
"bytes"
"io"
"net/netip"
"testing"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestLogFormatRefusalIsShown: a format the logger does not know is reported to the user
// as a failure, rather than returned bare, which over ssh was silence.
func TestLogFormatRefusalIsShown(t *testing.T) {
l := logging.NewLogger(io.Discard)
out := &bytes.Buffer{}
w := commands.NewStringWriter(out)
var f *commands.Failure
require.ErrorAs(t, logFormatCommand(l, nil, []string{"bogus"}, w), &f)
assert.Equal(t, "Unable to set log format: unknown log format `bogus`. possible formats: [text json]\n", out.String())
out.Reset()
require.NoError(t, logFormatCommand(l, nil, []string{"JSON"}, w))
assert.Equal(t, "Log format is: json\n", out.String())
}
// TestPrintRelaysReportsErrorsAsText: the Error field was an error value, which encodes
// as {} and lost the message. It is the message now, and absent when there is none.
func TestPrintRelaysReportsErrorsAsText(t *testing.T) {
l := test.NewLogger()
hm := newHostMap(l)
relay := netip.MustParseAddr("10.0.0.1")
peer := netip.MustParseAddr("10.0.0.2")
// A relay host that is in Relays but not in Hosts, so its hostinfo cannot be found.
hm.Relays[1] = &HostInfo{vpnAddrs: []netip.Addr{relay}}
ifce := &Interface{hostMap: hm}
out := &bytes.Buffer{}
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.JSONEq(t, `{"Relays":[{"NebulaAddr":"10.0.0.1","RelayForAddrs":[{"Error":"could not find hostinfo","Type":"","State":"","PeerAddr":"","LocalIndex":0,"RemoteIndex":0,"RelayedThrough":null}]}]}`, out.String())
// The same relay host, found, relaying for a peer whose index matches: no Error key.
hi := hm.Relays[1]
r := &Relay{Type: ForwardingType, State: Established, PeerAddr: peer, LocalIndex: 1, RemoteIndex: 2}
hi.relayState = RelayState{
relayForByAddr: map[netip.Addr]*Relay{peer: r},
relayForByIdx: map[uint32]*Relay{r.LocalIndex: r},
}
hm.Hosts[relay] = hi
out.Reset()
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.JSONEq(t, `{"Relays":[{"NebulaAddr":"10.0.0.1","RelayForAddrs":[{"Type":"forwarding","State":"established","PeerAddr":"10.0.0.2","LocalIndex":1,"RemoteIndex":2,"RelayedThrough":null}]}]}`, out.String())
// And an index mismatch names it.
hm.Relays[7] = hi
delete(hm.Relays, 1)
out.Reset()
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.Contains(t, out.String(), `"Error":"hostmap LocalIndex '7' does not match RelayState LocalIndex"`)
}
+15
View File
@@ -86,6 +86,21 @@ func (c *C) InitialLoad() bool {
return c.oldSettings == nil
}
// Render returns keys and their values as yaml, for a subsystem to remember what it applied. Empty means they couldn't
// be rendered and should never match.
func (c *C) Render(keys ...string) string {
m := make(map[string]any, len(keys))
for _, k := range keys {
m[k] = c.Get(k)
}
b, err := yaml.Marshal(m)
if err != nil {
c.l.Debug("Failed to render config", "keys", keys, "error", err)
return ""
}
return string(b)
}
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
// k in both the old and new settings will be serialized, the result of the string comparison is returned.
// If k is an empty string the entire config is tested.
+1 -10
View File
@@ -14,9 +14,6 @@ import (
)
const (
// ReplayWindow is the build-time receive window and the assumed peer window
// for TX reservation credit. Use a power of two; peers must match or have a
// larger receive window. The bitmap costs ReplayWindow/8 bytes per tunnel.
ReplayWindow = 8192
// RehandshakeAfterMessages rolls keys inside the AES-GCM data-volume margin (~2^-36 advantage at 64KB frames).
@@ -44,7 +41,6 @@ type ConnectionState struct {
peerCert *cert.CachedCertificate
initiator bool
messageCounter atomic.Uint64
txBudget txBudget
window *Bits
decryptLock sync.Mutex
writeLock sync.Mutex
@@ -108,7 +104,7 @@ func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, packet
return nil, ErrAlreadySeen
}
out, err := cs.decryptPacket(messageCounter, packet, nb)
out, err := cs.dKey.DecryptDanger(packet[header.Len:header.Len], packet[:header.Len], packet[header.Len:], messageCounter, nb)
if err != nil {
return nil, err
}
@@ -122,11 +118,6 @@ func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, packet
return out, nil
}
// decryptPacket authenticates and decrypts packet in place, leaving the replay window to the caller.
func (cs *ConnectionState) decryptPacket(messageCounter uint64, packet, nb []byte) ([]byte, error) {
return cs.dKey.DecryptDanger(packet[header.Len:header.Len], packet[:header.Len], packet[header.Len:], messageCounter, nb)
}
func (cs *ConnectionState) VerifyRelay(l *slog.Logger, messageCounter uint64, packet []byte, nb []byte) error {
cs.decryptLock.Lock()
result := cs.window.Check(l, messageCounter)
+3 -12
View File
@@ -23,20 +23,11 @@ import (
// runTestHandshake runs a complete IX handshake between two freshly-built
// peers and returns the initiator and responder Results. Used to produce
// real cipher states for tests that need to exercise post-handshake glue.
func runTestHandshake(t testing.TB) (initR, respR *handshake.Result) {
t.Helper()
return runTestHandshakeWith(t, noise.CipherChaChaPoly)
}
// runTestHandshakeWith is runTestHandshake with the data plane cipher picked by the caller.
func runTestHandshakeWith(t testing.TB, cipher noise.CipherFunc) (initR, respR *handshake.Result) {
func runTestHandshake(t *testing.T) (initR, respR *handshake.Result) {
t.Helper()
// Valid around now rather than the cert_test default, which is fixed at process start and lapses a minute
// later, inside a long -count run
now := time.Now()
ca, _, caKey, _ := ct.NewTestCaCert(
cert.Version2, cert.Curve_CURVE25519, now.Add(-time.Minute), now.Add(time.Hour), nil, nil, nil,
cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
)
caPool := ct.NewTestCAPool(ca)
@@ -49,7 +40,7 @@ func runTestHandshakeWith(t testing.TB, cipher noise.CipherFunc) (initR, respR *
require.NoError(t, err)
hsBytes, err := c.MarshalForHandshakes()
require.NoError(t, err)
ncs := noise.NewCipherSuite(noise.DH25519, cipher, noise.HashSHA256)
ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
cred := handshake.NewCredential(c, hsBytes, priv, ncs)
return func(v cert.Version) *handshake.Credential {
if v == cert.Version2 {
+9
View File
@@ -11,6 +11,7 @@ import (
"syscall"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay"
)
@@ -47,6 +48,7 @@ type Control struct {
f *Interface
l *slog.Logger
commands *commands.Registry
ctx context.Context
cancel context.CancelFunc
sshStart func()
@@ -372,6 +374,13 @@ func (c *Control) CloseAllTunnels(excludeLighthouses bool) (closed int) {
return
}
// Commands returns the registry the daemon's surfaces serve from. It is the live
// registry, not a copy: a command registered on it is served to every ssh session that
// starts afterwards. Clone it to run commands without that reach.
func (c *Control) Commands() *commands.Registry {
return c.commands
}
func (c *Control) Device() overlay.Device {
return c.f.inside
}
+164
View File
@@ -0,0 +1,164 @@
//go:build e2e_testing
package e2e
import (
"bytes"
"fmt"
"path/filepath"
"regexp"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/require"
)
// volatile lists everything in command output that legitimately differs between runs.
// Each run mints fresh certificates and indexes, so these are replaced with a marker
// before comparison. Keep the list short: anything added here is something the golden
// stops guarding.
var volatile = []struct {
re *regexp.Regexp
with string
}{
{regexp.MustCompile(`[0-9a-f]{40,}`), "<hex>"}, // fingerprints, keys, signatures
{regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})`), "<time>"}, // RFC 3339
{regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)? [+-]\d{4} \w+`), "<time>"}, // time.Time.String()
{regexp.MustCompile(`(?s)(-----BEGIN [A-Z ]+-----\n).*?(\n-----END [A-Z ]+-----)`), "${1}<pem>${2}"}, // print-cert -raw
{regexp.MustCompile(`(-----BEGIN [A-Z ]+-----\\n).*?(\\n-----END [A-Z ]+-----)`), "${1}<pem>${2}"}, // the same, inside a JSON string
{regexp.MustCompile(`"(localIndex|remoteIndex|messageCounter)":\d+`), `"$1":<n>`},
{regexp.MustCompile(`(?m)^(\s*"?(?:LocalIndex|RemoteIndex|localIndex|remoteIndex|messageCounter)"?:?\s*)\d+`), "${1}<n>"}, // and the -pretty form
}
func normalise(s string) string {
for _, v := range volatile {
s = v.re.ReplaceAllString(s, v.with)
}
return s
}
// outputCase is one command line to run through the ssh dispatch path.
type outputCase struct {
name string
args []string
}
// TestCommandOutput pins the bytes every command produces, in every format, through the
// ssh dispatch path. It is the safety net for moving rendering out of the commands: a
// change here is a change someone at an ssh prompt would see. Regenerate with:
//
// go test -tags e2e_testing ./e2e -run TestCommandOutput -update
func TestCommandOutput(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(
cert.Version1, cert.Curve_CURVE25519,
time.Now(), time.Now().Add(10*time.Minute),
nil, nil, []string{},
)
control, _, _, _ := newSimpleServer(cert.Version1, ca, caKey, "output-golden", "10.222.0.1/24", nil)
control.Start()
t.Cleanup(func() { control.Stop() })
// As an ssh session does: a clone of the daemon's registry with logout added, which
// help on the clone lists and describes. There is no session to end here.
reg := control.Commands().Clone()
reg.Register(sshd.LogoutCommand(func() {}))
// run dispatches as the ssh exec path does and returns the output and the error, which
// over ssh is exit status 1.
run := func(args ...string) (string, error) {
buf := &bytes.Buffer{}
err := reg.DispatchArgs(args, commands.NewStringWriter(buf))
return buf.String(), err
}
// Seed a pending tunnel so the listing and per-address commands have something to show.
seed, err := run("create-tunnel", "10.222.0.9")
require.NoError(t, err)
require.Equal(t, "Created\n", seed, "seeding a tunnel should succeed before anything else is recorded")
cases := []outputCase{
{"version", nil},
{"device-info", nil},
{"list-hostmap", nil},
{"list-hostmap", []string{"-by-index"}},
{"list-pending-hostmap", nil},
{"list-pending-hostmap", []string{"-by-index"}},
{"list-lighthouse-addrmap", nil},
{"print-cert", nil},
{"print-cert", []string{"-raw"}},
{"print-cert", []string{"10.222.0.9"}},
{"print-cert", []string{"10.222.0.250"}},
{"print-cert", []string{"not-an-address"}},
{"print-tunnel", nil},
{"print-tunnel", []string{"10.222.0.9"}},
{"print-tunnel", []string{"10.222.0.250"}},
{"print-relays", nil},
{"query-lighthouse", nil},
{"query-lighthouse", []string{"10.222.0.9"}},
{"query-lighthouse", []string{"not-an-address"}},
// The test logger cannot change level or format, so these cases record the
// refusal each command prints on it, not a successful change.
{"log-level", nil},
{"log-level", []string{"debug"}},
{"log-level", []string{"bogus"}},
{"log-level", []string{"info"}},
{"log-format", nil},
{"log-format", []string{"json"}},
{"log-format", []string{"text"}},
{"change-remote", nil},
{"change-remote", []string{"10.222.0.9"}},
{"change-remote", []string{"-address", "10.94.0.10:4242", "10.222.0.9"}},
{"change-remote", []string{"-address", "10.94.0.10:4242", "10.222.0.250"}},
{"create-tunnel", nil},
{"create-tunnel", []string{"10.222.0.9"}},
// A fresh address, so this exercises the handshake start with a remote attached
// rather than the "already handshaking" path above.
{"create-tunnel", []string{"-address", "10.94.0.10:4242", "10.222.0.10"}},
{"create-tunnel", []string{"not-an-address"}},
{"close-tunnel", nil},
{"close-tunnel", []string{"10.222.0.250"}},
{"close-tunnel", []string{"not-an-address"}},
{"help", nil},
{"help", []string{"print-cert"}},
{"help", []string{"logout"}},
{"logout", []string{"-h"}},
{"not-a-command", []string{"10.222.0.9"}},
// reload is last so that whatever a reload does to the daemon cannot leak into an
// earlier case. In this harness it fails: the config was loaded from a string and
// has no path to re-read, which is what the recorded result says.
{"reload", nil},
}
got := &strings.Builder{}
for _, c := range cases {
for _, fmtFlags := range [][]string{nil, {"-json"}, {"-pretty"}} {
args := append([]string{c.name}, fmtFlags...)
args = append(args, c.args...)
fmt.Fprintf(got, "$ %s\n", strings.Join(args, " "))
out, err := run(args...)
out = normalise(out)
got.WriteString(out)
// Output that does not end in a newline is recorded as such, the way diff marks
// it, so the golden sees that byte too and the next prompt still starts a line.
if out != "" && !strings.HasSuffix(out, "\n") {
got.WriteString("\\ No newline at end of output\n")
}
// A failed command is recorded as the exit status an ssh caller sees, so the
// golden pins which outcomes count as failures.
if err != nil {
got.WriteString("exit status 1\n")
}
got.WriteString("\n")
}
}
test.Golden(t, filepath.Join("testdata", "commands_output.golden"), got.String())
}
+720
View File
@@ -0,0 +1,720 @@
$ version
e2e-test
$ version -json
e2e-test
$ version -pretty
e2e-test
$ device-info
name= cidr=[10.222.0.1/24]
$ device-info -json
{"name":"","cidr":["10.222.0.1/24"]}
$ device-info -pretty
{
"name": "",
"cidr": [
"10.222.0.1/24"
]
}
$ list-hostmap
$ list-hostmap -json
[]
$ list-hostmap -pretty
[]
$ list-hostmap -by-index
$ list-hostmap -json -by-index
[]
$ list-hostmap -pretty -by-index
[]
$ list-pending-hostmap
[10.222.0.9]: []
$ list-pending-hostmap -json
[{"vpnAddrs":["10.222.0.9"],"localIndex":<n>,"remoteIndex":<n>,"remoteAddrs":[],"cert":null,"messageCounter":<n>,"currentRemote":"","currentRelaysToMe":[],"currentRelaysThroughMe":[]}]
$ list-pending-hostmap -pretty
[
{
"vpnAddrs": [
"10.222.0.9"
],
"localIndex": <n>,
"remoteIndex": <n>,
"remoteAddrs": [],
"cert": null,
"messageCounter": <n>,
"currentRemote": "",
"currentRelaysToMe": [],
"currentRelaysThroughMe": []
}
]
$ list-pending-hostmap -by-index
$ list-pending-hostmap -json -by-index
[]
$ list-pending-hostmap -pretty -by-index
[]
$ list-lighthouse-addrmap
10.222.0.9: {}
$ list-lighthouse-addrmap -json
[{"vpnAddr":"10.222.0.9","addrs":{}}]
$ list-lighthouse-addrmap -pretty
[
{
"vpnAddr": "10.222.0.9",
"addrs": {}
}
]
$ print-cert
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}
$ print-cert -json
{"details":{"curve":"CURVE25519","groups":[],"isCa":false,"issuer":"<hex>","name":"output-golden","networks":["10.222.0.1/24"],"notAfter":"<time>","notBefore":"<time>","publicKey":"<hex>","unsafeNetworks":[]},"fingerprint":"<hex>","signature":"<hex>","version":1}\ No newline at end of output
$ print-cert -pretty
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}\ No newline at end of output
$ print-cert -raw
-----BEGIN NEBULA CERTIFICATE-----
<pem>
-----END NEBULA CERTIFICATE-----
$ print-cert -json -raw
{"details":{"curve":"CURVE25519","groups":[],"isCa":false,"issuer":"<hex>","name":"output-golden","networks":["10.222.0.1/24"],"notAfter":"<time>","notBefore":"<time>","publicKey":"<hex>","unsafeNetworks":[]},"fingerprint":"<hex>","signature":"<hex>","version":1}\ No newline at end of output
$ print-cert -pretty -raw
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}\ No newline at end of output
$ print-cert 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert -json 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert -pretty 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert -json 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert -pretty 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-cert -json not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-cert -pretty not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-tunnel
No vpn address was provided
exit status 1
$ print-tunnel -json
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty
No vpn address was provided
exit status 1
$ print-tunnel 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-tunnel -json 10.222.0.9
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-tunnel 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-tunnel -json 10.222.0.250
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-relays
{"Relays":null}
$ print-relays -json
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-relays -pretty
{
"Relays": null
}
$ query-lighthouse
No vpn address was provided
exit status 1
$ query-lighthouse -json
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty
The provided vpn address could not be parsed: -pretty
exit status 1
$ query-lighthouse 10.222.0.9
{}
$ query-lighthouse -json 10.222.0.9
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty 10.222.0.9
The provided vpn address could not be parsed: -pretty
exit status 1
$ query-lighthouse not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ query-lighthouse -json not-an-address
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty not-an-address
The provided vpn address could not be parsed: -pretty
exit status 1
$ log-level
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty
Log level is not reconfigurable on this logger
exit status 1
$ log-level debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level info
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json info
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty info
Log level is not reconfigurable on this logger
exit status 1
$ log-format
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty
Log format is not reconfigurable on this logger
exit status 1
$ log-format json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty json
Log format is not reconfigurable on this logger
exit status 1
$ log-format text
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json text
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty text
Log format is not reconfigurable on this logger
exit status 1
$ change-remote
No vpn address was provided
exit status 1
$ change-remote -json
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote 10.222.0.9
No address was provided
exit status 1
$ change-remote -json 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -address 10.94.0.10:4242 10.222.0.9
Could not find tunnel for vpn address: 10.222.0.9
exit status 1
$ change-remote -json -address 10.94.0.10:4242 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty -address 10.94.0.10:4242 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -address 10.94.0.10:4242 10.222.0.250
Could not find tunnel for vpn address: 10.222.0.250
exit status 1
$ change-remote -json -address 10.94.0.10:4242 10.222.0.250
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty -address 10.94.0.10:4242 10.222.0.250
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ create-tunnel
No vpn address was provided
exit status 1
$ create-tunnel -json
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel 10.222.0.9
Tunnel already handshaking
exit status 1
$ create-tunnel -json 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -address 10.94.0.10:4242 10.222.0.10
Created
$ create-tunnel -json -address 10.94.0.10:4242 10.222.0.10
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty -address 10.94.0.10:4242 10.222.0.10
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ create-tunnel -json not-an-address
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty not-an-address
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ close-tunnel
No vpn address was provided
exit status 1
$ close-tunnel -json
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel 10.222.0.250
Could not find tunnel for vpn address: 10.222.0.250
exit status 1
$ close-tunnel -json 10.222.0.250
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty 10.222.0.250
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ close-tunnel -json not-an-address
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty not-an-address
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ help
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
$ help -json
Command not available -json
exit status 1
$ help -pretty
Command not available -pretty
exit status 1
$ help print-cert
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
-json
outputs as json
-pretty
pretty prints json, assumes -json
-raw
raw prints the PEM encoded certificate, not compatible with -json or -pretty
$ help -json print-cert
Command not available -json
exit status 1
$ help -pretty print-cert
Command not available -pretty
exit status 1
$ help logout
logout - Ends the current session
$ help -json logout
Command not available -json
exit status 1
$ help -pretty logout
Command not available -pretty
exit status 1
$ logout -h
logout - Ends the current session
$ logout -json -h
logout - Ends the current session
$ logout -pretty -h
logout - Ends the current session
$ not-a-command 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ not-a-command -json 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ not-a-command -pretty 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ reload
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
$ reload -json
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
$ reload -pretty
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
-29
View File
@@ -5,11 +5,9 @@ package e2e
import (
"fmt"
"net/netip"
"runtime"
"testing"
"time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/e2e/router"
@@ -572,30 +570,3 @@ func TestCloseTunnelAuthenticated(t *testing.T) {
myControl.Stop()
theirControl.Stop()
}
// TestStagedTunnel completes a handshake and traffic both ways with staged receive. The test tun cannot hand off
// batches, so transmit stays inline; every received packet, handshakes included, uses the receive pipeline on Linux.
// Other platforms must do the same over the inline fallback.
func TestStagedTunnel(t *testing.T) {
t.Parallel()
received := metrics.GetOrRegisterCounter("flow.rx.inline", nil)
before := received.Count()
staged := m{"tun": m{"staged": m{"workers": 2}}}
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", staged)
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", staged)
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
myControl.Start()
theirControl.Start()
r := router.NewR(t, myControl, theirControl)
defer r.RenderFlow()
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
myControl.Stop()
theirControl.Stop()
if runtime.GOOS != "linux" {
assert.Zero(t, received.Count()-before)
return
}
assert.GreaterOrEqual(t, received.Count()-before, int64(2))
}
+7 -38
View File
@@ -39,8 +39,9 @@ static_host_map:
# public address. Other valid options are "ip6" and "ip" (returns both.)
#network: ip4
# lookup_timeout is the DNS query timeout.
#lookup_timeout: 250ms
# lookup_timeout is the DNS query timeout. A failed lookup keeps the last known good addresses, only a not found
# answer removes them.
#lookup_timeout: 5s
lighthouse:
# am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
@@ -275,42 +276,9 @@ tun:
# the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable.
#use_offloads: false
# Linux only. staged.workers enables staged packet processing: each routine hands its batches to this many crypto
# workers per direction, then sends or writes them in order. 0, the default, keeps processing inline. Not reloadable.
# Transmit needs use_offloads; BoringCrypto and FIPS builds keep it inline because AES-GCM there needs ordered nonces.
# Receive needs a listener that can hand off its batches, which Linux has.
# IMPORTANT: staged senders require peers with at least the 8192-entry replay window of this release. Released
# v1.11.x peers use 1024 and are incompatible. The window is not negotiated.
# Staged reservations may use half the window, the other half absorbs inline and control traffic, relay forwarding
# and network reordering; unlimited forwarding or delay can still exceed it. Waiting for credit on a busy tunnel
# also delays the other tunnels of the same read, and readers are not guaranteed fairness.
# Small batches, below inline_below TX segments or RX packets (default 256), can skip the workers. On transmit each
# routine holds one mode, inline on the reader or fanout to the workers and the ordered sender, and switches only
# after gate_hold (default 64) reads in a row want the other one. Before going inline it waits for every batch in
# flight to be submitted to the socket; flow.tx.gate_switches counts the switches. This does not drain NIC queues.
# Relay data always stays on the reader, so gate switches and oversized reads cannot move it between NIC queues.
# Direct data retains the adaptive gate and independent sender placement.
# On receive a listener with nothing in flight writes a small batch itself. With multiple routines, inline sends
# still take tunnel reservation credit. 0 hands direct TX and all RX batches to the workers and ordered stages;
# relayed TX and reads too large for a tunnel's whole credit are still sent by the reader.
# tx: false keeps transmit inline and stages only receive.
# tx_slots and rx_slots bound the batches each routine has in flight per direction, minimum 2. A TX slot holds one
# tun read and its ciphertext, about 1.1 MiB; the default is 8, or workers + 2 if more. An RX slot holds one
# recvmmsg batch, listen.batch * 64 KiB with udp_offloads or * 9 KiB without; the default keeps up to 384 receive
# entries in flight in 4 to 6 slots: 6 slots or 24 MiB at the default listen.batch of 64. Lower them to save memory.
#staged:
# workers: 0
# tx: true
# inline_below: 256
# gate_hold: 64
# tx_slots: 8
# rx_slots: 6
# Linux only. pin_threads pins each tun reader/encrypt OS thread to a single CPU. This keeps every goroutine's
# batched sends flowing through one XPS-selected NIC TX ring, so packets within a flow stay ordered on the wire
# instead of being sprayed across multiple TX rings and reordered. Not reloadable. Coerced to false if routines <= 1.
# Staged senders are only pinned with an explicit cpu_affinity, see below; otherwise the scheduler places them.
# Small batches a reader sends itself can use a different TX ring than its sender.
#pin_threads: true
# pin_threads_key helps the CPU-auto-selector shuffle which CPUs are chosen for pinning.
@@ -320,9 +288,7 @@ tun:
# Linux only. cpu_affinity overrides which CPUs the tun reader threads pin to: a list of CPU IDs, one per routine
# (see the top-level `routines` setting). Lists shorter than `routines` are modulo-cycled across the queues; extra
# entries are ignored. Staged sender i pins to entry i+1, modulo-cycled the same way, unless that is its reader's
# CPU, as with a one-entry list: a sender never shares its reader's CPU and stays unpinned instead.
# IDs must be within the process's allowed CPU set, so this respects taskset / cgroup cpusets;
# entries are ignored. IDs must be within the process's allowed CPU set, so this respects taskset / cgroup cpusets;
# a non-integer or not-allowed entry disables the override, leaving the default pin selection described below.
# Only meaningful while pin_threads is true. Not reloadable.
# When unset (or rejected), the default spread prefers performance cores on heterogeneous CPUs (ARM big.LITTLE,
@@ -366,6 +332,9 @@ tun:
# `mtu`: will default to tun mtu if this option is not specified
# `metric`: will default to 0 if this option is not specified
# `install`: will default to true, controls whether this route is installed in the systems routing table.
# On macOS, a route whose destination another interface already routes, like the local network, is installed once
# that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another
# interface routes its destination and lets it go.
# This setting is reloadable.
unsafe_routes:
#- route: 172.16.1.0/24
+4
View File
@@ -66,6 +66,9 @@ type Firewall struct {
rules string
rulesVersion uint16
// source is the rendered firewall config this firewall was built from, the cert networks are kept above
source string
defaultLocalCIDRAny bool
incomingMetrics firewallMetrics
outgoingMetrics firewallMetrics
@@ -246,6 +249,7 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
return nil, err
}
fw.source = c.Render("firewall")
return fw, nil
}
-123
View File
@@ -1,123 +0,0 @@
package nebula
import (
"log/slog"
"runtime"
"sync"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/noiseutil"
)
// Staged processing (tun.staged.workers) moves crypto off each tun reader and udp listener onto worker goroutines,
// behind which an ordered stage sends or writes every batch in read order. It reuses the inline packet handlers:
// staging changes ownership and scheduling, not policy. Each routine has its own workers and ordered stages, so order
// is per routine. Full queues block the reader. A read error drains the batches in flight, while shutdown waits for
// the crypto and then drops packets whose destination has closed. A TX slot holds one tun read and its ciphertext,
// about 1.1 MiB; there are eight, or two more than the workers. An RX slot holds one recvmmsg batch, listen.batch
// times 64 KiB with GRO, and the count keeps up to 384 entries in flight in four to six slots: six and 24 MiB at
// listen.batch=64.
// tun.staged.tx_slots and rx_slots override the counts. Small-MSS TX arenas can grow.
// Relay data stays on the tun reader, regardless of the TX gate, so it never switches between reader and sender
// NIC queues. Direct data keeps its adaptive placement; relay and direct batches both share tunnel counter credit.
//
// Metrics are registered only for active pipelines. flow.{tx,rx}.batches counts reads handed to the ordered stage and
// flow.{tx,rx}.inline those the reader handled itself (TX inline also counts oversized reads); their rates show
// whether staging is used. flow.tx.gate_switches counts transmit switches between inline and fanout mode.
// stagedConfig is tun.staged. workers is per routine and direction; zero keeps processing inline.
type stagedConfig struct {
workers int
tx bool
rx bool
// inlineBelow is the TX segment or RX packet count below which a read may skip the worker handoff, 0 never: a TX
// read in inline mode, an RX read with nothing in flight
inlineBelow int
// txSlots and rxSlots are how many batches each routine can have in flight per direction
txSlots, rxSlots int
// gateHold is how many TX reads in a row must want the other mode before a routine switches between inline and
// fanout transmit
gateHold int
}
// stagedGateHold is the default tun.staged.gate_hold
const stagedGateHold = 64
const (
// stagedTxSlots is the default TX slot count, unless the workers need more
stagedTxSlots = 8
// stagedRxEntries is how many recvmmsg entries the RX slots hold in flight by default
stagedRxEntries = 6 * 64
// stagedMinSlots lets the reader fill one batch while another is in flight
stagedMinSlots = 2
// stagedMinRxSlots keeps the default RX pipeline deep enough for large listen.batch values
stagedMinRxSlots = 4
// stagedMaxRxSlots is the measured default, a smaller listen.batch does not take more slots
stagedMaxRxSlots = 6
)
// stagedSlots sizes the TX and RX slots. TX reads are bounded by the tun queue, so one slot per worker plus the one
// being read and the one being sent, and at least the eight measured. RX slots keep stagedRxEntries in flight, four
// to six, so a larger listen.batch means fewer slots and RX memory never exceeds the six measured. A positive override replaces either, down to stagedMinSlots.
func stagedSlots(workers, batch, txOverride, rxOverride int) (tx, rx int) {
tx = max(stagedTxSlots, workers+2)
rx = min(max(stagedMinRxSlots, (stagedRxEntries+batch-1)/max(batch, 1)), stagedMaxRxSlots)
if txOverride > 0 {
tx = max(txOverride, stagedMinSlots)
}
if rxOverride > 0 {
rx = max(rxOverride, stagedMinSlots)
}
return tx, rx
}
// parseStagedConfig enables staging on Linux only; queues and conns that cannot hand off batches also stay inline.
func parseStagedConfig(c *config.C, l *slog.Logger) stagedConfig {
sc := stagedConfig{workers: c.GetInt("tun.staged.workers", 0)}
if sc.workers <= 0 {
return stagedConfig{}
}
if runtime.GOOS != "linux" {
l.Warn("tun.staged is only supported on Linux; packet processing stays inline")
return stagedConfig{}
}
sc.tx, sc.rx = c.GetBool("tun.staged.tx", true), true
// 256 is the measured crossover, 0 hands every batch to the workers
sc.inlineBelow = max(c.GetInt("tun.staged.inline_below", 256), 0)
if sc.tx && noiseutil.EncryptLockNeeded {
l.Warn("tun.staged cannot encrypt in parallel with boringcrypto or fips140; transmit stays inline")
sc.tx = false
}
sc.gateHold = max(c.GetInt("tun.staged.gate_hold", stagedGateHold), 1)
sc.txSlots, sc.rxSlots = stagedSlots(sc.workers, c.GetInt("listen.batch", 64),
c.GetInt("tun.staged.tx_slots", 0), c.GetInt("tun.staged.rx_slots", 0))
l.Info("Staged packet processing enabled", "workers", sc.workers, "tx", sc.tx, "rx", sc.rx,
"inlineBelow", sc.inlineBelow, "gateHold", sc.gateHold,
"txSlots", sc.txSlots, "rxSlots", sc.rxSlots)
return sc
}
// flowJob is one batch of crypto work. It owns its packet buffers until run returns; jobs and queues are reused.
type flowJob interface{ run(nb []byte) }
// flowPool runs one reader's crypto jobs. The reader is the only producer and closes the pool once its ordered stage
// has drained.
type flowPool struct {
jobs chan flowJob
wg sync.WaitGroup
}
func newFlowPool(workers, queueLen int) *flowPool {
p := &flowPool{jobs: make(chan flowJob, queueLen)}
for range workers {
p.wg.Go(func() {
nb := make([]byte, 12)
for job := range p.jobs {
job.run(nb)
}
})
}
return p
}
func (p *flowPool) close() { close(p.jobs); p.wg.Wait() }
-177
View File
@@ -1,177 +0,0 @@
package nebula
import (
"sync/atomic"
"time"
"github.com/slackhq/nebula/overlay/tio"
)
// txBudget is the reservation credit of one tunnel, shared by every staged reader sending on it. The low 24 bits of
// state are outstanding counters, the next 24 spent credit, and the top 16 the sole ordered reader (q+1); owner zero
// means shared. Readers beyond the owner field's range always take the conservative shared path.
//
// A cap on outstanding counters alone is not enough: a fast reader could repeatedly send and reacquire credit,
// overtaking a paused reader by arbitrarily far. So once readers overlap, or another reader is blocked, spent credit
// is only recycled when ALL reservations have drained. A sole reader instead gets each ordered completion refunded,
// its later batches cannot overtake its earlier ones. All transitions are one CAS, without locks.
//
// The credit is half the peer's replay window. The other half is left for uncredited sends and transport reordering:
// inline and control traffic, and relay forwarding through SendVia, which takes counters on the relay tunnel without
// credit. Unlimited bypass traffic or network delay can still exceed it. The window is not negotiated, so
// ci.window.length, from the build-time ReplayWindow, is also the assumed peer window.
type txBudget struct{ state atomic.Uint64 }
const txBudgetMask = uint64(1<<24 - 1)
// Fail compilation for unsupported build-time windows: a power of two, at least 8192, fitting the credit fields (up
// to 16777216).
const (
_ = uint64(txBudgetMask) - ReplayWindow/2
_ = uint64(ReplayWindow) - 8192
_ = uint64(0) - (ReplayWindow & (ReplayWindow - 1))
)
func txBudgetOwner(q int) uint64 {
if q < 0 || q >= 65535 {
return 0
}
return uint64(q + 1)
}
func (b *txBudget) acquire(n, limit, owner uint64) bool {
for {
old := b.state.Load()
pending, spent, who := old&txBudgetMask, (old>>24)&txBudgetMask, old>>48
if n > limit || spent > limit-n {
// A blocked reader ends sole ownership so ordered completions stop refunding credit. Otherwise a busy
// owner could keep a larger waiter blocked forever without ever sharing a reservation.
if who != 0 && who != owner && !b.state.CompareAndSwap(old, old&^(uint64(0xffff)<<48)) {
continue
}
return false
}
if pending == 0 {
who = owner
} else if who != owner {
who = 0
}
next := who<<48 | (spent+n)<<24 | (pending + n)
if b.state.CompareAndSwap(old, next) {
return true
}
}
}
// release runs only after the socket flush or an explicit drop. A rollback returns credit before any message counter
// was allocated, as when a later tunnel of a relay or multi-tunnel batch could not acquire its own.
func (b *txBudget) release(n uint64, rollback bool) {
for {
old := b.state.Load()
pending, spent, owner := old&txBudgetMask, (old>>24)&txBudgetMask, old>>48
pending -= n
if rollback || owner != 0 {
spent -= n
}
next := owner<<48 | spent<<24 | pending
if pending == 0 {
next = 0
}
if b.state.CompareAndSwap(old, next) {
return
}
}
}
func (ci *ConnectionState) txBudgetLimit() uint64 { return ci.window.length / 2 }
func (r *txReserver) oversized() bool {
for _, t := range r.tunnels {
if t.n > t.ci.txBudgetLimit() {
return true
}
}
return false
}
func (r *txReserver) tryBudget() bool {
for i, t := range r.tunnels {
if !t.ci.txBudget.acquire(t.n, t.ci.txBudgetLimit(), r.owner) {
for _, acquired := range r.tunnels[:i] {
acquired.ci.txBudget.release(acquired.n, true)
}
return false
}
}
return true
}
// waitBudget acquires credit for every tunnel of the batch, never holding a partial acquisition while it waits. Each
// reader first flushes its own oldest batch, flushes need no credit; with none, another reader holds all outstanding
// work and can flush it independently. So there is no circular wait, even with relay tunnels acquired in opposite
// orders. The sleep is only for readers with no local work, not the uncontended hot path. Acquisition is
// all-or-nothing within each direct or relay batch, so pressure on one tunnel also delays packets for idle tunnels
// in that batch. Reader fairness is not guaranteed.
func (p *stagedTx) waitBudget(r *txReserver) bool {
if r.tryBudget() {
return true
}
for {
if p.f.closed.Load() {
return false
}
if !p.flushOldest() {
time.Sleep(50 * time.Microsecond)
}
if r.tryBudget() {
return true
}
}
}
func (s *txSlot) releaseBudget() {
for _, t := range s.reservations {
t.ci.txBudget.release(t.n, false)
}
clear(s.reservations)
s.reservations = s.reservations[:0]
}
// sendLarge handles the unusual reads that exceed a whole tunnel's credit, such as tiny-MSS superpackets. It drains
// the pipeline, then segments and sends them synchronously with credit for each segment, never exceeding the cap and
// never filtering or running connection manager side effects twice. This deliberately slow path avoids a lone-batch
// exception to the shared invariant.
func (p *stagedTx) sendLarge(r *txReserver, items []txItem) {
for p.flushOldest() {
}
for _, it := range items {
err := tio.SegmentSuperpacket(it.pkt, func(seg []byte) error {
if p.f.closed.Load() {
return nil
}
part := it
part.pkt, part.segs = tio.Packet{Bytes: seg}, 1
r.tunnels = r.tunnels[:0]
part.counter = r.need(part.hostinfo, part.ci, 1)
if part.relayHI != nil {
part.relayCounter = r.need(part.relayHI, part.relayHI.ConnectionState, 1)
}
if !p.waitBudget(r) {
return nil
}
one := [1]txItem{part}
if reserved := r.reserve(one[:]); len(reserved) != 0 {
p.f.encryptTxItem(&reserved[0], p.nb, p.sb)
p.f.flushSendBatch(p.sb, p.q)
}
for _, t := range r.tunnels {
t.ci.txBudget.release(t.n, false)
}
return nil
})
if err != nil {
it.hostinfo.logger(p.f.l).Error("Failed to segment superpacket for send", "error", err)
}
}
clear(items)
}
-104
View File
@@ -1,104 +0,0 @@
package nebula
import (
"fmt"
"testing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/require"
)
var budgetWindows = []uint64{8192, 65536}
// TestTXBudgetOvertaking: a fast sender must not recycle credit around a stalled older reservation. A pure in-flight
// count passes the cap assertion but fails this replay test.
func TestTXBudgetOvertaking(t *testing.T) {
for _, window := range budgetWindows {
t.Run(fmt.Sprint(window), func(t *testing.T) {
var b txBudget
limit := window / 2
w := NewBits(window)
l := test.NewLogger()
require.True(t, b.acquire(1, limit, 1)) // counter 1 stalls
for counter := uint64(2); counter <= limit; counter++ {
require.True(t, b.acquire(1, limit, 2))
require.True(t, w.Update(l, counter))
b.release(1, false)
}
require.Equal(t, uint64(1), b.state.Load()&txBudgetMask)
require.False(t, b.acquire(1, limit, 2), "spent credit cannot be recycled while the oldest sender stalls")
// The other half of the window allows this many unreserved sends.
for counter := limit + 1; counter <= window; counter++ {
require.True(t, w.Update(l, counter))
}
require.True(t, w.Update(l, 1), "oldest reservation must still fit after using the whole margin")
b.release(1, false)
require.Zero(t, b.state.Load())
require.True(t, b.acquire(limit, limit, 2))
require.False(t, b.acquire(1, limit, 2))
b.release(limit, false)
})
}
}
// TestTXBudgetBlockedReader models eight 512-segment reads of a busy owner and a 568-segment waiter, without socket
// timing. Alone, the owner gets each completion refunded, so it keeps eight batches in flight forever, even across its
// own failed acquisitions. But even when the owner always retries first, its completions must stop refunding credit
// once the waiter is blocked, and let the budget drain. Waiter 0 is the conservative ID of readers beyond the owner
// field's range.
func TestTXBudgetBlockedReader(t *testing.T) {
for _, waiter := range []uint64{1, 0} {
t.Run(fmt.Sprintf("waiter=%d", waiter), func(t *testing.T) {
var b txBudget
const limit, n, want = uint64(4096), uint64(512), uint64(568)
for range 8 {
require.True(t, b.acquire(n, limit, 2))
}
for range 100 {
require.False(t, b.acquire(n, limit, 2))
b.release(n, false) // the sole sender flushes its oldest batch
require.True(t, b.acquire(n, limit, 2))
require.Equal(t, 8*n, b.state.Load()&txBudgetMask)
}
require.False(t, b.acquire(want, limit, waiter))
for range 7 {
b.release(n, false)
require.False(t, b.acquire(n, limit, 2), "blocked reader must stop the owner's credit recycling")
require.False(t, b.acquire(want, limit, waiter))
}
b.release(n, false)
require.Zero(t, b.state.Load())
// Even if the former owner wins the next acquisition, both now fit.
require.True(t, b.acquire(n, limit, 2))
require.True(t, b.acquire(want, limit, waiter))
b.release(n, false)
b.release(want, false)
require.Zero(t, b.state.Load())
})
}
}
// TestTXBudgetRelayRollback: when a later tunnel of a batch cannot acquire, the earlier ones give their credit back,
// spent credit included.
func TestTXBudgetRelayRollback(t *testing.T) {
for _, window := range budgetWindows {
inner := &ConnectionState{window: NewBits(window)}
outer := &ConnectionState{window: NewBits(window)}
limit := inner.txBudgetLimit()
require.True(t, inner.txBudget.acquire(10, limit, 1))
require.True(t, outer.txBudget.acquire(limit, limit, 1))
r := txReserver{owner: 2, tunnels: []txTunnel{{ci: inner, n: 20}, {ci: outer, n: 20}}}
for range 100 {
require.False(t, r.tryBudget())
require.Equal(t, uint64(10), inner.txBudget.state.Load()&txBudgetMask)
require.Equal(t, uint64(10), (inner.txBudget.state.Load()>>24)&txBudgetMask, "rollback must refund spent credit too")
}
outer.txBudget.release(limit, false)
require.True(t, r.tryBudget())
inner.txBudget.release(20, false)
outer.txBudget.release(20, false)
inner.txBudget.release(10, false)
require.Zero(t, inner.txBudget.state.Load())
require.Zero(t, outer.txBudget.state.Load())
}
}
-135
View File
@@ -1,135 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
import (
"fmt"
"testing"
"time"
"github.com/flynn/noise"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
// gated is staging with an inline threshold of 4 segments and the given gate hold.
func gated(hold int) stagedConfig {
sc := adaptive(2)
sc.inlineBelow, sc.gateHold = 4, hold
return sc
}
// gateCounters snapshots the transmit counters a gate test checks.
type gateCounters struct{ inline, batches, switches int64 }
func readGateCounters() gateCounters {
return gateCounters{
inline: metrics.GetOrRegisterCounter("flow.tx.inline", nil).Count(),
batches: metrics.GetOrRegisterCounter("flow.tx.batches", nil).Count(),
switches: metrics.GetOrRegisterCounter("flow.tx.gate_switches", nil).Count(),
}
}
func (c gateCounters) since(c0 gateCounters) gateCounters {
return gateCounters{c.inline - c0.inline, c.batches - c0.batches, c.switches - c0.switches}
}
// gateReads builds reads of small (3 segments, below the threshold) or large (2 superpackets of 9) shape, in order,
// for one flow, and the segments they carry.
func gateReads(large ...bool) ([][]tio.Packet, []decodedSeg) {
var reads [][]tio.Packet
var want []decodedSeg
seq := uint32(0)
for _, l := range large {
per, segs := 1, 3
if l {
per, segs = 2, 9
}
var b []tio.Packet
for range per {
b = append(b, flowTCPSuperpacket(0, seq, segs))
for s := range segs {
want = append(want, decodedSeg{seq: seq, idx: uint16(s)})
}
seq++
}
reads = append(reads, b)
}
return reads, want
}
func repeatBool(v bool, n int) []bool {
out := make([]bool, n)
for i := range out {
out[i] = v
}
return out
}
// TestFlowStagedTXGateSwitch forces the gate from inline to fanout and back, with a slow socket and slow
// workers so the fanout batches are still in flight when the small reads arrive: wire order is read order, counters
// are contiguous, and each switch happens on exactly the gate_hold-th read that wants it.
func TestFlowStagedTXGateSwitch(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
const hold = 3
for _, routines := range []int{1, 2} {
t.Run(fmt.Sprintf("routines=%d", routines), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, gated(hold))
sender.f.routines = routines
sender.f.writers = []udp.Conn{sender.conn, sender.conn}
delayCrypto(sender, func(n uint64) {
if n%7 == 0 {
time.Sleep(time.Millisecond)
}
})
sender.conn.delay = 4 * time.Millisecond
// 4 small inline, 2 large held inline, 6 large fanout from the switch on, 2 small held in fanout, then
// the switch back drains and 3 small go inline
shape := append(repeatBool(false, 4), repeatBool(true, 8)...)
shape = append(shape, repeatBool(false, 5)...)
reads, want := gateReads(shape...)
ci := sender.hostinfo.ConnectionState
start := ci.messageCounter.Load()
c0 := readGateCounters()
p := newStagedTx(sender.f, newFlowBatchQueue(nil), 0)
for i, r := range reads {
p.send(r)
if i == 13 {
require.Positive(t, p.inFlight, "fanout batches must still be in flight when the small reads start")
}
if i == 14 {
require.Zero(t, p.inFlight, "switching to inline drains the pipeline")
}
}
p.close()
require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured()), "wire order must match read order")
require.Equal(t, start+uint64(len(want)), ci.messageCounter.Load(), "every reserved counter is used once")
require.Equal(t, gateCounters{inline: 4 + 2 + 3, batches: 6 + 2, switches: 2}, readGateCounters().since(c0))
require.Zero(t, ci.txBudget.state.Load())
})
}
}
// TestFlowStagedTXGateHold: the gate never switches before gate_hold consecutive reads want the other mode, and
// a read that wants the current mode restarts the count.
func TestFlowStagedTXGateHold(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
const hold = 5
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, gated(hold))
shape := append(repeatBool(true, hold-1), false)
shape = append(shape, repeatBool(true, hold-1)...)
reads, want := gateReads(append(shape, true)...)
c0 := readGateCounters()
p := newStagedTx(sender.f, newFlowBatchQueue(nil), 0)
for _, r := range reads[:len(reads)-1] {
p.send(r)
}
require.Equal(t, gateCounters{inline: 2*hold - 1}, readGateCounters().since(c0), "no switch before gate_hold")
p.send(reads[len(reads)-1])
require.Equal(t, gateCounters{inline: 2*hold - 1, batches: 1, switches: 1}, readGateCounters().since(c0))
p.close()
require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured()))
}
-371
View File
@@ -1,371 +0,0 @@
package nebula
import (
"encoding/binary"
"net/netip"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
)
// handoffConn is a udp conn that can hand each received batch over. The callback returns replacement buffers with
// the same count and at least the same lengths.
type handoffConn interface {
ListenOutHandoff(udp.EncReader, func([][]byte) [][]byte) error
}
const rxChunkMin = 16
// rxItem is one received packet. A data packet for an established tunnel has ci set; out and err are written by
// whoever decrypts it.
type rxItem struct {
packet []byte
hostinfo *HostInfo
ci *ConnectionState
counter uint64
via netip.AddrPort
out []byte
err error
}
// rxChunk is a contiguous run of a batch's items for one worker to decrypt.
type rxChunk struct {
items []rxItem
done chan struct{}
}
func (c *rxChunk) run(nb []byte) {
decryptRxItems(c.items, nb)
c.done <- struct{}{}
}
// decryptRxItems decrypts the data packets among items in place. The replay window is left to the ordered stage, as
// are packets without a tunnel and the known replays precheck marked.
func decryptRxItems(items []rxItem, nb []byte) {
for i := range items {
it := &items[i]
if it.ci == nil || it.err != nil {
continue
}
it.out, it.err = it.ci.decryptPacket(it.counter, it.packet, nb)
}
}
// rxBatch is one recvmmsg batch moving through the staged receive pipeline.
type rxBatch struct {
// items is every packet of the batch in arrival order. Data packets are decrypted by a worker; anything else is
// handled whole by the writer, in its place.
items []rxItem
data int
// bufs is the set of recvmmsg buffers the items point into, owned by the batch from handoff until it is reused.
bufs [][]byte
// chunks are the decrypt jobs, nChunks of them in use. With none the writer decrypts the batch itself.
chunks []rxChunk
nChunks int
}
func (b *rxBatch) reset() {
clear(b.items)
b.items = b.items[:0]
b.data = 0
for i := range b.nChunks {
b.chunks[i].items = nil
}
b.nChunks = 0
}
// stagedRx is the staged receive pipeline of one udp listener. deliver and handoff run on the listener goroutine.
type stagedRx struct {
f *Interface
q int
pool *flowPool
// h and hostmapCache are the listener's own. wrxc is the commit context: the writer's, or the listener's while
// nothing is in flight.
h *header.H
hostmapCache map[uint32]*HostInfo
wrxc *rxContext
cur *rxBatch
free []*rxBatch
ordered chan *rxBatch
returned chan *rxBatch
inFlight int
writerDone chan struct{}
metricBatches metrics.Counter
metricInline metrics.Counter
}
func newStagedRx(f *Interface, q int, wrxc *rxContext) *stagedRx {
p := &stagedRx{
f: f,
q: q,
h: &header.H{},
hostmapCache: map[uint32]*HostInfo{},
wrxc: wrxc,
ordered: make(chan *rxBatch, f.staged.rxSlots),
returned: make(chan *rxBatch, f.staged.rxSlots),
writerDone: make(chan struct{}),
metricBatches: metrics.GetOrRegisterCounter("flow.rx.batches", nil),
metricInline: metrics.GetOrRegisterCounter("flow.rx.inline", nil),
}
workers := f.staged.workers
p.pool = newFlowPool(workers, f.staged.rxSlots*2*workers)
for range f.staged.rxSlots {
// Two chunks per worker, so the writer can start committing while the workers are still on the rest
b := &rxBatch{items: make([]rxItem, 0, 256), chunks: make([]rxChunk, 2*workers)}
for i := range b.chunks {
b.chunks[i].done = make(chan struct{}, 1)
}
p.free = append(p.free, b)
}
p.cur = p.free[len(p.free)-1]
p.free = p.free[:len(p.free)-1]
go p.writer()
return p
}
// listen runs the conn's receive loop through the pipeline until it fails, then drains the pipeline. The conn hands
// its buffers over, so a batch moves between stages without a copy.
func (p *stagedRx) listen(li handoffConn) error {
defer p.close()
return li.ListenOutHandoff(p.deliver, p.handoff)
}
// deliver is the listener's per-packet work: classify the packet and add it to the current batch. Nothing here
// decrypts or touches the tun.
func (p *stagedRx) deliver(via netip.AddrPort, packet []byte) {
b := p.cur
it := rxItem{packet: packet, via: via}
p.classify(&it)
if it.ci != nil {
b.data++
}
b.items = append(b.items, it)
}
// classify fills in the tunnel for a plain data packet of an established, non-relayed tunnel, leaving anything else
// for the writer to handle whole. The replay pre-check waits for the end of the batch, see precheck.
func (p *stagedRx) classify(it *rxItem) {
f := p.f
h := p.h
packet := it.packet
if h.Parse(packet) != nil || h.Version != header.Version || h.Type != header.Message || h.Subtype != header.MessageNone {
return
}
if f.myVpnNetworksTable.Contains(it.via.Addr()) {
return
}
hostinfo := f.hostMap.QueryIndexCached(h.RemoteIndex, p.hostmapCache)
if hostinfo == nil || hostinfo.ConnectionState == nil {
return
}
ci := hostinfo.ConnectionState
if len(packet) < header.Len+ci.dKey.Overhead() {
return
}
it.hostinfo, it.ci, it.counter = hostinfo, ci, h.MessageCounter
}
// precheck marks obvious replays before anything decrypts them, like ConnectionState.Decrypt does, taking each
// tunnel's lock once per run of its packets rather than once per packet: the writer takes the same lock for its
// window updates, from another core. The writer checks again when it commits. The window only moves forward in
// between, so a stale check here can only let a replay through to that second check. Marked packets keep their
// place: an earlier RecvError may delete their tunnel, and inline receive then replies with a RecvError rather than
// dropping a replay.
func (p *stagedRx) precheck(b *rxBatch) {
f := p.f
items := b.items
for i := 0; i < len(items); {
ci := items[i].ci
if ci == nil {
i++
continue
}
ci.decryptLock.Lock()
for ; i < len(items) && items[i].ci == ci; i++ {
if ci.window.Check(f.l, items[i].counter) {
continue
}
b.data--
items[i].err = ErrAlreadySeen
}
ci.decryptLock.Unlock()
}
}
// handoff ends a batch: the batch takes bufs, and the conn reads the next batch into the buffers of the next free
// batch.
func (p *stagedRx) handoff(bufs [][]byte) [][]byte {
p.cur.bufs = bufs
p.endBatch()
b := p.cur
if len(b.bufs) != len(bufs) || len(b.bufs[0]) < len(bufs[0]) {
b.bufs = make([][]byte, len(bufs))
for i := range b.bufs {
b.bufs[i] = make([]byte, len(bufs[0]))
}
}
return b.bufs
}
// endBatch hands the batch to the workers and the ordered writer, then takes a free batch.
func (p *stagedRx) endBatch() {
clear(p.hostmapCache)
p.reap()
b := p.cur
p.precheck(b)
n := len(b.items)
if n == 0 {
b.reset()
return
}
if p.inFlight == 0 && n < p.f.staged.inlineBelow {
// The writer is idle with nothing in flight, so the listener may commit with its context and flush the tun
p.metricInline.Inc(1)
p.process(b, p.wrxc)
b.reset()
return
}
parts := min(len(b.chunks), b.data/rxChunkMin)
p.metricBatches.Inc(1)
if parts > 0 {
size := (n + parts - 1) / parts
for off := 0; off < n; off += size {
c := &b.chunks[b.nChunks]
c.items = b.items[off:min(off+size, n)]
b.nChunks++
}
}
// The jobs go out before the batch does: once the writer has it, it may finish and reset it at any moment
for i := range b.nChunks {
p.pool.jobs <- &b.chunks[i]
}
p.inFlight++
p.ordered <- b
if len(p.free) == 0 {
// Every batch is in flight, wait for the writer: the socket buffer absorbs the backlog meanwhile.
p.release(<-p.returned)
}
p.cur = p.free[len(p.free)-1]
p.free = p.free[:len(p.free)-1]
}
// reap takes back every batch the writer has finished with, without waiting.
func (p *stagedRx) reap() {
for {
select {
case b := <-p.returned:
p.release(b)
default:
return
}
}
}
func (p *stagedRx) release(b *rxBatch) {
p.inFlight--
p.free = append(p.free, b)
}
// process is the ordered stage for one batch: decrypt whatever no worker is decrypting, commit every item in arrival
// order, and flush the tun batcher.
func (p *stagedRx) process(b *rxBatch, rxc *rxContext) {
if b.nChunks == 0 {
decryptRxItems(b.items, rxc.nb)
p.commit(b.items, rxc)
} else {
// Commit chunk by chunk, so the commit overlaps the workers still on later chunks
for i := range b.nChunks {
c := &b.chunks[i]
<-c.done
p.commit(c.items, rxc)
}
}
p.f.flushOutside(rxc)
}
// commit runs the ordered stage over items in arrival order. A non-data packet goes through readOutsidePackets in
// its place, a run of data packets on one tunnel through commitRun.
func (p *stagedRx) commit(items []rxItem, rxc *rxContext) {
for i := 0; i < len(items); {
ci := items[i].ci
if ci == nil {
p.f.readOutsidePackets(ViaSender{UdpAddr: items[i].via}, items[i].packet, rxc)
i++
continue
}
j := i + 1
for j < len(items) && items[j].ci == ci {
j++
}
p.commitRun(items[i:j], rxc)
i = j
}
}
// commitRun commits a run of decrypted data packets on one tunnel. The window updates go first, under one lock, then
// the rest; handling a data packet never touches the window, so that is the same as one packet at a time.
func (p *stagedRx) commitRun(run []rxItem, rxc *rxContext) {
f := p.f
ci := run[0].ci
// Classification is speculative: earlier control packets may have removed this tunnel. Resolve it through the
// writer's batch-local cache, exactly where inline processing would, before touching replay state.
index := binary.BigEndian.Uint32(run[0].packet[4:8])
hostinfo := f.hostMap.QueryIndexCached(index, rxc.hostmapCache)
if hostinfo != run[0].hostinfo || hostinfo.ConnectionState != ci {
if hostinfo == nil {
for i := range run {
f.maybeSendRecvError(run[i].via, index)
}
}
return
}
ci.decryptLock.Lock()
for i := range run {
// A packet that failed to decrypt does not touch the window, like ConnectionState.Decrypt
if run[i].err == nil && !ci.window.Update(f.l, run[i].counter) {
run[i].err = ErrAlreadySeen
}
}
ci.decryptLock.Unlock()
for i := range run {
it := &run[i]
if f.finishOutsideDecrypt(it.hostinfo, ViaSender{UdpAddr: it.via}, it.counter, it.err) {
f.handleOutsideMessagePacket(it.hostinfo, it.counter, it.out, rxc)
}
}
}
// writer is the ordered receive stage: window, firewall, coalescer and tun write, one batch at a time in arrival
// order.
func (p *stagedRx) writer() {
defer close(p.writerDone)
for b := range p.ordered {
if p.f.closed.Load() {
// The tun is going away, a write would only log errors. Still wait out the workers on the batch.
for i := range b.nChunks {
<-b.chunks[i].done
}
} else {
p.process(b, p.wrxc)
}
b.reset()
p.returned <- b
}
}
// close commits anything still in flight, in order, and stops the writer and the workers.
func (p *stagedRx) close() {
if len(p.cur.items) > 0 {
p.endBatch()
}
close(p.ordered)
<-p.writerDone
p.pool.close()
}
-237
View File
@@ -1,237 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
import (
"fmt"
"net/netip"
"sync/atomic"
"testing"
"time"
"github.com/flynn/noise"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
// TestFlowStagedRXOrderAndReplay feeds every packet behind a forgery of it, from another address, and a replay of
// it, then a test packet that jumps the window and the whole stream again, with shuffled workers and a slow tun,
// through small and large batches: the tun sees each segment once, in order, the window marks them all, the forgeries
// neither consume a counter nor roam, and the jump does not get the earlier data packets of its batch rejected.
func TestFlowStagedRXOrderAndReplay(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
for _, batchLen := range []int{8, 1000} {
t.Run(fmt.Sprintf("%s/batch=%d", cipher.CipherName(), batchLen), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, cipher, stagedOn(4))
delayCrypto(receiver, func(n uint64) {
if n%32 == 0 {
time.Sleep(time.Millisecond)
}
})
receiver.tun.delay = 100 * time.Microsecond
batches, want := flowTestBatches(30, 4, 1, 10)
wire := wireFor(t, sender, batches)
var feed [][]byte
for i, p := range wire {
// Corrupt the tag or the payload
bad := append([]byte(nil), p...)
bad[[]int{len(p) - 1, header.Len + 3}[i%2]] ^= 1
feed = append(feed, bad, p, p)
}
// The jump lands in a batch with fresh data before it, at batch=1000
sender.hostinfo.ConnectionState.messageCounter.Add(2 * ReplayWindow)
feed = append(append(feed, sendControl(sender, header.Test, header.TestReply)), wire...)
forged := func(i int) netip.AddrPort {
if i < 3*len(wire) && i%3 == 0 {
return netip.MustParseAddrPort("198.51.100.9:4242")
}
return flowTestUnder
}
rx0 := rxBatches.Count()
conn := &rxConn{wire: feed, batchLen: batchLen, from: forged}
receiver.receive(conn)
require.Positive(t, rxBatches.Count()-rx0)
require.Equal(t, (len(feed)+batchLen-1)/batchLen, conn.swaps, "every batch is handed off")
require.Equal(t, want, receiver.tun.segments(t), "every packet exactly once, in order")
require.Equal(t, flowTestUnder, receiver.hostinfo.GetRemote())
for _, p := range wire {
h := &header.H{}
require.NoError(t, h.Parse(p))
require.False(t, receiver.hostinfo.ConnectionState.window.Check(receiver.f.l, h.MessageCounter), "counter %d must be marked seen", h.MessageCounter)
}
})
}
}
}
// TestFlowStagedRXClose: a CloseTunnel in the middle of a batch is handled in its place, by the writer. The data after
// it in the batch is still delivered from the batch's hostmap cache, as inline receive does.
func TestFlowStagedRXClose(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
batches, want := flowTestBatches(4, 4, 1, 8)
wire := wireFor(t, sender, batches)
half := len(wire) / 2
feed := append(append(append([][]byte{}, wire[:half]...), sendControl(sender, header.CloseTunnel, 0)), wire[half:]...)
receiver.receive(&rxConn{wire: feed, batchLen: len(feed)})
require.Equal(t, want, receiver.tun.segments(t))
require.Nil(t, receiver.f.hostMap.QueryIndex(receiver.hostinfo.localIndexId))
}
// sendControl returns the wire packet of a control message from sender.
func sendControl(sender *flowTestPeer, typ header.MessageType, st header.MessageSubType) []byte {
sender.f.sendNoMetrics(typ, st, sender.hostinfo.ConnectionState, sender.hostinfo, flowTestUnder, nil, make([]byte, 12), make([]byte, mtu), 0)
wire := sender.conn.captured()
sender.conn.pkts, sender.conn.addrs = nil, nil
return wire[len(wire)-1]
}
// receivePaths is inline, staged and, with the threshold, adaptive receive.
func receivePaths() []stagedConfig {
return []stagedConfig{{}, stagedOn(2), adaptive(2)}
}
// TestFlowStagedRXRecvError feeds a RecvError that deletes the tunnel, then a known replay and new data for it, in one
// batch. Every receive path must drop the data through its own view of the hostmap, without touching the replay window,
// and answer each packet with a RecvError, as inline receive does, without decrypting the replay again.
func TestFlowStagedRXRecvError(t *testing.T) {
for _, sc := range receivePaths() {
t.Run(stagedName(sc), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
receiver.f.messageMetrics = newMessageMetrics()
receiver.f.acceptRecvErrorConfig = recvErrorAlways
receiver.f.sendRecvErrorConfig = recvErrorAlways
batches, want := flowTestBatches(1, 1, 1, 33)
wire := wireFor(t, sender, batches)
rxc := newRxContext(receiver.f, 0)
receiver.f.readOutsidePackets(ViaSender{UdpAddr: flowTestUnder}, append([]byte(nil), wire[0]...), rxc)
receiver.f.flushOutside(rxc)
h := &header.H{}
require.NoError(t, h.Parse(wire[0]))
replay := h.MessageCounter
var decrypts atomic.Int64
delayCrypto(receiver, func(n uint64) {
if n == replay {
decrypts.Add(1)
}
})
ctl := header.Encode(make([]byte, header.Len), header.Version, header.RecvError, 0, receiver.hostinfo.remoteIndexId, 0)
conn := &rxConn{wire: append([][]byte{ctl}, wire...), batchLen: len(wire) + 1}
receiver.receive(conn)
require.Nil(t, receiver.f.hostMap.QueryIndex(receiver.hostinfo.localIndexId))
require.Equal(t, want[:1], receiver.tun.segments(t))
require.Zero(t, decrypts.Load(), "known replays must not be decrypted again")
replies := conn.captured()
require.Len(t, replies, len(wire))
for i, reply := range replies {
require.NoError(t, h.Parse(reply))
require.Equal(t, header.RecvError, h.Type)
require.NoError(t, h.Parse(wire[i]))
require.True(t, i == 0 || receiver.hostinfo.ConnectionState.window.Check(nil, h.MessageCounter), "dropped data must not update replay state")
}
})
}
}
// TestFlowStagedRXReplacedTunnel classifies a data packet, then a close and a new handshake replace its tunnel at the
// same index before the writer commits it, as when the listener runs ahead: it must be dropped untouched.
func TestFlowStagedRXReplacedTunnel(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
receiver.f.messageMetrics = newMessageMetrics()
batches, _ := flowTestBatches(1, 1, 1, 1)
wire := wireFor(t, sender, batches)
closePacket := sendControl(sender, header.CloseTunnel, 0)
p := &stagedRx{f: receiver.f, h: &header.H{}, hostmapCache: map[uint32]*HostInfo{}}
item := rxItem{packet: wire[0], via: flowTestUnder}
p.classify(&item)
require.NotNil(t, item.ci)
rxc := newRxContext(receiver.f, 0)
p.process(&rxBatch{items: []rxItem{{packet: closePacket, via: flowTestUnder}}}, rxc)
require.Nil(t, receiver.f.hostMap.QueryIndex(receiver.hostinfo.localIndexId))
_, replacement := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
receiver.f.hostMap.Lock()
receiver.f.hostMap.unlockedAddHostInfo(replacement.hostinfo, receiver.f)
receiver.f.hostMap.Unlock()
p.process(&rxBatch{items: []rxItem{item}}, rxc)
require.Empty(t, receiver.tun.writes)
require.True(t, item.ci.window.Check(nil, item.counter))
require.True(t, replacement.hostinfo.ConnectionState.window.Check(nil, item.counter))
}
// TestFlowStagedRXRealSocket exercises buffer handoff through the real Linux UDP receive loop.
func TestFlowStagedRXRealSocket(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(3))
batches, want := flowTestBatches(60, 4, 1, 8)
wire := wireFor(t, sender, batches)
l := receiver.f.l
conn, err := udp.NewListener(l, udp.Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Batch: 16, Offloads: true})
require.NoError(t, err)
addr, err := conn.LocalAddr()
require.NoError(t, err)
out, err := udp.NewListener(l, udp.Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Batch: 16, Offloads: true})
require.NoError(t, err)
defer out.Close()
// The sender's address is the tunnel's remote, so no roaming
from, err := out.LocalAddr()
require.NoError(t, err)
receiver.hostinfo.remote.Store(&from)
done := make(chan struct{})
go func() { receiver.receive(conn); close(done) }()
// Send in bursts small enough for the default socket buffer
for off := 0; off < len(wire); off += 32 {
end := min(off+32, len(wire))
addrs := make([]netip.AddrPort, end-off)
for i := range addrs {
addrs[i] = addr
}
_, err := out.WriteBatch(wire[off:end], addrs)
require.NoError(t, err)
require.Eventually(t, func() bool { return len(receiver.tun.segments(t)) >= end }, 5*time.Second, 100*time.Microsecond)
}
receiver.f.closed.Store(true)
require.NoError(t, conn.Close())
<-done
require.Equal(t, want, receiver.tun.segments(t))
}
// TestFlowStagedRXCloseFlushesPartial: packets the listener holds in a batch it never got to end when the conn fails
// are still committed, in order, on a live interface.
func TestFlowStagedRXCloseFlushesPartial(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
batches, want := flowTestBatches(11, 4, 1, 8)
wire := wireFor(t, sender, batches)
p := newStagedRx(receiver.f, 0, newRxContext(receiver.f, 0))
for i, packet := range wire {
p.deliver(flowTestUnder, packet)
if (i+1)%64 == 0 {
p.endBatch()
}
}
p.close()
require.Equal(t, want, receiver.tun.segments(t))
}
// TestFlowStagedRXInlineBelow: small idle batches stay on the listener, larger batches still use the ordered writer.
func TestFlowStagedRXInlineBelow(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sc := adaptive(3)
sc.inlineBelow = 16
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
batches, want := flowTestBatches(40, 4, 1, 16)
wire := wireFor(t, sender, batches)
inline := metrics.GetOrRegisterCounter("flow.rx.inline", nil)
inline0, rx0 := inline.Count(), rxBatches.Count()
receiver.receive(&rxConn{wire: wire, sizes: []int{8, 8, 8, 128, 128, 8, 8, 128, 4, 128}})
require.Equal(t, want, receiver.tun.segments(t))
require.Positive(t, inline.Count()-inline0)
require.Positive(t, rxBatches.Count()-rx0)
}
-536
View File
@@ -1,536 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
import (
"encoding/binary"
"io"
"math"
"net/netip"
"os"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
// flowTestMSS is the segment payload size of the synthetic TSO superpackets.
const flowTestMSS = 1200
var (
flowTestLocal = netip.MustParseAddr("10.0.0.1")
flowTestRemote = netip.MustParseAddr("10.0.0.2")
flowTestUnder = netip.MustParseAddrPort("192.0.2.2:4242")
)
// flowTCPSuperpacket builds an IPv4/TCP TSO superpacket from local to remote carrying segs segments. Segment i's
// payload starts with the flow id, the superpacket seq and i, so a receiver can check what it got and in which order.
func flowTCPSuperpacket(flow uint16, seq uint32, segs int) tio.Packet {
const hdr = 40
pay := segs * flowTestMSS
b := make([]byte, hdr+pay)
b[0] = 0x45
binary.BigEndian.PutUint16(b[2:4], uint16(min(hdr+pay, 65535)))
b[8] = 64
b[9] = iputil.IPProtocolTCP
copy(b[12:16], flowTestLocal.AsSlice())
copy(b[16:20], flowTestRemote.AsSlice())
binary.BigEndian.PutUint16(b[20:22], 10000+flow)
binary.BigEndian.PutUint16(b[22:24], 80)
binary.BigEndian.PutUint32(b[24:28], seq*uint32(pay))
b[32] = 0x50
b[33] = 0x10 // ACK
binary.BigEndian.PutUint16(b[34:36], 65535)
for i := range segs {
p := b[hdr+i*flowTestMSS:]
binary.BigEndian.PutUint16(p[0:2], flow)
binary.BigEndian.PutUint32(p[2:6], seq)
binary.BigEndian.PutUint16(p[6:8], uint16(i))
}
if segs == 1 {
return tio.Packet{Bytes: b}
}
return tio.Packet{Bytes: b, GSO: tio.GSOInfo{Size: flowTestMSS, HdrLen: hdr, CsumStart: 20, Proto: tio.GSOProtoTCP}}
}
// decodedSeg identifies one segment by the marker flowTCPSuperpacket put in its payload.
type decodedSeg struct {
flow uint16
seq uint32
idx uint16
}
func decodeSeg(p []byte) decodedSeg {
return decodedSeg{flow: binary.BigEndian.Uint16(p[0:2]), seq: binary.BigEndian.Uint32(p[2:6]), idx: binary.BigEndian.Uint16(p[6:8])}
}
// flowTestBatches builds n reads of per superpackets of segs segments, across flows interleaved flows, and the
// segments they carry in order.
func flowTestBatches(n, per, flows, segs int) ([][]tio.Packet, []decodedSeg) {
var batches [][]tio.Packet
var want []decodedSeg
seq := make([]uint32, flows)
for range n {
var b []tio.Packet
for j := range per {
flow := uint16(j % flows)
b = append(b, flowTCPSuperpacket(flow, seq[flow], segs))
for s := range segs {
want = append(want, decodedSeg{flow: flow, seq: seq[flow], idx: uint16(s)})
}
seq[flow]++
}
batches = append(batches, b)
}
return batches, want
}
// flowTestPeer is one end of a tunnel: the interface, the hostinfo for the other end, and its capture points.
type flowTestPeer struct {
f *Interface
hostinfo *HostInfo
conn *flowCaptureConn
tun *flowTunCapture
}
// newFlowTestPair builds two interfaces joined by one real tunnel (keys from an actual handshake). sender sends from
// flowTestLocal to flowTestRemote, receiver is the far side.
func newFlowTestPair(t testing.TB, cipher noise.CipherFunc, sc stagedConfig) (sender, receiver *flowTestPeer) {
t.Helper()
initR, respR := runTestHandshakeWith(t, cipher)
sci, err := newConnectionStateFromResult(initR)
require.NoError(t, err)
rci, err := newConnectionStateFromResult(respR)
require.NoError(t, err)
sender = newFlowTestPeer(t, flowTestLocal, flowTestRemote, sci, 100, 200)
receiver = newFlowTestPeer(t, flowTestRemote, flowTestLocal, rci, 200, 100)
sender.f.staged, receiver.f.staged = sc, sc
return sender, receiver
}
func newFlowTestPeer(t testing.TB, me, them netip.Addr, ci *ConnectionState, localIdx, remoteIdx uint32) *flowTestPeer {
l := test.NewLogger()
hm := newHostMap(l)
hm.preferredRanges.Store(&[]netip.Prefix{})
myNets, myAddrs := &bart.Lite{}, &bart.Lite{}
myNets.Insert(netip.PrefixFrom(me, 24))
myAddrs.Insert(netip.PrefixFrom(me, 32))
myCert := &dummyCert{version: cert.Version1, networks: []netip.Prefix{netip.PrefixFrom(me, 24)}}
theirCert := &dummyCert{version: cert.Version1, networks: []netip.Prefix{netip.PrefixFrom(them, 24)}, name: "peer"}
ci.peerCert = &cert.CachedCertificate{Certificate: theirCert, InvertedGroups: map[string]struct{}{}}
fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, myCert)
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
conn, tun := &flowCaptureConn{}, &flowTunCapture{}
lh := newTestLighthouse()
lh.remoteAllowList.Store(&RemoteAllowList{})
f := &Interface{
hostMap: hm,
outside: conn,
firewall: fw,
lightHouse: lh,
pki: &PKI{},
myVpnNetworksTable: myNets,
myVpnAddrsTable: myAddrs,
routines: 1,
writers: []udp.Conn{conn},
messageMetrics: &MessageMetrics{txExhausted: metrics.NewCounter()},
metricTxDropped: metrics.NilCounter{},
l: l,
}
f.ctx = t.Context()
f.pki.caPool.Store(cert.NewCAPool())
// Never promote or requery, those would put extra test packets on the wire
f.tryPromoteEvery.Store(math.MaxUint32)
f.reQueryEvery.Store(math.MaxUint32)
f.batchers = []*batch.MultiCoalescer{batch.NewMultiCoalescer(tun, l)}
f.connectionManager = &connectionManager{
intf: f,
relayUsed: map[uint32]struct{}{},
relayUsedLock: &sync.RWMutex{},
trafficTimer: NewLockingTimerWheel[uint32](time.Second, time.Minute),
}
f.handshakeManager = NewHandshakeManager(l, hm, lh, conn, defaultHandshakeConfig)
f.handshakeManager.f = f
hostinfo := &HostInfo{
ConnectionState: ci,
vpnAddrs: []netip.Addr{them},
localIndexId: localIdx,
remoteIndexId: remoteIdx,
relayState: RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}},
}
hostinfo.remote.Store(&flowTestUnder)
hostinfo.buildNetworks(myNets, theirCert)
hm.unlockedAddHostInfo(hostinfo, f)
return &flowTestPeer{f: f, hostinfo: hostinfo, conn: conn, tun: tun}
}
// flowCaptureConn records every packet sent, in order, as copies.
type flowCaptureConn struct {
udp.NoopConn
mu sync.Mutex
pkts [][]byte
addrs []netip.AddrPort
// delay, when set, slows every WriteBatch down so batches pile up in flight
delay time.Duration
}
func (c *flowCaptureConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
time.Sleep(c.delay)
c.mu.Lock()
defer c.mu.Unlock()
for i, b := range bufs {
c.pkts = append(c.pkts, append([]byte(nil), b...))
c.addrs = append(c.addrs, addrs[i])
}
return len(bufs), nil
}
func (c *flowCaptureConn) WriteTo(b []byte, addr netip.AddrPort) error {
_, err := c.WriteBatch([][]byte{b}, []netip.AddrPort{addr})
return err
}
func (c *flowCaptureConn) captured() [][]byte {
c.mu.Lock()
defer c.mu.Unlock()
return append([][]byte(nil), c.pkts...)
}
// flowTunCapture is a tun queue that records every plaintext write.
type flowTunCapture struct {
mu sync.Mutex
writes [][]byte
// delay, when set, slows every write down
delay time.Duration
}
func (c *flowTunCapture) Read() ([]tio.Packet, error) { return nil, io.EOF }
func (c *flowTunCapture) Close() error { return nil }
func (c *flowTunCapture) Write(b []byte) (int, error) {
time.Sleep(c.delay)
c.mu.Lock()
defer c.mu.Unlock()
c.writes = append(c.writes, append([]byte(nil), b...))
return len(b), nil
}
// segments decodes what reached the tun. The coalescer may have merged segments back into superpackets, so this
// walks each payload in MSS steps.
func (c *flowTunCapture) segments(t testing.TB) []decodedSeg {
t.Helper()
c.mu.Lock()
defer c.mu.Unlock()
var out []decodedSeg
for _, w := range c.writes {
pay := w[40:]
require.Zero(t, len(pay)%flowTestMSS, "tun write payload %d bytes", len(pay))
for off := 0; off < len(pay); off += flowTestMSS {
out = append(out, decodeSeg(pay[off:]))
}
}
return out
}
// decryptWire checks every wire packet decrypts on the receiver, in order, with strictly increasing counters, and
// returns what each one carried. It uses its own replay window and copies, so the wire can still be fed to the
// receiver afterwards.
func decryptWire(t testing.TB, receiver *flowTestPeer, wire [][]byte) []decodedSeg {
t.Helper()
peer := receiver.hostinfo.ConnectionState
ci := &ConnectionState{dKey: peer.dKey, window: NewBits(peer.window.length)}
nb := make([]byte, 12)
var last uint64
out := make([]decodedSeg, 0, len(wire))
for i, pkt := range wire {
h := &header.H{}
require.NoError(t, h.Parse(pkt))
require.Equal(t, header.Message, h.Type)
require.Equal(t, receiver.hostinfo.localIndexId, h.RemoteIndex)
require.Greater(t, h.MessageCounter, last, "wire packet %d: counter %d after %d", i, h.MessageCounter, last)
last = h.MessageCounter
plain, err := ci.Decrypt(receiver.f.l, h.MessageCounter, append([]byte(nil), pkt...), nb)
require.NoError(t, err, "wire packet %d", i)
out = append(out, decodeSeg(plain[40:]))
}
return out
}
// flowBatchQueue is a handoffQueue that hands out scripted Read batches, then fails with io.EOF. Every batch is copied
// into the current receive buffer, so a test sees the buffer ownership the Offload queue has.
type flowBatchQueue struct {
batches [][]tio.Packet
rxBuf []byte
out []tio.Packet
closed chan struct{}
// reads counts the batches handed out
reads atomic.Int64
}
func newFlowBatchQueue(batches [][]tio.Packet) *flowBatchQueue {
return &flowBatchQueue{batches: batches, rxBuf: make([]byte, 8*65536), closed: make(chan struct{})}
}
func (q *flowBatchQueue) Read() ([]tio.Packet, error) {
select {
case <-q.closed:
return nil, os.ErrClosed
default:
}
n := int(q.reads.Load())
if n == len(q.batches) {
return nil, io.EOF
}
q.out = q.out[:0]
off := 0
for _, p := range q.batches[n] {
l := copy(q.rxBuf[off:], p.Bytes)
q.out = append(q.out, tio.Packet{Bytes: q.rxBuf[off : off+l : off+l], GSO: p.GSO})
off += l
}
q.reads.Add(1)
return q.out, nil
}
func (q *flowBatchQueue) SwapRxBuf(buf []byte) []byte {
if buf == nil {
buf = make([]byte, len(q.rxBuf))
}
old := q.rxBuf
q.rxBuf = buf
// Scribble over the buffer we are handed, a staged batch that still pointed into it would be corrupted
for i := range buf {
buf[i] = 0xa5
}
return old
}
func (q *flowBatchQueue) Write(b []byte) (int, error) { return len(b), nil }
func (q *flowBatchQueue) Close() error { close(q.closed); return nil }
// delayedCipher changes scheduling through the real cipher interface, only in tests.
type delayedCipher struct {
noiseutil.CipherState
before func(uint64)
}
func (c delayedCipher) EncryptDanger(out, ad, plain []byte, n uint64, nb []byte) ([]byte, error) {
c.before(n)
return c.CipherState.EncryptDanger(out, ad, plain, n, nb)
}
func (c delayedCipher) DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error) {
c.before(n)
return c.CipherState.DecryptDanger(out, ad, ciphertext, n, nb)
}
func delayCrypto(peer *flowTestPeer, before func(uint64)) {
ci := peer.hostinfo.ConnectionState
ci.eKey = delayedCipher{ci.eKey, before}
ci.dKey = delayedCipher{ci.dKey, before}
}
// stagedOn enables staging with n workers.
func stagedOn(n int) stagedConfig {
return stagedConfig{workers: n, tx: true, rx: true, txSlots: stagedTxSlots, rxSlots: 6}
}
// adaptive is staging with the inline threshold and gate hold parseStagedConfig sets.
func adaptive(n int) stagedConfig {
return stagedConfig{workers: n, tx: true, rx: true, inlineBelow: 256, gateHold: stagedGateHold,
txSlots: stagedTxSlots, rxSlots: 6}
}
func stagedName(sc stagedConfig) string {
switch {
case sc.workers == 0:
return "inline"
case sc.inlineBelow > 0:
return "adaptive"
default:
return "staged"
}
}
// send takes one read through the pipeline, as run does.
func (p *stagedTx) send(pkts []tio.Packet) {
p.handle(pkts, &firewall.ParsedPacket{}, make([]byte, mtu), nil)
}
// drain waits until every batch p has in flight is on the wire.
func (p *stagedTx) drain() {
for p.inFlight > 0 {
p.release(<-p.returned)
}
}
// TestFlowStagedFallback: a tun queue that cannot hand off its buffer, or a conn that cannot hand off its batches,
// keeps that direction on the inline path.
func TestFlowStagedFallback(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(3))
batches, want := flowTestBatches(10, 2, 1, 4)
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
tx0, rx0 := tx.Count(), rxBatches.Count()
sender.f.listenIn(plainQueue{newFlowBatchQueue(batches)}, 0)
wire := sender.conn.captured()
require.Equal(t, want, decryptWire(t, receiver, wire))
require.Equal(t, tx0, tx.Count())
receiver.receive(plainConn{&rxConn{wire: wire, batchLen: 16}})
require.Equal(t, want, receiver.tun.segments(t))
require.Equal(t, rx0, rxBatches.Count())
}
// plainQueue and plainConn hide SwapRxBuf and ListenOutHandoff, so they cannot hand off their batches.
type plainQueue struct{ tio.Queue }
// wireFor produces the wire packets the sender emits for batches, inline.
func wireFor(t testing.TB, sender *flowTestPeer, batches [][]tio.Packet) [][]byte {
t.Helper()
sc := sender.f.staged
sender.f.staged = stagedConfig{}
sender.f.listenIn(newFlowBatchQueue(batches), 0)
sender.f.staged = sc
wire := sender.conn.captured()
sender.conn.pkts, sender.conn.addrs = nil, nil
return wire
}
// rxConn plays wire back to listenOut through the buffer handoff, in batches of batchLen or of the scripted sizes
// (the last one repeating), and captures what is sent back. Every buffer handed back is scribbled over before it is
// read into, so a stage still using one shows corruption. from, when set, picks each packet's source address by its
// index in wire.
type rxConn struct {
flowCaptureConn
wire [][]byte
batchLen int
sizes []int
from func(i int) netip.AddrPort
swaps int
}
func (c *rxConn) ListenOut(r udp.EncReader, flush func()) error {
return c.ListenOutHandoff(r, func(bufs [][]byte) [][]byte {
flush()
return bufs
})
}
func (c *rxConn) ListenOutHandoff(r udp.EncReader, handoff func(bufs [][]byte) [][]byte) error {
n := c.batchLen
for _, size := range c.sizes {
n = max(n, size)
}
bufs := make([][]byte, n)
for i := range bufs {
bufs[i] = make([]byte, udp.MTU)
}
for off, k := 0, 0; off < len(c.wire); k++ {
size := c.batchLen
if len(c.sizes) > 0 {
size = c.sizes[min(k, len(c.sizes)-1)]
}
end := min(off+size, len(c.wire))
for i, p := range c.wire[off:end] {
n := copy(bufs[i], p)
from := flowTestUnder
if c.from != nil {
from = c.from(off + i)
}
r(from, bufs[i][:n:n])
}
next := handoff(bufs)
if len(next) != len(bufs) {
panic("handoff returned the wrong number of buffers")
}
bufs = next
for _, b := range bufs {
for j := range b[:udp.MTU] {
b[j] = 0x5a
}
}
c.swaps++
off = end
}
return os.ErrClosed
}
type plainConn struct{ udp.Conn }
var rxBatches = metrics.GetOrRegisterCounter("flow.rx.batches", nil)
// receive runs the peer's listenOut over conn until conn runs out.
func (p *flowTestPeer) receive(conn udp.Conn) {
p.f.writers = []udp.Conn{conn}
p.f.outside = conn
p.f.listenOut(0)
}
// TestFlowStagedFirewallParity: both paths apply the same outbound and inbound policy. An authenticated inbound
// denial still consumes its replay counter, exactly as inline Decrypt does.
func TestFlowStagedFirewallParity(t *testing.T) {
for _, sc := range []stagedConfig{{}, stagedOn(2)} {
t.Run(stagedName(sc), func(t *testing.T) {
batches, _ := flowTestBatches(4, 3, 1, 8)
sender, _ := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
sender.f.firewall.OutRules = newFirewallTable()
sender.f.listenIn(newFlowBatchQueue(batches), 0)
require.Empty(t, sender.conn.captured())
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
wire := wireFor(t, sender, batches)
receiver.f.firewall.InRules = newFirewallTable()
receiver.receive(&rxConn{wire: wire, batchLen: 64})
require.Empty(t, receiver.tun.writes)
for _, pkt := range wire {
h := &header.H{}
require.NoError(t, h.Parse(pkt))
require.False(t, receiver.hostinfo.ConnectionState.window.Check(receiver.f.l, h.MessageCounter))
}
})
}
}
// TestFlowStagedCipherFallback runs the parsed configuration with the real AES cipher: transmit falls back to inline
// when the build needs nonce-ordered encryption (BoringCrypto, FIPS), receive decryption stays staged.
func TestFlowStagedCipherFallback(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
c.Settings = map[string]any{"tun": map[string]any{"staged": map[string]any{"workers": 2}}}
sender, receiver := newFlowTestPair(t, noiseutil.CipherAESGCM, parseStagedConfig(c, l))
require.Equal(t, !noiseutil.EncryptLockNeeded, sender.f.staged.tx)
require.True(t, sender.f.staged.rx)
batches, want := flowTestBatches(4, 8, 1, 45)
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
tx0, rx0 := tx.Count(), rxBatches.Count()
sender.f.listenIn(newFlowBatchQueue(batches), 0)
if noiseutil.EncryptLockNeeded {
require.Equal(t, tx0, tx.Count())
}
receiver.receive(&rxConn{wire: sender.conn.captured(), batchLen: 512})
require.Greater(t, rxBatches.Count(), rx0)
require.Equal(t, want, receiver.tun.segments(t))
}
-583
View File
@@ -1,583 +0,0 @@
package nebula
import (
"errors"
"net/netip"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/udp"
)
var errFlowSegmentOverrun = errors.New("superpacket has more segments than counters reserved for it")
// txItem is one firewall-approved inside packet (or superpacket) with everything needed to encrypt it. A nonzero
// counter is the first of the segs counters reserved for it on ci, relayCounter the same on the relay tunnel when
// relayHI is set. Zero counters are allocated per segment while encrypting, as inline sends do.
type txItem struct {
pkt tio.Packet
hostinfo *HostInfo
ci *ConnectionState
remote netip.AddrPort
segs uint64
counter uint64
relayHI *HostInfo
relay *Relay
relayCounter uint64
}
// prepareTxItem does the per-packet work sendInsideMessage does before it encrypts, and resolves where the packet goes.
func (f *Interface) prepareTxItem(it *txItem, hostinfo *HostInfo, pkt tio.Packet) bool {
ci := hostinfo.ConnectionState
if ci.eKey == nil {
return false
}
f.recordOutbound(hostinfo)
it.pkt, it.hostinfo, it.ci = pkt, hostinfo, ci
it.remote = hostinfo.GetRemote()
if it.remote.IsValid() {
return true
}
it.relayHI, it.relay = f.findRelay(hostinfo)
if it.relayHI == nil || it.relay == nil {
return false
}
it.remote = it.relayHI.GetRemote()
return true
}
// encryptTxItem segments and encrypts it into sb, for inline and staged sends alike.
func (f *Interface) encryptTxItem(it *txItem, nb []byte, sb *batch.SendBatch) {
reserved := it.counter != 0
c, rc, end := it.counter, it.relayCounter, it.counter+it.segs
err := tio.SegmentSuperpacket(it.pkt, func(seg []byte) error {
if reserved && c >= end {
// More segments than counters reserved for them, never encrypt past the reservation
return errFlowSegmentOverrun
}
if it.relayHI == nil {
// header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305)
if out := f.encryptTxSegment(it, c, seg, nb, sb.Reserve(header.Len+len(seg)+16)); out != nil {
sb.Commit(out, it.remote)
}
} else {
f.encryptTxSegmentVia(it, c, rc, seg, nb, sb)
}
if reserved {
c++
rc++
}
return nil
})
if err != nil {
it.hostinfo.logger(f.l).Error("Failed to segment superpacket for send", "error", err)
}
}
// encryptTxSegment encrypts one segment into out with counter c, or with the next counter of the tunnel when c is
// zero. It returns nil when encryption failed, which is already logged.
func (f *Interface) encryptTxSegment(it *txItem, c uint64, seg, nb, out []byte) []byte {
if c != 0 {
// Reserved counters are unique, so the encryption needs no lock
return f.encryptMessage(header.Message, 0, it.hostinfo, it.ci, c, seg, nb, out, it.remote)
}
if noiseutil.EncryptLockNeeded {
it.ci.writeLock.Lock()
}
c = it.ci.messageCounter.Add(1)
out = f.encryptMessage(header.Message, 0, it.hostinfo, it.ci, c, seg, nb, out, it.remote)
if noiseutil.EncryptLockNeeded {
it.ci.writeLock.Unlock()
}
return out
}
// encryptTxSegmentVia encrypts one segment for the relayed tunnel, then wraps it for the relay with counter rc, both
// allocated on demand when zero like encryptTxSegment does.
func (f *Interface) encryptTxSegmentVia(it *txItem, c, rc uint64, seg, nb []byte, sb *batch.SendBatch) {
//relay header + header + plaintext + AEAD tag + relay tag
scratch := sb.Reserve(len(seg) + header.MaxOverhead)
out := f.encryptTxSegment(it, c, seg, nb, scratch[header.Len:])
if out == nil {
return
}
toSend, err := f.prepareSendVia(it.relayHI, it.relay, out, nb, scratch, true, rc)
if err != nil {
return
}
remote := it.remote
if rc == 0 {
// Unreserved sends follow the relay's current remote, like sendNoMetrics
remote = it.relayHI.GetRemote()
}
sb.Commit(toSend, remote)
}
// txTunnel accumulates one tunnel's counter needs across a batch so the batch reserves them with one atomic add.
type txTunnel struct {
ci *ConnectionState
hostinfo *HostInfo
n uint64
base uint64
exhausted bool
}
// txReserver turns one tun read into txItems with message counters reserved for them. Only the reader goroutine
// collects and reserves counters.
type txReserver struct {
f *Interface
tunnels []txTunnel
owner uint64
// The staged reader separates relay data before reserving direct work. A nil pointer collects both paths.
inlineRelays *[]txItem
}
// collect runs the filter over pkts and appends the sendable ones to items, tallying each tunnel's counter needs.
func (r *txReserver) collect(items []txItem, pkts []tio.Packet, fwPacket *firewall.ParsedPacket, rejectBuf []byte, q int, localCache firewall.ConntrackCache) []txItem {
f := r.f
r.tunnels = r.tunnels[:0]
for _, pkt := range pkts {
hostinfo := f.filterInsidePacket(pkt, fwPacket, rejectBuf, q, localCache)
if hostinfo == nil {
continue
}
var it txItem
if !f.prepareTxItem(&it, hostinfo, pkt) {
continue
}
it.segs = uint64(pkt.SegmentCount())
if it.relayHI != nil {
// The relay bookkeeping inline sends do per segment in prepareSendVia, once per item
f.connectionManager.OutNoRebind(it.relayHI)
f.connectionManager.RelayUsed(it.relay.LocalIndex)
if r.inlineRelays != nil {
*r.inlineRelays = append(*r.inlineRelays, it)
continue
}
it.relayCounter = r.need(it.relayHI, it.relayHI.ConnectionState, it.segs)
}
it.counter = r.need(it.hostinfo, it.ci, it.segs)
items = append(items, it)
}
return items
}
// need adds n counters to ci's tally for this batch and returns the offset of the first one within it.
func (r *txReserver) need(hostinfo *HostInfo, ci *ConnectionState, n uint64) uint64 {
t := r.tunnel(ci)
if t == nil {
r.tunnels = append(r.tunnels, txTunnel{ci: ci, hostinfo: hostinfo})
t = &r.tunnels[len(r.tunnels)-1]
}
off := t.n
t.n += n
return off
}
// tunnel finds ci's tally, a batch rarely touches more than a couple of tunnels so a scan beats a map.
func (r *txReserver) tunnel(ci *ConnectionState) *txTunnel {
for i := range r.tunnels {
if r.tunnels[i].ci == ci {
return &r.tunnels[i]
}
}
return nil
}
// reserve takes the counters collect tallied and stamps them onto items, dropping the items of any tunnel whose
// counter is exhausted. It returns the items that are left.
func (r *txReserver) reserve(items []txItem) []txItem {
f := r.f
// One atomic add per tunnel reserves the whole batch's counters before any worker sees it, so counter order on a
// tunnel is dispatch order, which is the order the batches are flushed in. Reservation and the sender's order are
// both taken on the reader goroutine, so there is no window for another batch to slip in between.
for i := range r.tunnels {
t := &r.tunnels[i]
end := t.ci.messageCounter.Add(t.n)
t.base = end - t.n + 1
if end >= RejectAfterMessages {
t.ci.messageCounter.Store(RejectAfterMessages)
t.exhausted = true
f.messageMetrics.TxExhausted(int64(t.n))
if t.base <= RejectAfterMessages {
t.hostinfo.logger(f.l).Error("Dropping outbound packets, tunnel message counter is exhausted")
}
}
}
n := 0
for _, it := range items {
t := r.tunnel(it.ci)
if t.exhausted {
continue
}
it.counter += t.base
if it.relayHI != nil {
rt := r.tunnel(it.relayHI.ConnectionState)
if rt.exhausted {
continue
}
it.relayCounter += rt.base
}
items[n] = it
n++
}
clear(items[n:])
return items[:n]
}
// handoffQueue is a tun queue that can give up the buffer its last Read filled. SwapRxBuf(nil) allocates a
// correctly sized replacement; later swaps recycle the buffers handed back.
type handoffQueue interface {
tio.Queue
SwapRxBuf([]byte) []byte
}
// txSlot is one staged batch. The reader fills items, a worker encrypts them into sb and signals done, and the
// ordered sender flushes sb.
type txSlot struct {
f *Interface
items []txItem
// buf owns the queue receive buffer the items point into, swapped out at dispatch
buf []byte
sb *batch.SendBatch
done chan struct{}
// reservations hold shared tunnel credit until the flush, including for dropped counters
reservations []txTunnel
}
func (s *txSlot) run(nb []byte) {
for i := range s.items {
s.f.encryptTxItem(&s.items[i], nb, s.sb)
}
s.done <- struct{}{}
}
// stagedTx is the staged transmit pipeline of one tun reader. Everything but the workers and the sender runs on the
// reader goroutine.
type stagedTx struct {
txReserver
// Relay data always leaves on the reader, including small and oversized reads. Its independent reservation
// scratch keeps mixed reads from changing the direct pipeline's counters, gate or sender placement.
relayReserver txReserver
relaySlot txSlot
q int
cpu int // reader CPU, inline sends stay here
senderCPU int // -1 leaves the sender within its inherited affinity
queue handoffQueue
pool *flowPool
// sb and nb are the reader's own, for the sends it does itself
sb *batch.SendBatch
nb []byte
// free holds the reader's idle slots. A dispatched slot goes to the workers and, in the same order, to the
// sender on ordered; the sender gives it back on returned once it is on the wire.
free []*txSlot
ordered chan *txSlot
returned chan *txSlot
inFlight int
senderDone chan struct{}
// gateInline is the routine's transmit mode, gateStreak how many reads in a row wanted the other one
gateInline bool
gateStreak int
metricGateSwitches metrics.Counter
metricBatches metrics.Counter
metricInline metrics.Counter
}
// newStagedTx needs a queue that hands over whole read batches, the offload (TSO/USO) queue on linux. On any other
// queue a read is one packet, and staging it would cost a handoff per packet.
func newStagedTx(f *Interface, queue handoffQueue, q int) *stagedTx {
p := &stagedTx{
txReserver: txReserver{f: f, tunnels: make([]txTunnel, 0, 4), owner: txBudgetOwner(q)},
q: q,
senderCPU: -1,
queue: queue,
sb: batch.NewSendBatch(f.writers[q], batch.SendBatchCap, batch.SendBatchCap*(udp.MTU+32)),
nb: make([]byte, 12),
ordered: make(chan *txSlot, f.staged.txSlots),
returned: make(chan *txSlot, f.staged.txSlots),
senderDone: make(chan struct{}),
metricBatches: metrics.GetOrRegisterCounter("flow.tx.batches", nil),
metricInline: metrics.GetOrRegisterCounter("flow.tx.inline", nil),
// Starts inline, the pipeline is empty, unless the threshold never allows it
gateInline: f.staged.inlineBelow > 0,
metricGateSwitches: metrics.GetOrRegisterCounter("flow.tx.gate_switches", nil),
}
// Inline relay sends can share a tunnel with older direct work on this routine. They are not the ordered
// sender's sole owner, so they must not recycle credit around a stalled direct reservation.
p.relayReserver = txReserver{f: f, tunnels: make([]txTunnel, 0, 4)}
p.inlineRelays = &p.relaySlot.items
if f.pinThreads {
p.cpu = f.threadCPU(q)
// Only an explicit list pins the sender, and never onto its reader's CPU, where the two would serialise.
// Automatic placement by queue index can put a busy sender on a slow core; unpinned, the scheduler moves it.
if n := len(f.cpuAffinity); n > 0 && f.cpuAffinity[(q+1)%n] != p.cpu {
p.senderCPU = f.cpuAffinity[(q+1)%n]
}
}
p.pool = newFlowPool(f.staged.workers, f.staged.txSlots)
// Ciphertext adds per-segment headers and tags; the arena can grow for small segments.
buf := queue.SwapRxBuf(nil)
bufSize := len(buf)
for i := range f.staged.txSlots {
if i > 0 {
buf = make([]byte, bufSize)
}
p.free = append(p.free, &txSlot{
f: f,
buf: buf,
sb: batch.NewSendBatch(f.writers[q], 4*batch.SendBatchCap, bufSize+bufSize/4),
done: make(chan struct{}, 1),
})
}
go p.sender()
return p
}
// run is listenIn's loop with the staged pipeline: read, then hand each batch on, until the queue fails.
func (p *stagedTx) run(fwPacket *firewall.ParsedPacket, rejectBuf []byte, conntrackCache *firewall.ConntrackCacheTicker) error {
defer p.close()
if p.f.pinThreads {
p.f.pinThreadToCPU(p.q, p.cpu)
}
for {
pkts, err := p.queue.Read()
if err != nil {
return err
}
p.ready()
p.handle(pkts, fwPacket, rejectBuf, conntrackCache.Get())
}
}
// handle takes one tun read through the pipeline. It returns once the batch is queued, or sent when the reader sent
// it itself.
func (p *stagedTx) handle(pkts []tio.Packet, fwPacket *firewall.ParsedPacket, rejectBuf []byte, localCache firewall.ConntrackCache) {
p.ready()
inline := p.gate(p.small(pkts))
if inline && p.f.routines == 1 {
// With only one reader and an empty pipeline, no other staged reservation can be overtaken. Keep this common
// case inexpensive.
p.inlineSoleReader(pkts, fwPacket, rejectBuf, localCache)
return
}
slot := p.free[len(p.free)-1]
p.free = p.free[:len(p.free)-1]
items := p.collect(slot.items[:0], pkts, fwPacket, rejectBuf, p.q, localCache)
if len(p.relaySlot.items) != 0 {
p.sendRelays()
}
if len(items) == 0 {
slot.items = items
p.free = append(p.free, slot)
return
}
if p.oversized() {
p.metricInline.Inc(1)
p.sendLarge(&p.txReserver, items)
slot.items = items[:0]
p.free = append(p.free, slot)
return
}
if !p.waitBudget(&p.txReserver) {
clear(items)
slot.items = items[:0]
p.free = append(p.free, slot)
return
}
slot.reservations = append(slot.reservations[:0], p.tunnels...)
items = p.reserve(items)
slot.items = items
if len(items) == 0 {
slot.releaseBudget()
p.free = append(p.free, slot)
return
}
if inline {
p.inline(slot)
p.free = append(p.free, slot)
return
}
// The packets now live in the slot, and the queue reads into the slot's old buffer
slot.buf = p.queue.SwapRxBuf(slot.buf)
p.inFlight++
p.metricBatches.Inc(1)
// Counters were reserved above on this goroutine and the slot takes its place in the sender's order right here,
// before any other batch can, so the sender puts them on the wire in counter order.
p.ordered <- slot
p.pool.jobs <- slot
}
// sendRelays keeps relay data on one send path for the lifetime of this routine. The gate's software drain only
// waits for socket submission; switching CPUs can still overtake packets on another NIC queue. Direct data retains
// the existing gate and sender placement. Relay credit is still shared with all other users of both tunnels.
func (p *stagedTx) sendRelays() {
r, slot := &p.relayReserver, &p.relaySlot
r.tunnels = r.tunnels[:0]
for i := range slot.items {
it := &slot.items[i]
it.counter = r.need(it.hostinfo, it.ci, it.segs)
it.relayCounter = r.need(it.relayHI, it.relayHI.ConnectionState, it.segs)
}
if r.oversized() {
p.metricInline.Inc(1)
p.sendLarge(r, slot.items)
slot.items = slot.items[:0]
return
}
if !p.waitBudget(r) {
clear(slot.items)
slot.items = slot.items[:0]
return
}
slot.reservations = append(slot.reservations[:0], r.tunnels...)
slot.items = r.reserve(slot.items)
p.inline(slot)
}
// gate reports whether a read goes inline. Each routine holds one
// transmit mode, inline on the reader or fanout to the workers and the sender, until gateHold reads in a row want the
// other one, the last of which goes the new way. Choosing per read let small inline sends leave from the reader's CPU
// between staged ones leaving from the sender's, and the receiver's TCP saw them reordered. Going inline the reader
// first waits until everything in flight has been submitted to the socket and the sender is idle. This does not
// drain independent NIC queues; relay data therefore always uses the reader. Going to fanout nothing is in flight:
// inline sends finish before the reader moves on.
func (p *stagedTx) gate(small bool) bool {
if small == p.gateInline {
p.gateStreak = 0
return small
}
if p.gateStreak++; p.gateStreak < p.f.staged.gateHold {
return p.gateInline
}
p.gateInline, p.gateStreak = small, 0
p.metricGateSwitches.Inc(1)
if !small {
return false
}
for p.flushOldest() {
}
return true
}
// ready takes back every slot the sender has finished with, and waits for one when all of them are in flight.
func (p *stagedTx) ready() {
p.reap()
if len(p.free) == 0 {
p.release(<-p.returned)
}
}
// reap takes back every slot the sender has finished with, without waiting.
func (p *stagedTx) reap() {
for {
select {
case s := <-p.returned:
p.release(s)
default:
return
}
}
}
// flushOldest waits for the oldest batch in flight to reach the wire and takes its slot back. It reports false when
// nothing is in flight.
func (p *stagedTx) flushOldest() bool {
if p.inFlight == 0 {
return false
}
p.release(<-p.returned)
return true
}
func (p *stagedTx) release(s *txSlot) {
p.inFlight--
p.free = append(p.free, s)
}
// sender is the ordered send stage: it puts each batch on the wire once its worker is done, in dispatch order.
func (p *stagedTx) sender() {
defer close(p.senderDone)
if p.senderCPU >= 0 {
p.f.pinThreadToCPU(p.q, p.senderCPU)
}
for s := range p.ordered {
<-s.done
if p.f.closed.Load() {
// The sockets are gone, a flush would only log errors
s.sb.Reset()
} else {
p.f.flushSendBatch(s.sb, p.q)
}
s.releaseBudget()
clear(s.items)
s.items = s.items[:0]
p.returned <- s
}
}
// close sends everything still in flight, in order, and stops the sender and the workers.
func (p *stagedTx) close() {
close(p.ordered)
<-p.senderDone
p.pool.close()
}
// small counts segments against the inline threshold.
func (p *stagedTx) small(pkts []tio.Packet) bool {
below := uint64(p.f.staged.inlineBelow)
if below == 0 {
return false
}
var n uint64
for _, pkt := range pkts {
if n += uint64(pkt.SegmentCount()); n >= below {
return false
}
}
return true
}
// inline encrypts a budgeted batch on the reader, with no worker handoff or buffer swap. The local sender is idle,
// but other readers may hold reservations on these tunnels, so keep our credit until the flush completes.
func (p *stagedTx) inline(slot *txSlot) {
p.metricInline.Inc(1)
for i := range slot.items {
p.f.encryptTxItem(&slot.items[i], p.nb, p.sb)
if p.sb.Len() >= batch.SendBatchCap {
p.f.flushSendBatch(p.sb, p.q)
}
}
p.f.flushSendBatch(p.sb, p.q)
slot.releaseBudget()
clear(slot.items)
slot.items = slot.items[:0]
}
// inlineSoleReader needs no credit: the interface has no other tun reader and this reader has drained every
// reservation before taking the inline path.
func (p *stagedTx) inlineSoleReader(pkts []tio.Packet, fwPacket *firewall.ParsedPacket, rejectBuf []byte, localCache firewall.ConntrackCache) {
p.metricInline.Inc(1)
for _, pkt := range pkts {
p.f.consumeInsidePacket(pkt, fwPacket, p.nb, p.sb, rejectBuf, p.q, localCache)
if p.sb.Len() >= batch.SendBatchCap {
p.f.flushSendBatch(p.sb, p.q)
}
}
p.f.flushSendBatch(p.sb, p.q)
}
-231
View File
@@ -1,231 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
import (
"fmt"
"net/netip"
"sync"
"time"
"os"
"os/exec"
"testing"
"github.com/slackhq/nebula/overlay/tio"
"github.com/flynn/noise"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util"
"github.com/stretchr/testify/require"
)
// queuedWire models two CPU-selected NIC queues with independent completion: the reader's
// queue is congested, while another CPU's queue drains immediately. WriteBatch returning only
// means the kernel accepted the packets; draining staged slots does not drain a NIC queue.
// The sampled affinity makes the schedule deterministic without a real socket or NIC. An
// unpinned sender alternates between allowed CPUs, even if it happened not to in this run.
func (c *affinityConn) queuedWire(readerCPU int) [][]byte {
var slow, fast [][]byte
wire := c.captured()
off := 0
for i, n := range c.sizes {
mask := c.masks[i]
if mask.IsSet(readerCPU) && (mask.Count() == 1 || i%2 == 0) {
slow = append(slow, wire[off:off+n]...)
} else {
fast = append(fast, wire[off:off+n]...)
}
off += n
}
return append(fast, slow...)
}
// TestFlowStagedTXRelayQueuedWire reproduces reorder after successful, ordered socket
// submissions. Mixed reads switch the sticky gate with an empty software pipeline; relayed
// data must stay on the reader throughout, without moving the direct sender. Each routine owns a flow.
func TestFlowStagedTXRelayQueuedWire(t *testing.T) {
if os.Getenv("NEBULA_FLOW_RING_TEST") != "1" {
cmd := exec.Command(os.Args[0], "-test.run=^TestFlowStagedTXRelayQueuedWire$")
cmd.Env = append(os.Environ(), "NEBULA_FLOW_RING_TEST=1")
out, err := cmd.CombinedOutput()
require.NoError(t, err, "%s", out)
return
}
allowed, err := util.AllowedCPUs()
require.NoError(t, err)
if len(allowed) < 2 {
t.Skip("requires two allowed CPUs")
}
for _, mode := range []string{"gate", "always-staged", "oversized"} {
for _, explicit := range []bool{false, true} {
for _, routines := range []int{1, 2} {
for q := range routines {
t.Run(fmt.Sprintf("%s/explicit=%t/routines=%d/q=%d", mode, explicit, routines, q), func(t *testing.T) {
sc := gated(stagedGateHold)
if mode != "gate" {
sc.inlineBelow = 0
}
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
f := sender.f
f.pinThreads, f.routines = true, routines
if explicit {
f.cpuAffinity = allowed[:2]
}
_, relayHI, relayPeer := addTestRelay(t, sender, ReplayWindow)
conn := &affinityConn{}
f.writers = make([]udp.Conn, routines)
for i := range f.writers {
f.writers[i] = conn
}
shape := append(repeatBool(false, 2), repeatBool(true, stagedGateHold+2)...)
shape = append(shape, repeatBool(false, stagedGateHold+2)...)
if mode != "gate" {
shape = []bool{false, true, false, true}
}
reads, want := gateReads(shape...)
if mode == "oversized" {
// A deliberately tiny outer budget alone forces the oversized fallback.
relayHI.ConnectionState.window = NewBits(16)
}
queue := &affinityQueue{flowBatchQueue: newFlowBatchQueue(reads), conn: conn}
// Each Read waits for all previous socket submissions, making software drain
// irrelevant to the independently queued packets in queuedWire.
f.listenIn(queue, q)
unwrap := func(wire [][]byte) [][]byte {
inner := make([][]byte, 0, len(wire))
for _, pkt := range wire {
inner = append(inner, pkt[header.Len:len(pkt)-relayPeer.dKey.Overhead()])
}
return inner
}
require.Equal(t, want, decryptWire(t, receiver, unwrap(conn.captured())), "socket submission order")
require.Equal(t, want, decryptWire(t, receiver, unwrap(conn.queuedWire(f.threadCPU(q)))), "NIC queue completion order")
})
}
}
}
}
}
// Mixed reads must keep direct sends on the original staged sender, including when relay data is present.
func TestFlowStagedTXMixedPlacement(t *testing.T) {
if os.Getenv("NEBULA_FLOW_MIXED_TEST") != "1" {
cmd := exec.Command(os.Args[0], "-test.run=^TestFlowStagedTXMixedPlacement$")
cmd.Env = append(os.Environ(), "NEBULA_FLOW_MIXED_TEST=1")
out, err := cmd.CombinedOutput()
require.NoError(t, err, "%s", out)
return
}
allowed, err := util.AllowedCPUs()
require.NoError(t, err)
if len(allowed) < 2 {
t.Skip("requires two allowed CPUs")
}
for _, explicit := range []bool{false, true} {
t.Run(fmt.Sprintf("explicit=%t", explicit), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
_, _, relayPeer := addTestRelay(t, sender, ReplayWindow)
direct, directReceiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
directAddr := netip.MustParseAddr("10.0.0.4")
direct.hostinfo.vpnAddrs = []netip.Addr{directAddr}
f := sender.f
f.hostMap.Hosts[directAddr] = direct.hostinfo
f.pinThreads, f.routines = true, 2
sendCPU := -1
if explicit {
f.cpuAffinity = allowed[:2]
sendCPU = allowed[1]
}
conn := &affinityConn{}
f.writers = []udp.Conn{conn, conn}
var reads [][]tio.Packet
var wantRelay, wantDirect []decodedSeg
for seq := range uint32(6) {
r := flowTCPSuperpacket(0, seq, 3)
d := flowTCPSuperpacket(1, seq, 9)
copy(d.Bytes[16:20], directAddr.AsSlice())
reads = append(reads, []tio.Packet{r, d})
for i := range 3 {
wantRelay = append(wantRelay, decodedSeg{seq: seq, idx: uint16(i)})
}
for i := range 9 {
wantDirect = append(wantDirect, decodedSeg{flow: 1, seq: seq, idx: uint16(i)})
}
}
queue := &affinityQueue{flowBatchQueue: newFlowBatchQueue(reads), conn: conn}
f.listenIn(queue, 0)
wire := conn.captured()
var directWire, relayWire [][]byte
off := 0
for i, n := range conn.sizes {
for _, pkt := range wire[off : off+n] {
var h header.H
require.NoError(t, h.Parse(pkt))
cpu := sendCPU
if h.Subtype == header.MessageRelay {
cpu = f.threadCPU(0)
require.NoError(t, relayPeer.VerifyRelay(nil, h.MessageCounter, pkt, make([]byte, 12)))
relayWire = append(relayWire, pkt[header.Len:len(pkt)-relayPeer.dKey.Overhead()])
} else {
directWire = append(directWire, pkt)
}
requireFlowCPU(t, conn.masks[i:i+1], cpu, allowed)
}
off += n
}
require.Equal(t, wantDirect, decryptWire(t, directReceiver, directWire))
require.Equal(t, wantRelay, decryptWire(t, receiver, relayWire))
})
}
}
// A relayed batch shares its outer tunnel with older staged direct data on the SAME routine. Inline completion
// must not refund that direct sender's credit while its oldest reservation is stalled.
func TestFlowStagedTXRelaySharedBudget(t *testing.T) {
sender, _ := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
_, via, _ := addTestRelay(t, sender, ReplayWindow)
// Give the relay host the same ordinary peer metadata as a direct tunnel.
direct := newFlowTestPeer(t, flowTestLocal, via.vpnAddrs[0], via.ConnectionState, 300, 400)
via.buildNetworks(sender.f.myVpnNetworksTable, direct.hostinfo.ConnectionState.peerCert.Certificate)
via.remoteIndexId = direct.hostinfo.remoteIndexId
ci := via.ConnectionState
ci.window = NewBits(32)
start := ci.messageCounter.Load()
entered, unblock := make(chan struct{}), make(chan struct{})
var release sync.Once
delayCrypto(direct, func(n uint64) {
if n == start+1 {
close(entered)
<-unblock
}
})
p := newStagedTx(sender.f, newFlowBatchQueue(nil), 0)
defer p.close()
defer release.Do(func() { close(unblock) })
d := flowTCPSuperpacket(1, 0, 9)
copy(d.Bytes[16:20], via.vpnAddrs[0].AsSlice())
p.send([]tio.Packet{d})
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("direct worker did not start")
}
for seq := range uint32(7) {
p.send([]tio.Packet{flowTCPSuperpacket(0, seq, 1)})
}
require.Equal(t, uint64(16), (ci.txBudget.state.Load()>>24)&txBudgetMask)
done := make(chan struct{})
go func() { defer close(done); p.send([]tio.Packet{flowTCPSuperpacket(0, 7, 1)}) }()
select {
case <-done:
t.Error("relay recycled credit around a stalled direct reservation")
case <-time.After(50 * time.Millisecond):
}
require.Equal(t, start+16, ci.messageCounter.Load())
release.Do(func() { close(unblock) })
<-done
p.drain()
require.Zero(t, ci.txBudget.state.Load())
}
-604
View File
@@ -1,604 +0,0 @@
//go:build linux && !android && !e2e_testing
package nebula
import (
"fmt"
"net/netip"
"os"
"os/exec"
"slices"
"sync"
"testing"
"time"
"github.com/flynn/noise"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/cpupick"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
"golang.org/x/sys/unix"
)
// TestReservedEncryptionMatchesInline encrypts a batch with counters reserved up front: the receiver accepts every
// segment, in order.
func TestReservedEncryptionMatchesInline(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedConfig{})
packets, want := flowTestBatches(1, 3, 3, 9)
r := txReserver{f: sender.f}
items := r.collect(nil, packets[0], &firewall.ParsedPacket{}, make([]byte, mtu), 0, nil)
require.Equal(t, uint64(len(want)), r.tunnels[0].n)
sb := batch.NewSendBatch(sender.conn, 128, 65536)
for _, it := range r.reserve(items) {
sender.f.encryptTxItem(&it, make([]byte, 12), sb)
}
sender.f.flushSendBatch(sb, 0)
require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured()))
}
// TestSegmentCount checks tio.Packet.SegmentCount predicts exactly the segments tio.SegmentSuperpacket cuts, odd tails included.
func TestSegmentCount(t *testing.T) {
require.Equal(t, 1, tio.Packet{Bytes: make([]byte, 60)}.SegmentCount())
for _, segs := range []int{1, 2, 3, 45} {
for _, tail := range []int{0, 1, flowTestMSS - 1} {
pkt := flowTCPSuperpacket(0, 0, segs)
pkt.Bytes = pkt.Bytes[:len(pkt.Bytes)-tail]
pkt.GSO = tio.GSOInfo{Size: flowTestMSS, HdrLen: 40, CsumStart: 20, Proto: tio.GSOProtoTCP}
want, got := pkt.SegmentCount(), 0
require.NoError(t, tio.SegmentSuperpacket(pkt, func([]byte) error { got++; return nil }))
require.Equal(t, want, got, "segs %d tail %d", segs, tail)
}
}
}
// TestFlowStagedTXOrderAndCounters runs the staged transmit pipeline with shuffled workers and a slow socket, so the
// sender lags and the reader runs into backpressure, then fails the tun read with batches in flight: the wire must
// carry every segment once, in tun read order, under strictly increasing counters that were each reserved exactly
// once, and the read error is still reported.
func TestFlowStagedTXOrderAndCounters(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
t.Run(cipher.CipherName(), func(t *testing.T) {
if cipher.CipherName() == noiseutil.CipherAESGCM.CipherName() && noiseutil.EncryptLockNeeded {
t.Skip("AES-GCM requires encryption in nonce order")
}
sender, receiver := newFlowTestPair(t, cipher, stagedOn(4))
delayCrypto(sender, func(n uint64) {
if n%54 == 0 {
time.Sleep(time.Millisecond)
}
})
// Far slower than the reader, even under the race detector
sender.conn.delay = 8 * time.Millisecond
batches, want := flowTestBatches(60, 6, 3, 9)
start := sender.hostinfo.ConnectionState.messageCounter.Load()
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
tx0 := tx.Count()
sender.f.listenIn(newFlowBatchQueue(batches), 0)
require.Equal(t, int64(len(batches)), tx.Count()-tx0, "every batch uses the ordered sender")
require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured()), "wire order must match tun read order")
require.Equal(t, start+uint64(len(want)), sender.hostinfo.ConnectionState.messageCounter.Load())
require.NotNil(t, sender.f.fatalErr.Load())
})
}
}
// TestFlowStagedTXCloseInFlight closes the interface and tun while batches are in flight, 20 times over.
func TestFlowStagedTXCloseInFlight(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
for range 20 {
sender, _ := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(3))
sender.conn.delay = time.Millisecond
batches, _ := flowTestBatches(400, 4, 2, 8)
q := newFlowBatchQueue(batches)
done := make(chan struct{})
go func() { sender.f.listenIn(q, 0); close(done) }()
require.Eventually(t, func() bool { return q.reads.Load() > 5 }, 5*time.Second, 100*time.Microsecond)
sender.f.closed.Store(true)
require.NoError(t, q.Close())
<-done
}
}
// TestFlowStagedTXExhausted drops every batch of a tunnel whose counter runs out, like inline sends do.
func TestFlowStagedTXExhausted(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sender, _ := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
ci := sender.hostinfo.ConnectionState
ci.messageCounter.Store(RejectAfterMessages - 3)
sender.f.listenIn(newFlowBatchQueue([][]tio.Packet{{flowTCPSuperpacket(0, 0, 10)}, {flowTCPSuperpacket(0, 1, 2)}}), 0)
require.Empty(t, sender.conn.captured())
require.Equal(t, RejectAfterMessages, ci.messageCounter.Load())
require.Equal(t, int64(12), sender.f.messageMetrics.txExhausted.Count())
require.Zero(t, ci.txBudget.state.Load())
}
// TestFlowStagedTXRelay sends staged batches, and an empty read, through a relay: outer and inner layers both verify,
// in order, and go to the relay.
// With a tiny MSS, the middle superpacket alone exceeds the tunnels' whole credit: it must be split, not admitted as an
// unbounded lone batch, with either replay window.
func TestFlowStagedTXRelay(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
for _, window := range budgetWindows {
t.Run(fmt.Sprintf("window=%d", window), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(2))
sender.hostinfo.ConnectionState.window = NewBits(window)
receiver.hostinfo.ConnectionState.window = NewBits(window)
relayRemote, relayHI, relayPeer := addTestRelay(t, sender, window)
batches, want := flowTestBatches(2, 3, 1, 6)
var payload []byte
large := flowTCPSuperpacket(0, 9, 45)
large.GSO.Size = 1
payload = append([]byte(nil), large.Bytes[40:]...)
batches = [][]tio.Packet{batches[0], {large}, batches[1]}
sender.f.listenIn(newFlowBatchQueue(append(batches, nil)), 0)
wire := sender.conn.captured()
require.Len(t, wire, len(want)+len(payload))
var got []decodedSeg
var tiny []byte
var lastOuter, lastInner uint64
for i, pkt := range wire {
require.Equal(t, relayRemote, sender.conn.addrs[i])
h := &header.H{}
require.NoError(t, h.Parse(pkt))
require.Greater(t, h.MessageCounter, lastOuter)
lastOuter = h.MessageCounter
require.NoError(t, relayPeer.VerifyRelay(nil, h.MessageCounter, pkt, make([]byte, 12)))
pkt = pkt[header.Len : len(pkt)-16]
require.NoError(t, h.Parse(pkt))
require.Greater(t, h.MessageCounter, lastInner)
lastInner = h.MessageCounter
plain, err := receiver.hostinfo.ConnectionState.Decrypt(receiver.f.l, h.MessageCounter, pkt, make([]byte, 12))
require.NoError(t, err)
if len(plain) == 41 {
tiny = append(tiny, plain[40])
} else {
got = append(got, decodeSeg(plain[40:]))
}
}
require.Equal(t, want, got)
require.Equal(t, payload, tiny)
require.Zero(t, sender.hostinfo.ConnectionState.txBudget.state.Load())
require.Zero(t, relayHI.ConnectionState.txBudget.state.Load())
})
}
}
// addTestRelay makes sender reach its peer only through a relay tunnel with replay windows of window. It returns the
// relay's underlay address, its hostinfo and the relay's side of the tunnel.
func addTestRelay(t testing.TB, sender *flowTestPeer, window uint64) (netip.AddrPort, *HostInfo, *ConnectionState) {
relayInit, relayResp := runTestHandshake(t)
relayCI, err := newConnectionStateFromResult(relayInit)
require.NoError(t, err)
relayPeer, err := newConnectionStateFromResult(relayResp)
require.NoError(t, err)
relayCI.window, relayPeer.window = NewBits(window), NewBits(window)
relayAddr := netip.MustParseAddr("10.0.0.3")
relayRemote := netip.MustParseAddrPort("192.0.2.3:4242")
relay := &Relay{Type: TerminalType, State: Established, LocalIndex: 5, RemoteIndex: 6, PeerAddr: flowTestRemote}
relayHI := &HostInfo{
vpnAddrs: []netip.Addr{relayAddr},
ConnectionState: relayCI,
relayState: RelayState{
relayForByAddr: map[netip.Addr]*Relay{flowTestRemote: relay},
relayForByIdx: map[uint32]*Relay{relay.LocalIndex: relay},
},
}
relayHI.remote.Store(&relayRemote)
sender.f.hostMap.Hosts[relayAddr] = relayHI
sender.hostinfo.remote.Store(&netip.AddrPort{})
sender.hostinfo.relayState.relays = []netip.Addr{relayAddr}
return relayRemote, relayHI, relayPeer
}
// affinityQueue makes every read start with an idle pipeline: before each read it waits until conn has sent what was
// read before, recording how many sends that took, then samples the reader's actual thread affinity.
type affinityQueue struct {
*flowBatchQueue
conn *affinityConn
sent int
boundaries []int
masks []unix.CPUSet
}
func (q *affinityQueue) Read() ([]tio.Packet, error) {
if n := q.reads.Load(); n > 0 {
for _, p := range q.batches[n-1] {
q.sent += p.SegmentCount()
}
}
q.boundaries = append(q.boundaries, q.conn.waitFor(q.sent))
q.masks = append(q.masks, threadMask())
return q.flowBatchQueue.Read()
}
// affinityConn captures every send, sampling the sending thread's actual affinity in each WriteBatch.
type affinityConn struct {
flowCaptureConn
masks []unix.CPUSet
sizes []int
}
func (c *affinityConn) WriteBatch(p [][]byte, addrs []netip.AddrPort) (int, error) {
mask := threadMask()
c.mu.Lock()
c.masks = append(c.masks, mask)
c.sizes = append(c.sizes, len(p))
c.mu.Unlock()
return c.flowCaptureConn.WriteBatch(p, addrs)
}
// waitFor waits until n packets were sent and returns how many WriteBatch calls that took.
func (c *affinityConn) waitFor(n int) int {
for len(c.captured()) < n {
time.Sleep(50 * time.Microsecond)
}
c.mu.Lock()
defer c.mu.Unlock()
return len(c.masks)
}
func threadMask() unix.CPUSet {
var mask unix.CPUSet
if err := unix.SchedGetaffinity(0, &mask); err != nil {
panic(err)
}
return mask
}
// requireFlowCPU checks every mask is just cpu, or with cpu -1, the whole allowed set.
func requireFlowCPU(t *testing.T, masks []unix.CPUSet, cpu int, allowed []int) {
t.Helper()
require.NotEmpty(t, masks)
for _, mask := range masks {
if cpu >= 0 {
require.Equal(t, 1, mask.Count())
require.True(t, mask.IsSet(cpu), "expected CPU %d", cpu)
continue
}
require.Equal(t, len(allowed), mask.Count())
for _, c := range allowed {
require.True(t, mask.IsSet(c))
}
}
}
// TestFlowStagedTXAffinity sends large, small and large reads, each from an idle pipeline, and checks the order on the
// wire and where the reader and each send actually ran, for every CPU placement, with one routine and with several. A
// staged sender is only pinned to an explicit entry other than its reader's CPU, and inline sends stay on the reader.
// With the threshold, only the small read goes inline.
// The reader only reads again once its earlier reads are on the wire, so this also checks the sender never waits for
// the reader. Pinning threads changes the process, so the test runs in a child.
func TestFlowStagedTXAffinity(t *testing.T) {
if os.Getenv("NEBULA_FLOW_AFFINITY_TEST") != "1" {
cmd := exec.Command(os.Args[0], "-test.run=^TestFlowStagedTXAffinity$")
cmd.Env = append(os.Environ(), "NEBULA_FLOW_AFFINITY_TEST=1")
out, err := cmd.CombinedOutput()
require.NoError(t, err, "%s", out)
return
}
allowed, err := util.AllowedCPUs()
require.NoError(t, err)
for _, placement := range []string{"fallback", "automatic", "explicit", "explicit-single"} {
for _, pin := range []bool{false, true} {
for _, sc := range sendPaths() {
for _, routines := range []int{1, 2} {
t.Run(fmt.Sprintf("%s/pin=%t/%s/routines=%d", placement, pin, stagedName(sc), routines), func(t *testing.T) {
if pin && len(allowed) < 2 {
t.Skip("requires at least two allowed CPUs")
}
if sc.inlineBelow > 0 {
// Switch on every read that wants the other mode, so the small read goes inline between
// staged ones
sc.gateHold = 1
}
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
f := sender.f
f.pinThreads, f.routines = pin, routines
switch placement {
case "automatic":
f.defaultCpuAffinity = cpupick.Default(routines, 4242, nil)
case "explicit":
f.cpuAffinity = []int{allowed[len(allowed)-1], allowed[0]}
case "explicit-single":
f.cpuAffinity = allowed[len(allowed)-1:]
}
readerCPU, sendCPU := -1, -1
if pin {
readerCPU = f.threadCPU(0)
sendCPU = readerCPU
if sc.tx {
// Never the reader's CPU, and no automatic choice: unpinned unless an explicit entry fits
sendCPU = -1
if placement == "explicit" {
sendCPU = f.cpuAffinity[1]
}
}
}
smallCPU := sendCPU
if sc.inlineBelow > 0 {
smallCPU = readerCPU
}
conn := &affinityConn{}
f.writers = []udp.Conn{conn}
batches, want := flowTestBatches(3, 8, 1, 45)
batches[1] = batches[1][:1]
want = append(want[:405], want[720:]...)
q := &affinityQueue{flowBatchQueue: newFlowBatchQueue(batches), conn: conn}
inline := metrics.GetOrRegisterCounter("flow.tx.inline", nil)
inline0 := inline.Count()
f.listenIn(q, 0)
require.Equal(t, want, decryptWire(t, receiver, conn.captured()))
if sc.inlineBelow > 0 {
require.Equal(t, int64(1), inline.Count()-inline0)
}
b := q.boundaries
requireFlowCPU(t, q.masks, readerCPU, allowed)
requireFlowCPU(t, conn.masks[:b[1]], sendCPU, allowed)
requireFlowCPU(t, conn.masks[b[1]:b[2]], smallCPU, allowed)
requireFlowCPU(t, conn.masks[b[2]:], sendCPU, allowed)
})
}
}
}
}
}
// sendPaths is inline, staged and, with the threshold, adaptive transmit.
func sendPaths() []stagedConfig {
return []stagedConfig{{}, stagedOn(2), adaptive(2)}
}
// requireAccepted checks the receiver accepts every packet of wire in the order given. With several readers the wire
// order differs from counter order, but every packet must still authenticate and fit within the replay window.
func requireAccepted(t *testing.T, receiver *flowTestPeer, wire [][]byte) {
t.Helper()
for _, pkt := range wire {
h := &header.H{}
require.NoError(t, h.Parse(pkt))
_, err := receiver.hostinfo.ConnectionState.Decrypt(receiver.f.l, h.MessageCounter, pkt, make([]byte, 12))
require.NoError(t, err, "counter %d", h.MessageCounter)
}
}
// slowConn delays every WriteBatch, then writes to the shared capture.
type slowConn struct {
*flowCaptureConn
delay time.Duration
}
func (c slowConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
time.Sleep(c.delay)
return c.flowCaptureConn.WriteBatch(bufs, addrs)
}
// TestFlowTXBudgetManyReaders saturates one tunnel from eight readers, one of them on a slow socket that lets the
// others overtake it, alongside a burst of unreserved control messages: in the order they reach the wire, every packet
// is accepted by the receiver's replay window, so none was sent twice or displaced past the window.
func TestFlowTXBudgetManyReaders(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
for _, window := range budgetWindows {
t.Run(fmt.Sprintf("window=%d", window), func(t *testing.T) {
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, stagedOn(3))
ci := sender.hostinfo.ConnectionState
ci.window = NewBits(window)
receiver.hostinfo.ConnectionState.window = NewBits(window)
sender.f.routines = 8
sender.f.writers = append([]udp.Conn{slowConn{sender.conn, 4 * time.Millisecond}}, slices.Repeat(sender.f.writers, 8)...)
start := ci.messageCounter.Load()
var wg sync.WaitGroup
for q := range 8 {
wg.Go(func() {
batches, _ := flowTestBatches(28, 8, 2, 45)
p := newStagedTx(sender.f, newFlowBatchQueue(nil), q)
defer p.close()
for _, packets := range batches {
p.send(packets)
}
})
}
// A bounded burst of unreserved control messages uses the documented margin, on writer 8
wg.Go(func() {
for range 128 {
sender.f.sendNoMetrics(header.Test, header.TestRequest, ci, sender.hostinfo, flowTestUnder, []byte("x"), make([]byte, 12), make([]byte, mtu), 8)
time.Sleep(100 * time.Microsecond)
}
})
wg.Wait()
require.Zero(t, ci.txBudget.state.Load())
wire := sender.conn.captured()
require.Len(t, wire, 8*28*360+128)
require.Equal(t, start+uint64(len(wire)), ci.messageCounter.Load())
requireAccepted(t, receiver, wire)
})
}
}
// TestFlowStagedTXInlineOnlyWhenEmpty: a small read behind a staged batch still in flight must not go inline before
// that batch is on the wire, it would overtake it.
func TestFlowStagedTXInlineOnlyWhenEmpty(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sc := adaptive(2)
sc.inlineBelow, sc.gateHold = 4, 1
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
p := newStagedTx(sender.f, newFlowBatchQueue(nil), 0)
block := make(chan struct{})
delayCrypto(sender, func(uint64) { <-block })
batches, want := flowTestBatches(1, 4, 1, 3)
p.send(batches[0][:3]) // switches to fanout, its worker is blocked
require.Equal(t, 1, p.inFlight)
unblock := time.AfterFunc(20*time.Millisecond, func() { close(block) })
defer unblock.Stop()
p.send(batches[0][3:]) // switches to inline, which waits for the staged batch first
require.Zero(t, p.inFlight)
p.close()
require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured()))
}
// TestFlowStagedTXInlineBelowZero: with inline_below 0 an idle reader never sends a batch itself, in either
// direction: every read goes through the workers and the ordered sender or writer.
func TestFlowStagedTXInlineBelowZero(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
l := test.NewLogger()
c := config.NewC(l)
c.Settings = map[string]any{"tun": map[string]any{"staged": map[string]any{"workers": 2, "inline_below": 0}}}
sc := parseStagedConfig(c, l)
if !sc.tx {
t.Skip("transmit stays inline in this build")
}
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
batches, want := flowTestBatches(20, 2, 1, 4)
txInline := metrics.GetOrRegisterCounter("flow.tx.inline", nil)
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
rxInline := metrics.GetOrRegisterCounter("flow.rx.inline", nil)
txInline0, tx0, rxInline0, rx0 := txInline.Count(), tx.Count(), rxInline.Count(), rxBatches.Count()
sender.f.listenIn(newFlowBatchQueue(batches), 0)
wire := sender.conn.captured()
require.Equal(t, want, decryptWire(t, receiver, wire))
require.Zero(t, txInline.Count()-txInline0)
require.Equal(t, int64(len(batches)), tx.Count()-tx0)
receiver.receive(&rxConn{wire: wire, sizes: []int{1, 2, 4}})
require.Equal(t, want, receiver.tun.segments(t))
require.Zero(t, rxInline.Count()-rxInline0)
require.Positive(t, rxBatches.Count()-rx0)
}
// TestFlowStagedTXDisabled: with tx false the reader never builds a staged pipeline, even on a queue that could hand
// off its batches. Receive stays staged.
func TestFlowStagedTXDisabled(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
l := test.NewLogger()
c := config.NewC(l)
c.Settings = map[string]any{"tun": map[string]any{"staged": map[string]any{"workers": 2, "tx": false}}}
sc := parseStagedConfig(c, l)
require.False(t, sc.tx)
require.True(t, sc.rx)
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
batches, want := flowTestBatches(4, 8, 1, 45)
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
txInline := metrics.GetOrRegisterCounter("flow.tx.inline", nil)
tx0, txInline0, rx0 := tx.Count(), txInline.Count(), rxBatches.Count()
q := &swapCountQueue{flowBatchQueue: newFlowBatchQueue(batches)}
sender.f.listenIn(q, 0)
wire := sender.conn.captured()
require.Equal(t, want, decryptWire(t, receiver, wire))
require.Zero(t, q.swaps, "newStagedTx always takes the queue's buffer, so it never ran")
require.Equal(t, tx0, tx.Count())
require.Equal(t, txInline0, txInline.Count())
receiver.receive(&rxConn{wire: wire, batchLen: 512})
require.Equal(t, want, receiver.tun.segments(t))
require.Greater(t, rxBatches.Count(), rx0)
}
// swapCountQueue counts SwapRxBuf calls.
type swapCountQueue struct {
*flowBatchQueue
swaps int
}
func (q *swapCountQueue) SwapRxBuf(buf []byte) []byte {
q.swaps++
return q.flowBatchQueue.SwapRxBuf(buf)
}
// TestFlowStagedTXAdaptiveBudget: a fast reader's adaptive sends must stop spending credit until another reader's
// earlier reservation reaches the wire.
func TestFlowStagedTXAdaptiveBudget(t *testing.T) {
for _, shape := range []struct {
name string
packets, segments int
}{
{"single", 1, 1}, {"batch", 8, 30},
} {
t.Run(shape.name, func(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
sc := adaptive(2)
// p0's large read must go to the workers at once, p1 stays inline
sc.gateHold = 1
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
sender.f.routines = 2
sender.f.writers = []udp.Conn{sender.conn, sender.conn}
ci := sender.hostinfo.ConnectionState
start := ci.messageCounter.Load()
entered, unblock := make(chan struct{}), make(chan struct{})
delayCrypto(sender, func(n uint64) {
if n == start+1 {
close(entered)
<-unblock
}
})
p0 := newStagedTx(sender.f, newFlowBatchQueue(nil), 0)
p1 := newStagedTx(sender.f, newFlowBatchQueue(nil), 1)
defer p0.close()
defer p1.close()
var release sync.Once
defer release.Do(func() { close(unblock) })
large, _ := flowTestBatches(1, 8, 1, 45)
p0.send(large[0])
<-entered
perBatch := shape.packets * shape.segments
small, _ := flowTestBatches(9600/perBatch, shape.packets, 1, shape.segments)
fits := int(ci.txBudgetLimit()-360) / perBatch
for _, packets := range small[:fits] {
p1.send(packets)
}
require.Equal(t, uint64(360+fits*perBatch), (ci.txBudget.state.Load()>>24)&txBudgetMask, "inline sends must spend shared credit")
done := make(chan struct{})
go func() {
defer close(done)
for _, packets := range small[fits:] {
p1.send(packets)
}
}()
select {
case <-done:
t.Error("adaptive reader bypassed outstanding credit")
case <-time.After(50 * time.Millisecond):
}
require.LessOrEqual(t, ci.messageCounter.Load()-start, ci.txBudgetLimit())
release.Do(func() { close(unblock) })
<-done
p0.drain()
p1.drain()
wire := sender.conn.captured()
require.Len(t, wire, 360+40*240)
requireAccepted(t, receiver, wire)
require.Zero(t, ci.txBudget.state.Load())
})
}
}
// BenchmarkFlowStagedTX runs reads of eight 45-segment superpackets through listenIn into a socket that discards
// them, inline and staged.
func BenchmarkFlowStagedTX(b *testing.B) {
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
for _, sc := range []stagedConfig{{}, stagedOn(4)} {
b.Run(cipher.CipherName()+"/"+stagedName(sc), func(b *testing.B) {
if sc.tx && cipher.CipherName() == noiseutil.CipherAESGCM.CipherName() && noiseutil.EncryptLockNeeded {
b.Skip("AES-GCM requires encryption in nonce order")
}
sender, _ := newFlowTestPair(b, cipher, sc)
sender.f.writers = []udp.Conn{udp.NoopConn{}}
batch, _ := flowTestBatches(1, 8, 4, 45)
batches := make([][]tio.Packet, b.N)
for i := range batches {
batches[i] = batch[0]
}
b.SetBytes(int64(8 * 45 * flowTestMSS))
b.ReportAllocs()
b.ResetTimer()
sender.f.listenIn(newFlowBatchQueue(batches), 0)
})
}
}
}
-1
View File
@@ -7,7 +7,6 @@ require (
filippo.io/bigmod v0.1.0
github.com/DefinedNet/netlink v1.3.2-dn.1
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
github.com/armon/go-radix v1.0.0
github.com/flynn/noise v1.1.0
github.com/gaissmai/bart v0.29.0
github.com/gogo/protobuf v1.3.2
-2
View File
@@ -12,8 +12,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+35 -23
View File
@@ -61,16 +61,17 @@ type HandshakeManager struct {
vpnIps map[netip.Addr]*HandshakeHostInfo
indexes map[uint32]*HandshakeHostInfo
mainHostMap *HostMap
lightHouse *LightHouse
outside udp.Conn
config HandshakeConfig
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
messageMetrics *MessageMetrics
metricInitiated metrics.Counter
metricTimedOut metrics.Counter
f *Interface
l *slog.Logger
mainHostMap *HostMap
lightHouse *LightHouse
outside udp.Conn
config HandshakeConfig
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
messageMetrics *MessageMetrics
metricInitiated metrics.Counter
metricTimedOut metrics.Counter
metricRejectedBlocklisted metrics.Counter
f *Interface
l *slog.Logger
// can be used to trigger outbound handshake for the given vpnIp
trigger chan netip.Addr
@@ -118,18 +119,19 @@ func (hh *HandshakeHostInfo) cachePacket(l *slog.Logger, t header.MessageType, s
func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager {
return &HandshakeManager{
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
indexes: map[uint32]*HandshakeHostInfo{},
mainHostMap: mainHostMap,
lightHouse: lightHouse,
outside: outside,
config: config,
trigger: make(chan netip.Addr, config.triggerBuffer),
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
l: l,
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
indexes: map[uint32]*HandshakeHostInfo{},
mainHostMap: mainHostMap,
lightHouse: lightHouse,
outside: outside,
config: config,
trigger: make(chan netip.Addr, config.triggerBuffer),
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
metricRejectedBlocklisted: metrics.GetOrRegisterCounter("handshake_manager.rejected.blocklisted", nil),
l: l,
}
}
@@ -403,6 +405,8 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
hm.Unlock()
hm.lightHouse.QueryServer(vpnAddr)
// If this host came from a static_map hostname its addresses may have moved, retries will pick up anything new
hostinfo.remotes.ResolveHostnames()
return hostinfo
}
@@ -717,7 +721,12 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
response, result, err := machine.ProcessPacket(nil, packet)
if err != nil {
f.l.Error("Failed to process handshake packet", "from", via, "error", err)
level := slog.LevelError
if errors.Is(err, cert.ErrBlockListed) {
level = slog.LevelDebug
hm.metricRejectedBlocklisted.Inc(1)
}
f.l.Log(context.Background(), level, "Failed to process handshake packet", "from", via, "error", err)
return
}
@@ -849,6 +858,9 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
response, result, err := machine.ProcessPacket(nil, packet)
if err != nil {
if errors.Is(err, cert.ErrBlockListed) {
hm.metricRejectedBlocklisted.Inc(1)
}
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
if machine.Failed() {
f.l.Warn("Failed to process handshake packet, abandoning",
+88
View File
@@ -1,12 +1,17 @@
package nebula
import (
"log/slog"
"net/netip"
"testing"
"time"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
@@ -235,6 +240,89 @@ func TestHandleIncomingDispatch(t *testing.T) {
})
}
func TestBlocklistedHandshakeLogLevelAndCounter(t *testing.T) {
ca, _, caKey, _ := ct.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
pool := ct.NewTestCAPool(ca)
newCredential := func(name, network string) (cert.Certificate, *handshake.Credential) {
c, _, rawKey, _ := ct.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey,
name, ca.NotBefore(), ca.NotAfter(), []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
privateKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey)
require.NoError(t, err)
certBytes, err := c.MarshalForHandshakes()
require.NoError(t, err)
cipherSuite := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
return c, handshake.NewCredential(c, certBytes, privateKey, cipherSuite)
}
initiatorCert, initiatorCredential := newCredential("initiator", "10.0.0.1/24")
responderCert, responderCredential := newCredential("responder", "10.0.0.2/24")
fingerprint, err := initiatorCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(fingerprint)
initiator, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return initiatorCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return pool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 1, nil }, true, header.HandshakeIXPSK0)
require.NoError(t, err)
packet, err := initiator.Initiate(nil)
require.NoError(t, err)
handler := &capturingHandler{}
logger := slog.New(handler)
pki := &PKI{}
pki.cs.Store(&CertState{initiatingVersion: cert.Version2, v2Credential: responderCredential})
pki.caPool.Store(pool)
hm := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
hm.metricRejectedBlocklisted = metrics.NewCounter()
hm.f = &Interface{pki: pki, l: logger}
via := ViaSender{UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242")}
hm.beginHandshake(via, packet, nil)
entry := findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelDebug, entry.Level)
require.ErrorIs(t, entry.Attrs["error"].(error), cert.ErrBlockListed)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
handler.entries = nil
hm.beginHandshake(via, nil, nil)
entry = findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelError, entry.Level)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
// The initiator sees the same rejection in continueHandshake, after a
// responder has accepted its first packet and sent a valid reply.
cleanPool := ct.NewTestCAPool(ca)
responder, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return responderCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return cleanPool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 2, nil }, false, header.HandshakeIXPSK0)
require.NoError(t, err)
response, _, err := responder.ProcessPacket(nil, packet)
require.NoError(t, err)
responderFingerprint, err := responderCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(responderFingerprint)
initiatingHM := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
initiatingHM.metricRejectedBlocklisted = metrics.NewCounter()
initiatingHM.f = &Interface{l: logger}
hh := &HandshakeHostInfo{
machine: initiator,
hostinfo: &HostInfo{localIndexId: 1, vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}},
}
initiatingHM.indexes[1] = hh
via.IsRelayed = true // skip the remote allow list before certificate validation
initiatingHM.continueHandshake(via, hh, response)
assert.Equal(t, int64(1), initiatingHM.metricRejectedBlocklisted.Count())
}
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
+165 -89
View File
@@ -18,20 +18,12 @@ import (
)
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
if hostinfo := f.filterInsidePacket(pkt, fwPacket, rejectBuf, q, localCache); hostinfo != nil {
f.sendInsideMessage(hostinfo, pkt, nb, sendBatch)
}
}
// filterInsidePacket parses and filters pkt, handling local traffic, destination lookup, pending handshakes and
// the outbound firewall. It returns the hostinfo to send pkt to, or nil when pkt was dropped, rejected, forwarded
// to self or cached for a pending handshake.
func (f *Interface) filterInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, rejectBuf []byte, q int, localCache firewall.ConntrackCache) *HostInfo {
// borrowed: pkt.Bytes is owned by the originating tio.Queue and is
// only valid until the next Read on that queue. Filtering and sending
// must finish before then. Self-forwarding writes synchronously and the
// handshake cache copies each segment. To retain the packet beyond the
// next Read, use pkt.Clone() to detach it from the borrow.
// only valid until the next Read on that queue. Every consumer below
// (parse, self-forward, handshake cache, sendInsideMessage) reads it
// synchronously; do not retain pkt outside this call. If a future
// caller needs to keep the packet, use pkt.Clone() to detach it from
// the borrow.
//
// pkt.Bytes is either one IP datagram (GSO zero) or a TSO/USO
// superpacket. In both cases the L3+L4 headers at the start describe
@@ -46,13 +38,13 @@ func (f *Interface) filterInsidePacket(pkt tio.Packet, fwPacket *firewall.Parsed
"error", err,
)
}
return nil
return
}
// Ignore local broadcast packets
if f.dropLocalBroadcast {
if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
return nil
return
}
}
@@ -79,12 +71,12 @@ func (f *Interface) filterInsidePacket(pkt tio.Packet, fwPacket *firewall.Parsed
}
// Otherwise, drop. On linux, we should never see these packets - Linux
// routes packets from the nebula addr to the nebula addr through the loopback device.
return nil
return
}
// Ignore multicast packets
if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() {
return nil
return
}
hostinfo, ready := f.getOrHandshakeConsiderRouting(&fwPacket.Packet, func(hh *HandshakeHostInfo) {
@@ -111,26 +103,50 @@ func (f *Interface) filterInsidePacket(pkt tio.Packet, fwPacket *firewall.Parsed
"fwPacket", fwPacket,
)
}
return nil
return
}
if !ready {
return nil
return
}
dropReason := f.firewall.Drop(fwPacket.Packet, false, hostinfo, f.pki.GetCAPool(), localCache)
if dropReason == nil {
return hostinfo
f.sendInsideMessage(hostinfo, pkt, nb, sendBatch)
} else {
f.rejectInside(packet, rejectBuf, q)
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("dropping outbound packet",
"fwPacket", fwPacket,
"reason", dropReason,
)
}
}
}
func (f *Interface) sendInsideEncrypt(hostinfo *HostInfo, ci *ConnectionState, seg, scratch, nb []byte) []byte {
if noiseutil.EncryptLockNeeded {
ci.writeLock.Lock()
}
c := ci.messageCounter.Add(1)
out := header.Encode(scratch, header.Version, header.Message, 0, hostinfo.remoteIndexId, c)
out, encErr := ci.eKey.EncryptDanger(out, out, seg, c, nb)
if noiseutil.EncryptLockNeeded {
ci.writeLock.Unlock()
}
if encErr != nil {
hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet",
"error", encErr,
"udpAddr", hostinfo.GetRemote(),
"counter", c,
)
// Skip this segment; the rest of the superpacket can still go out. TCP will retransmit anything we drop here.
return nil
}
f.rejectInside(packet, rejectBuf, q)
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("dropping outbound packet",
"fwPacket", fwPacket,
"reason", dropReason,
)
}
return nil
return out
}
// sendInsideMessage encrypts a firewall-approved inside packet (or every
@@ -140,46 +156,88 @@ func (f *Interface) filterInsidePacket(pkt tio.Packet, fwPacket *firewall.Parsed
// scratch arena: SegmentSuperpacket builds each segment's plaintext in
// segScratch[:segLen] in turn, and we encrypt directly into a fresh SendBatch slot.
func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt tio.Packet, nb []byte, sendBatch *batch.SendBatch) {
var it txItem
if f.prepareTxItem(&it, hostinfo, pkt) {
f.encryptTxItem(&it, nb, sendBatch)
ci := hostinfo.ConnectionState
if ci.eKey == nil {
return
}
}
// recordOutbound marks traffic out on hostinfo, once per packet or superpacket, and when we rebound since this tunnel
// last sent, asks the lighthouse to get the far side punching at us again.
func (f *Interface) recordOutbound(hostinfo *HostInfo) {
// One traffic-out mark covers every segment of the superpacket; doing it
// per segment in sendInsideEncrypt paid an atomic store up to ~45 extra
// times per TSO packet, inside writeLock under boring crypto.
//
// We rebound since this tunnel last sent, ask the lighthouse to get the far side punching at us again
if f.connectionManager.Out(hostinfo) {
f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Lighthouse update triggered for punch due to rebind epoch", "vpnAddrs", hostinfo.vpnAddrs)
hostinfo.logger(f.l).Debug("Lighthouse update triggered for punch due to rebind epoch",
"vpnAddrs", hostinfo.vpnAddrs,
)
}
}
}
// findRelay returns the first relay that can reach hostinfo, forgetting the ones that no longer resolve.
func (f *Interface) findRelay(hostinfo *HostInfo) (*HostInfo, *Relay) {
for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
via, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
if err == nil {
return via, relay
remote := hostinfo.GetRemote()
if !remote.IsValid() { //the relay path
//first, find our relay hostinfo:
var relayHostInfo *HostInfo
var relay *Relay
var err error
for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
relayHostInfo, relay, err = f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
if err != nil {
hostinfo.relayState.DeleteRelay(relayIP)
hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo",
"relay", relayIP,
"error", err,
)
continue
}
break
}
if relayHostInfo == nil || relay == nil {
//failure already logged
return
}
hostinfo.relayState.DeleteRelay(relayIP)
hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo", "relay", relayIP, "error", err)
}
return nil, nil
}
// encryptMessage encodes the header for counter c and encrypts p behind it into out. The caller allocates c and
// holds writeLock when the cipher requires encryption in nonce order.
func (f *Interface) encryptMessage(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, ci *ConnectionState, c uint64, p, nb, out []byte, remote netip.AddrPort) []byte {
out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
out, err := ci.eKey.EncryptDanger(out, out, p, c, nb)
if err != nil {
hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet", "error", err, "udpAddr", remote, "counter", c)
err = tio.SegmentSuperpacket(pkt, func(seg []byte) error {
//relay header + header + plaintext + AEAD tag + relay tag
scratch := sendBatch.Reserve(len(seg) + header.MaxOverhead)
innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb)
if innerPacket == nil {
return nil
}
//now we need to do a relay-encrypt:
toSend, err := f.prepareSendVia(relayHostInfo, relay, innerPacket, nb, scratch, true)
if err != nil {
//already logged
return nil
}
sendBatch.Commit(toSend, relayHostInfo.GetRemote())
return nil
})
if err != nil {
hostinfo.logger(f.l).Error("Failed to segment superpacket for relay send", "error", err)
}
return
}
err := tio.SegmentSuperpacket(pkt, func(seg []byte) error {
// header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305)
scratch := sendBatch.Reserve(header.Len + len(seg) + 16)
out := f.sendInsideEncrypt(hostinfo, ci, seg, scratch, nb)
if out == nil {
return nil
}
sendBatch.Commit(out, remote)
return nil
})
if err != nil {
hostinfo.logger(f.l).Error("Failed to segment superpacket for send", "error", err)
}
return out
}
func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
@@ -388,33 +446,27 @@ func (f *Interface) prepareSendVia(via *HostInfo,
nb,
out []byte,
nocopy bool,
c uint64,
) ([]byte, error) {
// A nonzero c was reserved by the caller, who also did the connection manager bookkeeping
reserved := c != 0
locked := !reserved && noiseutil.EncryptLockNeeded
if locked {
if noiseutil.EncryptLockNeeded {
// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
via.ConnectionState.writeLock.Lock()
}
if !reserved {
var ok bool
c, ok = via.ConnectionState.NextMessageCounter()
if !ok {
if locked {
via.ConnectionState.writeLock.Unlock()
}
f.dropExhausted(via, c, "Dropping outbound relay packets, tunnel message counter is exhausted")
return nil, fmt.Errorf("tunnel message counter is exhausted")
c, ok := via.ConnectionState.NextMessageCounter()
if !ok {
if noiseutil.EncryptLockNeeded {
via.ConnectionState.writeLock.Unlock()
}
f.connectionManager.OutNoRebind(via)
f.dropExhausted(via, c, "Dropping outbound relay packets, tunnel message counter is exhausted")
return nil, fmt.Errorf("tunnel message counter is exhausted")
}
out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
f.connectionManager.OutNoRebind(via)
// Authenticate the header and payload, but do not encrypt for this message type.
// The payload consists of the inner, unencrypted Nebula header, as well as the end-to-end encrypted payload.
if len(out)+len(ad)+via.ConnectionState.eKey.Overhead() > cap(out) {
if locked {
if noiseutil.EncryptLockNeeded {
via.ConnectionState.writeLock.Unlock()
}
via.logger(f.l).Error("SendVia out buffer not large enough for relay",
@@ -438,16 +490,14 @@ func (f *Interface) prepareSendVia(via *HostInfo,
var err error
out, err = via.ConnectionState.eKey.EncryptDanger(out, out, nil, c, nb)
if locked {
if noiseutil.EncryptLockNeeded {
via.ConnectionState.writeLock.Unlock()
}
if err != nil {
via.logger(f.l).Info("Failed to EncryptDanger in sendVia", "error", err)
return nil, err
}
if !reserved {
f.connectionManager.RelayUsed(relay.LocalIndex)
}
f.connectionManager.RelayUsed(relay.LocalIndex)
return out, nil
}
@@ -460,7 +510,7 @@ func (f *Interface) prepareSendVia(via *HostInfo,
// out is a buffer used to store the result of the Encrypt operation
// q indicates which writer to use to send the packet.
func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, nocopy bool, q int) {
toSend, err := f.prepareSendVia(via, relay, ad, nb, out, nocopy, 0)
toSend, err := f.prepareSendVia(via, relay, ad, nb, out, nocopy)
if err != nil {
// already logged by prepareSendVia
return
@@ -514,19 +564,34 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
return
}
// A closing tunnel needs neither traffic tracking nor another punch.
if t != header.CloseTunnel {
f.recordOutbound(hostinfo)
}
out = f.encryptMessage(t, st, hostinfo, ci, c, p, nb, out, remote)
if noiseutil.EncryptLockNeeded {
ci.writeLock.Unlock()
}
if out == nil {
return
//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
// A closing tunnel is torn down right after this, so skip the connection manager entirely: no point recording
// traffic or asking the lighthouse for a punch. Otherwise, if we rebound since this tunnel last sent, ask the
// lighthouse to get the far side punching at us again.
if t != header.CloseTunnel && f.connectionManager.Out(hostinfo) {
f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
if f.l.Enabled(context.Background(), slog.LevelDebug) {
f.l.Debug("Lighthouse update triggered for punch due to rebind epoch",
"vpnAddrs", hostinfo.vpnAddrs,
)
}
}
var err error
out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
if noiseutil.EncryptLockNeeded {
ci.writeLock.Unlock()
}
if err != nil {
hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet",
"error", err,
"udpAddr", remote,
"counter", c,
)
return
}
if remote.IsValid() {
err = f.writers[q].WriteTo(out, remote)
if err != nil {
@@ -548,8 +613,19 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
}
}
} else {
if via, relay := f.findRelay(hostinfo); via != nil && relay != nil {
f.SendVia(via, relay, out, nb, fullOut[:header.Len+len(out)], true, q)
// Try to send via a relay
for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
if err != nil {
hostinfo.relayState.DeleteRelay(relayIP)
hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo",
"relay", relayIP,
"error", err,
)
continue
}
f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true, q)
break
}
}
}
-180
View File
@@ -1,180 +0,0 @@
package nebula
import (
"net/netip"
"testing"
"time"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/overlaytest"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newInsideFilterTestInterface(t *testing.T, allow bool) (*Interface, *HostInfo, *capturingTun) {
t.Helper()
local := netip.MustParseAddr("192.0.2.1")
remote := netip.MustParseAddr("192.0.2.2")
f, tun := newSelfForwardInterface(local)
f.inside = overlaytest.NoopTun{}
f.myVpnNetworksTable = &bart.Lite{}
f.myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.0/24"))
f.myBroadcastAddrsTable.Insert(netip.MustParsePrefix("192.0.2.255/32"))
f.dropLocalBroadcast, f.dropMulticast = true, true
localCert := &dummyCert{version: cert.Version1, networks: []netip.Prefix{netip.PrefixFrom(local, 24)}}
peerCert := &dummyCert{version: cert.Version1, networks: []netip.Prefix{netip.PrefixFrom(remote, 24)}}
f.firewall = NewFirewall(f.l, time.Minute, time.Minute, time.Minute, localCert)
if allow {
require.NoError(t, f.firewall.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
}
f.pki = &PKI{}
f.pki.caPool.Store(cert.NewCAPool())
f.hostMap = newHostMap(f.l)
f.lightHouse = newTestLighthouse()
f.lightHouse.amLighthouse = true // Keep destination lookup from probing other remotes.
f.handshakeManager = NewHandshakeManager(f.l, f.hostMap, f.lightHouse, udp.NoopConn{}, defaultHandshakeConfig)
f.handshakeManager.f = f
f.connectionManager = &connectionManager{intf: f}
h := &HostInfo{
vpnAddrs: []netip.Addr{remote},
ConnectionState: &ConnectionState{
peerCert: &cert.CachedCertificate{Certificate: peerCert},
},
}
f.hostMap.Hosts[remote] = h
return f, h, tun
}
func TestFilterInsidePacket(t *testing.T) {
local := netip.MustParseAddr("192.0.2.1")
remote := netip.MustParseAddr("192.0.2.2")
for _, tt := range []struct {
name string
packet []byte
allow bool
reject bool
ready bool
writes int
}{
{name: "allowed", packet: buildIPv4(local, remote, tcpSyn), allow: true, ready: true},
{name: "malformed", packet: []byte{0x45}, allow: true},
{name: "broadcast", packet: buildIPv4(local, netip.MustParseAddr("192.0.2.255"), udpDatagram), allow: true},
{name: "multicast", packet: buildIPv4(local, netip.MustParseAddr("224.0.0.1"), udpDatagram), allow: true},
{name: "firewall drop", packet: buildIPv4(local, remote, tcpSyn)},
{name: "firewall reject", packet: buildIPv4(local, remote, tcpSyn), reject: true, writes: 1},
{name: "unroutable drop", packet: buildIPv4(local, netip.MustParseAddr("198.51.100.1"), tcpSyn), allow: true},
{name: "unroutable reject", packet: buildIPv4(local, netip.MustParseAddr("198.51.100.1"), tcpSyn), allow: true, reject: true, writes: 1},
} {
t.Run(tt.name, func(t *testing.T) {
f, h, tun := newInsideFilterTestInterface(t, tt.allow)
f.firewall.OutboundSendReject = tt.reject
fp := &firewall.ParsedPacket{}
got := f.filterInsidePacket(tio.Packet{Bytes: tt.packet}, fp, make([]byte, mtu), 0, nil)
if tt.ready {
require.Same(t, h, got)
assert.Equal(t, local, fp.LocalAddr)
assert.Equal(t, remote, fp.RemoteAddr)
} else {
require.Nil(t, got)
}
assert.Zero(t, h.ConnectionState.messageCounter.Load(), "filtering must not allocate encryption counters")
assert.False(t, h.sentSinceCheck(), "filtering must not mark outbound traffic")
require.Len(t, tun.writes, tt.writes)
if tt.writes > 0 {
require.GreaterOrEqual(t, len(tun.writes[0]), 40)
assert.Equal(t, tt.packet[16:20], tun.writes[0][12:16])
assert.Equal(t, tt.packet[12:16], tun.writes[0][16:20])
assert.NotZero(t, tun.writes[0][33]&0x04, "reject must carry a TCP reset")
}
})
}
}
func TestFilterInsidePacketSelf(t *testing.T) {
local := netip.MustParseAddr("192.0.2.1")
f, _, tun := newInsideFilterTestInterface(t, true)
pkt := buildIPv4(local, local, tcpSyn)
require.Nil(t, f.filterInsidePacket(tio.Packet{Bytes: pkt}, &firewall.ParsedPacket{}, nil, 0, nil))
if immediatelyForwardToSelf {
require.Len(t, tun.writes, 1)
assert.Equal(t, pkt, tun.writes[0])
} else {
assert.Empty(t, tun.writes)
}
}
func TestFilterInsidePacketPendingHandshake(t *testing.T) {
f, h, tun := newInsideFilterTestInterface(t, true)
remote := h.vpnAddrs[0]
delete(f.hostMap.Hosts, remote)
hh := &HandshakeHostInfo{hostinfo: h}
f.handshakeManager.vpnIps[remote] = hh
pkt := buildIPv4(netip.MustParseAddr("192.0.2.1"), remote, udpDatagram)
want := append([]byte(nil), pkt...)
require.Nil(t, f.filterInsidePacket(tio.Packet{Bytes: pkt}, &firewall.ParsedPacket{}, nil, 0, nil))
require.Len(t, hh.packetStore, 1)
clear(pkt)
assert.Equal(t, want, hh.packetStore[0].packet, "the handshake cache must own its packet")
assert.Equal(t, header.Message, hh.packetStore[0].messageType)
assert.Empty(t, tun.writes)
assert.Zero(t, h.ConnectionState.messageCounter.Load())
}
type insideFilterConn struct {
udp.NoopConn
packets [][]byte
}
func (c *insideFilterConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
for _, b := range bufs {
c.packets = append(c.packets, append([]byte(nil), b...))
}
return len(bufs), nil
}
func TestConsumeInsidePacketFilterAndSend(t *testing.T) {
for _, allow := range []bool{false, true} {
t.Run(map[bool]string{false: "denied", true: "allowed"}[allow], func(t *testing.T) {
f, h, _ := newInsideFilterTestInterface(t, allow)
initR, respR := runTestHandshake(t)
ci, err := newConnectionStateFromResult(initR)
require.NoError(t, err)
rci, err := newConnectionStateFromResult(respR)
require.NoError(t, err)
ci.peerCert = h.ConnectionState.peerCert
h.ConnectionState = ci
h.remoteIndexId = 100
remote := netip.MustParseAddrPort("198.51.100.2:4242")
h.remote.Store(&remote)
conn := &insideFilterConn{}
sb := batch.NewSendBatch(conn, 1, mtu)
pkt := buildIPv4(netip.MustParseAddr("192.0.2.1"), h.vpnAddrs[0], udpDatagram)
before := ci.messageCounter.Load()
f.consumeInsidePacket(tio.Packet{Bytes: pkt}, &firewall.ParsedPacket{}, make([]byte, 12), sb, nil, 0, nil)
_, err = sb.Flush()
require.NoError(t, err)
if !allow {
assert.Empty(t, conn.packets)
assert.Equal(t, before, ci.messageCounter.Load())
assert.False(t, h.sentSinceCheck())
return
}
require.Len(t, conn.packets, 1)
assert.True(t, h.sentSinceCheck())
hdr := &header.H{}
require.NoError(t, hdr.Parse(conn.packets[0]))
assert.Equal(t, header.Message, hdr.Type)
assert.Equal(t, h.remoteIndexId, hdr.RemoteIndex)
assert.Equal(t, before+1, hdr.MessageCounter)
plain, err := rci.Decrypt(f.l, hdr.MessageCounter, conn.packets[0], make([]byte, 12))
require.NoError(t, err)
assert.Equal(t, pkt, plain)
})
}
}
+86 -67
View File
@@ -57,19 +57,14 @@ type InterfaceConfig struct {
// CpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
// should pin to. Queue i pins to CpuAffinity[i % len(CpuAffinity)] —
// shorter lists than `routines` cycle. Staged sender i uses entry i+1,
// unless that is its reader's CPU. Empty uses the automatic order. Only
// consulted when PinThreads is true.
// shorter lists than `routines` cycle. Empty list keeps the default
// pin-to-(i % NumCPU) behavior. Only consulted when PinThreads is true.
CpuAffinity []int
// defaultCpuAffinity is the automatic pin order for readers, kept apart
// from CpuAffinity because staged senders are only pinned to an explicit list.
defaultCpuAffinity []int
// PinThreads controls whether each TUN reader OS thread is pinned to a
// single CPU (via tun.pin_threads, default true). Pinning keeps each
// goroutine's sendmmsg on one XPS-selected NIC TX ring so per-flow
// packets stay ordered on the wire.
PinThreads bool
staged stagedConfig
l *slog.Logger
}
@@ -99,13 +94,11 @@ type Interface struct {
// should pin to. Queue i pins to cpuAffinity[i % len(cpuAffinity)].
// Empty falls back to the default pin-to-(allowed CPU) behavior.
// Only consulted when pinThreads is true.
cpuAffinity []int
defaultCpuAffinity []int
cpuAffinity []int
// pinThreads controls whether listenIn pins each TUN reader OS thread to
// a CPU at all (tun.pin_threads, default true). When false, threads are
// left free to migrate as on stock nebula.
pinThreads bool
staged stagedConfig
relayManager *relayManager
tryPromoteEvery atomic.Uint32
@@ -131,6 +124,8 @@ type Interface struct {
// reused (every udp.Conn ListenOut guarantees that ordering).
batchers []*batch.MultiCoalescer
wg sync.WaitGroup
// readers covers just the listenIn and listenOut goroutines, wg also holds the construction token Close releases
readers sync.WaitGroup
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
@@ -254,9 +249,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
connectionManager: c.connectionManager,
conntrackCacheTimeout: c.ConntrackCacheTimeout,
cpuAffinity: c.CpuAffinity,
defaultCpuAffinity: c.defaultCpuAffinity,
pinThreads: c.PinThreads,
staged: c.staged,
udpTxErrLimiter: udpTxErrLimiter,
udpRelayTxErrLimiter: udpRelayTxErrLimiter,
txOversizeLimiter: txOversizeLimiter,
@@ -350,14 +343,18 @@ func (f *Interface) activate() error {
func (f *Interface) run() {
// Launch n queues to read packets from udp
for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() {
defer f.readers.Done()
f.listenOut(i)
})
}
// Launch n queues to read packets from tun dev
for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() {
defer f.readers.Done()
f.listenIn(f.queues[i], i)
})
}
@@ -422,19 +419,15 @@ func (f *Interface) listenOut(i int) {
f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, payload, rxc)
}
flusher := func() { f.flushOutside(rxc) }
var err error
if hc, ok := li.(handoffConn); ok && f.staged.rx {
// Every stage on its own goroutine, see flow_rx.go. rxc becomes the ordered writer's context.
err = newStagedRx(f, i, rxc).listen(hc)
} else {
if f.staged.rx {
f.l.Info("udp conn cannot hand off receive batches, receive stays inline", "reader", i)
flusher := func() {
if err := f.batchers[i].Flush(); err != nil {
f.l.Error("Failed to flush tun coalescer", "error", err)
}
err = li.ListenOut(listener, flusher)
clear(rxc.hostmapCache)
}
err := li.ListenOut(listener, flusher)
// An error after teardown began is shutdown noise, the closed flag covers resources
// Close releases itself and the cancelled ctx covers ones torn down by their owners
// reacting to it, like the user device pipes
@@ -446,30 +439,32 @@ func (f *Interface) listenOut(i int) {
f.l.Debug("underlay reader is done", "reader", i)
}
// threadCPU is the CPU tun reader i pins to.
func (f *Interface) threadCPU(i int) int {
func (f *Interface) pinThisThread(i int) {
var cpu int
if n := len(f.cpuAffinity); n > 0 {
// Explicit tun.cpu_affinity list wins; parseCpuAffinity already
// validated the entries against the allowed CPU set.
return f.cpuAffinity[i%n]
} else if n := len(f.defaultCpuAffinity); n > 0 {
return f.defaultCpuAffinity[i%n]
cpu = f.cpuAffinity[i%n]
} else if allowed, err := util.AllowedCPUs(); err == nil && len(allowed) > 0 {
// Default: spread queues across the CPUs we're actually allowed to
// run on. Under a cpuset/taskset mask these aren't 0..NumCPU-1, so
// i % NumCPU would pick unrunnable IDs and every pin would fail.
return allowed[i%len(allowed)]
cpu = allowed[i%len(allowed)]
} else {
cpu = i % runtime.NumCPU()
}
return i % runtime.NumCPU()
}
func (f *Interface) pinThreadToCPU(i, cpu int) {
if err := util.PinThreadToCPU(cpu); err != nil {
f.l.Warn("failed to pin tun processing thread to CPU", "queue", i, "cpu", cpu, "err", err)
f.l.Warn("failed to pin tun reader to CPU", "queue", i, "cpu", cpu, "err", err)
}
}
func (f *Interface) listenIn(queue tio.Queue, i int) {
// Pinning this thread (and goroutine) to a single CPU keeps every sendmmsg from this goroutine going through the
// same TX ring on the nic, so the wire sees per-flow order. Skip entirely when tun.pin_threads is false.
if f.pinThreads {
f.pinThisThread(i)
}
rejectBuf := make([]byte, mtu)
arenaSize := batch.SendBatchCap * (udp.MTU + 32)
sb := batch.NewSendBatch(f.writers[i], batch.SendBatchCap, arenaSize)
@@ -478,26 +473,6 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
if sq, ok := queue.(handoffQueue); ok && f.staged.tx {
// Every stage on its own goroutine, see flow_tx.go
err := newStagedTx(f, sq, i).run(fwPacket, rejectBuf, conntrackCache)
if !f.closed.Load() && f.ctx.Err() == nil {
f.l.Error("Error while reading outbound packet, closing", "error", err, "reader", i)
f.onFatal(err)
}
f.l.Debug("overlay reader is done", "reader", i)
return
}
if f.staged.tx {
f.l.Info("tun queue cannot hand off read batches, transmit stays inline", "reader", i)
}
// Pinning this thread (and goroutine) to a single CPU keeps every sendmmsg from this goroutine going through the
// same TX ring on the nic, so the wire sees per-flow order. Skip entirely when tun.pin_threads is false.
if f.pinThreads {
f.pinThreadToCPU(i, f.threadCPU(i))
}
for {
pkts, err := queue.Read()
if err != nil {
@@ -568,11 +543,12 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
curCert = cs.getCertificate(cert.Version1)
}
// The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction.
// Check to see if that set has changed, and if so, rebuild the firewall.
certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks)
if !c.HasChanged("firewall") && !certUnsafeChanged {
// Compare with what the running firewall was built from, not the last config, so a firewall that failed to build
// is tried again. The cert is the one this reload brought, PKI registers its reload callback before this one
src := c.Render("firewall")
if src != "" && src == f.firewall.source && curCert != nil &&
slices.Equal(curCert.Networks(), f.firewall.assignedNetworks) &&
slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) {
f.l.Debug("No firewall config change detected")
return
}
@@ -713,6 +689,14 @@ func (f *Interface) GetCertState() *CertState {
return f.pki.getCertState()
}
// shutdowner is a udp.Conn or overlay.Device whose readers can be woken without releasing their fds
type shutdowner interface {
Shutdown() error
}
// readerShutdownTimeout bounds how long Close waits on woken readers before releasing their fds anyway
const readerShutdownTimeout = 5 * time.Second
// Close releases the interface's resources: the udp sockets and the tun device.
// It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated,
// calls after the first return nil without doing anything.
@@ -722,20 +706,55 @@ func (f *Interface) Close() error {
}
var errs []error
// Release the udp readers
for i, u := range f.writers {
err := u.Close()
if err != nil {
closeUdp := func(i int, u udp.Conn) {
if err := u.Close(); err != nil {
f.l.Error("Error while closing udp socket", "error", err, "writer", i)
errs = append(errs, err)
}
}
closeTun := func() {
if err := f.inside.Close(); err != nil {
errs = append(errs, err)
}
}
// Release the tun device (closing the tun also closes all readers)
closeErr := f.inside.Close()
if closeErr != nil {
errs = append(errs, closeErr)
// Wake every reader before releasing any fd, one that loops back into a syscall on a reused fd number never wakes.
// Anything without Shutdown is closed right away, as before.
var deferred []func()
for i, u := range f.writers {
if s, ok := u.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
f.l.Error("Error while shutting down udp socket", "error", err, "writer", i)
errs = append(errs, err)
}
deferred = append(deferred, func() { closeUdp(i, u) })
} else {
closeUdp(i, u)
}
}
if s, ok := f.inside.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
errs = append(errs, err)
}
deferred = append(deferred, closeTun)
} else {
closeTun()
}
if len(deferred) > 0 {
readersDone := make(chan struct{})
go func() {
f.readers.Wait()
close(readersDone)
}()
select {
case <-readersDone:
case <-time.After(readerShutdownTimeout):
f.l.Warn("Readers did not stop, closing their fds anyway", "timeout", readerShutdownTimeout)
}
for _, closeFn := range deferred {
closeFn()
}
}
// Release the construction token so waiters know the resources are gone
+58 -3
View File
@@ -135,8 +135,63 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
res := cfg.ReloadConfigString(rules("nope"))
// The running firewall still isn't built from what the config asks for, so the same reload fails again
for range 2 {
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
res := cfg.ReloadConfigString(rules("tcp"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
assert.False(t, res.Failed)
assert.NotSame(t, fw, f.firewall)
}
// Every input the firewall is built from rebuilds it when it changes, and nothing else does
func TestReloadFirewall_RebuildsOnAnyInput(t *testing.T) {
l := test.NewLogger()
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
base := "firewall:\n inbound:\n - port: any\n proto: any\n host: any\n"
setup := func() (*config.C, *Interface, *PKI) {
pki := &PKI{}
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks}, initiatingVersion: cert.Version2})
cfg := config.NewC(l)
require.NoError(t, cfg.LoadString(base))
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
require.NoError(t, err)
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
return cfg, f, pki
}
setCert := func(pki *PKI, networks, unsafe []netip.Prefix) {
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks, unsafeNetworks: unsafe}, initiatingVersion: cert.Version2})
}
cases := map[string]struct {
cfg string
cert func(*PKI)
want bool
}{
"nothing": {base + "punchy:\n punch: true\n", nil, false},
"rules": {base + " outbound:\n - port: any\n proto: any\n host: any\n", nil, true},
"inbound_action": {base + " inbound_action: reject\n", nil, true},
"conntrack": {base + " conntrack:\n tcp_timeout: 1m\n", nil, true},
"networks": {base, func(p *PKI) { setCert(p, []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil) }, true},
"unsafe_networks": {base, func(p *PKI) { setCert(p, networks, []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}) }, true},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
cfg, f, pki := setup()
old := f.firewall
if tc.cert != nil {
tc.cert(pki)
}
res := cfg.ReloadConfigString(tc.cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, tc.want, f.firewall != old)
})
}
}
+5 -3
View File
@@ -452,7 +452,7 @@ func getStaticMapCadence(c *config.C) (time.Duration, error) {
}
func getStaticMapLookupTimeout(c *config.C) (time.Duration, error) {
lookupTimeout := c.GetString("static_map.lookup_timeout", "250ms")
lookupTimeout := c.GetString("static_map.lookup_timeout", "5s")
d, err := time.ParseDuration(lookupTimeout)
if err != nil {
return 0, err
@@ -630,7 +630,8 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
ctx := lh.ctx
lh.Unlock()
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, func() {
// On reload am.hr is the previous resolver, its last known good addresses carry over in case DNS is down right now
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, am.hr, func() {
// This callback runs whenever the DNS hostname resolver finds a different set of addr's
// in its resolution for hostnames.
am.Lock()
@@ -642,7 +643,8 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
}
am.unlockedSetHostnamesResults(hr)
for _, addrPort := range hr.GetAddrs() {
// Only literals belong in the cache, anything from DNS has to stay in hr where a not found answer can remove it
for _, addrPort := range hr.literalAddrs() {
if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) {
continue
}
+9 -7
View File
@@ -12,6 +12,7 @@ import (
"strings"
"time"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/cpupick"
"github.com/slackhq/nebula/noiseutil"
@@ -71,7 +72,11 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
}
l.Info("Firewall started", "firewallHashes", fw.GetRuleHashes())
ssh, err := sshd.NewSSHServer(ctx, l.With("subsystem", "sshd"))
// The command registry is shared by every surface that exposes these commands, so a
// command cannot reach one of them without reaching the others.
cmdRegistry := commands.New()
ssh, err := sshd.NewSSHServer(ctx, l.With("subsystem", "sshd"), cmdRegistry)
if err != nil {
return nil, util.ContextualizeIfNeeded("Error while creating SSH server", err)
}
@@ -244,8 +249,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
pinThreads := c.GetBool("tun.pin_threads", true)
cpuAffinity := parseCpuAffinity(c, l, routines)
var defaultCpuAffinity []int
staged := parseStagedConfig(c, l)
if pinThreads && routines > 1 && len(cpuAffinity) == 0 && !configTest {
// The operator didn't choose pin CPUs, so pick a default set that
// prefers performance cores and doesn't stack co-located instances
@@ -272,7 +275,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
l.Warn("tun.pin_threads_key is invalid, using PID")
}
defaultCpuAffinity = cpupick.Default(routines, key, l)
cpuAffinity = cpupick.Default(routines, key, l)
}
ifConfig := &InterfaceConfig{
@@ -297,9 +300,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
punchy: punchy,
ConntrackCacheTimeout: conntrackCacheTimeout,
CpuAffinity: cpuAffinity,
defaultCpuAffinity: defaultCpuAffinity,
PinThreads: pinThreads,
staged: staged,
l: l,
}
@@ -347,13 +348,14 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
return nil, nil
}
attachCommands(l, c, ssh, ifce)
attachCommands(l, c, cmdRegistry, ifce)
networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
return &Control{
state: StateReady,
f: ifce,
commands: cmdRegistry,
l: l,
ctx: ctx,
cancel: cancel,
-69
View File
@@ -5,14 +5,12 @@ import (
"net/netip"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/slackhq/nebula/cert"
cert_test "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/util"
"github.com/stretchr/testify/assert"
@@ -125,70 +123,3 @@ firewall:
require.NoError(t, err)
require.Nil(t, ctrl, "a config test hands back nothing to stop, so it must stop itself")
}
func TestParseStagedConfig(t *testing.T) {
l := test.NewLogger()
staged := func(workers any) stagedConfig {
c := config.NewC(l)
if workers != nil {
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": workers}}
}
return parseStagedConfig(c, l)
}
for _, workers := range []any{nil, -1, 0} {
require.Equal(t, stagedConfig{}, staged(workers))
}
if runtime.GOOS != "linux" {
require.Equal(t, stagedConfig{}, staged(2))
return
}
want := stagedConfig{workers: 2, tx: !noiseutil.EncryptLockNeeded, rx: true, inlineBelow: 256, gateHold: stagedGateHold,
txSlots: 8, rxSlots: 6}
require.Equal(t, want, staged(2))
c := config.NewC(l)
c.Settings["listen"] = map[string]any{"batch": 128}
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "tx_slots": 3, "rx_slots": 1}}
got := parseStagedConfig(c, l)
require.Equal(t, []int{3, 2}, []int{got.txSlots, got.rxSlots}, "overrides apply, down to the minimum")
c = config.NewC(l)
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "tx": false, "inline_below": 0}}
got = parseStagedConfig(c, l)
require.False(t, got.tx, "tx: false keeps transmit inline")
require.True(t, got.rx)
require.Zero(t, got.inlineBelow, "inline_below: 0 hands every batch off")
c = config.NewC(l)
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "inline_below": 32}}
require.Equal(t, 32, parseStagedConfig(c, l).inlineBelow)
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "inline_below": -1}}
require.Zero(t, parseStagedConfig(c, l).inlineBelow, "a negative threshold is 0")
for _, tt := range []struct {
hold any
want int
}{{nil, stagedGateHold}, {8, 8}, {0, 1}, {-3, 1}} {
c = config.NewC(l)
settings := map[string]any{"workers": 2}
if tt.hold != nil {
settings["gate_hold"] = tt.hold
}
c.Settings["tun"] = map[string]any{"staged": settings}
require.Equal(t, tt.want, parseStagedConfig(c, l).gateHold, "gate_hold %v", tt.hold)
}
}
func TestStagedSlots(t *testing.T) {
for _, tt := range []struct{ workers, batch, tx, rx int }{
{workers: 2, batch: 64, tx: 8, rx: 6},
{workers: 12, batch: 64, tx: 14, rx: 6},
{workers: 2, batch: 96, tx: 8, rx: 4},
{workers: 2, batch: 512, tx: 8, rx: 4},
{workers: 2, batch: 32, tx: 8, rx: 6},
{workers: 2, batch: 1, tx: 8, rx: 6},
} {
tx, rx := stagedSlots(tt.workers, tt.batch, 0, 0)
require.Equal(t, []int{tt.tx, tt.rx}, []int{tx, rx}, "workers %d batch %d", tt.workers, tt.batch)
}
}
+9 -25
View File
@@ -130,10 +130,17 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
}
out, err := hostinfo.ConnectionState.Decrypt(f.l, h.MessageCounter, packet, rxc.nb)
if !f.finishOutsideDecrypt(hostinfo, via, h.MessageCounter, err) {
if err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Failed to decrypt packet", "error", err, "from", via, "header", h)
}
return
}
// Roam before we respond
f.handleHostRoaming(hostinfo, via)
f.connectionManager.In(hostinfo)
switch h.Type {
case header.Message:
switch h.Subtype {
@@ -466,36 +473,13 @@ func parseV4(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
return nil
}
// finishOutsideDecrypt logs a failed decrypt, or roams and records the traffic for an authenticated packet before it
// is handled. It reports whether the packet authenticated.
func (f *Interface) finishOutsideDecrypt(hostinfo *HostInfo, via ViaSender, counter uint64, err error) bool {
if err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Failed to decrypt packet", "error", err, "from", via, "counter", counter)
}
return false
}
// Roam before we respond
f.handleHostRoaming(hostinfo, via)
f.connectionManager.In(hostinfo)
return true
}
// flushOutside ends a receive batch: it writes out the tun coalescer and drops the batch's hostmap cache.
func (f *Interface) flushOutside(rxc *rxContext) {
if err := f.batchers[rxc.q].Flush(); err != nil {
f.l.Error("Failed to flush tun coalescer", "error", err)
}
clear(rxc.hostmapCache)
}
func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounter uint64, out []byte, rxc *rxContext) {
err := newPacket(out, true, rxc.fwPacket)
if err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out)
return
}
return
}
dropReason := f.firewall.Drop(rxc.fwPacket.Packet, true, hostinfo, f.pki.GetCAPool(), rxc.ctCache.Get())
+4 -9
View File
@@ -43,14 +43,6 @@ func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort) {
b.dsts = append(b.dsts, dst)
}
// Reset drops every queued packet without sending it.
func (b *SendBatch) Reset() {
clear(b.bufs)
b.bufs = b.bufs[:0]
b.dsts = b.dsts[:0]
b.arena.Reset()
}
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations
// were undeliverable; the batch is drained either way.
func (b *SendBatch) Flush() (int, error) {
@@ -59,6 +51,9 @@ func (b *SendBatch) Flush() (int, error) {
if len(b.bufs) > 0 {
written, err = b.out.WriteBatch(b.bufs, b.dsts)
}
b.Reset()
clear(b.bufs)
b.bufs = b.bufs[:0]
b.dsts = b.dsts[:0]
b.arena.Reset()
return written, err
}
+13 -3
View File
@@ -20,6 +20,7 @@ type offloadQueueSet struct {
// usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel.
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
usoEnabled bool
shutdown atomic.Bool
closed atomic.Bool
// l is handed to each queue for its bad-vnet-header drop logging.
l *slog.Logger
@@ -69,6 +70,17 @@ func (c *offloadQueueSet) wakeForShutdown() error {
return err
}
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *offloadQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *offloadQueueSet) Close() error {
if c.closed.Swap(true) {
return nil
@@ -76,9 +88,7 @@ func (c *offloadQueueSet) Close() error {
errs := []error{}
// Signal all readers blocked in poll to wake up and exit.
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
if err := c.Shutdown(); err != nil {
errs = append(errs, err)
}
+13 -3
View File
@@ -16,6 +16,7 @@ type pollQueueSet struct {
// pqi is exactly the same as pq, but stored as the interface type
pqi []Queue
shutdownFd int
shutdown atomic.Bool
closed atomic.Bool
}
@@ -59,6 +60,17 @@ func (c *pollQueueSet) wakeForShutdown() error {
return err
}
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *pollQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *pollQueueSet) Close() error {
if c.closed.Swap(true) {
return nil
@@ -66,9 +78,7 @@ func (c *pollQueueSet) Close() error {
errs := []error{}
// Signal all readers blocked in poll to wake up and exit.
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
if err := c.Shutdown(); err != nil {
errs = append(errs, err)
}
+2 -9
View File
@@ -7,6 +7,8 @@ import (
// QueueSet holds one or many Queue objects and helps close them in an orderly way.
type QueueSet interface {
io.Closer
// Shutdown wakes every reader without releasing any fds, Close only once they have exited
Shutdown() error
Queues() []Queue
// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
@@ -75,15 +77,6 @@ type GSOInfo struct {
// superpacket that needs segmentation before its bytes can be encrypted and sent on the wire.
func (g GSOInfo) IsSuperpacket() bool { return g.Size > 0 }
// SegmentCount reports how many segments SegmentSuperpacket will cut p into, without touching its bytes.
func (p Packet) SegmentCount() int {
if !p.GSO.IsSuperpacket() {
return 1
}
n := (len(p.Bytes) - int(p.GSO.HdrLen) + int(p.GSO.Size) - 1) / int(p.GSO.Size)
return max(n, 1)
}
// Clone returns a Packet whose Bytes is a freshly allocated copy of p.Bytes,
// safe to retain past the next Read or Close on the originating Queue.
// GSO metadata is copied verbatim.
+6 -16
View File
@@ -66,7 +66,9 @@ type Offload struct {
// usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD,
// so writers can decide whether emitting GSO_UDP_L4 superpackets is safe.
usoEnabled bool
closed atomic.Bool
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool
// gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted
// by WriteGSO. Kept separate from the read-only package-level validVnetHdr
@@ -176,6 +178,9 @@ func (r *Offload) readPacket(block bool) (int, error) {
// This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs).
// Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue.
func (r *Offload) Read() ([]Packet, error) {
if r.shutdown.Load() {
return nil, os.ErrClosed
}
r.pending = r.pending[:0]
r.rxOff = 0
@@ -219,21 +224,6 @@ func (r *Offload) Read() ([]Packet, error) {
return r.pending, nil
}
// SwapRxBuf installs buf as the receive buffer for the following Reads and returns the one backing the packets
// of the most recent Read, so the caller can keep them past the next Read. A nil buf allocates a replacement of
// the required size. Single-reader only, like Read.
func (r *Offload) SwapRxBuf(buf []byte) []byte {
if buf == nil {
buf = make([]byte, tunRxBufCap)
}
if len(buf) < tunRxBufCap {
panic(fmt.Sprintf("tio: SwapRxBuf buffer is %d bytes, need %d", len(buf), tunRxBufCap))
}
old := r.rxBuf
r.rxBuf = buf
return old
}
// logDroppedRead reports a tun packet dropped for a bad/unsupported virtio
// header. Debug-gated so the happy path never pays for attribute assembly.
func (r *Offload) logDroppedRead(err error) {
+6 -1
View File
@@ -13,7 +13,9 @@ import (
type Poll struct {
fd int
shutdownFd int
closed atomic.Bool
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool
readBuf []byte
batchRet [1]Packet
@@ -52,6 +54,9 @@ func (t *Poll) blockOnWrite() error {
// config currently lose that batching. blockOn and the EAGAIN plumbing are
// already shared; kept one-packet-per-Read for now to preserve behavior.
func (t *Poll) Read() ([]Packet, error) {
if t.shutdown.Load() {
return nil, os.ErrClosed
}
n, err := t.readOne(t.readBuf)
if err != nil {
return nil, err
+27
View File
@@ -223,3 +223,30 @@ func TestOffloadQueueSet_Close_ClosesShutdownFd(t *testing.T) {
// Second Close must not touch fds (shutdownFd is now -1) and must return nil.
require.NoError(t, qs.Close())
}
// A busy reader never polls the shutdown eventfd, so Read has to stop on its own with data still waiting
func TestQueueSet_ShutdownStopsReaderWithDataWaiting(t *testing.T) {
sets := map[string]func() (QueueSet, error){
"poll": NewPollQueueSet,
"offload": func() (QueueSet, error) { return NewOffloadQueueSet(false, slog.New(slog.DiscardHandler)) },
}
for name, newSet := range sets {
t.Run(name, func(t *testing.T) {
var fds [2]int
require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
t.Cleanup(func() { _ = unix.Close(fds[1]) })
qs, err := newSet()
require.NoError(t, err)
require.NoError(t, qs.Add(fds[0]))
t.Cleanup(func() { _ = qs.Close() })
_, err = unix.Write(fds[1], make([]byte, 64))
require.NoError(t, err)
require.NoError(t, qs.Shutdown())
_, err = qs.Queues()[0].Read()
require.ErrorIs(t, err, os.ErrClosed)
})
}
}
-19
View File
@@ -911,25 +911,6 @@ func TestOffloadWriteZeroLength(t *testing.T) {
}
}
// TestOffloadSwapRxBuf: SwapRxBuf hands over the buffer the last Read filled and reads into the one it is given.
func TestOffloadSwapRxBuf(t *testing.T) {
o := &Offload{rxBuf: make([]byte, tunRxBufCap)}
read := o.rxBuf
owned := o.SwapRxBuf(nil)
if &owned[0] != &read[0] || len(o.rxBuf) != tunRxBufCap || &o.rxBuf[0] == &read[0] {
t.Fatal("SwapRxBuf(nil) must hand over the last read buffer and allocate a new one")
}
if back := o.SwapRxBuf(owned); &back[0] == &owned[0] || &o.rxBuf[0] != &owned[0] {
t.Fatal("SwapRxBuf must install the buffer it is given")
}
defer func() {
if recover() == nil {
t.Fatal("SwapRxBuf must reject a short buffer")
}
}()
o.SwapRxBuf(make([]byte, 10))
}
// TestWriteGSOSuperpacketGeometry decodes the vnet header the kernel would see for a multi-segment write:
// the GSO type must match the proto and IP version
// gso_size must be the per-segment size (the kernel rejects a superpacket with gso_size == 0),
+54 -84
View File
@@ -3,7 +3,6 @@
package overlay
import (
"errors"
"fmt"
"log/slog"
"math/bits"
@@ -12,6 +11,7 @@ import (
"sync"
"sync/atomic"
"syscall"
"time"
"unsafe"
"github.com/gaissmai/bart"
@@ -35,6 +35,26 @@ type tun struct {
// and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool
l *slog.Logger
// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
// fields below it
routesMu sync.Mutex
// routeStates is who held each installable unsafe_route prefix when we last looked
routeStates map[netip.Prefix]routeState
// routeWatch is the routing socket watchRoutes reads, Close closes it
routeWatch *os.File
// routeSweep reconciles the unsafe_routes every routeSweepEvery
routeSweep *time.Timer
routeSweepEvery time.Duration
closed bool
// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
// lock so the route watcher never waits on a reconcile
routeMoversMu sync.Mutex
routeMovers map[netip.Prefix]routeMover
// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
lastRouteEvent atomic.Int64
}
type ifReq struct {
@@ -139,11 +159,12 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
}
t := &tun{
f: os.NewFile(uintptr(fd), ""),
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c),
l: l,
f: os.NewFile(uintptr(fd), ""),
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c),
routeSweepEvery: routeSweepInterval,
l: l,
}
err = t.reload(c, true, nil)
@@ -219,6 +240,16 @@ func utunNameFromFd(fd int) string {
}
func (t *tun) Close() error {
t.routesMu.Lock()
t.closed = true
if t.routeWatch != nil {
_ = t.routeWatch.Close()
}
if t.routeSweep != nil {
t.routeSweep.Stop()
}
t.routesMu.Unlock()
if t.f != nil {
return t.f.Close()
}
@@ -287,8 +318,22 @@ func (t *tun) Activate() error {
return fmt.Errorf("failed to run tun device: %s", err)
}
// Watch before adding so a route removed in between is still noticed
watch, err := openRouteSocket()
if err != nil {
t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
// Unsafe path routes
return t.addRoutes(false, nil)
if err = t.addRoutes(false, nil); err != nil {
if watch != nil {
_ = watch.Close()
}
return err
}
t.startRouteWatch(watch)
return nil
}
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
@@ -393,7 +438,8 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
}
if !initial && !change {
return nil
// Any reload puts back the unsafe_routes someone deleted
return t.addRoutes(true, r)
}
routeTree, err := makeRouteTree(t.l, routes, false)
@@ -511,82 +557,6 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
return nil, nil
}
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
err := addRoute(r.Cidr, linkAddr)
if err != nil {
if errors.Is(err, unix.EEXIST) {
// Every route we added already exists on a reload, only warn about someone else's
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
retErr.Log(t.l)
rl.Failed()
} else {
return retErr
}
}
} else {
t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
}
}
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
linkAddr := t.linkAddr.Load()
if t.hostOwned || linkAddr == nil {
return nil
}
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue
}
err := delRoute(r.Cidr, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
+366
View File
@@ -3,12 +3,17 @@
package overlay
import (
"encoding/binary"
"encoding/hex"
"fmt"
"net"
"net/netip"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
@@ -104,6 +109,367 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
const (
// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
// With no interface address, which x/net/route skips
interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
)
func TestRoutesMayHaveMoved(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
routes := []Route{{Cidr: cidr, Via: via, Install: true}}
// A request like route(8) sends, the kernel echoes it to every routing socket
request := func(typ, flags int, dst, mask [4]byte) []byte {
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: typ,
Flags: flags,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: dst},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
},
}
b, err := m.Marshal()
require.NoError(t, err)
return b
}
raw := func(s string) []byte {
b, err := hex.DecodeString(s)
require.NoError(t, err)
return b
}
lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
routesMayHaveMoved := func(b []byte, routes []Route) bool {
moved, _, _ := routesMayHaveMoved(b, routes)
return moved
}
assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
neighbor := raw(kernelSubnetDelete)
binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
}
// A request names its prefix and who made it, the kernel's own and interface changes name neither
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: unix.RTM_CHANGE,
Flags: unix.RTF_UP,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
},
}
b, err := m.Marshal()
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved)
assert.Equal(t, cidr, p)
assert.Equal(t, int32(1234), pid)
for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
b, err := hex.DecodeString(s)
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved, name)
assert.False(t, p.IsValid(), name)
assert.Zero(t, pid, name)
}
}
func TestProcessName(t *testing.T) {
assert.Equal(t, "launchd", processName(1))
self := filepath.Base(os.Args[0])
assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
assert.Empty(t, processName(-1))
}
// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
// the minute between sweeps when it isn't zero.
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
requireRoot(t)
cfg := "tun:\n unsafe_routes:\n"
for _, cidr := range cidrs {
via := "10.250.0.2"
if netip.MustParsePrefix(cidr).Addr().Is6() {
via = "fd00:250::2"
}
cfg += " - route: " + cidr + "\n via: " + via + "\n"
}
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString(cfg))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
require.NoError(t, err)
if sweep != 0 {
tn.routeSweepEvery = sweep
}
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
return tn, c, cfg
}
func requireRoot(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating interfaces and adding routes needs root")
}
}
// routeVia is the index of the interface that routes cidr, 0 if none does
func routeVia(t *testing.T, cidr string) int {
msgs, err := fetchRoutes()
require.NoError(t, err)
return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
}
// routeStateOf is what tn made of cidr when it last looked
func routeStateOf(tn *tun, cidr string) routeState {
tn.routesMu.Lock()
defer tn.routesMu.Unlock()
return tn.routeStates[netip.MustParsePrefix(cidr)]
}
func routeArgs(action, cidr string, extra ...string) []string {
args := []string{"-q", "-n", action}
if netip.MustParsePrefix(cidr).Addr().Is6() {
args = append(args, "-inet6")
}
return append(append(args, "-net", cidr), extra...)
}
func routeCmd(t *testing.T, action, cidr string, extra ...string) {
requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
}
// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
// says so
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
require.NotContains(t, string(out), "writing to routing socket")
}
// cleanupRoute deletes cidr's route when the test ends, whoever holds it
func cleanupRoute(t *testing.T, cidr string) {
t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
}
// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
func routeThroughLo0(t *testing.T, cidr string) int {
requireRoot(t)
routeCmd(t, "add", cidr, "-interface", "lo0")
cleanupRoute(t, cidr)
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
return lo0.Index
}
// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
t.Run(cidr, func(t *testing.T) {
lo0 := routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
})
}
}
// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
cidr := "10.251.1.0/24"
lo0 := routeThroughLo0(t, cidr)
_, c, _ := newRouteTestTun(t, 0, cidr)
res := c.ReloadConfigString("tun: {}\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, lo0, routeVia(t, cidr))
}
// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
cidr := "10.251.3.0/24"
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
time.Sleep(2 * routeRetryDelay)
require.Equal(t, lo0.Index, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
}
// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
cidr := "10.251.4.0/24"
tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
// A reload with nothing changed is how to ask for it back
res := c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, ours, routeVia(t, cidr))
}
// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
for name, tc := range map[string]struct {
cidr string
sweep time.Duration
misfile func(t *testing.T, tn *tun, cidr string)
}{
// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
}},
// The sweep that caps a table that never settles
"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
tn.reconcileRoutes()
require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
}},
} {
t.Run(name, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, tc.cidr))
routeCmd(t, "delete", tc.cidr)
tc.misfile(t, tn, tc.cidr)
lo0 := routeThroughLo0(t, tc.cidr)
require.Eventually(t, func() bool {
s := routeStateOf(tn, tc.cidr)
return s.kind == routeYielded && s.holder == lo0
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", tc.cidr)
assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
// taken for a hand delete
func TestTunYieldsToARouteChangedAway(t *testing.T) {
for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
t.Run(cidr, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
routeCmd(t, "change", cidr, "-interface", "lo0")
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
require.Equal(t, lo0.Index, routeVia(t, cidr))
require.Eventually(t, func() bool {
s := routeStateOf(tn, cidr)
return s.kind == routeYielded && s.holder == lo0.Index
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
// ethernet configd routed the lan through
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
requireRoot(t)
var feth string
for i := 100; i < 200 && feth == ""; i++ {
if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
feth = fmt.Sprintf("feth%d", i)
}
}
require.NotEmpty(t, feth)
require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
// refuses one named by the feth itself
cidr := "10.251.2.0/24"
routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
cleanupRoute(t, cidr)
ifc, err := net.InterfaceByName(feth)
require.NoError(t, err)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Losing the address first leaves the route, the reconcile finds it still held
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
time.Sleep(2 * routeRetryDelay)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Only RTM_IFINFO says the interface went, its route goes with no message of its own
require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
cidr := "10.251.5.0/24"
routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
tn.routesMu.Lock()
require.NoError(t, tn.routeWatch.Close())
tn.routesMu.Unlock()
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
func TestTunDevNeedsRestart(t *testing.T) {
for name, want := range map[string]bool{
+121
View File
@@ -0,0 +1,121 @@
//go:build !ios && !e2e_testing
package overlay
import (
"bytes"
"os"
"syscall"
"testing"
"github.com/stretchr/testify/require"
)
func TestTun_WriteFraming(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: w}
for _, tc := range []struct {
name string
pkt []byte
af byte
}{
// Literal families rather than syscall.AF_*, so a wrong constant in Write can't also fix the expectation.
{"ipv4", []byte{0x45, 1, 2, 3, 4, 5}, 2},
{"ipv6", []byte{0x60, 1, 2, 3, 4, 5}, 30},
} {
t.Run(tc.name, func(t *testing.T) {
n, err := tn.Write(tc.pkt)
require.NoError(t, err)
require.Equal(t, len(tc.pkt), n)
got := make([]byte, 64)
n, err = r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, tc.af}, tc.pkt...), got[:n])
})
}
_, err = tn.Write([]byte{0x10, 1, 2, 3})
require.Error(t, err)
_, err = tn.Write([]byte{})
require.ErrorIs(t, err, syscall.EIO)
// The rejected writes must not have put anything on the fd, so the next read sees only this packet.
pkt := []byte{0x45, 9, 9}
_, err = tn.Write(pkt)
require.NoError(t, err)
got := make([]byte, 64)
n, err := r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, 2}, pkt...), got[:n])
}
func TestTun_ReadStripsHeader(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
pkt := []byte{0x45, 1, 2, 3, 4, 5}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err := tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
func TestTun_ReadShort(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
// Fewer bytes than the protocol header is not a packet.
_, err = w.Write([]byte{0, 0})
require.NoError(t, err)
n, err := tn.Read(make([]byte, 64))
require.NoError(t, err)
require.Zero(t, n)
// The next packet reads cleanly only if the short read consumed exactly what was written.
pkt := []byte{0x45, 1, 2, 3}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err = tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
// A heap allocation per packet rarely shows in a short throughput run, only across GC pauses, so
// this pins Read and Write to zero allocations directly.
func TestTun_ReadWriteDoNotAllocate(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tw := &tun{f: w}
tr := &tun{f: r}
pkts := [][]byte{bytes.Repeat([]byte{0x45}, 1300), bytes.Repeat([]byte{0x60}, 1300)}
buf := make([]byte, 1500)
i := 0
allocs := testing.AllocsPerRun(1000, func() {
i++
if _, err := tw.Write(pkts[i%len(pkts)]); err != nil {
t.Fatal(err)
}
if _, err := tr.Read(buf); err != nil {
t.Fatal(err)
}
})
require.Zero(t, allocs)
}
+5
View File
@@ -832,6 +832,11 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
t.routeTree.Store(newTree)
}
// Shutdown wakes the tun readers without releasing their fds, Close releases them
func (t *tun) Shutdown() error {
return t.readers.Shutdown()
}
func (t *tun) Close() error {
t.closeLock.Lock()
defer t.closeLock.Unlock()
+451
View File
@@ -0,0 +1,451 @@
//go:build !ios && !e2e_testing
package overlay
import (
"errors"
"fmt"
"net"
"net/netip"
"os"
"strconv"
"sync/atomic"
"time"
"unsafe"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/util"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
// or until another interface takes the prefix, after which ours follows theirs like any other.
const (
// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
// replaces a route by deleting and adding, looking in between would find the prefix free
routeRetryDelay = time.Second
// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
// a message for each
routeSweepInterval = time.Minute
// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
routeSocketRcvBuf = 1 << 20
)
type routeStateKind int
const (
routeUnknown routeStateKind = iota
// routeInstalled is our route holding the prefix
routeInstalled
// routeYielded is another interface holding the prefix, ours is added once it lets go
routeYielded
// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
// another interface takes the prefix
routeRemoved
// routeFailed is adding our route failing for some other reason, it is retried
routeFailed
)
type routeState struct {
kind routeStateKind
// holder is the index of the interface holding the prefix when kind is routeYielded
holder int
}
// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
// is looked up as the message arrives, route(8) has exited by the time the table settles
type routeMover struct {
pid int32
name string
}
// logArgs appends m to a log line's args, when we know who it was
func (m routeMover) logArgs(args ...any) []any {
if m.pid == 0 {
return args
}
args = append(args, "pid", m.pid)
if m.name != "" {
args = append(args, "process", m.name)
}
return args
}
// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
func processName(pid int32) string {
kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
if err != nil || kp.Proc.P_pid != pid {
return ""
}
return unix.ByteSliceToString(kp.Proc.P_comm[:])
}
// installRoutes returns the routes we put in the system routing table, one per prefix
func installRoutes(routes []Route) []Route {
seen := make(map[netip.Prefix]struct{}, len(routes))
var out []Route
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
p := r.Cidr.Masked()
if _, ok := seen[p]; ok {
continue
}
seen[p] = struct{}{}
out = append(out, r)
}
return out
}
// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
// interface holds its prefix. It runs on start and on every reload.
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
var msgs []netroute.Message
var msgsErr error
fetched := false
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
err := addRoute(p, linkAddr)
if err == nil {
t.l.Info("Added route", "route", r)
states[p] = routeState{kind: routeInstalled}
continue
}
if !errors.Is(err, unix.EEXIST) {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if !logErrors {
return retErr
}
retErr.Log(t.l)
rl.Failed()
states[p] = routeState{kind: routeFailed}
continue
}
// Every route we added already exists on a reload, only warn about someone else's
if !fetched {
msgs, msgsErr = fetchRoutes()
fetched = true
}
if msgsErr != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
states[p] = prev
continue
}
switch holder := routeInterfaceIn(msgs, p); holder {
case linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case 0:
// Removed since the add, the watcher will see to it
states[p] = prev
default:
if prev.kind != routeYielded || prev.holder != holder {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
}
states[p] = routeState{kind: routeYielded, holder: holder}
}
}
t.routeStates = states
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
msgs, err := fetchRoutes()
if err != nil {
var left []netip.Prefix
for _, r := range installRoutes(routes) {
left = append(left, r.Cidr)
}
return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
}
for _, r := range installRoutes(routes) {
p := r.Cidr.Masked()
// Whatever happens next, the prefix is no longer one we know about
delete(t.routeStates, p)
if routeInterfaceIn(msgs, p) != linkAddr.Index {
continue
}
err = delRoute(p, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
// changed since it last looked, it runs every routeSweepEvery.
func (t *tun) reconcileRoutes() {
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return
}
t.routeMoversMu.Lock()
movers := t.routeMovers
t.routeMovers = nil
t.routeMoversMu.Unlock()
msgs, err := fetchRoutes()
if err != nil {
t.l.Error("Failed to read the system routing table", "error", err)
return
}
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
by := movers[p]
holder := routeInterfaceIn(msgs, p)
switch {
case holder == linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case holder != 0:
// Another interface taking a prefix we left alone puts ours back in line behind theirs
if prev.kind != routeYielded || prev.holder != holder {
t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
}
states[p] = routeState{kind: routeYielded, holder: holder}
case prev.kind == routeInstalled:
t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
states[p] = routeState{kind: routeRemoved}
case prev.kind == routeRemoved:
states[p] = prev
default:
err := addRoute(p, linkAddr)
switch {
case err == nil:
if prev.kind == routeYielded {
t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
} else {
t.l.Info("Added route", "route", r)
}
states[p] = routeState{kind: routeInstalled}
case errors.Is(err, unix.EEXIST):
// Taken since the fetch, the next look will see by whom
states[p] = prev
default:
if prev.kind != routeFailed {
util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
}
states[p] = routeState{kind: routeFailed}
}
}
}
t.routeStates = states
}
// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
// routing table settles after anything that may have moved a prefix
func (t *tun) startRouteWatch(watch *os.File) {
t.routesMu.Lock()
defer t.routesMu.Unlock()
if t.closed {
if watch != nil {
_ = watch.Close()
}
return
}
t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
if watch != nil {
t.routeWatch = watch
go t.watchRoutes(watch)
}
}
func (t *tun) sweepRoutes() {
// A table still changing gets reconciled once it settles
if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
t.reconcileRoutes()
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
if !t.closed {
t.routeSweep.Reset(t.routeSweepEvery)
}
}
func openRouteSocket() (*os.File, error) {
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
}
// Fewer drops, the sweep catches the rest
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
// Non-blocking puts it on the poller, so Close interrupts a Read
if err = unix.SetNonblock(fd, true); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
}
return os.NewFile(uintptr(fd), "route"), nil
}
// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
func (t *tun) watchRoutes(f *os.File) {
var reconciled atomic.Int64
settle := time.AfterFunc(time.Hour, func() {
reconciled.Store(time.Now().UnixNano())
t.reconcileRoutes()
})
settle.Stop()
defer settle.Stop()
var burst time.Time
self := int32(os.Getpid())
b := make([]byte, os.Getpagesize())
for {
n, err := f.Read(b)
if err != nil {
if !errors.Is(err, os.ErrClosed) {
t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
return
}
moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
if !moved {
continue
}
// Our own are already in routeStates
if p.IsValid() && pid != self {
t.routeMoversMu.Lock()
if t.routeMovers == nil {
t.routeMovers = make(map[netip.Prefix]routeMover)
}
t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
t.routeMoversMu.Unlock()
}
now := time.Now()
t.lastRouteEvent.Store(now.UnixNano())
if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
burst = now
}
settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
}
}
// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
// and the pid of the process that made it.
func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
if len(b) < 4 {
return false, p, 0
}
switch int(b[3]) {
case unix.RTM_IFINFO, unix.RTM_DELADDR:
return true, p, 0
case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
default:
return false, p, 0
}
if len(b) < unix.SizeofRtMsghdr {
return false, p, 0
}
hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
// A failed request is reported too, and a scoped route never held the prefix
if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
return false, p, 0
}
if hdr.Pid == 0 {
// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
// could be holding a prefix
return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
}
// Replies to a request carry the requester's padded sockaddrs
msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
if err != nil || len(msgs) != 1 {
return false, p, 0
}
m, ok := msgs[0].(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
return false, p, 0
}
p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
if !ok {
return false, netip.Prefix{}, 0
}
p = p.Masked()
for _, r := range installRoutes(routes) {
if r.Cidr.Masked() == p {
return true, p, hdr.Pid
}
}
return false, netip.Prefix{}, 0
}
// interfaceName is the name of the interface with index i, for logs
func interfaceName(i int) string {
if ifc, err := net.InterfaceByIndex(i); err == nil {
return ifc.Name
}
return strconv.Itoa(i)
}
+4
View File
@@ -528,6 +528,10 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
if !rm.GetAmRelay() {
return
}
if !slices.Contains(h.vpnAddrs, from) {
logMsg.Info("Discarding relay request where relayFrom is not the sending peer")
return
}
peer := rm.hostmap.QueryVpnAddr(target)
if peer == nil {
// Try to establish a connection to this host. If we get a future relay request,
+57
View File
@@ -7,8 +7,10 @@ import (
"testing"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestStartRelaysLogDedupe verifies that repeated attempts with the same relay set drop the log
@@ -95,3 +97,58 @@ func TestStartRelaysLogDedupe(t *testing.T) {
assert.NotContains(t, buf.String(), msg, "should not log when we shortcut out")
})
}
// TestHandleCreateRelayRequestRejectsSpoofedFrom verifies that a relay only forwards a CreateRelayRequest
// when relayFrom is the peer that sent it. Otherwise any peer could reset another host's established
// relay back to Requested, or allocate unbounded relay entries on the target's hostinfo.
func TestHandleCreateRelayRequestRejectsSpoofedFrom(t *testing.T) {
l := test.NewLogger()
relayAddr := netip.MustParseAddr("10.128.0.128")
malloryAddr := netip.MustParseAddr("10.128.0.3")
victimAddr := netip.MustParseAddr("10.128.0.1")
targetAddr := netip.MustParseAddr("10.128.0.2")
addrTable := new(bart.Lite)
addrTable.Insert(netip.PrefixFrom(relayAddr, relayAddr.BitLen()))
hm := newHostMap(l)
f := &Interface{myVpnAddrsTable: addrTable, hostMap: hm}
rm := &relayManager{l: l, hostmap: hm}
rm.amRelay.Store(true)
newRelayState := func() RelayState {
return RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}}
}
mallory := &HostInfo{vpnAddrs: []netip.Addr{malloryAddr}, localIndexId: 1, relayState: newRelayState()}
target := &HostInfo{
vpnAddrs: []netip.Addr{targetAddr},
localIndexId: 2,
remotes: NewRemoteList([]netip.Addr{targetAddr}, nil),
relayState: newRelayState(),
}
target.SetRemote(netip.MustParseAddrPort("192.0.2.2:4242"))
hm.unlockedAddHostInfo(mallory, f)
hm.unlockedAddHostInfo(target, f)
// The victim already has a working relay to the target through us.
_, err := AddRelay(l, target, hm, victimAddr, nil, ForwardingType, Established)
require.NoError(t, err)
relaysBefore := len(hm.Relays)
for i := range 10 {
spoofed := netip.AddrFrom4([4]byte{10, 128, 1, byte(i)})
for _, from := range []netip.Addr{victimAddr, spoofed} {
rm.handleCreateRelayRequest(cert.Version2, mallory, f, &NebulaControl{
Type: NebulaControl_CreateRelayRequest,
InitiatorRelayIndex: uint32(100 + i),
RelayFromAddr: netAddrToProtoAddr(from),
RelayToAddr: netAddrToProtoAddr(targetAddr),
})
}
}
r, ok := target.relayState.QueryRelayForByIp(victimAddr)
require.True(t, ok)
assert.Equal(t, Established, r.State, "victim's relay must not be reset by a request from another peer")
assert.Len(t, hm.Relays, relaysBefore, "spoofed requests must not allocate relay entries")
assert.Empty(t, mallory.relayState.CopyAllRelayFor(), "spoofed requests must not create state on the sender")
}
+124 -19
View File
@@ -2,6 +2,7 @@ package nebula
import (
"context"
"errors"
"log/slog"
"net"
"net/netip"
@@ -58,6 +59,8 @@ type cacheV6 struct {
type hostnamePort struct {
name string
port uint16
// addr is set when name is an ip literal, these never need a lookup
addr netip.Addr
}
type hostnamesResults struct {
@@ -65,15 +68,20 @@ type hostnamesResults struct {
network string
lookupTimeout time.Duration
cancelFn func()
resolveNow chan struct{}
l *slog.Logger
ips atomic.Pointer[map[netip.AddrPort]struct{}]
// known is the last known good addresses per hostname, kept so a reload can carry them forward
known atomic.Pointer[map[hostnamePort][]netip.AddrPort]
}
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, onUpdate func()) (*hostnamesResults, error) {
// NewHostnameResults starts resolving hostPorts. Any hostnames also in prev start with its last known good addresses.
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, prev *hostnamesResults, onUpdate func()) (*hostnamesResults, error) {
r := &hostnamesResults{
hostnames: make([]hostnamePort, len(hostPorts)),
network: network,
lookupTimeout: timeout,
resolveNow: make(chan struct{}, 1),
l: l,
}
@@ -81,6 +89,13 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
// DNS lookups for hostnames that aren't hardcoded IP's will happen in a background goroutine.
performBackgroundLookup := false
ips := map[netip.AddrPort]struct{}{}
known := make([][]netip.AddrPort, len(hostPorts))
var prevKnown map[hostnamePort][]netip.AddrPort
if prev != nil && prev.network == network {
if p := prev.known.Load(); p != nil {
prevKnown = *p
}
}
for idx, hostPort := range hostPorts {
rIp, sPort, err := net.SplitHostPort(hostPort)
@@ -98,13 +113,19 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
if err != nil {
// This address is a hostname, not an IP address
performBackgroundLookup = true
known[idx] = prevKnown[r.hostnames[idx]]
for _, a := range known[idx] {
ips[a] = struct{}{}
}
continue
}
// Save the IP address immediately
r.hostnames[idx].addr = addr
ips[netip.AddrPortFrom(addr, uint16(iPort))] = struct{}{}
}
r.ips.Store(&ips)
r.storeKnown(known)
// Time for the DNS lookup goroutine
if performBackgroundLookup {
@@ -114,21 +135,12 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
go func() {
defer ticker.Stop()
for {
r.resolve(newCtx, known)
r.storeKnown(known)
netipAddrs := map[netip.AddrPort]struct{}{}
for _, hostPort := range r.hostnames {
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, r.lookupTimeout)
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, r.network, hostPort.name)
timeoutCancel()
if err != nil {
l.Error("DNS resolution failed for static_map host",
"hostname", hostPort.name,
"network", r.network,
"error", err,
)
continue
}
for _, addrs := range known {
for _, a := range addrs {
netipAddrs[netip.AddrPortFrom(a.Unmap(), hostPort.port)] = struct{}{}
netipAddrs[a] = struct{}{}
}
}
origSet := r.ips.Load()
@@ -159,7 +171,7 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
case <-newCtx.Done():
return
case <-ticker.C:
continue
case <-r.resolveNow:
}
}
}()
@@ -168,6 +180,84 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
return r, nil
}
// resolve looks up every hostname in parallel and folds the results into known
func (hr *hostnamesResults) resolve(ctx context.Context, known [][]netip.AddrPort) {
var wg sync.WaitGroup
for i, hostPort := range hr.hostnames {
if hostPort.addr.IsValid() {
known[i] = []netip.AddrPort{netip.AddrPortFrom(hostPort.addr, hostPort.port)}
continue
}
wg.Go(func() {
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, hr.lookupTimeout)
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, hr.network, hostPort.name)
timeoutCancel()
known[i] = applyLookup(known[i], hostPort.port, addrs, err)
// A canceled ctx means we are reloading or shutting down, not that DNS failed
if err != nil && ctx.Err() == nil {
hr.l.Error("DNS resolution failed for static_map host",
"hostname", hostPort.name,
"network", hr.network,
"error", err,
"usingAddrs", known[i],
)
}
})
}
wg.Wait()
}
// applyLookup returns the addresses to use for a hostname given the previous ones and a lookup result.
// Only a not found answer clears them, a stale lighthouse address costs a handshake but an empty set costs everything.
// darwin also reports not found while offline, that's harmless since nothing is reachable and the next handshake
// triggers a fresh lookup once we're back.
func applyLookup(prev []netip.AddrPort, port uint16, addrs []netip.Addr, err error) []netip.AddrPort {
if err != nil {
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
return nil
}
return prev
}
out := make([]netip.AddrPort, 0, len(addrs))
for _, a := range addrs {
out = append(out, netip.AddrPortFrom(a.Unmap(), port))
}
return out
}
func (hr *hostnamesResults) storeKnown(known [][]netip.AddrPort) {
byName := make(map[hostnamePort][]netip.AddrPort, len(known))
for i, addrs := range known {
byName[hr.hostnames[i]] = addrs
}
hr.known.Store(&byName)
}
// literalAddrs returns the entries that were configured as ip addresses rather than hostnames
func (hr *hostnamesResults) literalAddrs() []netip.AddrPort {
var out []netip.AddrPort
for _, hostPort := range hr.hostnames {
if hostPort.addr.IsValid() {
out = append(out, netip.AddrPortFrom(hostPort.addr, hostPort.port))
}
}
return out
}
// ResolveNow asks the background goroutine to look up the hostnames again without waiting for the next tick
func (hr *hostnamesResults) ResolveNow() {
if hr == nil || hr.resolveNow == nil {
return
}
select {
case hr.resolveNow <- struct{}{}:
default:
}
}
func (hr *hostnamesResults) Cancel() {
if hr != nil && hr.cancelFn != nil {
hr.cancelFn()
@@ -256,6 +346,13 @@ func (r *RemoteList) ResetForOwner(ownerVpnAddr netip.Addr) {
r.shouldRebuild = true
}
// ResolveHostnames kicks a fresh DNS lookup of any static_map hostnames for this host
func (r *RemoteList) ResolveHostnames() {
r.RLock()
defer r.RUnlock()
r.hr.ResolveNow()
}
// ClearHostnameResults cancels the in-flight DNS resolver goroutine (if any) and drops the resolved IP cache.
func (r *RemoteList) ClearHostnameResults() {
r.Lock()
@@ -460,10 +557,14 @@ func (r *RemoteList) unlockedSetV4(ownerVpnIp, vpnIp netip.Addr, to []*V4AddrPor
// Reset the slice
c.reported = c.reported[:0]
// We can't take their array but we can take their pointers
for _, v := range to[:minInt(len(to), MaxRemotes)] {
// We can't take their array but we can take their pointers.
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
for _, v := range to {
if check(vpnIp, v) {
c.reported = append(c.reported, v)
if len(c.reported) == MaxRemotes {
break
}
}
}
}
@@ -508,10 +609,14 @@ func (r *RemoteList) unlockedSetV6(ownerVpnIp, vpnIp netip.Addr, to []*V6AddrPor
// Reset the slice
c.reported = c.reported[:0]
// We can't take their array but we can take their pointers
for _, v := range to[:minInt(len(to), MaxRemotes)] {
// We can't take their array but we can take their pointers.
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
for _, v := range to {
if check(vpnIp, v) {
c.reported = append(c.reported, v)
if len(c.reported) == MaxRemotes {
break
}
}
}
}
+89
View File
@@ -1,10 +1,15 @@
package nebula
import (
"context"
"encoding/binary"
"fmt"
"net"
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -328,6 +333,55 @@ func BenchmarkSortRebuild(b *testing.B) {
})
}
func TestApplyLookup(t *testing.T) {
prev := []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}
// A successful lookup replaces the previous addresses
got := applyLookup(prev, 4242, []netip.Addr{netip.MustParseAddr("::ffff:2.2.2.2")}, nil)
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("2.2.2.2:4242")}, got)
// A not found answer means the record is gone
assert.Empty(t, applyLookup(prev, 4242, nil, &net.DNSError{Err: "no such host", IsNotFound: true}))
// Anything else is not new information, keep what we had
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "i/o timeout", IsTimeout: true}))
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "server misbehaving", IsTemporary: true}))
assert.Equal(t, prev, applyLookup(prev, 4242, nil, context.DeadlineExceeded))
}
func TestNewHostnameResults_CarriesKnown(t *testing.T) {
// A canceled context keeps the background lookup from learning anything, as if DNS were down
ctx, cancel := context.WithCancel(context.Background())
cancel()
known := netip.MustParseAddrPort("1.1.1.1:4242")
prev := &hostnamesResults{network: "ip4"}
prev.known.Store(&map[hostnamePort][]netip.AddrPort{{name: "lh.invalid", port: 4242}: {known}})
hostPorts := []string{"lh.invalid:4242", "10.0.0.1:4242"}
literal := netip.MustParseAddrPort("10.0.0.1:4242")
hr, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, prev, func() {})
require.NoError(t, err)
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr.GetAddrs())
assert.Equal(t, []netip.AddrPort{literal}, hr.literalAddrs())
// And again from the new one, before it has finished a lookup of its own
hr2, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, hr, func() {})
require.NoError(t, err)
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr2.GetAddrs())
// A port change is a different entry
hr3, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, []string{"lh.invalid:4243"}, prev, func() {})
require.NoError(t, err)
assert.Empty(t, hr3.GetAddrs())
// So is a network change
hr4, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip6", time.Second, hostPorts, prev, func() {})
require.NoError(t, err)
assert.Equal(t, []netip.AddrPort{literal}, hr4.GetAddrs())
}
func newIp4AndPortFromString(s string) *V4AddrPort {
a := netip.MustParseAddrPort(s)
v4Addr := a.Addr().As4()
@@ -346,3 +400,38 @@ func newIp6AndPortFromString(s string) *V6AddrPort {
Port: uint32(a.Port()),
}
}
func TestRemoteList_SetFiltersBeforeCap(t *testing.T) {
owner := netip.MustParseAddr("0.0.0.0")
rl := NewRemoteList([]netip.Addr{owner}, nil)
// MaxRemotes denied addrs first, then the only allowed one, then enough allowed to exceed the cap
var v4 []*V4AddrPort
for i := 0; i < MaxRemotes; i++ {
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("172.17.0.%d:4242", i+1)))
}
for i := 0; i < MaxRemotes+1; i++ {
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("70.199.182.%d:4242", i+1)))
}
rl.unlockedSetV4(owner, owner, v4, func(_ netip.Addr, a *V4AddrPort) bool {
return protoV4AddrPortToNetAddrPort(a).Addr().As4()[0] != 172
})
var v6 []*V6AddrPort
for i := 0; i < MaxRemotes; i++ {
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[fd00::%d]:4242", i+1)))
}
for i := 0; i < MaxRemotes+1; i++ {
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[2001:db8::%d]:4242", i+1)))
}
rl.unlockedSetV6(owner, owner, v6, func(_ netip.Addr, a *V6AddrPort) bool {
return protoV6AddrPortToNetAddrPort(a).Addr().As16()[0] != 0xfd
})
rl.Rebuild([]netip.Prefix{})
assert.Len(t, rl.addrs, MaxRemotes*2)
for _, a := range rl.addrs {
assert.NotEqual(t, byte(172), a.Addr().As16()[12], "denied v4 addr %v was kept", a)
assert.NotEqual(t, byte(0xfd), a.Addr().As16()[0], "denied v6 addr %v was kept", a)
}
}
+4 -2
View File
@@ -36,11 +36,13 @@ func (l *tcpListener) Accept() (net.Conn, error) {
func (l *tcpListener) Close() error {
l.closeOnce.Do(func() {
// Close before leaving the map so a concurrent closeListeners that
// no longer sees this listener knows Accept is already released.
close(l.closed)
l.s.mu.Lock()
delete(l.s.mu.listeners, l.port)
l.s.mu.Unlock()
close(l.closed)
})
return nil
}
+70 -7
View File
@@ -32,13 +32,18 @@ import (
const nicID = 1
type Service struct {
eg *errgroup.Group
control *nebula.Control
ipstack *stack.Stack
// done is closed once the forwarding goroutines have exited and the
// netstack is torn down, at which point err holds the first fatal error.
done chan struct{}
err error
mu struct {
sync.Mutex
closed bool
listeners map[uint16]*tcpListener
}
}
@@ -65,7 +70,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
ctx := control.Context()
eg, ctx := errgroup.WithContext(ctx)
s := Service{
eg: eg,
control: control,
}
s.mu.listeners = map[uint16]*tcpListener{}
@@ -74,6 +78,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6},
})
// The teardown goroutine only owns the stack once New succeeds
defer func() {
if reterr != nil {
s.ipstack.Destroy()
}
}()
sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default
tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt)
if tcpipErr != nil {
@@ -123,6 +133,12 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
// this will read exactly one packet
n, err := reader.Read(buf)
if err != nil {
// An error after shutdown began is noise from the device pipes
// closing, anything else is fatal and must stop nebula too
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err
}
packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{
@@ -130,8 +146,8 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
})
linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf)
if err := ctx.Err(); err != nil {
return err
if ctx.Err() != nil {
return nil
}
}
})
@@ -139,13 +155,18 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
for {
packet := linkEP.ReadContext(ctx)
if packet == nil {
if err := ctx.Err(); err != nil {
return err
if ctx.Err() != nil {
return nil
}
continue
}
bufView := packet.ToView()
if _, err := bufView.WriteTo(writer); err != nil {
// Same shutdown noise handling as the inbound reader
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err
}
bufView.Release()
@@ -158,6 +179,22 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
return control.Wait()
})
// Tear down here rather than in Close so it also happens when nebula stops
// on its own. The forwarding goroutines must exit before the stack they feed
// is destroyed.
s.done = make(chan struct{})
go func() {
s.err = eg.Wait()
s.closeListeners()
// Destroy aborts the endpoints that exist when it starts, then waits on
// every endpoint, so a Dial registering in between would hold Wait until
// its connect timed out. Without the NIC new dials fail route lookup.
// RemoveNIC also closes linkEP.
_ = s.ipstack.RemoveNIC(nicID)
s.ipstack.Destroy()
close(s.done)
}()
return &s, nil
}
@@ -237,6 +274,9 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.mu.closed {
return nil, net.ErrClosed
}
if _, ok := s.mu.listeners[port]; ok {
return nil, fmt.Errorf("already listening on port %d", port)
}
@@ -245,15 +285,38 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
return l, nil
}
// Wait blocks until the service has fully stopped, either via Close or a fatal
// error, and returns the first fatal error if there was one.
func (s *Service) Wait() error {
return s.eg.Wait()
<-s.done
return s.err
}
// Close closes all listeners and stops nebula. Use Wait to block until
// shutdown has completed.
func (s *Service) Close() error {
s.closeListeners()
s.control.Stop()
return nil
}
// closeListeners unblocks pending Accept calls and rejects new listeners.
func (s *Service) closeListeners() {
// Every caller closes every listener rather than returning early once
// closed is set, so none returns while another is still closing them.
s.mu.Lock()
s.mu.closed = true
listeners := make([]*tcpListener, 0, len(s.mu.listeners))
for _, l := range s.mu.listeners {
listeners = append(listeners, l)
}
s.mu.Unlock()
for _, l := range listeners {
_ = l.Close()
}
}
func (s *Service) tcpHandler(r *tcp.ForwarderRequest) {
endpointID := r.ID()
+159
View File
@@ -327,3 +327,162 @@ func TestListenerAcceptAfterClose(t *testing.T) {
}
}
}
// Close must release blocked Accept calls and let Wait report a clean shutdown.
func TestServiceClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
if err := a.Close(); err != nil {
t.Fatal(err)
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after Close")
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after Close")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
if err := a.Close(); err != nil {
t.Fatalf("second close: %v", err)
}
if _, err := a.Listen("tcp", ":1235"); !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Listen after Close, got %v", err)
}
}
// If nebula stops without Close being called, the service must still release
// Accept and tear down the netstack.
func TestServiceNebulaStopped(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
a.control.Stop()
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after nebula stopped")
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after nebula stopped")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
}
// Dials racing Close must not hold Wait until their connect times out.
func TestServiceDialDuringClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
stop := make(chan struct{})
var eg errgroup.Group
for range 8 {
eg.Go(func() error {
for {
select {
case <-stop:
return nil
default:
}
// 10.0.0.2 has no tunnel, so each dial sits in SYN-SENT until aborted
if c, err := a.Dial("tcp", "10.0.0.2:80"); err == nil {
_ = c.Close()
}
}
})
}
time.Sleep(50 * time.Millisecond)
if err := a.Close(); err != nil {
t.Fatal(err)
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait blocked behind a dial that raced Close")
}
close(stop)
dialersDone := make(chan error, 1)
go func() { dialersDone <- eg.Wait() }()
select {
case <-dialersDone:
case <-time.After(5 * time.Second):
t.Fatal("Dial still blocked after Wait")
}
}
-743
View File
@@ -1,59 +1,17 @@
package nebula
import (
"bytes"
"encoding/json"
"errors"
"flag"
"fmt"
"log/slog"
"maps"
"net"
"net/netip"
"os"
"slices"
"sort"
"strings"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/sshd"
)
type sshListHostMapFlags struct {
Json bool
Pretty bool
ByIndex bool
}
type sshPrintCertFlags struct {
Json bool
Pretty bool
Raw bool
}
type sshPrintTunnelFlags struct {
Pretty bool
}
type sshChangeRemoteFlags struct {
Address string
}
type sshCloseTunnelFlags struct {
LocalOnly bool
}
type sshCreateTunnelFlags struct {
Address string
}
type sshDeviceInfoFlags struct {
Json bool
Pretty bool
}
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
if c.GetBool("sshd.enabled", false) {
@@ -227,704 +185,3 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
return runner, nil
}
func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
ssh.RegisterCommand(&sshd.Command{
Name: "list-hostmap",
ShortDescription: "List all known previously connected hosts",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshListHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshListHostMap(f.hostMap, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "list-pending-hostmap",
ShortDescription: "List all handshaking hosts",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshListHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshListHostMap(f.handshakeManager, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "list-lighthouse-addrmap",
ShortDescription: "List all lighthouse map entries",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshListHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshListLighthouseMap(f.lightHouse, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "reload",
ShortDescription: "Reloads configuration from disk, same as sending HUP to the process",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshReload(l, c, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "log-level",
ShortDescription: "Gets or sets the current log level",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshLogLevel(l, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "log-format",
ShortDescription: "Gets or sets the current log format",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshLogFormat(l, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "version",
ShortDescription: "Prints the currently running version of nebula",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshVersion(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "device-info",
ShortDescription: "Prints information about the network device.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshDeviceInfoFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshDeviceInfo(f, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "print-cert",
ShortDescription: "Prints the current certificate being used or the certificate for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshPrintCertFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.Raw, "raw", false, "raw prints the PEM encoded certificate, not compatible with -json or -pretty")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshPrintCert(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "print-tunnel",
ShortDescription: "Prints json details about a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshPrintTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshPrintTunnel(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "print-relays",
ShortDescription: "Prints json details about all relay info",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshPrintTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshPrintRelays(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "change-remote",
ShortDescription: "Changes the remote address used in the tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshChangeRemoteFlags{}
fl.StringVar(&s.Address, "address", "", "The new remote address, ip:port")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshChangeRemote(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "close-tunnel",
ShortDescription: "Closes a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshCloseTunnelFlags{}
fl.BoolVar(&s.LocalOnly, "local-only", false, "Disables notifying the remote that the tunnel is shutting down")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshCloseTunnel(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "create-tunnel",
ShortDescription: "Creates a tunnel for the provided vpn address",
Help: "The lighthouses will be queried for real addresses but you can provide one as well.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshCreateTunnelFlags{}
fl.StringVar(&s.Address, "address", "", "Optionally provide a real remote address, ip:port ")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshCreateTunnel(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "query-lighthouse",
ShortDescription: "Query the lighthouses for the provided vpn address",
Help: "This command is asynchronous. Only currently known udp addresses will be printed.",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshQueryLighthouse(f, fs, a, w)
},
})
}
func sshListHostMap(hl controlHostLister, a any, w sshd.StringWriter) error {
fs, ok := a.(*sshListHostMapFlags)
if !ok {
return nil
}
var hm []ControlHostInfo
if fs.ByIndex {
hm = listHostMapIndexes(hl)
} else {
hm = listHostMapHosts(hl)
}
sort.Slice(hm, func(i, j int) bool {
return hm[i].VpnAddrs[0].Compare(hm[j].VpnAddrs[0]) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
err := js.Encode(hm)
if err != nil {
return nil
}
} else {
for _, v := range hm {
err := w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddrs, v.RemoteAddrs))
if err != nil {
return err
}
}
}
return nil
}
func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) error {
fs, ok := a.(*sshListHostMapFlags)
if !ok {
return nil
}
type lighthouseInfo struct {
VpnAddr string `json:"vpnAddr"`
Addrs *CacheMap `json:"addrs"`
}
lightHouse.RLock()
addrMap := make([]lighthouseInfo, len(lightHouse.addrMap))
x := 0
for k, v := range lightHouse.addrMap {
addrMap[x] = lighthouseInfo{
VpnAddr: k.String(),
Addrs: v.CopyCache(),
}
x++
}
lightHouse.RUnlock()
sort.Slice(addrMap, func(i, j int) bool {
return strings.Compare(addrMap[i].VpnAddr, addrMap[j].VpnAddr) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
err := js.Encode(addrMap)
if err != nil {
return nil
}
} else {
for _, v := range addrMap {
b, err := json.Marshal(v.Addrs)
if err != nil {
return err
}
err = w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddr, string(b)))
if err != nil {
return err
}
}
}
return nil
}
func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
return w.WriteLine(fmt.Sprintf("%s", ifce.version))
}
func sshQueryLighthouse(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
var cm *CacheMap
rl := ifce.lightHouse.Query(vpnAddr)
if rl != nil {
cm = rl.CopyCache()
}
return json.NewEncoder(w.GetWriter()).Encode(cm)
}
func sshCloseTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
flags, ok := fs.(*sshCloseTunnelFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
if !flags.LocalOnly {
ifce.send(
header.CloseTunnel,
0,
hostInfo.ConnectionState,
hostInfo,
[]byte{},
make([]byte, 12, 12),
make([]byte, mtu),
)
}
ifce.closeTunnel(hostInfo)
return w.WriteLine("Closed")
}
func sshCreateTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
flags, ok := fs.(*sshCreateTunnelFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return w.WriteLine(fmt.Sprintf("Tunnel already exists"))
}
hostInfo = ifce.handshakeManager.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return w.WriteLine(fmt.Sprintf("Tunnel already handshaking"))
}
var addr netip.AddrPort
if flags.Address != "" {
addr, err = netip.ParseAddrPort(flags.Address)
if err != nil {
return w.WriteLine("Address could not be parsed")
}
}
// Set the remote from the callback, which StartHandshake runs under the handshake manager lock
// before the hostinfo reaches the handshake loop or the static-host trigger, so the address is
// in the RemoteList before the first handshake is sent.
ifce.handshakeManager.StartHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
if addr.IsValid() {
hh.hostinfo.SetRemote(addr)
}
})
return w.WriteLine("Created")
}
func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
flags, ok := fs.(*sshChangeRemoteFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
if flags.Address == "" {
return w.WriteLine("No address was provided")
}
addr, err := netip.ParseAddrPort(flags.Address)
if err != nil {
return w.WriteLine("Address could not be parsed")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
hostInfo.SetRemote(addr)
return w.WriteLine("Changed")
}
func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level
SetLevel(slog.Level)
})
if !ok {
return w.WriteLine("Log level is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
level, err := logging.ParseLevel(strings.ToLower(a[0]))
if err != nil {
return w.WriteLine(fmt.Sprintf("Unknown log level %s. Possible log levels: trace, debug, info, warn, error", a))
}
ctrl.SetLevel(level)
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
func sshLogFormat(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetFormat() string
SetFormat(string) error
})
if !ok {
return w.WriteLine("Log format is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
if err := ctrl.SetFormat(strings.ToLower(a[0])); err != nil {
return err
}
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
func sshPrintCert(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
args, ok := fs.(*sshPrintCertFlags)
if !ok {
return nil
}
cert := ifce.pki.getCertState().GetDefaultCertificate()
if len(a) > 0 {
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
cert = hostInfo.GetCert().Certificate
}
if args.Json || args.Pretty {
b, err := cert.MarshalJSON()
if err != nil {
return nil
}
if args.Pretty {
buf := new(bytes.Buffer)
err := json.Indent(buf, b, "", " ")
b = buf.Bytes()
if err != nil {
return nil
}
}
return w.WriteBytes(b)
}
if args.Raw {
b, err := cert.MarshalPEM()
if err != nil {
return nil
}
return w.WriteBytes(b)
}
return w.WriteLine(cert.String())
}
func sshPrintRelays(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
args, ok := fs.(*sshPrintTunnelFlags)
if !ok {
w.WriteLine(fmt.Sprintf("sshPrintRelays failed to convert args type"))
return nil
}
relays := map[uint32]*HostInfo{}
ifce.hostMap.Lock()
maps.Copy(relays, ifce.hostMap.Relays)
ifce.hostMap.Unlock()
type RelayFor struct {
Error error
Type string
State string
PeerAddr netip.Addr
LocalIndex uint32
RemoteIndex uint32
RelayedThrough []netip.Addr
}
type RelayOutput struct {
NebulaAddr netip.Addr
RelayForAddrs []RelayFor
}
type CmdOutput struct {
Relays []*RelayOutput
}
co := CmdOutput{}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
for k, v := range relays {
ro := RelayOutput{NebulaAddr: v.vpnAddrs[0]}
co.Relays = append(co.Relays, &ro)
relayHI := ifce.hostMap.QueryVpnAddr(v.vpnAddrs[0])
if relayHI == nil {
ro.RelayForAddrs = append(ro.RelayForAddrs, RelayFor{Error: errors.New("could not find hostinfo")})
continue
}
for _, vpnAddr := range relayHI.relayState.CopyRelayForIps() {
rf := RelayFor{Error: nil}
r, ok := relayHI.relayState.GetRelayForByAddr(vpnAddr)
if ok {
t := ""
switch r.Type {
case ForwardingType:
t = "forwarding"
case TerminalType:
t = "terminal"
default:
t = "unknown"
}
s := ""
switch r.State {
case Requested:
s = "requested"
case Established:
s = "established"
default:
s = "unknown"
}
rf.LocalIndex = r.LocalIndex
rf.RemoteIndex = r.RemoteIndex
rf.PeerAddr = r.PeerAddr
rf.Type = t
rf.State = s
if rf.LocalIndex != k {
rf.Error = fmt.Errorf("hostmap LocalIndex '%v' does not match RelayState LocalIndex", k)
}
}
relayedHI := ifce.hostMap.QueryVpnAddr(vpnAddr)
if relayedHI != nil {
rf.RelayedThrough = append(rf.RelayedThrough, relayedHI.relayState.CopyRelayIps()...)
}
ro.RelayForAddrs = append(ro.RelayForAddrs, rf)
}
}
err := enc.Encode(co)
if err != nil {
return err
}
return nil
}
func sshPrintTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
args, ok := fs.(*sshPrintTunnelFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.GetPreferredRanges()))
}
func sshDeviceInfo(ifce *Interface, fs any, w sshd.StringWriter) error {
data := struct {
Name string `json:"name"`
Cidr []netip.Prefix `json:"cidr"`
}{
Name: ifce.inside.Name(),
Cidr: make([]netip.Prefix, len(ifce.inside.Networks())),
}
copy(data.Cidr, ifce.inside.Networks())
flags, ok := fs.(*sshDeviceInfoFlags)
if !ok {
return fmt.Errorf("internal error: expected flags to be sshDeviceInfoFlags but was %+v", fs)
}
if flags.Json || flags.Pretty {
js := json.NewEncoder(w.GetWriter())
if flags.Pretty {
js.SetIndent("", " ")
}
return js.Encode(data)
} else {
return w.WriteLine(fmt.Sprintf("name=%v cidr=%v", data.Name, data.Cidr))
}
}
// sshReload reloads the config and writes the result to the client. It reloads even if the client has gone away
func sshReload(l *slog.Logger, c *config.C, w sshd.StringWriter) error {
writeErr := w.WriteLine("Reloading config")
res := c.ReloadConfig()
res.Log(l)
if err := w.WriteLine(res.String()); writeErr == nil {
writeErr = err
}
return writeErr
}
+57
View File
@@ -0,0 +1,57 @@
package nebula
import (
"bytes"
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
)
// TestSSHCommandSurface is a characterization test for the ssh command surface.
// It pins the registered command set, their descriptions, help text and flag definitions
// so that moving the registry to a transport neutral package is provably a relocation
// rather than a rewrite. Regenerate with `go test . -run TestSSHCommandSurface -update`.
func TestSSHCommandSurface(t *testing.T) {
got := renderCommandSurface(registeredCommands(t).All())
test.Golden(t, filepath.Join("testdata", "ssh_commands.golden"), got)
}
// registeredCommands returns a registry holding every command nebula serves. The
// Interface is never dereferenced: attachCommands only closes over it.
func registeredCommands(t *testing.T) *commands.Registry {
t.Helper()
l := test.NewLogger()
reg := commands.New()
attachCommands(l, config.NewC(l), reg, &Interface{})
return reg
}
func renderCommandSurface(cmds []*commands.Command) string {
lines := make([]string, 0, len(cmds))
for _, cmd := range cmds {
b := &bytes.Buffer{}
fmt.Fprintf(b, "%s\n", cmd.Name)
fmt.Fprintf(b, " short: %s\n", cmd.ShortDescription)
fmt.Fprintf(b, " help: %s\n", cmd.Help)
if cmd.Flags != nil {
fl, _ := cmd.Flags()
if fl != nil {
fb := &bytes.Buffer{}
fl.SetOutput(fb)
fl.PrintDefaults()
fmt.Fprintf(b, " flags:\n%s", fb.String())
}
}
lines = append(lines, b.String())
}
// In the order All gives, so the golden pins display order too.
return strings.Join(lines, "")
}
+20 -9
View File
@@ -7,6 +7,7 @@ import (
"path/filepath"
"testing"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
@@ -26,7 +27,7 @@ func (w *lineWriter) Write(s string) error { return w.WriteLine(s) }
func (w *lineWriter) WriteBytes(b []byte) error { return w.WriteLine(string(b)) }
func (w *lineWriter) GetWriter() io.Writer { return io.Discard }
func TestSSHReloadReportsTheResult(t *testing.T) {
func TestReloadCommandReportsTheResult(t *testing.T) {
dir := t.TempDir()
write := func(body string) {
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(body), 0o600))
@@ -43,32 +44,42 @@ func TestSSHReloadReportsTheResult(t *testing.T) {
}
})
reload := func() []string {
// reload returns what the client was told. A reload that failed is a commands.Failure
// to the surface, exit 1 over ssh, with the result line as its message; one that
// merely needs a restart is not.
reload := func(failed bool) []string {
w := &lineWriter{}
require.NoError(t, sshReload(l, c, w))
err := reloadCommand(l, c, w)
if failed {
var f *commands.Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, w.lines[len(w.lines)-1], f.Msg)
} else {
require.NoError(t, err)
}
return w.lines
}
write("a: 2\nb: 1")
assert.Equal(t, []string{"Reloading config", "Config reload complete"}, reload())
assert.Equal(t, []string{"Reloading config", "Config reload complete"}, reload(false))
fail = true
write("a: 3\nb: 1")
assert.Equal(t, []string{"Reloading config", "Config reload completed with errors, review the logs"}, reload())
assert.Equal(t, []string{"Reloading config", "Config reload completed with errors, review the logs"}, reload(true))
fail = false
write("a: 3\nb: 2")
assert.Equal(t, []string{"Reloading config", "Config reload was completed but requires a restart to fully apply, keys [b]"}, reload())
assert.Equal(t, []string{"Reloading config", "Config reload was completed but requires a restart to fully apply, keys [b]"}, reload(false))
write("a: [")
lines := reload()
lines := reload(true)
require.Len(t, lines, 2)
assert.Contains(t, lines[1], "Config reload failed, the new config did not load")
assert.Contains(t, lines[1], dir, "names the config that did not load")
}
// A client that went away still gets the reload it asked for
func TestSSHReloadReloadsWhenTheClientIsGone(t *testing.T) {
func TestReloadCommandReloadsWhenTheClientIsGone(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte("a: 1"), 0o600))
l := test.NewLogger()
@@ -78,6 +89,6 @@ func TestSSHReloadReloadsWhenTheClientIsGone(t *testing.T) {
c.RegisterReloadCallback(func(*config.C, *config.Reload) { reloaded = true })
gone := errors.New("client is gone")
require.ErrorIs(t, sshReload(l, c, &lineWriter{err: gone}), gone)
require.ErrorIs(t, reloadCommand(l, c, &lineWriter{err: gone}), gone)
assert.True(t, reloaded)
}
-161
View File
@@ -1,161 +0,0 @@
package sshd
import (
"errors"
"flag"
"fmt"
"sort"
"strings"
"github.com/armon/go-radix"
)
// CommandFlags is a function called before help or command execution to parse command line flags
// It should return a flag.FlagSet instance and a pointer to the struct that will contain parsed flags
type CommandFlags func() (*flag.FlagSet, any)
// CommandCallback is the function called when your command should execute.
// fs will be a a pointer to the struct provided by Command.Flags callback, if there was one. -h and -help are reserved
// and handled automatically for you.
// a will be any unconsumed arguments, if no Command.Flags was available this will be all the flags passed in.
// w is the writer to use when sending messages back to the client.
// If an error is returned by the callback it is logged locally, the callback should handle messaging errors to the user
// where appropriate
type CommandCallback func(fs any, a []string, w StringWriter) error
type Command struct {
Name string
ShortDescription string
Help string
Flags CommandFlags
Callback CommandCallback
}
func execCommand(c *Command, args []string, w StringWriter) error {
var (
fl *flag.FlagSet
fs any
)
if c.Flags != nil {
fl, fs = c.Flags()
if fl != nil {
// SetOutput() here in case fl.Parse dumps usage.
fl.SetOutput(w.GetWriter())
err := fl.Parse(args)
if err != nil {
// fl.Parse has dumped error information to the user via the w writer.
return err
}
args = fl.Args()
}
}
return c.Callback(fs, args, w)
}
func dumpCommands(c *radix.Tree, w StringWriter) {
err := w.WriteLine("Available commands:")
if err != nil {
return
}
cmds := make([]string, 0)
for _, l := range allCommands(c) {
cmds = append(cmds, fmt.Sprintf("%s - %s", l.Name, l.ShortDescription))
}
sort.Strings(cmds)
_ = w.Write(strings.Join(cmds, "\n") + "\n\n")
}
func lookupCommand(c *radix.Tree, sCmd string) (*Command, error) {
cmd, ok := c.Get(sCmd)
if !ok {
return nil, nil
}
command, ok := cmd.(*Command)
if !ok {
return nil, errors.New("failed to cast command")
}
return command, nil
}
func matchCommand(c *radix.Tree, cmd string) []string {
cmds := make([]string, 0)
c.WalkPrefix(cmd, func(found string, v any) bool {
cmds = append(cmds, found)
return false
})
sort.Strings(cmds)
return cmds
}
func allCommands(c *radix.Tree) []*Command {
cmds := make([]*Command, 0)
c.WalkPrefix("", func(found string, v any) bool {
cmd, ok := v.(*Command)
if ok {
cmds = append(cmds, cmd)
}
return false
})
return cmds
}
func helpCallback(commands *radix.Tree, a []string, w StringWriter) (err error) {
// Just typed help
if len(a) == 0 {
dumpCommands(commands, w)
return nil
}
// We are printing a specific commands help text
cmd, err := lookupCommand(commands, a[0])
if err != nil {
return
}
if cmd != nil {
err = w.WriteLine(fmt.Sprintf("%s - %s", cmd.Name, cmd.ShortDescription))
if err != nil {
return err
}
if cmd.Help != "" {
err = w.WriteLine(fmt.Sprintf(" %s", cmd.Help))
if err != nil {
return err
}
}
if cmd.Flags != nil {
fs, _ := cmd.Flags()
if fs != nil {
fs.SetOutput(w.GetWriter())
fs.PrintDefaults()
}
}
return nil
}
err = w.WriteLine("Command not available " + a[0])
if err != nil {
return err
}
return nil
}
func checkHelpArgs(args []string) bool {
for _, a := range args {
if a == "-h" || a == "-help" {
return true
}
}
return false
}
+9 -23
View File
@@ -10,7 +10,7 @@ import (
"sync"
"sync/atomic"
"github.com/armon/go-radix"
"github.com/slackhq/nebula/commands"
"golang.org/x/crypto/ssh"
)
@@ -29,10 +29,9 @@ type SSHServer struct {
trustedKeys map[string]map[string]bool
trustedCAs []ssh.PublicKey
// List of available commands
helpCommand *Command
commands *radix.Tree
listener atomic.Pointer[net.Listener]
// Available commands.
commands *commands.Registry
listener atomic.Pointer[net.Listener]
// runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned
runs atomic.Int32
@@ -44,14 +43,14 @@ type SSHServer struct {
ctx context.Context
}
// NewSSHServer creates a new ssh server rigged with default commands and prepares to listen.
// The ssh server's context is parented off the supplied ctx so cancelling it
// (e.g. on Control.Stop) tears down active sessions and closes the listener.
func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
// NewSSHServer creates a new ssh server serving the supplied command registry and
// prepares to listen. The ssh server's context is parented off the supplied ctx so
// cancelling it (e.g. on Control.Stop) tears down active sessions and closes the listener.
func NewSSHServer(ctx context.Context, l *slog.Logger, reg *commands.Registry) (*SSHServer, error) {
s := &SSHServer{
trustedKeys: make(map[string]map[string]bool),
l: l,
commands: radix.New(),
commands: reg,
ctx: ctx,
}
@@ -97,14 +96,6 @@ func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
s.authenticate = cc.Authenticate
s.config.Store(s.newConfig())
s.RegisterCommand(&Command{
Name: "help",
ShortDescription: "prints available commands or help <command> for specific usage info",
Callback: func(a any, args []string, w StringWriter) error {
return helpCallback(s.commands, args, w)
},
})
return s, nil
}
@@ -189,11 +180,6 @@ func (s *SSHServer) AddAuthorizedKey(user, pubKey string) error {
return nil
}
// RegisterCommand adds a command that can be run by a user, by default only `help` is available
func (s *SSHServer) RegisterCommand(c *Command) {
s.commands.Insert(c.Name, c)
}
// Run begins listening and accepting connections. Each invocation derives a fresh per-Run context
// from the constructor-supplied ctx so a Stop+Run sequence (used by config reload) starts clean
// rather than carrying a permanently-cancelled context across runs.
+37 -46
View File
@@ -1,13 +1,11 @@
package sshd
import (
"fmt"
"errors"
"log/slog"
"sort"
"strings"
"github.com/anmitsu/go-shlex"
"github.com/armon/go-radix"
"github.com/slackhq/nebula/commands"
"golang.org/x/crypto/ssh"
"golang.org/x/term"
)
@@ -16,31 +14,37 @@ type session struct {
l *slog.Logger
c *ssh.ServerConn
term *term.Terminal
commands *radix.Tree
commands *commands.Registry
cancel func()
}
func NewSession(commands *radix.Tree, conn *ssh.ServerConn, chans <-chan ssh.NewChannel, cancel func(), l *slog.Logger) *session {
func NewSession(reg *commands.Registry, conn *ssh.ServerConn, chans <-chan ssh.NewChannel, cancel func(), l *slog.Logger) *session {
s := &session{
commands: radix.NewFromMap(commands.ToMap()),
commands: reg.Clone(),
l: l,
c: conn,
cancel: cancel,
}
s.commands.Insert("logout", &Command{
Name: "logout",
ShortDescription: "Ends the current session",
Callback: func(a any, args []string, w StringWriter) error {
s.Close()
return nil
},
})
s.commands.Register(LogoutCommand(s.Close))
go s.handleChannels(chans)
return s
}
// LogoutCommand is the logout command a session registers on its clone of the registry,
// with end as what it does. Exported so a test can serve what a session serves.
func LogoutCommand(end func()) *commands.Command {
return &commands.Command{
Name: commands.Logout,
ShortDescription: "Ends the current session",
Callback: func(fs any, a []string, w commands.StringWriter) error {
end()
return nil
},
}
}
func (s *session) handleChannels(chans <-chan ssh.NewChannel) {
defer s.Close()
for newChannel := range chans {
@@ -87,9 +91,13 @@ func (s *session) handleRequests(in <-chan *ssh.Request, channel ssh.Channel) {
}
req.Reply(true, nil)
s.dispatchCommand(payload.Value, &stringWriter{channel})
err := s.dispatchCommand(payload.Value, commands.NewStringWriter(channel))
// A command that failed exits 1, so `ssh host cmd` callers can tell.
status := struct{ Status uint32 }{uint32(0)}
if err != nil {
status.Status = 1
}
channel.SendRequest("exit-status", false, ssh.Marshal(status))
channel.Close()
return
@@ -111,12 +119,11 @@ func (s *session) createTerm(channel ssh.Channel) *term.Terminal {
term.AutoCompleteCallback = func(line string, pos int, key rune) (newLine string, newPos int, ok bool) {
// key 9 is tab
if key == 9 {
cmds := matchCommand(s.commands, line)
cmds := s.commands.Match(line)
if len(cmds) == 1 {
return cmds[0] + " ", len(cmds[0]) + 1, true
}
sort.Strings(cmds)
term.Write([]byte(strings.Join(cmds, "\n") + "\n\n"))
}
@@ -128,7 +135,7 @@ func (s *session) createTerm(channel ssh.Channel) *term.Terminal {
}
func (s *session) handleInput() {
w := &stringWriter{w: s.term}
w := commands.NewStringWriter(s.term)
for {
line, err := s.term.ReadLine()
if err != nil {
@@ -139,36 +146,20 @@ func (s *session) handleInput() {
}
}
func (s *session) dispatchCommand(line string, w StringWriter) {
args, err := shlex.Split(line, true)
if err != nil {
return
// dispatchCommand runs line and returns the command's error, which it has logged unless
// the command already told the user everything: the log line is for the operator reading
// the daemon's log, who learns nothing from a failure the user caused.
func (s *session) dispatchCommand(line string, w commands.StringWriter) error {
err := s.commands.Dispatch(line, w)
if err == nil {
return nil
}
if len(args) == 0 {
dumpCommands(s.commands, w)
return
var f *commands.Failure
if !errors.As(err, &f) {
s.l.Debug("Command returned an error", "line", line, "error", err)
}
c, err := lookupCommand(s.commands, args[0])
if err != nil {
return
}
if c == nil {
err := w.WriteLine(fmt.Sprintf("did not understand: %s", line))
_ = err
dumpCommands(s.commands, w)
return
}
if checkHelpArgs(args) {
s.dispatchCommand(fmt.Sprintf("%s %s", "help", c.Name), w)
return
}
_ = execCommand(c, args[1:], w)
return err
}
func (s *session) Close() {
+42 -1
View File
@@ -6,13 +6,16 @@ import (
"crypto/rand"
"crypto/rsa"
"encoding/pem"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
@@ -23,6 +26,7 @@ import (
type sshdReloadHarness struct {
c *config.C
reg *commands.Registry
server *sshd.SSHServer
keyFile string
signer ssh.Signer
@@ -44,7 +48,8 @@ func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
h.c = config.NewC(l)
require.NoError(t, h.c.LoadString(h.config()))
h.server, err = sshd.NewSSHServer(t.Context(), l)
h.reg = commands.New()
h.server, err = sshd.NewSSHServer(t.Context(), l, h.reg)
require.NoError(t, err)
wireSSHReload(l, h.server, h.c)
run, err := configSSH(l, h.server, h.c, nil)
@@ -238,3 +243,39 @@ func TestSSHDReload_SkippedEntriesFail(t *testing.T) {
})
}
}
// TestSSHExecExitStatusReflectsTheCommand: an exec request exits 1 when the command
// returned an error and 0 otherwise, with the command's output either way.
func TestSSHExecExitStatusReflectsTheCommand(t *testing.T) {
h := newSSHDReloadHarness(t)
h.reg.Register(&commands.Command{
Name: "fails",
Callback: func(fs any, a []string, w commands.StringWriter) error {
_ = w.WriteLine("nope")
return errors.New("failed")
},
})
client, _ := h.connect(t)
sess, err := client.NewSession()
require.NoError(t, err)
out, err := sess.CombinedOutput("fails")
var exit *ssh.ExitError
require.ErrorAs(t, err, &exit)
assert.Equal(t, 1, exit.ExitStatus())
assert.Equal(t, "nope\n", string(out))
sess, err = client.NewSession()
require.NoError(t, err)
out, err = sess.CombinedOutput("help fails")
require.NoError(t, err)
assert.Equal(t, "fails - \n", string(out))
// A word that is not a command is a failure too, with the command list as before.
sess, err = client.NewSession()
require.NoError(t, err)
out, err = sess.CombinedOutput("nope")
require.ErrorAs(t, err, &exit)
assert.Equal(t, 1, exit.ExitStatus())
assert.True(t, strings.HasPrefix(string(out), "did not understand: nope\nAvailable commands:\n"), string(out))
}
+29
View File
@@ -0,0 +1,29 @@
package test
import (
"flag"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Update is the -update flag: Golden rewrites its file instead of comparing against it.
var Update = flag.Bool("update", false, "rewrite golden files instead of comparing against them")
// Golden compares got with the file at path, or rewrites the file when -update is set.
func Golden(t *testing.T, path, got string) {
t.Helper()
if *Update {
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755))
require.NoError(t, os.WriteFile(path, []byte(got), 0644))
return
}
want, err := os.ReadFile(path)
require.NoError(t, err, "missing golden file, run with -update")
assert.Equal(t, string(want), got)
}
+94
View File
@@ -0,0 +1,94 @@
change-remote
short: Changes the remote address used in the tunnel for the provided vpn addr
help:
flags:
-address string
The new remote address, ip:port
close-tunnel
short: Closes a tunnel for the provided vpn addr
help:
flags:
-local-only
Disables notifying the remote that the tunnel is shutting down
create-tunnel
short: Creates a tunnel for the provided vpn address
help: The lighthouses will be queried for real addresses but you can provide one as well.
flags:
-address string
Optionally provide a real remote address, ip:port
device-info
short: Prints information about the network device.
help:
flags:
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
help
short: prints available commands or help <command> for specific usage info
help:
list-hostmap
short: List all known previously connected hosts
help:
flags:
-by-index
gets all hosts in the hostmap from the index table
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
list-lighthouse-addrmap
short: List all lighthouse map entries
help:
flags:
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
list-pending-hostmap
short: List all handshaking hosts
help:
flags:
-by-index
gets all hosts in the hostmap from the index table
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
log-format
short: Gets or sets the current log format
help:
log-level
short: Gets or sets the current log level
help:
print-cert
short: Prints the current certificate being used or the certificate for the provided vpn addr
help:
flags:
-json
outputs as json
-pretty
pretty prints json, assumes -json
-raw
raw prints the PEM encoded certificate, not compatible with -json or -pretty
print-relays
short: Prints json details about all relay info
help:
flags:
-pretty
pretty prints json
print-tunnel
short: Prints json details about a tunnel for the provided vpn addr
help:
flags:
-pretty
pretty prints json
query-lighthouse
short: Query the lighthouses for the provided vpn address
help: This command is asynchronous. Only currently known udp addresses will be printed.
reload
short: Reloads configuration from disk, same as sending HUP to the process
help:
version
short: Prints the currently running version of nebula
help:
+13 -31
View File
@@ -246,18 +246,6 @@ func getFrom(names [][]byte, i int, isV4 bool) netip.AddrPort {
}
func (u *StdConn) ListenOut(r EncReader, flush func()) error {
return u.listenOut(r, flush, nil)
}
// ListenOutHandoff is ListenOut for a reader that keeps the packets: after each batch, handoff takes the batch's
// buffers and returns replacements, of the same count and at least the same size, to receive the next one into.
func (u *StdConn) ListenOutHandoff(r EncReader, handoff func(bufs [][]byte) [][]byte) error {
return u.listenOut(r, nil, handoff)
}
// listenOut is the receive loop. After each batch it calls flush, or with handoff set, hands the batch's buffers
// over and points the recvmmsg iovecs at the ones it gets back.
func (u *StdConn) listenOut(r EncReader, flush func(), handoff func(bufs [][]byte) [][]byte) error {
bufSize := MTU
cmsgSpace := 0
if u.groSupported {
@@ -267,6 +255,10 @@ func (u *StdConn) listenOut(r EncReader, flush func(), handoff func(bufs [][]byt
msgs, buffers, names, _ := prepareRawMessages(u.batch, bufSize, cmsgSpace)
for {
// A shut down socket still queues and returns datagrams, a busy reader would never see an empty read
if u.closed.Load() {
return net.ErrClosed
}
if cmsgSpace > 0 {
for i := range msgs {
setMsgControllen(&msgs[i].Hdr, cmsgSpace)
@@ -295,21 +287,7 @@ func (u *StdConn) listenOut(r EncReader, flush func(), handoff func(bufs [][]byt
deliverSegments(r, from, payload, segSize)
}
if handoff == nil {
flush()
continue
}
next := handoff(buffers)
if len(next) != len(buffers) {
panic(fmt.Sprintf("udp: handoff returned %d buffers, need %d", len(next), len(buffers)))
}
for i := range next {
if len(next[i]) < bufSize {
panic(fmt.Sprintf("udp: handoff buffer is %d bytes, need %d", len(next[i]), bufSize))
}
msgs[i].Hdr.Iov.Base = &next[i][0]
}
buffers = next
flush()
}
}
@@ -472,12 +450,16 @@ func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
return nil
}
func (u *StdConn) Close() error {
// Shutdown wakes the reader without releasing the fd, Close only once it has exited
func (u *StdConn) Shutdown() error {
u.closed.Store(true)
// Wake the reader parked in recvmmsg. shutdown(2) on an unconnected socket
// returns ENOTCONN but still wakes it, so ignore the error.
// The reader then sees closed and stops touching the fd, making the Close below safe.
// shutdown(2) on an unconnected socket returns ENOTCONN but still wakes the reader, so ignore the error
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
return nil
}
func (u *StdConn) Close() error {
_ = u.Shutdown()
return unix.Close(u.sysFd)
}
+74 -5
View File
@@ -76,7 +76,7 @@ func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
}
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
// reader must tear down cleanly on Close() regardless of what the socket has
// reader must tear down cleanly on Shutdown() regardless of what the socket has
// carried. The three cases the report called out:
//
// no traffic ever -> works (shutdown wakes recvmmsg with n==0)
@@ -162,11 +162,12 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
// empty queue with the socket already having received a packet.
time.Sleep(500 * time.Millisecond)
// Tear down the way Interface.Close does, the fd stays open until the reader is gone
start := time.Now()
if err := sc.Close(); err != nil {
t.Fatalf("Close: %v", err)
if err := sc.Shutdown(); err != nil {
t.Fatalf("Shutdown: %v", err)
}
close(stop)
defer func() { _ = sc.Close() }()
select {
case err := <-loopDone:
@@ -179,7 +180,75 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
case <-time.After(3 * time.Second):
buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true)
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Shutdown\n%s", name, buf[:n])
}
close(stop)
<-trafficDone
}
// A shut down socket still returns queued datagrams, so a reader that is behind has to stop on its own within the
// batch it was working on
func TestListenOutShutdown_BusyReader(t *testing.T) {
const batch = 64
udpSettings := Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Multi: true, Batch: batch}
c, err := NewListener(testLogger(), udpSettings)
if err != nil {
t.Fatalf("NewListener: %v", err)
}
sc := c.(*StdConn)
defer sc.Close()
addr, err := sc.LocalAddr()
if err != nil {
t.Fatalf("LocalAddr: %v", err)
}
stop := make(chan struct{})
defer close(stop)
for range 8 {
send, err := net.Dial("udp", addr.String())
if err != nil {
t.Fatalf("dial: %v", err)
}
defer send.Close()
go func() {
b := make([]byte, 1200)
for {
select {
case <-stop:
return
default:
_, _ = send.Write(b)
}
}
}()
}
var afterShutdown atomic.Int64
loopDone := make(chan error, 1)
go func() {
loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
if sc.closed.Load() {
afterShutdown.Add(1)
}
// Slower than the senders so the queue never drains
for start := time.Now(); time.Since(start) < 200*time.Microsecond; {
}
}, func() {})
}()
time.Sleep(200 * time.Millisecond)
if err := sc.Shutdown(); err != nil {
t.Fatalf("Shutdown: %v", err)
}
select {
case err := <-loopDone:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("ListenOut returned %v, want net.ErrClosed", err)
}
case <-time.After(3 * time.Second):
t.Fatalf("ListenOut still running 3s after Shutdown, delivered %d packets since", afterShutdown.Load())
}
if n := afterShutdown.Load(); n > batch {
t.Fatalf("delivered %d packets after Shutdown, more than the one batch in flight", n)
}
}
+3 -12
View File
@@ -183,24 +183,15 @@ func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, err
}
func (u *TesterConn) ListenOut(r EncReader, flush func()) error {
// The batcher borrows plaintext decrypted in place inside p.Data
// until Flush, so the packet must stay alive across flush()
return u.listenOut(r, func(p *Packet) { flush() })
}
// ListenOutHandoff hands each packet's buffer over, the replacement goes back to the pool.
func (u *TesterConn) ListenOutHandoff(r EncReader, handoff func(bufs [][]byte) [][]byte) error {
return u.listenOut(r, func(p *Packet) { p.Data = handoff([][]byte{p.Data})[0] })
}
func (u *TesterConn) listenOut(r EncReader, finish func(*Packet)) error {
for {
select {
case <-u.done:
return os.ErrClosed
case p := <-u.RxPackets:
r(p.From, p.Data[:len(p.Data):len(p.Data)])
finish(p)
// The batcher borrows plaintext decrypted in place inside p.Data
// until Flush, so the packet must stay alive across flush()
flush()
p.Release()
}
}