Compare commits

..
68 changed files with 1284 additions and 5152 deletions

No files matched your search

-2
View File
@@ -1,2 +0,0 @@
# Goldens are compared byte for byte, so they must come out of git the way they went in.
*.golden -text
-13
View File
@@ -7,17 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Changed
- A tunnel whose remote becomes unreachable, such as an IPv6 remote after moving to a network without IPv6, now
moves to another of the peer's addresses within a round trip or two instead of waiting 20-35s to be torn down
and rehandshaked. When a send fails because there is no route to the remote, the tunnel sends a test packet to
every other address it knows for the peer and asks the lighthouse to have the peer punch its new address.
Whichever path answers carries the tunnel. With `punchy.punch` enabled, idle tunnels notice on their next
keepalive punch. An idle tunnel whose remote is unreachable and none of whose other addresses answer is now
torn down by the connection manager rather than left in place. Not yet effective for relayed traffic or on
Windows with the default RIO backend.
### Removed
- The SSH debug server's `start-cpu-profile`, `stop-cpu-profile`, `save-heap-profile`, `save-mutex-profile`, and
@@ -25,8 +14,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- Signing a v2 certificate with an empty name, a name longer than 253 bytes, or an empty group now fails.
These certificates were signed successfully but could not be loaded.
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
+1 -10
View File
@@ -389,16 +389,7 @@ func (c *certificateV2) fromTBSCertificate(t *TBSCertificate) error {
}
func (c *certificateV2) validate() error {
// Mirror the name and group rules in unmarshalDetails so signing fails instead of producing an unparseable certificate
if c.details.name == "" || len(c.details.name) > MaxNameLength {
return NewErrInvalidCertificateProperties("name must be between 1 and %d bytes", MaxNameLength)
}
for _, group := range c.details.groups {
if group == "" {
return NewErrInvalidCertificateProperties("groups may not be empty")
}
}
// Empty names are allowed
if len(c.publicKey) == 0 {
return ErrInvalidPublicKey
-45
View File
@@ -6,7 +6,6 @@ import (
"crypto/elliptic"
"crypto/rand"
"net/netip"
"strings"
"testing"
"time"
@@ -136,47 +135,3 @@ func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
assert.True(t, normie)
}
}
func TestCertificateV2_SignRejectsUnparseable(t *testing.T) {
_, priv, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
var invalid *ErrInvalidCertificateProperties
newTBS := func() TBSCertificate {
return TBSCertificate{
Version: Version2,
Name: "testing",
Networks: []netip.Prefix{mustParsePrefixUnmapped("10.1.1.1/24")},
Groups: []string{"test-group1"},
NotBefore: time.Now().Add(-time.Minute),
NotAfter: time.Now().Add(time.Minute),
PublicKey: []byte("1234567890abcedfghij1234567890ab"),
Curve: Curve_CURVE25519,
IsCA: true,
}
}
tbs := newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength)
c, err := tbs.Sign(nil, Curve_CURVE25519, priv)
require.NoError(t, err)
b, err := c.Marshal()
require.NoError(t, err)
_, err = unmarshalCertificateV2(b, nil, Curve_CURVE25519)
require.NoError(t, err)
tbs = newTBS()
tbs.Name = strings.Repeat("a", MaxNameLength+1)
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Name = ""
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
tbs = newTBS()
tbs.Groups = []string{"test-group1", ""}
_, err = tbs.Sign(nil, Curve_CURVE25519, priv)
require.ErrorAs(t, err, &invalid)
}
-710
View File
@@ -1,710 +0,0 @@
package nebula
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"log/slog"
"maps"
"net/netip"
"sort"
"strings"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/logging"
)
// The commands nebula serves. attachCommands builds the one registry every surface
// reads. See the commands package for what a Command is.
type listHostMapFlags struct {
Json bool
Pretty bool
ByIndex bool
}
// listHostMapFlagSet is the flag set list-hostmap and list-pending-hostmap share.
func listHostMapFlagSet() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := listHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table")
return fl, &s
}
type printCertFlags struct {
Json bool
Pretty bool
Raw bool
}
type printTunnelFlags struct {
Pretty bool
}
type changeRemoteFlags struct {
Address string
}
type closeTunnelFlags struct {
LocalOnly bool
}
type createTunnelFlags struct {
Address string
}
type deviceInfoFlags struct {
Json bool
Pretty bool
}
func attachCommands(l *slog.Logger, c *config.C, reg *commands.Registry, f *Interface) {
reg.Register(&commands.Command{
Name: commands.ListHostmap,
ShortDescription: "List all known previously connected hosts",
Flags: listHostMapFlagSet,
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listHostMapCommand(f.hostMap, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.ListPendingHostmap,
ShortDescription: "List all handshaking hosts",
Flags: listHostMapFlagSet,
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listHostMapCommand(f.handshakeManager, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.ListLighthouseAddrmap,
ShortDescription: "List all lighthouse map entries",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := listHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listLighthouseAddrmapCommand(f.lightHouse, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.Reload,
ShortDescription: "Reloads configuration from disk, same as sending HUP to the process",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return reloadCommand(l, c, w)
},
})
reg.Register(&commands.Command{
Name: commands.LogLevel,
ShortDescription: "Gets or sets the current log level",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return logLevelCommand(l, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.LogFormat,
ShortDescription: "Gets or sets the current log format",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return logFormatCommand(l, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.Version,
ShortDescription: "Prints the currently running version of nebula",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return versionCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.DeviceInfo,
ShortDescription: "Prints information about the network device.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := deviceInfoFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return deviceInfoCommand(f, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintCert,
ShortDescription: "Prints the current certificate being used or the certificate for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printCertFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.Raw, "raw", false, "raw prints the PEM encoded certificate, not compatible with -json or -pretty")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printCertCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintTunnel,
ShortDescription: "Prints json details about a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintRelays,
ShortDescription: "Prints json details about all relay info",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printRelaysCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.ChangeRemote,
ShortDescription: "Changes the remote address used in the tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := changeRemoteFlags{}
fl.StringVar(&s.Address, "address", "", "The new remote address, ip:port")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return changeRemoteCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.CloseTunnel,
ShortDescription: "Closes a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := closeTunnelFlags{}
fl.BoolVar(&s.LocalOnly, "local-only", false, "Disables notifying the remote that the tunnel is shutting down")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return closeTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.CreateTunnel,
ShortDescription: "Creates a tunnel for the provided vpn address",
Help: "The lighthouses will be queried for real addresses but you can provide one as well.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := createTunnelFlags{}
fl.StringVar(&s.Address, "address", "", "Optionally provide a real remote address, ip:port ")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return createTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.QueryLighthouse,
ShortDescription: "Query the lighthouses for the provided vpn address",
Help: "This command is asynchronous. Only currently known udp addresses will be printed.",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return queryLighthouseCommand(f, fs, a, w)
},
})
}
func listHostMapCommand(hl controlHostLister, a any, w commands.StringWriter) error {
fs, ok := a.(*listHostMapFlags)
if !ok {
return fmt.Errorf("internal error: expected *listHostMapFlags but was %T", a)
}
var hm []ControlHostInfo
if fs.ByIndex {
hm = listHostMapIndexes(hl)
} else {
hm = listHostMapHosts(hl)
}
sort.Slice(hm, func(i, j int) bool {
return hm[i].VpnAddrs[0].Compare(hm[j].VpnAddrs[0]) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
if err := js.Encode(hm); err != nil {
return err
}
} else {
for _, v := range hm {
err := w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddrs, v.RemoteAddrs))
if err != nil {
return err
}
}
}
return nil
}
func listLighthouseAddrmapCommand(lightHouse *LightHouse, a any, w commands.StringWriter) error {
fs, ok := a.(*listHostMapFlags)
if !ok {
return fmt.Errorf("internal error: expected *listHostMapFlags but was %T", a)
}
type lighthouseInfo struct {
VpnAddr string `json:"vpnAddr"`
Addrs *CacheMap `json:"addrs"`
}
lightHouse.RLock()
addrMap := make([]lighthouseInfo, len(lightHouse.addrMap))
x := 0
for k, v := range lightHouse.addrMap {
addrMap[x] = lighthouseInfo{
VpnAddr: k.String(),
Addrs: v.CopyCache(),
}
x++
}
lightHouse.RUnlock()
sort.Slice(addrMap, func(i, j int) bool {
return strings.Compare(addrMap[i].VpnAddr, addrMap[j].VpnAddr) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
if err := js.Encode(addrMap); err != nil {
return err
}
} else {
for _, v := range addrMap {
b, err := json.Marshal(v.Addrs)
if err != nil {
return err
}
err = w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddr, string(b)))
if err != nil {
return err
}
}
}
return nil
}
// parseVpnAddr parses the vpn address a command takes as its first operand, failing to w
// when there is none or it does not parse. noun is how the command's messages name it,
// "vpn address" or "vpn addr", which the output golden pins per command.
func parseVpnAddr(a []string, noun string, w commands.StringWriter) (netip.Addr, error) {
if len(a) == 0 {
return netip.Addr{}, commands.Fail(w, "No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return netip.Addr{}, commands.Fail(w, fmt.Sprintf("The provided %s could not be parsed: %s", noun, a[0]))
}
return vpnAddr, nil
}
func versionCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
return w.WriteLine(ifce.version)
}
func queryLighthouseCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
var cm *CacheMap
rl := ifce.lightHouse.Query(vpnAddr)
if rl != nil {
cm = rl.CopyCache()
}
return json.NewEncoder(w.GetWriter()).Encode(cm)
}
func closeTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*closeTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *closeTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
if !flags.LocalOnly {
ifce.sendCloseTunnel(hostInfo)
}
ifce.closeTunnel(hostInfo)
return w.WriteLine("Closed")
}
func createTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*createTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *createTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return commands.Fail(w, "Tunnel already exists")
}
hostInfo = ifce.handshakeManager.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return commands.Fail(w, "Tunnel already handshaking")
}
var addr netip.AddrPort
if flags.Address != "" {
addr, err = netip.ParseAddrPort(flags.Address)
if err != nil {
return commands.Fail(w, "Address could not be parsed")
}
}
// Set the remote from the callback, which StartHandshake runs under the handshake manager lock
// before the hostinfo reaches the handshake loop or the static-host trigger, so the address is
// in the RemoteList before the first handshake is sent.
ifce.handshakeManager.StartHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
if addr.IsValid() {
hh.hostinfo.SetRemote(addr)
}
})
return w.WriteLine("Created")
}
func changeRemoteCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*changeRemoteFlags)
if !ok {
return fmt.Errorf("internal error: expected *changeRemoteFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
if flags.Address == "" {
return commands.Fail(w, "No address was provided")
}
addr, err := netip.ParseAddrPort(flags.Address)
if err != nil {
return commands.Fail(w, "Address could not be parsed")
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
hostInfo.SetRemote(addr)
return w.WriteLine("Changed")
}
func logLevelCommand(l *slog.Logger, fs any, a []string, w commands.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level
SetLevel(slog.Level)
})
if !ok {
return commands.Fail(w, "Log level is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
level, err := logging.ParseLevel(strings.ToLower(a[0]))
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unknown log level %s. Possible log levels: trace, debug, info, warn, error", a))
}
ctrl.SetLevel(level)
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
func logFormatCommand(l *slog.Logger, fs any, a []string, w commands.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetFormat() string
SetFormat(string) error
})
if !ok {
return commands.Fail(w, "Log format is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
if err := ctrl.SetFormat(strings.ToLower(a[0])); err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to set log format: %s", err))
}
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
func printCertCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printCertFlags)
if !ok {
return fmt.Errorf("internal error: expected *printCertFlags but was %T", fs)
}
cert := ifce.pki.getCertState().GetDefaultCertificate()
if len(a) > 0 {
vpnAddr, err := parseVpnAddr(a, "vpn addr", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
cert = hostInfo.GetCert().Certificate
}
if args.Json || args.Pretty {
b, err := cert.MarshalJSON()
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
if args.Pretty {
buf := new(bytes.Buffer)
if err := json.Indent(buf, b, "", " "); err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
b = buf.Bytes()
}
return w.WriteBytes(b)
}
if args.Raw {
b, err := cert.MarshalPEM()
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
return w.WriteBytes(b)
}
return w.WriteLine(cert.String())
}
func printRelaysCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *printTunnelFlags but was %T", fs)
}
ifce.hostMap.RLock()
relays := maps.Clone(ifce.hostMap.Relays)
ifce.hostMap.RUnlock()
type RelayFor struct {
Error string `json:",omitempty"`
Type string
State string
PeerAddr netip.Addr
LocalIndex uint32
RemoteIndex uint32
RelayedThrough []netip.Addr
}
type RelayOutput struct {
NebulaAddr netip.Addr
RelayForAddrs []RelayFor
}
type CmdOutput struct {
Relays []*RelayOutput
}
co := CmdOutput{}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
for k, v := range relays {
ro := RelayOutput{NebulaAddr: v.vpnAddrs[0]}
co.Relays = append(co.Relays, &ro)
relayHI := ifce.hostMap.QueryVpnAddr(v.vpnAddrs[0])
if relayHI == nil {
ro.RelayForAddrs = append(ro.RelayForAddrs, RelayFor{Error: "could not find hostinfo"})
continue
}
for _, vpnAddr := range relayHI.relayState.CopyRelayForIps() {
rf := RelayFor{}
r, ok := relayHI.relayState.GetRelayForByAddr(vpnAddr)
if ok {
t := ""
switch r.Type {
case ForwardingType:
t = "forwarding"
case TerminalType:
t = "terminal"
default:
t = "unknown"
}
s := ""
switch r.State {
case Requested:
s = "requested"
case Established:
s = "established"
default:
s = "unknown"
}
rf.LocalIndex = r.LocalIndex
rf.RemoteIndex = r.RemoteIndex
rf.PeerAddr = r.PeerAddr
rf.Type = t
rf.State = s
if rf.LocalIndex != k {
rf.Error = fmt.Sprintf("hostmap LocalIndex '%v' does not match RelayState LocalIndex", k)
}
}
relayedHI := ifce.hostMap.QueryVpnAddr(vpnAddr)
if relayedHI != nil {
rf.RelayedThrough = append(rf.RelayedThrough, relayedHI.relayState.CopyRelayIps()...)
}
ro.RelayForAddrs = append(ro.RelayForAddrs, rf)
}
}
return enc.Encode(co)
}
func printTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *printTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn addr", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.GetPreferredRanges()))
}
func deviceInfoCommand(ifce *Interface, fs any, w commands.StringWriter) error {
flags, ok := fs.(*deviceInfoFlags)
if !ok {
return fmt.Errorf("internal error: expected flags to be deviceInfoFlags but was %+v", fs)
}
data := struct {
Name string `json:"name"`
Cidr []netip.Prefix `json:"cidr"`
}{
Name: ifce.inside.Name(),
Cidr: make([]netip.Prefix, len(ifce.inside.Networks())),
}
copy(data.Cidr, ifce.inside.Networks())
if flags.Json || flags.Pretty {
js := json.NewEncoder(w.GetWriter())
if flags.Pretty {
js.SetIndent("", " ")
}
return js.Encode(data)
} else {
return w.WriteLine(fmt.Sprintf("name=%v cidr=%v", data.Name, data.Cidr))
}
}
// reloadCommand reloads the config and writes the result to the client. It reloads even if the client has gone away
func reloadCommand(l *slog.Logger, c *config.C, w commands.StringWriter) error {
writeErr := w.WriteLine("Reloading config")
res := c.ReloadConfig()
res.Log(l)
if err := w.WriteLine(res.String()); writeErr == nil {
writeErr = err
}
if writeErr != nil {
return writeErr
}
if res.Err != nil || res.Failed {
return &commands.Failure{Msg: res.String()}
}
return nil
}
-83
View File
@@ -1,83 +0,0 @@
package commands
import (
"flag"
)
// CommandFlags is called before help or command execution to parse command line flags.
// It should return a flag.FlagSet instance and a pointer to the struct that will contain parsed flags
type CommandFlags func() (*flag.FlagSet, any)
// CommandCallback is called when your command should execute.
// fs will be a pointer to the struct provided by Command.Flags callback, if there was one. -h and -help are reserved
// and handled automatically for you.
// a will be any unconsumed arguments, if no Command.Flags was available this will be all the flags passed in.
// w is the writer to use when sending messages back to the client.
// An error returned by the callback is handed back to whoever dispatched the command, which reports a failure: exit
// status 1 over ssh. The error is not shown to the user. For a failure the user caused or can act on, write the
// message and return it as a *Failure, which Fail does in one step; the surface then reports the failure without
// logging what the user was already told. Any other error is logged by the surface as well.
type CommandCallback func(fs any, a []string, w StringWriter) error
type Command struct {
Name Name
ShortDescription string
Help string
Flags CommandFlags
Callback CommandCallback
}
// Failure is the error a command returns for a failure it has already explained to the user:
// the message it wrote is the whole story. A surface reports it as a failure and does not
// log it.
type Failure struct {
Msg string
}
func (f *Failure) Error() string {
return f.Msg
}
// Fail writes msg to w and returns it as a *Failure, so a command reports a failure to the
// user and to its surface in one line. If the write itself fails, that error is returned
// instead: the user was not told anything.
func Fail(w StringWriter, msg string) error {
if err := w.WriteLine(msg); err != nil {
return err
}
return &Failure{Msg: msg}
}
func execCommand(c *Command, args []string, w StringWriter) error {
var (
fl *flag.FlagSet
fs any
)
if c.Flags != nil {
fl, fs = c.Flags()
if fl != nil {
// SetOutput() here in case fl.Parse dumps usage.
fl.SetOutput(w.GetWriter())
err := fl.Parse(args)
if err != nil {
// fl.Parse has dumped error information to the user via the w writer.
return err
}
args = fl.Args()
}
}
return c.Callback(fs, args, w)
}
func checkHelpArgs(args []string) bool {
for _, a := range args {
if a == "-h" || a == "-help" {
return true
}
}
return false
}
-28
View File
@@ -1,28 +0,0 @@
package commands
// Name identifies a command. Every command nebula serves is a constant here, so a
// surface, a client and a test spell it the same way and are told at compile time when
// they do not. It is a string rather than an integer so a test can register a throwaway
// command without adding it here, and so the wire carries the name as the string it has
// always been.
type Name string
const (
ChangeRemote Name = "change-remote"
CloseTunnel Name = "close-tunnel"
CreateTunnel Name = "create-tunnel"
DeviceInfo Name = "device-info"
Help Name = "help"
ListHostmap Name = "list-hostmap"
ListLighthouseAddrmap Name = "list-lighthouse-addrmap"
ListPendingHostmap Name = "list-pending-hostmap"
LogFormat Name = "log-format"
LogLevel Name = "log-level"
Logout Name = "logout"
PrintCert Name = "print-cert"
PrintRelays Name = "print-relays"
PrintTunnel Name = "print-tunnel"
QueryLighthouse Name = "query-lighthouse"
Reload Name = "reload"
Version Name = "version"
)
-181
View File
@@ -1,181 +0,0 @@
package commands
import (
"cmp"
"fmt"
"maps"
"slices"
"strings"
"sync"
"github.com/anmitsu/go-shlex"
)
// Registry holds a set of commands. New is the only way to make one.
//
// It is safe for concurrent use. Commands may be registered while a surface is serving
// from the registry: the ssh server clones it per session, and a clone may overlap a
// Register.
type Registry struct {
mu sync.RWMutex
commands map[Name]*Command
}
// New returns an empty Registry with only `help` available.
func New() *Registry {
r := &Registry{commands: map[Name]*Command{}}
r.registerHelp()
return r
}
func (r *Registry) registerHelp() {
r.Register(&Command{
Name: Help,
ShortDescription: "prints available commands or help <command> for specific usage info",
Callback: func(a any, args []string, w StringWriter) error {
return r.help(args, w)
},
})
}
// Register adds a command that can be run by a user. A command registered under a name
// already held replaces it; see Clone for what that means for Help.
func (r *Registry) Register(c *Command) {
r.mu.Lock()
defer r.mu.Unlock()
r.commands[c.Name] = c
}
// Get returns the command registered under name, or nil if there is no such command.
func (r *Registry) Get(name Name) *Command {
r.mu.RLock()
defer r.mu.RUnlock()
return r.commands[name]
}
// Match returns the names of every command starting with prefix, sorted, as strings for
// a prompt to complete and print.
func (r *Registry) Match(prefix string) []string {
cmds := make([]string, 0)
r.mu.RLock()
for n := range r.commands {
if strings.HasPrefix(string(n), prefix) {
cmds = append(cmds, string(n))
}
}
r.mu.RUnlock()
slices.Sort(cmds)
return cmds
}
// All returns every registered command sorted by name, which is the order help lists them.
func (r *Registry) All() []*Command {
r.mu.RLock()
defer r.mu.RUnlock()
cmds := slices.Collect(maps.Values(r.commands))
slices.SortFunc(cmds, func(a, b *Command) int { return cmp.Compare(a.Name, b.Name) })
return cmds
}
// Clone returns a copy holding the same commands, with help bound to the copy: a
// command registered on the clone alone is listed and described there, and one the
// clone withholds is not. That rebinding replaces whatever was registered under Help on
// the source, built-in or not.
func (r *Registry) Clone() *Registry {
r.mu.RLock()
c := &Registry{commands: maps.Clone(r.commands)}
r.mu.RUnlock()
c.registerHelp()
return c
}
// Dispatch parses a command line and runs it, reporting problems to w. The error is for
// the caller to log, not to show; see CommandCallback.
func (r *Registry) Dispatch(line string, w StringWriter) error {
// A line shlex cannot split, an unbalanced quote.
args, err := shlex.Split(line, true)
if err != nil {
return Fail(w, "could not parse the command line: "+err.Error())
}
return r.DispatchArgs(args, w)
}
// DispatchArgs runs an already tokenized command line. The error is as for Dispatch.
func (r *Registry) DispatchArgs(args []string, w StringWriter) error {
if len(args) == 0 {
r.dumpCommands(w)
return nil
}
c := r.Get(Name(args[0]))
if c == nil {
// Only the first word was looked up, so only it is named.
msg := fmt.Sprintf("did not understand: %s", args[0])
if err := w.WriteLine(msg); err != nil {
return err
}
r.dumpCommands(w)
return &Failure{Msg: msg}
}
if checkHelpArgs(args) {
// Through the registered help command, so -h and `help <name>` are one path.
return execCommand(r.Get(Help), []string{string(c.Name)}, w)
}
return execCommand(c, args[1:], w)
}
// dumpCommands writes the list of available commands to w.
func (r *Registry) dumpCommands(w StringWriter) {
err := w.WriteLine("Available commands:")
if err != nil {
return
}
// In the order All gives, which is the display order.
cmds := make([]string, 0)
for _, l := range r.All() {
cmds = append(cmds, fmt.Sprintf("%s - %s", l.Name, l.ShortDescription))
}
_ = w.Write(strings.Join(cmds, "\n") + "\n\n")
}
// help writes the command list, or usage for the single command named in a.
func (r *Registry) help(a []string, w StringWriter) error {
// Just typed help
if len(a) == 0 {
r.dumpCommands(w)
return nil
}
// We are printing a specific commands help text
cmd := r.Get(Name(a[0]))
if cmd == nil {
return Fail(w, "Command not available "+a[0])
}
if err := w.WriteLine(fmt.Sprintf("%s - %s", cmd.Name, cmd.ShortDescription)); err != nil {
return err
}
if cmd.Help != "" {
if err := w.WriteLine(fmt.Sprintf(" %s", cmd.Help)); err != nil {
return err
}
}
if cmd.Flags != nil {
fs, _ := cmd.Flags()
if fs != nil {
fs.SetOutput(w.GetWriter())
fs.PrintDefaults()
}
}
return nil
}
-176
View File
@@ -1,176 +0,0 @@
package commands
import (
"bytes"
"errors"
"flag"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testRegistry() *Registry {
r := New()
r.Register(&Command{
Name: "served",
ShortDescription: "a command this surface serves",
Callback: func(fs any, a []string, w StringWriter) error {
return w.WriteLine("read " + strings.Join(a, ","))
},
})
r.Register(&Command{
Name: "withheld",
ShortDescription: "a command this surface withholds",
Callback: func(fs any, a []string, w StringWriter) error {
return w.WriteLine("wrote")
},
})
return r
}
// TestFlagUsageGoesToTheWriter pins that a parse failure's usage lands on the command's
// writer.
func TestFlagUsageGoesToTheWriter(t *testing.T) {
r := New()
r.Register(&Command{
Name: "emit",
ShortDescription: "emits",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
fl.Bool("json", false, "emit json")
return fl, nil
},
Callback: func(fs any, a []string, w StringWriter) error { return nil },
})
merged := &bytes.Buffer{}
require.Error(t, r.DispatchArgs([]string{"emit", "-nope"}, NewStringWriter(merged)))
assert.Contains(t, merged.String(), "flag provided but not defined")
}
// TestDispatchReportsUnparseableLine: a line shlex rejects, an unbalanced quote, is a
// failure the user is told about.
func TestDispatchReportsUnparseableLine(t *testing.T) {
out := &bytes.Buffer{}
err := testRegistry().Dispatch(`served "unterminated`, NewStringWriter(out))
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, f.Msg+"\n", out.String())
assert.Contains(t, out.String(), "could not parse the command line: ")
}
// TestFailTellsTheUserAndTheSurface: Fail writes the message and returns it as a Failure;
// when the write fails, the user heard nothing, so the write error is what comes back.
func TestFailTellsTheUserAndTheSurface(t *testing.T) {
out := &bytes.Buffer{}
err := Fail(NewStringWriter(out), "no such tunnel")
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, "no such tunnel", f.Msg)
assert.Equal(t, "no such tunnel\n", out.String())
err = Fail(NewStringWriter(failingWriter{}), "no such tunnel")
require.EqualError(t, err, "gone")
assert.NotErrorAs(t, err, &f)
}
type failingWriter struct{}
func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("gone") }
// TestAllSortsByName: registration order does not matter.
func TestAllSortsByName(t *testing.T) {
r := New()
for _, n := range []Name{"zz", Version, "aa", ChangeRemote} {
r.Register(&Command{Name: n})
}
var got []Name
for _, c := range r.All() {
got = append(got, c.Name)
}
assert.Equal(t, []Name{"aa", ChangeRemote, Help, Version, "zz"}, got)
}
// TestHelpFlagAgreesWithHelpCommand pins that -h answers through the registered help
// command, and that help on a clone describes the clone: a command registered only there
// is known to both -h and `help <name>`, and one withheld from it is known to neither.
func TestHelpFlagAgreesWithHelpCommand(t *testing.T) {
r := testRegistry()
c := r.Clone()
c.Register(&Command{
Name: Logout,
ShortDescription: "ends the session",
Callback: func(fs any, a []string, w StringWriter) error { return nil },
})
flag := &bytes.Buffer{}
c.Dispatch("logout -h", NewStringWriter(flag))
cmd := &bytes.Buffer{}
c.Dispatch("help logout", NewStringWriter(cmd))
assert.Equal(t, cmd.String(), flag.String())
assert.Equal(t, "logout - ends the session\n", flag.String())
// The source never learns of it.
src := &bytes.Buffer{}
r.Dispatch("help logout", NewStringWriter(src))
assert.Equal(t, "Command not available logout\n", src.String())
// And a command both hold is described either way.
flag.Reset()
c.Dispatch("served -h", NewStringWriter(flag))
assert.Equal(t, "served - a command this surface serves\n", flag.String())
}
// TestCloneHelpListsTheClone: the command list help prints is the clone's, so a clone
// that withholds a command does not name it, and the source still does.
func TestCloneHelpListsTheClone(t *testing.T) {
r := testRegistry()
c := New()
c.Register(r.Get("served"))
c = c.Clone()
out := &bytes.Buffer{}
c.Dispatch("help", NewStringWriter(out))
assert.Equal(t, "Available commands:\n"+
"help - prints available commands or help <command> for specific usage info\n"+
"served - a command this surface serves\n\n", out.String())
out.Reset()
c.Dispatch("help withheld", NewStringWriter(out))
assert.Equal(t, "Command not available withheld\n", out.String())
out.Reset()
r.Dispatch("help withheld", NewStringWriter(out))
assert.Equal(t, "withheld - a command this surface withholds\n", out.String())
}
// TestDispatchReturnsTheCommandsError: what the callback returns reaches the caller, so
// a surface can log it.
func TestDispatchReturnsTheCommandsError(t *testing.T) {
r := New()
r.Register(&Command{
Name: "fails",
Callback: func(fs any, a []string, w StringWriter) error {
return errors.New("boom")
},
})
out := &bytes.Buffer{}
require.EqualError(t, r.Dispatch("fails", NewStringWriter(out)), "boom")
assert.Empty(t, out.String(), "a command's own error is not shown; the command speaks for itself")
}
// TestUnknownCommandNamesTheWord: only the first word is looked up, so only it is
// reported, followed by the command list, and the line is a failure to the surface.
func TestUnknownCommandNamesTheWord(t *testing.T) {
out := &bytes.Buffer{}
err := testRegistry().Dispatch(`nope "with args"`, NewStringWriter(out))
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, "did not understand: nope", f.Msg)
assert.True(t, strings.HasPrefix(out.String(), "did not understand: nope\nAvailable commands:\n"), out.String())
}
-67
View File
@@ -1,67 +0,0 @@
package nebula
import (
"bytes"
"io"
"net/netip"
"testing"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestLogFormatRefusalIsShown: a format the logger does not know is reported to the user
// as a failure, rather than returned bare, which over ssh was silence.
func TestLogFormatRefusalIsShown(t *testing.T) {
l := logging.NewLogger(io.Discard)
out := &bytes.Buffer{}
w := commands.NewStringWriter(out)
var f *commands.Failure
require.ErrorAs(t, logFormatCommand(l, nil, []string{"bogus"}, w), &f)
assert.Equal(t, "Unable to set log format: unknown log format `bogus`. possible formats: [text json]\n", out.String())
out.Reset()
require.NoError(t, logFormatCommand(l, nil, []string{"JSON"}, w))
assert.Equal(t, "Log format is: json\n", out.String())
}
// TestPrintRelaysReportsErrorsAsText: the Error field was an error value, which encodes
// as {} and lost the message. It is the message now, and absent when there is none.
func TestPrintRelaysReportsErrorsAsText(t *testing.T) {
l := test.NewLogger()
hm := newHostMap(l)
relay := netip.MustParseAddr("10.0.0.1")
peer := netip.MustParseAddr("10.0.0.2")
// A relay host that is in Relays but not in Hosts, so its hostinfo cannot be found.
hm.Relays[1] = &HostInfo{vpnAddrs: []netip.Addr{relay}}
ifce := &Interface{hostMap: hm}
out := &bytes.Buffer{}
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.JSONEq(t, `{"Relays":[{"NebulaAddr":"10.0.0.1","RelayForAddrs":[{"Error":"could not find hostinfo","Type":"","State":"","PeerAddr":"","LocalIndex":0,"RemoteIndex":0,"RelayedThrough":null}]}]}`, out.String())
// The same relay host, found, relaying for a peer whose index matches: no Error key.
hi := hm.Relays[1]
r := &Relay{Type: ForwardingType, State: Established, PeerAddr: peer, LocalIndex: 1, RemoteIndex: 2}
hi.relayState = RelayState{
relayForByAddr: map[netip.Addr]*Relay{peer: r},
relayForByIdx: map[uint32]*Relay{r.LocalIndex: r},
}
hm.Hosts[relay] = hi
out.Reset()
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.JSONEq(t, `{"Relays":[{"NebulaAddr":"10.0.0.1","RelayForAddrs":[{"Type":"forwarding","State":"established","PeerAddr":"10.0.0.2","LocalIndex":1,"RemoteIndex":2,"RelayedThrough":null}]}]}`, out.String())
// And an index mismatch names it.
hm.Relays[7] = hi
delete(hm.Relays, 1)
out.Reset()
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.Contains(t, out.String(), `"Error":"hostmap LocalIndex '7' does not match RelayState LocalIndex"`)
}
-15
View File
@@ -86,21 +86,6 @@ func (c *C) InitialLoad() bool {
return c.oldSettings == nil
}
// Render returns keys and their values as yaml, for a subsystem to remember what it applied. Empty means they couldn't
// be rendered and should never match.
func (c *C) Render(keys ...string) string {
m := make(map[string]any, len(keys))
for _, k := range keys {
m[k] = c.Get(k)
}
b, err := yaml.Marshal(m)
if err != nil {
c.l.Debug("Failed to render config", "keys", keys, "error", err)
return ""
}
return string(b)
}
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
// k in both the old and new settings will be serialized, the result of the string comparison is returned.
// If k is an empty string the entire config is tested.
+12 -28
View File
@@ -14,7 +14,6 @@ import (
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
)
type trafficDecision int
@@ -172,7 +171,7 @@ func (cm *connectionManager) Start(ctx context.Context) {
}
func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
decision, hostinfo, primary, punch := cm.makeTrafficDecision(localIndex, now)
decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now)
switch decision {
case deleteTunnel:
@@ -197,20 +196,6 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
}
cm.resetRelayTrafficCheck(hostinfo)
if punch != nil {
cm.sendPunch(punch)
}
}
// sendPunch keeps a tunnel's NAT state alive when we are not sending on it. When we have no route to its remote any
// more, typically after moving to a network without that address family, the tunnel probes its other remotes so the
// peer roams over to us. Nothing else would notice, a tunnel we are not sending on has nothing else that could fail.
func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
remote, err := cm.punchy.SendPunch(hostinfo)
if udp.IsUnreachable(err) {
hostinfo.remoteUnreachable(cm.intf, remote)
}
}
func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
@@ -328,9 +313,7 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
}
}
// makeTrafficDecision decides what to do with a tunnel under the hostmap lock. It also returns the tunnel to punch when
// we are not sending on it, for the caller to send once the lock is released.
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo, *HostInfo) {
func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) {
// Read lock the main hostmap to order decisions based on tunnels being the primary tunnel
cm.hostMap.RLock()
defer cm.hostMap.RUnlock()
@@ -338,17 +321,17 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
hostinfo := cm.hostMap.Indexes[localIndex]
if hostinfo == nil {
cm.l.Debug("Not found in hostmap", "localIndex", localIndex)
return doNothing, nil, nil, nil
return doNothing, nil, nil
}
if cm.isInvalidCertificate(now, hostinfo) {
return closeTunnel, hostinfo, nil, nil
return closeTunnel, hostinfo, nil
}
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
return deleteTunnel, hostinfo, nil, nil
return deleteTunnel, hostinfo, nil
}
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
@@ -385,10 +368,10 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
if !outTraffic {
// Send a punch packet to keep the NAT state alive
return decision, hostinfo, primary, hostinfo
cm.punchy.SendPunch(hostinfo)
}
return decision, hostinfo, primary, nil
return decision, hostinfo, primary
}
if hostinfo.isPendingDeletion() {
@@ -397,7 +380,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
"tunnelCheck", m{"state": "dead", "method": "active"},
)
return deleteTunnel, hostinfo, nil, nil
return deleteTunnel, hostinfo, nil
}
decision := doNothing
@@ -411,13 +394,14 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
"primary", mainHostInfo,
)
return closeTunnel, hostinfo, primary, nil
return closeTunnel, hostinfo, primary
}
// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
// Just maintain NAT state if configured to do so.
cm.punchy.SendPunch(hostinfo)
cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
return doNothing, nil, nil, hostinfo
return doNothing, nil, nil
}
// We aren't receiving traffic but we are sending it. The outbound
@@ -443,7 +427,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
hostinfo.setPendingDeletion(true)
cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
return decision, hostinfo, nil, nil
return decision, hostinfo, nil
}
func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) {
+6 -247
View File
@@ -3,18 +3,13 @@ package nebula
import (
"crypto/ed25519"
"crypto/rand"
"io"
"net/netip"
"testing"
"time"
"github.com/flynn/noise"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/overlaytest"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
@@ -271,12 +266,12 @@ func Test_NewConnectionManager_CounterLimits(t *testing.T) {
// Still below the reject limit, the tunnel stays up
nc.In(hostinfo)
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, tryRehandshake, decision)
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, deleteTunnel, decision)
}
@@ -338,14 +333,14 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.True(t, (hostinfo.state.Load()&stateIn != 0))
now := time.Now()
decision, _, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
assert.False(t, hostinfo.sentSinceCheck())
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -353,7 +348,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.False(t, (hostinfo.state.Load()&stateIn != 0))
// Do another traffic check tick, should still not be pending deletion
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
assert.Equal(t, doNothing, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -363,7 +358,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
// Finally advance beyond the inactivity timeout
decision, _, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
assert.Equal(t, closeTunnel, decision)
assert.Equal(t, now, hostinfo.lastUsed)
assert.False(t, hostinfo.isPendingDeletion())
@@ -583,239 +578,3 @@ func (d *dummyCert) MarshalJSON() ([]byte, error) {
func (d *dummyCert) Copy() cert.Certificate {
return d
}
// newPunchTestConnectionManager is a connection manager punching through conn, holding the tunnel from newPunchTarget
func newPunchTestConnectionManager(t *testing.T, conn udp.Conn) (*connectionManager, *HostInfo) {
l := test.NewLogger()
hostMap := newHostMap(l)
hostMap.preferredRanges.Store(&[]netip.Prefix{})
lh := newTestLighthouse()
ifce := &Interface{
hostMap: hostMap,
inside: &overlaytest.NoopTun{},
outside: &udp.NoopConn{},
firewall: &Firewall{},
lightHouse: lh,
pki: &PKI{},
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
l: l,
}
ifce.pki.cs.Store(&CertState{
initiatingVersion: cert.Version1,
privateKey: []byte{},
v1Cert: &dummyCert{version: cert.Version1},
})
punchy := newPunchyForTest(t, map[string]any{"punch": true}, conn)
nc := newConnectionManagerFromConfig(l, config.NewC(l), hostMap, punchy)
nc.intf = ifce
ifce.connectionManager = nc
// Tunnel sends go through conn too, though without a key from withTestKey they never get as far as writing
ifce.writers = []udp.Conn{conn}
ifce.metricTxDropped = metrics.NilCounter{}
hostinfo := newPunchTarget()
hostinfo.localIndexId = 1099
hostinfo.remoteIndexId = 9901
hostinfo.ConnectionState = &ConnectionState{myCert: &dummyCert{version: cert.Version1}}
nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
return nc, hostinfo
}
// A tunnel we are not sending on gets punched whether or not it is receiving, the punch is the only thing that
// would notice its remote has become unreachable.
func Test_makeTrafficDecision_punchesWhenNotSending(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
now := time.Now()
// Receiving but not sending
hostinfo.takeTraffic()
nc.In(hostinfo)
decision, _, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Same(t, hostinfo, punch)
// Sending and receiving, our own traffic keeps the NAT state alive
nc.Out(hostinfo)
nc.In(hostinfo)
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, tryRehandshake, decision)
assert.Nil(t, punch)
// Unused
decision, decided, _, punch := nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, doNothing, decision)
assert.Nil(t, decided, "an unused tunnel has no relay usage to reset")
assert.Same(t, hostinfo, punch)
// Sending but not receiving gets a test packet instead, which finds an unreachable remote on its own
nc.Out(hostinfo)
decision, _, _, punch = nc.makeTrafficDecision(hostinfo.localIndexId, now)
assert.Equal(t, sendTestPacket, decision)
assert.Nil(t, punch)
}
func Test_sendPunch_probesWhenRemoteUnreachable(t *testing.T) {
t.Run("unreachable", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
"the probe should ask the lighthouse to have the peer punch us")
// Within the probe interval nothing more goes out, however many sends fail
nc.sendPunch(hostinfo)
nc.sendPunch(hostinfo)
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("delivered", func(t *testing.T) {
conn := &recordingConn{}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("other failure", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: io.ErrShortWrite}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
nc.sendPunch(hostinfo)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "only a missing route says to look elsewhere")
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
}
// A packet the data path had no route for names its tunnel in its header, the peer's index for it
func Test_sendUnreachable_findsTunnelFromHeader(t *testing.T) {
head := func(st header.MessageSubType, remoteIndex uint32) []byte {
return header.Encode(make([]byte, header.Len), header.Version, header.Message, st, remoteIndex, 1)
}
t.Run("direct", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV6)
assert.NotZero(t, hostinfo.nextUnreachableProbe.Load(), "the tunnel should have probed its other remotes")
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("not its remote", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId), punchTargetV4)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load(), "a failure on a remote the tunnel is not using is stale")
})
t.Run("relayed", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
// A relay header's index is the relay session's, it must not be taken for a tunnel's even when they collide
nc.intf.sendUnreachable(head(header.MessageRelay, hostinfo.remoteIndexId), punchTargetV6)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("unknown tunnel", func(t *testing.T) {
nc, hostinfo := newPunchTestConnectionManager(t, &recordingConn{})
nc.intf.sendUnreachable(head(header.MessageNone, hostinfo.remoteIndexId+1), punchTargetV6)
assert.Zero(t, hostinfo.nextUnreachableProbe.Load())
})
}
// withTestKey gives a tunnel a real key so its sends get encrypted and written instead of returning early
func withTestKey(hostinfo *HostInfo) {
cs := noise.NewCipherSuite(noise.DH25519, noise.CipherAESGCM, noise.HashSHA256)
hostinfo.ConnectionState.eKey = noiseutil.NewCipherState(noise.UnsafeNewCipherState(cs, [32]byte{}, 0), noise.CipherAESGCM)
}
// The probe itself, end to end from a failed send: a test packet to every remote but the one we cannot reach, and a
// lighthouse query, through both places a send can fail.
func Test_remoteUnreachable_probesOtherRemotes(t *testing.T) {
t.Run("control send", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV4}, conn.wrote, "the failed send, then the probe")
assert.Equal(t, []header.MessageType{header.Test, header.Test}, conn.types)
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("data path batch", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
// Two of the tunnel's packets fail in one batch, they probe once
sb := batch.NewSendBatch(conn, 4, 4096)
for c := uint64(1); c <= 2; c++ {
pkt := header.Encode(sb.Reserve(64), header.Version, header.Message, header.MessageNone, hostinfo.remoteIndexId, c)
sb.Commit(append(pkt, make([]byte, 32)...), punchTargetV6)
}
nc.intf.flushSendBatch(sb, 0)
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6, punchTargetV4}, conn.wrote,
"the failed packets, then one probe")
assert.Equal(t, header.Test, conn.types[2])
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse))
})
t.Run("closing", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.SendMessageToHostInfo(header.CloseTunnel, 0, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a tunnel being torn down does not probe")
assert.Empty(t, drainQueries(nc.intf.lightHouse))
})
t.Run("lighthouse", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.lightHouse.amLighthouse = true
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote, "a lighthouse does not roam its clients")
})
t.Run("nothing else to probe", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
nc, hostinfo := newPunchTestConnectionManager(t, conn)
withTestKey(hostinfo)
nc.intf.reQueryWait.Store(int64(time.Minute))
// Forget the IPv4 remote, all the tunnel knows is the one it cannot reach
hostinfo.remotes = NewRemoteList(hostinfo.vpnAddrs, nil)
hostinfo.remotes.LearnRemote(hostinfo.vpnAddrs[0], punchTargetV6)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Equal(t, []netip.Addr{hostinfo.vpnAddrs[0]}, drainQueries(nc.intf.lightHouse),
"the lighthouse is the only way to learn of another remote")
// Past the probe interval, but the lighthouse is only asked again on the requery schedule
hostinfo.nextUnreachableProbe.Store(0)
nc.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, []byte{}, make([]byte, 12), make([]byte, mtu))
assert.Empty(t, drainQueries(nc.intf.lightHouse))
assert.Equal(t, []netip.AddrPort{punchTargetV6, punchTargetV6}, conn.wrote)
})
}
// drainQueries returns every lighthouse query waiting to go out
func drainQueries(lh *LightHouse) []netip.Addr {
var queries []netip.Addr
for {
select {
case addr := <-lh.queryChan:
queries = append(queries, addr)
default:
return queries
}
}
}
-9
View File
@@ -11,7 +11,6 @@ import (
"syscall"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay"
)
@@ -48,7 +47,6 @@ type Control struct {
f *Interface
l *slog.Logger
commands *commands.Registry
ctx context.Context
cancel context.CancelFunc
sshStart func()
@@ -374,13 +372,6 @@ func (c *Control) CloseAllTunnels(excludeLighthouses bool) (closed int) {
return
}
// Commands returns the registry the daemon's surfaces serve from. It is the live
// registry, not a copy: a command registered on it is served to every ssh session that
// starts afterwards. Clone it to run commands without that reach.
func (c *Control) Commands() *commands.Registry {
return c.commands
}
func (c *Control) Device() overlay.Device {
return c.f.inside
}
+1 -1
View File
@@ -148,7 +148,7 @@ func (c *fakeConn) Rebind() error { c.rebinds++; ret
func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil }
func (c *fakeConn) ListenOut(_ udp.EncReader, _ func()) error { return nil }
func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
return len(bufs), nil
}
func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {}
-12
View File
@@ -117,18 +117,6 @@ func (c *Control) SetUDPAddr(addr netip.AddrPort) {
c.f.outside.(*udp.TesterConn).SetAddr(addr)
}
// GetUDPAddrFor returns the underlay address this node sends to dst from, which differs by family for a dual stack node
func (c *Control) GetUDPAddrFor(dst netip.AddrPort) netip.AddrPort {
return c.f.outside.(*udp.TesterConn).GetAddrFor(dst)
}
// SetDualStackUDPAddrs gives this node one underlay address per family, sending from whichever matches the
// destination. An invalid AddrPort leaves the node without that family, sends to it fail with ENETUNREACH. Register
// any address the node did not start on with the router as well or nothing will route back.
func (c *Control) SetDualStackUDPAddrs(v4, v6 netip.AddrPort) {
c.f.outside.(*udp.TesterConn).SetDualStackAddrs(v4, v6)
}
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
-164
View File
@@ -1,164 +0,0 @@
//go:build e2e_testing
package e2e
import (
"bytes"
"fmt"
"path/filepath"
"regexp"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/require"
)
// volatile lists everything in command output that legitimately differs between runs.
// Each run mints fresh certificates and indexes, so these are replaced with a marker
// before comparison. Keep the list short: anything added here is something the golden
// stops guarding.
var volatile = []struct {
re *regexp.Regexp
with string
}{
{regexp.MustCompile(`[0-9a-f]{40,}`), "<hex>"}, // fingerprints, keys, signatures
{regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})`), "<time>"}, // RFC 3339
{regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)? [+-]\d{4} \w+`), "<time>"}, // time.Time.String()
{regexp.MustCompile(`(?s)(-----BEGIN [A-Z ]+-----\n).*?(\n-----END [A-Z ]+-----)`), "${1}<pem>${2}"}, // print-cert -raw
{regexp.MustCompile(`(-----BEGIN [A-Z ]+-----\\n).*?(\\n-----END [A-Z ]+-----)`), "${1}<pem>${2}"}, // the same, inside a JSON string
{regexp.MustCompile(`"(localIndex|remoteIndex|messageCounter)":\d+`), `"$1":<n>`},
{regexp.MustCompile(`(?m)^(\s*"?(?:LocalIndex|RemoteIndex|localIndex|remoteIndex|messageCounter)"?:?\s*)\d+`), "${1}<n>"}, // and the -pretty form
}
func normalise(s string) string {
for _, v := range volatile {
s = v.re.ReplaceAllString(s, v.with)
}
return s
}
// outputCase is one command line to run through the ssh dispatch path.
type outputCase struct {
name string
args []string
}
// TestCommandOutput pins the bytes every command produces, in every format, through the
// ssh dispatch path. It is the safety net for moving rendering out of the commands: a
// change here is a change someone at an ssh prompt would see. Regenerate with:
//
// go test -tags e2e_testing ./e2e -run TestCommandOutput -update
func TestCommandOutput(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(
cert.Version1, cert.Curve_CURVE25519,
time.Now(), time.Now().Add(10*time.Minute),
nil, nil, []string{},
)
control, _, _, _ := newSimpleServer(cert.Version1, ca, caKey, "output-golden", "10.222.0.1/24", nil)
control.Start()
t.Cleanup(func() { control.Stop() })
// As an ssh session does: a clone of the daemon's registry with logout added, which
// help on the clone lists and describes. There is no session to end here.
reg := control.Commands().Clone()
reg.Register(sshd.LogoutCommand(func() {}))
// run dispatches as the ssh exec path does and returns the output and the error, which
// over ssh is exit status 1.
run := func(args ...string) (string, error) {
buf := &bytes.Buffer{}
err := reg.DispatchArgs(args, commands.NewStringWriter(buf))
return buf.String(), err
}
// Seed a pending tunnel so the listing and per-address commands have something to show.
seed, err := run("create-tunnel", "10.222.0.9")
require.NoError(t, err)
require.Equal(t, "Created\n", seed, "seeding a tunnel should succeed before anything else is recorded")
cases := []outputCase{
{"version", nil},
{"device-info", nil},
{"list-hostmap", nil},
{"list-hostmap", []string{"-by-index"}},
{"list-pending-hostmap", nil},
{"list-pending-hostmap", []string{"-by-index"}},
{"list-lighthouse-addrmap", nil},
{"print-cert", nil},
{"print-cert", []string{"-raw"}},
{"print-cert", []string{"10.222.0.9"}},
{"print-cert", []string{"10.222.0.250"}},
{"print-cert", []string{"not-an-address"}},
{"print-tunnel", nil},
{"print-tunnel", []string{"10.222.0.9"}},
{"print-tunnel", []string{"10.222.0.250"}},
{"print-relays", nil},
{"query-lighthouse", nil},
{"query-lighthouse", []string{"10.222.0.9"}},
{"query-lighthouse", []string{"not-an-address"}},
// The test logger cannot change level or format, so these cases record the
// refusal each command prints on it, not a successful change.
{"log-level", nil},
{"log-level", []string{"debug"}},
{"log-level", []string{"bogus"}},
{"log-level", []string{"info"}},
{"log-format", nil},
{"log-format", []string{"json"}},
{"log-format", []string{"text"}},
{"change-remote", nil},
{"change-remote", []string{"10.222.0.9"}},
{"change-remote", []string{"-address", "10.94.0.10:4242", "10.222.0.9"}},
{"change-remote", []string{"-address", "10.94.0.10:4242", "10.222.0.250"}},
{"create-tunnel", nil},
{"create-tunnel", []string{"10.222.0.9"}},
// A fresh address, so this exercises the handshake start with a remote attached
// rather than the "already handshaking" path above.
{"create-tunnel", []string{"-address", "10.94.0.10:4242", "10.222.0.10"}},
{"create-tunnel", []string{"not-an-address"}},
{"close-tunnel", nil},
{"close-tunnel", []string{"10.222.0.250"}},
{"close-tunnel", []string{"not-an-address"}},
{"help", nil},
{"help", []string{"print-cert"}},
{"help", []string{"logout"}},
{"logout", []string{"-h"}},
{"not-a-command", []string{"10.222.0.9"}},
// reload is last so that whatever a reload does to the daemon cannot leak into an
// earlier case. In this harness it fails: the config was loaded from a string and
// has no path to re-read, which is what the recorded result says.
{"reload", nil},
}
got := &strings.Builder{}
for _, c := range cases {
for _, fmtFlags := range [][]string{nil, {"-json"}, {"-pretty"}} {
args := append([]string{c.name}, fmtFlags...)
args = append(args, c.args...)
fmt.Fprintf(got, "$ %s\n", strings.Join(args, " "))
out, err := run(args...)
out = normalise(out)
got.WriteString(out)
// Output that does not end in a newline is recorded as such, the way diff marks
// it, so the golden sees that byte too and the next prompt still starts a line.
if out != "" && !strings.HasSuffix(out, "\n") {
got.WriteString("\\ No newline at end of output\n")
}
// A failed command is recorded as the exit status an ssh caller sees, so the
// golden pins which outcomes count as failures.
if err != nil {
got.WriteString("exit status 1\n")
}
got.WriteString("\n")
}
}
test.Golden(t, filepath.Join("testdata", "commands_output.golden"), got.String())
}
-305
View File
@@ -1,305 +0,0 @@
//go:build e2e_testing
package e2e
import (
"net/netip"
"sync/atomic"
"testing"
"time"
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/e2e/router"
"github.com/slackhq/nebula/udp"
)
// TestDualStackRoamToV4Only measures how long a laptop's tunnels take to come back after it moves from a dual stack
// network, where its tunnels formed over IPv6, to one with no IPv6 service at all: the office to hotel wifi. Every
// peer still has an IPv4 address the laptop can reach, so nothing about the move should strand a tunnel.
//
// It is a measurement, not a pass/fail assertion, and it runs on the production connection manager timers.
//
// go test -tags e2e_testing -v -run TestDualStackRoamToV4Only ./e2e/
func TestDualStackRoamToV4Only(t *testing.T) {
for _, tc := range []dualStackRoamCase{
{name: "both send", meSends: true, themSends: true},
{name: "both send, rebind", meSends: true, themSends: true, rebind: true},
{name: "both send, two tunnels", meSends: true, themSends: true, simultaneous: true},
{name: "laptop sends", meSends: true},
{name: "peer sends", themSends: true},
{name: "peer sends, rebind", themSends: true, rebind: true},
{name: "peer sends, lh over v4", themSends: true, lhOverV4: true},
{name: "peer sends, punchy", themSends: true, punchy: true},
{name: "peer sends, punchy all", themSends: true, punchy: true, punchAll: true},
{name: "laptop sends, peer NAT", meSends: true, peerNAT: true},
{name: "laptop sends, peer NAT, lh v4", meSends: true, peerNAT: true, lhOverV4: true},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fwd, rev := measureDualStackRoam(t, tc)
t.Logf("RESULT %-24s laptop->peer %-9v peer->laptop %-9v", tc.name, fmtRecovery(fwd), fmtRecovery(rev))
})
}
}
// TestDualStackRoamRecovers holds the line TestDualStackRoamToV4Only measures: a tunnel whose remote became unreachable
// moves to the peer's other address instead of waiting out the connection manager to rehandshake, which takes 20s.
func TestDualStackRoamRecovers(t *testing.T) {
for _, tc := range []struct {
dualStackRoamCase
// within bounds recovery in each direction that has traffic
within time.Duration
}{
// Measures 0.4s. Under the 5s connection manager check, so it fails if the data path stops probing.
{dualStackRoamCase{name: "both send", meSends: true, themSends: true}, 2 * time.Second},
// Measures 2.6s or 3.8s, depending on whether a probe beats the peer's punch. Half the 20s it used to take.
{dualStackRoamCase{name: "laptop sends, peer NAT", meSends: true, peerNAT: true}, 10 * time.Second},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fwd, rev := measureDualStackRoam(t, tc.dualStackRoamCase)
if fwd == 0 || fwd > tc.within {
t.Errorf("laptop->peer recovered in %v, want within %v", fmtRecovery(fwd), tc.within)
}
if tc.themSends && (rev == 0 || rev > tc.within) {
t.Errorf("peer->laptop recovered in %v, want within %v", fmtRecovery(rev), tc.within)
}
})
}
}
type dualStackRoamCase struct {
name string
// meSends and themSends say who keeps traffic flowing after the move
meSends, themSends bool
// rebind fires RebindUDPServer on the move, as the darwin network change monitor would
rebind bool
// lhOverV4 forms the lighthouse tunnel over IPv4 at the office, so the move does not break it
lhOverV4 bool
// simultaneous has both sides initiate at once at the office, which can leave each side a different primary
// tunnel for the other
simultaneous bool
// punchy turns on punchy.punch everywhere, punchAll adds punchy.target_all_remotes
punchy, punchAll bool
// peerNAT puts the peer behind a NAT that, after the move, only lets in traffic from addresses the peer has sent to
peerNAT bool
}
func fmtRecovery(d time.Duration) string {
if d == 0 {
return "never"
}
return d.Round(time.Millisecond).String()
}
type dualStackNode struct {
c *nebula.Control
vpnAddr netip.Addr
v4, v6 netip.AddrPort
adv atomic.Pointer[[]netip.Addr]
}
// move puts the node on a new network, advertising only what it can be reached on there
func (n *dualStackNode) move(v4, v6 netip.AddrPort) {
n.v4, n.v6 = v4, v6
n.c.SetDualStackUDPAddrs(v4, v6)
var adv []netip.Addr
for _, a := range []netip.AddrPort{v4, v6} {
if a.IsValid() {
adv = append(adv, a.Addr())
}
}
n.adv.Store(&adv)
}
func newDualStackNode(ca cert.Certificate, caKey []byte, name, vpnNet string, v4, v6 netip.AddrPort, overrides m) *dualStackNode {
c, vpnNets, _, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, name, vpnNet, v4, overrides)
n := &dualStackNode{c: c, vpnAddr: vpnNets[0].Addr()}
n.move(v4, v6)
c.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
return *n.adv.Load()
})
return n
}
// measureDualStackRoam returns how long after the move the first packet made it laptop->peer and peer->laptop, zero
// for a direction that never recovered.
func measureDualStackRoam(t *testing.T, tc dualStackRoamCase) (time.Duration, time.Duration) {
t.Helper()
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
ap := netip.MustParseAddrPort
lhV4, lhV6 := ap("10.0.0.1:4242"), ap("[fd00::1]:4242")
lh := newDualStackNode(ca, caKey, "lh", "10.128.0.1/24", lhV4, lhV6, m{
"lighthouse": m{"am_lighthouse": true},
})
// Production timers rather than the 2s the harness defaults to, this is about how long a real laptop waits
peerCfg := m{
"lighthouse": m{
"hosts": []any{lh.vpnAddr.String()},
"interval": 10,
},
"static_host_map": m{
lh.vpnAddr.String(): []any{lhV4.String(), lhV6.String()},
},
"timers": m{
"connection_alive_interval": 5,
"pending_deletion_interval": 10,
},
"punchy": m{
"punch": tc.punchy,
"target_all_remotes": tc.punchAll,
},
}
me := newDualStackNode(ca, caKey, "me", "10.128.0.2/24", ap("10.0.0.2:4242"), ap("[fd00::2]:4242"), peerCfg)
them := newDualStackNode(ca, caKey, "them", "10.128.0.3/24", ap("10.0.0.3:4242"), ap("[fd00::3]:4242"), peerCfg)
r := router.NewR(t, lh.c, me.c, them.c)
r.CancelFlowLogs()
for _, n := range []*dualStackNode{lh, me, them} {
r.AddRoute(n.v6.Addr(), n.v6.Port(), n.c)
}
defer func() {
lh.c.Stop()
me.c.Stop()
them.c.Stop()
}()
lh.c.Start()
me.c.Start()
them.c.Start()
// At the office IPv6 wins every race, so everything forms over it. The lighthouse tunnel optionally does not,
// for a static_host_map that only lists the lighthouse's IPv4 address.
officeV6 := func(p *udp.Packet, _ *nebula.Control) router.ExitType {
lhTraffic := p.To == lhV4 || p.To == lhV6 || p.From == lhV4 || p.From == lhV6
if p.To.Addr().Is4() == (tc.lhOverV4 && lhTraffic) {
return router.KeepRouting
}
return router.Drop
}
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
// One side at a time, so both sides hold the same single tunnel, unless the case wants them racing
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("establish")))
if !tc.simultaneous {
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
}
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("establish")))
r.RouteForAllExitFuncOrTimeout(time.Second, officeV6)
if tc.simultaneous && me.c.GetHostmapIndexCount() < 3 {
// A tunnel to the lighthouse and two to the peer, or the race did not happen
t.Logf("simultaneous handshakes settled on a single tunnel this run")
}
drainTun(me.c)
drainTun(them.c)
for _, want := range []struct {
name string
c *nebula.Control
vpnAddr netip.Addr
v6 bool
}{
{"laptop->peer", me.c, them.vpnAddr, true},
{"peer->laptop", them.c, me.vpnAddr, true},
{"laptop->lh", me.c, lh.vpnAddr, !tc.lhOverV4},
} {
hi := want.c.GetHostInfoByVpnAddr(want.vpnAddr, false)
if hi == nil {
t.Fatalf("%s: no tunnel at the office", want.name)
}
if hi.CurrentRemote.Addr().Is6() != want.v6 {
t.Fatalf("%s: tunnel formed over %v, the scenario wants v6=%v", want.name, hi.CurrentRemote, want.v6)
}
}
// Hotel wifi: a new NAT'd IPv4 address and no IPv6 at all. The office addresses are gone, nothing reaches them.
officeV4, officeV6Addr := me.v4, me.v6
hotel := ap("10.0.0.99:4242")
me.move(hotel, netip.AddrPort{})
r.AddRoute(hotel.Addr(), hotel.Port(), me.c)
start := time.Now()
if tc.rebind {
me.c.RebindUDPServer()
}
var fwd, rev atomic.Int64
stop := make(chan struct{})
defer close(stop)
watchTun(them.c, stop, start, &fwd)
watchTun(me.c, stop, start, &rev)
go func() {
tick := time.NewTicker(time.Millisecond * 200)
defer tick.Stop()
for {
select {
case <-stop:
return
case <-tick.C:
if tc.meSends {
me.c.InjectTunPacket(BuildTunUDPPacket(them.vpnAddr, 80, me.vpnAddr, 80, []byte("retry")))
}
if tc.themSends {
them.c.InjectTunPacket(BuildTunUDPPacket(me.vpnAddr, 80, them.vpnAddr, 80, []byte("retry")))
}
}
}
}()
// When only one side has traffic, the other direction recovering is a bonus we still report but do not wait for
done := func() bool {
return (!tc.meSends || fwd.Load() != 0) && (!tc.themSends || rev.Load() != 0)
}
// The peer's NAT has state for the lighthouse, which its own updates keep fresh, and for wherever it sends from here
natOpen := map[netip.AddrPort]bool{lhV4: true, lhV6: true}
isPeer := func(a netip.AddrPort) bool { return a == them.v4 || a == them.v6 }
r.RouteForAllExitFuncOrTimeout(time.Second*90, func(p *udp.Packet, _ *nebula.Control) router.ExitType {
if p.To == officeV4 || p.To == officeV6Addr {
return router.Drop
}
if tc.peerNAT {
if isPeer(p.From) {
natOpen[p.To] = true
} else if isPeer(p.To) && !natOpen[p.From] {
return router.Drop
}
}
if done() {
return router.ExitNow
}
return router.KeepRouting
})
return time.Duration(fwd.Load()), time.Duration(rev.Load())
}
func drainTun(c *nebula.Control) {
for {
select {
case <-c.GetTunTxChan():
default:
return
}
}
}
// watchTun records how long after start the first packet came out of c's tun, and keeps the tun drained after
func watchTun(c *nebula.Control, stop chan struct{}, start time.Time, first *atomic.Int64) {
go func() {
for {
select {
case <-stop:
return
case <-c.GetTunTxChan():
first.CompareAndSwap(0, int64(time.Since(start)))
}
}
}()
}
+13 -14
View File
@@ -497,10 +497,11 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []
// Nope, lets push the sender along
case p := <-udpTx:
r.Lock()
c := r.getControl(p)
a := sender.GetUDPAddr()
c := r.getControl(a, p.To, p)
if c == nil {
r.Unlock()
panic("No control for udp tx " + p.To.String())
panic("No control for udp tx " + a.String())
}
fp := r.unlockedInjectFlow(sender, c, p, false)
c.InjectUDPPacket(p) // copies internally; original is ours to release
@@ -621,7 +622,8 @@ func (r *R) stopFanInWorkers() {
func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
r.Lock()
defer r.Unlock()
c := r.getControl(p)
a := from.GetUDPAddr()
c := r.getControl(a, p.To, p)
if c == nil {
panic(fmt.Sprintf("No control for udp tx %s", p.To))
}
@@ -669,7 +671,7 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
panic(err)
}
receiver := r.getControl(p)
receiver := r.getControl(sender.GetUDPAddr(), p.To, p)
if receiver == nil {
r.Unlock()
panic("Can't RouteExitFunc for host: " + p.To.String())
@@ -765,7 +767,7 @@ func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc)
r.Lock()
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(p)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
if receiver == nil {
r.Unlock()
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
@@ -874,7 +876,7 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
r.Lock()
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(p)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
if receiver == nil {
r.Unlock()
panic("Can't RouteForAllExitFunc for host: " + p.To.String())
@@ -937,7 +939,7 @@ func (r *R) FlushAll() {
})
for {
_, rx, ok := reflect.Select(sc)
x, rx, ok := reflect.Select(sc)
if !ok {
return
}
@@ -945,7 +947,7 @@ func (r *R) FlushAll() {
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(p)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
if receiver == nil {
r.Unlock()
panic("Can't FlushAll for host: " + p.To.String())
@@ -956,19 +958,16 @@ func (r *R) FlushAll() {
}
}
// getControl performs or seeds NAT translation and returns the control for p.To, p from fields may change
// getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change
// This is an internal router function, the caller must hold the lock
func (r *R) getControl(p *udp.Packet) *nebula.Control {
// Key on the address the packet actually left from, a dual stack sender has one per family
fromAddr, toAddr := p.From, p.To
func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control {
if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
p.From = newAddr
}
c, ok := r.inNat[toAddr]
if ok {
// Key the return path on the address the receiver will answer from, which is per family for a dual stack node
r.outNat[outNatKey{from: c.GetUDPAddrFor(fromAddr), to: fromAddr}] = toAddr
r.outNat[outNatKey{from: c.GetUDPAddr(), to: fromAddr}] = toAddr
return c
}
-720
View File
@@ -1,720 +0,0 @@
$ version
e2e-test
$ version -json
e2e-test
$ version -pretty
e2e-test
$ device-info
name= cidr=[10.222.0.1/24]
$ device-info -json
{"name":"","cidr":["10.222.0.1/24"]}
$ device-info -pretty
{
"name": "",
"cidr": [
"10.222.0.1/24"
]
}
$ list-hostmap
$ list-hostmap -json
[]
$ list-hostmap -pretty
[]
$ list-hostmap -by-index
$ list-hostmap -json -by-index
[]
$ list-hostmap -pretty -by-index
[]
$ list-pending-hostmap
[10.222.0.9]: []
$ list-pending-hostmap -json
[{"vpnAddrs":["10.222.0.9"],"localIndex":<n>,"remoteIndex":<n>,"remoteAddrs":[],"cert":null,"messageCounter":<n>,"currentRemote":"","currentRelaysToMe":[],"currentRelaysThroughMe":[]}]
$ list-pending-hostmap -pretty
[
{
"vpnAddrs": [
"10.222.0.9"
],
"localIndex": <n>,
"remoteIndex": <n>,
"remoteAddrs": [],
"cert": null,
"messageCounter": <n>,
"currentRemote": "",
"currentRelaysToMe": [],
"currentRelaysThroughMe": []
}
]
$ list-pending-hostmap -by-index
$ list-pending-hostmap -json -by-index
[]
$ list-pending-hostmap -pretty -by-index
[]
$ list-lighthouse-addrmap
10.222.0.9: {}
$ list-lighthouse-addrmap -json
[{"vpnAddr":"10.222.0.9","addrs":{}}]
$ list-lighthouse-addrmap -pretty
[
{
"vpnAddr": "10.222.0.9",
"addrs": {}
}
]
$ print-cert
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}
$ print-cert -json
{"details":{"curve":"CURVE25519","groups":[],"isCa":false,"issuer":"<hex>","name":"output-golden","networks":["10.222.0.1/24"],"notAfter":"<time>","notBefore":"<time>","publicKey":"<hex>","unsafeNetworks":[]},"fingerprint":"<hex>","signature":"<hex>","version":1}\ No newline at end of output
$ print-cert -pretty
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}\ No newline at end of output
$ print-cert -raw
-----BEGIN NEBULA CERTIFICATE-----
<pem>
-----END NEBULA CERTIFICATE-----
$ print-cert -json -raw
{"details":{"curve":"CURVE25519","groups":[],"isCa":false,"issuer":"<hex>","name":"output-golden","networks":["10.222.0.1/24"],"notAfter":"<time>","notBefore":"<time>","publicKey":"<hex>","unsafeNetworks":[]},"fingerprint":"<hex>","signature":"<hex>","version":1}\ No newline at end of output
$ print-cert -pretty -raw
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}\ No newline at end of output
$ print-cert 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert -json 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert -pretty 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert -json 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert -pretty 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-cert -json not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-cert -pretty not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-tunnel
No vpn address was provided
exit status 1
$ print-tunnel -json
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty
No vpn address was provided
exit status 1
$ print-tunnel 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-tunnel -json 10.222.0.9
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-tunnel 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-tunnel -json 10.222.0.250
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-relays
{"Relays":null}
$ print-relays -json
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-relays -pretty
{
"Relays": null
}
$ query-lighthouse
No vpn address was provided
exit status 1
$ query-lighthouse -json
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty
The provided vpn address could not be parsed: -pretty
exit status 1
$ query-lighthouse 10.222.0.9
{}
$ query-lighthouse -json 10.222.0.9
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty 10.222.0.9
The provided vpn address could not be parsed: -pretty
exit status 1
$ query-lighthouse not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ query-lighthouse -json not-an-address
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty not-an-address
The provided vpn address could not be parsed: -pretty
exit status 1
$ log-level
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty
Log level is not reconfigurable on this logger
exit status 1
$ log-level debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level info
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json info
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty info
Log level is not reconfigurable on this logger
exit status 1
$ log-format
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty
Log format is not reconfigurable on this logger
exit status 1
$ log-format json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty json
Log format is not reconfigurable on this logger
exit status 1
$ log-format text
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json text
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty text
Log format is not reconfigurable on this logger
exit status 1
$ change-remote
No vpn address was provided
exit status 1
$ change-remote -json
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote 10.222.0.9
No address was provided
exit status 1
$ change-remote -json 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -address 10.94.0.10:4242 10.222.0.9
Could not find tunnel for vpn address: 10.222.0.9
exit status 1
$ change-remote -json -address 10.94.0.10:4242 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty -address 10.94.0.10:4242 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -address 10.94.0.10:4242 10.222.0.250
Could not find tunnel for vpn address: 10.222.0.250
exit status 1
$ change-remote -json -address 10.94.0.10:4242 10.222.0.250
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty -address 10.94.0.10:4242 10.222.0.250
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ create-tunnel
No vpn address was provided
exit status 1
$ create-tunnel -json
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel 10.222.0.9
Tunnel already handshaking
exit status 1
$ create-tunnel -json 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -address 10.94.0.10:4242 10.222.0.10
Created
$ create-tunnel -json -address 10.94.0.10:4242 10.222.0.10
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty -address 10.94.0.10:4242 10.222.0.10
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ create-tunnel -json not-an-address
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty not-an-address
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ close-tunnel
No vpn address was provided
exit status 1
$ close-tunnel -json
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel 10.222.0.250
Could not find tunnel for vpn address: 10.222.0.250
exit status 1
$ close-tunnel -json 10.222.0.250
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty 10.222.0.250
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ close-tunnel -json not-an-address
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty not-an-address
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ help
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
$ help -json
Command not available -json
exit status 1
$ help -pretty
Command not available -pretty
exit status 1
$ help print-cert
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
-json
outputs as json
-pretty
pretty prints json, assumes -json
-raw
raw prints the PEM encoded certificate, not compatible with -json or -pretty
$ help -json print-cert
Command not available -json
exit status 1
$ help -pretty print-cert
Command not available -pretty
exit status 1
$ help logout
logout - Ends the current session
$ help -json logout
Command not available -json
exit status 1
$ help -pretty logout
Command not available -pretty
exit status 1
$ logout -h
logout - Ends the current session
$ logout -json -h
logout - Ends the current session
$ logout -pretty -h
logout - Ends the current session
$ not-a-command 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ not-a-command -json 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ not-a-command -pretty 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ reload
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
$ reload -json
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
$ reload -pretty
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
-3
View File
@@ -331,9 +331,6 @@ tun:
# `mtu`: will default to tun mtu if this option is not specified
# `metric`: will default to 0 if this option is not specified
# `install`: will default to true, controls whether this route is installed in the systems routing table.
# On macOS, a route whose destination another interface already routes, like the local network, is installed once
# that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another
# interface routes its destination and lets it go.
# This setting is reloadable.
unsafe_routes:
#- route: 172.16.1.0/24
-4
View File
@@ -66,9 +66,6 @@ type Firewall struct {
rules string
rulesVersion uint16
// source is the rendered firewall config this firewall was built from, the cert networks are kept above
source string
defaultLocalCIDRAny bool
incomingMetrics firewallMetrics
outgoingMetrics firewallMetrics
@@ -249,7 +246,6 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
return nil, err
}
fw.source = c.Render("firewall")
return fw, nil
}
+1
View File
@@ -7,6 +7,7 @@ require (
filippo.io/bigmod v0.1.0
github.com/DefinedNet/netlink v1.3.2-dn.1
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
github.com/armon/go-radix v1.0.0
github.com/flynn/noise v1.1.0
github.com/gaissmai/bart v0.29.0
github.com/gogo/protobuf v1.3.2
+2
View File
@@ -12,6 +12,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+43 -48
View File
@@ -61,17 +61,16 @@ type HandshakeManager struct {
vpnIps map[netip.Addr]*HandshakeHostInfo
indexes map[uint32]*HandshakeHostInfo
mainHostMap *HostMap
lightHouse *LightHouse
outside udp.Conn
config HandshakeConfig
OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
messageMetrics *MessageMetrics
metricInitiated metrics.Counter
metricTimedOut metrics.Counter
metricRejectedBlocklisted metrics.Counter
f *Interface
l *slog.Logger
mainHostMap *HostMap
lightHouse *LightHouse
outside udp.Conn
config HandshakeConfig
OutboundHandshakeTimer *LockingTimerWheel[*HandshakeHostInfo]
messageMetrics *MessageMetrics
metricInitiated metrics.Counter
metricTimedOut metrics.Counter
f *Interface
l *slog.Logger
// can be used to trigger outbound handshake for the given vpnIp
trigger chan netip.Addr
@@ -82,6 +81,7 @@ type HandshakeHostInfo struct {
startTime time.Time // Time that we first started trying with this handshake
ready bool // Is the handshake ready
deleted bool // Has the handshake been removed from the pending maps, deleters hold the lock
initiatingVersionOverride cert.Version // Should we use a non-default cert version for this handshake?
counter int64 // How many attempts have we made so far
lastRemotes []netip.AddrPort // Remotes that we sent to during the previous attempt
@@ -119,19 +119,18 @@ func (hh *HandshakeHostInfo) cachePacket(l *slog.Logger, t header.MessageType, s
func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager {
return &HandshakeManager{
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
indexes: map[uint32]*HandshakeHostInfo{},
mainHostMap: mainHostMap,
lightHouse: lightHouse,
outside: outside,
config: config,
trigger: make(chan netip.Addr, config.triggerBuffer),
OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
metricRejectedBlocklisted: metrics.GetOrRegisterCounter("handshake_manager.rejected.blocklisted", nil),
l: l,
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
indexes: map[uint32]*HandshakeHostInfo{},
mainHostMap: mainHostMap,
lightHouse: lightHouse,
outside: outside,
config: config,
trigger: make(chan netip.Addr, config.triggerBuffer),
OutboundHandshakeTimer: NewLockingTimerWheel[*HandshakeHostInfo](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
l: l,
}
}
@@ -144,7 +143,9 @@ func (hm *HandshakeManager) Run(ctx context.Context) {
case <-ctx.Done():
return
case vpnIP := <-hm.trigger:
hm.handleOutbound(vpnIP, true)
if hh := hm.queryVpnIp(vpnIP); hh != nil {
hm.handleOutbound(hh, true)
}
case now := <-clockSource.C:
hm.NextOutboundHandshakeTimerTick(now)
}
@@ -199,23 +200,25 @@ func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *head
func (hm *HandshakeManager) NextOutboundHandshakeTimerTick(now time.Time) {
hm.OutboundHandshakeTimer.Advance(now)
for {
vpnIp, has := hm.OutboundHandshakeTimer.Purge()
hh, has := hm.OutboundHandshakeTimer.Purge()
if !has {
break
}
hm.handleOutbound(vpnIp, false)
hm.handleOutbound(hh, false)
}
}
func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered bool) {
hh := hm.queryVpnIp(vpnIp)
if hh == nil {
return
}
func (hm *HandshakeManager) handleOutbound(hh *HandshakeHostInfo, lighthouseTriggered bool) {
hh.Lock()
defer hh.Unlock()
// Timer entries outlive their handshake. A new handshake for the same vpn addr runs its own timer chain.
if hh.deleted {
return
}
hostinfo := hh.hostinfo
vpnIp := hostinfo.vpnAddrs[0]
// If we are out of time, clean up
if hh.counter >= hm.config.retries {
fields := []any{
@@ -241,10 +244,14 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
// Increment the counter to increase our delay, linear backoff
hh.counter++
// A lighthouse triggered attempt is still in the timer wheel, adding it again would start a second retry chain
if !lighthouseTriggered {
defer hm.OutboundHandshakeTimer.Add(hh, hm.config.tryInterval*time.Duration(hh.counter))
}
// Check if we have a handshake packet to transmit yet
if !hh.ready {
if !hm.buildStage0Packet(hh) {
hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval*time.Duration(hh.counter))
return
}
}
@@ -325,11 +332,6 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
}
hm.f.relayManager.StartRelays(hm.f, vpnIp, hh, stage0)
// If a lighthouse triggered this attempt then we are still in the timer wheel and do not need to re-add
if !lighthouseTriggered {
hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval*time.Duration(hh.counter))
}
}
// GetOrHandshake will try to find a hostinfo with a fully formed tunnel or start a new handshake if one is not present
@@ -382,7 +384,7 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
}
hm.vpnIps[vpnAddr] = hh
hm.metricInitiated.Inc(1)
hm.OutboundHandshakeTimer.Add(vpnAddr, hm.config.tryInterval)
hm.OutboundHandshakeTimer.Add(hh, hm.config.tryInterval)
if cacheCb != nil {
cacheCb(hh)
@@ -534,6 +536,7 @@ func (hm *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) {
func (hm *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) {
for _, addr := range hostinfo.vpnAddrs {
if cur, ok := hm.vpnIps[addr]; ok && cur.hostinfo == hostinfo {
cur.deleted = true
delete(hm.vpnIps, addr)
}
}
@@ -719,12 +722,7 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
response, result, err := machine.ProcessPacket(nil, packet)
if err != nil {
level := slog.LevelError
if errors.Is(err, cert.ErrBlockListed) {
level = slog.LevelDebug
hm.metricRejectedBlocklisted.Inc(1)
}
f.l.Log(context.Background(), level, "Failed to process handshake packet", "from", via, "error", err)
f.l.Error("Failed to process handshake packet", "from", via, "error", err)
return
}
@@ -856,9 +854,6 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
response, result, err := machine.ProcessPacket(nil, packet)
if err != nil {
if errors.Is(err, cert.ErrBlockListed) {
hm.metricRejectedBlocklisted.Inc(1)
}
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
if machine.Failed() {
f.l.Warn("Failed to process handshake packet, abandoning",
+63 -89
View File
@@ -1,17 +1,12 @@
package nebula
import (
"log/slog"
"net/netip"
"testing"
"time"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
@@ -70,7 +65,69 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
assert.NotContains(t, blah.vpnIps, ip)
}
func testCountTimerWheelEntries(tw *LockingTimerWheel[netip.Addr]) (c int) {
// newTestHandshakeManager returns a manager whose handshakes never build a stage 0 packet (the cert has no key), so
// every attempt just bumps the counter and schedules the next one. tick advances its timer wheel by one tryInterval.
func newTestHandshakeManager() (hm *HandshakeManager, tick func()) {
l := test.NewLogger()
mainHM := newHostMap(l)
mainHM.preferredRanges.Store(&[]netip.Prefix{})
hm = NewHandshakeManager(l, mainHM, newTestLighthouse(), &udp.NoopConn{}, defaultHandshakeConfig)
hm.f = &Interface{handshakeManager: hm, pki: &PKI{}, l: l}
hm.f.pki.cs.Store(&CertState{
initiatingVersion: cert.Version1,
privateKey: []byte{},
v1Cert: &dummyCert{version: cert.Version1},
})
now := time.Now()
hm.NextOutboundHandshakeTimerTick(now)
return hm, func() {
now = now.Add(DefaultHandshakeTryInterval)
hm.NextOutboundHandshakeTimerTick(now)
}
}
func Test_HandshakeManager_RestartDropsOldTimer(t *testing.T) {
ip := netip.MustParseAddr("172.1.1.2")
hm, tick := newTestHandshakeManager()
hm.StartHandshake(ip, nil)
old := hm.queryVpnIp(ip)
tick()
tick()
require.Equal(t, int64(1), old.counter)
// Restart the handshake while the old one still has a retry queued
hm.DeleteHostInfo(old.hostinfo)
hm.StartHandshake(ip, nil)
hh := hm.queryVpnIp(ip)
require.NotSame(t, old, hh)
// An entry for timeout d fires d/tryInterval+1 ticks after it is added, so linear backoff lands attempts on ticks
// 2, 4 and 7
var counters []int64
for range 7 {
tick()
counters = append(counters, hh.counter)
}
assert.Equal(t, []int64{0, 1, 1, 2, 2, 2, 3}, counters)
// The old handshake's queued retry ended its chain instead of retrying a deleted handshake
assert.Equal(t, int64(1), old.counter)
assert.Equal(t, 1, testCountTimerWheelEntries(hm.OutboundHandshakeTimer))
}
func Test_HandshakeManager_LighthouseTriggerKeepsOneTimer(t *testing.T) {
ip := netip.MustParseAddr("172.1.1.2")
hm, _ := newTestHandshakeManager()
hm.StartHandshake(ip, nil)
hm.handleOutbound(hm.queryVpnIp(ip), true)
assert.Equal(t, 1, testCountTimerWheelEntries(hm.OutboundHandshakeTimer))
}
func testCountTimerWheelEntries(tw *LockingTimerWheel[*HandshakeHostInfo]) (c int) {
for _, i := range tw.t.wheel {
n := i.Head
for n != nil {
@@ -240,89 +297,6 @@ func TestHandleIncomingDispatch(t *testing.T) {
})
}
func TestBlocklistedHandshakeLogLevelAndCounter(t *testing.T) {
ca, _, caKey, _ := ct.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
pool := ct.NewTestCAPool(ca)
newCredential := func(name, network string) (cert.Certificate, *handshake.Credential) {
c, _, rawKey, _ := ct.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey,
name, ca.NotBefore(), ca.NotAfter(), []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
privateKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey)
require.NoError(t, err)
certBytes, err := c.MarshalForHandshakes()
require.NoError(t, err)
cipherSuite := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
return c, handshake.NewCredential(c, certBytes, privateKey, cipherSuite)
}
initiatorCert, initiatorCredential := newCredential("initiator", "10.0.0.1/24")
responderCert, responderCredential := newCredential("responder", "10.0.0.2/24")
fingerprint, err := initiatorCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(fingerprint)
initiator, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return initiatorCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return pool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 1, nil }, true, header.HandshakeIXPSK0)
require.NoError(t, err)
packet, err := initiator.Initiate(nil)
require.NoError(t, err)
handler := &capturingHandler{}
logger := slog.New(handler)
pki := &PKI{}
pki.cs.Store(&CertState{initiatingVersion: cert.Version2, v2Credential: responderCredential})
pki.caPool.Store(pool)
hm := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
hm.metricRejectedBlocklisted = metrics.NewCounter()
hm.f = &Interface{pki: pki, l: logger}
via := ViaSender{UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242")}
hm.beginHandshake(via, packet, nil)
entry := findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelDebug, entry.Level)
require.ErrorIs(t, entry.Attrs["error"].(error), cert.ErrBlockListed)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
handler.entries = nil
hm.beginHandshake(via, nil, nil)
entry = findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelError, entry.Level)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
// The initiator sees the same rejection in continueHandshake, after a
// responder has accepted its first packet and sent a valid reply.
cleanPool := ct.NewTestCAPool(ca)
responder, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return responderCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return cleanPool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 2, nil }, false, header.HandshakeIXPSK0)
require.NoError(t, err)
response, _, err := responder.ProcessPacket(nil, packet)
require.NoError(t, err)
responderFingerprint, err := responderCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(responderFingerprint)
initiatingHM := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
initiatingHM.metricRejectedBlocklisted = metrics.NewCounter()
initiatingHM.f = &Interface{l: logger}
hh := &HandshakeHostInfo{
machine: initiator,
hostinfo: &HostInfo{localIndexId: 1, vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}},
}
initiatingHM.indexes[1] = hh
via.IsRelayed = true // skip the remote allow list before certificate validation
initiatingHM.continueHandshake(via, hh, response)
assert.Equal(t, int64(1), initiatingHM.metricRejectedBlocklisted.Count())
}
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
-62
View File
@@ -264,9 +264,6 @@ type HostInfo struct {
// This is used to limit lighthouse re-queries in chatty clients
nextLHQuery atomic.Int64
// nextUnreachableProbe is the earliest we will probe our other remotes again after failing to reach the current one
nextUnreachableProbe atomic.Int64
// lastHandshakeTime records the time the remote side told us about at the stage when the handshake was completed locally
// Stage 1 packet will contain it if I am a responder, stage 2 packet if I am an initiator
// This is used to avoid an attack where a handshake packet is replayed after some time
@@ -769,65 +766,6 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
}
}
// unreachableProbeInterval bounds how often a tunnel whose remote we cannot reach probes its other remotes
const unreachableProbeInterval = time.Second
// remoteUnreachable is called when a send to our current remote failed because we have no route to it, typically
// after moving to a network without that address family. Sends a test packet to every other remote we know for this
// host: the first one that arrives roams the peer over to wherever we are now, and its reply roams us onto that path.
//
// A peer behind a NAT drops a test from an address it has never sent to, so every probe also asks the lighthouse
// about the peer. The lighthouse tells the peer where we are now, the peer punches us, and the next probe gets in.
func (i *HostInfo) remoteUnreachable(ifce *Interface, dead netip.AddrPort) {
if i.remotes == nil || i.ConnectionState == nil || i.GetRemote() != dead {
// Nothing to probe with, or we already moved on from the remote that failed
return
}
if ifce.lightHouse == nil || ifce.lightHouse.amLighthouse {
// A lighthouse hands out the address it reaches a client on, it must not go roaming its clients itself
return
}
now := time.Now().UnixNano()
next := i.nextUnreachableProbe.Load()
if now < next || !i.nextUnreachableProbe.CompareAndSwap(next, now+int64(unreachableProbeInterval)) {
return
}
nb := make([]byte, 12)
out := make([]byte, mtu)
probed := 0
i.remotes.ForEach(ifce.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) {
if addr == dead {
return
}
probed++
ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), nb, out)
})
if ifce.l.Enabled(context.Background(), slog.LevelDebug) {
i.logger(ifce.l).Debug("Remote unreachable, probing other remotes",
"udpAddr", dead,
"probed", probed,
)
}
if probed == 0 {
// With nothing else to try, the lighthouse is our only way to learn of another remote. The peer may not have
// one, so ask on the usual requery schedule rather than every probe for as long as the tunnel sits here.
if now < i.nextLHQuery.Load() {
return
}
i.nextLHQuery.Store(now + ifce.reQueryWait.Load())
}
// With other remotes to probe we ask on every probe, without backing off. A repeat re-sends the peer its punch
// notification in case the last one or its punches were lost. If the tunnel stays up on inbound traffic while
// our sends to its remote keep failing (only one direction broken), this continues once a second for as long as
// that lasts. That is a bounded cost for a rare state, and the probe that would clear it has to keep running.
ifce.lightHouse.QueryServer(i.vpnAddrs[0])
}
// Bits within HostInfo.state, everything above stateEpochShift is the epoch
const (
stateIn uint32 = 1 << iota
-5
View File
@@ -15,7 +15,6 @@ import (
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/udp"
)
func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.ParsedPacket, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
@@ -612,10 +611,6 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
"udpAddr", hr,
)
}
// A closing tunnel is about to be torn down, there is no point looking for another path for it
if t != header.CloseTunnel && udp.IsUnreachable(err) {
hostinfo.remoteUnreachable(f, hr)
}
}
} else {
// Try to send via a relay
+6 -47
View File
@@ -1,7 +1,6 @@
package nebula
import (
"bytes"
"context"
"crypto/fips140"
"errors"
@@ -496,8 +495,7 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
// flushSendBatch drains sb to the underlay and accounts for anything it could not deliver. A shortfall means
// specific destinations were undeliverable (a stale remote, a reject rule), which the backend logs per peer at
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch. Tunnels whose remote
// we no longer have a route to go looking for another path.
// debug; here it is only a counter, so one unreachable peer cannot spam a log line per batch.
func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
queued := sb.Len()
written, err := sb.Flush()
@@ -507,44 +505,6 @@ func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) {
if dropped := queued - written; dropped > 0 {
f.metricTxDropped.Inc(int64(dropped))
}
if f.lightHouse == nil || f.lightHouse.amLighthouse {
// Lighthouses never probe, see remoteUnreachable, so skip finding the tunnels
return
}
// A tunnel's packets tend to fail together, so skip straight past a repeat of the one before. The first 8 bytes
// of a header say whose packet it is, the counter after them changes with every packet.
var last batch.Failure
for i, fail := range sb.Failures() {
if i > 0 && fail.Dst == last.Dst && bytes.Equal(fail.Head[:8], last.Head[:8]) {
continue
}
last = fail
if udp.IsUnreachable(fail.Err) {
f.sendUnreachable(fail.Head[:], fail.Dst)
}
}
}
// sendUnreachable finds the tunnel a packet we had no route for was sent on, from the start of the packet, and has it
// look for another path to its peer.
func (f *Interface) sendUnreachable(head []byte, dst netip.AddrPort) {
var h header.H
if h.Parse(head) != nil {
return
}
if h.Type == header.Message && h.Subtype == header.MessageRelay {
// TODO: a relayed packet's header carries the relay session's index, not the relay tunnel's, so it does not
// lead us to the tunnel whose remote failed. The relay tunnel's own traffic finds out soon enough.
return
}
if hostinfo := f.hostMap.QueryReverseIndex(h.RemoteIndex); hostinfo != nil {
hostinfo.remoteUnreachable(f, dst)
}
}
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
@@ -577,12 +537,11 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
curCert = cs.getCertificate(cert.Version1)
}
// Compare with what the running firewall was built from, not the last config, so a firewall that failed to build
// is tried again. The cert is the one this reload brought, PKI registers its reload callback before this one
src := c.Render("firewall")
if src != "" && src == f.firewall.source && curCert != nil &&
slices.Equal(curCert.Networks(), f.firewall.assignedNetworks) &&
slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) {
// The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction.
// Check to see if that set has changed, and if so, rebuild the firewall.
certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks)
if !c.HasChanged("firewall") && !certUnsafeChanged {
f.l.Debug("No firewall config change detected")
return
}
+3 -58
View File
@@ -135,63 +135,8 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
// The running firewall still isn't built from what the config asks for, so the same reload fails again
for range 2 {
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
res := cfg.ReloadConfigString(rules("tcp"))
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.NotSame(t, fw, f.firewall)
}
// Every input the firewall is built from rebuilds it when it changes, and nothing else does
func TestReloadFirewall_RebuildsOnAnyInput(t *testing.T) {
l := test.NewLogger()
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
base := "firewall:\n inbound:\n - port: any\n proto: any\n host: any\n"
setup := func() (*config.C, *Interface, *PKI) {
pki := &PKI{}
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks}, initiatingVersion: cert.Version2})
cfg := config.NewC(l)
require.NoError(t, cfg.LoadString(base))
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
require.NoError(t, err)
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
return cfg, f, pki
}
setCert := func(pki *PKI, networks, unsafe []netip.Prefix) {
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks, unsafeNetworks: unsafe}, initiatingVersion: cert.Version2})
}
cases := map[string]struct {
cfg string
cert func(*PKI)
want bool
}{
"nothing": {base + "punchy:\n punch: true\n", nil, false},
"rules": {base + " outbound:\n - port: any\n proto: any\n host: any\n", nil, true},
"inbound_action": {base + " inbound_action: reject\n", nil, true},
"conntrack": {base + " conntrack:\n tcp_timeout: 1m\n", nil, true},
"networks": {base, func(p *PKI) { setCert(p, []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil) }, true},
"unsafe_networks": {base, func(p *PKI) { setCert(p, networks, []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}) }, true},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
cfg, f, pki := setup()
old := f.firewall
if tc.cert != nil {
tc.cert(pki)
}
res := cfg.ReloadConfigString(tc.cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, tc.want, f.firewall != old)
})
}
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
+2 -8
View File
@@ -12,7 +12,6 @@ import (
"strings"
"time"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/cpupick"
"github.com/slackhq/nebula/noiseutil"
@@ -72,11 +71,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
}
l.Info("Firewall started", "firewallHashes", fw.GetRuleHashes())
// The command registry is shared by every surface that exposes these commands, so a
// command cannot reach one of them without reaching the others.
cmdRegistry := commands.New()
ssh, err := sshd.NewSSHServer(ctx, l.With("subsystem", "sshd"), cmdRegistry)
ssh, err := sshd.NewSSHServer(ctx, l.With("subsystem", "sshd"))
if err != nil {
return nil, util.ContextualizeIfNeeded("Error while creating SSH server", err)
}
@@ -348,14 +343,13 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
return nil, nil
}
attachCommands(l, c, cmdRegistry, ifce)
attachCommands(l, c, ssh, ifce)
networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
return &Control{
state: StateReady,
f: ifce,
commands: cmdRegistry,
l: l,
ctx: ctx,
cancel: cancel,
+1 -1
View File
@@ -478,8 +478,8 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, messageCounte
if err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Error while validating inbound packet", "error", err, "packet", out)
return
}
return
}
dropReason := f.firewall.Drop(rxc.fwPacket.Packet, true, hostinfo, f.pki.GetCAPool(), rxc.ctCache.Get())
+5 -41
View File
@@ -1,27 +1,12 @@
package batch
import (
"net/netip"
"github.com/slackhq/nebula/header"
)
import "net/netip"
const SendBatchCap = 128
// FailureHeadLen is how much of a refused packet a Failure keeps, nebula's header, which says whose packet it was
const FailureHeadLen = header.Len
// batchWriter is the minimal subset of udp.Conn needed by SendBatch to flush.
type batchWriter interface {
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
}
// Failure is a packet the underlay would not send
type Failure struct {
// Head is the start of the packet, zero padded if the packet was shorter
Head [FailureHeadLen]byte
Dst netip.AddrPort
Err error
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
}
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
@@ -32,28 +17,16 @@ type SendBatch struct {
bufs [][]byte
dsts []netip.AddrPort
arena *Arena
failures []Failure
// recordFailure is bound once so handing it to WriteBatch does not allocate on every Flush
recordFailure func(i int, err error)
}
// NewSendBatch makes a SendBatch with batchCap slots and an arenaSize byte buffer for slices to back those slots
func NewSendBatch(out batchWriter, batchCap, arenaSize int) *SendBatch {
b := &SendBatch{
return &SendBatch{
out: out,
bufs: make([][]byte, 0, batchCap),
dsts: make([]netip.AddrPort, 0, batchCap),
arena: NewArena(arenaSize),
}
b.recordFailure = b.failed
return b
}
func (b *SendBatch) failed(i int, err error) {
f := Failure{Dst: b.dsts[i], Err: err}
copy(f.Head[:], b.bufs[i])
b.failures = append(b.failures, f)
}
func (b *SendBatch) Reserve(sz int) []byte {
@@ -71,15 +44,12 @@ func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort) {
}
// Flush writes every queued packet and reports how many actually went out. A short count means some destinations
// were undeliverable, Failures says which; the batch is drained either way.
// were undeliverable; the batch is drained either way.
func (b *SendBatch) Flush() (int, error) {
clear(b.failures)
b.failures = b.failures[:0]
var err error
written := 0
if len(b.bufs) > 0 {
written, err = b.out.WriteBatch(b.bufs, b.dsts, b.recordFailure)
written, err = b.out.WriteBatch(b.bufs, b.dsts)
}
clear(b.bufs)
b.bufs = b.bufs[:0]
@@ -87,9 +57,3 @@ func (b *SendBatch) Flush() (int, error) {
b.arena.Reset()
return written, err
}
// Failures returns every packet the last Flush could not send, in the order they were committed. Valid until the
// next Flush.
func (b *SendBatch) Failures() []Failure {
return b.failures
}
+1 -84
View File
@@ -1,9 +1,7 @@
package batch
import (
"errors"
"net/netip"
"slices"
"testing"
)
@@ -12,7 +10,7 @@ type fakeBatchWriter struct {
addrs []netip.AddrPort
}
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, _ func(int, error)) (int, error) {
func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// Snapshot — SendBatch.Flush nils its slot pointers right after WriteBatch
// returns, so tests must capture data before that happens.
w.bufs = make([][]byte, len(bufs))
@@ -122,84 +120,3 @@ func TestSendBatchGrowPreservesCommitted(t *testing.T) {
t.Errorf("second packet on the wire: %x", fw.bufs[1])
}
}
// failingBatchWriter refuses every packet whose first byte is in refuse
type failingBatchWriter struct {
refuse map[byte]error
}
func (w *failingBatchWriter) WriteBatch(bufs [][]byte, _ []netip.AddrPort, failed func(int, error)) (int, error) {
written := 0
for i, b := range bufs {
if err, ok := w.refuse[b[0]]; ok {
failed(i, err)
continue
}
written++
}
return written, nil
}
func TestSendBatchReportsFailures(t *testing.T) {
errNoRoute := errors.New("no route")
fw := &failingBatchWriter{refuse: map[byte]error{2: errNoRoute, 3: errNoRoute}}
b := NewSendBatch(fw, 4, 128)
// pkt is n bytes starting with the refuse key, each later byte its offset plus 0x10 so the head is recognizable
pkt := func(key byte, n int) []byte {
p := b.Reserve(n)[:n]
p[0] = key
for i := 1; i < n; i++ {
p[i] = byte(0x10 + i)
}
return p
}
head := func(p []byte) (h [FailureHeadLen]byte) {
copy(h[:], p)
return h
}
dsts := []netip.AddrPort{
netip.MustParseAddrPort("10.0.0.1:4242"),
netip.MustParseAddrPort("[fd00::2]:4242"),
netip.MustParseAddrPort("[fd00::3]:4242"),
netip.MustParseAddrPort("10.0.0.4:4242"),
}
long := pkt(2, FailureHeadLen+8) // only its head is kept
short := pkt(3, 4) // zero padded
b.Commit(pkt(1, 20), dsts[0])
b.Commit(long, dsts[1])
b.Commit(short, dsts[2])
b.Commit(pkt(1, 20), dsts[3])
written, err := b.Flush()
if err != nil {
t.Fatalf("Flush: %v", err)
}
if written != 2 {
t.Errorf("written = %d, want 2", written)
}
want := []Failure{{head(long), dsts[1], errNoRoute}, {head(short), dsts[2], errNoRoute}}
if got := b.Failures(); !slices.Equal(got, want) {
t.Errorf("failures = %v, want %v", got, want)
}
// Failures do not leak into the next flush
again := pkt(2, 20)
b.Commit(again, dsts[0])
if _, err := b.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if want := []Failure{{head(again), dsts[0], errNoRoute}}; !slices.Equal(b.Failures(), want) {
t.Errorf("second flush failures = %v, want %v", b.Failures(), want)
}
// A clean flush forgets the last one's failures
b.Commit(pkt(1, 20), dsts[0])
if _, err := b.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if got := b.Failures(); len(got) != 0 {
t.Errorf("clean flush failures = %v, want none", got)
}
}
+84 -54
View File
@@ -3,6 +3,7 @@
package overlay
import (
"errors"
"fmt"
"log/slog"
"math/bits"
@@ -11,7 +12,6 @@ import (
"sync"
"sync/atomic"
"syscall"
"time"
"unsafe"
"github.com/gaissmai/bart"
@@ -35,26 +35,6 @@ type tun struct {
// and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool
l *slog.Logger
// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
// fields below it
routesMu sync.Mutex
// routeStates is who held each installable unsafe_route prefix when we last looked
routeStates map[netip.Prefix]routeState
// routeWatch is the routing socket watchRoutes reads, Close closes it
routeWatch *os.File
// routeSweep reconciles the unsafe_routes every routeSweepEvery
routeSweep *time.Timer
routeSweepEvery time.Duration
closed bool
// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
// lock so the route watcher never waits on a reconcile
routeMoversMu sync.Mutex
routeMovers map[netip.Prefix]routeMover
// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
lastRouteEvent atomic.Int64
}
type ifReq struct {
@@ -159,12 +139,11 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
}
t := &tun{
f: os.NewFile(uintptr(fd), ""),
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c),
routeSweepEvery: routeSweepInterval,
l: l,
f: os.NewFile(uintptr(fd), ""),
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c),
l: l,
}
err = t.reload(c, true, nil)
@@ -240,16 +219,6 @@ func utunNameFromFd(fd int) string {
}
func (t *tun) Close() error {
t.routesMu.Lock()
t.closed = true
if t.routeWatch != nil {
_ = t.routeWatch.Close()
}
if t.routeSweep != nil {
t.routeSweep.Stop()
}
t.routesMu.Unlock()
if t.f != nil {
return t.f.Close()
}
@@ -318,22 +287,8 @@ func (t *tun) Activate() error {
return fmt.Errorf("failed to run tun device: %s", err)
}
// Watch before adding so a route removed in between is still noticed
watch, err := openRouteSocket()
if err != nil {
t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
// Unsafe path routes
if err = t.addRoutes(false, nil); err != nil {
if watch != nil {
_ = watch.Close()
}
return err
}
t.startRouteWatch(watch)
return nil
return t.addRoutes(false, nil)
}
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
@@ -438,8 +393,7 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
}
if !initial && !change {
// Any reload puts back the unsafe_routes someone deleted
return t.addRoutes(true, r)
return nil
}
routeTree, err := makeRouteTree(t.l, routes, false)
@@ -557,6 +511,82 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
return nil, nil
}
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
err := addRoute(r.Cidr, linkAddr)
if err != nil {
if errors.Is(err, unix.EEXIST) {
// Every route we added already exists on a reload, only warn about someone else's
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
retErr.Log(t.l)
rl.Failed()
} else {
return retErr
}
}
} else {
t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
}
}
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
linkAddr := t.linkAddr.Load()
if t.hostOwned || linkAddr == nil {
return nil
}
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue
}
err := delRoute(r.Cidr, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
-366
View File
@@ -3,17 +3,12 @@
package overlay
import (
"encoding/binary"
"encoding/hex"
"fmt"
"net"
"net/netip"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
@@ -109,367 +104,6 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
const (
// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
// With no interface address, which x/net/route skips
interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
)
func TestRoutesMayHaveMoved(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
routes := []Route{{Cidr: cidr, Via: via, Install: true}}
// A request like route(8) sends, the kernel echoes it to every routing socket
request := func(typ, flags int, dst, mask [4]byte) []byte {
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: typ,
Flags: flags,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: dst},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
},
}
b, err := m.Marshal()
require.NoError(t, err)
return b
}
raw := func(s string) []byte {
b, err := hex.DecodeString(s)
require.NoError(t, err)
return b
}
lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
routesMayHaveMoved := func(b []byte, routes []Route) bool {
moved, _, _ := routesMayHaveMoved(b, routes)
return moved
}
assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
neighbor := raw(kernelSubnetDelete)
binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
}
// A request names its prefix and who made it, the kernel's own and interface changes name neither
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: unix.RTM_CHANGE,
Flags: unix.RTF_UP,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
},
}
b, err := m.Marshal()
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved)
assert.Equal(t, cidr, p)
assert.Equal(t, int32(1234), pid)
for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
b, err := hex.DecodeString(s)
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved, name)
assert.False(t, p.IsValid(), name)
assert.Zero(t, pid, name)
}
}
func TestProcessName(t *testing.T) {
assert.Equal(t, "launchd", processName(1))
self := filepath.Base(os.Args[0])
assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
assert.Empty(t, processName(-1))
}
// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
// the minute between sweeps when it isn't zero.
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
requireRoot(t)
cfg := "tun:\n unsafe_routes:\n"
for _, cidr := range cidrs {
via := "10.250.0.2"
if netip.MustParsePrefix(cidr).Addr().Is6() {
via = "fd00:250::2"
}
cfg += " - route: " + cidr + "\n via: " + via + "\n"
}
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString(cfg))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
require.NoError(t, err)
if sweep != 0 {
tn.routeSweepEvery = sweep
}
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
return tn, c, cfg
}
func requireRoot(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating interfaces and adding routes needs root")
}
}
// routeVia is the index of the interface that routes cidr, 0 if none does
func routeVia(t *testing.T, cidr string) int {
msgs, err := fetchRoutes()
require.NoError(t, err)
return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
}
// routeStateOf is what tn made of cidr when it last looked
func routeStateOf(tn *tun, cidr string) routeState {
tn.routesMu.Lock()
defer tn.routesMu.Unlock()
return tn.routeStates[netip.MustParsePrefix(cidr)]
}
func routeArgs(action, cidr string, extra ...string) []string {
args := []string{"-q", "-n", action}
if netip.MustParsePrefix(cidr).Addr().Is6() {
args = append(args, "-inet6")
}
return append(append(args, "-net", cidr), extra...)
}
func routeCmd(t *testing.T, action, cidr string, extra ...string) {
requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
}
// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
// says so
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
require.NotContains(t, string(out), "writing to routing socket")
}
// cleanupRoute deletes cidr's route when the test ends, whoever holds it
func cleanupRoute(t *testing.T, cidr string) {
t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
}
// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
func routeThroughLo0(t *testing.T, cidr string) int {
requireRoot(t)
routeCmd(t, "add", cidr, "-interface", "lo0")
cleanupRoute(t, cidr)
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
return lo0.Index
}
// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
t.Run(cidr, func(t *testing.T) {
lo0 := routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
})
}
}
// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
cidr := "10.251.1.0/24"
lo0 := routeThroughLo0(t, cidr)
_, c, _ := newRouteTestTun(t, 0, cidr)
res := c.ReloadConfigString("tun: {}\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, lo0, routeVia(t, cidr))
}
// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
cidr := "10.251.3.0/24"
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
time.Sleep(2 * routeRetryDelay)
require.Equal(t, lo0.Index, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
}
// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
cidr := "10.251.4.0/24"
tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
// A reload with nothing changed is how to ask for it back
res := c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, ours, routeVia(t, cidr))
}
// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
for name, tc := range map[string]struct {
cidr string
sweep time.Duration
misfile func(t *testing.T, tn *tun, cidr string)
}{
// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
}},
// The sweep that caps a table that never settles
"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
tn.reconcileRoutes()
require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
}},
} {
t.Run(name, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, tc.cidr))
routeCmd(t, "delete", tc.cidr)
tc.misfile(t, tn, tc.cidr)
lo0 := routeThroughLo0(t, tc.cidr)
require.Eventually(t, func() bool {
s := routeStateOf(tn, tc.cidr)
return s.kind == routeYielded && s.holder == lo0
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", tc.cidr)
assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
// taken for a hand delete
func TestTunYieldsToARouteChangedAway(t *testing.T) {
for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
t.Run(cidr, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
routeCmd(t, "change", cidr, "-interface", "lo0")
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
require.Equal(t, lo0.Index, routeVia(t, cidr))
require.Eventually(t, func() bool {
s := routeStateOf(tn, cidr)
return s.kind == routeYielded && s.holder == lo0.Index
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
// ethernet configd routed the lan through
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
requireRoot(t)
var feth string
for i := 100; i < 200 && feth == ""; i++ {
if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
feth = fmt.Sprintf("feth%d", i)
}
}
require.NotEmpty(t, feth)
require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
// refuses one named by the feth itself
cidr := "10.251.2.0/24"
routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
cleanupRoute(t, cidr)
ifc, err := net.InterfaceByName(feth)
require.NoError(t, err)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Losing the address first leaves the route, the reconcile finds it still held
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
time.Sleep(2 * routeRetryDelay)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Only RTM_IFINFO says the interface went, its route goes with no message of its own
require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
cidr := "10.251.5.0/24"
routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
tn.routesMu.Lock()
require.NoError(t, tn.routeWatch.Close())
tn.routesMu.Unlock()
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
func TestTunDevNeedsRestart(t *testing.T) {
for name, want := range map[string]bool{
-121
View File
@@ -1,121 +0,0 @@
//go:build !ios && !e2e_testing
package overlay
import (
"bytes"
"os"
"syscall"
"testing"
"github.com/stretchr/testify/require"
)
func TestTun_WriteFraming(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: w}
for _, tc := range []struct {
name string
pkt []byte
af byte
}{
// Literal families rather than syscall.AF_*, so a wrong constant in Write can't also fix the expectation.
{"ipv4", []byte{0x45, 1, 2, 3, 4, 5}, 2},
{"ipv6", []byte{0x60, 1, 2, 3, 4, 5}, 30},
} {
t.Run(tc.name, func(t *testing.T) {
n, err := tn.Write(tc.pkt)
require.NoError(t, err)
require.Equal(t, len(tc.pkt), n)
got := make([]byte, 64)
n, err = r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, tc.af}, tc.pkt...), got[:n])
})
}
_, err = tn.Write([]byte{0x10, 1, 2, 3})
require.Error(t, err)
_, err = tn.Write([]byte{})
require.ErrorIs(t, err, syscall.EIO)
// The rejected writes must not have put anything on the fd, so the next read sees only this packet.
pkt := []byte{0x45, 9, 9}
_, err = tn.Write(pkt)
require.NoError(t, err)
got := make([]byte, 64)
n, err := r.Read(got)
require.NoError(t, err)
require.Equal(t, append([]byte{0, 0, 0, 2}, pkt...), got[:n])
}
func TestTun_ReadStripsHeader(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
pkt := []byte{0x45, 1, 2, 3, 4, 5}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err := tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
func TestTun_ReadShort(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tn := &tun{f: r}
// Fewer bytes than the protocol header is not a packet.
_, err = w.Write([]byte{0, 0})
require.NoError(t, err)
n, err := tn.Read(make([]byte, 64))
require.NoError(t, err)
require.Zero(t, n)
// The next packet reads cleanly only if the short read consumed exactly what was written.
pkt := []byte{0x45, 1, 2, 3}
_, err = w.Write(append([]byte{0, 0, 0, syscall.AF_INET}, pkt...))
require.NoError(t, err)
got := make([]byte, 64)
n, err = tn.Read(got)
require.NoError(t, err)
require.Equal(t, pkt, got[:n])
}
// A heap allocation per packet rarely shows in a short throughput run, only across GC pauses, so
// this pins Read and Write to zero allocations directly.
func TestTun_ReadWriteDoNotAllocate(t *testing.T) {
r, w, err := os.Pipe()
require.NoError(t, err)
defer r.Close()
defer w.Close()
tw := &tun{f: w}
tr := &tun{f: r}
pkts := [][]byte{bytes.Repeat([]byte{0x45}, 1300), bytes.Repeat([]byte{0x60}, 1300)}
buf := make([]byte, 1500)
i := 0
allocs := testing.AllocsPerRun(1000, func() {
i++
if _, err := tw.Write(pkts[i%len(pkts)]); err != nil {
t.Fatal(err)
}
if _, err := tr.Read(buf); err != nil {
t.Fatal(err)
}
})
require.Zero(t, allocs)
}
-451
View File
@@ -1,451 +0,0 @@
//go:build !ios && !e2e_testing
package overlay
import (
"errors"
"fmt"
"net"
"net/netip"
"os"
"strconv"
"sync/atomic"
"time"
"unsafe"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/util"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
// or until another interface takes the prefix, after which ours follows theirs like any other.
const (
// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
// replaces a route by deleting and adding, looking in between would find the prefix free
routeRetryDelay = time.Second
// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
// a message for each
routeSweepInterval = time.Minute
// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
routeSocketRcvBuf = 1 << 20
)
type routeStateKind int
const (
routeUnknown routeStateKind = iota
// routeInstalled is our route holding the prefix
routeInstalled
// routeYielded is another interface holding the prefix, ours is added once it lets go
routeYielded
// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
// another interface takes the prefix
routeRemoved
// routeFailed is adding our route failing for some other reason, it is retried
routeFailed
)
type routeState struct {
kind routeStateKind
// holder is the index of the interface holding the prefix when kind is routeYielded
holder int
}
// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
// is looked up as the message arrives, route(8) has exited by the time the table settles
type routeMover struct {
pid int32
name string
}
// logArgs appends m to a log line's args, when we know who it was
func (m routeMover) logArgs(args ...any) []any {
if m.pid == 0 {
return args
}
args = append(args, "pid", m.pid)
if m.name != "" {
args = append(args, "process", m.name)
}
return args
}
// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
func processName(pid int32) string {
kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
if err != nil || kp.Proc.P_pid != pid {
return ""
}
return unix.ByteSliceToString(kp.Proc.P_comm[:])
}
// installRoutes returns the routes we put in the system routing table, one per prefix
func installRoutes(routes []Route) []Route {
seen := make(map[netip.Prefix]struct{}, len(routes))
var out []Route
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
p := r.Cidr.Masked()
if _, ok := seen[p]; ok {
continue
}
seen[p] = struct{}{}
out = append(out, r)
}
return out
}
// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
// interface holds its prefix. It runs on start and on every reload.
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
var msgs []netroute.Message
var msgsErr error
fetched := false
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
err := addRoute(p, linkAddr)
if err == nil {
t.l.Info("Added route", "route", r)
states[p] = routeState{kind: routeInstalled}
continue
}
if !errors.Is(err, unix.EEXIST) {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if !logErrors {
return retErr
}
retErr.Log(t.l)
rl.Failed()
states[p] = routeState{kind: routeFailed}
continue
}
// Every route we added already exists on a reload, only warn about someone else's
if !fetched {
msgs, msgsErr = fetchRoutes()
fetched = true
}
if msgsErr != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
states[p] = prev
continue
}
switch holder := routeInterfaceIn(msgs, p); holder {
case linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case 0:
// Removed since the add, the watcher will see to it
states[p] = prev
default:
if prev.kind != routeYielded || prev.holder != holder {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
}
states[p] = routeState{kind: routeYielded, holder: holder}
}
}
t.routeStates = states
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
msgs, err := fetchRoutes()
if err != nil {
var left []netip.Prefix
for _, r := range installRoutes(routes) {
left = append(left, r.Cidr)
}
return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
}
for _, r := range installRoutes(routes) {
p := r.Cidr.Masked()
// Whatever happens next, the prefix is no longer one we know about
delete(t.routeStates, p)
if routeInterfaceIn(msgs, p) != linkAddr.Index {
continue
}
err = delRoute(p, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
// changed since it last looked, it runs every routeSweepEvery.
func (t *tun) reconcileRoutes() {
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return
}
t.routeMoversMu.Lock()
movers := t.routeMovers
t.routeMovers = nil
t.routeMoversMu.Unlock()
msgs, err := fetchRoutes()
if err != nil {
t.l.Error("Failed to read the system routing table", "error", err)
return
}
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
by := movers[p]
holder := routeInterfaceIn(msgs, p)
switch {
case holder == linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case holder != 0:
// Another interface taking a prefix we left alone puts ours back in line behind theirs
if prev.kind != routeYielded || prev.holder != holder {
t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
}
states[p] = routeState{kind: routeYielded, holder: holder}
case prev.kind == routeInstalled:
t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
states[p] = routeState{kind: routeRemoved}
case prev.kind == routeRemoved:
states[p] = prev
default:
err := addRoute(p, linkAddr)
switch {
case err == nil:
if prev.kind == routeYielded {
t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
} else {
t.l.Info("Added route", "route", r)
}
states[p] = routeState{kind: routeInstalled}
case errors.Is(err, unix.EEXIST):
// Taken since the fetch, the next look will see by whom
states[p] = prev
default:
if prev.kind != routeFailed {
util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
}
states[p] = routeState{kind: routeFailed}
}
}
}
t.routeStates = states
}
// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
// routing table settles after anything that may have moved a prefix
func (t *tun) startRouteWatch(watch *os.File) {
t.routesMu.Lock()
defer t.routesMu.Unlock()
if t.closed {
if watch != nil {
_ = watch.Close()
}
return
}
t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
if watch != nil {
t.routeWatch = watch
go t.watchRoutes(watch)
}
}
func (t *tun) sweepRoutes() {
// A table still changing gets reconciled once it settles
if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
t.reconcileRoutes()
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
if !t.closed {
t.routeSweep.Reset(t.routeSweepEvery)
}
}
func openRouteSocket() (*os.File, error) {
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
}
// Fewer drops, the sweep catches the rest
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
// Non-blocking puts it on the poller, so Close interrupts a Read
if err = unix.SetNonblock(fd, true); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
}
return os.NewFile(uintptr(fd), "route"), nil
}
// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
func (t *tun) watchRoutes(f *os.File) {
var reconciled atomic.Int64
settle := time.AfterFunc(time.Hour, func() {
reconciled.Store(time.Now().UnixNano())
t.reconcileRoutes()
})
settle.Stop()
defer settle.Stop()
var burst time.Time
self := int32(os.Getpid())
b := make([]byte, os.Getpagesize())
for {
n, err := f.Read(b)
if err != nil {
if !errors.Is(err, os.ErrClosed) {
t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
return
}
moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
if !moved {
continue
}
// Our own are already in routeStates
if p.IsValid() && pid != self {
t.routeMoversMu.Lock()
if t.routeMovers == nil {
t.routeMovers = make(map[netip.Prefix]routeMover)
}
t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
t.routeMoversMu.Unlock()
}
now := time.Now()
t.lastRouteEvent.Store(now.UnixNano())
if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
burst = now
}
settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
}
}
// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
// and the pid of the process that made it.
func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
if len(b) < 4 {
return false, p, 0
}
switch int(b[3]) {
case unix.RTM_IFINFO, unix.RTM_DELADDR:
return true, p, 0
case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
default:
return false, p, 0
}
if len(b) < unix.SizeofRtMsghdr {
return false, p, 0
}
hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
// A failed request is reported too, and a scoped route never held the prefix
if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
return false, p, 0
}
if hdr.Pid == 0 {
// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
// could be holding a prefix
return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
}
// Replies to a request carry the requester's padded sockaddrs
msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
if err != nil || len(msgs) != 1 {
return false, p, 0
}
m, ok := msgs[0].(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
return false, p, 0
}
p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
if !ok {
return false, netip.Prefix{}, 0
}
p = p.Masked()
for _, r := range installRoutes(routes) {
if r.Cidr.Masked() == p {
return true, p, hdr.Pid
}
}
return false, netip.Prefix{}, 0
}
// interfaceName is the name of the interface with index i, for logs
func interfaceName(i int) string {
if ifc, err := net.InterfaceByIndex(i); err == nil {
return ifc.Name
}
return strconv.Itoa(i)
}
+10 -19
View File
@@ -165,24 +165,20 @@ func (p *Punchy) scheduleJob(job holepunchJob, delay time.Duration) {
// SendPunch sends an immediate keepalive punch for an idle hostinfo.
// The configured punchy.target_all_remotes mode picks the targets. Gated on punchy.punch and the lighthouse-skip rule
// (lighthouses don't get keepalive punches because the regular update interval keeps their NAT state warm).
// Returns the hostinfo's current remote and the error from punching it, an invalid remote if it was not punched.
func (p *Punchy) SendPunch(hostinfo *HostInfo) (netip.AddrPort, error) {
func (p *Punchy) SendPunch(hostinfo *HostInfo) {
if !p.punch.Load() {
return netip.AddrPort{}, nil
return
}
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
return netip.AddrPort{}, nil
return
}
hr := hostinfo.GetRemote()
if p.punchEverything.Load() {
return hr, p.sendPunchToAllRemotes(hostinfo, hr)
p.sendPunchToAllRemotes(hostinfo)
} else if hr := hostinfo.GetRemote(); hr.IsValid() {
p.metricPunchyTx.Inc(1)
p.punchConn.WriteTo([]byte{1}, hr)
}
if !hr.IsValid() {
return hr, nil
}
p.metricPunchyTx.Inc(1)
return hr, p.punchConn.WriteTo([]byte{1}, hr)
}
// SendPunchToAll punches every known remote for hostinfo, but only when punchy.target_all_remotes is enabled.
@@ -199,19 +195,14 @@ func (p *Punchy) SendPunchToAll(hostinfo *HostInfo) {
if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
return
}
p.sendPunchToAllRemotes(hostinfo, netip.AddrPort{})
p.sendPunchToAllRemotes(hostinfo)
}
// sendPunchToAllRemotes punches every known remote for hostinfo and returns the error from punching current
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo, current netip.AddrPort) error {
var currentErr error
func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo) {
hostinfo.remotes.ForEach(p.hm.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
p.metricPunchyTx.Inc(1)
if err := p.punchConn.WriteTo([]byte{1}, addr); addr == current {
currentErr = err
}
p.punchConn.WriteTo([]byte{1}, addr)
})
return currentErr
}
// Start wires the runtime dependencies and spawns the scheduler worker.
-112
View File
@@ -3,14 +3,11 @@ package nebula
import (
"context"
"log/slog"
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -245,112 +242,3 @@ func TestPunchyReloadRestartRequired(t *testing.T) {
require.NoError(t, res.Err)
assert.True(t, res.RestartRequired)
}
// errNoRoute is the unreachable error every platform classifies the same way
var errNoRoute = udp.ErrInvalidIPv6RemoteForSocket
// recordingConn records where every write went and what kind of message it was, and fails the ones to an address in
// fail
type recordingConn struct {
udp.NoopConn
fail map[netip.AddrPort]error
wrote []netip.AddrPort
// types is the message type of each write, a punch has none
types []header.MessageType
}
func (c *recordingConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.wrote = append(c.wrote, addr)
var h header.H
if h.Parse(b) == nil {
c.types = append(c.types, h.Type)
}
return c.fail[addr]
}
func (c *recordingConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(int, error)) (int, error) {
written := 0
for i := range bufs {
if err := c.WriteTo(bufs[i], addrs[i]); err != nil {
failed(i, err)
continue
}
written++
}
return written, nil
}
type noLighthouses struct{}
func (noLighthouses) IsAnyLighthouseAddr([]netip.Addr) bool { return false }
var (
punchTargetV4 = netip.MustParseAddrPort("10.0.0.3:4242")
punchTargetV6 = netip.MustParseAddrPort("[fd00::3]:4242")
)
// newPunchTarget is a tunnel on its IPv6 remote that also knows the peer's IPv4 one
func newPunchTarget() *HostInfo {
vpnAddr := netip.MustParseAddr("10.128.0.3")
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
remotes: NewRemoteList([]netip.Addr{vpnAddr}, nil),
}
hostinfo.remotes.LearnRemote(vpnAddr, punchTargetV4)
hostinfo.SetRemote(punchTargetV6)
return hostinfo
}
func newPunchyForTest(t *testing.T, settings map[string]any, conn udp.Conn) *Punchy {
c := config.NewC(test.NewLogger())
c.Settings["punchy"] = settings
p := NewPunchyFromConfig(test.NewLogger(), c, conn)
hm := newHostMap(test.NewLogger())
hm.preferredRanges.Store(&[]netip.Prefix{})
p.hm = hm
p.lh = noLighthouses{}
return p
}
func TestPunchy_SendPunchReportsCurrentRemote(t *testing.T) {
t.Run("disabled", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": false}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.False(t, remote.IsValid())
require.NoError(t, err)
assert.Empty(t, conn.wrote)
})
t.Run("current remote", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.ErrorIs(t, err, errNoRoute)
assert.Equal(t, []netip.AddrPort{punchTargetV6}, conn.wrote)
})
t.Run("all remotes, current fails", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV6: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.ErrorIs(t, err, errNoRoute)
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
})
t.Run("all remotes, only another fails", func(t *testing.T) {
conn := &recordingConn{fail: map[netip.AddrPort]error{punchTargetV4: errNoRoute}}
p := newPunchyForTest(t, map[string]any{"punch": true, "target_all_remotes": true}, conn)
remote, err := p.SendPunch(newPunchTarget())
assert.Equal(t, punchTargetV6, remote)
require.NoError(t, err, "a failure on a remote we are not using says nothing about the current one")
assert.ElementsMatch(t, []netip.AddrPort{punchTargetV4, punchTargetV6}, conn.wrote)
})
}
+2 -4
View File
@@ -36,13 +36,11 @@ func (l *tcpListener) Accept() (net.Conn, error) {
func (l *tcpListener) Close() error {
l.closeOnce.Do(func() {
// Close before leaving the map so a concurrent closeListeners that
// no longer sees this listener knows Accept is already released.
close(l.closed)
l.s.mu.Lock()
delete(l.s.mu.listeners, l.port)
l.s.mu.Unlock()
close(l.closed)
})
return nil
}
+7 -70
View File
@@ -32,18 +32,13 @@ import (
const nicID = 1
type Service struct {
eg *errgroup.Group
control *nebula.Control
ipstack *stack.Stack
// done is closed once the forwarding goroutines have exited and the
// netstack is torn down, at which point err holds the first fatal error.
done chan struct{}
err error
mu struct {
sync.Mutex
closed bool
listeners map[uint16]*tcpListener
}
}
@@ -70,6 +65,7 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
ctx := control.Context()
eg, ctx := errgroup.WithContext(ctx)
s := Service{
eg: eg,
control: control,
}
s.mu.listeners = map[uint16]*tcpListener{}
@@ -78,12 +74,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6},
})
// The teardown goroutine only owns the stack once New succeeds
defer func() {
if reterr != nil {
s.ipstack.Destroy()
}
}()
sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default
tcpipErr := s.ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt)
if tcpipErr != nil {
@@ -133,12 +123,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
// this will read exactly one packet
n, err := reader.Read(buf)
if err != nil {
// An error after shutdown began is noise from the device pipes
// closing, anything else is fatal and must stop nebula too
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err
}
packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{
@@ -146,8 +130,8 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
})
linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf)
if ctx.Err() != nil {
return nil
if err := ctx.Err(); err != nil {
return err
}
}
})
@@ -155,18 +139,13 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
for {
packet := linkEP.ReadContext(ctx)
if packet == nil {
if ctx.Err() != nil {
return nil
if err := ctx.Err(); err != nil {
return err
}
continue
}
bufView := packet.ToView()
if _, err := bufView.WriteTo(writer); err != nil {
// Same shutdown noise handling as the inbound reader
if ctx.Err() != nil {
return nil
}
s.control.Stop()
return err
}
bufView.Release()
@@ -179,22 +158,6 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
return control.Wait()
})
// Tear down here rather than in Close so it also happens when nebula stops
// on its own. The forwarding goroutines must exit before the stack they feed
// is destroyed.
s.done = make(chan struct{})
go func() {
s.err = eg.Wait()
s.closeListeners()
// Destroy aborts the endpoints that exist when it starts, then waits on
// every endpoint, so a Dial registering in between would hold Wait until
// its connect timed out. Without the NIC new dials fail route lookup.
// RemoveNIC also closes linkEP.
_ = s.ipstack.RemoveNIC(nicID)
s.ipstack.Destroy()
close(s.done)
}()
return &s, nil
}
@@ -274,9 +237,6 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.mu.closed {
return nil, net.ErrClosed
}
if _, ok := s.mu.listeners[port]; ok {
return nil, fmt.Errorf("already listening on port %d", port)
}
@@ -285,38 +245,15 @@ func (s *Service) Listen(network, address string) (net.Listener, error) {
return l, nil
}
// Wait blocks until the service has fully stopped, either via Close or a fatal
// error, and returns the first fatal error if there was one.
func (s *Service) Wait() error {
<-s.done
return s.err
return s.eg.Wait()
}
// Close closes all listeners and stops nebula. Use Wait to block until
// shutdown has completed.
func (s *Service) Close() error {
s.closeListeners()
s.control.Stop()
return nil
}
// closeListeners unblocks pending Accept calls and rejects new listeners.
func (s *Service) closeListeners() {
// Every caller closes every listener rather than returning early once
// closed is set, so none returns while another is still closing them.
s.mu.Lock()
s.mu.closed = true
listeners := make([]*tcpListener, 0, len(s.mu.listeners))
for _, l := range s.mu.listeners {
listeners = append(listeners, l)
}
s.mu.Unlock()
for _, l := range listeners {
_ = l.Close()
}
}
func (s *Service) tcpHandler(r *tcp.ForwarderRequest) {
endpointID := r.ID()
-159
View File
@@ -327,162 +327,3 @@ func TestListenerAcceptAfterClose(t *testing.T) {
}
}
}
// Close must release blocked Accept calls and let Wait report a clean shutdown.
func TestServiceClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
if err := a.Close(); err != nil {
t.Fatal(err)
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after Close")
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after Close")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
if err := a.Close(); err != nil {
t.Fatalf("second close: %v", err)
}
if _, err := a.Listen("tcp", ":1235"); !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Listen after Close, got %v", err)
}
}
// If nebula stops without Close being called, the service must still release
// Accept and tear down the netstack.
func TestServiceNebulaStopped(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
ln, err := a.Listen("tcp", ":1234")
if err != nil {
t.Fatal(err)
}
accepted := make(chan error, 1)
go func() {
_, err := ln.Accept()
accepted <- err
}()
a.control.Stop()
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait still blocked after nebula stopped")
}
select {
case err := <-accepted:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("expected net.ErrClosed from Accept, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Accept still blocked after nebula stopped")
}
if a.ipstack.HasNIC(nicID) {
t.Fatal("netstack not torn down after Wait")
}
}
// Dials racing Close must not hold Wait until their connect times out.
func TestServiceDialDuringClose(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{
"static_host_map": m{},
"listen": m{
"host": "127.0.0.1",
"port": 0,
},
})
stop := make(chan struct{})
var eg errgroup.Group
for range 8 {
eg.Go(func() error {
for {
select {
case <-stop:
return nil
default:
}
// 10.0.0.2 has no tunnel, so each dial sits in SYN-SENT until aborted
if c, err := a.Dial("tcp", "10.0.0.2:80"); err == nil {
_ = c.Close()
}
}
})
}
time.Sleep(50 * time.Millisecond)
if err := a.Close(); err != nil {
t.Fatal(err)
}
waited := make(chan error, 1)
go func() { waited <- a.Wait() }()
select {
case err := <-waited:
if err != nil {
t.Fatalf("expected clean shutdown from Wait, got %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Wait blocked behind a dial that raced Close")
}
close(stop)
dialersDone := make(chan error, 1)
go func() { dialersDone <- eg.Wait() }()
select {
case <-dialersDone:
case <-time.After(5 * time.Second):
t.Fatal("Dial still blocked after Wait")
}
}
+743
View File
@@ -1,17 +1,59 @@
package nebula
import (
"bytes"
"encoding/json"
"errors"
"flag"
"fmt"
"log/slog"
"maps"
"net"
"net/netip"
"os"
"slices"
"sort"
"strings"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/sshd"
)
type sshListHostMapFlags struct {
Json bool
Pretty bool
ByIndex bool
}
type sshPrintCertFlags struct {
Json bool
Pretty bool
Raw bool
}
type sshPrintTunnelFlags struct {
Pretty bool
}
type sshChangeRemoteFlags struct {
Address string
}
type sshCloseTunnelFlags struct {
LocalOnly bool
}
type sshCreateTunnelFlags struct {
Address string
}
type sshDeviceInfoFlags struct {
Json bool
Pretty bool
}
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
if c.GetBool("sshd.enabled", false) {
@@ -185,3 +227,704 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
return runner, nil
}
func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
ssh.RegisterCommand(&sshd.Command{
Name: "list-hostmap",
ShortDescription: "List all known previously connected hosts",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshListHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshListHostMap(f.hostMap, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "list-pending-hostmap",
ShortDescription: "List all handshaking hosts",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshListHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshListHostMap(f.handshakeManager, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "list-lighthouse-addrmap",
ShortDescription: "List all lighthouse map entries",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshListHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshListLighthouseMap(f.lightHouse, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "reload",
ShortDescription: "Reloads configuration from disk, same as sending HUP to the process",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshReload(l, c, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "log-level",
ShortDescription: "Gets or sets the current log level",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshLogLevel(l, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "log-format",
ShortDescription: "Gets or sets the current log format",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshLogFormat(l, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "version",
ShortDescription: "Prints the currently running version of nebula",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshVersion(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "device-info",
ShortDescription: "Prints information about the network device.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshDeviceInfoFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshDeviceInfo(f, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "print-cert",
ShortDescription: "Prints the current certificate being used or the certificate for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshPrintCertFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.Raw, "raw", false, "raw prints the PEM encoded certificate, not compatible with -json or -pretty")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshPrintCert(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "print-tunnel",
ShortDescription: "Prints json details about a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshPrintTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshPrintTunnel(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "print-relays",
ShortDescription: "Prints json details about all relay info",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshPrintTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshPrintRelays(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "change-remote",
ShortDescription: "Changes the remote address used in the tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshChangeRemoteFlags{}
fl.StringVar(&s.Address, "address", "", "The new remote address, ip:port")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshChangeRemote(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "close-tunnel",
ShortDescription: "Closes a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshCloseTunnelFlags{}
fl.BoolVar(&s.LocalOnly, "local-only", false, "Disables notifying the remote that the tunnel is shutting down")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshCloseTunnel(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "create-tunnel",
ShortDescription: "Creates a tunnel for the provided vpn address",
Help: "The lighthouses will be queried for real addresses but you can provide one as well.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshCreateTunnelFlags{}
fl.StringVar(&s.Address, "address", "", "Optionally provide a real remote address, ip:port ")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshCreateTunnel(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "query-lighthouse",
ShortDescription: "Query the lighthouses for the provided vpn address",
Help: "This command is asynchronous. Only currently known udp addresses will be printed.",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshQueryLighthouse(f, fs, a, w)
},
})
}
func sshListHostMap(hl controlHostLister, a any, w sshd.StringWriter) error {
fs, ok := a.(*sshListHostMapFlags)
if !ok {
return nil
}
var hm []ControlHostInfo
if fs.ByIndex {
hm = listHostMapIndexes(hl)
} else {
hm = listHostMapHosts(hl)
}
sort.Slice(hm, func(i, j int) bool {
return hm[i].VpnAddrs[0].Compare(hm[j].VpnAddrs[0]) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
err := js.Encode(hm)
if err != nil {
return nil
}
} else {
for _, v := range hm {
err := w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddrs, v.RemoteAddrs))
if err != nil {
return err
}
}
}
return nil
}
func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) error {
fs, ok := a.(*sshListHostMapFlags)
if !ok {
return nil
}
type lighthouseInfo struct {
VpnAddr string `json:"vpnAddr"`
Addrs *CacheMap `json:"addrs"`
}
lightHouse.RLock()
addrMap := make([]lighthouseInfo, len(lightHouse.addrMap))
x := 0
for k, v := range lightHouse.addrMap {
addrMap[x] = lighthouseInfo{
VpnAddr: k.String(),
Addrs: v.CopyCache(),
}
x++
}
lightHouse.RUnlock()
sort.Slice(addrMap, func(i, j int) bool {
return strings.Compare(addrMap[i].VpnAddr, addrMap[j].VpnAddr) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
err := js.Encode(addrMap)
if err != nil {
return nil
}
} else {
for _, v := range addrMap {
b, err := json.Marshal(v.Addrs)
if err != nil {
return err
}
err = w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddr, string(b)))
if err != nil {
return err
}
}
}
return nil
}
func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
return w.WriteLine(fmt.Sprintf("%s", ifce.version))
}
func sshQueryLighthouse(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
var cm *CacheMap
rl := ifce.lightHouse.Query(vpnAddr)
if rl != nil {
cm = rl.CopyCache()
}
return json.NewEncoder(w.GetWriter()).Encode(cm)
}
func sshCloseTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
flags, ok := fs.(*sshCloseTunnelFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
if !flags.LocalOnly {
ifce.send(
header.CloseTunnel,
0,
hostInfo.ConnectionState,
hostInfo,
[]byte{},
make([]byte, 12, 12),
make([]byte, mtu),
)
}
ifce.closeTunnel(hostInfo)
return w.WriteLine("Closed")
}
func sshCreateTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
flags, ok := fs.(*sshCreateTunnelFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return w.WriteLine(fmt.Sprintf("Tunnel already exists"))
}
hostInfo = ifce.handshakeManager.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return w.WriteLine(fmt.Sprintf("Tunnel already handshaking"))
}
var addr netip.AddrPort
if flags.Address != "" {
addr, err = netip.ParseAddrPort(flags.Address)
if err != nil {
return w.WriteLine("Address could not be parsed")
}
}
// Set the remote from the callback, which StartHandshake runs under the handshake manager lock
// before the hostinfo reaches the handshake loop or the static-host trigger, so the address is
// in the RemoteList before the first handshake is sent.
ifce.handshakeManager.StartHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
if addr.IsValid() {
hh.hostinfo.SetRemote(addr)
}
})
return w.WriteLine("Created")
}
func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
flags, ok := fs.(*sshChangeRemoteFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
if flags.Address == "" {
return w.WriteLine("No address was provided")
}
addr, err := netip.ParseAddrPort(flags.Address)
if err != nil {
return w.WriteLine("Address could not be parsed")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
hostInfo.SetRemote(addr)
return w.WriteLine("Changed")
}
func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level
SetLevel(slog.Level)
})
if !ok {
return w.WriteLine("Log level is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
level, err := logging.ParseLevel(strings.ToLower(a[0]))
if err != nil {
return w.WriteLine(fmt.Sprintf("Unknown log level %s. Possible log levels: trace, debug, info, warn, error", a))
}
ctrl.SetLevel(level)
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
func sshLogFormat(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetFormat() string
SetFormat(string) error
})
if !ok {
return w.WriteLine("Log format is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
if err := ctrl.SetFormat(strings.ToLower(a[0])); err != nil {
return err
}
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
func sshPrintCert(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
args, ok := fs.(*sshPrintCertFlags)
if !ok {
return nil
}
cert := ifce.pki.getCertState().GetDefaultCertificate()
if len(a) > 0 {
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
cert = hostInfo.GetCert().Certificate
}
if args.Json || args.Pretty {
b, err := cert.MarshalJSON()
if err != nil {
return nil
}
if args.Pretty {
buf := new(bytes.Buffer)
err := json.Indent(buf, b, "", " ")
b = buf.Bytes()
if err != nil {
return nil
}
}
return w.WriteBytes(b)
}
if args.Raw {
b, err := cert.MarshalPEM()
if err != nil {
return nil
}
return w.WriteBytes(b)
}
return w.WriteLine(cert.String())
}
func sshPrintRelays(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
args, ok := fs.(*sshPrintTunnelFlags)
if !ok {
w.WriteLine(fmt.Sprintf("sshPrintRelays failed to convert args type"))
return nil
}
relays := map[uint32]*HostInfo{}
ifce.hostMap.Lock()
maps.Copy(relays, ifce.hostMap.Relays)
ifce.hostMap.Unlock()
type RelayFor struct {
Error error
Type string
State string
PeerAddr netip.Addr
LocalIndex uint32
RemoteIndex uint32
RelayedThrough []netip.Addr
}
type RelayOutput struct {
NebulaAddr netip.Addr
RelayForAddrs []RelayFor
}
type CmdOutput struct {
Relays []*RelayOutput
}
co := CmdOutput{}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
for k, v := range relays {
ro := RelayOutput{NebulaAddr: v.vpnAddrs[0]}
co.Relays = append(co.Relays, &ro)
relayHI := ifce.hostMap.QueryVpnAddr(v.vpnAddrs[0])
if relayHI == nil {
ro.RelayForAddrs = append(ro.RelayForAddrs, RelayFor{Error: errors.New("could not find hostinfo")})
continue
}
for _, vpnAddr := range relayHI.relayState.CopyRelayForIps() {
rf := RelayFor{Error: nil}
r, ok := relayHI.relayState.GetRelayForByAddr(vpnAddr)
if ok {
t := ""
switch r.Type {
case ForwardingType:
t = "forwarding"
case TerminalType:
t = "terminal"
default:
t = "unknown"
}
s := ""
switch r.State {
case Requested:
s = "requested"
case Established:
s = "established"
default:
s = "unknown"
}
rf.LocalIndex = r.LocalIndex
rf.RemoteIndex = r.RemoteIndex
rf.PeerAddr = r.PeerAddr
rf.Type = t
rf.State = s
if rf.LocalIndex != k {
rf.Error = fmt.Errorf("hostmap LocalIndex '%v' does not match RelayState LocalIndex", k)
}
}
relayedHI := ifce.hostMap.QueryVpnAddr(vpnAddr)
if relayedHI != nil {
rf.RelayedThrough = append(rf.RelayedThrough, relayedHI.relayState.CopyRelayIps()...)
}
ro.RelayForAddrs = append(ro.RelayForAddrs, rf)
}
}
err := enc.Encode(co)
if err != nil {
return err
}
return nil
}
func sshPrintTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
args, ok := fs.(*sshPrintTunnelFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.GetPreferredRanges()))
}
func sshDeviceInfo(ifce *Interface, fs any, w sshd.StringWriter) error {
data := struct {
Name string `json:"name"`
Cidr []netip.Prefix `json:"cidr"`
}{
Name: ifce.inside.Name(),
Cidr: make([]netip.Prefix, len(ifce.inside.Networks())),
}
copy(data.Cidr, ifce.inside.Networks())
flags, ok := fs.(*sshDeviceInfoFlags)
if !ok {
return fmt.Errorf("internal error: expected flags to be sshDeviceInfoFlags but was %+v", fs)
}
if flags.Json || flags.Pretty {
js := json.NewEncoder(w.GetWriter())
if flags.Pretty {
js.SetIndent("", " ")
}
return js.Encode(data)
} else {
return w.WriteLine(fmt.Sprintf("name=%v cidr=%v", data.Name, data.Cidr))
}
}
// sshReload reloads the config and writes the result to the client. It reloads even if the client has gone away
func sshReload(l *slog.Logger, c *config.C, w sshd.StringWriter) error {
writeErr := w.WriteLine("Reloading config")
res := c.ReloadConfig()
res.Log(l)
if err := w.WriteLine(res.String()); writeErr == nil {
writeErr = err
}
return writeErr
}
-57
View File
@@ -1,57 +0,0 @@
package nebula
import (
"bytes"
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
)
// TestSSHCommandSurface is a characterization test for the ssh command surface.
// It pins the registered command set, their descriptions, help text and flag definitions
// so that moving the registry to a transport neutral package is provably a relocation
// rather than a rewrite. Regenerate with `go test . -run TestSSHCommandSurface -update`.
func TestSSHCommandSurface(t *testing.T) {
got := renderCommandSurface(registeredCommands(t).All())
test.Golden(t, filepath.Join("testdata", "ssh_commands.golden"), got)
}
// registeredCommands returns a registry holding every command nebula serves. The
// Interface is never dereferenced: attachCommands only closes over it.
func registeredCommands(t *testing.T) *commands.Registry {
t.Helper()
l := test.NewLogger()
reg := commands.New()
attachCommands(l, config.NewC(l), reg, &Interface{})
return reg
}
func renderCommandSurface(cmds []*commands.Command) string {
lines := make([]string, 0, len(cmds))
for _, cmd := range cmds {
b := &bytes.Buffer{}
fmt.Fprintf(b, "%s\n", cmd.Name)
fmt.Fprintf(b, " short: %s\n", cmd.ShortDescription)
fmt.Fprintf(b, " help: %s\n", cmd.Help)
if cmd.Flags != nil {
fl, _ := cmd.Flags()
if fl != nil {
fb := &bytes.Buffer{}
fl.SetOutput(fb)
fl.PrintDefaults()
fmt.Fprintf(b, " flags:\n%s", fb.String())
}
}
lines = append(lines, b.String())
}
// In the order All gives, so the golden pins display order too.
return strings.Join(lines, "")
}
+9 -20
View File
@@ -7,7 +7,6 @@ import (
"path/filepath"
"testing"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
@@ -27,7 +26,7 @@ func (w *lineWriter) Write(s string) error { return w.WriteLine(s) }
func (w *lineWriter) WriteBytes(b []byte) error { return w.WriteLine(string(b)) }
func (w *lineWriter) GetWriter() io.Writer { return io.Discard }
func TestReloadCommandReportsTheResult(t *testing.T) {
func TestSSHReloadReportsTheResult(t *testing.T) {
dir := t.TempDir()
write := func(body string) {
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(body), 0o600))
@@ -44,42 +43,32 @@ func TestReloadCommandReportsTheResult(t *testing.T) {
}
})
// reload returns what the client was told. A reload that failed is a commands.Failure
// to the surface, exit 1 over ssh, with the result line as its message; one that
// merely needs a restart is not.
reload := func(failed bool) []string {
reload := func() []string {
w := &lineWriter{}
err := reloadCommand(l, c, w)
if failed {
var f *commands.Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, w.lines[len(w.lines)-1], f.Msg)
} else {
require.NoError(t, err)
}
require.NoError(t, sshReload(l, c, w))
return w.lines
}
write("a: 2\nb: 1")
assert.Equal(t, []string{"Reloading config", "Config reload complete"}, reload(false))
assert.Equal(t, []string{"Reloading config", "Config reload complete"}, reload())
fail = true
write("a: 3\nb: 1")
assert.Equal(t, []string{"Reloading config", "Config reload completed with errors, review the logs"}, reload(true))
assert.Equal(t, []string{"Reloading config", "Config reload completed with errors, review the logs"}, reload())
fail = false
write("a: 3\nb: 2")
assert.Equal(t, []string{"Reloading config", "Config reload was completed but requires a restart to fully apply, keys [b]"}, reload(false))
assert.Equal(t, []string{"Reloading config", "Config reload was completed but requires a restart to fully apply, keys [b]"}, reload())
write("a: [")
lines := reload(true)
lines := reload()
require.Len(t, lines, 2)
assert.Contains(t, lines[1], "Config reload failed, the new config did not load")
assert.Contains(t, lines[1], dir, "names the config that did not load")
}
// A client that went away still gets the reload it asked for
func TestReloadCommandReloadsWhenTheClientIsGone(t *testing.T) {
func TestSSHReloadReloadsWhenTheClientIsGone(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte("a: 1"), 0o600))
l := test.NewLogger()
@@ -89,6 +78,6 @@ func TestReloadCommandReloadsWhenTheClientIsGone(t *testing.T) {
c.RegisterReloadCallback(func(*config.C, *config.Reload) { reloaded = true })
gone := errors.New("client is gone")
require.ErrorIs(t, reloadCommand(l, c, &lineWriter{err: gone}), gone)
require.ErrorIs(t, sshReload(l, c, &lineWriter{err: gone}), gone)
assert.True(t, reloaded)
}
+161
View File
@@ -0,0 +1,161 @@
package sshd
import (
"errors"
"flag"
"fmt"
"sort"
"strings"
"github.com/armon/go-radix"
)
// CommandFlags is a function called before help or command execution to parse command line flags
// It should return a flag.FlagSet instance and a pointer to the struct that will contain parsed flags
type CommandFlags func() (*flag.FlagSet, any)
// CommandCallback is the function called when your command should execute.
// fs will be a a pointer to the struct provided by Command.Flags callback, if there was one. -h and -help are reserved
// and handled automatically for you.
// a will be any unconsumed arguments, if no Command.Flags was available this will be all the flags passed in.
// w is the writer to use when sending messages back to the client.
// If an error is returned by the callback it is logged locally, the callback should handle messaging errors to the user
// where appropriate
type CommandCallback func(fs any, a []string, w StringWriter) error
type Command struct {
Name string
ShortDescription string
Help string
Flags CommandFlags
Callback CommandCallback
}
func execCommand(c *Command, args []string, w StringWriter) error {
var (
fl *flag.FlagSet
fs any
)
if c.Flags != nil {
fl, fs = c.Flags()
if fl != nil {
// SetOutput() here in case fl.Parse dumps usage.
fl.SetOutput(w.GetWriter())
err := fl.Parse(args)
if err != nil {
// fl.Parse has dumped error information to the user via the w writer.
return err
}
args = fl.Args()
}
}
return c.Callback(fs, args, w)
}
func dumpCommands(c *radix.Tree, w StringWriter) {
err := w.WriteLine("Available commands:")
if err != nil {
return
}
cmds := make([]string, 0)
for _, l := range allCommands(c) {
cmds = append(cmds, fmt.Sprintf("%s - %s", l.Name, l.ShortDescription))
}
sort.Strings(cmds)
_ = w.Write(strings.Join(cmds, "\n") + "\n\n")
}
func lookupCommand(c *radix.Tree, sCmd string) (*Command, error) {
cmd, ok := c.Get(sCmd)
if !ok {
return nil, nil
}
command, ok := cmd.(*Command)
if !ok {
return nil, errors.New("failed to cast command")
}
return command, nil
}
func matchCommand(c *radix.Tree, cmd string) []string {
cmds := make([]string, 0)
c.WalkPrefix(cmd, func(found string, v any) bool {
cmds = append(cmds, found)
return false
})
sort.Strings(cmds)
return cmds
}
func allCommands(c *radix.Tree) []*Command {
cmds := make([]*Command, 0)
c.WalkPrefix("", func(found string, v any) bool {
cmd, ok := v.(*Command)
if ok {
cmds = append(cmds, cmd)
}
return false
})
return cmds
}
func helpCallback(commands *radix.Tree, a []string, w StringWriter) (err error) {
// Just typed help
if len(a) == 0 {
dumpCommands(commands, w)
return nil
}
// We are printing a specific commands help text
cmd, err := lookupCommand(commands, a[0])
if err != nil {
return
}
if cmd != nil {
err = w.WriteLine(fmt.Sprintf("%s - %s", cmd.Name, cmd.ShortDescription))
if err != nil {
return err
}
if cmd.Help != "" {
err = w.WriteLine(fmt.Sprintf(" %s", cmd.Help))
if err != nil {
return err
}
}
if cmd.Flags != nil {
fs, _ := cmd.Flags()
if fs != nil {
fs.SetOutput(w.GetWriter())
fs.PrintDefaults()
}
}
return nil
}
err = w.WriteLine("Command not available " + a[0])
if err != nil {
return err
}
return nil
}
func checkHelpArgs(args []string) bool {
for _, a := range args {
if a == "-h" || a == "-help" {
return true
}
}
return false
}
+23 -9
View File
@@ -10,7 +10,7 @@ import (
"sync"
"sync/atomic"
"github.com/slackhq/nebula/commands"
"github.com/armon/go-radix"
"golang.org/x/crypto/ssh"
)
@@ -29,9 +29,10 @@ type SSHServer struct {
trustedKeys map[string]map[string]bool
trustedCAs []ssh.PublicKey
// Available commands.
commands *commands.Registry
listener atomic.Pointer[net.Listener]
// List of available commands
helpCommand *Command
commands *radix.Tree
listener atomic.Pointer[net.Listener]
// runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned
runs atomic.Int32
@@ -43,14 +44,14 @@ type SSHServer struct {
ctx context.Context
}
// NewSSHServer creates a new ssh server serving the supplied command registry and
// prepares to listen. The ssh server's context is parented off the supplied ctx so
// cancelling it (e.g. on Control.Stop) tears down active sessions and closes the listener.
func NewSSHServer(ctx context.Context, l *slog.Logger, reg *commands.Registry) (*SSHServer, error) {
// NewSSHServer creates a new ssh server rigged with default commands and prepares to listen.
// The ssh server's context is parented off the supplied ctx so cancelling it
// (e.g. on Control.Stop) tears down active sessions and closes the listener.
func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
s := &SSHServer{
trustedKeys: make(map[string]map[string]bool),
l: l,
commands: reg,
commands: radix.New(),
ctx: ctx,
}
@@ -96,6 +97,14 @@ func NewSSHServer(ctx context.Context, l *slog.Logger, reg *commands.Registry) (
s.authenticate = cc.Authenticate
s.config.Store(s.newConfig())
s.RegisterCommand(&Command{
Name: "help",
ShortDescription: "prints available commands or help <command> for specific usage info",
Callback: func(a any, args []string, w StringWriter) error {
return helpCallback(s.commands, args, w)
},
})
return s, nil
}
@@ -180,6 +189,11 @@ func (s *SSHServer) AddAuthorizedKey(user, pubKey string) error {
return nil
}
// RegisterCommand adds a command that can be run by a user, by default only `help` is available
func (s *SSHServer) RegisterCommand(c *Command) {
s.commands.Insert(c.Name, c)
}
// Run begins listening and accepting connections. Each invocation derives a fresh per-Run context
// from the constructor-supplied ctx so a Stop+Run sequence (used by config reload) starts clean
// rather than carrying a permanently-cancelled context across runs.
+46 -37
View File
@@ -1,11 +1,13 @@
package sshd
import (
"errors"
"fmt"
"log/slog"
"sort"
"strings"
"github.com/slackhq/nebula/commands"
"github.com/anmitsu/go-shlex"
"github.com/armon/go-radix"
"golang.org/x/crypto/ssh"
"golang.org/x/term"
)
@@ -14,37 +16,31 @@ type session struct {
l *slog.Logger
c *ssh.ServerConn
term *term.Terminal
commands *commands.Registry
commands *radix.Tree
cancel func()
}
func NewSession(reg *commands.Registry, conn *ssh.ServerConn, chans <-chan ssh.NewChannel, cancel func(), l *slog.Logger) *session {
func NewSession(commands *radix.Tree, conn *ssh.ServerConn, chans <-chan ssh.NewChannel, cancel func(), l *slog.Logger) *session {
s := &session{
commands: reg.Clone(),
commands: radix.NewFromMap(commands.ToMap()),
l: l,
c: conn,
cancel: cancel,
}
s.commands.Register(LogoutCommand(s.Close))
s.commands.Insert("logout", &Command{
Name: "logout",
ShortDescription: "Ends the current session",
Callback: func(a any, args []string, w StringWriter) error {
s.Close()
return nil
},
})
go s.handleChannels(chans)
return s
}
// LogoutCommand is the logout command a session registers on its clone of the registry,
// with end as what it does. Exported so a test can serve what a session serves.
func LogoutCommand(end func()) *commands.Command {
return &commands.Command{
Name: commands.Logout,
ShortDescription: "Ends the current session",
Callback: func(fs any, a []string, w commands.StringWriter) error {
end()
return nil
},
}
}
func (s *session) handleChannels(chans <-chan ssh.NewChannel) {
defer s.Close()
for newChannel := range chans {
@@ -91,13 +87,9 @@ func (s *session) handleRequests(in <-chan *ssh.Request, channel ssh.Channel) {
}
req.Reply(true, nil)
err := s.dispatchCommand(payload.Value, commands.NewStringWriter(channel))
s.dispatchCommand(payload.Value, &stringWriter{channel})
// A command that failed exits 1, so `ssh host cmd` callers can tell.
status := struct{ Status uint32 }{uint32(0)}
if err != nil {
status.Status = 1
}
channel.SendRequest("exit-status", false, ssh.Marshal(status))
channel.Close()
return
@@ -119,11 +111,12 @@ func (s *session) createTerm(channel ssh.Channel) *term.Terminal {
term.AutoCompleteCallback = func(line string, pos int, key rune) (newLine string, newPos int, ok bool) {
// key 9 is tab
if key == 9 {
cmds := s.commands.Match(line)
cmds := matchCommand(s.commands, line)
if len(cmds) == 1 {
return cmds[0] + " ", len(cmds[0]) + 1, true
}
sort.Strings(cmds)
term.Write([]byte(strings.Join(cmds, "\n") + "\n\n"))
}
@@ -135,7 +128,7 @@ func (s *session) createTerm(channel ssh.Channel) *term.Terminal {
}
func (s *session) handleInput() {
w := commands.NewStringWriter(s.term)
w := &stringWriter{w: s.term}
for {
line, err := s.term.ReadLine()
if err != nil {
@@ -146,20 +139,36 @@ func (s *session) handleInput() {
}
}
// dispatchCommand runs line and returns the command's error, which it has logged unless
// the command already told the user everything: the log line is for the operator reading
// the daemon's log, who learns nothing from a failure the user caused.
func (s *session) dispatchCommand(line string, w commands.StringWriter) error {
err := s.commands.Dispatch(line, w)
if err == nil {
return nil
func (s *session) dispatchCommand(line string, w StringWriter) {
args, err := shlex.Split(line, true)
if err != nil {
return
}
var f *commands.Failure
if !errors.As(err, &f) {
s.l.Debug("Command returned an error", "line", line, "error", err)
if len(args) == 0 {
dumpCommands(s.commands, w)
return
}
return err
c, err := lookupCommand(s.commands, args[0])
if err != nil {
return
}
if c == nil {
err := w.WriteLine(fmt.Sprintf("did not understand: %s", line))
_ = err
dumpCommands(s.commands, w)
return
}
if checkHelpArgs(args) {
s.dispatchCommand(fmt.Sprintf("%s %s", "help", c.Name), w)
return
}
_ = execCommand(c, args[1:], w)
}
func (s *session) Close() {
+1 -8
View File
@@ -1,9 +1,7 @@
package commands
package sshd
import "io"
// StringWriter is how a command talks back to whoever invoked it: one stream, which the
// ssh surface points at its channel.
type StringWriter interface {
WriteLine(string) error
Write(string) error
@@ -15,11 +13,6 @@ type stringWriter struct {
w io.Writer
}
// NewStringWriter returns a StringWriter that writes to w.
func NewStringWriter(w io.Writer) StringWriter {
return &stringWriter{w: w}
}
func (w *stringWriter) WriteLine(s string) error {
return w.Write(s + "\n")
}
+1 -42
View File
@@ -6,16 +6,13 @@ import (
"crypto/rand"
"crypto/rsa"
"encoding/pem"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
@@ -26,7 +23,6 @@ import (
type sshdReloadHarness struct {
c *config.C
reg *commands.Registry
server *sshd.SSHServer
keyFile string
signer ssh.Signer
@@ -48,8 +44,7 @@ func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
h.c = config.NewC(l)
require.NoError(t, h.c.LoadString(h.config()))
h.reg = commands.New()
h.server, err = sshd.NewSSHServer(t.Context(), l, h.reg)
h.server, err = sshd.NewSSHServer(t.Context(), l)
require.NoError(t, err)
wireSSHReload(l, h.server, h.c)
run, err := configSSH(l, h.server, h.c, nil)
@@ -243,39 +238,3 @@ func TestSSHDReload_SkippedEntriesFail(t *testing.T) {
})
}
}
// TestSSHExecExitStatusReflectsTheCommand: an exec request exits 1 when the command
// returned an error and 0 otherwise, with the command's output either way.
func TestSSHExecExitStatusReflectsTheCommand(t *testing.T) {
h := newSSHDReloadHarness(t)
h.reg.Register(&commands.Command{
Name: "fails",
Callback: func(fs any, a []string, w commands.StringWriter) error {
_ = w.WriteLine("nope")
return errors.New("failed")
},
})
client, _ := h.connect(t)
sess, err := client.NewSession()
require.NoError(t, err)
out, err := sess.CombinedOutput("fails")
var exit *ssh.ExitError
require.ErrorAs(t, err, &exit)
assert.Equal(t, 1, exit.ExitStatus())
assert.Equal(t, "nope\n", string(out))
sess, err = client.NewSession()
require.NoError(t, err)
out, err = sess.CombinedOutput("help fails")
require.NoError(t, err)
assert.Equal(t, "fails - \n", string(out))
// A word that is not a command is a failure too, with the command list as before.
sess, err = client.NewSession()
require.NoError(t, err)
out, err = sess.CombinedOutput("nope")
require.ErrorAs(t, err, &exit)
assert.Equal(t, 1, exit.ExitStatus())
assert.True(t, strings.HasPrefix(string(out), "did not understand: nope\nAvailable commands:\n"), string(out))
}
-29
View File
@@ -1,29 +0,0 @@
package test
import (
"flag"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Update is the -update flag: Golden rewrites its file instead of comparing against it.
var Update = flag.Bool("update", false, "rewrite golden files instead of comparing against them")
// Golden compares got with the file at path, or rewrites the file when -update is set.
func Golden(t *testing.T, path, got string) {
t.Helper()
if *Update {
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755))
require.NoError(t, os.WriteFile(path, []byte(got), 0644))
return
}
want, err := os.ReadFile(path)
require.NoError(t, err, "missing golden file, run with -update")
assert.Equal(t, string(want), got)
}
-94
View File
@@ -1,94 +0,0 @@
change-remote
short: Changes the remote address used in the tunnel for the provided vpn addr
help:
flags:
-address string
The new remote address, ip:port
close-tunnel
short: Closes a tunnel for the provided vpn addr
help:
flags:
-local-only
Disables notifying the remote that the tunnel is shutting down
create-tunnel
short: Creates a tunnel for the provided vpn address
help: The lighthouses will be queried for real addresses but you can provide one as well.
flags:
-address string
Optionally provide a real remote address, ip:port
device-info
short: Prints information about the network device.
help:
flags:
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
help
short: prints available commands or help <command> for specific usage info
help:
list-hostmap
short: List all known previously connected hosts
help:
flags:
-by-index
gets all hosts in the hostmap from the index table
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
list-lighthouse-addrmap
short: List all lighthouse map entries
help:
flags:
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
list-pending-hostmap
short: List all handshaking hosts
help:
flags:
-by-index
gets all hosts in the hostmap from the index table
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
log-format
short: Gets or sets the current log format
help:
log-level
short: Gets or sets the current log level
help:
print-cert
short: Prints the current certificate being used or the certificate for the provided vpn addr
help:
flags:
-json
outputs as json
-pretty
pretty prints json, assumes -json
-raw
raw prints the PEM encoded certificate, not compatible with -json or -pretty
print-relays
short: Prints json details about all relay info
help:
flags:
-pretty
pretty prints json
print-tunnel
short: Prints json details about a tunnel for the provided vpn addr
help:
flags:
-pretty
pretty prints json
query-lighthouse
short: Query the lighthouses for the provided vpn address
help: This command is asynchronous. Only currently known udp addresses will be printed.
reload
short: Reloads configuration from disk, same as sending HUP to the process
help:
version
short: Prints the currently running version of nebula
help:
+5 -2
View File
@@ -147,7 +147,10 @@ func (tw *TimerWheel[T]) Purge() (T, bool) {
tw.expired.Tail = nil
}
// Clear out the items references
// Clear out the items references, a cached item must not keep its value alive
v := ti.Item
var zero T
ti.Item = zero
ti.Next = nil
// Maybe cache it for later
@@ -157,7 +160,7 @@ func (tw *TimerWheel[T]) Purge() (T, bool) {
tw.itemsCached++
}
return ti.Item, true
return v, true
}
// findWheel find the next position in the wheel for the provided timeout given the current tick
+3 -1
View File
@@ -7,6 +7,7 @@ import (
"github.com/slackhq/nebula/firewall"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewTimerWheel(t *testing.T) {
@@ -150,7 +151,8 @@ func TestTimerWheel_Purge(t *testing.T) {
assert.Equal(t, 4, tw.itemsCached)
ci := tw.itemCache
for range 4 {
assert.NotNil(t, ci)
require.NotNil(t, ci)
assert.Zero(t, ci.Item)
ci = ci.Next
}
assert.Nil(t, ci)
+2 -3
View File
@@ -41,9 +41,8 @@ type Conn interface {
//
// Returns the number of packets successfully written. A destination the kernel rejects costs only
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
// failed, when not nil, is called with the index of every packet a destination cost us and why.
// Not safe for concurrent use on the same Conn.
WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error)
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
// ReloadConfig applies the listen settings that can change, r is nil at startup
ReloadConfig(c *config.C, r *config.Reload)
SupportsMultipleReaders() bool
@@ -67,7 +66,7 @@ func (NoopConn) SupportsMultipleReaders() bool {
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
return nil
}
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort, _ func(int, error)) (int, error) {
func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) {
return len(bufs), nil
}
func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) {
+1 -4
View File
@@ -139,7 +139,7 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
}
}
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
// silently drops the rest of a burst (linux blocks instead). Poll for
@@ -150,9 +150,6 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+1 -4
View File
@@ -44,7 +44,7 @@ func (u *GenericConn) WriteTo(b []byte, addr netip.AddrPort) error {
return err
}
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
written := 0
for i, b := range bufs {
@@ -52,9 +52,6 @@ func (u *GenericConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed f
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+2 -2
View File
@@ -358,8 +358,8 @@ func sendto(fd int, b []byte, addr netip.AddrPort, isV4 bool) error {
// WriteBatch sends bufs via sendmmsg(2), coalescing same-destination runs into UDP-GSO superpackets when supported.
// See batchWriter in udp_linux_writebatch.go for the mechanics.
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
return u.bw.WriteBatch(bufs, addrs, failed)
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
return u.bw.WriteBatch(bufs, addrs)
}
// writeSockaddr encodes addr into buf (which must be at least SizeofSockaddrInet6 bytes).
+9 -38
View File
@@ -3,7 +3,6 @@
package udp
import (
"errors"
"fmt"
"log/slog"
"net"
@@ -97,22 +96,13 @@ func TestWriteBatchBadFamilyDeliversOthers(t *testing.T) {
bufs := [][]byte{[]byte("AAA"), []byte("BBB"), []byte("CCC")}
addrs := []netip.AddrPort{good, bad, good}
var failed []int
n, err := sender.WriteBatch(bufs, addrs, func(i int, err error) {
if !IsUnreachable(err) {
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
}
failed = append(failed, i)
})
n, err := sender.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch returned error, want nil (bad dest should be isolated): %v", err)
}
if n != 2 {
t.Errorf("WriteBatch wrote %d packets, want 2 of 3 (the bad-family dest is the only casualty)", n)
}
if want := []int{1}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
got := map[string]bool{}
rx.SetReadDeadline(time.Now().Add(2 * time.Second))
@@ -164,7 +154,7 @@ func TestWriteBatchUnreachableDestDeliversOthers(t *testing.T) {
addrs := []netip.AddrPort{good, good, bad, good, good}
// The bad destination is reported, but only after every other packet has been attempted.
if _, err := sender.WriteBatch(bufs, addrs, nil); err == nil {
if _, err := sender.WriteBatch(bufs, addrs); err == nil {
t.Log("WriteBatch returned nil; kernel accepted the reserved address, delivery assertions still apply")
}
@@ -386,7 +376,7 @@ func TestWriteBatchPartialSendRewind(t *testing.T) {
return accept, nil
}
written, err := w.WriteBatch(bufs, addrs, nil)
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -439,22 +429,13 @@ func TestWriteBatchSkipUnroutableRunAccounting(t *testing.T) {
return accept, nil
}
var failed []int
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
if !IsUnreachable(err) {
t.Errorf("packet %d failed with %v, want an unreachable error", i, err)
}
failed = append(failed, i)
})
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
if written != 4 {
t.Errorf("written = %d, want 4 (the unroutable run is the only casualty)", written)
}
if want := []int{2}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
wantTags := []byte{1, 2, 4, 5}
if len(wire) != len(wantTags) {
t.Fatalf("wire got %d packets, want %d (dup or loss around the skip)", len(wire), len(wantTags))
@@ -505,23 +486,13 @@ func TestWriteBatchMidChunkRejectResumes(t *testing.T) {
}
}
var failed []int
written, err := w.WriteBatch(bufs, addrs, func(i int, err error) {
if !errors.Is(err, unix.EPERM) {
t.Errorf("packet %d failed with %v, want EPERM", i, err)
}
failed = append(failed, i)
})
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
if written != 3 {
t.Errorf("written = %d, want 3 (B's rejected run is the only casualty)", written)
}
// Every packet in B's run is reported, not just the entry
if want := []int{2, 3}; !slices.Equal(failed, want) {
t.Errorf("failed reported %v, want %v", failed, want)
}
wantTags := []byte{1, 2, 5}
if len(wire) != len(wantTags) {
t.Fatalf("wire got %d packets, want %d (dup or loss around the mid-chunk reject)", len(wire), len(wantTags))
@@ -572,7 +543,7 @@ func TestWriteBatchMidChunkEIODisablesGSOWithoutDup(t *testing.T) {
}
}
written, err := w.WriteBatch(bufs, addrs, nil)
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -600,7 +571,7 @@ func TestWriteBatchZeroProgress(t *testing.T) {
w.sendFn = func(start, n int) (int, error) { return 0, nil }
bufs := [][]byte{make([]byte, 100)}
addrs := []netip.AddrPort{netip.MustParseAddrPort("127.0.0.1:4242")}
if _, err := w.WriteBatch(bufs, addrs, nil); err == nil {
if _, err := w.WriteBatch(bufs, addrs); err == nil {
t.Fatal("WriteBatch = nil error on zero progress, want error")
}
}
@@ -626,7 +597,7 @@ func TestWriteBatchEIODisablesGSOAndReplays(t *testing.T) {
return n, nil
}
written, err := w.WriteBatch(bufs, addrs, nil)
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -703,7 +674,7 @@ func TestGSOEngagesOnLoopback(t *testing.T) {
addrs[i] = dst
}
written, err := sc.WriteBatch(bufs, addrs, nil)
written, err := sc.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
+2 -2
View File
@@ -86,7 +86,7 @@ func TestWriteBatchOffloadsDisabledScripted(t *testing.T) {
addrs[i] = dst
}
written, err := w.WriteBatch(bufs, addrs, nil)
written, err := w.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
@@ -163,7 +163,7 @@ func TestOffloadsDisabledOnLoopback(t *testing.T) {
addrs[i] = dst
}
written, err := sc.WriteBatch(bufs, addrs, nil)
written, err := sc.WriteBatch(bufs, addrs)
if err != nil {
t.Fatalf("WriteBatch: %v", err)
}
+2 -14
View File
@@ -190,8 +190,8 @@ func parseRelease(r string) (major, minor int) {
// its packets are dropped and the rest of the chunk resumes in place.
//
// Returns the number of packets sent. An error means the call itself failed.
// A short count means some destinations were undeliverable, failed hears about each of their packets.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
// A short count means some destinations were undeliverable.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
if len(bufs) != len(addrs) {
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
}
@@ -233,7 +233,6 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed f
if w.l.Enabled(context.Background(), slog.LevelDebug) {
w.l.Debug("skipping unroutable batch destination", "udpAddr", addrs[i], "packets", runLen, "error", err)
}
reportFailed(failed, i, i+runLen, err)
i += runLen
continue
}
@@ -301,23 +300,12 @@ func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed f
"gso", w.gsoSupported,
)
}
reportFailed(failed, w.entryEnd[done]-w.entryPkts[done], w.entryEnd[done], serr)
done++
}
}
return written, nil
}
// reportFailed tells failed, if anyone is listening, that every packet in bufs[start:end] was lost to err
func reportFailed(failed func(i int, err error), start, end int, err error) {
if failed == nil {
return
}
for i := start; i < end; i++ {
failed(i, err)
}
}
// planRun returns the length of the run starting at start and its segment
// size (len(bufs[start])). A run of length 1 carries no UDP_SEGMENT cmsg
// and is sent as a plain datagram; without GSO support planRun always returns 1.
+2 -2
View File
@@ -80,11 +80,11 @@ func TestWriteBatchNoAllocs(t *testing.T) {
var werr error
// Warm-up outside the measured runs.
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
t.Fatalf("WriteBatch warm-up: %v", err)
}
allocs := testing.AllocsPerRun(100, func() {
if _, err := tx.WriteBatch(bufs, addrs, nil); err != nil {
if _, err := tx.WriteBatch(bufs, addrs); err != nil {
werr = err
}
})
+1 -4
View File
@@ -319,7 +319,7 @@ func (u *RIOConn) WriteTo(buf []byte, ip netip.AddrPort) error {
return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
}
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
written := 0
for i, b := range bufs {
@@ -327,9 +327,6 @@ func (u *RIOConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
+7 -64
View File
@@ -6,7 +6,6 @@ import (
"context"
"io"
"log/slog"
"net"
"net/netip"
"os"
"sync"
@@ -64,19 +63,10 @@ func acquirePacket() *Packet {
}
}
// testerAddrs is where a TesterConn sends from. A single address host sends everything from addr, whatever the
// destination. A dual stack host sends from the address matching the destination's family and has no route at all
// for a family it holds no address in.
type testerAddrs struct {
addr netip.AddrPort
v4, v6 netip.AddrPort
dual bool
}
type TesterConn struct {
// addrs is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
addrs atomic.Pointer[testerAddrs]
addr atomic.Pointer[netip.AddrPort]
RxPackets chan *Packet // Packets to receive into nebula
TxPackets chan *Packet // Packets transmitted outside by nebula
@@ -104,53 +94,14 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
return c, nil
}
// GetAddr returns the underlay address this conn currently sends from. For a dual stack conn that is the IPv4
// address, or the IPv6 one when there is no IPv4 service.
// GetAddr returns the underlay address this conn currently sends from.
func (u *TesterConn) GetAddr() netip.AddrPort {
return u.addrs.Load().addr
return *u.addr.Load()
}
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
func (u *TesterConn) SetAddr(addr netip.AddrPort) {
u.addrs.Store(&testerAddrs{addr: addr})
}
// SetDualStackAddrs gives this conn one underlay address per family, standing in for a host whose routing table picks
// the source by destination. Pass an invalid AddrPort for a family the network does not offer, sends to that family
// then fail like they would on a host with no route for it.
func (u *TesterConn) SetDualStackAddrs(v4, v6 netip.AddrPort) {
a := &testerAddrs{v4: v4, v6: v6, dual: true}
if v4.IsValid() {
a.addr = v4
} else {
a.addr = v6
}
u.addrs.Store(a)
}
// GetAddrFor returns the underlay address a send to dst leaves from, GetAddr when there is no route to dst
func (u *TesterConn) GetAddrFor(dst netip.AddrPort) netip.AddrPort {
if src, err := u.sourceFor(dst); err == nil {
return src
}
return u.GetAddr()
}
// sourceFor returns the address a send to dst leaves from
func (u *TesterConn) sourceFor(dst netip.AddrPort) (netip.AddrPort, error) {
a := u.addrs.Load()
if !a.dual {
return a.addr, nil
}
src := a.v6
if dst.Addr().Unmap().Is4() {
src = a.v4
}
if !src.IsValid() {
return netip.AddrPort{}, &net.OpError{Op: "write", Net: "udp", Addr: net.UDPAddrFromAddrPort(dst), Err: errNetUnreachable}
}
return src, nil
u.addr.Store(&addr)
}
// Send will place a UdpPacket onto the receive queue for nebula to consume
@@ -202,11 +153,6 @@ func (u *TesterConn) Get(block bool) *Packet {
//********************************************************************************************************************//
func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
from, err := u.sourceFor(addr)
if err != nil {
return err
}
p := acquirePacket()
if cap(p.Data) < len(b) {
p.Data = make([]byte, len(b))
@@ -214,7 +160,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
p.Data = p.Data[:len(b)]
}
copy(p.Data, b)
p.From = from
p.From = u.GetAddr()
p.To = addr
select {
case <-u.done:
@@ -224,16 +170,13 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
return nil
}
}
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, failed func(i int, err error)) (int, error) {
func (u *TesterConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
written := 0
for i, b := range bufs {
if err := u.WriteTo(b, addrs[i]); err == nil {
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
if failed != nil {
failed(i, err)
}
}
}
return written, nil
-22
View File
@@ -1,22 +0,0 @@
//go:build !windows
package udp
import (
"errors"
"syscall"
)
// errNetUnreachable is the error this platform's sockets give for a destination with no route
var errNetUnreachable error = syscall.ENETUNREACH
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
// to an IPv6 remote into one of these.
func IsUnreachable(err error) bool {
return errors.Is(err, syscall.ENETUNREACH) ||
errors.Is(err, syscall.EHOSTUNREACH) ||
errors.Is(err, syscall.EADDRNOTAVAIL) ||
errors.Is(err, syscall.EAFNOSUPPORT) ||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
}
-37
View File
@@ -1,37 +0,0 @@
//go:build !windows
package udp
import (
"net"
"syscall"
"testing"
)
func TestIsUnreachable(t *testing.T) {
wrap := func(err error) error { return &net.OpError{Op: "write", Net: "udp", Err: err} }
for _, err := range []error{
wrap(syscall.ENETUNREACH),
wrap(syscall.EHOSTUNREACH),
wrap(syscall.EADDRNOTAVAIL),
wrap(syscall.EAFNOSUPPORT),
ErrInvalidIPv6RemoteForSocket,
} {
if !IsUnreachable(err) {
t.Errorf("IsUnreachable(%v) = false, want true", err)
}
}
for _, err := range []error{
nil,
wrap(syscall.EPERM),
wrap(syscall.ENOBUFS),
wrap(syscall.EMSGSIZE),
net.ErrClosed,
} {
if IsUnreachable(err) {
t.Errorf("IsUnreachable(%v) = true, want false", err)
}
}
}
-21
View File
@@ -1,21 +0,0 @@
package udp
import (
"errors"
"golang.org/x/sys/windows"
)
// errNetUnreachable is the error this platform's sockets give for a destination with no route
var errNetUnreachable error = windows.WSAENETUNREACH
// IsUnreachable reports whether a send failed because this host has no way to reach the destination from where it
// is now, rather than for something transient. Moving from a dual stack network to one without IPv6 turns every send
// to an IPv6 remote into one of these.
func IsUnreachable(err error) bool {
return errors.Is(err, windows.WSAENETUNREACH) ||
errors.Is(err, windows.WSAEHOSTUNREACH) ||
errors.Is(err, windows.WSAEADDRNOTAVAIL) ||
errors.Is(err, windows.WSAEAFNOSUPPORT) ||
errors.Is(err, ErrInvalidIPv6RemoteForSocket)
}