Compare commits

..
53 changed files with 1447 additions and 3997 deletions

No files matched your search

-2
View File
@@ -1,2 +0,0 @@
# Goldens are compared byte for byte, so they must come out of git the way they went in.
*.golden -text
-710
View File
@@ -1,710 +0,0 @@
package nebula
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"log/slog"
"maps"
"net/netip"
"sort"
"strings"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/logging"
)
// The commands nebula serves. attachCommands builds the one registry every surface
// reads. See the commands package for what a Command is.
type listHostMapFlags struct {
Json bool
Pretty bool
ByIndex bool
}
// listHostMapFlagSet is the flag set list-hostmap and list-pending-hostmap share.
func listHostMapFlagSet() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := listHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table")
return fl, &s
}
type printCertFlags struct {
Json bool
Pretty bool
Raw bool
}
type printTunnelFlags struct {
Pretty bool
}
type changeRemoteFlags struct {
Address string
}
type closeTunnelFlags struct {
LocalOnly bool
}
type createTunnelFlags struct {
Address string
}
type deviceInfoFlags struct {
Json bool
Pretty bool
}
func attachCommands(l *slog.Logger, c *config.C, reg *commands.Registry, f *Interface) {
reg.Register(&commands.Command{
Name: commands.ListHostmap,
ShortDescription: "List all known previously connected hosts",
Flags: listHostMapFlagSet,
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listHostMapCommand(f.hostMap, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.ListPendingHostmap,
ShortDescription: "List all handshaking hosts",
Flags: listHostMapFlagSet,
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listHostMapCommand(f.handshakeManager, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.ListLighthouseAddrmap,
ShortDescription: "List all lighthouse map entries",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := listHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return listLighthouseAddrmapCommand(f.lightHouse, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.Reload,
ShortDescription: "Reloads configuration from disk, same as sending HUP to the process",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return reloadCommand(l, c, w)
},
})
reg.Register(&commands.Command{
Name: commands.LogLevel,
ShortDescription: "Gets or sets the current log level",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return logLevelCommand(l, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.LogFormat,
ShortDescription: "Gets or sets the current log format",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return logFormatCommand(l, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.Version,
ShortDescription: "Prints the currently running version of nebula",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return versionCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.DeviceInfo,
ShortDescription: "Prints information about the network device.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := deviceInfoFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return deviceInfoCommand(f, fs, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintCert,
ShortDescription: "Prints the current certificate being used or the certificate for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printCertFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.Raw, "raw", false, "raw prints the PEM encoded certificate, not compatible with -json or -pretty")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printCertCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintTunnel,
ShortDescription: "Prints json details about a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.PrintRelays,
ShortDescription: "Prints json details about all relay info",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := printTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return printRelaysCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.ChangeRemote,
ShortDescription: "Changes the remote address used in the tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := changeRemoteFlags{}
fl.StringVar(&s.Address, "address", "", "The new remote address, ip:port")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return changeRemoteCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.CloseTunnel,
ShortDescription: "Closes a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := closeTunnelFlags{}
fl.BoolVar(&s.LocalOnly, "local-only", false, "Disables notifying the remote that the tunnel is shutting down")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return closeTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.CreateTunnel,
ShortDescription: "Creates a tunnel for the provided vpn address",
Help: "The lighthouses will be queried for real addresses but you can provide one as well.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := createTunnelFlags{}
fl.StringVar(&s.Address, "address", "", "Optionally provide a real remote address, ip:port ")
return fl, &s
},
Callback: func(fs any, a []string, w commands.StringWriter) error {
return createTunnelCommand(f, fs, a, w)
},
})
reg.Register(&commands.Command{
Name: commands.QueryLighthouse,
ShortDescription: "Query the lighthouses for the provided vpn address",
Help: "This command is asynchronous. Only currently known udp addresses will be printed.",
Callback: func(fs any, a []string, w commands.StringWriter) error {
return queryLighthouseCommand(f, fs, a, w)
},
})
}
func listHostMapCommand(hl controlHostLister, a any, w commands.StringWriter) error {
fs, ok := a.(*listHostMapFlags)
if !ok {
return fmt.Errorf("internal error: expected *listHostMapFlags but was %T", a)
}
var hm []ControlHostInfo
if fs.ByIndex {
hm = listHostMapIndexes(hl)
} else {
hm = listHostMapHosts(hl)
}
sort.Slice(hm, func(i, j int) bool {
return hm[i].VpnAddrs[0].Compare(hm[j].VpnAddrs[0]) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
if err := js.Encode(hm); err != nil {
return err
}
} else {
for _, v := range hm {
err := w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddrs, v.RemoteAddrs))
if err != nil {
return err
}
}
}
return nil
}
func listLighthouseAddrmapCommand(lightHouse *LightHouse, a any, w commands.StringWriter) error {
fs, ok := a.(*listHostMapFlags)
if !ok {
return fmt.Errorf("internal error: expected *listHostMapFlags but was %T", a)
}
type lighthouseInfo struct {
VpnAddr string `json:"vpnAddr"`
Addrs *CacheMap `json:"addrs"`
}
lightHouse.RLock()
addrMap := make([]lighthouseInfo, len(lightHouse.addrMap))
x := 0
for k, v := range lightHouse.addrMap {
addrMap[x] = lighthouseInfo{
VpnAddr: k.String(),
Addrs: v.CopyCache(),
}
x++
}
lightHouse.RUnlock()
sort.Slice(addrMap, func(i, j int) bool {
return strings.Compare(addrMap[i].VpnAddr, addrMap[j].VpnAddr) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
if err := js.Encode(addrMap); err != nil {
return err
}
} else {
for _, v := range addrMap {
b, err := json.Marshal(v.Addrs)
if err != nil {
return err
}
err = w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddr, string(b)))
if err != nil {
return err
}
}
}
return nil
}
// parseVpnAddr parses the vpn address a command takes as its first operand, failing to w
// when there is none or it does not parse. noun is how the command's messages name it,
// "vpn address" or "vpn addr", which the output golden pins per command.
func parseVpnAddr(a []string, noun string, w commands.StringWriter) (netip.Addr, error) {
if len(a) == 0 {
return netip.Addr{}, commands.Fail(w, "No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return netip.Addr{}, commands.Fail(w, fmt.Sprintf("The provided %s could not be parsed: %s", noun, a[0]))
}
return vpnAddr, nil
}
func versionCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
return w.WriteLine(ifce.version)
}
func queryLighthouseCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
var cm *CacheMap
rl := ifce.lightHouse.Query(vpnAddr)
if rl != nil {
cm = rl.CopyCache()
}
return json.NewEncoder(w.GetWriter()).Encode(cm)
}
func closeTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*closeTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *closeTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
if !flags.LocalOnly {
ifce.sendCloseTunnel(hostInfo)
}
ifce.closeTunnel(hostInfo)
return w.WriteLine("Closed")
}
func createTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*createTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *createTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return commands.Fail(w, "Tunnel already exists")
}
hostInfo = ifce.handshakeManager.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return commands.Fail(w, "Tunnel already handshaking")
}
var addr netip.AddrPort
if flags.Address != "" {
addr, err = netip.ParseAddrPort(flags.Address)
if err != nil {
return commands.Fail(w, "Address could not be parsed")
}
}
// Set the remote from the callback, which StartHandshake runs under the handshake manager lock
// before the hostinfo reaches the handshake loop or the static-host trigger, so the address is
// in the RemoteList before the first handshake is sent.
ifce.handshakeManager.StartHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
if addr.IsValid() {
hh.hostinfo.SetRemote(addr)
}
})
return w.WriteLine("Created")
}
func changeRemoteCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
flags, ok := fs.(*changeRemoteFlags)
if !ok {
return fmt.Errorf("internal error: expected *changeRemoteFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn address", w)
if err != nil {
return err
}
if flags.Address == "" {
return commands.Fail(w, "No address was provided")
}
addr, err := netip.ParseAddrPort(flags.Address)
if err != nil {
return commands.Fail(w, "Address could not be parsed")
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
hostInfo.SetRemote(addr)
return w.WriteLine("Changed")
}
func logLevelCommand(l *slog.Logger, fs any, a []string, w commands.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level
SetLevel(slog.Level)
})
if !ok {
return commands.Fail(w, "Log level is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
level, err := logging.ParseLevel(strings.ToLower(a[0]))
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unknown log level %s. Possible log levels: trace, debug, info, warn, error", a))
}
ctrl.SetLevel(level)
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
func logFormatCommand(l *slog.Logger, fs any, a []string, w commands.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetFormat() string
SetFormat(string) error
})
if !ok {
return commands.Fail(w, "Log format is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
if err := ctrl.SetFormat(strings.ToLower(a[0])); err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to set log format: %s", err))
}
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
func printCertCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printCertFlags)
if !ok {
return fmt.Errorf("internal error: expected *printCertFlags but was %T", fs)
}
cert := ifce.pki.getCertState().GetDefaultCertificate()
if len(a) > 0 {
vpnAddr, err := parseVpnAddr(a, "vpn addr", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
cert = hostInfo.GetCert().Certificate
}
if args.Json || args.Pretty {
b, err := cert.MarshalJSON()
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
if args.Pretty {
buf := new(bytes.Buffer)
if err := json.Indent(buf, b, "", " "); err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
b = buf.Bytes()
}
return w.WriteBytes(b)
}
if args.Raw {
b, err := cert.MarshalPEM()
if err != nil {
return commands.Fail(w, fmt.Sprintf("Unable to marshal the certificate: %s", err))
}
return w.WriteBytes(b)
}
return w.WriteLine(cert.String())
}
func printRelaysCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *printTunnelFlags but was %T", fs)
}
ifce.hostMap.RLock()
relays := maps.Clone(ifce.hostMap.Relays)
ifce.hostMap.RUnlock()
type RelayFor struct {
Error string `json:",omitempty"`
Type string
State string
PeerAddr netip.Addr
LocalIndex uint32
RemoteIndex uint32
RelayedThrough []netip.Addr
}
type RelayOutput struct {
NebulaAddr netip.Addr
RelayForAddrs []RelayFor
}
type CmdOutput struct {
Relays []*RelayOutput
}
co := CmdOutput{}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
for k, v := range relays {
ro := RelayOutput{NebulaAddr: v.vpnAddrs[0]}
co.Relays = append(co.Relays, &ro)
relayHI := ifce.hostMap.QueryVpnAddr(v.vpnAddrs[0])
if relayHI == nil {
ro.RelayForAddrs = append(ro.RelayForAddrs, RelayFor{Error: "could not find hostinfo"})
continue
}
for _, vpnAddr := range relayHI.relayState.CopyRelayForIps() {
rf := RelayFor{}
r, ok := relayHI.relayState.GetRelayForByAddr(vpnAddr)
if ok {
t := ""
switch r.Type {
case ForwardingType:
t = "forwarding"
case TerminalType:
t = "terminal"
default:
t = "unknown"
}
s := ""
switch r.State {
case Requested:
s = "requested"
case Established:
s = "established"
default:
s = "unknown"
}
rf.LocalIndex = r.LocalIndex
rf.RemoteIndex = r.RemoteIndex
rf.PeerAddr = r.PeerAddr
rf.Type = t
rf.State = s
if rf.LocalIndex != k {
rf.Error = fmt.Sprintf("hostmap LocalIndex '%v' does not match RelayState LocalIndex", k)
}
}
relayedHI := ifce.hostMap.QueryVpnAddr(vpnAddr)
if relayedHI != nil {
rf.RelayedThrough = append(rf.RelayedThrough, relayedHI.relayState.CopyRelayIps()...)
}
ro.RelayForAddrs = append(ro.RelayForAddrs, rf)
}
}
return enc.Encode(co)
}
func printTunnelCommand(ifce *Interface, fs any, a []string, w commands.StringWriter) error {
args, ok := fs.(*printTunnelFlags)
if !ok {
return fmt.Errorf("internal error: expected *printTunnelFlags but was %T", fs)
}
vpnAddr, err := parseVpnAddr(a, "vpn addr", w)
if err != nil {
return err
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return commands.Fail(w, fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.GetPreferredRanges()))
}
func deviceInfoCommand(ifce *Interface, fs any, w commands.StringWriter) error {
flags, ok := fs.(*deviceInfoFlags)
if !ok {
return fmt.Errorf("internal error: expected flags to be deviceInfoFlags but was %+v", fs)
}
data := struct {
Name string `json:"name"`
Cidr []netip.Prefix `json:"cidr"`
}{
Name: ifce.inside.Name(),
Cidr: make([]netip.Prefix, len(ifce.inside.Networks())),
}
copy(data.Cidr, ifce.inside.Networks())
if flags.Json || flags.Pretty {
js := json.NewEncoder(w.GetWriter())
if flags.Pretty {
js.SetIndent("", " ")
}
return js.Encode(data)
} else {
return w.WriteLine(fmt.Sprintf("name=%v cidr=%v", data.Name, data.Cidr))
}
}
// reloadCommand reloads the config and writes the result to the client. It reloads even if the client has gone away
func reloadCommand(l *slog.Logger, c *config.C, w commands.StringWriter) error {
writeErr := w.WriteLine("Reloading config")
res := c.ReloadConfig()
res.Log(l)
if err := w.WriteLine(res.String()); writeErr == nil {
writeErr = err
}
if writeErr != nil {
return writeErr
}
if res.Err != nil || res.Failed {
return &commands.Failure{Msg: res.String()}
}
return nil
}
-83
View File
@@ -1,83 +0,0 @@
package commands
import (
"flag"
)
// CommandFlags is called before help or command execution to parse command line flags.
// It should return a flag.FlagSet instance and a pointer to the struct that will contain parsed flags
type CommandFlags func() (*flag.FlagSet, any)
// CommandCallback is called when your command should execute.
// fs will be a pointer to the struct provided by Command.Flags callback, if there was one. -h and -help are reserved
// and handled automatically for you.
// a will be any unconsumed arguments, if no Command.Flags was available this will be all the flags passed in.
// w is the writer to use when sending messages back to the client.
// An error returned by the callback is handed back to whoever dispatched the command, which reports a failure: exit
// status 1 over ssh. The error is not shown to the user. For a failure the user caused or can act on, write the
// message and return it as a *Failure, which Fail does in one step; the surface then reports the failure without
// logging what the user was already told. Any other error is logged by the surface as well.
type CommandCallback func(fs any, a []string, w StringWriter) error
type Command struct {
Name Name
ShortDescription string
Help string
Flags CommandFlags
Callback CommandCallback
}
// Failure is the error a command returns for a failure it has already explained to the user:
// the message it wrote is the whole story. A surface reports it as a failure and does not
// log it.
type Failure struct {
Msg string
}
func (f *Failure) Error() string {
return f.Msg
}
// Fail writes msg to w and returns it as a *Failure, so a command reports a failure to the
// user and to its surface in one line. If the write itself fails, that error is returned
// instead: the user was not told anything.
func Fail(w StringWriter, msg string) error {
if err := w.WriteLine(msg); err != nil {
return err
}
return &Failure{Msg: msg}
}
func execCommand(c *Command, args []string, w StringWriter) error {
var (
fl *flag.FlagSet
fs any
)
if c.Flags != nil {
fl, fs = c.Flags()
if fl != nil {
// SetOutput() here in case fl.Parse dumps usage.
fl.SetOutput(w.GetWriter())
err := fl.Parse(args)
if err != nil {
// fl.Parse has dumped error information to the user via the w writer.
return err
}
args = fl.Args()
}
}
return c.Callback(fs, args, w)
}
func checkHelpArgs(args []string) bool {
for _, a := range args {
if a == "-h" || a == "-help" {
return true
}
}
return false
}
-28
View File
@@ -1,28 +0,0 @@
package commands
// Name identifies a command. Every command nebula serves is a constant here, so a
// surface, a client and a test spell it the same way and are told at compile time when
// they do not. It is a string rather than an integer so a test can register a throwaway
// command without adding it here, and so the wire carries the name as the string it has
// always been.
type Name string
const (
ChangeRemote Name = "change-remote"
CloseTunnel Name = "close-tunnel"
CreateTunnel Name = "create-tunnel"
DeviceInfo Name = "device-info"
Help Name = "help"
ListHostmap Name = "list-hostmap"
ListLighthouseAddrmap Name = "list-lighthouse-addrmap"
ListPendingHostmap Name = "list-pending-hostmap"
LogFormat Name = "log-format"
LogLevel Name = "log-level"
Logout Name = "logout"
PrintCert Name = "print-cert"
PrintRelays Name = "print-relays"
PrintTunnel Name = "print-tunnel"
QueryLighthouse Name = "query-lighthouse"
Reload Name = "reload"
Version Name = "version"
)
-181
View File
@@ -1,181 +0,0 @@
package commands
import (
"cmp"
"fmt"
"maps"
"slices"
"strings"
"sync"
"github.com/anmitsu/go-shlex"
)
// Registry holds a set of commands. New is the only way to make one.
//
// It is safe for concurrent use. Commands may be registered while a surface is serving
// from the registry: the ssh server clones it per session, and a clone may overlap a
// Register.
type Registry struct {
mu sync.RWMutex
commands map[Name]*Command
}
// New returns an empty Registry with only `help` available.
func New() *Registry {
r := &Registry{commands: map[Name]*Command{}}
r.registerHelp()
return r
}
func (r *Registry) registerHelp() {
r.Register(&Command{
Name: Help,
ShortDescription: "prints available commands or help <command> for specific usage info",
Callback: func(a any, args []string, w StringWriter) error {
return r.help(args, w)
},
})
}
// Register adds a command that can be run by a user. A command registered under a name
// already held replaces it; see Clone for what that means for Help.
func (r *Registry) Register(c *Command) {
r.mu.Lock()
defer r.mu.Unlock()
r.commands[c.Name] = c
}
// Get returns the command registered under name, or nil if there is no such command.
func (r *Registry) Get(name Name) *Command {
r.mu.RLock()
defer r.mu.RUnlock()
return r.commands[name]
}
// Match returns the names of every command starting with prefix, sorted, as strings for
// a prompt to complete and print.
func (r *Registry) Match(prefix string) []string {
cmds := make([]string, 0)
r.mu.RLock()
for n := range r.commands {
if strings.HasPrefix(string(n), prefix) {
cmds = append(cmds, string(n))
}
}
r.mu.RUnlock()
slices.Sort(cmds)
return cmds
}
// All returns every registered command sorted by name, which is the order help lists them.
func (r *Registry) All() []*Command {
r.mu.RLock()
defer r.mu.RUnlock()
cmds := slices.Collect(maps.Values(r.commands))
slices.SortFunc(cmds, func(a, b *Command) int { return cmp.Compare(a.Name, b.Name) })
return cmds
}
// Clone returns a copy holding the same commands, with help bound to the copy: a
// command registered on the clone alone is listed and described there, and one the
// clone withholds is not. That rebinding replaces whatever was registered under Help on
// the source, built-in or not.
func (r *Registry) Clone() *Registry {
r.mu.RLock()
c := &Registry{commands: maps.Clone(r.commands)}
r.mu.RUnlock()
c.registerHelp()
return c
}
// Dispatch parses a command line and runs it, reporting problems to w. The error is for
// the caller to log, not to show; see CommandCallback.
func (r *Registry) Dispatch(line string, w StringWriter) error {
// A line shlex cannot split, an unbalanced quote.
args, err := shlex.Split(line, true)
if err != nil {
return Fail(w, "could not parse the command line: "+err.Error())
}
return r.DispatchArgs(args, w)
}
// DispatchArgs runs an already tokenized command line. The error is as for Dispatch.
func (r *Registry) DispatchArgs(args []string, w StringWriter) error {
if len(args) == 0 {
r.dumpCommands(w)
return nil
}
c := r.Get(Name(args[0]))
if c == nil {
// Only the first word was looked up, so only it is named.
msg := fmt.Sprintf("did not understand: %s", args[0])
if err := w.WriteLine(msg); err != nil {
return err
}
r.dumpCommands(w)
return &Failure{Msg: msg}
}
if checkHelpArgs(args) {
// Through the registered help command, so -h and `help <name>` are one path.
return execCommand(r.Get(Help), []string{string(c.Name)}, w)
}
return execCommand(c, args[1:], w)
}
// dumpCommands writes the list of available commands to w.
func (r *Registry) dumpCommands(w StringWriter) {
err := w.WriteLine("Available commands:")
if err != nil {
return
}
// In the order All gives, which is the display order.
cmds := make([]string, 0)
for _, l := range r.All() {
cmds = append(cmds, fmt.Sprintf("%s - %s", l.Name, l.ShortDescription))
}
_ = w.Write(strings.Join(cmds, "\n") + "\n\n")
}
// help writes the command list, or usage for the single command named in a.
func (r *Registry) help(a []string, w StringWriter) error {
// Just typed help
if len(a) == 0 {
r.dumpCommands(w)
return nil
}
// We are printing a specific commands help text
cmd := r.Get(Name(a[0]))
if cmd == nil {
return Fail(w, "Command not available "+a[0])
}
if err := w.WriteLine(fmt.Sprintf("%s - %s", cmd.Name, cmd.ShortDescription)); err != nil {
return err
}
if cmd.Help != "" {
if err := w.WriteLine(fmt.Sprintf(" %s", cmd.Help)); err != nil {
return err
}
}
if cmd.Flags != nil {
fs, _ := cmd.Flags()
if fs != nil {
fs.SetOutput(w.GetWriter())
fs.PrintDefaults()
}
}
return nil
}
-176
View File
@@ -1,176 +0,0 @@
package commands
import (
"bytes"
"errors"
"flag"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testRegistry() *Registry {
r := New()
r.Register(&Command{
Name: "served",
ShortDescription: "a command this surface serves",
Callback: func(fs any, a []string, w StringWriter) error {
return w.WriteLine("read " + strings.Join(a, ","))
},
})
r.Register(&Command{
Name: "withheld",
ShortDescription: "a command this surface withholds",
Callback: func(fs any, a []string, w StringWriter) error {
return w.WriteLine("wrote")
},
})
return r
}
// TestFlagUsageGoesToTheWriter pins that a parse failure's usage lands on the command's
// writer.
func TestFlagUsageGoesToTheWriter(t *testing.T) {
r := New()
r.Register(&Command{
Name: "emit",
ShortDescription: "emits",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
fl.Bool("json", false, "emit json")
return fl, nil
},
Callback: func(fs any, a []string, w StringWriter) error { return nil },
})
merged := &bytes.Buffer{}
require.Error(t, r.DispatchArgs([]string{"emit", "-nope"}, NewStringWriter(merged)))
assert.Contains(t, merged.String(), "flag provided but not defined")
}
// TestDispatchReportsUnparseableLine: a line shlex rejects, an unbalanced quote, is a
// failure the user is told about.
func TestDispatchReportsUnparseableLine(t *testing.T) {
out := &bytes.Buffer{}
err := testRegistry().Dispatch(`served "unterminated`, NewStringWriter(out))
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, f.Msg+"\n", out.String())
assert.Contains(t, out.String(), "could not parse the command line: ")
}
// TestFailTellsTheUserAndTheSurface: Fail writes the message and returns it as a Failure;
// when the write fails, the user heard nothing, so the write error is what comes back.
func TestFailTellsTheUserAndTheSurface(t *testing.T) {
out := &bytes.Buffer{}
err := Fail(NewStringWriter(out), "no such tunnel")
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, "no such tunnel", f.Msg)
assert.Equal(t, "no such tunnel\n", out.String())
err = Fail(NewStringWriter(failingWriter{}), "no such tunnel")
require.EqualError(t, err, "gone")
assert.NotErrorAs(t, err, &f)
}
type failingWriter struct{}
func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("gone") }
// TestAllSortsByName: registration order does not matter.
func TestAllSortsByName(t *testing.T) {
r := New()
for _, n := range []Name{"zz", Version, "aa", ChangeRemote} {
r.Register(&Command{Name: n})
}
var got []Name
for _, c := range r.All() {
got = append(got, c.Name)
}
assert.Equal(t, []Name{"aa", ChangeRemote, Help, Version, "zz"}, got)
}
// TestHelpFlagAgreesWithHelpCommand pins that -h answers through the registered help
// command, and that help on a clone describes the clone: a command registered only there
// is known to both -h and `help <name>`, and one withheld from it is known to neither.
func TestHelpFlagAgreesWithHelpCommand(t *testing.T) {
r := testRegistry()
c := r.Clone()
c.Register(&Command{
Name: Logout,
ShortDescription: "ends the session",
Callback: func(fs any, a []string, w StringWriter) error { return nil },
})
flag := &bytes.Buffer{}
c.Dispatch("logout -h", NewStringWriter(flag))
cmd := &bytes.Buffer{}
c.Dispatch("help logout", NewStringWriter(cmd))
assert.Equal(t, cmd.String(), flag.String())
assert.Equal(t, "logout - ends the session\n", flag.String())
// The source never learns of it.
src := &bytes.Buffer{}
r.Dispatch("help logout", NewStringWriter(src))
assert.Equal(t, "Command not available logout\n", src.String())
// And a command both hold is described either way.
flag.Reset()
c.Dispatch("served -h", NewStringWriter(flag))
assert.Equal(t, "served - a command this surface serves\n", flag.String())
}
// TestCloneHelpListsTheClone: the command list help prints is the clone's, so a clone
// that withholds a command does not name it, and the source still does.
func TestCloneHelpListsTheClone(t *testing.T) {
r := testRegistry()
c := New()
c.Register(r.Get("served"))
c = c.Clone()
out := &bytes.Buffer{}
c.Dispatch("help", NewStringWriter(out))
assert.Equal(t, "Available commands:\n"+
"help - prints available commands or help <command> for specific usage info\n"+
"served - a command this surface serves\n\n", out.String())
out.Reset()
c.Dispatch("help withheld", NewStringWriter(out))
assert.Equal(t, "Command not available withheld\n", out.String())
out.Reset()
r.Dispatch("help withheld", NewStringWriter(out))
assert.Equal(t, "withheld - a command this surface withholds\n", out.String())
}
// TestDispatchReturnsTheCommandsError: what the callback returns reaches the caller, so
// a surface can log it.
func TestDispatchReturnsTheCommandsError(t *testing.T) {
r := New()
r.Register(&Command{
Name: "fails",
Callback: func(fs any, a []string, w StringWriter) error {
return errors.New("boom")
},
})
out := &bytes.Buffer{}
require.EqualError(t, r.Dispatch("fails", NewStringWriter(out)), "boom")
assert.Empty(t, out.String(), "a command's own error is not shown; the command speaks for itself")
}
// TestUnknownCommandNamesTheWord: only the first word is looked up, so only it is
// reported, followed by the command list, and the line is a failure to the surface.
func TestUnknownCommandNamesTheWord(t *testing.T) {
out := &bytes.Buffer{}
err := testRegistry().Dispatch(`nope "with args"`, NewStringWriter(out))
var f *Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, "did not understand: nope", f.Msg)
assert.True(t, strings.HasPrefix(out.String(), "did not understand: nope\nAvailable commands:\n"), out.String())
}
-67
View File
@@ -1,67 +0,0 @@
package nebula
import (
"bytes"
"io"
"net/netip"
"testing"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestLogFormatRefusalIsShown: a format the logger does not know is reported to the user
// as a failure, rather than returned bare, which over ssh was silence.
func TestLogFormatRefusalIsShown(t *testing.T) {
l := logging.NewLogger(io.Discard)
out := &bytes.Buffer{}
w := commands.NewStringWriter(out)
var f *commands.Failure
require.ErrorAs(t, logFormatCommand(l, nil, []string{"bogus"}, w), &f)
assert.Equal(t, "Unable to set log format: unknown log format `bogus`. possible formats: [text json]\n", out.String())
out.Reset()
require.NoError(t, logFormatCommand(l, nil, []string{"JSON"}, w))
assert.Equal(t, "Log format is: json\n", out.String())
}
// TestPrintRelaysReportsErrorsAsText: the Error field was an error value, which encodes
// as {} and lost the message. It is the message now, and absent when there is none.
func TestPrintRelaysReportsErrorsAsText(t *testing.T) {
l := test.NewLogger()
hm := newHostMap(l)
relay := netip.MustParseAddr("10.0.0.1")
peer := netip.MustParseAddr("10.0.0.2")
// A relay host that is in Relays but not in Hosts, so its hostinfo cannot be found.
hm.Relays[1] = &HostInfo{vpnAddrs: []netip.Addr{relay}}
ifce := &Interface{hostMap: hm}
out := &bytes.Buffer{}
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.JSONEq(t, `{"Relays":[{"NebulaAddr":"10.0.0.1","RelayForAddrs":[{"Error":"could not find hostinfo","Type":"","State":"","PeerAddr":"","LocalIndex":0,"RemoteIndex":0,"RelayedThrough":null}]}]}`, out.String())
// The same relay host, found, relaying for a peer whose index matches: no Error key.
hi := hm.Relays[1]
r := &Relay{Type: ForwardingType, State: Established, PeerAddr: peer, LocalIndex: 1, RemoteIndex: 2}
hi.relayState = RelayState{
relayForByAddr: map[netip.Addr]*Relay{peer: r},
relayForByIdx: map[uint32]*Relay{r.LocalIndex: r},
}
hm.Hosts[relay] = hi
out.Reset()
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.JSONEq(t, `{"Relays":[{"NebulaAddr":"10.0.0.1","RelayForAddrs":[{"Type":"forwarding","State":"established","PeerAddr":"10.0.0.2","LocalIndex":1,"RemoteIndex":2,"RelayedThrough":null}]}]}`, out.String())
// And an index mismatch names it.
hm.Relays[7] = hi
delete(hm.Relays, 1)
out.Reset()
require.NoError(t, printRelaysCommand(ifce, &printTunnelFlags{}, nil, commands.NewStringWriter(out)))
assert.Contains(t, out.String(), `"Error":"hostmap LocalIndex '7' does not match RelayState LocalIndex"`)
}
-15
View File
@@ -86,21 +86,6 @@ func (c *C) InitialLoad() bool {
return c.oldSettings == nil
}
// Render returns keys and their values as yaml, for a subsystem to remember what it applied. Empty means they couldn't
// be rendered and should never match.
func (c *C) Render(keys ...string) string {
m := make(map[string]any, len(keys))
for _, k := range keys {
m[k] = c.Get(k)
}
b, err := yaml.Marshal(m)
if err != nil {
c.l.Debug("Failed to render config", "keys", keys, "error", err)
return ""
}
return string(b)
}
// HasChanged checks if the underlying structure of the provided key has changed after a config reload. The value of
// k in both the old and new settings will be serialized, the result of the string comparison is returned.
// If k is an empty string the entire config is tested.
-9
View File
@@ -11,7 +11,6 @@ import (
"syscall"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay"
)
@@ -48,7 +47,6 @@ type Control struct {
f *Interface
l *slog.Logger
commands *commands.Registry
ctx context.Context
cancel context.CancelFunc
sshStart func()
@@ -374,13 +372,6 @@ func (c *Control) CloseAllTunnels(excludeLighthouses bool) (closed int) {
return
}
// Commands returns the registry the daemon's surfaces serve from. It is the live
// registry, not a copy: a command registered on it is served to every ssh session that
// starts afterwards. Clone it to run commands without that reach.
func (c *Control) Commands() *commands.Registry {
return c.commands
}
func (c *Control) Device() overlay.Device {
return c.f.inside
}
-164
View File
@@ -1,164 +0,0 @@
//go:build e2e_testing
package e2e
import (
"bytes"
"fmt"
"path/filepath"
"regexp"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/require"
)
// volatile lists everything in command output that legitimately differs between runs.
// Each run mints fresh certificates and indexes, so these are replaced with a marker
// before comparison. Keep the list short: anything added here is something the golden
// stops guarding.
var volatile = []struct {
re *regexp.Regexp
with string
}{
{regexp.MustCompile(`[0-9a-f]{40,}`), "<hex>"}, // fingerprints, keys, signatures
{regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})`), "<time>"}, // RFC 3339
{regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)? [+-]\d{4} \w+`), "<time>"}, // time.Time.String()
{regexp.MustCompile(`(?s)(-----BEGIN [A-Z ]+-----\n).*?(\n-----END [A-Z ]+-----)`), "${1}<pem>${2}"}, // print-cert -raw
{regexp.MustCompile(`(-----BEGIN [A-Z ]+-----\\n).*?(\\n-----END [A-Z ]+-----)`), "${1}<pem>${2}"}, // the same, inside a JSON string
{regexp.MustCompile(`"(localIndex|remoteIndex|messageCounter)":\d+`), `"$1":<n>`},
{regexp.MustCompile(`(?m)^(\s*"?(?:LocalIndex|RemoteIndex|localIndex|remoteIndex|messageCounter)"?:?\s*)\d+`), "${1}<n>"}, // and the -pretty form
}
func normalise(s string) string {
for _, v := range volatile {
s = v.re.ReplaceAllString(s, v.with)
}
return s
}
// outputCase is one command line to run through the ssh dispatch path.
type outputCase struct {
name string
args []string
}
// TestCommandOutput pins the bytes every command produces, in every format, through the
// ssh dispatch path. It is the safety net for moving rendering out of the commands: a
// change here is a change someone at an ssh prompt would see. Regenerate with:
//
// go test -tags e2e_testing ./e2e -run TestCommandOutput -update
func TestCommandOutput(t *testing.T) {
ca, _, caKey, _ := cert_test.NewTestCaCert(
cert.Version1, cert.Curve_CURVE25519,
time.Now(), time.Now().Add(10*time.Minute),
nil, nil, []string{},
)
control, _, _, _ := newSimpleServer(cert.Version1, ca, caKey, "output-golden", "10.222.0.1/24", nil)
control.Start()
t.Cleanup(func() { control.Stop() })
// As an ssh session does: a clone of the daemon's registry with logout added, which
// help on the clone lists and describes. There is no session to end here.
reg := control.Commands().Clone()
reg.Register(sshd.LogoutCommand(func() {}))
// run dispatches as the ssh exec path does and returns the output and the error, which
// over ssh is exit status 1.
run := func(args ...string) (string, error) {
buf := &bytes.Buffer{}
err := reg.DispatchArgs(args, commands.NewStringWriter(buf))
return buf.String(), err
}
// Seed a pending tunnel so the listing and per-address commands have something to show.
seed, err := run("create-tunnel", "10.222.0.9")
require.NoError(t, err)
require.Equal(t, "Created\n", seed, "seeding a tunnel should succeed before anything else is recorded")
cases := []outputCase{
{"version", nil},
{"device-info", nil},
{"list-hostmap", nil},
{"list-hostmap", []string{"-by-index"}},
{"list-pending-hostmap", nil},
{"list-pending-hostmap", []string{"-by-index"}},
{"list-lighthouse-addrmap", nil},
{"print-cert", nil},
{"print-cert", []string{"-raw"}},
{"print-cert", []string{"10.222.0.9"}},
{"print-cert", []string{"10.222.0.250"}},
{"print-cert", []string{"not-an-address"}},
{"print-tunnel", nil},
{"print-tunnel", []string{"10.222.0.9"}},
{"print-tunnel", []string{"10.222.0.250"}},
{"print-relays", nil},
{"query-lighthouse", nil},
{"query-lighthouse", []string{"10.222.0.9"}},
{"query-lighthouse", []string{"not-an-address"}},
// The test logger cannot change level or format, so these cases record the
// refusal each command prints on it, not a successful change.
{"log-level", nil},
{"log-level", []string{"debug"}},
{"log-level", []string{"bogus"}},
{"log-level", []string{"info"}},
{"log-format", nil},
{"log-format", []string{"json"}},
{"log-format", []string{"text"}},
{"change-remote", nil},
{"change-remote", []string{"10.222.0.9"}},
{"change-remote", []string{"-address", "10.94.0.10:4242", "10.222.0.9"}},
{"change-remote", []string{"-address", "10.94.0.10:4242", "10.222.0.250"}},
{"create-tunnel", nil},
{"create-tunnel", []string{"10.222.0.9"}},
// A fresh address, so this exercises the handshake start with a remote attached
// rather than the "already handshaking" path above.
{"create-tunnel", []string{"-address", "10.94.0.10:4242", "10.222.0.10"}},
{"create-tunnel", []string{"not-an-address"}},
{"close-tunnel", nil},
{"close-tunnel", []string{"10.222.0.250"}},
{"close-tunnel", []string{"not-an-address"}},
{"help", nil},
{"help", []string{"print-cert"}},
{"help", []string{"logout"}},
{"logout", []string{"-h"}},
{"not-a-command", []string{"10.222.0.9"}},
// reload is last so that whatever a reload does to the daemon cannot leak into an
// earlier case. In this harness it fails: the config was loaded from a string and
// has no path to re-read, which is what the recorded result says.
{"reload", nil},
}
got := &strings.Builder{}
for _, c := range cases {
for _, fmtFlags := range [][]string{nil, {"-json"}, {"-pretty"}} {
args := append([]string{c.name}, fmtFlags...)
args = append(args, c.args...)
fmt.Fprintf(got, "$ %s\n", strings.Join(args, " "))
out, err := run(args...)
out = normalise(out)
got.WriteString(out)
// Output that does not end in a newline is recorded as such, the way diff marks
// it, so the golden sees that byte too and the next prompt still starts a line.
if out != "" && !strings.HasSuffix(out, "\n") {
got.WriteString("\\ No newline at end of output\n")
}
// A failed command is recorded as the exit status an ssh caller sees, so the
// golden pins which outcomes count as failures.
if err != nil {
got.WriteString("exit status 1\n")
}
got.WriteString("\n")
}
}
test.Golden(t, filepath.Join("testdata", "commands_output.golden"), got.String())
}
-720
View File
@@ -1,720 +0,0 @@
$ version
e2e-test
$ version -json
e2e-test
$ version -pretty
e2e-test
$ device-info
name= cidr=[10.222.0.1/24]
$ device-info -json
{"name":"","cidr":["10.222.0.1/24"]}
$ device-info -pretty
{
"name": "",
"cidr": [
"10.222.0.1/24"
]
}
$ list-hostmap
$ list-hostmap -json
[]
$ list-hostmap -pretty
[]
$ list-hostmap -by-index
$ list-hostmap -json -by-index
[]
$ list-hostmap -pretty -by-index
[]
$ list-pending-hostmap
[10.222.0.9]: []
$ list-pending-hostmap -json
[{"vpnAddrs":["10.222.0.9"],"localIndex":<n>,"remoteIndex":<n>,"remoteAddrs":[],"cert":null,"messageCounter":<n>,"currentRemote":"","currentRelaysToMe":[],"currentRelaysThroughMe":[]}]
$ list-pending-hostmap -pretty
[
{
"vpnAddrs": [
"10.222.0.9"
],
"localIndex": <n>,
"remoteIndex": <n>,
"remoteAddrs": [],
"cert": null,
"messageCounter": <n>,
"currentRemote": "",
"currentRelaysToMe": [],
"currentRelaysThroughMe": []
}
]
$ list-pending-hostmap -by-index
$ list-pending-hostmap -json -by-index
[]
$ list-pending-hostmap -pretty -by-index
[]
$ list-lighthouse-addrmap
10.222.0.9: {}
$ list-lighthouse-addrmap -json
[{"vpnAddr":"10.222.0.9","addrs":{}}]
$ list-lighthouse-addrmap -pretty
[
{
"vpnAddr": "10.222.0.9",
"addrs": {}
}
]
$ print-cert
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}
$ print-cert -json
{"details":{"curve":"CURVE25519","groups":[],"isCa":false,"issuer":"<hex>","name":"output-golden","networks":["10.222.0.1/24"],"notAfter":"<time>","notBefore":"<time>","publicKey":"<hex>","unsafeNetworks":[]},"fingerprint":"<hex>","signature":"<hex>","version":1}\ No newline at end of output
$ print-cert -pretty
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}\ No newline at end of output
$ print-cert -raw
-----BEGIN NEBULA CERTIFICATE-----
<pem>
-----END NEBULA CERTIFICATE-----
$ print-cert -json -raw
{"details":{"curve":"CURVE25519","groups":[],"isCa":false,"issuer":"<hex>","name":"output-golden","networks":["10.222.0.1/24"],"notAfter":"<time>","notBefore":"<time>","publicKey":"<hex>","unsafeNetworks":[]},"fingerprint":"<hex>","signature":"<hex>","version":1}\ No newline at end of output
$ print-cert -pretty -raw
{
"details": {
"curve": "CURVE25519",
"groups": [],
"isCa": false,
"issuer": "<hex>",
"name": "output-golden",
"networks": [
"10.222.0.1/24"
],
"notAfter": "<time>",
"notBefore": "<time>",
"publicKey": "<hex>",
"unsafeNetworks": []
},
"fingerprint": "<hex>",
"signature": "<hex>",
"version": 1
}\ No newline at end of output
$ print-cert 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert -json 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert -pretty 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-cert 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert -json 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert -pretty 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-cert not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-cert -json not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-cert -pretty not-an-address
The provided vpn addr could not be parsed: not-an-address
exit status 1
$ print-tunnel
No vpn address was provided
exit status 1
$ print-tunnel -json
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty
No vpn address was provided
exit status 1
$ print-tunnel 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-tunnel -json 10.222.0.9
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty 10.222.0.9
Could not find tunnel for vpn addr: 10.222.0.9
exit status 1
$ print-tunnel 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-tunnel -json 10.222.0.250
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-tunnel -pretty 10.222.0.250
Could not find tunnel for vpn addr: 10.222.0.250
exit status 1
$ print-relays
{"Relays":null}
$ print-relays -json
flag provided but not defined: -json
Usage:
-pretty
pretty prints json
exit status 1
$ print-relays -pretty
{
"Relays": null
}
$ query-lighthouse
No vpn address was provided
exit status 1
$ query-lighthouse -json
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty
The provided vpn address could not be parsed: -pretty
exit status 1
$ query-lighthouse 10.222.0.9
{}
$ query-lighthouse -json 10.222.0.9
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty 10.222.0.9
The provided vpn address could not be parsed: -pretty
exit status 1
$ query-lighthouse not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ query-lighthouse -json not-an-address
The provided vpn address could not be parsed: -json
exit status 1
$ query-lighthouse -pretty not-an-address
The provided vpn address could not be parsed: -pretty
exit status 1
$ log-level
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty
Log level is not reconfigurable on this logger
exit status 1
$ log-level debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty debug
Log level is not reconfigurable on this logger
exit status 1
$ log-level bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty bogus
Log level is not reconfigurable on this logger
exit status 1
$ log-level info
Log level is not reconfigurable on this logger
exit status 1
$ log-level -json info
Log level is not reconfigurable on this logger
exit status 1
$ log-level -pretty info
Log level is not reconfigurable on this logger
exit status 1
$ log-format
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty
Log format is not reconfigurable on this logger
exit status 1
$ log-format json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json json
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty json
Log format is not reconfigurable on this logger
exit status 1
$ log-format text
Log format is not reconfigurable on this logger
exit status 1
$ log-format -json text
Log format is not reconfigurable on this logger
exit status 1
$ log-format -pretty text
Log format is not reconfigurable on this logger
exit status 1
$ change-remote
No vpn address was provided
exit status 1
$ change-remote -json
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote 10.222.0.9
No address was provided
exit status 1
$ change-remote -json 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -address 10.94.0.10:4242 10.222.0.9
Could not find tunnel for vpn address: 10.222.0.9
exit status 1
$ change-remote -json -address 10.94.0.10:4242 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty -address 10.94.0.10:4242 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -address 10.94.0.10:4242 10.222.0.250
Could not find tunnel for vpn address: 10.222.0.250
exit status 1
$ change-remote -json -address 10.94.0.10:4242 10.222.0.250
flag provided but not defined: -json
Usage:
-address string
The new remote address, ip:port
exit status 1
$ change-remote -pretty -address 10.94.0.10:4242 10.222.0.250
flag provided but not defined: -pretty
Usage:
-address string
The new remote address, ip:port
exit status 1
$ create-tunnel
No vpn address was provided
exit status 1
$ create-tunnel -json
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel 10.222.0.9
Tunnel already handshaking
exit status 1
$ create-tunnel -json 10.222.0.9
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty 10.222.0.9
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -address 10.94.0.10:4242 10.222.0.10
Created
$ create-tunnel -json -address 10.94.0.10:4242 10.222.0.10
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty -address 10.94.0.10:4242 10.222.0.10
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ create-tunnel -json not-an-address
flag provided but not defined: -json
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ create-tunnel -pretty not-an-address
flag provided but not defined: -pretty
Usage:
-address string
Optionally provide a real remote address, ip:port
exit status 1
$ close-tunnel
No vpn address was provided
exit status 1
$ close-tunnel -json
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel 10.222.0.250
Could not find tunnel for vpn address: 10.222.0.250
exit status 1
$ close-tunnel -json 10.222.0.250
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty 10.222.0.250
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel not-an-address
The provided vpn address could not be parsed: not-an-address
exit status 1
$ close-tunnel -json not-an-address
flag provided but not defined: -json
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ close-tunnel -pretty not-an-address
flag provided but not defined: -pretty
Usage:
-local-only
Disables notifying the remote that the tunnel is shutting down
exit status 1
$ help
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
$ help -json
Command not available -json
exit status 1
$ help -pretty
Command not available -pretty
exit status 1
$ help print-cert
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
-json
outputs as json
-pretty
pretty prints json, assumes -json
-raw
raw prints the PEM encoded certificate, not compatible with -json or -pretty
$ help -json print-cert
Command not available -json
exit status 1
$ help -pretty print-cert
Command not available -pretty
exit status 1
$ help logout
logout - Ends the current session
$ help -json logout
Command not available -json
exit status 1
$ help -pretty logout
Command not available -pretty
exit status 1
$ logout -h
logout - Ends the current session
$ logout -json -h
logout - Ends the current session
$ logout -pretty -h
logout - Ends the current session
$ not-a-command 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ not-a-command -json 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ not-a-command -pretty 10.222.0.9
did not understand: not-a-command
Available commands:
change-remote - Changes the remote address used in the tunnel for the provided vpn addr
close-tunnel - Closes a tunnel for the provided vpn addr
create-tunnel - Creates a tunnel for the provided vpn address
device-info - Prints information about the network device.
help - prints available commands or help <command> for specific usage info
list-hostmap - List all known previously connected hosts
list-lighthouse-addrmap - List all lighthouse map entries
list-pending-hostmap - List all handshaking hosts
log-format - Gets or sets the current log format
log-level - Gets or sets the current log level
logout - Ends the current session
print-cert - Prints the current certificate being used or the certificate for the provided vpn addr
print-relays - Prints json details about all relay info
print-tunnel - Prints json details about a tunnel for the provided vpn addr
query-lighthouse - Query the lighthouses for the provided vpn address
reload - Reloads configuration from disk, same as sending HUP to the process
version - Prints the currently running version of nebula
exit status 1
$ reload
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
$ reload -json
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
$ reload -pretty
Reloading config
Config reload failed, the new config did not load: no config files found at
exit status 1
+2 -6
View File
@@ -39,9 +39,8 @@ static_host_map:
# public address. Other valid options are "ip6" and "ip" (returns both.)
#network: ip4
# lookup_timeout is the DNS query timeout. A failed lookup keeps the last known good addresses, only a not found
# answer removes them.
#lookup_timeout: 5s
# lookup_timeout is the DNS query timeout.
#lookup_timeout: 250ms
lighthouse:
# am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
@@ -332,9 +331,6 @@ tun:
# `mtu`: will default to tun mtu if this option is not specified
# `metric`: will default to 0 if this option is not specified
# `install`: will default to true, controls whether this route is installed in the systems routing table.
# On macOS, a route whose destination another interface already routes, like the local network, is installed once
# that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another
# interface routes its destination and lets it go.
# This setting is reloadable.
unsafe_routes:
#- route: 172.16.1.0/24
-4
View File
@@ -66,9 +66,6 @@ type Firewall struct {
rules string
rulesVersion uint16
// source is the rendered firewall config this firewall was built from, the cert networks are kept above
source string
defaultLocalCIDRAny bool
incomingMetrics firewallMetrics
outgoingMetrics firewallMetrics
@@ -249,7 +246,6 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
return nil, err
}
fw.source = c.Render("firewall")
return fw, nil
}
+1
View File
@@ -7,6 +7,7 @@ require (
filippo.io/bigmod v0.1.0
github.com/DefinedNet/netlink v1.3.2-dn.1
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
github.com/armon/go-radix v1.0.0
github.com/flynn/noise v1.1.0
github.com/gaissmai/bart v0.29.0
github.com/gogo/protobuf v1.3.2
+2
View File
@@ -12,6 +12,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+1 -13
View File
@@ -69,7 +69,6 @@ type HandshakeManager struct {
messageMetrics *MessageMetrics
metricInitiated metrics.Counter
metricTimedOut metrics.Counter
metricRejectedBlocklisted metrics.Counter
f *Interface
l *slog.Logger
@@ -130,7 +129,6 @@ func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *Light
messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
metricRejectedBlocklisted: metrics.GetOrRegisterCounter("handshake_manager.rejected.blocklisted", nil),
l: l,
}
}
@@ -405,8 +403,6 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
hm.Unlock()
hm.lightHouse.QueryServer(vpnAddr)
// If this host came from a static_map hostname its addresses may have moved, retries will pick up anything new
hostinfo.remotes.ResolveHostnames()
return hostinfo
}
@@ -721,12 +717,7 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
response, result, err := machine.ProcessPacket(nil, packet)
if err != nil {
level := slog.LevelError
if errors.Is(err, cert.ErrBlockListed) {
level = slog.LevelDebug
hm.metricRejectedBlocklisted.Inc(1)
}
f.l.Log(context.Background(), level, "Failed to process handshake packet", "from", via, "error", err)
f.l.Error("Failed to process handshake packet", "from", via, "error", err)
return
}
@@ -858,9 +849,6 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
response, result, err := machine.ProcessPacket(nil, packet)
if err != nil {
if errors.Is(err, cert.ErrBlockListed) {
hm.metricRejectedBlocklisted.Inc(1)
}
// Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
if machine.Failed() {
f.l.Warn("Failed to process handshake packet, abandoning",
-88
View File
@@ -1,17 +1,12 @@
package nebula
import (
"log/slog"
"net/netip"
"testing"
"time"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
@@ -240,89 +235,6 @@ func TestHandleIncomingDispatch(t *testing.T) {
})
}
func TestBlocklistedHandshakeLogLevelAndCounter(t *testing.T) {
ca, _, caKey, _ := ct.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
pool := ct.NewTestCAPool(ca)
newCredential := func(name, network string) (cert.Certificate, *handshake.Credential) {
c, _, rawKey, _ := ct.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey,
name, ca.NotBefore(), ca.NotAfter(), []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil)
privateKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey)
require.NoError(t, err)
certBytes, err := c.MarshalForHandshakes()
require.NoError(t, err)
cipherSuite := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
return c, handshake.NewCredential(c, certBytes, privateKey, cipherSuite)
}
initiatorCert, initiatorCredential := newCredential("initiator", "10.0.0.1/24")
responderCert, responderCredential := newCredential("responder", "10.0.0.2/24")
fingerprint, err := initiatorCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(fingerprint)
initiator, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return initiatorCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return pool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 1, nil }, true, header.HandshakeIXPSK0)
require.NoError(t, err)
packet, err := initiator.Initiate(nil)
require.NoError(t, err)
handler := &capturingHandler{}
logger := slog.New(handler)
pki := &PKI{}
pki.cs.Store(&CertState{initiatingVersion: cert.Version2, v2Credential: responderCredential})
pki.caPool.Store(pool)
hm := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
hm.metricRejectedBlocklisted = metrics.NewCounter()
hm.f = &Interface{pki: pki, l: logger}
via := ViaSender{UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242")}
hm.beginHandshake(via, packet, nil)
entry := findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelDebug, entry.Level)
require.ErrorIs(t, entry.Attrs["error"].(error), cert.ErrBlockListed)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
handler.entries = nil
hm.beginHandshake(via, nil, nil)
entry = findEntry(t, handler.entries, "Failed to process handshake packet")
assert.Equal(t, slog.LevelError, entry.Level)
assert.Equal(t, int64(1), hm.metricRejectedBlocklisted.Count())
// The initiator sees the same rejection in continueHandshake, after a
// responder has accepted its first packet and sent a valid reply.
cleanPool := ct.NewTestCAPool(ca)
responder, err := handshake.NewMachine(cert.Version2,
func(cert.Version) *handshake.Credential { return responderCredential },
func(c cert.Certificate) (*cert.CachedCertificate, error) {
return cleanPool.VerifyCertificate(time.Now(), c)
},
func() (uint32, error) { return 2, nil }, false, header.HandshakeIXPSK0)
require.NoError(t, err)
response, _, err := responder.ProcessPacket(nil, packet)
require.NoError(t, err)
responderFingerprint, err := responderCert.Fingerprint()
require.NoError(t, err)
pool.BlocklistFingerprint(responderFingerprint)
initiatingHM := NewHandshakeManager(logger, nil, nil, nil, defaultHandshakeConfig)
initiatingHM.metricRejectedBlocklisted = metrics.NewCounter()
initiatingHM.f = &Interface{l: logger}
hh := &HandshakeHostInfo{
machine: initiator,
hostinfo: &HostInfo{localIndexId: 1, vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}},
}
initiatingHM.indexes[1] = hh
via.IsRelayed = true // skip the remote allow list before certificate validation
initiatingHM.continueHandshake(via, hh, response)
assert.Equal(t, int64(1), initiatingHM.metricRejectedBlocklisted.Count())
}
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
+18 -64
View File
@@ -124,8 +124,6 @@ type Interface struct {
// reused (every udp.Conn ListenOut guarantees that ordering).
batchers []*batch.MultiCoalescer
wg sync.WaitGroup
// readers covers just the listenIn and listenOut goroutines, wg also holds the construction token Close releases
readers sync.WaitGroup
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
@@ -343,18 +341,14 @@ func (f *Interface) activate() error {
func (f *Interface) run() {
// Launch n queues to read packets from udp
for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() {
defer f.readers.Done()
f.listenOut(i)
})
}
// Launch n queues to read packets from tun dev
for i := 0; i < f.routines; i++ {
f.readers.Add(1)
f.wg.Go(func() {
defer f.readers.Done()
f.listenIn(f.queues[i], i)
})
}
@@ -390,6 +384,8 @@ type rxContext struct {
hostmapCache map[uint32]*HostInfo
lhh *LightHouseHandler
ctCache *firewall.ConntrackCacheTicker
// relayBatch holds forwarded relay packets, which borrow their receive buffers until flushed
relayBatch *batch.SendBatch
}
func newRxContext(f *Interface, q int) *rxContext {
@@ -402,6 +398,7 @@ func newRxContext(f *Interface, q int) *rxContext {
hostmapCache: map[uint32]*HostInfo{},
lhh: f.lightHouse.NewRequestHandler(),
ctCache: firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout),
relayBatch: batch.NewSendBatch(f.writers[q], batch.SendBatchCap, 0),
}
}
@@ -420,6 +417,7 @@ func (f *Interface) listenOut(i int) {
}
flusher := func() {
f.flushSendBatch(rxc.relayBatch, i)
if err := f.batchers[i].Flush(); err != nil {
f.l.Error("Failed to flush tun coalescer", "error", err)
}
@@ -543,12 +541,11 @@ func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) {
curCert = cs.getCertificate(cert.Version1)
}
// Compare with what the running firewall was built from, not the last config, so a firewall that failed to build
// is tried again. The cert is the one this reload brought, PKI registers its reload callback before this one
src := c.Render("firewall")
if src != "" && src == f.firewall.source && curCert != nil &&
slices.Equal(curCert.Networks(), f.firewall.assignedNetworks) &&
slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) {
// The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction.
// Check to see if that set has changed, and if so, rebuild the firewall.
certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks)
if !c.HasChanged("firewall") && !certUnsafeChanged {
f.l.Debug("No firewall config change detected")
return
}
@@ -689,14 +686,6 @@ func (f *Interface) GetCertState() *CertState {
return f.pki.getCertState()
}
// shutdowner is a udp.Conn or overlay.Device whose readers can be woken without releasing their fds
type shutdowner interface {
Shutdown() error
}
// readerShutdownTimeout bounds how long Close waits on woken readers before releasing their fds anyway
const readerShutdownTimeout = 5 * time.Second
// Close releases the interface's resources: the udp sockets and the tun device.
// It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated,
// calls after the first return nil without doing anything.
@@ -706,55 +695,20 @@ func (f *Interface) Close() error {
}
var errs []error
closeUdp := func(i int, u udp.Conn) {
if err := u.Close(); err != nil {
// Release the udp readers
for i, u := range f.writers {
err := u.Close()
if err != nil {
f.l.Error("Error while closing udp socket", "error", err, "writer", i)
errs = append(errs, err)
}
}
closeTun := func() {
if err := f.inside.Close(); err != nil {
errs = append(errs, err)
}
}
// Wake every reader before releasing any fd, one that loops back into a syscall on a reused fd number never wakes.
// Anything without Shutdown is closed right away, as before.
var deferred []func()
for i, u := range f.writers {
if s, ok := u.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
f.l.Error("Error while shutting down udp socket", "error", err, "writer", i)
errs = append(errs, err)
}
deferred = append(deferred, func() { closeUdp(i, u) })
} else {
closeUdp(i, u)
}
}
if s, ok := f.inside.(shutdowner); ok {
if err := s.Shutdown(); err != nil {
errs = append(errs, err)
}
deferred = append(deferred, closeTun)
} else {
closeTun()
}
if len(deferred) > 0 {
readersDone := make(chan struct{})
go func() {
f.readers.Wait()
close(readersDone)
}()
select {
case <-readersDone:
case <-time.After(readerShutdownTimeout):
f.l.Warn("Readers did not stop, closing their fds anyway", "timeout", readerShutdownTimeout)
}
for _, closeFn := range deferred {
closeFn()
}
// Release the tun device (closing the tun also closes all readers)
closeErr := f.inside.Close()
if closeErr != nil {
errs = append(errs, closeErr)
}
// Release the construction token so waiters know the resources are gone
-55
View File
@@ -135,63 +135,8 @@ func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) {
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
// The running firewall still isn't built from what the config asks for, so the same reload fails again
for range 2 {
res := cfg.ReloadConfigString(rules("nope"))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
assert.Same(t, fw, f.firewall)
}
res := cfg.ReloadConfigString(rules("tcp"))
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.NotSame(t, fw, f.firewall)
}
// Every input the firewall is built from rebuilds it when it changes, and nothing else does
func TestReloadFirewall_RebuildsOnAnyInput(t *testing.T) {
l := test.NewLogger()
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
base := "firewall:\n inbound:\n - port: any\n proto: any\n host: any\n"
setup := func() (*config.C, *Interface, *PKI) {
pki := &PKI{}
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks}, initiatingVersion: cert.Version2})
cfg := config.NewC(l)
require.NoError(t, cfg.LoadString(base))
fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
require.NoError(t, err)
f := &Interface{pki: pki, firewall: fw, l: l}
cfg.RegisterReloadCallback(f.reloadFirewall)
return cfg, f, pki
}
setCert := func(pki *PKI, networks, unsafe []netip.Prefix) {
pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: networks, unsafeNetworks: unsafe}, initiatingVersion: cert.Version2})
}
cases := map[string]struct {
cfg string
cert func(*PKI)
want bool
}{
"nothing": {base + "punchy:\n punch: true\n", nil, false},
"rules": {base + " outbound:\n - port: any\n proto: any\n host: any\n", nil, true},
"inbound_action": {base + " inbound_action: reject\n", nil, true},
"conntrack": {base + " conntrack:\n tcp_timeout: 1m\n", nil, true},
"networks": {base, func(p *PKI) { setCert(p, []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil) }, true},
"unsafe_networks": {base, func(p *PKI) { setCert(p, networks, []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}) }, true},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
cfg, f, pki := setup()
old := f.firewall
if tc.cert != nil {
tc.cert(pki)
}
res := cfg.ReloadConfigString(tc.cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, tc.want, f.firewall != old)
})
}
}
+3 -5
View File
@@ -452,7 +452,7 @@ func getStaticMapCadence(c *config.C) (time.Duration, error) {
}
func getStaticMapLookupTimeout(c *config.C) (time.Duration, error) {
lookupTimeout := c.GetString("static_map.lookup_timeout", "5s")
lookupTimeout := c.GetString("static_map.lookup_timeout", "250ms")
d, err := time.ParseDuration(lookupTimeout)
if err != nil {
return 0, err
@@ -630,8 +630,7 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
ctx := lh.ctx
lh.Unlock()
// On reload am.hr is the previous resolver, its last known good addresses carry over in case DNS is down right now
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, am.hr, func() {
hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, func() {
// This callback runs whenever the DNS hostname resolver finds a different set of addr's
// in its resolution for hostnames.
am.Lock()
@@ -643,8 +642,7 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
}
am.unlockedSetHostnamesResults(hr)
// Only literals belong in the cache, anything from DNS has to stay in hr where a not found answer can remove it
for _, addrPort := range hr.literalAddrs() {
for _, addrPort := range hr.GetAddrs() {
if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) {
continue
}
+2 -8
View File
@@ -12,7 +12,6 @@ import (
"strings"
"time"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/cpupick"
"github.com/slackhq/nebula/noiseutil"
@@ -72,11 +71,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
}
l.Info("Firewall started", "firewallHashes", fw.GetRuleHashes())
// The command registry is shared by every surface that exposes these commands, so a
// command cannot reach one of them without reaching the others.
cmdRegistry := commands.New()
ssh, err := sshd.NewSSHServer(ctx, l.With("subsystem", "sshd"), cmdRegistry)
ssh, err := sshd.NewSSHServer(ctx, l.With("subsystem", "sshd"))
if err != nil {
return nil, util.ContextualizeIfNeeded("Error while creating SSH server", err)
}
@@ -348,14 +343,13 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
return nil, nil
}
attachCommands(l, c, cmdRegistry, ifce)
attachCommands(l, c, ssh, ifce)
networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
return &Control{
state: StateReady,
f: ifce,
commands: cmdRegistry,
l: l,
ctx: ctx,
cancel: cancel,
+9 -3
View File
@@ -25,7 +25,7 @@ var ErrOutOfWindow = errors.New("out of window packet")
// readOutsidePackets processes one received underlay packet.
// Message payloads are decrypted IN PLACE, so packet must stay untouched
// by the caller until the batcher for queue q has been flushed
// by the caller until the TUN and relay batches for queue q have been flushed.
func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxContext) {
h := rxc.h
err := h.Parse(packet)
@@ -233,8 +233,14 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
// Forward this packet through the relay tunnel, rebuilding it in place.
// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
fwdBuf := packet[:0]
//todo it would potentially be nice to batch these
f.SendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true, rxc.q)
out, err := f.prepareSendVia(targetHI, targetRelay, signedPayload, rxc.nb, fwdBuf, true)
if err == nil {
// Borrows the receive buffer, the listener's flush callback drains it before the buffer is reused
rxc.relayBatch.Commit(out, targetHI.GetRemote())
if rxc.relayBatch.Len() >= batch.SendBatchCap {
f.flushSendBatch(rxc.relayBatch, rxc.q)
}
}
case TerminalType:
hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
return
+242
View File
@@ -0,0 +1,242 @@
package nebula
import (
"bytes"
"net/netip"
"sync"
"testing"
"github.com/flynn/noise"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/require"
)
type relayBatchWrite struct {
packets [][]byte
remotes []netip.AddrPort
batched bool
}
type relayBatchConn struct {
udp.NoopConn
read func(udp.EncReader, func())
writes []relayBatchWrite
}
func (c *relayBatchConn) ListenOut(r udp.EncReader, flush func()) error {
c.read(r, flush)
return nil
}
func (c *relayBatchConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
w := relayBatchWrite{remotes: append([]netip.AddrPort(nil), addrs...), batched: true}
for _, b := range bufs {
w.packets = append(w.packets, bytes.Clone(b))
}
c.writes = append(c.writes, w)
return len(bufs), nil
}
func (c *relayBatchConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.writes = append(c.writes, relayBatchWrite{packets: [][]byte{bytes.Clone(b)}, remotes: []netip.AddrPort{addr}})
return nil
}
func (c *relayBatchConn) packets() [][]byte {
var packets [][]byte
for _, w := range c.writes {
packets = append(packets, w.packets...)
}
return packets
}
type relayBatchPath struct {
f *Interface
conn *relayBatchConn
source, target *HostInfo
incoming, outgoing *Relay
sourceKey noiseutil.CipherState
cipher noise.CipherFunc
counter uint64
}
// Separate cipher instances let the reference SendVia path use the same keys and
// counters without rewinding an AEAD instance (including FIPS nonce checks).
func relayBatchKey(cipher noise.CipherFunc, id byte) noiseutil.CipherState {
suite := noise.NewCipherSuite(noise.DH25519, cipher, noise.HashSHA256)
return noiseutil.NewCipherState(noise.UnsafeNewCipherState(suite, [32]byte{id}, 0), cipher)
}
func newRelayBatchPath(t *testing.T, cipher noise.CipherFunc) *relayBatchPath {
t.Helper()
l := test.NewLogger()
conn := &relayBatchConn{}
f := &Interface{
ctx: t.Context(), l: l, hostMap: newHostMap(l), outside: conn,
writers: []udp.Conn{conn}, lightHouse: &LightHouse{l: l},
myVpnNetworksTable: &bart.Lite{}, metricTxDropped: metrics.NewCounter(),
batchers: []*batch.MultiCoalescer{batch.NewMultiCoalescer(&capturingTun{}, l)},
}
f.connectionManager = &connectionManager{
intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{},
}
newHost := func(addr string, remote string, id byte) *HostInfo {
h := &HostInfo{
vpnAddrs: []netip.Addr{netip.MustParseAddr(addr)},
ConnectionState: &ConnectionState{
eKey: relayBatchKey(cipher, id), dKey: relayBatchKey(cipher, id), window: NewBits(ReplayWindow),
},
relayState: RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}},
}
u := netip.MustParseAddrPort(remote)
h.remote.Store(&u)
f.hostMap.Hosts[h.vpnAddrs[0]] = h
return h
}
source := newHost("192.0.2.1", "198.51.100.1:4242", 1)
target := newHost("192.0.2.2", "198.51.100.2:4242", 2)
incoming := &Relay{Type: ForwardingType, State: Established, LocalIndex: 301, RemoteIndex: 302, PeerAddr: target.vpnAddrs[0]}
outgoing := &Relay{Type: ForwardingType, State: Established, LocalIndex: 303, RemoteIndex: 304, PeerAddr: source.vpnAddrs[0]}
source.relayState.InsertRelay(incoming.PeerAddr, incoming.LocalIndex, incoming)
target.relayState.InsertRelay(outgoing.PeerAddr, outgoing.LocalIndex, outgoing)
f.hostMap.Relays[incoming.LocalIndex] = source
f.hostMap.Indexes[100] = source
return &relayBatchPath{f: f, conn: conn, source: source, target: target, incoming: incoming, outgoing: outgoing, sourceKey: relayBatchKey(cipher, 1), cipher: cipher}
}
func (p *relayBatchPath) wrap(t *testing.T, payload []byte) []byte {
t.Helper()
p.counter++
b := header.Encode(make([]byte, header.Len), header.Version, header.Message, header.MessageRelay, p.incoming.LocalIndex, p.counter)
b = append(b, payload...)
b, err := p.sourceKey.EncryptDanger(b, b, nil, p.counter, make([]byte, 12))
require.NoError(t, err)
return b
}
func (p *relayBatchPath) reference(payloads [][]byte) [][]byte {
conn := &relayBatchConn{}
f := &Interface{writers: []udp.Conn{conn}, connectionManager: p.f.connectionManager, l: p.f.l}
h := &HostInfo{ConnectionState: &ConnectionState{eKey: relayBatchKey(p.cipher, 2)}}
u := p.target.GetRemote()
h.remote.Store(&u)
for _, payload := range payloads {
f.SendVia(h, p.outgoing, payload, make([]byte, 12), make([]byte, 0, len(payload)+header.MaxOverhead), false, 0)
}
return conn.packets()
}
func TestRelayReceiveBatch(t *testing.T) {
for _, cipher := range []noise.CipherFunc{noiseutil.CipherAESGCM, noise.CipherChaChaPoly} {
t.Run(cipher.CipherName(), func(t *testing.T) {
p := newRelayBatchPath(t, cipher)
const count = 8
var payloads, wire [][]byte
for i := range count * 2 {
payload := bytes.Repeat([]byte{byte(i)}, 64)
payloads = append(payloads, payload)
wire = append(wire, p.wrap(t, payload))
}
want := p.reference(payloads)
// Model a listener reusing the same slots for the next receive batch.
buffers := make([][]byte, count)
for i := range buffers {
buffers[i] = make([]byte, len(wire[i]))
}
p.conn.read = func(r udp.EncReader, flush func()) {
for b := range 2 {
for i := range count {
copy(buffers[i], wire[b*count+i])
r(p.source.GetRemote(), buffers[i])
require.Len(t, p.conn.writes, b, "forwarding waits for the batch boundary")
}
flush()
require.Len(t, p.conn.writes, b+1, "flush must send before receive slots are reused")
for _, buffer := range buffers {
clear(buffer)
}
}
}
p.f.listenOut(0)
require.Equal(t, want, p.conn.packets())
for _, w := range p.conn.writes {
require.True(t, w.batched)
require.Len(t, w.packets, count)
for _, remote := range w.remotes {
require.Equal(t, p.target.GetRemote(), remote)
}
}
})
}
}
func TestRelayBatchMixedDestinations(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, second := p.target.GetRemote(), netip.MustParseAddrPort("198.51.100.3:4242")
remotes := []netip.AddrPort{first, first, second, second, first}
var payloads, wire [][]byte
for i := range remotes {
payload := []byte{byte(i)}
payloads = append(payloads, payload)
wire = append(wire, p.wrap(t, payload))
}
want := p.reference(payloads)
p.conn.read = func(r udp.EncReader, flush func()) {
for i, remote := range remotes {
p.target.remote.Store(&remote)
r(p.source.GetRemote(), wire[i])
}
flush()
}
p.f.listenOut(0)
require.Equal(t, want, p.conn.packets())
require.Len(t, p.conn.writes, 1, "WriteBatch splits runs by destination, the batch does not need to")
require.Equal(t, remotes, p.conn.writes[0].remotes)
}
func TestRelayBatchCapacity(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
p.conn.read = func(r udp.EncReader, flush func()) {
for i := range batch.SendBatchCap + 1 {
r(p.source.GetRemote(), p.wrap(t, []byte{byte(i)}))
}
require.Len(t, p.conn.writes, 1)
require.Len(t, p.conn.writes[0].packets, batch.SendBatchCap)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 2)
require.Len(t, p.conn.writes[1].packets, 1)
}
func TestRelayBatchRemovedTarget(t *testing.T) {
for _, remove := range []string{"host", "relay", "disestablished"} {
t.Run(remove, func(t *testing.T) {
p := newRelayBatchPath(t, noise.CipherChaChaPoly)
first, removed := p.wrap(t, []byte("first")), p.wrap(t, []byte("removed"))
want := p.reference([][]byte{[]byte("first")})
p.conn.read = func(r udp.EncReader, flush func()) {
r(p.source.GetRemote(), first)
switch remove {
case "host":
delete(p.f.hostMap.Hosts, p.target.vpnAddrs[0])
case "relay":
delete(p.target.relayState.relayForByAddr, p.source.vpnAddrs[0])
case "disestablished":
p.target.relayState.UpdateRelayForByIpState(p.source.vpnAddrs[0], Disestablished)
}
r(p.source.GetRemote(), removed)
flush()
}
p.f.listenOut(0)
require.Len(t, p.conn.writes, 1)
require.Equal(t, want, p.conn.packets(), "only the packet accepted before removal is sent")
})
}
}
+2 -2
View File
@@ -10,8 +10,8 @@ type batchWriter interface {
}
// SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
// One SendBatch is owned by each listenIn goroutine; no locking is needed.
// Slots are backed by an Arena (see its docs)
// Each reader goroutine owns its own SendBatch; no locking is needed.
// Slots can borrow receive buffers or be backed by an Arena (see its docs).
type SendBatch struct {
out batchWriter
bufs [][]byte
+3 -13
View File
@@ -20,7 +20,6 @@ type offloadQueueSet struct {
// usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel.
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
usoEnabled bool
shutdown atomic.Bool
closed atomic.Bool
// l is handed to each queue for its bad-vnet-header drop logging.
l *slog.Logger
@@ -70,17 +69,6 @@ func (c *offloadQueueSet) wakeForShutdown() error {
return err
}
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *offloadQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *offloadQueueSet) Close() error {
if c.closed.Swap(true) {
return nil
@@ -88,7 +76,9 @@ func (c *offloadQueueSet) Close() error {
errs := []error{}
if err := c.Shutdown(); err != nil {
// Signal all readers blocked in poll to wake up and exit.
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
errs = append(errs, err)
}
+3 -13
View File
@@ -16,7 +16,6 @@ type pollQueueSet struct {
// pqi is exactly the same as pq, but stored as the interface type
pqi []Queue
shutdownFd int
shutdown atomic.Bool
closed atomic.Bool
}
@@ -60,17 +59,6 @@ func (c *pollQueueSet) wakeForShutdown() error {
return err
}
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
func (c *pollQueueSet) Shutdown() error {
if c.shutdown.Swap(true) {
return nil
}
for _, x := range c.pq {
x.shutdown.Store(true)
}
return c.wakeForShutdown()
}
func (c *pollQueueSet) Close() error {
if c.closed.Swap(true) {
return nil
@@ -78,7 +66,9 @@ func (c *pollQueueSet) Close() error {
errs := []error{}
if err := c.Shutdown(); err != nil {
// Signal all readers blocked in poll to wake up and exit.
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
if err := c.wakeForShutdown(); err != nil {
errs = append(errs, err)
}
-2
View File
@@ -7,8 +7,6 @@ import (
// QueueSet holds one or many Queue objects and helps close them in an orderly way.
type QueueSet interface {
io.Closer
// Shutdown wakes every reader without releasing any fds, Close only once they have exited
Shutdown() error
Queues() []Queue
// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
-5
View File
@@ -66,8 +66,6 @@ type Offload struct {
// usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD,
// so writers can decide whether emitting GSO_UDP_L4 superpackets is safe.
usoEnabled bool
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool
// gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted
@@ -178,9 +176,6 @@ func (r *Offload) readPacket(block bool) (int, error) {
// This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs).
// Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue.
func (r *Offload) Read() ([]Packet, error) {
if r.shutdown.Load() {
return nil, os.ErrClosed
}
r.pending = r.pending[:0]
r.rxOff = 0
-5
View File
@@ -13,8 +13,6 @@ import (
type Poll struct {
fd int
shutdownFd int
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
shutdown atomic.Bool
closed atomic.Bool
readBuf []byte
@@ -54,9 +52,6 @@ func (t *Poll) blockOnWrite() error {
// config currently lose that batching. blockOn and the EAGAIN plumbing are
// already shared; kept one-packet-per-Read for now to preserve behavior.
func (t *Poll) Read() ([]Packet, error) {
if t.shutdown.Load() {
return nil, os.ErrClosed
}
n, err := t.readOne(t.readBuf)
if err != nil {
return nil, err
-27
View File
@@ -223,30 +223,3 @@ func TestOffloadQueueSet_Close_ClosesShutdownFd(t *testing.T) {
// Second Close must not touch fds (shutdownFd is now -1) and must return nil.
require.NoError(t, qs.Close())
}
// A busy reader never polls the shutdown eventfd, so Read has to stop on its own with data still waiting
func TestQueueSet_ShutdownStopsReaderWithDataWaiting(t *testing.T) {
sets := map[string]func() (QueueSet, error){
"poll": NewPollQueueSet,
"offload": func() (QueueSet, error) { return NewOffloadQueueSet(false, slog.New(slog.DiscardHandler)) },
}
for name, newSet := range sets {
t.Run(name, func(t *testing.T) {
var fds [2]int
require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
t.Cleanup(func() { _ = unix.Close(fds[1]) })
qs, err := newSet()
require.NoError(t, err)
require.NoError(t, qs.Add(fds[0]))
t.Cleanup(func() { _ = qs.Close() })
_, err = unix.Write(fds[1], make([]byte, 64))
require.NoError(t, err)
require.NoError(t, qs.Shutdown())
_, err = qs.Queues()[0].Read()
require.ErrorIs(t, err, os.ErrClosed)
})
}
}
+79 -49
View File
@@ -3,6 +3,7 @@
package overlay
import (
"errors"
"fmt"
"log/slog"
"math/bits"
@@ -11,7 +12,6 @@ import (
"sync"
"sync/atomic"
"syscall"
"time"
"unsafe"
"github.com/gaissmai/bart"
@@ -35,26 +35,6 @@ type tun struct {
// and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool
l *slog.Logger
// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
// fields below it
routesMu sync.Mutex
// routeStates is who held each installable unsafe_route prefix when we last looked
routeStates map[netip.Prefix]routeState
// routeWatch is the routing socket watchRoutes reads, Close closes it
routeWatch *os.File
// routeSweep reconciles the unsafe_routes every routeSweepEvery
routeSweep *time.Timer
routeSweepEvery time.Duration
closed bool
// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
// lock so the route watcher never waits on a reconcile
routeMoversMu sync.Mutex
routeMovers map[netip.Prefix]routeMover
// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
lastRouteEvent atomic.Int64
}
type ifReq struct {
@@ -163,7 +143,6 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: getMTU(l, c),
routeSweepEvery: routeSweepInterval,
l: l,
}
@@ -240,16 +219,6 @@ func utunNameFromFd(fd int) string {
}
func (t *tun) Close() error {
t.routesMu.Lock()
t.closed = true
if t.routeWatch != nil {
_ = t.routeWatch.Close()
}
if t.routeSweep != nil {
t.routeSweep.Stop()
}
t.routesMu.Unlock()
if t.f != nil {
return t.f.Close()
}
@@ -318,22 +287,8 @@ func (t *tun) Activate() error {
return fmt.Errorf("failed to run tun device: %s", err)
}
// Watch before adding so a route removed in between is still noticed
watch, err := openRouteSocket()
if err != nil {
t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
// Unsafe path routes
if err = t.addRoutes(false, nil); err != nil {
if watch != nil {
_ = watch.Close()
}
return err
}
t.startRouteWatch(watch)
return nil
return t.addRoutes(false, nil)
}
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
@@ -438,8 +393,7 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
}
if !initial && !change {
// Any reload puts back the unsafe_routes someone deleted
return t.addRoutes(true, r)
return nil
}
routeTree, err := makeRouteTree(t.l, routes, false)
@@ -557,6 +511,82 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
return nil, nil
}
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
err := addRoute(r.Cidr, linkAddr)
if err != nil {
if errors.Is(err, unix.EEXIST) {
// Every route we added already exists on a reload, only warn about someone else's
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
retErr.Log(t.l)
rl.Failed()
} else {
return retErr
}
}
} else {
t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
}
}
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
linkAddr := t.linkAddr.Load()
if t.hostOwned || linkAddr == nil {
return nil
}
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue
}
err := delRoute(r.Cidr, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
-366
View File
@@ -3,17 +3,12 @@
package overlay
import (
"encoding/binary"
"encoding/hex"
"fmt"
"net"
"net/netip"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
@@ -109,367 +104,6 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
const (
// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
// With no interface address, which x/net/route skips
interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
)
func TestRoutesMayHaveMoved(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
routes := []Route{{Cidr: cidr, Via: via, Install: true}}
// A request like route(8) sends, the kernel echoes it to every routing socket
request := func(typ, flags int, dst, mask [4]byte) []byte {
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: typ,
Flags: flags,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: dst},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
},
}
b, err := m.Marshal()
require.NoError(t, err)
return b
}
raw := func(s string) []byte {
b, err := hex.DecodeString(s)
require.NoError(t, err)
return b
}
lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
routesMayHaveMoved := func(b []byte, routes []Route) bool {
moved, _, _ := routesMayHaveMoved(b, routes)
return moved
}
assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
neighbor := raw(kernelSubnetDelete)
binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
}
// A request names its prefix and who made it, the kernel's own and interface changes name neither
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
cidr := netip.MustParsePrefix("192.168.1.0/24")
routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
m := &netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: unix.RTM_CHANGE,
Flags: unix.RTF_UP,
ID: 1234,
Addrs: []netroute.Addr{
unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
},
}
b, err := m.Marshal()
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved)
assert.Equal(t, cidr, p)
assert.Equal(t, int32(1234), pid)
for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
b, err := hex.DecodeString(s)
require.NoError(t, err)
moved, p, pid := routesMayHaveMoved(b, routes)
assert.True(t, moved, name)
assert.False(t, p.IsValid(), name)
assert.Zero(t, pid, name)
}
}
func TestProcessName(t *testing.T) {
assert.Equal(t, "launchd", processName(1))
self := filepath.Base(os.Args[0])
assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
assert.Empty(t, processName(-1))
}
// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
// the minute between sweeps when it isn't zero.
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
requireRoot(t)
cfg := "tun:\n unsafe_routes:\n"
for _, cidr := range cidrs {
via := "10.250.0.2"
if netip.MustParsePrefix(cidr).Addr().Is6() {
via = "fd00:250::2"
}
cfg += " - route: " + cidr + "\n via: " + via + "\n"
}
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString(cfg))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
require.NoError(t, err)
if sweep != 0 {
tn.routeSweepEvery = sweep
}
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
return tn, c, cfg
}
func requireRoot(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating interfaces and adding routes needs root")
}
}
// routeVia is the index of the interface that routes cidr, 0 if none does
func routeVia(t *testing.T, cidr string) int {
msgs, err := fetchRoutes()
require.NoError(t, err)
return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
}
// routeStateOf is what tn made of cidr when it last looked
func routeStateOf(tn *tun, cidr string) routeState {
tn.routesMu.Lock()
defer tn.routesMu.Unlock()
return tn.routeStates[netip.MustParsePrefix(cidr)]
}
func routeArgs(action, cidr string, extra ...string) []string {
args := []string{"-q", "-n", action}
if netip.MustParsePrefix(cidr).Addr().Is6() {
args = append(args, "-inet6")
}
return append(append(args, "-net", cidr), extra...)
}
func routeCmd(t *testing.T, action, cidr string, extra ...string) {
requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
}
// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
// says so
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
require.NotContains(t, string(out), "writing to routing socket")
}
// cleanupRoute deletes cidr's route when the test ends, whoever holds it
func cleanupRoute(t *testing.T, cidr string) {
t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
}
// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
func routeThroughLo0(t *testing.T, cidr string) int {
requireRoot(t)
routeCmd(t, "add", cidr, "-interface", "lo0")
cleanupRoute(t, cidr)
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
return lo0.Index
}
// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
t.Run(cidr, func(t *testing.T) {
lo0 := routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
})
}
}
// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
cidr := "10.251.1.0/24"
lo0 := routeThroughLo0(t, cidr)
_, c, _ := newRouteTestTun(t, 0, cidr)
res := c.ReloadConfigString("tun: {}\n")
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, lo0, routeVia(t, cidr))
}
// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
cidr := "10.251.3.0/24"
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
time.Sleep(2 * routeRetryDelay)
require.Equal(t, lo0.Index, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
}
// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
cidr := "10.251.4.0/24"
tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
routeCmd(t, "delete", cidr)
assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
// A reload with nothing changed is how to ask for it back
res := c.ReloadConfigString(cfg)
require.NoError(t, res.Err)
assert.False(t, res.Failed)
assert.Equal(t, ours, routeVia(t, cidr))
}
// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
for name, tc := range map[string]struct {
cidr string
sweep time.Duration
misfile func(t *testing.T, tn *tun, cidr string)
}{
// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
}},
// The sweep that caps a table that never settles
"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
tn.reconcileRoutes()
require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
}},
} {
t.Run(name, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, tc.cidr))
routeCmd(t, "delete", tc.cidr)
tc.misfile(t, tn, tc.cidr)
lo0 := routeThroughLo0(t, tc.cidr)
require.Eventually(t, func() bool {
s := routeStateOf(tn, tc.cidr)
return s.kind == routeYielded && s.holder == lo0
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", tc.cidr)
assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
// taken for a hand delete
func TestTunYieldsToARouteChangedAway(t *testing.T) {
for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
t.Run(cidr, func(t *testing.T) {
tn, _, _ := newRouteTestTun(t, 0, cidr)
ours := tn.linkAddr.Load().Index
require.Equal(t, ours, routeVia(t, cidr))
cleanupRoute(t, cidr)
routeCmd(t, "change", cidr, "-interface", "lo0")
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
require.Equal(t, lo0.Index, routeVia(t, cidr))
require.Eventually(t, func() bool {
s := routeStateOf(tn, cidr)
return s.kind == routeYielded && s.holder == lo0.Index
}, 5*time.Second, 50*time.Millisecond)
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
})
}
}
// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
// ethernet configd routed the lan through
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
requireRoot(t)
var feth string
for i := 100; i < 200 && feth == ""; i++ {
if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
feth = fmt.Sprintf("feth%d", i)
}
}
require.NotEmpty(t, feth)
require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
// refuses one named by the feth itself
cidr := "10.251.2.0/24"
routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
cleanupRoute(t, cidr)
ifc, err := net.InterfaceByName(feth)
require.NoError(t, err)
tn, _, _ := newRouteTestTun(t, 0, cidr)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Losing the address first leaves the route, the reconcile finds it still held
require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
time.Sleep(2 * routeRetryDelay)
require.Equal(t, ifc.Index, routeVia(t, cidr))
// Only RTM_IFINFO says the interface went, its route goes with no message of its own
require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
cidr := "10.251.5.0/24"
routeThroughLo0(t, cidr)
tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
tn.routesMu.Lock()
require.NoError(t, tn.routeWatch.Close())
tn.routesMu.Unlock()
routeCmd(t, "delete", cidr)
assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}
// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
func TestTunDevNeedsRestart(t *testing.T) {
for name, want := range map[string]bool{
-5
View File
@@ -832,11 +832,6 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
t.routeTree.Store(newTree)
}
// Shutdown wakes the tun readers without releasing their fds, Close releases them
func (t *tun) Shutdown() error {
return t.readers.Shutdown()
}
func (t *tun) Close() error {
t.closeLock.Lock()
defer t.closeLock.Unlock()
-451
View File
@@ -1,451 +0,0 @@
//go:build !ios && !e2e_testing
package overlay
import (
"errors"
"fmt"
"net"
"net/netip"
"os"
"strconv"
"sync/atomic"
"time"
"unsafe"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/util"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
// or until another interface takes the prefix, after which ours follows theirs like any other.
const (
// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
// replaces a route by deleting and adding, looking in between would find the prefix free
routeRetryDelay = time.Second
// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
// a message for each
routeSweepInterval = time.Minute
// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
routeSocketRcvBuf = 1 << 20
)
type routeStateKind int
const (
routeUnknown routeStateKind = iota
// routeInstalled is our route holding the prefix
routeInstalled
// routeYielded is another interface holding the prefix, ours is added once it lets go
routeYielded
// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
// another interface takes the prefix
routeRemoved
// routeFailed is adding our route failing for some other reason, it is retried
routeFailed
)
type routeState struct {
kind routeStateKind
// holder is the index of the interface holding the prefix when kind is routeYielded
holder int
}
// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
// is looked up as the message arrives, route(8) has exited by the time the table settles
type routeMover struct {
pid int32
name string
}
// logArgs appends m to a log line's args, when we know who it was
func (m routeMover) logArgs(args ...any) []any {
if m.pid == 0 {
return args
}
args = append(args, "pid", m.pid)
if m.name != "" {
args = append(args, "process", m.name)
}
return args
}
// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
func processName(pid int32) string {
kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
if err != nil || kp.Proc.P_pid != pid {
return ""
}
return unix.ByteSliceToString(kp.Proc.P_comm[:])
}
// installRoutes returns the routes we put in the system routing table, one per prefix
func installRoutes(routes []Route) []Route {
seen := make(map[netip.Prefix]struct{}, len(routes))
var out []Route
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// We don't allow route MTUs so only install routes with a via
continue
}
p := r.Cidr.Masked()
if _, ok := seen[p]; ok {
continue
}
seen[p] = struct{}{}
out = append(out, r)
}
return out
}
// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
// interface holds its prefix. It runs on start and on every reload.
func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
// The route tree is still ours, the system routing table is not
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
var msgs []netroute.Message
var msgsErr error
fetched := false
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
err := addRoute(p, linkAddr)
if err == nil {
t.l.Info("Added route", "route", r)
states[p] = routeState{kind: routeInstalled}
continue
}
if !errors.Is(err, unix.EEXIST) {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if !logErrors {
return retErr
}
retErr.Log(t.l)
rl.Failed()
states[p] = routeState{kind: routeFailed}
continue
}
// Every route we added already exists on a reload, only warn about someone else's
if !fetched {
msgs, msgsErr = fetchRoutes()
fetched = true
}
if msgsErr != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
states[p] = prev
continue
}
switch holder := routeInterfaceIn(msgs, p); holder {
case linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case 0:
// Removed since the add, the watcher will see to it
states[p] = prev
default:
if prev.kind != routeYielded || prev.holder != holder {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
}
states[p] = routeState{kind: routeYielded, holder: holder}
}
}
t.routeStates = states
return nil
}
func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
if t.hostOwned {
return nil
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return nil
}
// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
msgs, err := fetchRoutes()
if err != nil {
var left []netip.Prefix
for _, r := range installRoutes(routes) {
left = append(left, r.Cidr)
}
return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
}
for _, r := range installRoutes(routes) {
p := r.Cidr.Masked()
// Whatever happens next, the prefix is no longer one we know about
delete(t.routeStates, p)
if routeInterfaceIn(msgs, p) != linkAddr.Index {
continue
}
err = delRoute(p, linkAddr)
if errors.Is(err, unix.ESRCH) {
// Already gone, which is what we wanted
t.l.Info("Route was already removed", "route", r)
} else if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
rl.Failed()
} else {
t.l.Info("Removed route", "route", r)
}
}
return nil
}
// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
// changed since it last looked, it runs every routeSweepEvery.
func (t *tun) reconcileRoutes() {
t.routesMu.Lock()
defer t.routesMu.Unlock()
linkAddr := t.linkAddr.Load()
if linkAddr == nil || t.closed {
return
}
t.routeMoversMu.Lock()
movers := t.routeMovers
t.routeMovers = nil
t.routeMoversMu.Unlock()
msgs, err := fetchRoutes()
if err != nil {
t.l.Error("Failed to read the system routing table", "error", err)
return
}
states := make(map[netip.Prefix]routeState)
for _, r := range installRoutes(*t.Routes.Load()) {
p := r.Cidr.Masked()
prev := t.routeStates[p]
by := movers[p]
holder := routeInterfaceIn(msgs, p)
switch {
case holder == linkAddr.Index:
states[p] = routeState{kind: routeInstalled}
case holder != 0:
// Another interface taking a prefix we left alone puts ours back in line behind theirs
if prev.kind != routeYielded || prev.holder != holder {
t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
}
states[p] = routeState{kind: routeYielded, holder: holder}
case prev.kind == routeInstalled:
t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
states[p] = routeState{kind: routeRemoved}
case prev.kind == routeRemoved:
states[p] = prev
default:
err := addRoute(p, linkAddr)
switch {
case err == nil:
if prev.kind == routeYielded {
t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
} else {
t.l.Info("Added route", "route", r)
}
states[p] = routeState{kind: routeInstalled}
case errors.Is(err, unix.EEXIST):
// Taken since the fetch, the next look will see by whom
states[p] = prev
default:
if prev.kind != routeFailed {
util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
}
states[p] = routeState{kind: routeFailed}
}
}
}
t.routeStates = states
}
// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
// routing table settles after anything that may have moved a prefix
func (t *tun) startRouteWatch(watch *os.File) {
t.routesMu.Lock()
defer t.routesMu.Unlock()
if t.closed {
if watch != nil {
_ = watch.Close()
}
return
}
t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
if watch != nil {
t.routeWatch = watch
go t.watchRoutes(watch)
}
}
func (t *tun) sweepRoutes() {
// A table still changing gets reconciled once it settles
if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
t.reconcileRoutes()
}
t.routesMu.Lock()
defer t.routesMu.Unlock()
if !t.closed {
t.routeSweep.Reset(t.routeSweepEvery)
}
}
func openRouteSocket() (*os.File, error) {
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
}
// Fewer drops, the sweep catches the rest
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
// Non-blocking puts it on the poller, so Close interrupts a Read
if err = unix.SetNonblock(fd, true); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
}
return os.NewFile(uintptr(fd), "route"), nil
}
// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
func (t *tun) watchRoutes(f *os.File) {
var reconciled atomic.Int64
settle := time.AfterFunc(time.Hour, func() {
reconciled.Store(time.Now().UnixNano())
t.reconcileRoutes()
})
settle.Stop()
defer settle.Stop()
var burst time.Time
self := int32(os.Getpid())
b := make([]byte, os.Getpagesize())
for {
n, err := f.Read(b)
if err != nil {
if !errors.Is(err, os.ErrClosed) {
t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
}
return
}
moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
if !moved {
continue
}
// Our own are already in routeStates
if p.IsValid() && pid != self {
t.routeMoversMu.Lock()
if t.routeMovers == nil {
t.routeMovers = make(map[netip.Prefix]routeMover)
}
t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
t.routeMoversMu.Unlock()
}
now := time.Now()
t.lastRouteEvent.Store(now.UnixNano())
if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
burst = now
}
settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
}
}
// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
// and the pid of the process that made it.
func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
if len(b) < 4 {
return false, p, 0
}
switch int(b[3]) {
case unix.RTM_IFINFO, unix.RTM_DELADDR:
return true, p, 0
case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
default:
return false, p, 0
}
if len(b) < unix.SizeofRtMsghdr {
return false, p, 0
}
hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
// A failed request is reported too, and a scoped route never held the prefix
if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
return false, p, 0
}
if hdr.Pid == 0 {
// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
// could be holding a prefix
return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
}
// Replies to a request carry the requester's padded sockaddrs
msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
if err != nil || len(msgs) != 1 {
return false, p, 0
}
m, ok := msgs[0].(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
return false, p, 0
}
p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
if !ok {
return false, netip.Prefix{}, 0
}
p = p.Masked()
for _, r := range installRoutes(routes) {
if r.Cidr.Masked() == p {
return true, p, hdr.Pid
}
}
return false, netip.Prefix{}, 0
}
// interfaceName is the name of the interface with index i, for logs
func interfaceName(i int) string {
if ifc, err := net.InterfaceByIndex(i); err == nil {
return ifc.Name
}
return strconv.Itoa(i)
}
-4
View File
@@ -528,10 +528,6 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
if !rm.GetAmRelay() {
return
}
if !slices.Contains(h.vpnAddrs, from) {
logMsg.Info("Discarding relay request where relayFrom is not the sending peer")
return
}
peer := rm.hostmap.QueryVpnAddr(target)
if peer == nil {
// Try to establish a connection to this host. If we get a future relay request,
-57
View File
@@ -7,10 +7,8 @@ import (
"testing"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestStartRelaysLogDedupe verifies that repeated attempts with the same relay set drop the log
@@ -97,58 +95,3 @@ func TestStartRelaysLogDedupe(t *testing.T) {
assert.NotContains(t, buf.String(), msg, "should not log when we shortcut out")
})
}
// TestHandleCreateRelayRequestRejectsSpoofedFrom verifies that a relay only forwards a CreateRelayRequest
// when relayFrom is the peer that sent it. Otherwise any peer could reset another host's established
// relay back to Requested, or allocate unbounded relay entries on the target's hostinfo.
func TestHandleCreateRelayRequestRejectsSpoofedFrom(t *testing.T) {
l := test.NewLogger()
relayAddr := netip.MustParseAddr("10.128.0.128")
malloryAddr := netip.MustParseAddr("10.128.0.3")
victimAddr := netip.MustParseAddr("10.128.0.1")
targetAddr := netip.MustParseAddr("10.128.0.2")
addrTable := new(bart.Lite)
addrTable.Insert(netip.PrefixFrom(relayAddr, relayAddr.BitLen()))
hm := newHostMap(l)
f := &Interface{myVpnAddrsTable: addrTable, hostMap: hm}
rm := &relayManager{l: l, hostmap: hm}
rm.amRelay.Store(true)
newRelayState := func() RelayState {
return RelayState{relayForByAddr: map[netip.Addr]*Relay{}, relayForByIdx: map[uint32]*Relay{}}
}
mallory := &HostInfo{vpnAddrs: []netip.Addr{malloryAddr}, localIndexId: 1, relayState: newRelayState()}
target := &HostInfo{
vpnAddrs: []netip.Addr{targetAddr},
localIndexId: 2,
remotes: NewRemoteList([]netip.Addr{targetAddr}, nil),
relayState: newRelayState(),
}
target.SetRemote(netip.MustParseAddrPort("192.0.2.2:4242"))
hm.unlockedAddHostInfo(mallory, f)
hm.unlockedAddHostInfo(target, f)
// The victim already has a working relay to the target through us.
_, err := AddRelay(l, target, hm, victimAddr, nil, ForwardingType, Established)
require.NoError(t, err)
relaysBefore := len(hm.Relays)
for i := range 10 {
spoofed := netip.AddrFrom4([4]byte{10, 128, 1, byte(i)})
for _, from := range []netip.Addr{victimAddr, spoofed} {
rm.handleCreateRelayRequest(cert.Version2, mallory, f, &NebulaControl{
Type: NebulaControl_CreateRelayRequest,
InitiatorRelayIndex: uint32(100 + i),
RelayFromAddr: netAddrToProtoAddr(from),
RelayToAddr: netAddrToProtoAddr(targetAddr),
})
}
}
r, ok := target.relayState.QueryRelayForByIp(victimAddr)
require.True(t, ok)
assert.Equal(t, Established, r.State, "victim's relay must not be reset by a request from another peer")
assert.Len(t, hm.Relays, relaysBefore, "spoofed requests must not allocate relay entries")
assert.Empty(t, mallory.relayState.CopyAllRelayFor(), "spoofed requests must not create state on the sender")
}
+19 -124
View File
@@ -2,7 +2,6 @@ package nebula
import (
"context"
"errors"
"log/slog"
"net"
"net/netip"
@@ -59,8 +58,6 @@ type cacheV6 struct {
type hostnamePort struct {
name string
port uint16
// addr is set when name is an ip literal, these never need a lookup
addr netip.Addr
}
type hostnamesResults struct {
@@ -68,20 +65,15 @@ type hostnamesResults struct {
network string
lookupTimeout time.Duration
cancelFn func()
resolveNow chan struct{}
l *slog.Logger
ips atomic.Pointer[map[netip.AddrPort]struct{}]
// known is the last known good addresses per hostname, kept so a reload can carry them forward
known atomic.Pointer[map[hostnamePort][]netip.AddrPort]
}
// NewHostnameResults starts resolving hostPorts. Any hostnames also in prev start with its last known good addresses.
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, prev *hostnamesResults, onUpdate func()) (*hostnamesResults, error) {
func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, onUpdate func()) (*hostnamesResults, error) {
r := &hostnamesResults{
hostnames: make([]hostnamePort, len(hostPorts)),
network: network,
lookupTimeout: timeout,
resolveNow: make(chan struct{}, 1),
l: l,
}
@@ -89,13 +81,6 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
// DNS lookups for hostnames that aren't hardcoded IP's will happen in a background goroutine.
performBackgroundLookup := false
ips := map[netip.AddrPort]struct{}{}
known := make([][]netip.AddrPort, len(hostPorts))
var prevKnown map[hostnamePort][]netip.AddrPort
if prev != nil && prev.network == network {
if p := prev.known.Load(); p != nil {
prevKnown = *p
}
}
for idx, hostPort := range hostPorts {
rIp, sPort, err := net.SplitHostPort(hostPort)
@@ -113,19 +98,13 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
if err != nil {
// This address is a hostname, not an IP address
performBackgroundLookup = true
known[idx] = prevKnown[r.hostnames[idx]]
for _, a := range known[idx] {
ips[a] = struct{}{}
}
continue
}
// Save the IP address immediately
r.hostnames[idx].addr = addr
ips[netip.AddrPortFrom(addr, uint16(iPort))] = struct{}{}
}
r.ips.Store(&ips)
r.storeKnown(known)
// Time for the DNS lookup goroutine
if performBackgroundLookup {
@@ -135,12 +114,21 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
go func() {
defer ticker.Stop()
for {
r.resolve(newCtx, known)
r.storeKnown(known)
netipAddrs := map[netip.AddrPort]struct{}{}
for _, addrs := range known {
for _, hostPort := range r.hostnames {
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, r.lookupTimeout)
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, r.network, hostPort.name)
timeoutCancel()
if err != nil {
l.Error("DNS resolution failed for static_map host",
"hostname", hostPort.name,
"network", r.network,
"error", err,
)
continue
}
for _, a := range addrs {
netipAddrs[a] = struct{}{}
netipAddrs[netip.AddrPortFrom(a.Unmap(), hostPort.port)] = struct{}{}
}
}
origSet := r.ips.Load()
@@ -171,7 +159,7 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
case <-newCtx.Done():
return
case <-ticker.C:
case <-r.resolveNow:
continue
}
}
}()
@@ -180,84 +168,6 @@ func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, ne
return r, nil
}
// resolve looks up every hostname in parallel and folds the results into known
func (hr *hostnamesResults) resolve(ctx context.Context, known [][]netip.AddrPort) {
var wg sync.WaitGroup
for i, hostPort := range hr.hostnames {
if hostPort.addr.IsValid() {
known[i] = []netip.AddrPort{netip.AddrPortFrom(hostPort.addr, hostPort.port)}
continue
}
wg.Go(func() {
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, hr.lookupTimeout)
addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, hr.network, hostPort.name)
timeoutCancel()
known[i] = applyLookup(known[i], hostPort.port, addrs, err)
// A canceled ctx means we are reloading or shutting down, not that DNS failed
if err != nil && ctx.Err() == nil {
hr.l.Error("DNS resolution failed for static_map host",
"hostname", hostPort.name,
"network", hr.network,
"error", err,
"usingAddrs", known[i],
)
}
})
}
wg.Wait()
}
// applyLookup returns the addresses to use for a hostname given the previous ones and a lookup result.
// Only a not found answer clears them, a stale lighthouse address costs a handshake but an empty set costs everything.
// darwin also reports not found while offline, that's harmless since nothing is reachable and the next handshake
// triggers a fresh lookup once we're back.
func applyLookup(prev []netip.AddrPort, port uint16, addrs []netip.Addr, err error) []netip.AddrPort {
if err != nil {
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
return nil
}
return prev
}
out := make([]netip.AddrPort, 0, len(addrs))
for _, a := range addrs {
out = append(out, netip.AddrPortFrom(a.Unmap(), port))
}
return out
}
func (hr *hostnamesResults) storeKnown(known [][]netip.AddrPort) {
byName := make(map[hostnamePort][]netip.AddrPort, len(known))
for i, addrs := range known {
byName[hr.hostnames[i]] = addrs
}
hr.known.Store(&byName)
}
// literalAddrs returns the entries that were configured as ip addresses rather than hostnames
func (hr *hostnamesResults) literalAddrs() []netip.AddrPort {
var out []netip.AddrPort
for _, hostPort := range hr.hostnames {
if hostPort.addr.IsValid() {
out = append(out, netip.AddrPortFrom(hostPort.addr, hostPort.port))
}
}
return out
}
// ResolveNow asks the background goroutine to look up the hostnames again without waiting for the next tick
func (hr *hostnamesResults) ResolveNow() {
if hr == nil || hr.resolveNow == nil {
return
}
select {
case hr.resolveNow <- struct{}{}:
default:
}
}
func (hr *hostnamesResults) Cancel() {
if hr != nil && hr.cancelFn != nil {
hr.cancelFn()
@@ -346,13 +256,6 @@ func (r *RemoteList) ResetForOwner(ownerVpnAddr netip.Addr) {
r.shouldRebuild = true
}
// ResolveHostnames kicks a fresh DNS lookup of any static_map hostnames for this host
func (r *RemoteList) ResolveHostnames() {
r.RLock()
defer r.RUnlock()
r.hr.ResolveNow()
}
// ClearHostnameResults cancels the in-flight DNS resolver goroutine (if any) and drops the resolved IP cache.
func (r *RemoteList) ClearHostnameResults() {
r.Lock()
@@ -557,14 +460,10 @@ func (r *RemoteList) unlockedSetV4(ownerVpnIp, vpnIp netip.Addr, to []*V4AddrPor
// Reset the slice
c.reported = c.reported[:0]
// We can't take their array but we can take their pointers.
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
for _, v := range to {
// We can't take their array but we can take their pointers
for _, v := range to[:minInt(len(to), MaxRemotes)] {
if check(vpnIp, v) {
c.reported = append(c.reported, v)
if len(c.reported) == MaxRemotes {
break
}
}
}
}
@@ -609,14 +508,10 @@ func (r *RemoteList) unlockedSetV6(ownerVpnIp, vpnIp netip.Addr, to []*V6AddrPor
// Reset the slice
c.reported = c.reported[:0]
// We can't take their array but we can take their pointers.
// Filter before capping so denied addrs do not crowd out allowed ones reported later in the list.
for _, v := range to {
// We can't take their array but we can take their pointers
for _, v := range to[:minInt(len(to), MaxRemotes)] {
if check(vpnIp, v) {
c.reported = append(c.reported, v)
if len(c.reported) == MaxRemotes {
break
}
}
}
}
-89
View File
@@ -1,15 +1,10 @@
package nebula
import (
"context"
"encoding/binary"
"fmt"
"net"
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -333,55 +328,6 @@ func BenchmarkSortRebuild(b *testing.B) {
})
}
func TestApplyLookup(t *testing.T) {
prev := []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}
// A successful lookup replaces the previous addresses
got := applyLookup(prev, 4242, []netip.Addr{netip.MustParseAddr("::ffff:2.2.2.2")}, nil)
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("2.2.2.2:4242")}, got)
// A not found answer means the record is gone
assert.Empty(t, applyLookup(prev, 4242, nil, &net.DNSError{Err: "no such host", IsNotFound: true}))
// Anything else is not new information, keep what we had
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "i/o timeout", IsTimeout: true}))
assert.Equal(t, prev, applyLookup(prev, 4242, nil, &net.DNSError{Err: "server misbehaving", IsTemporary: true}))
assert.Equal(t, prev, applyLookup(prev, 4242, nil, context.DeadlineExceeded))
}
func TestNewHostnameResults_CarriesKnown(t *testing.T) {
// A canceled context keeps the background lookup from learning anything, as if DNS were down
ctx, cancel := context.WithCancel(context.Background())
cancel()
known := netip.MustParseAddrPort("1.1.1.1:4242")
prev := &hostnamesResults{network: "ip4"}
prev.known.Store(&map[hostnamePort][]netip.AddrPort{{name: "lh.invalid", port: 4242}: {known}})
hostPorts := []string{"lh.invalid:4242", "10.0.0.1:4242"}
literal := netip.MustParseAddrPort("10.0.0.1:4242")
hr, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, prev, func() {})
require.NoError(t, err)
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr.GetAddrs())
assert.Equal(t, []netip.AddrPort{literal}, hr.literalAddrs())
// And again from the new one, before it has finished a lookup of its own
hr2, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, hostPorts, hr, func() {})
require.NoError(t, err)
assert.ElementsMatch(t, []netip.AddrPort{known, literal}, hr2.GetAddrs())
// A port change is a different entry
hr3, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip4", time.Second, []string{"lh.invalid:4243"}, prev, func() {})
require.NoError(t, err)
assert.Empty(t, hr3.GetAddrs())
// So is a network change
hr4, err := NewHostnameResults(ctx, test.NewLogger(), time.Hour, "ip6", time.Second, hostPorts, prev, func() {})
require.NoError(t, err)
assert.Equal(t, []netip.AddrPort{literal}, hr4.GetAddrs())
}
func newIp4AndPortFromString(s string) *V4AddrPort {
a := netip.MustParseAddrPort(s)
v4Addr := a.Addr().As4()
@@ -400,38 +346,3 @@ func newIp6AndPortFromString(s string) *V6AddrPort {
Port: uint32(a.Port()),
}
}
func TestRemoteList_SetFiltersBeforeCap(t *testing.T) {
owner := netip.MustParseAddr("0.0.0.0")
rl := NewRemoteList([]netip.Addr{owner}, nil)
// MaxRemotes denied addrs first, then the only allowed one, then enough allowed to exceed the cap
var v4 []*V4AddrPort
for i := 0; i < MaxRemotes; i++ {
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("172.17.0.%d:4242", i+1)))
}
for i := 0; i < MaxRemotes+1; i++ {
v4 = append(v4, newIp4AndPortFromString(fmt.Sprintf("70.199.182.%d:4242", i+1)))
}
rl.unlockedSetV4(owner, owner, v4, func(_ netip.Addr, a *V4AddrPort) bool {
return protoV4AddrPortToNetAddrPort(a).Addr().As4()[0] != 172
})
var v6 []*V6AddrPort
for i := 0; i < MaxRemotes; i++ {
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[fd00::%d]:4242", i+1)))
}
for i := 0; i < MaxRemotes+1; i++ {
v6 = append(v6, newIp6AndPortFromString(fmt.Sprintf("[2001:db8::%d]:4242", i+1)))
}
rl.unlockedSetV6(owner, owner, v6, func(_ netip.Addr, a *V6AddrPort) bool {
return protoV6AddrPortToNetAddrPort(a).Addr().As16()[0] != 0xfd
})
rl.Rebuild([]netip.Prefix{})
assert.Len(t, rl.addrs, MaxRemotes*2)
for _, a := range rl.addrs {
assert.NotEqual(t, byte(172), a.Addr().As16()[12], "denied v4 addr %v was kept", a)
assert.NotEqual(t, byte(0xfd), a.Addr().As16()[0], "denied v6 addr %v was kept", a)
}
}
+743
View File
@@ -1,17 +1,59 @@
package nebula
import (
"bytes"
"encoding/json"
"errors"
"flag"
"fmt"
"log/slog"
"maps"
"net"
"net/netip"
"os"
"slices"
"sort"
"strings"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/sshd"
)
type sshListHostMapFlags struct {
Json bool
Pretty bool
ByIndex bool
}
type sshPrintCertFlags struct {
Json bool
Pretty bool
Raw bool
}
type sshPrintTunnelFlags struct {
Pretty bool
}
type sshChangeRemoteFlags struct {
Address string
}
type sshCloseTunnelFlags struct {
LocalOnly bool
}
type sshCreateTunnelFlags struct {
Address string
}
type sshDeviceInfoFlags struct {
Json bool
Pretty bool
}
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
if c.GetBool("sshd.enabled", false) {
@@ -185,3 +227,704 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reloa
return runner, nil
}
func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
ssh.RegisterCommand(&sshd.Command{
Name: "list-hostmap",
ShortDescription: "List all known previously connected hosts",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshListHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshListHostMap(f.hostMap, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "list-pending-hostmap",
ShortDescription: "List all handshaking hosts",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshListHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshListHostMap(f.handshakeManager, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "list-lighthouse-addrmap",
ShortDescription: "List all lighthouse map entries",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshListHostMapFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshListLighthouseMap(f.lightHouse, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "reload",
ShortDescription: "Reloads configuration from disk, same as sending HUP to the process",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshReload(l, c, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "log-level",
ShortDescription: "Gets or sets the current log level",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshLogLevel(l, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "log-format",
ShortDescription: "Gets or sets the current log format",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshLogFormat(l, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "version",
ShortDescription: "Prints the currently running version of nebula",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshVersion(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "device-info",
ShortDescription: "Prints information about the network device.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshDeviceInfoFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshDeviceInfo(f, fs, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "print-cert",
ShortDescription: "Prints the current certificate being used or the certificate for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshPrintCertFlags{}
fl.BoolVar(&s.Json, "json", false, "outputs as json")
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
fl.BoolVar(&s.Raw, "raw", false, "raw prints the PEM encoded certificate, not compatible with -json or -pretty")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshPrintCert(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "print-tunnel",
ShortDescription: "Prints json details about a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshPrintTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshPrintTunnel(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "print-relays",
ShortDescription: "Prints json details about all relay info",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshPrintTunnelFlags{}
fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshPrintRelays(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "change-remote",
ShortDescription: "Changes the remote address used in the tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshChangeRemoteFlags{}
fl.StringVar(&s.Address, "address", "", "The new remote address, ip:port")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshChangeRemote(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "close-tunnel",
ShortDescription: "Closes a tunnel for the provided vpn addr",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshCloseTunnelFlags{}
fl.BoolVar(&s.LocalOnly, "local-only", false, "Disables notifying the remote that the tunnel is shutting down")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshCloseTunnel(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "create-tunnel",
ShortDescription: "Creates a tunnel for the provided vpn address",
Help: "The lighthouses will be queried for real addresses but you can provide one as well.",
Flags: func() (*flag.FlagSet, any) {
fl := flag.NewFlagSet("", flag.ContinueOnError)
s := sshCreateTunnelFlags{}
fl.StringVar(&s.Address, "address", "", "Optionally provide a real remote address, ip:port ")
return fl, &s
},
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshCreateTunnel(f, fs, a, w)
},
})
ssh.RegisterCommand(&sshd.Command{
Name: "query-lighthouse",
ShortDescription: "Query the lighthouses for the provided vpn address",
Help: "This command is asynchronous. Only currently known udp addresses will be printed.",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshQueryLighthouse(f, fs, a, w)
},
})
}
func sshListHostMap(hl controlHostLister, a any, w sshd.StringWriter) error {
fs, ok := a.(*sshListHostMapFlags)
if !ok {
return nil
}
var hm []ControlHostInfo
if fs.ByIndex {
hm = listHostMapIndexes(hl)
} else {
hm = listHostMapHosts(hl)
}
sort.Slice(hm, func(i, j int) bool {
return hm[i].VpnAddrs[0].Compare(hm[j].VpnAddrs[0]) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
err := js.Encode(hm)
if err != nil {
return nil
}
} else {
for _, v := range hm {
err := w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddrs, v.RemoteAddrs))
if err != nil {
return err
}
}
}
return nil
}
func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) error {
fs, ok := a.(*sshListHostMapFlags)
if !ok {
return nil
}
type lighthouseInfo struct {
VpnAddr string `json:"vpnAddr"`
Addrs *CacheMap `json:"addrs"`
}
lightHouse.RLock()
addrMap := make([]lighthouseInfo, len(lightHouse.addrMap))
x := 0
for k, v := range lightHouse.addrMap {
addrMap[x] = lighthouseInfo{
VpnAddr: k.String(),
Addrs: v.CopyCache(),
}
x++
}
lightHouse.RUnlock()
sort.Slice(addrMap, func(i, j int) bool {
return strings.Compare(addrMap[i].VpnAddr, addrMap[j].VpnAddr) < 0
})
if fs.Json || fs.Pretty {
js := json.NewEncoder(w.GetWriter())
if fs.Pretty {
js.SetIndent("", " ")
}
err := js.Encode(addrMap)
if err != nil {
return nil
}
} else {
for _, v := range addrMap {
b, err := json.Marshal(v.Addrs)
if err != nil {
return err
}
err = w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddr, string(b)))
if err != nil {
return err
}
}
}
return nil
}
func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
return w.WriteLine(fmt.Sprintf("%s", ifce.version))
}
func sshQueryLighthouse(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
var cm *CacheMap
rl := ifce.lightHouse.Query(vpnAddr)
if rl != nil {
cm = rl.CopyCache()
}
return json.NewEncoder(w.GetWriter()).Encode(cm)
}
func sshCloseTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
flags, ok := fs.(*sshCloseTunnelFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
if !flags.LocalOnly {
ifce.send(
header.CloseTunnel,
0,
hostInfo.ConnectionState,
hostInfo,
[]byte{},
make([]byte, 12, 12),
make([]byte, mtu),
)
}
ifce.closeTunnel(hostInfo)
return w.WriteLine("Closed")
}
func sshCreateTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
flags, ok := fs.(*sshCreateTunnelFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return w.WriteLine(fmt.Sprintf("Tunnel already exists"))
}
hostInfo = ifce.handshakeManager.QueryVpnAddr(vpnAddr)
if hostInfo != nil {
return w.WriteLine(fmt.Sprintf("Tunnel already handshaking"))
}
var addr netip.AddrPort
if flags.Address != "" {
addr, err = netip.ParseAddrPort(flags.Address)
if err != nil {
return w.WriteLine("Address could not be parsed")
}
}
// Set the remote from the callback, which StartHandshake runs under the handshake manager lock
// before the hostinfo reaches the handshake loop or the static-host trigger, so the address is
// in the RemoteList before the first handshake is sent.
ifce.handshakeManager.StartHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
if addr.IsValid() {
hh.hostinfo.SetRemote(addr)
}
})
return w.WriteLine("Created")
}
func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
flags, ok := fs.(*sshChangeRemoteFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
if flags.Address == "" {
return w.WriteLine("No address was provided")
}
addr, err := netip.ParseAddrPort(flags.Address)
if err != nil {
return w.WriteLine("Address could not be parsed")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0]))
}
hostInfo.SetRemote(addr)
return w.WriteLine("Changed")
}
func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetLevel() slog.Level
SetLevel(slog.Level)
})
if !ok {
return w.WriteLine("Log level is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
level, err := logging.ParseLevel(strings.ToLower(a[0]))
if err != nil {
return w.WriteLine(fmt.Sprintf("Unknown log level %s. Possible log levels: trace, debug, info, warn, error", a))
}
ctrl.SetLevel(level)
return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel())))
}
func sshLogFormat(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error {
ctrl, ok := l.Handler().(interface {
GetFormat() string
SetFormat(string) error
})
if !ok {
return w.WriteLine("Log format is not reconfigurable on this logger")
}
if len(a) == 0 {
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
if err := ctrl.SetFormat(strings.ToLower(a[0])); err != nil {
return err
}
return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat()))
}
func sshPrintCert(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
args, ok := fs.(*sshPrintCertFlags)
if !ok {
return nil
}
cert := ifce.pki.getCertState().GetDefaultCertificate()
if len(a) > 0 {
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
cert = hostInfo.GetCert().Certificate
}
if args.Json || args.Pretty {
b, err := cert.MarshalJSON()
if err != nil {
return nil
}
if args.Pretty {
buf := new(bytes.Buffer)
err := json.Indent(buf, b, "", " ")
b = buf.Bytes()
if err != nil {
return nil
}
}
return w.WriteBytes(b)
}
if args.Raw {
b, err := cert.MarshalPEM()
if err != nil {
return nil
}
return w.WriteBytes(b)
}
return w.WriteLine(cert.String())
}
func sshPrintRelays(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
args, ok := fs.(*sshPrintTunnelFlags)
if !ok {
w.WriteLine(fmt.Sprintf("sshPrintRelays failed to convert args type"))
return nil
}
relays := map[uint32]*HostInfo{}
ifce.hostMap.Lock()
maps.Copy(relays, ifce.hostMap.Relays)
ifce.hostMap.Unlock()
type RelayFor struct {
Error error
Type string
State string
PeerAddr netip.Addr
LocalIndex uint32
RemoteIndex uint32
RelayedThrough []netip.Addr
}
type RelayOutput struct {
NebulaAddr netip.Addr
RelayForAddrs []RelayFor
}
type CmdOutput struct {
Relays []*RelayOutput
}
co := CmdOutput{}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
for k, v := range relays {
ro := RelayOutput{NebulaAddr: v.vpnAddrs[0]}
co.Relays = append(co.Relays, &ro)
relayHI := ifce.hostMap.QueryVpnAddr(v.vpnAddrs[0])
if relayHI == nil {
ro.RelayForAddrs = append(ro.RelayForAddrs, RelayFor{Error: errors.New("could not find hostinfo")})
continue
}
for _, vpnAddr := range relayHI.relayState.CopyRelayForIps() {
rf := RelayFor{Error: nil}
r, ok := relayHI.relayState.GetRelayForByAddr(vpnAddr)
if ok {
t := ""
switch r.Type {
case ForwardingType:
t = "forwarding"
case TerminalType:
t = "terminal"
default:
t = "unknown"
}
s := ""
switch r.State {
case Requested:
s = "requested"
case Established:
s = "established"
default:
s = "unknown"
}
rf.LocalIndex = r.LocalIndex
rf.RemoteIndex = r.RemoteIndex
rf.PeerAddr = r.PeerAddr
rf.Type = t
rf.State = s
if rf.LocalIndex != k {
rf.Error = fmt.Errorf("hostmap LocalIndex '%v' does not match RelayState LocalIndex", k)
}
}
relayedHI := ifce.hostMap.QueryVpnAddr(vpnAddr)
if relayedHI != nil {
rf.RelayedThrough = append(rf.RelayedThrough, relayedHI.relayState.CopyRelayIps()...)
}
ro.RelayForAddrs = append(ro.RelayForAddrs, rf)
}
}
err := enc.Encode(co)
if err != nil {
return err
}
return nil
}
func sshPrintTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error {
args, ok := fs.(*sshPrintTunnelFlags)
if !ok {
return nil
}
if len(a) == 0 {
return w.WriteLine("No vpn address was provided")
}
vpnAddr, err := netip.ParseAddr(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
if !vpnAddr.IsValid() {
return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0]))
}
hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr)
if hostInfo == nil {
return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0]))
}
enc := json.NewEncoder(w.GetWriter())
if args.Pretty {
enc.SetIndent("", " ")
}
return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.GetPreferredRanges()))
}
func sshDeviceInfo(ifce *Interface, fs any, w sshd.StringWriter) error {
data := struct {
Name string `json:"name"`
Cidr []netip.Prefix `json:"cidr"`
}{
Name: ifce.inside.Name(),
Cidr: make([]netip.Prefix, len(ifce.inside.Networks())),
}
copy(data.Cidr, ifce.inside.Networks())
flags, ok := fs.(*sshDeviceInfoFlags)
if !ok {
return fmt.Errorf("internal error: expected flags to be sshDeviceInfoFlags but was %+v", fs)
}
if flags.Json || flags.Pretty {
js := json.NewEncoder(w.GetWriter())
if flags.Pretty {
js.SetIndent("", " ")
}
return js.Encode(data)
} else {
return w.WriteLine(fmt.Sprintf("name=%v cidr=%v", data.Name, data.Cidr))
}
}
// sshReload reloads the config and writes the result to the client. It reloads even if the client has gone away
func sshReload(l *slog.Logger, c *config.C, w sshd.StringWriter) error {
writeErr := w.WriteLine("Reloading config")
res := c.ReloadConfig()
res.Log(l)
if err := w.WriteLine(res.String()); writeErr == nil {
writeErr = err
}
return writeErr
}
-57
View File
@@ -1,57 +0,0 @@
package nebula
import (
"bytes"
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
)
// TestSSHCommandSurface is a characterization test for the ssh command surface.
// It pins the registered command set, their descriptions, help text and flag definitions
// so that moving the registry to a transport neutral package is provably a relocation
// rather than a rewrite. Regenerate with `go test . -run TestSSHCommandSurface -update`.
func TestSSHCommandSurface(t *testing.T) {
got := renderCommandSurface(registeredCommands(t).All())
test.Golden(t, filepath.Join("testdata", "ssh_commands.golden"), got)
}
// registeredCommands returns a registry holding every command nebula serves. The
// Interface is never dereferenced: attachCommands only closes over it.
func registeredCommands(t *testing.T) *commands.Registry {
t.Helper()
l := test.NewLogger()
reg := commands.New()
attachCommands(l, config.NewC(l), reg, &Interface{})
return reg
}
func renderCommandSurface(cmds []*commands.Command) string {
lines := make([]string, 0, len(cmds))
for _, cmd := range cmds {
b := &bytes.Buffer{}
fmt.Fprintf(b, "%s\n", cmd.Name)
fmt.Fprintf(b, " short: %s\n", cmd.ShortDescription)
fmt.Fprintf(b, " help: %s\n", cmd.Help)
if cmd.Flags != nil {
fl, _ := cmd.Flags()
if fl != nil {
fb := &bytes.Buffer{}
fl.SetOutput(fb)
fl.PrintDefaults()
fmt.Fprintf(b, " flags:\n%s", fb.String())
}
}
lines = append(lines, b.String())
}
// In the order All gives, so the golden pins display order too.
return strings.Join(lines, "")
}
+9 -20
View File
@@ -7,7 +7,6 @@ import (
"path/filepath"
"testing"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
@@ -27,7 +26,7 @@ func (w *lineWriter) Write(s string) error { return w.WriteLine(s) }
func (w *lineWriter) WriteBytes(b []byte) error { return w.WriteLine(string(b)) }
func (w *lineWriter) GetWriter() io.Writer { return io.Discard }
func TestReloadCommandReportsTheResult(t *testing.T) {
func TestSSHReloadReportsTheResult(t *testing.T) {
dir := t.TempDir()
write := func(body string) {
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(body), 0o600))
@@ -44,42 +43,32 @@ func TestReloadCommandReportsTheResult(t *testing.T) {
}
})
// reload returns what the client was told. A reload that failed is a commands.Failure
// to the surface, exit 1 over ssh, with the result line as its message; one that
// merely needs a restart is not.
reload := func(failed bool) []string {
reload := func() []string {
w := &lineWriter{}
err := reloadCommand(l, c, w)
if failed {
var f *commands.Failure
require.ErrorAs(t, err, &f)
assert.Equal(t, w.lines[len(w.lines)-1], f.Msg)
} else {
require.NoError(t, err)
}
require.NoError(t, sshReload(l, c, w))
return w.lines
}
write("a: 2\nb: 1")
assert.Equal(t, []string{"Reloading config", "Config reload complete"}, reload(false))
assert.Equal(t, []string{"Reloading config", "Config reload complete"}, reload())
fail = true
write("a: 3\nb: 1")
assert.Equal(t, []string{"Reloading config", "Config reload completed with errors, review the logs"}, reload(true))
assert.Equal(t, []string{"Reloading config", "Config reload completed with errors, review the logs"}, reload())
fail = false
write("a: 3\nb: 2")
assert.Equal(t, []string{"Reloading config", "Config reload was completed but requires a restart to fully apply, keys [b]"}, reload(false))
assert.Equal(t, []string{"Reloading config", "Config reload was completed but requires a restart to fully apply, keys [b]"}, reload())
write("a: [")
lines := reload(true)
lines := reload()
require.Len(t, lines, 2)
assert.Contains(t, lines[1], "Config reload failed, the new config did not load")
assert.Contains(t, lines[1], dir, "names the config that did not load")
}
// A client that went away still gets the reload it asked for
func TestReloadCommandReloadsWhenTheClientIsGone(t *testing.T) {
func TestSSHReloadReloadsWhenTheClientIsGone(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte("a: 1"), 0o600))
l := test.NewLogger()
@@ -89,6 +78,6 @@ func TestReloadCommandReloadsWhenTheClientIsGone(t *testing.T) {
c.RegisterReloadCallback(func(*config.C, *config.Reload) { reloaded = true })
gone := errors.New("client is gone")
require.ErrorIs(t, reloadCommand(l, c, &lineWriter{err: gone}), gone)
require.ErrorIs(t, sshReload(l, c, &lineWriter{err: gone}), gone)
assert.True(t, reloaded)
}
+161
View File
@@ -0,0 +1,161 @@
package sshd
import (
"errors"
"flag"
"fmt"
"sort"
"strings"
"github.com/armon/go-radix"
)
// CommandFlags is a function called before help or command execution to parse command line flags
// It should return a flag.FlagSet instance and a pointer to the struct that will contain parsed flags
type CommandFlags func() (*flag.FlagSet, any)
// CommandCallback is the function called when your command should execute.
// fs will be a a pointer to the struct provided by Command.Flags callback, if there was one. -h and -help are reserved
// and handled automatically for you.
// a will be any unconsumed arguments, if no Command.Flags was available this will be all the flags passed in.
// w is the writer to use when sending messages back to the client.
// If an error is returned by the callback it is logged locally, the callback should handle messaging errors to the user
// where appropriate
type CommandCallback func(fs any, a []string, w StringWriter) error
type Command struct {
Name string
ShortDescription string
Help string
Flags CommandFlags
Callback CommandCallback
}
func execCommand(c *Command, args []string, w StringWriter) error {
var (
fl *flag.FlagSet
fs any
)
if c.Flags != nil {
fl, fs = c.Flags()
if fl != nil {
// SetOutput() here in case fl.Parse dumps usage.
fl.SetOutput(w.GetWriter())
err := fl.Parse(args)
if err != nil {
// fl.Parse has dumped error information to the user via the w writer.
return err
}
args = fl.Args()
}
}
return c.Callback(fs, args, w)
}
func dumpCommands(c *radix.Tree, w StringWriter) {
err := w.WriteLine("Available commands:")
if err != nil {
return
}
cmds := make([]string, 0)
for _, l := range allCommands(c) {
cmds = append(cmds, fmt.Sprintf("%s - %s", l.Name, l.ShortDescription))
}
sort.Strings(cmds)
_ = w.Write(strings.Join(cmds, "\n") + "\n\n")
}
func lookupCommand(c *radix.Tree, sCmd string) (*Command, error) {
cmd, ok := c.Get(sCmd)
if !ok {
return nil, nil
}
command, ok := cmd.(*Command)
if !ok {
return nil, errors.New("failed to cast command")
}
return command, nil
}
func matchCommand(c *radix.Tree, cmd string) []string {
cmds := make([]string, 0)
c.WalkPrefix(cmd, func(found string, v any) bool {
cmds = append(cmds, found)
return false
})
sort.Strings(cmds)
return cmds
}
func allCommands(c *radix.Tree) []*Command {
cmds := make([]*Command, 0)
c.WalkPrefix("", func(found string, v any) bool {
cmd, ok := v.(*Command)
if ok {
cmds = append(cmds, cmd)
}
return false
})
return cmds
}
func helpCallback(commands *radix.Tree, a []string, w StringWriter) (err error) {
// Just typed help
if len(a) == 0 {
dumpCommands(commands, w)
return nil
}
// We are printing a specific commands help text
cmd, err := lookupCommand(commands, a[0])
if err != nil {
return
}
if cmd != nil {
err = w.WriteLine(fmt.Sprintf("%s - %s", cmd.Name, cmd.ShortDescription))
if err != nil {
return err
}
if cmd.Help != "" {
err = w.WriteLine(fmt.Sprintf(" %s", cmd.Help))
if err != nil {
return err
}
}
if cmd.Flags != nil {
fs, _ := cmd.Flags()
if fs != nil {
fs.SetOutput(w.GetWriter())
fs.PrintDefaults()
}
}
return nil
}
err = w.WriteLine("Command not available " + a[0])
if err != nil {
return err
}
return nil
}
func checkHelpArgs(args []string) bool {
for _, a := range args {
if a == "-h" || a == "-help" {
return true
}
}
return false
}
+22 -8
View File
@@ -10,7 +10,7 @@ import (
"sync"
"sync/atomic"
"github.com/slackhq/nebula/commands"
"github.com/armon/go-radix"
"golang.org/x/crypto/ssh"
)
@@ -29,8 +29,9 @@ type SSHServer struct {
trustedKeys map[string]map[string]bool
trustedCAs []ssh.PublicKey
// Available commands.
commands *commands.Registry
// List of available commands
helpCommand *Command
commands *radix.Tree
listener atomic.Pointer[net.Listener]
// runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned
@@ -43,14 +44,14 @@ type SSHServer struct {
ctx context.Context
}
// NewSSHServer creates a new ssh server serving the supplied command registry and
// prepares to listen. The ssh server's context is parented off the supplied ctx so
// cancelling it (e.g. on Control.Stop) tears down active sessions and closes the listener.
func NewSSHServer(ctx context.Context, l *slog.Logger, reg *commands.Registry) (*SSHServer, error) {
// NewSSHServer creates a new ssh server rigged with default commands and prepares to listen.
// The ssh server's context is parented off the supplied ctx so cancelling it
// (e.g. on Control.Stop) tears down active sessions and closes the listener.
func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
s := &SSHServer{
trustedKeys: make(map[string]map[string]bool),
l: l,
commands: reg,
commands: radix.New(),
ctx: ctx,
}
@@ -96,6 +97,14 @@ func NewSSHServer(ctx context.Context, l *slog.Logger, reg *commands.Registry) (
s.authenticate = cc.Authenticate
s.config.Store(s.newConfig())
s.RegisterCommand(&Command{
Name: "help",
ShortDescription: "prints available commands or help <command> for specific usage info",
Callback: func(a any, args []string, w StringWriter) error {
return helpCallback(s.commands, args, w)
},
})
return s, nil
}
@@ -180,6 +189,11 @@ func (s *SSHServer) AddAuthorizedKey(user, pubKey string) error {
return nil
}
// RegisterCommand adds a command that can be run by a user, by default only `help` is available
func (s *SSHServer) RegisterCommand(c *Command) {
s.commands.Insert(c.Name, c)
}
// Run begins listening and accepting connections. Each invocation derives a fresh per-Run context
// from the constructor-supplied ctx so a Stop+Run sequence (used by config reload) starts clean
// rather than carrying a permanently-cancelled context across runs.
+46 -37
View File
@@ -1,11 +1,13 @@
package sshd
import (
"errors"
"fmt"
"log/slog"
"sort"
"strings"
"github.com/slackhq/nebula/commands"
"github.com/anmitsu/go-shlex"
"github.com/armon/go-radix"
"golang.org/x/crypto/ssh"
"golang.org/x/term"
)
@@ -14,37 +16,31 @@ type session struct {
l *slog.Logger
c *ssh.ServerConn
term *term.Terminal
commands *commands.Registry
commands *radix.Tree
cancel func()
}
func NewSession(reg *commands.Registry, conn *ssh.ServerConn, chans <-chan ssh.NewChannel, cancel func(), l *slog.Logger) *session {
func NewSession(commands *radix.Tree, conn *ssh.ServerConn, chans <-chan ssh.NewChannel, cancel func(), l *slog.Logger) *session {
s := &session{
commands: reg.Clone(),
commands: radix.NewFromMap(commands.ToMap()),
l: l,
c: conn,
cancel: cancel,
}
s.commands.Register(LogoutCommand(s.Close))
s.commands.Insert("logout", &Command{
Name: "logout",
ShortDescription: "Ends the current session",
Callback: func(a any, args []string, w StringWriter) error {
s.Close()
return nil
},
})
go s.handleChannels(chans)
return s
}
// LogoutCommand is the logout command a session registers on its clone of the registry,
// with end as what it does. Exported so a test can serve what a session serves.
func LogoutCommand(end func()) *commands.Command {
return &commands.Command{
Name: commands.Logout,
ShortDescription: "Ends the current session",
Callback: func(fs any, a []string, w commands.StringWriter) error {
end()
return nil
},
}
}
func (s *session) handleChannels(chans <-chan ssh.NewChannel) {
defer s.Close()
for newChannel := range chans {
@@ -91,13 +87,9 @@ func (s *session) handleRequests(in <-chan *ssh.Request, channel ssh.Channel) {
}
req.Reply(true, nil)
err := s.dispatchCommand(payload.Value, commands.NewStringWriter(channel))
s.dispatchCommand(payload.Value, &stringWriter{channel})
// A command that failed exits 1, so `ssh host cmd` callers can tell.
status := struct{ Status uint32 }{uint32(0)}
if err != nil {
status.Status = 1
}
channel.SendRequest("exit-status", false, ssh.Marshal(status))
channel.Close()
return
@@ -119,11 +111,12 @@ func (s *session) createTerm(channel ssh.Channel) *term.Terminal {
term.AutoCompleteCallback = func(line string, pos int, key rune) (newLine string, newPos int, ok bool) {
// key 9 is tab
if key == 9 {
cmds := s.commands.Match(line)
cmds := matchCommand(s.commands, line)
if len(cmds) == 1 {
return cmds[0] + " ", len(cmds[0]) + 1, true
}
sort.Strings(cmds)
term.Write([]byte(strings.Join(cmds, "\n") + "\n\n"))
}
@@ -135,7 +128,7 @@ func (s *session) createTerm(channel ssh.Channel) *term.Terminal {
}
func (s *session) handleInput() {
w := commands.NewStringWriter(s.term)
w := &stringWriter{w: s.term}
for {
line, err := s.term.ReadLine()
if err != nil {
@@ -146,20 +139,36 @@ func (s *session) handleInput() {
}
}
// dispatchCommand runs line and returns the command's error, which it has logged unless
// the command already told the user everything: the log line is for the operator reading
// the daemon's log, who learns nothing from a failure the user caused.
func (s *session) dispatchCommand(line string, w commands.StringWriter) error {
err := s.commands.Dispatch(line, w)
if err == nil {
return nil
func (s *session) dispatchCommand(line string, w StringWriter) {
args, err := shlex.Split(line, true)
if err != nil {
return
}
var f *commands.Failure
if !errors.As(err, &f) {
s.l.Debug("Command returned an error", "line", line, "error", err)
if len(args) == 0 {
dumpCommands(s.commands, w)
return
}
return err
c, err := lookupCommand(s.commands, args[0])
if err != nil {
return
}
if c == nil {
err := w.WriteLine(fmt.Sprintf("did not understand: %s", line))
_ = err
dumpCommands(s.commands, w)
return
}
if checkHelpArgs(args) {
s.dispatchCommand(fmt.Sprintf("%s %s", "help", c.Name), w)
return
}
_ = execCommand(c, args[1:], w)
}
func (s *session) Close() {
+1 -8
View File
@@ -1,9 +1,7 @@
package commands
package sshd
import "io"
// StringWriter is how a command talks back to whoever invoked it: one stream, which the
// ssh surface points at its channel.
type StringWriter interface {
WriteLine(string) error
Write(string) error
@@ -15,11 +13,6 @@ type stringWriter struct {
w io.Writer
}
// NewStringWriter returns a StringWriter that writes to w.
func NewStringWriter(w io.Writer) StringWriter {
return &stringWriter{w: w}
}
func (w *stringWriter) WriteLine(s string) error {
return w.Write(s + "\n")
}
+1 -42
View File
@@ -6,16 +6,13 @@ import (
"crypto/rand"
"crypto/rsa"
"encoding/pem"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/commands"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
@@ -26,7 +23,6 @@ import (
type sshdReloadHarness struct {
c *config.C
reg *commands.Registry
server *sshd.SSHServer
keyFile string
signer ssh.Signer
@@ -48,8 +44,7 @@ func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
h.c = config.NewC(l)
require.NoError(t, h.c.LoadString(h.config()))
h.reg = commands.New()
h.server, err = sshd.NewSSHServer(t.Context(), l, h.reg)
h.server, err = sshd.NewSSHServer(t.Context(), l)
require.NoError(t, err)
wireSSHReload(l, h.server, h.c)
run, err := configSSH(l, h.server, h.c, nil)
@@ -243,39 +238,3 @@ func TestSSHDReload_SkippedEntriesFail(t *testing.T) {
})
}
}
// TestSSHExecExitStatusReflectsTheCommand: an exec request exits 1 when the command
// returned an error and 0 otherwise, with the command's output either way.
func TestSSHExecExitStatusReflectsTheCommand(t *testing.T) {
h := newSSHDReloadHarness(t)
h.reg.Register(&commands.Command{
Name: "fails",
Callback: func(fs any, a []string, w commands.StringWriter) error {
_ = w.WriteLine("nope")
return errors.New("failed")
},
})
client, _ := h.connect(t)
sess, err := client.NewSession()
require.NoError(t, err)
out, err := sess.CombinedOutput("fails")
var exit *ssh.ExitError
require.ErrorAs(t, err, &exit)
assert.Equal(t, 1, exit.ExitStatus())
assert.Equal(t, "nope\n", string(out))
sess, err = client.NewSession()
require.NoError(t, err)
out, err = sess.CombinedOutput("help fails")
require.NoError(t, err)
assert.Equal(t, "fails - \n", string(out))
// A word that is not a command is a failure too, with the command list as before.
sess, err = client.NewSession()
require.NoError(t, err)
out, err = sess.CombinedOutput("nope")
require.ErrorAs(t, err, &exit)
assert.Equal(t, 1, exit.ExitStatus())
assert.True(t, strings.HasPrefix(string(out), "did not understand: nope\nAvailable commands:\n"), string(out))
}
-29
View File
@@ -1,29 +0,0 @@
package test
import (
"flag"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Update is the -update flag: Golden rewrites its file instead of comparing against it.
var Update = flag.Bool("update", false, "rewrite golden files instead of comparing against them")
// Golden compares got with the file at path, or rewrites the file when -update is set.
func Golden(t *testing.T, path, got string) {
t.Helper()
if *Update {
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755))
require.NoError(t, os.WriteFile(path, []byte(got), 0644))
return
}
want, err := os.ReadFile(path)
require.NoError(t, err, "missing golden file, run with -update")
assert.Equal(t, string(want), got)
}
-94
View File
@@ -1,94 +0,0 @@
change-remote
short: Changes the remote address used in the tunnel for the provided vpn addr
help:
flags:
-address string
The new remote address, ip:port
close-tunnel
short: Closes a tunnel for the provided vpn addr
help:
flags:
-local-only
Disables notifying the remote that the tunnel is shutting down
create-tunnel
short: Creates a tunnel for the provided vpn address
help: The lighthouses will be queried for real addresses but you can provide one as well.
flags:
-address string
Optionally provide a real remote address, ip:port
device-info
short: Prints information about the network device.
help:
flags:
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
help
short: prints available commands or help <command> for specific usage info
help:
list-hostmap
short: List all known previously connected hosts
help:
flags:
-by-index
gets all hosts in the hostmap from the index table
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
list-lighthouse-addrmap
short: List all lighthouse map entries
help:
flags:
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
list-pending-hostmap
short: List all handshaking hosts
help:
flags:
-by-index
gets all hosts in the hostmap from the index table
-json
outputs as json with more information
-pretty
pretty prints json, assumes -json
log-format
short: Gets or sets the current log format
help:
log-level
short: Gets or sets the current log level
help:
print-cert
short: Prints the current certificate being used or the certificate for the provided vpn addr
help:
flags:
-json
outputs as json
-pretty
pretty prints json, assumes -json
-raw
raw prints the PEM encoded certificate, not compatible with -json or -pretty
print-relays
short: Prints json details about all relay info
help:
flags:
-pretty
pretty prints json
print-tunnel
short: Prints json details about a tunnel for the provided vpn addr
help:
flags:
-pretty
pretty prints json
query-lighthouse
short: Query the lighthouses for the provided vpn address
help: This command is asynchronous. Only currently known udp addresses will be printed.
reload
short: Reloads configuration from disk, same as sending HUP to the process
help:
version
short: Prints the currently running version of nebula
help:
+1 -1
View File
@@ -41,7 +41,7 @@ type Conn interface {
//
// Returns the number of packets successfully written. A destination the kernel rejects costs only
// its own packet, so a short count means some peers were undeliverable, not that the batch failed.
// Not safe for concurrent use on the same Conn.
// Safe for concurrent use on the same Conn.
WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
// ReloadConfig applies the listen settings that can change, r is nil at startup
ReloadConfig(c *config.C, r *config.Reload)
+5 -13
View File
@@ -255,10 +255,6 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
msgs, buffers, names, _ := prepareRawMessages(u.batch, bufSize, cmsgSpace)
for {
// A shut down socket still queues and returns datagrams, a busy reader would never see an empty read
if u.closed.Load() {
return net.ErrClosed
}
if cmsgSpace > 0 {
for i := range msgs {
setMsgControllen(&msgs[i].Hdr, cmsgSpace)
@@ -450,16 +446,12 @@ func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
return nil
}
// Shutdown wakes the reader without releasing the fd, Close only once it has exited
func (u *StdConn) Shutdown() error {
u.closed.Store(true)
// shutdown(2) on an unconnected socket returns ENOTCONN but still wakes the reader, so ignore the error
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
return nil
}
func (u *StdConn) Close() error {
_ = u.Shutdown()
u.closed.Store(true)
// Wake the reader parked in recvmmsg. shutdown(2) on an unconnected socket
// returns ENOTCONN but still wakes it, so ignore the error.
// The reader then sees closed and stops touching the fd, making the Close below safe.
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
return unix.Close(u.sysFd)
}
+5 -74
View File
@@ -76,7 +76,7 @@ func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
}
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
// reader must tear down cleanly on Shutdown() regardless of what the socket has
// reader must tear down cleanly on Close() regardless of what the socket has
// carried. The three cases the report called out:
//
// no traffic ever -> works (shutdown wakes recvmmsg with n==0)
@@ -162,12 +162,11 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
// empty queue with the socket already having received a packet.
time.Sleep(500 * time.Millisecond)
// Tear down the way Interface.Close does, the fd stays open until the reader is gone
start := time.Now()
if err := sc.Shutdown(); err != nil {
t.Fatalf("Shutdown: %v", err)
if err := sc.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
defer func() { _ = sc.Close() }()
close(stop)
select {
case err := <-loopDone:
@@ -180,75 +179,7 @@ func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net
case <-time.After(3 * time.Second):
buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true)
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Shutdown\n%s", name, buf[:n])
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
}
close(stop)
<-trafficDone
}
// A shut down socket still returns queued datagrams, so a reader that is behind has to stop on its own within the
// batch it was working on
func TestListenOutShutdown_BusyReader(t *testing.T) {
const batch = 64
udpSettings := Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Multi: true, Batch: batch}
c, err := NewListener(testLogger(), udpSettings)
if err != nil {
t.Fatalf("NewListener: %v", err)
}
sc := c.(*StdConn)
defer sc.Close()
addr, err := sc.LocalAddr()
if err != nil {
t.Fatalf("LocalAddr: %v", err)
}
stop := make(chan struct{})
defer close(stop)
for range 8 {
send, err := net.Dial("udp", addr.String())
if err != nil {
t.Fatalf("dial: %v", err)
}
defer send.Close()
go func() {
b := make([]byte, 1200)
for {
select {
case <-stop:
return
default:
_, _ = send.Write(b)
}
}
}()
}
var afterShutdown atomic.Int64
loopDone := make(chan error, 1)
go func() {
loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
if sc.closed.Load() {
afterShutdown.Add(1)
}
// Slower than the senders so the queue never drains
for start := time.Now(); time.Since(start) < 200*time.Microsecond; {
}
}, func() {})
}()
time.Sleep(200 * time.Millisecond)
if err := sc.Shutdown(); err != nil {
t.Fatalf("Shutdown: %v", err)
}
select {
case err := <-loopDone:
if !errors.Is(err, net.ErrClosed) {
t.Fatalf("ListenOut returned %v, want net.ErrClosed", err)
}
case <-time.After(3 * time.Second):
t.Fatalf("ListenOut still running 3s after Shutdown, delivered %d packets since", afterShutdown.Load())
}
if n := afterShutdown.Load(); n > batch {
t.Fatalf("delivered %d packets after Shutdown, more than the one batch in flight", n)
}
}
+6 -1
View File
@@ -12,6 +12,7 @@ import (
"net/netip"
"strconv"
"strings"
"sync"
"unsafe"
"golang.org/x/sys/unix"
@@ -20,7 +21,7 @@ import (
// batchWriter owns the sendmmsg(2)/UDP-GSO transmit path for a StdConn: the
// scratch WriteBatch packs mmsghdr entries into, plus the GSO capability
// state probed at socket creation. Each queue has its own StdConn and
// batchWriter, so no locking is needed.
// batchWriter; mu serializes batches from the TUN reader and relay forwarding.
//
// Terminology, smallest to largest:
//
@@ -39,6 +40,7 @@ import (
// chunk the entries packed for one sendmmsg call, at most MaxWriteBatch.
// batch the caller's whole bufs/addrs pair, processed as one or more chunks.
type batchWriter struct {
mu sync.Mutex
fd int
isV4 bool
@@ -192,6 +194,9 @@ func parseRelease(r string) (major, minor int) {
// Returns the number of packets sent. An error means the call itself failed.
// A short count means some destinations were undeliverable.
func (w *batchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()
if len(bufs) != len(addrs) {
return 0, fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
}
@@ -0,0 +1,61 @@
//go:build linux && !android && !e2e_testing
package udp
import (
"net/netip"
"runtime"
"sync"
"testing"
"github.com/stretchr/testify/require"
)
// The TUN reader and relay forwarding share a socket. Exercise their concurrent
// batches against the actual GSO planner and scratch, without relying on a NIC.
func TestWriteBatchConcurrent(t *testing.T) {
const batches = 64
w := newRewindTestWriter()
sent := make(chan [][]byte, 2*batches)
w.sendFn = func(start, n int) (int, error) {
runtime.Gosched()
sent <- capturePrepared(w, start, n)
return n, nil
}
conn := &StdConn{bw: w}
remote := netip.MustParseAddrPort("192.0.2.1:4242")
var wg sync.WaitGroup
start := make(chan struct{})
for producer := range 2 {
wg.Go(func() {
<-start
for i := range batches {
var bufs [][]byte
var addrs []netip.AddrPort
for j := range 4 {
bufs = append(bufs, []byte{byte(producer), byte(i), byte(j), 0})
addrs = append(addrs, remote)
}
n, err := conn.WriteBatch(bufs, addrs)
if err != nil || n != len(bufs) {
t.Errorf("WriteBatch = %d, %v; want %d, nil", n, err, len(bufs))
return
}
}
})
}
close(start)
wg.Wait()
close(sent)
var counts [2]int
for packets := range sent {
require.Len(t, packets, 4)
producer := packets[0][0]
require.Less(t, int(producer), len(counts))
for j, packet := range packets {
require.Equal(t, []byte{producer, byte(counts[producer]), byte(j), 0}, packet)
}
counts[producer]++
}
require.Equal(t, [2]int{batches, batches}, counts)
}