Add a local control socket and nebula ctl (#1975)

Add support for accessing the previously ssh-only debug commands via
`nebula ctl <command>.`

- unix socket (named pipe on windows) for communication
- sockets scoped to the user + an optional group
- in cmd/nebula and cmd/nebula-service, but not nebula.Main

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

cmd/internal/ctl/ctl.go

@@ -0,0 +1,420 @@
+package ctl
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"strings"
+
+	"github.com/slackhq/nebula/commands"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/localapi"
+)
+
+// Word is the argument that selects the control client instead of the daemon.
+const Word = "ctl"
+
+// Subcommand handles the one bare word a daemon binary accepts before its flags. `nebula
+// ctl <command>` runs the client. Any other first argument that is not a flag is a
+// mistyped subcommand, reported with a usage line: the daemon's flag parser would ignore
+// the stray word and start nebula, so `nebula clt` would bring up a second daemon
+// instead of reporting a typo. It returns the exit code and true when the process is
+// done, and false when argv is the daemon's to parse. Both daemon binaries, cmd/nebula
+// and cmd/nebula-service, call it before anything else, so each has a nebula ctl and
+// neither has the hazard. A word that follows the daemon's flags instead is Trailing's.
+func Subcommand(argv []string) (code int, done bool) {
+	if len(argv) < 2 || strings.HasPrefix(argv[1], "-") {
+		return 0, false
+	}
+
+	if argv[1] == Word {
+		return Run(argv[2:]), true
+	}
+
+	return unknownCommand(argv[1]), true
+}
+
+// Trailing handles the words the daemon's flag parser leaves unparsed: flag.Parse stops
+// at the first argument that is not a flag and hands the rest back. rest is flag.Args()
+// after the daemon has parsed, and configPath is the config the daemon's -config named,
+// "" when it did not. Both daemon binaries call it right after flag.Parse, so that
+// `nebula -config x ctl identity` runs the client and `nebula -config x clt` reports a
+// typo, as Subcommand does for a word at the front of the line; a daemon that ignored
+// the words would start in their place.
+//
+// A leading ctl word runs the client on what follows, with the daemon's -config carried
+// over: an operator who typed it meant the daemon running on that config, and the client
+// derives the same socket from it. Any other word is a mistake, reported as Subcommand
+// reports one.
+func Trailing(rest []string, configPath string) (code int, done bool) {
+	if len(rest) == 0 {
+		return 0, false
+	}
+
+	if rest[0] != Word {
+		return unknownCommand(rest[0]), true
+	}
+
+	args := rest[1:]
+	if configPath != "" {
+		// First, so a -config on the ctl side of the line still wins: Parse applies
+		// flags in order and the last value set is the one read.
+		args = append([]string{"-" + commands.FlagConfig, configPath}, args...)
+	}
+
+	return Run(args), true
+}
+
+// unknownCommand reports a bare word that is neither a flag nor ctl, with the one usage
+// line that says what the two forms of the command line are.
+func unknownCommand(word string) int {
+	fmt.Fprintf(os.Stderr, "nebula: unknown command %q\n\n", word)
+	fmt.Fprintf(os.Stderr, "Run nebula with flags to start it, or `nebula %s <command>` to control one that is already running.\n", Word)
+	return 1
+}
+
+// Run implements `nebula ctl <command> [args]`
+//
+// args is everything after the `ctl` word.
+func Run(args []string) int {
+	fs := flag.NewFlagSet("nebula ctl", flag.ContinueOnError)
+	fs.SetOutput(os.Stderr)
+
+	configPath := fs.String(commands.FlagConfig, "", "Path to the config of the running nebula, used to locate its control socket")
+	socketPath := fs.String(commands.FlagSocket, "", "Path to the control socket, instead of the one the config implies")
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage: nebula ctl [flags] <command> [command flags]\n\n")
+		fmt.Fprintf(os.Stderr, "Inspects and controls a nebula already running on this machine.\n")
+		fmt.Fprintf(os.Stderr, "Run it as the user who ran nebula, or as a member of the group %s names.\n\n", localapi.CtlGroupKey)
+		fmt.Fprintf(os.Stderr, "Flags, which may come before or after the command:\n")
+		fs.PrintDefaults()
+		fmt.Fprintf(os.Stderr, "\nRun `nebula ctl help` for the available commands.\n")
+	}
+
+	own, cmdArgs := splitLine(fs, args)
+	if err := fs.Parse(own); err != nil {
+		// fs has already printed the usage, or the problem. Asking for help is not a
+		// failure, and a script that chains on `nebula ctl -h` should not see one.
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 1
+	}
+
+	if len(cmdArgs) == 0 {
+		cmdArgs = []string{"help"}
+	}
+
+	path, err := resolveSocket(*socketPath, *configPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		return 1
+	}
+
+	client := &localapi.Client{Path: path}
+
+	// Ask the daemon what it serves before parsing anything. The flags belong to the
+	// command, which lives over there, so this is what lets the CLI keep its usual shape
+	// while the wire protocol stays a named-parameter API rather than a shipped argv.
+	schema, err := client.Describe()
+	if err != nil {
+		// A daemon that answered was reached, and what it said, or that it could not be
+		// read, is the whole message.
+		var refused *localapi.DaemonError
+		var unread *localapi.ReplyError
+		if errors.As(err, &refused) || errors.As(err, &unread) {
+			fmt.Fprintf(os.Stderr, "nebula: %s\n", err)
+			return 1
+		}
+
+		// Otherwise the operating system said why, and it is passed on as it came: a
+		// permission denied or a missing socket says more about what to do than a
+		// guess at the operator's setup would.
+		fmt.Fprintf(os.Stderr, "nebula: could not reach the control socket at %s: %s\n", path, err)
+		return 1
+	}
+
+	req, printedHelp, err := parseCommand(schema, cmdArgs)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		return 1
+	}
+
+	if printedHelp {
+		return 0
+	}
+
+	res, err := client.Invoke(string(req.name), req.Params, req.Args, wireFormat(req.Format))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "nebula: %s\n", err)
+		return 1
+	}
+
+	out := output(res, req.Format)
+	if out != "" {
+		fmt.Fprint(os.Stdout, out)
+		if !strings.HasSuffix(out, "\n") {
+			fmt.Fprintln(os.Stdout)
+		}
+	}
+
+	// Error set is the one failure signal, whether the request never ran or the command
+	// ran and refused. A refusal asked for in JSON has already gone to stdout as the
+	// {ok, reason} object, and is not said again on stderr; the exit status carries it.
+	if res.Error == "" {
+		return 0
+	}
+
+	if len(res.Result) == 0 && len(res.Text) == 0 {
+		msg := res.Error
+		if !strings.HasSuffix(msg, "\n") {
+			msg += "\n"
+		}
+		fmt.Fprint(os.Stderr, msg)
+	}
+
+	return 1
+}
+
+// output renders what the daemon answered for the format the line asked for. A text
+// request is answered in Text with the bytes the command wrote, printed as they are; a
+// JSON request is answered in Result with the value itself, printed as it came, since a
+// value that happens to be a string is still JSON to the script reading it. Indentation
+// is the client's business: whitespace does not survive a JSON envelope, and the daemon
+// has no reason to care how its answer is laid out. The bytes are re-indented rather
+// than decoded and encoded again, which would sort the keys and round the numbers.
+func output(res *localapi.Response, format commands.Format) string {
+	switch format {
+	case commands.FormatJSON:
+		return string(res.Result)
+	case commands.FormatPrettyJSON:
+		var buf bytes.Buffer
+		if err := json.Indent(&buf, res.Result, "", "    "); err != nil {
+			return string(res.Result)
+		}
+		return buf.String()
+	default:
+		return string(res.Text)
+	}
+}
+
+// ctlRequest is one parsed nebula ctl command line: the command's name, which only the
+// daemon can resolve, and the line as every surface parses it. The Request's Command is
+// nil here, since the CLI holds a schema rather than the command, and its Format is
+// FormatUnset when the line named none, which to nebula ctl means text: its default
+// differs from the API's.
+type ctlRequest struct {
+	name commands.Name
+	commands.Request
+}
+
+// wireFormat is what a request asks the daemon for. Pretty is the client's business,
+// since whitespace does not survive a JSON envelope, so it travels as json. No
+// preference is text, nebula ctl's default: a person gets the human rendering wherever
+// one exists.
+func wireFormat(f commands.Format) string {
+	switch f {
+	case commands.FormatJSON, commands.FormatPrettyJSON:
+		return "json"
+	default:
+		return "text"
+	}
+}
+
+// parseCommand turns the user's command line into a ctlRequest, using the schema the
+// daemon supplied. The line is parsed with the same rules the daemon applies to an ssh
+// line, through ParseArgs, and ParseLine reads what was set into the Request to send, so
+// a line means the same on both.
+//
+// printedHelp reports that help was written for the command instead, and there is
+// nothing to send.
+func parseCommand(schema []commands.CommandSchema, argv []string) (req ctlRequest, printedHelp bool, err error) {
+	name := commands.Name(argv[0])
+
+	var found *commands.CommandSchema
+	for i := range schema {
+		if schema[i].Name == name {
+			found = &schema[i]
+			break
+		}
+	}
+
+	if found == nil {
+		// Let the daemon answer. It knows whether this is a typo or a command it holds
+		// back, and its message for the second case is better than anything guessable
+		// from the schema alone.
+		return ctlRequest{name: name, Request: commands.Request{Args: argv[1:]}}, false, nil
+	}
+
+	// The flag package's own report is a bare message followed by a usage dump in a layout
+	// that does not match WriteCommandHelp, so it is silenced and the error is reported
+	// once, below, with a pointer at the help this command actually prints.
+	fs := flag.NewFlagSet(string(name), flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+
+	for _, f := range found.Flags {
+		if f.Bool {
+			fs.Bool(f.Name, f.Default == "true", f.Usage)
+			continue
+		}
+		fs.String(f.Name, f.Default, f.Usage)
+	}
+
+	operands, err := commands.ParseArgs(fs, argv[1:])
+	if err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			// The same rendering the daemon uses for `help <command>`, so -h and help agree.
+			_ = commands.WriteCommandHelp(*found, os.Stdout)
+			return ctlRequest{}, true, nil
+		}
+
+		// Say what flag did: an unknown flag, a missing value and a malformed value are
+		// different mistakes, and "does not take" would be wrong for two of them.
+		return ctlRequest{}, false, fmt.Errorf("nebula: %s: %s (see 'nebula ctl %s -h')", name, err, name)
+	}
+
+	// Only what the user actually set travels. A parameter left alone is the daemon's
+	// default to apply, not the client's to assert, and the presentation flags are the
+	// client's to interpret, never sent.
+	//
+	// Where the command has no human form the daemon says so, and only once the command
+	// has run: a refusal ("No vpn address was provided") always has a human form, so
+	// telling the user to pass -json before finding out whether the command would even
+	// accept the line sends them to fix the wrong thing. That is why the schema does not
+	// say which commands have a human form: nothing should decide on it before running.
+	return ctlRequest{name: name, Request: commands.ParseLine(fs, operands)}, false, nil
+}
+
+// splitLine divides a nebula ctl line into the client's own flags, for fs to parse, and
+// the command line to send, which begins with the command word. The client's flags may
+// come before the command or after it: `nebula ctl -config x identity` and
+// `nebula ctl identity -config x` mean the same, so an operator who has typed a command
+// and then remembers which daemon they meant does not have to retype the line.
+//
+// Before the command word every flag is the client's, whatever it is, so an unknown one
+// is reported by fs, and -h is the client's usage. After it only the
+// flags fs defines are lifted out, and only those: the command's own flags are unknown
+// here, since they come from the daemon the line has yet to name, and -h after the
+// command asks for that command's help. Which command flags take a value is unknown too,
+// so the client's names are reserved in the commands package and no command can give
+// one a value that would be lifted from here. A "--" before the command is consumed, as
+// fs.Parse would have; one after it belongs to the command line and ends the lifting,
+// so an operand spelled like a client flag can still be passed.
+func splitLine(fs *flag.FlagSet, args []string) (own, cmd []string) {
+	seenCommand := false
+	for i := 0; i < len(args); i++ {
+		a := args[i]
+
+		if a == "--" {
+			if seenCommand {
+				return own, append(cmd, args[i:]...)
+			}
+			seenCommand = true
+			continue
+		}
+
+		if !seenCommand {
+			if len(a) < 2 || a[0] != '-' {
+				seenCommand = true
+				cmd = append(cmd, a)
+				continue
+			}
+
+			own = append(own, a)
+			if f, hasValue := commands.LookupArg(fs, a); f != nil && !hasValue && !commands.IsBoolFlag(f) && i+1 < len(args) {
+				i++
+				own = append(own, args[i])
+			}
+			continue
+		}
+
+		if len(a) >= 2 && a[0] == '-' {
+			// The same reading of the argument ParseArgs makes, so a word this lifts as a
+			// client flag's value is one the daemon's parser would have read the same way.
+			if f, hasValue := commands.LookupArg(fs, a); f != nil {
+				own = append(own, a)
+				if !hasValue && !commands.IsBoolFlag(f) && i+1 < len(args) {
+					i++
+					own = append(own, args[i])
+				}
+				continue
+			}
+		}
+
+		cmd = append(cmd, a)
+	}
+
+	return own, cmd
+}
+
+// resolveSocket decides which socket to talk to. It never searches: the answer is the
+// socket the operator named, or the one the config implies, and nothing else.
+//
+// An explicit flag wins. Otherwise the config is read, the one the operator named or the
+// one found next to the executable, and the path is the socket that config implies: its
+// local_api.control.socket if set, else the same derivation the daemon applies.
+func resolveSocket(socketFlag, configFlag string) (string, error) {
+	if socketFlag != "" {
+		return socketFlag, nil
+	}
+
+	return pathFromConfig(configFlag)
+}
+
+// pathFromConfig returns the socket path the daemon would be serving: the one
+// local_api.control.socket names in the config, so an operator who moved the socket does
+// not have to repeat themselves on every invocation, else the default.
+//
+// A config the operator did not name is a guess: config.DefaultPath looks next to the
+// executable, and a package that installs a default config there while systemd starts
+// the daemon with another will make it the wrong guess. It is still the only answer
+// that involves no searching. A guess that cannot be read is not an error, it is no
+// guess: a member of the socket's group cannot read a root daemon's config, and is
+// answered by the default path, where the daemon serves unless that config moved it.
+// A config the operator named is theirs to make readable.
+func pathFromConfig(configFlag string) (string, error) {
+	return socketFromConfigFile(configFlag, configFlag != "")
+}
+
+func socketFromConfigFile(configPath string, named bool) (string, error) {
+	if !named {
+		p, err := config.DefaultPath()
+		if err != nil {
+			return localapi.DefaultSocketPath, nil
+		}
+		configPath = p
+	}
+
+	// The config is loaded only to find the socket path. Loading logs lines that belong
+	// to the daemon, not to someone running a read only command, and mixing them into
+	// the CLI's output would corrupt a -json result, so they are discarded.
+	c := config.NewC(slog.New(slog.DiscardHandler))
+	if err := c.Load(configPath); err != nil {
+		if !named {
+			return localapi.DefaultSocketPath, nil
+		}
+		// The operating system's error, as it came: permission denied on a root daemon's
+		// config says what to do.
+		return "", fmt.Errorf("nebula: could not read %s: %w", configPath, err)
+	}
+
+	// A daemon running this config serves no socket; dialing the path would only say
+	// that nothing is there.
+	if !localapi.ControlEnabled(c) {
+		return "", fmt.Errorf("nebula: the control socket is disabled in %s by %s", configPath, localapi.CtlEnabledKey)
+	}
+
+	// The same call the daemon makes, so the two agree on the path and on what makes a
+	// path unusable.
+	p, err := localapi.SocketPathFromConfig(c)
+	if err != nil {
+		return "", fmt.Errorf("nebula: %w", err)
+	}
+
+	return p, nil
+}

cmd/internal/ctl/ctl_test.go

@@ -0,0 +1,294 @@
+package ctl
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/slackhq/nebula/commands"
+	"github.com/slackhq/nebula/localapi"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestResolveSocketReportsAnUnreadableConfig: an operator who named a config must hear
+// that it could not be read, rather than have the command steered anywhere else.
+func TestResolveSocketReportsAnUnreadableConfig(t *testing.T) {
+	missing := filepath.Join(t.TempDir(), "typo.yml")
+
+	_, err := resolveSocket("", missing)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), missing)
+}
+
+// TestResolveSocketGuessedConfigNeedNotBeReadable: a config nobody named is a guess, and
+// one that cannot be read, a root daemon's config read by a member of the socket's
+// group, means the default path rather than an error.
+func TestResolveSocketGuessedConfigNeedNotBeReadable(t *testing.T) {
+	missing := filepath.Join(t.TempDir(), "config.yml")
+
+	p, err := socketFromConfigFile(missing, false)
+	require.NoError(t, err)
+	assert.Equal(t, localapi.DefaultSocketPath, p)
+
+	_, err = socketFromConfigFile(missing, true)
+	require.ErrorContains(t, err, missing)
+}
+
+// nowhereSocket is a socket name nothing listens on, spelled the way the platform
+// requires: a full pipe name on windows, an absolute path elsewhere.
+var nowhereSocket = map[bool]string{true: `\\.\pipe\nebula-nowhere`, false: "/nowhere/ctl.sock"}[runtime.GOOS == "windows"]
+
+// namedSocketConfig is a config that names nowhereSocket as the control socket. Single
+// quotes keep a pipe name's backslashes literal in YAML.
+var namedSocketConfig = "local_api:\n  control:\n    socket: '" + nowhereSocket + "'\n"
+
+// TestResolveSocketTrustsANamedConfig: a config the operator named yields the socket it
+// implies even when nothing is listening there, so a dial failure names that socket.
+func TestResolveSocketTrustsANamedConfig(t *testing.T) {
+	cfg := filepath.Join(t.TempDir(), "config.yml")
+	require.NoError(t, os.WriteFile(cfg, []byte(namedSocketConfig), 0600))
+
+	p, err := resolveSocket("", cfg)
+	require.NoError(t, err)
+	assert.Equal(t, nowhereSocket, p)
+}
+
+// TestResolveSocketSaysTheSocketIsDisabled: a config that turns the control socket off
+// is answered with that, not with a dial to a path nothing serves. -socket still wins.
+func TestResolveSocketSaysTheSocketIsDisabled(t *testing.T) {
+	cfg := filepath.Join(t.TempDir(), "config.yml")
+	require.NoError(t, os.WriteFile(cfg, []byte("local_api:\n  control:\n    enabled: false\n"), 0600))
+
+	_, err := resolveSocket("", cfg)
+	require.ErrorContains(t, err, "the control socket is disabled in "+cfg+" by local_api.control.enabled")
+
+	p, err := resolveSocket("/explicit.sock", cfg)
+	require.NoError(t, err)
+	assert.Equal(t, "/explicit.sock", p)
+}
+
+// TestResolveSocketUsesTheDefaultWithoutAnOverride: a config that does not name the
+// socket means the default path, the same one the daemon serves with that config, and
+// no config at all means the default too, since that is where a daemon started without
+// an override serves. Nothing is searched for.
+func TestResolveSocketUsesTheDefaultWithoutAnOverride(t *testing.T) {
+	cfg := filepath.Join(t.TempDir(), "config.yml")
+	require.NoError(t, os.WriteFile(cfg, []byte("static_host_map: {}\n"), 0600))
+
+	p, err := resolveSocket("", cfg)
+	require.NoError(t, err)
+	assert.Equal(t, localapi.DefaultSocketPath, p)
+
+	// The test binary has no config next to it.
+	p, err = resolveSocket("", "")
+	require.NoError(t, err)
+	assert.Equal(t, localapi.DefaultSocketPath, p)
+}
+
+func TestResolveSocketFlagWins(t *testing.T) {
+	p, err := resolveSocket("/explicit.sock", "/does/not/matter.yml")
+	require.NoError(t, err)
+	assert.Equal(t, "/explicit.sock", p)
+}
+
+// TestParseCommandReportsWhatFlagSaid covers the three ways a flag line goes wrong, which
+// are different mistakes: an unknown flag, a valid flag with a missing value, and a
+// valid flag with a value it cannot parse.
+func TestParseCommandReportsWhatFlagSaid(t *testing.T) {
+	schema := []commands.CommandSchema{{
+		Name: "change-remote",
+		Flags: []commands.FlagSchema{
+			{Name: "address"},
+			{Name: "by-index", Bool: true},
+		},
+	}}
+
+	cases := map[string]struct {
+		argv []string
+		want string
+	}{
+		"missing value": {[]string{"change-remote", "-address"}, "flag needs an argument: -address"},
+		"bad bool":      {[]string{"change-remote", "-by-index=maybe"}, "invalid boolean value \"maybe\" for -by-index"},
+		"unknown flag":  {[]string{"change-remote", "-nope"}, "flag provided but not defined: -nope"},
+	}
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			_, _, err := parseCommand(schema, tc.argv)
+			require.Error(t, err)
+			require.ErrorContains(t, err, "nebula: change-remote: "+tc.want)
+			require.ErrorContains(t, err, "see 'nebula ctl change-remote -h'")
+			assert.NotContains(t, err.Error(), "does not take")
+		})
+	}
+}
+
+// TestParseCommandOffPresentationFlagIsNoPreference: the daemon treats -json=false as
+// no preference, so nebula ctl must too, or the same line renders differently on the
+// two surfaces. Only a true value asks for JSON, and -pretty implies it.
+func TestParseCommandOffPresentationFlagIsNoPreference(t *testing.T) {
+	schema := []commands.CommandSchema{{
+		Name: "device-info",
+		Flags: []commands.FlagSchema{
+			{Name: commands.FlagJSON, Bool: true, Default: "false"},
+			{Name: commands.FlagPretty, Bool: true, Default: "false"},
+		},
+	}}
+
+	for _, tc := range []struct {
+		argv []string
+		want commands.Format
+		wire string
+	}{
+		{[]string{"device-info"}, commands.FormatUnset, "text"},
+		{[]string{"device-info", "-json=false"}, commands.FormatUnset, "text"},
+		{[]string{"device-info", "-json"}, commands.FormatJSON, "json"},
+		{[]string{"device-info", "-pretty"}, commands.FormatPrettyJSON, "json"},
+	} {
+		t.Run(strings.Join(tc.argv[1:], " "), func(t *testing.T) {
+			req, printedHelp, err := parseCommand(schema, tc.argv)
+			require.NoError(t, err)
+			assert.False(t, printedHelp)
+			assert.Equal(t, tc.want, req.Format)
+			assert.Equal(t, tc.wire, wireFormat(req.Format))
+			assert.Nil(t, req.Params, "presentation flags are never sent as parameters")
+		})
+	}
+}
+
+// TestParseCommandReadsTheLineAsTheDaemonDoes: nebula ctl parses the line with the same
+// rules as the daemon, so an operand after "--" reaches the command on both and a
+// negative number before it is the flag it would be at an ssh prompt.
+func TestParseCommandReadsTheLineAsTheDaemonDoes(t *testing.T) {
+	schema := []commands.CommandSchema{{Name: "set"}}
+
+	req, printedHelp, err := parseCommand(schema, []string{"set", "--", "-1"})
+	require.NoError(t, err)
+	assert.False(t, printedHelp)
+	assert.Equal(t, commands.Name("set"), req.name)
+	assert.Nil(t, req.Params)
+	assert.Equal(t, []string{"-1"}, req.Args)
+
+	_, _, err = parseCommand(schema, []string{"set", "-1"})
+	require.ErrorContains(t, err, "flag provided but not defined: -1")
+}
+
+// TestOutputRendersForTheFormatAsked: what the daemon answered is read by the format the
+// line asked for, never by sniffing the bytes. A text answer is the command's rendering,
+// printed as it came; a JSON answer whose value is a string stays quoted, since to the
+// script reading it that string is the JSON.
+func TestOutputRendersForTheFormatAsked(t *testing.T) {
+	text := &localapi.Response{Text: []byte("Log level is: info\n")}
+	assert.Equal(t, "Log level is: info\n", output(text, commands.FormatUnset))
+	assert.Equal(t, "Log level is: info\n", output(text, commands.FormatText))
+
+	str := &localapi.Response{Result: json.RawMessage(`"some string"`)}
+	assert.Equal(t, `"some string"`, output(str, commands.FormatJSON))
+	assert.Equal(t, `"some string"`, output(str, commands.FormatPrettyJSON))
+
+	// Keys stay in the order the daemon wrote them and a uint64 past 2^53 is not rounded,
+	// which decoding into `any` and encoding again would do.
+	obj := &localapi.Response{Result: json.RawMessage(`{"z":18446744073709551615,"a":{"y":[1,2],"b":true}}`)}
+	//nolint:testifylint // the layout is what is under test, which JSONEq would not see
+	assert.Equal(t, `{"z":18446744073709551615,"a":{"y":[1,2],"b":true}}`, output(obj, commands.FormatJSON))
+	//nolint:testifylint // as above
+	assert.Equal(t, "{\n    \"z\": 18446744073709551615,\n    \"a\": {\n        \"y\": [\n            1,\n            2\n        ],\n        \"b\": true\n    }\n}", output(obj, commands.FormatPrettyJSON))
+
+	bad := &localapi.Response{Result: json.RawMessage(`{"a":`)}
+	assert.Equal(t, `{"a":`, output(bad, commands.FormatPrettyJSON), "what cannot be indented is printed as it came")
+
+	none := &localapi.Response{}
+	assert.Empty(t, output(none, commands.FormatJSON))
+	assert.Empty(t, output(none, commands.FormatUnset))
+}
+
+// TestSplitLineLiftsOwnFlagsFromAnywhere: -config and -socket are the client's wherever
+// they appear, before the command or after it, in either spelling, so the two orders
+// mean the same line. Everything else after the command word stays in order for the
+// command's own parse, including flags and operands that happen to look like values.
+func TestSplitLineLiftsOwnFlagsFromAnywhere(t *testing.T) {
+	newFS := func() *flag.FlagSet {
+		fs := flag.NewFlagSet("nebula ctl", flag.ContinueOnError)
+		fs.String(commands.FlagConfig, "", "")
+		fs.String(commands.FlagSocket, "", "")
+		return fs
+	}
+
+	for _, tc := range []struct {
+		args    []string
+		wantOwn []string
+		wantCmd []string
+	}{
+		{nil, nil, nil},
+		{[]string{"identity"}, nil, []string{"identity"}},
+		{[]string{"-config", "x", "identity"}, []string{"-config", "x"}, []string{"identity"}},
+		{[]string{"identity", "-config", "x"}, []string{"-config", "x"}, []string{"identity"}},
+		{[]string{"identity", "-config=x"}, []string{"-config=x"}, []string{"identity"}},
+		{[]string{"identity", "--socket", "s"}, []string{"--socket", "s"}, []string{"identity"}},
+		{[]string{"-socket", "s", "identity", "-config", "x"}, []string{"-socket", "s", "-config", "x"}, []string{"identity"}},
+		// The command's flags and operands are left where they are, around the lifted ones.
+		{[]string{"print-tunnel", "10.0.0.1", "-config", "x", "-json"}, []string{"-config", "x"}, []string{"print-tunnel", "10.0.0.1", "-json"}},
+		{[]string{"change-remote", "-address", "1.2.3.4:5", "-socket", "s", "10.0.0.1"}, []string{"-socket", "s"}, []string{"change-remote", "-address", "1.2.3.4:5", "10.0.0.1"}},
+		// -h before the command is the client's usage; after it, the command's help.
+		{[]string{"-h"}, []string{"-h"}, nil},
+		{[]string{"identity", "-h"}, nil, []string{"identity", "-h"}},
+		// An unknown flag before the command is the client's to report, and does not
+		// swallow the command word as its value.
+		{[]string{"-nope", "identity"}, []string{"-nope"}, []string{"identity"}},
+		// A -config with nothing after it is left for fs.Parse to report.
+		{[]string{"identity", "-config"}, []string{"-config"}, []string{"identity"}},
+		// "--" before the command is consumed, as fs.Parse consumed it; after the command
+		// it belongs to the line and ends the lifting.
+		{[]string{"--", "identity", "-config", "x"}, []string{"-config", "x"}, []string{"identity"}},
+		{[]string{"identity", "--", "-config", "x"}, nil, []string{"identity", "--", "-config", "x"}},
+	} {
+		t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
+			own, cmd := splitLine(newFS(), tc.args)
+			assert.Equal(t, tc.wantOwn, own)
+			assert.Equal(t, tc.wantCmd, cmd)
+		})
+	}
+}
+
+// TestTrailingHandlesWordsAfterTheDaemonsFlags: what flag.Parse leaves behind is never
+// a daemon to start. A leading ctl runs the client on the rest, with the daemon's
+// -config carried over; anything else is a mistyped subcommand. Only -h is run here,
+// since the client's help needs no daemon: it proves the words reach Run and that a
+// carried -config is accepted by it.
+func TestTrailingHandlesWordsAfterTheDaemonsFlags(t *testing.T) {
+	code, done := Trailing(nil, "")
+	assert.False(t, done)
+	assert.Equal(t, 0, code)
+
+	code, done = Trailing([]string{"clt"}, "")
+	assert.True(t, done)
+	assert.Equal(t, 1, code)
+
+	code, done = Trailing([]string{Word, "-h"}, "")
+	assert.True(t, done)
+	assert.Equal(t, 0, code)
+
+	code, done = Trailing([]string{Word, "-h"}, "/etc/nebula/config.yml")
+	assert.True(t, done)
+	assert.Equal(t, 0, code)
+}
+
+// TestSubcommandOnlyReadsTheFirstWord: a flag first means the line is the daemon's to
+// parse, and whatever follows the flags is Trailing's to judge.
+func TestSubcommandOnlyReadsTheFirstWord(t *testing.T) {
+	_, done := Subcommand([]string{"nebula", "-config", "x", Word})
+	assert.False(t, done)
+
+	code, done := Subcommand([]string{"nebula", "clt"})
+	assert.True(t, done)
+	assert.Equal(t, 1, code)
+
+	code, done = Subcommand([]string{"nebula", Word, "-h"})
+	assert.True(t, done)
+	assert.Equal(t, 0, code)
+}

cmd/internal/ctl/ctl_unix_test.go

@@ -0,0 +1,42 @@
+//go:build !windows
+
+package ctl
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/slackhq/nebula/localapi"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestResolveSocketDiagnosesAnOverlongOverride: the CLI applies the daemon's own rule to
+// the override, so it says what the daemon's log says rather than dialing a path the
+// kernel would truncate and reporting the generic failure. The rule is sun_path's, which
+// a named pipe does not have, so this is a unix test.
+func TestResolveSocketDiagnosesAnOverlongOverride(t *testing.T) {
+	cfg := filepath.Join(t.TempDir(), "config.yml")
+	long := "/" + strings.Repeat("x", 120)
+	require.NoError(t, os.WriteFile(cfg, []byte("local_api:\n  control:\n    socket: "+long+"\n"), 0600))
+
+	_, err := resolveSocket("", cfg)
+	require.Error(t, err)
+	require.ErrorContains(t, err, "too long for this platform")
+	assert.ErrorContains(t, err, localapi.CtlSocketKey)
+}
+
+// TestResolveSocketRefusesARelativeOverride: the CLI applies the daemon's rule here too,
+// so a relative path in the config is reported as the daemon's log reports it rather
+// than resolved against wherever nebula ctl happened to be run from.
+func TestResolveSocketRefusesARelativeOverride(t *testing.T) {
+	cfg := filepath.Join(t.TempDir(), "config.yml")
+	require.NoError(t, os.WriteFile(cfg, []byte("local_api:\n  control:\n    socket: run/ctl.sock\n"), 0600))
+
+	_, err := resolveSocket("", cfg)
+	require.Error(t, err)
+	require.ErrorContains(t, err, "must be an absolute path")
+	assert.ErrorContains(t, err, localapi.CtlSocketKey)
+}

cmd/nebula-service/main.go

@@ -8,7 +8,9 @@ import (
 	"strings"
 
 	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cmd/internal/ctl"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/localapi"
 	"github.com/slackhq/nebula/logging"
 	"github.com/slackhq/nebula/util"
 )
@@ -32,6 +34,11 @@ func init() {
 }
 
 func main() {
+	// commands are either `nebula ctl ...` or `nebula -<flag> ...`
+	if code, done := ctl.Subcommand(os.Args); done {
+		os.Exit(code)
+	}
+
 	serviceFlag := flag.String("service", "", "Control the system service.")
 	configPath := flag.String("config", "", "Path to either a file or directory to load configuration from")
 	configTest := flag.Bool("test", false, "Test the config and print the end result. Non zero exit indicates a faulty config")
@@ -40,6 +47,12 @@ func main() {
 
 	flag.Parse()
 
+	// A word after the daemon's flags is `ctl` in the other place an operator puts it,
+	// or a mistyped subcommand, and either way not a daemon to start; see Trailing.
+	if code, done := ctl.Trailing(flag.Args(), *configPath); done {
+		os.Exit(code)
+	}
+
 	if *printVersion {
 		fmt.Printf("Version: %s\n", Build)
 		os.Exit(0)
@@ -104,9 +117,17 @@ func main() {
 			os.Exit(1)
 		}
 
+		ctl := localapi.Wire(ctrl.Context(), l, c, ctrl.Commands())
+
 		go ctrl.ShutdownBlock()
 
-		if err := ctrl.Wait(); err != nil {
+		err := ctrl.Wait()
+
+		// os.Exit skips deferred calls, so the socket is removed here, before either exit
+		// below, rather than left to a deferred Stop that would never run.
+		ctl.Stop()
+
+		if err != nil {
 			l.Error("Nebula stopped due to fatal error", "error", err)
 			os.Exit(2)
 		}

cmd/nebula-service/service.go

@@ -8,6 +8,7 @@ import (
 	"github.com/kardianos/service"
 	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/localapi"
 	"github.com/slackhq/nebula/logging"
 )
 
@@ -17,6 +18,7 @@ type program struct {
 	configPath *string
 	build      string
 	control    *nebula.Control
+	ctl        *localapi.Server
 }
 
 func (p *program) Start(s service.Service) error {
@@ -50,10 +52,16 @@ func (p *program) Start(s service.Service) error {
 		return err
 	}
 
+	// Now that we're started, stand up the local control socket.
+	p.ctl = localapi.Wire(p.control.Context(), l, c, p.control.Commands())
+
 	// Nebula can stop itself on a fatal packet reader error, make sure to log it if it happens.
 	go func() {
 		if err := p.control.Wait(); err != nil {
 			logger.Error(fmt.Sprintf("Nebula stopped due to fatal error: %v", err))
+			// os.Exit skips deferred calls and races the socket's own context watcher,
+			// so the socket is removed here, as cmd/nebula does.
+			p.ctl.Stop()
 			os.Exit(2)
 		}
 	}()
@@ -63,6 +71,11 @@ func (p *program) Start(s service.Service) error {
 
 func (p *program) Stop(s service.Service) error {
 	logger.Info("Nebula service stopping.")
+
+	if p.ctl != nil {
+		p.ctl.Stop()
+	}
+
 	if p.control == nil {
 		return nil
 	}

cmd/nebula/main.go

@@ -8,7 +8,9 @@ import (
 	"strings"
 
 	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cmd/internal/ctl"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/localapi"
 	"github.com/slackhq/nebula/logging"
 	"github.com/slackhq/nebula/util"
 )
@@ -32,6 +34,11 @@ func init() {
 }
 
 func main() {
+	// commands are either `nebula ctl ...` or `nebula -<flag> ...`
+	if code, done := ctl.Subcommand(os.Args); done {
+		os.Exit(code)
+	}
+
 	configPath := flag.String("config", "", "Path to either a file or directory to load configuration from")
 	configTest := flag.Bool("test", false, "Test the config and print the end result. Non zero exit indicates a faulty config")
 	printVersion := flag.Bool("version", false, "Print version")
@@ -39,6 +46,12 @@ func main() {
 
 	flag.Parse()
 
+	// A word after the daemon's flags is `ctl` in the other place an operator puts it,
+	// or a mistyped subcommand, and either way not a daemon to start; see Trailing.
+	if code, done := ctl.Trailing(flag.Args(), *configPath); done {
+		os.Exit(code)
+	}
+
 	if *printVersion {
 		fmt.Printf("Version: %s\n", Build)
 		os.Exit(0)
@@ -90,10 +103,18 @@ func main() {
 			os.Exit(1)
 		}
 
+		ctl := localapi.Wire(ctrl.Context(), l, c, ctrl.Commands())
+
 		go ctrl.ShutdownBlock()
 		notifyReady(l)
 
-		if err := ctrl.Wait(); err != nil {
+		err := ctrl.Wait()
+
+		// os.Exit skips deferred calls, so the socket is removed here, before either exit
+		// below, rather than left to a deferred Stop that would never run.
+		ctl.Stop()
+
+		if err != nil {
 			l.Error("Nebula stopped due to fatal error", "error", err)
 			os.Exit(2)
 		}