Compare commits

...
7 changed files with 184 additions and 51 deletions

No files matched your search

+49 -22
View File
@@ -38,8 +38,8 @@ type dnsServer struct {
serverMu sync.Mutex serverMu sync.Mutex
server *dns.Server server *dns.Server
// started is closed once `server` has finished binding (or after // started is closed once `server` is serving (or after serve returns
// ListenAndServe returns on a bind failure). Stop waits on it before // without ever starting). Stop waits on it before
// calling Shutdown to avoid the miekg/dns "server not started" race // calling Shutdown to avoid the miekg/dns "server not started" race
// where a Shutdown that arrives before bind completes is silently // where a Shutdown that arrives before bind completes is silently
// ignored, leaving the listener running forever. // ignored, leaving the listener running forever.
@@ -84,7 +84,7 @@ func newDnsServerFromConfig(ctx context.Context, l *slog.Logger, pki *PKI, hostM
} }
// reload applies the latest config and reconciles the running state with it: // reload applies the latest config and reconciles the running state with it:
// - enabled toggled on -> spawn a runner // - enabled toggled on -> bind, then serve in the background
// - enabled toggled off -> stop the runner // - enabled toggled off -> stop the runner
// - listen address changed (while running) -> restart on the new address // - listen address changed (while running) -> restart on the new address
// - everything else -> no-op // - everything else -> no-op
@@ -121,13 +121,20 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
return nil return nil
} }
if !running { if running && !sameAddr {
// Was disabled (or never started); bring it up now. // Stop clears the slot before shutting down, otherwise the listen below can find the dying server and refuse
go d.Start()
} else if !sameAddr {
// Stop clears the slot before shutting down, otherwise the Start below can find the dying server and refuse
d.Stop() d.Stop()
go d.Start() running = false
}
if !running {
// Bind here rather than in the background, so a reload that can't listen says so
server, started, err := d.listen()
if err != nil {
return err
}
if server != nil {
go d.serve(server, started)
}
} }
// Refresh the self entry every enabled reload so cert renewals that change our name or VPN addresses are picked up. // Refresh the self entry every enabled reload so cert renewals that change our name or VPN addresses are picked up.
@@ -156,30 +163,50 @@ func (d *dnsServer) shutdownServer(srv *dns.Server, started chan struct{}, reaso
// Must be invoked after the tun device is active so that lighthouse.dns.host // Must be invoked after the tun device is active so that lighthouse.dns.host
// may bind to a nebula IP. // may bind to a nebula IP.
func (d *dnsServer) Start() { func (d *dnsServer) Start() {
if !d.enabled.Load() { server, started, err := d.listen()
if err != nil {
d.l.Warn("Failed to run the DNS responder", "error", err)
return return
} }
if server != nil {
d.serve(server, started)
}
}
// listen binds the responder and claims the server slot for serve. It returns a nil server, and no error, when DNS is
// disabled or a server is already running.
func (d *dnsServer) listen() (*dns.Server, chan struct{}, error) {
if !d.enabled.Load() {
return nil, nil, nil
}
started := make(chan struct{}) started := make(chan struct{})
d.serverMu.Lock() d.serverMu.Lock()
defer d.serverMu.Unlock()
// Re-check enabled under the lock, a disable that raced our check above snapshots the slot under it too. // Re-check enabled under the lock, a disable that raced our check above snapshots the slot under it too.
// Two reloads in quick succession can both spawn a Start, the loser would orphan the live listener past Stop // A reload and Control.Start can both get here, the loser would orphan the live listener past Stop
if d.ctx.Err() != nil || d.server != nil || !d.enabled.Load() { if d.ctx.Err() != nil || d.server != nil || !d.enabled.Load() {
d.serverMu.Unlock() return nil, nil, nil
return }
pc, err := net.ListenPacket("udp", d.addr)
if err != nil {
return nil, nil, err
} }
addr := d.addr
server := &dns.Server{ server := &dns.Server{
Addr: addr, Addr: d.addr,
Net: "udp", PacketConn: pc,
Handler: d.mux, Handler: d.mux,
NotifyStartedFunc: func() { close(started) }, NotifyStartedFunc: func() { close(started) },
} }
d.server = server d.server = server
d.started = started d.started = started
d.serverMu.Unlock() d.l.Info("Starting DNS responder", "dnsListener", pc.LocalAddr())
return server, started, nil
}
// Per-invocation ctx watcher. Exits when Start does, so we don't leak a // serve answers queries on a server from listen until Stop or the context ends it.
func (d *dnsServer) serve(server *dns.Server, started chan struct{}) {
// Per-invocation ctx watcher. Exits when serve does, so we don't leak a
// watcher per reload-driven restart. // watcher per reload-driven restart.
done := make(chan struct{}) done := make(chan struct{})
go func() { go func() {
@@ -190,16 +217,16 @@ func (d *dnsServer) Start() {
} }
}() }()
d.l.Info("Starting DNS responder", "dnsListener", addr) err := server.ActivateAndServe()
err := server.ListenAndServe()
close(done) close(done)
// If the listener never bound (bind error) NotifyStartedFunc never fires, // If serving failed before it started NotifyStartedFunc never fires, so close started here to release any Stop
// so close started here to release any Stop caller waiting on it. // caller waiting on it. ActivateAndServe doesn't close the socket on that path either
select { select {
case <-started: case <-started:
default: default:
close(started) close(started)
_ = server.PacketConn.Close()
} }
// Release our slot, unless a reload already replaced us, so a dead listener can't block a future Start // Release our slot, unless a reload already replaced us, so a dead listener can't block a future Start
+19
View File
@@ -629,3 +629,22 @@ func TestDnsServer_Start_refusesWhenDisabledUnderLock(t *testing.T) {
require.NoError(t, err, "an orphaned listener is still holding the port") require.NoError(t, err, "an orphaned listener is still holding the port")
_ = pc.Close() _ = pc.Close()
} }
// A reload that can't bind says so, rather than finding out in the background
func TestDnsServer_reload_busyPortFails(t *testing.T) {
busy, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = busy.Close() })
_, port, err := net.SplitHostPort(busy.LocalAddr().String())
require.NoError(t, err)
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", port, false, true)
require.NoError(t, ds.reload(c, true))
setDnsConfig(c, "127.0.0.1", port, true, true)
require.Error(t, ds.reload(c, false))
ds.serverMu.Lock()
assert.Nil(t, ds.server, "a failed bind kept the slot")
ds.serverMu.Unlock()
}
+9 -4
View File
@@ -30,15 +30,20 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
} }
return return
} }
sshRun, err := configSSH(l, ssh, c, r) if _, err := configSSH(l, ssh, c, r); err != nil {
if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err) l.Error("Failed to reconfigure the sshd", "error", err)
ssh.Stop() ssh.Stop()
r.Failed() r.Failed()
return
} }
if sshRun != nil { // Bind here rather than in the background, so a reload that can't listen says so
go sshRun() listener, err := ssh.Listen(c.GetString("sshd.listen", ""))
if err != nil {
l.Error("Failed to run the SSH server", "error", err)
r.Failed()
return
} }
go ssh.Serve(listener)
} else { } else {
ssh.Stop() ssh.Stop()
} }
+20 -7
View File
@@ -184,19 +184,34 @@ func (s *SSHServer) AddAuthorizedKey(user, pubKey string) error {
// from the constructor-supplied ctx so a Stop+Run sequence (used by config reload) starts clean // from the constructor-supplied ctx so a Stop+Run sequence (used by config reload) starts clean
// rather than carrying a permanently-cancelled context across runs. // rather than carrying a permanently-cancelled context across runs.
func (s *SSHServer) Run(addr string) error { func (s *SSHServer) Run(addr string) error {
if s.ctx.Err() != nil { listener, err := s.Listen(addr)
return s.ctx.Err()
}
listener, err := net.Listen("tcp", addr)
if err != nil { if err != nil {
return err return err
} }
s.Serve(listener)
return nil
}
// Listen binds addr for Serve, so a caller can tell the bind failed before serving in the background. Stop can close
// the listener from here on, Serve then returns as soon as it starts.
func (s *SSHServer) Listen(addr string) (net.Listener, error) {
if s.ctx.Err() != nil {
return nil, s.ctx.Err()
}
listener, err := net.Listen("tcp", addr)
if err != nil {
return nil, err
}
// s.listener is the public handle Stop uses to interrupt the active run; listener (the local) is what // s.listener is the public handle Stop uses to interrupt the active run; listener (the local) is what
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's // this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
// listener before this run's watcher fires, so each run must close its own listener via the local // listener before this run's watcher fires, so each run must close its own listener via the local
// reference. // reference.
s.listener.Store(&listener) s.listener.Store(&listener)
return listener, nil
}
// Serve accepts connections on a listener from Listen until Stop, or the context passed to NewSSHServer, ends it.
func (s *SSHServer) Serve(listener net.Listener) {
s.runs.Add(1) s.runs.Add(1)
bound := listener.Addr() bound := listener.Addr()
s.addr.Store(&bound) s.addr.Store(&bound)
@@ -224,8 +239,6 @@ func (s *SSHServer) Run(addr string) error {
s.run(runCtx, listener) s.run(runCtx, listener)
s.l.Info("SSH server stopped listening") s.l.Info("SSH server stopped listening")
// We don't return an error because run logs for us
return nil
} }
func (s *SSHServer) run(ctx context.Context, listener net.Listener) { func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
+12
View File
@@ -279,3 +279,15 @@ func TestSSHExecExitStatusReflectsTheCommand(t *testing.T) {
assert.Equal(t, 1, exit.ExitStatus()) assert.Equal(t, 1, exit.ExitStatus())
assert.True(t, strings.HasPrefix(string(out), "did not understand: nope\nAvailable commands:\n"), string(out)) assert.True(t, strings.HasPrefix(string(out), "did not understand: nope\nAvailable commands:\n"), string(out))
} }
// A reload that can't bind the new address says so, rather than finding out in the background
func TestSSHDReload_BusyPortFails(t *testing.T) {
h := newSSHDReloadHarness(t)
busy, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = busy.Close() })
res := h.c.ReloadConfigString(strings.Replace(h.config(), "127.0.0.1:0", busy.Addr().String(), 1))
require.NoError(t, res.Err)
assert.True(t, res.Failed)
}
+50 -18
View File
@@ -140,8 +140,13 @@ func (s *statsServer) reload(c *config.C, initial bool) error {
// Even with nothing running, a runtime whose listener failed can still be finishing its last pass // Even with nothing running, a runtime whose listener failed can still be finishing its last pass
s.Stop() s.Stop()
if enabled && !s.configTest { // Bind here rather than in the background, so a reload that can't listen says so
go s.Start() run, err := s.start()
if err != nil {
return err
}
if run != nil {
go run()
} }
return nil return nil
} }
@@ -151,36 +156,64 @@ func (s *statsServer) reload(c *config.C, initial bool) error {
// the HTTP listener. For graphite it blocks on the capture loop's context. // the HTTP listener. For graphite it blocks on the capture loop's context.
// Safe to call when stats are disabled or already running (both no-op). // Safe to call when stats are disabled or already running (both no-op).
func (s *statsServer) Start() { func (s *statsServer) Start() {
if !s.enabled.Load() || s.configTest { run, err := s.start()
if err != nil {
s.l.Error("Failed to start the prometheus stats listener", "error", err)
return return
} }
if run != nil {
run()
}
}
// start claims the runtime and binds the prometheus listener. The returned run serves it and blocks until Stop or ctx
// ends it. A nil run and no error means stats are off or already running.
func (s *statsServer) start() (func(), error) {
if !s.enabled.Load() || s.configTest {
return nil, nil
}
s.runMu.Lock() s.runMu.Lock()
defer s.runMu.Unlock()
if s.ctx.Err() != nil || s.run != nil || s.runCfg == nil { if s.ctx.Err() != nil || s.run != nil || s.runCfg == nil {
s.runMu.Unlock() return nil, nil
return
} }
cfg := *s.runCfg cfg := *s.runCfg
runCtx, cancel := context.WithCancel(s.ctx) runCtx, cancel := context.WithCancel(s.ctx)
captureFns, listener := s.buildRuntime(runCtx, cfg) captureFns, listener := s.buildRuntime(runCtx, cfg)
var ln net.Listener
if listener != nil {
var err error
ln, err = net.Listen("tcp", listener.Addr)
if err != nil {
cancel()
// Drop the config so a SIGHUP with the same one tries again once the problem is fixed
s.runCfg = nil
return nil, err
}
}
rt := &statsRuntime{cancel: cancel, listener: listener} rt := &statsRuntime{cancel: cancel, listener: listener}
s.run = rt s.run = rt
done := make(chan struct{}) done := make(chan struct{})
s.loopDone = done s.loopDone = done
s.runMu.Unlock()
return func() { s.serve(runCtx, rt, captureFns, cfg.interval, ln, done) }, nil
}
// serve runs the capture loop, and the prometheus listener on ln, until Stop or ctx ends them.
func (s *statsServer) serve(runCtx context.Context, rt *statsRuntime, captureFns []func(), interval time.Duration, ln net.Listener, done chan struct{}) {
// Prime before serving, so a scrape that lands before the first tick sees real values instead of zeros (issue #907) // Prime before serving, so a scrape that lands before the first tick sees real values instead of zeros (issue #907)
for _, fn := range captureFns { for _, fn := range captureFns {
fn() fn()
} }
go captureStatsLoop(runCtx, cfg.interval, captureFns, done) go captureStatsLoop(runCtx, interval, captureFns, done)
cleanExit := true cleanExit := true
if listener == nil { if rt.listener == nil {
// Graphite: no HTTP listener to serve; block until teardown. // Graphite: no HTTP listener to serve; block until teardown.
<-runCtx.Done() <-runCtx.Done()
} else { } else {
cleanExit = s.serveListener(listener) cleanExit = s.serveListener(rt.listener, ln)
} }
// Clear our runtime only if nothing has replaced it. Stop races through // Clear our runtime only if nothing has replaced it. Stop races through
@@ -189,10 +222,9 @@ func (s *statsServer) Start() {
if s.run == rt { if s.run == rt {
rt.cancel() rt.cancel()
s.run = nil s.run = nil
// A listener that exited with an error (e.g., bind conflict) leaves // A listener that exited with an error leaves runCfg cached as if it
// runCfg cached as if it were applied. Drop it so a SIGHUP with the // were applied. Drop it so a SIGHUP with the same config re-triggers
// same config re-triggers Start once the user fixes the underlying // Start once the user fixes the underlying problem.
// problem.
if !cleanExit { if !cleanExit {
s.runCfg = nil s.runCfg = nil
} }
@@ -200,12 +232,12 @@ func (s *statsServer) Start() {
s.runMu.Unlock() s.runMu.Unlock()
} }
// serveListener runs ListenAndServe and ensures ctx cancellation unblocks it. // serveListener serves listener on ln and ensures ctx cancellation unblocks it.
// Returns true if the listener exited cleanly (Stop, ctx cancellation, or any // Returns true if the listener exited cleanly (Stop, ctx cancellation, or any
// other http.ErrServerClosed path), false on an unexpected error. // other http.ErrServerClosed path), false on an unexpected error.
func (s *statsServer) serveListener(listener *http.Server) bool { func (s *statsServer) serveListener(listener *http.Server, ln net.Listener) bool {
// Per-invocation watcher: ctx cancellation triggers a listener shutdown // Per-invocation watcher: ctx cancellation triggers a listener shutdown
// which in turn unblocks ListenAndServe. Closing `done` on exit keeps // which in turn unblocks Serve. Closing `done` on exit keeps
// the watcher from outliving this call. // the watcher from outliving this call.
done := make(chan struct{}) done := make(chan struct{})
go func() { go func() {
@@ -221,8 +253,8 @@ func (s *statsServer) serveListener(listener *http.Server) bool {
}() }()
defer close(done) defer close(done)
s.l.Info("Starting prometheus stats listener", "addr", listener.Addr) s.l.Info("Starting prometheus stats listener", "addr", ln.Addr())
err := listener.ListenAndServe() err := listener.Serve(ln)
if err == nil || errors.Is(err, http.ErrServerClosed) { if err == nil || errors.Is(err, http.ErrServerClosed) {
return true return true
} }
+25
View File
@@ -673,3 +673,28 @@ func TestStatsServer_Start_duringStopWaitsForThePass(t *testing.T) {
waitFor(t, func() bool { return len(entered) == 1 }) waitFor(t, func() bool { return len(entered) == 1 })
assert.Equal(t, int64(1), most.Load(), "a Start captured alongside the pass Stop was waiting out") assert.Equal(t, int64(1), most.Load(), "a Start captured alongside the pass Stop was waiting out")
} }
// A reload that can't bind says so rather than finding out in the background, and the same config tries again once
// the port is free
func TestStatsServer_reload_bindFailureFails(t *testing.T) {
blocker, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
port := strconv.Itoa(blocker.Addr().(*net.TCPAddr).Port)
s, c := newTestStatsServer(t)
stopAtCleanup(t, &s)
require.NoError(t, s.reload(c, true))
setStatsConfig(c, map[string]any{
"type": "prometheus",
"interval": "1s",
"listen": "127.0.0.1:" + port,
"path": "/metrics",
})
require.Error(t, s.reload(c, false))
assert.Nil(t, currentRuntime(s))
require.NoError(t, blocker.Close())
require.NoError(t, s.reload(c, false))
waitForListening(t, "127.0.0.1:"+port)
}