graphite: Time out a send, and abandon it on stop (#1920)

2026-11-15 16:26:52 +01:00 · 2026-10-07 18:01:09 -05:00
1 parent 94be454f4f
commit f3705cbb67
@@ -0,0 +1,315 @@
+package nebula
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/rcrowley/go-metrics"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// graphiteHost accepts connections and hands each one to the test
+type graphiteHost struct {
+	ln    net.Listener
+	conns chan net.Conn
+
+	mu     sync.Mutex
+	closed bool
+	open   []net.Conn
+}
+
+// newGraphiteHost accepts connections for the test. A non-zero window shrinks each connection's receive buffer, so a
+// sender that is not read from blocks after a little whatever the host's TCP tuning. Only for tests that need a stall,
+// a small window makes reading very slow on Linux.
+func newGraphiteHost(t *testing.T, window int) *graphiteHost {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	h := &graphiteHost{ln: ln, conns: make(chan net.Conn, 8)}
+	done := make(chan struct{})
+	t.Cleanup(func() {
+		_ = ln.Close()
+		h.mu.Lock()
+		h.closed = true
+		for _, c := range h.open {
+			_ = c.Close()
+		}
+		h.mu.Unlock()
+		close(done)
+	})
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			h.mu.Lock()
+			if h.closed {
+				h.mu.Unlock()
+				_ = c.Close()
+				return
+			}
+			h.open = append(h.open, c)
+			h.mu.Unlock()
+			if tc, ok := c.(*net.TCPConn); ok && window > 0 {
+				_ = tc.SetReadBuffer(window)
+			}
+			select {
+			case h.conns <- c:
+			case <-done:
+				return
+			}
+		}
+	}()
+	return h
+}
+
+func (h *graphiteHost) addr(t *testing.T) *net.TCPAddr {
+	a, err := net.ResolveTCPAddr("tcp", h.ln.Addr().String())
+	require.NoError(t, err)
+	return a
+}
+
+// readExport reads what one send wrote, bounded so a send that never finishes fails the test rather than hanging it
+func readExport(t *testing.T, c net.Conn) []byte {
+	t.Helper()
+	require.NoError(t, c.SetReadDeadline(time.Now().Add(10*time.Second)))
+	b, err := io.ReadAll(c)
+	require.NoError(t, err)
+	return b
+}
+
+func (h *graphiteHost) next(t *testing.T, wait time.Duration) net.Conn {
+	t.Helper()
+	select {
+	case c := <-h.conns:
+		return c
+	case <-time.After(wait):
+		t.Fatal("no connection")
+		return nil
+	}
+}
+
+// lockedBuffer is a log destination a test can read while the code under test writes to it from another goroutine
+type lockedBuffer struct {
+	mu  sync.Mutex
+	buf bytes.Buffer
+}
+
+func (b *lockedBuffer) Write(p []byte) (int, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.buf.Write(p)
+}
+
+func (b *lockedBuffer) String() string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.buf.String()
+}
+
+func testGraphiteConfig(addr *net.TCPAddr, r metrics.Registry) graphiteConfigExport {
+	return graphiteConfigExport{Addr: addr, Registry: r, FlushInterval: time.Second, DurationUnit: time.Nanosecond, Prefix: "nebula"}
+}
+
+// Enough metrics that one export overflows the socket buffers on any platform, so a send to a host that stops
+// reading blocks
+func bigGraphiteRegistry(r metrics.Registry, t *testing.T) {
+	for i := range 200000 {
+		name := fmt.Sprintf("big.gauge.%d", i)
+		metrics.GetOrRegisterGauge(name, r).Update(int64(i))
+		t.Cleanup(func() { r.Unregister(name) })
+	}
+}
+
+func TestGraphiteOnce_sends(t *testing.T) {
+	r := metrics.NewRegistry()
+	metrics.GetOrRegisterGauge("g", r).Update(7)
+	metrics.GetOrRegisterCounter("c", r).Inc(3)
+	h := newGraphiteHost(t, 0)
+
+	require.NoError(t, graphiteOnce(t.Context(), testGraphiteConfig(h.addr(t), r), time.Minute))
+	b := readExport(t, h.next(t, 5*time.Second))
+	assert.Contains(t, string(b), "nebula.g.value 7 ")
+	assert.Contains(t, string(b), "nebula.c.count 3 ")
+	assert.Contains(t, string(b), "nebula.c.count_ps 3.00 ")
+}
+
+// A host that accepts and never reads is given up on, and the failed write is reported
+func TestGraphiteOnce_stalledHostTimesOut(t *testing.T) {
+	r := metrics.NewRegistry()
+	bigGraphiteRegistry(r, t)
+	h := newGraphiteHost(t, 4096)
+	done := make(chan error, 1)
+	cfg := testGraphiteConfig(h.addr(t), r)
+	go func() { done <- graphiteOnce(t.Context(), cfg, 300*time.Millisecond) }()
+	h.next(t, 5*time.Second) // accepted, never read
+
+	select {
+	case err := <-done:
+		require.Error(t, err)
+	case <-time.After(10 * time.Second):
+		t.Fatal("the send did not give up on a host that stopped reading")
+	}
+}
+
+// The timeout covers the whole send, a host that keeps reading but too slowly is still given up on
+func TestGraphiteOnce_timeoutCoversTheWholeSend(t *testing.T) {
+	r := metrics.NewRegistry()
+	bigGraphiteRegistry(r, t)
+	h := newGraphiteHost(t, 0)
+	done := make(chan error, 1)
+	start := time.Now()
+	cfg := testGraphiteConfig(h.addr(t), r)
+	go func() { done <- graphiteOnce(t.Context(), cfg, 500*time.Millisecond) }()
+	c := h.next(t, 5*time.Second)
+	stopReading := make(chan struct{})
+	defer close(stopReading)
+	// 32KB every 10ms keeps every write well inside the timeout, the whole export takes seconds
+	go func() {
+		buf := make([]byte, 32<<10)
+		for {
+			select {
+			case <-stopReading:
+				return
+			case <-time.After(10 * time.Millisecond):
+			}
+			if _, err := c.Read(buf); err != nil {
+				return
+			}
+		}
+	}()
+
+	select {
+	case err := <-done:
+		require.Error(t, err)
+		assert.Less(t, time.Since(start), 5*time.Second)
+	case <-time.After(10 * time.Second):
+		t.Fatal("a host that kept reading slowly held the send past its timeout")
+	}
+}
+
+func TestGraphiteOnce_cancelledCtxDoesNotConnect(t *testing.T) {
+	h := newGraphiteHost(t, 0)
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	require.Error(t, graphiteOnce(ctx, testGraphiteConfig(h.addr(t), metrics.NewRegistry()), time.Minute))
+	select {
+	case <-h.conns:
+		t.Fatal("connected on a cancelled ctx")
+	case <-time.After(200 * time.Millisecond):
+	}
+}
+
+// A stop or reload abandons a stuck send rather than waiting out its timeout
+func TestGraphiteOnce_cancelAbandonsSend(t *testing.T) {
+	r := metrics.NewRegistry()
+	bigGraphiteRegistry(r, t)
+	h := newGraphiteHost(t, 4096)
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan error, 1)
+	cfg := testGraphiteConfig(h.addr(t), r)
+	go func() { done <- graphiteOnce(ctx, cfg, time.Minute) }()
+	h.next(t, 5*time.Second)
+	select {
+	case err := <-done:
+		t.Fatalf("the send should be stuck on a host that is not reading, it ended with %v", err)
+	case <-time.After(200 * time.Millisecond):
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		require.Error(t, err)
+	case <-time.After(5 * time.Second):
+		t.Fatal("the send is still stuck after its ctx was cancelled")
+	}
+}
+
+// buildRuntime hands graphite the configured prefix and interval
+func TestStatsServer_graphiteExportUsesConfig(t *testing.T) {
+	const name = "graphite.wiring.test"
+	metrics.GetOrRegisterCounter(name, nil).Inc(8)
+	t.Cleanup(func() { metrics.DefaultRegistry.Unregister(name) })
+	h := newGraphiteHost(t, 0)
+	s, c := newTestStatsServer(t)
+	setStatsConfig(c, map[string]any{"type": "graphite", "interval": "4s", "prefix": "pfx", "host": h.ln.Addr().String()})
+	cfg, err := loadStatsConfig(c)
+	require.NoError(t, err)
+
+	s.runMu.Lock()
+	fns, _ := s.buildRuntime(t.Context(), cfg)
+	s.runMu.Unlock()
+	for _, fn := range fns {
+		fn()
+	}
+	b := readExport(t, h.next(t, 5*time.Second))
+	assert.Contains(t, string(b), "pfx."+name+".count 8 ")
+	assert.Contains(t, string(b), "pfx."+name+".count_ps 2.00 ")
+}
+
+func TestStatsServer_graphiteFailureIsLogged(t *testing.T) {
+	logs := &lockedBuffer{}
+	s, c := newTestStatsServer(t)
+	s.l = test.NewLoggerWithOutput(logs)
+	setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)})
+	cfg, err := loadStatsConfig(c)
+	require.NoError(t, err)
+
+	s.runMu.Lock()
+	fns, _ := s.buildRuntime(t.Context(), cfg)
+	s.runMu.Unlock()
+	for _, fn := range fns {
+		fn()
+	}
+	assert.Contains(t, logs.String(), "Graphite export failed")
+}
+
+// Stop abandons the running export, so it goes through Start rather than handing buildRuntime a ctx, and a stop is
+// not logged as a failure
+func TestStatsServer_stopAbandonsGraphiteSend(t *testing.T) {
+	bigGraphiteRegistry(metrics.DefaultRegistry, t)
+	h := newGraphiteHost(t, 4096)
+	logs := &lockedBuffer{}
+	s, c := newTestStatsServer(t)
+	s.l = test.NewLoggerWithOutput(logs)
+	setStatsConfig(c, map[string]any{"type": "graphite", "interval": "50ms", "host": h.ln.Addr().String()})
+	require.NoError(t, s.reload(c, true))
+
+	started := make(chan struct{})
+	go func() {
+		s.Start()
+		close(started)
+	}()
+	stuck := h.next(t, 30*time.Second)
+	s.Stop()
+	select {
+	case <-started:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not return after Stop")
+	}
+
+	// A closed sender resets the connection once it sees this, one still sending just buffers it. Either way the
+	// read below doesn't have to drain the export through the small window to find out
+	_, err := stuck.Write([]byte("x"))
+	require.NoError(t, err)
+	require.NoError(t, stuck.SetReadDeadline(time.Now().Add(10*time.Second)))
+	n, err := io.Copy(io.Discard, stuck)
+	var ne net.Error
+	if errors.As(err, &ne) && ne.Timeout() {
+		t.Fatalf("the send kept the connection after Stop, read %d bytes", n)
+	}
+	assert.Less(t, n, int64(1<<20), "the send kept going after Stop")
+	// The capture loop logs on its own goroutine after Start has returned, give it the chance
+	assert.Never(t, func() bool { return strings.Contains(logs.String(), "Graphite export failed") }, 500*time.Millisecond, 10*time.Millisecond, "a stop is not a failure")
+}