overlay/tio: KeepAlive poll-path readv/writev buffers too

The Poll fallback (used when IFF_VNET_HDR can't be enabled) has the
same unsafe-pointer-via-uintptr pattern as Offload.rawWrite:
readOne/writeOne build a [2]syscall.Iovec on the stack, pass it to
syscall.Syscall as uintptr, and the kernel then DMAs in/out of the
to/from slices whose Base pointers the iovec holds.

Escape analysis can't see that use, so under GC pressure the
backing memory could be collected or moved mid-syscall.

Add runtime.KeepAlive on the iovec and the user buffer around both
the SYS_READV and SYS_WRITEV syscalls. Same pattern and rationale
as the prior commit on the offload path.

.github/workflows/smoke/smoke.sh

@@ -82,7 +82,7 @@ docker exec host4 tcpdump -i eth0 -q -w - -U 2>logs/host4.outside.log >logs/host
 
 docker exec host2 ncat -nklv 0.0.0.0 2000 &
 docker exec host3 ncat -nklv 0.0.0.0 2000 &
-docker exec host4 ncat -e '/usr/bin/echo helloagainfromhost4' -nkluv 0.0.0.0 4000 &
+docker exec host4 ncat -nkluv 0.0.0.0 4000 &
 docker exec host2 ncat -e '/usr/bin/echo host2' -nkluv 0.0.0.0 3000 &
 docker exec host3 ncat -e '/usr/bin/echo host3' -nkluv 0.0.0.0 3000 &
 
@@ -155,11 +155,11 @@ echo " *** Testing conntrack"
 echo
 set -x
 
-# host4's outbound firewall only allows ICMP to the lighthouse, so host4
-# cannot initiate UDP to host2. Once host2 initiates a flow to host4:4000,
-# conntrack must let host4's listener reply on that flow. If it doesn't,
-# the echo back from host4 never reaches host2.
-docker exec host2 sh -c "(/usr/bin/echo host2; sleep 2) | ncat -nuv 192.168.100.4 4000" | grep -q helloagainfromhost4
+# host2 speaking to host4 on UDP 4000 should allow it to reply, when firewall rules would normally not permit this
+docker exec host2 sh -c "/usr/bin/echo host2 | ncat -nuv 192.168.100.4 4000"
+docker exec host2 ncat -e '/usr/bin/echo helloagainfromhost2' -nkluv 0.0.0.0 4000 &
+sleep 1
+docker exec host4 sh -c "/usr/bin/echo host4 | ncat -nuv 192.168.100.2 4000"
 
 docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'

.golangci.yaml

@@ -2,21 +2,7 @@ version: "2"
 linters:
   default: none
   enable:
-    - sloglint
     - testifylint
-  settings:
-    sloglint:
-      # Enforce key-value pair form for Info/Debug/Warn/Error/Log/With and
-      # the package-level slog equivalents. Use l.Log(ctx, level, ...) for
-      # custom levels instead of LogAttrs when you can.
-      #
-      # LogAttrs is also flagged by this rule because it takes ...slog.Attr;
-      # the few legitimate sites (where attrs is built up as a []slog.Attr)
-      # carry a //nolint:sloglint with rationale.
-      kv-only: true
-      # no-mixed-args is on by default: forbids mixing kv and attrs in one call.
-      # discard-handler is on by default (since Go 1.24): suggests
-      # slog.DiscardHandler over slog.NewTextHandler(io.Discard, nil).
   exclusions:
     generated: lax
     presets:

batch.go

@@ -0,0 +1,70 @@
+package nebula
+
+import "net/netip"
+
+// sendBatchCap is the maximum number of encrypted packets accumulated before a
+// flush is forced. TSO superpackets segment to at most ~45 packets on
+// reasonable MTUs, so 128 leaves headroom without bloating the backing
+// allocation.
+const sendBatchCap = 128
+
+// sendBatch accumulates encrypted UDP packets for a single sendmmsg flush.
+// One sendBatch is owned by each listenIn goroutine; no locking is needed.
+// The backing storage holds up to batchCap packets of slotCap bytes each;
+// bufs and dsts are parallel slices of committed slots.
+type sendBatch struct {
+	bufs     [][]byte
+	dsts     []netip.AddrPort
+	backing  []byte
+	slotCap  int
+	batchCap int
+	nextSlot int
+}
+
+func newSendBatch(batchCap, slotCap int) *sendBatch {
+	return &sendBatch{
+		bufs:     make([][]byte, 0, batchCap),
+		dsts:     make([]netip.AddrPort, 0, batchCap),
+		backing:  make([]byte, batchCap*slotCap),
+		slotCap:  slotCap,
+		batchCap: batchCap,
+	}
+}
+
+// Next returns a zero-length slice with slotCap capacity over the next unused
+// slot's backing bytes. The caller writes into the returned slice and then
+// calls Commit with the final length and destination. Next returns nil when
+// the batch is full.
+func (b *sendBatch) Next() []byte {
+	if b.nextSlot >= b.batchCap {
+		return nil
+	}
+	start := b.nextSlot * b.slotCap
+	return b.backing[start : start : start+b.slotCap]
+}
+
+// Commit records the slot just returned by Next as a packet of length n
+// destined for dst.
+func (b *sendBatch) Commit(n int, dst netip.AddrPort) {
+	start := b.nextSlot * b.slotCap
+	b.bufs = append(b.bufs, b.backing[start:start+n])
+	b.dsts = append(b.dsts, dst)
+	b.nextSlot++
+}
+
+// Reset clears committed slots; backing storage is retained for reuse.
+func (b *sendBatch) Reset() {
+	b.bufs = b.bufs[:0]
+	b.dsts = b.dsts[:0]
+	b.nextSlot = 0
+}
+
+// Len returns the number of committed packets.
+func (b *sendBatch) Len() int {
+	return len(b.bufs)
+}
+
+// Cap returns the maximum number of slots in the batch.
+func (b *sendBatch) Cap() int {
+	return b.batchCap
+}

batch_test.go

@@ -0,0 +1,69 @@
+package nebula
+
+import (
+	"net/netip"
+	"testing"
+)
+
+func TestSendBatchBookkeeping(t *testing.T) {
+	b := newSendBatch(4, 32)
+	if b.Len() != 0 || b.Cap() != 4 {
+		t.Fatalf("fresh batch: len=%d cap=%d", b.Len(), b.Cap())
+	}
+
+	ap := netip.MustParseAddrPort("10.0.0.1:4242")
+	for i := 0; i < 4; i++ {
+		slot := b.Next()
+		if slot == nil {
+			t.Fatalf("slot %d: Next returned nil before cap", i)
+		}
+		if cap(slot) != 32 || len(slot) != 0 {
+			t.Fatalf("slot %d: got len=%d cap=%d want len=0 cap=32", i, len(slot), cap(slot))
+		}
+		// Write a marker byte.
+		slot = append(slot, byte(i), byte(i+1), byte(i+2))
+		b.Commit(len(slot), ap)
+	}
+	if b.Next() != nil {
+		t.Fatalf("Next should return nil when full")
+	}
+	if b.Len() != 4 {
+		t.Fatalf("Len=%d want 4", b.Len())
+	}
+	for i, buf := range b.bufs {
+		if len(buf) != 3 || buf[0] != byte(i) {
+			t.Errorf("buf %d: %x", i, buf)
+		}
+		if b.dsts[i] != ap {
+			t.Errorf("dst %d: got %v want %v", i, b.dsts[i], ap)
+		}
+	}
+
+	// Reset returns empty and Next works again.
+	b.Reset()
+	if b.Len() != 0 {
+		t.Fatalf("after Reset Len=%d want 0", b.Len())
+	}
+	slot := b.Next()
+	if slot == nil || cap(slot) != 32 {
+		t.Fatalf("after Reset Next nil or wrong cap: %v cap=%d", slot == nil, cap(slot))
+	}
+}
+
+func TestSendBatchSlotsDoNotOverlap(t *testing.T) {
+	b := newSendBatch(3, 8)
+	ap := netip.MustParseAddrPort("10.0.0.1:80")
+
+	// Fill three slots, each with its own sentinel byte.
+	for i := 0; i < 3; i++ {
+		s := b.Next()
+		s = append(s, byte(0xA0+i), byte(0xB0+i))
+		b.Commit(len(s), ap)
+	}
+
+	for i, buf := range b.bufs {
+		if buf[0] != byte(0xA0+i) || buf[1] != byte(0xB0+i) {
+			t.Errorf("slot %d corrupted: %x", i, buf)
+		}
+	}
+}

bits.go

@@ -1,10 +1,8 @@
 package nebula
 
 import (
-	"context"
-	"log/slog"
-
 	"github.com/rcrowley/go-metrics"
+	"github.com/sirupsen/logrus"
 )
 
 type Bits struct {
@@ -32,7 +30,7 @@ func NewBits(bits uint64) *Bits {
 	return b
 }
 
-func (b *Bits) Check(l *slog.Logger, i uint64) bool {
+func (b *Bits) Check(l *logrus.Logger, i uint64) bool {
 	// If i is the next number, return true.
 	if i > b.current {
 		return true
@@ -44,16 +42,13 @@ func (b *Bits) Check(l *slog.Logger, i uint64) bool {
 	}
 
 	// Not within the window
-	if l.Enabled(context.Background(), slog.LevelDebug) {
-		l.Debug("rejected a packet (top)",
-			"current", b.current,
-			"incoming", i,
-		)
+	if l.Level >= logrus.DebugLevel {
+		l.Debugf("rejected a packet (top) %d %d\n", b.current, i)
 	}
 	return false
 }
 
-func (b *Bits) Update(l *slog.Logger, i uint64) bool {
+func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 	// If i is the next number, return true and update current.
 	if i == b.current+1 {
 		// Check if the oldest bit was lost since we are shifting the window by 1 and occupying it with this counter
@@ -92,13 +87,9 @@ func (b *Bits) Update(l *slog.Logger, i uint64) bool {
 	// Check to see if it's a duplicate
 	if i > b.current-b.length || i < b.length && b.current < b.length {
 		if b.current == i || b.bits[i%b.length] == true {
-			if l.Enabled(context.Background(), slog.LevelDebug) {
-				l.Debug("Receive window",
-					"accepted", false,
-					"currentCounter", b.current,
-					"incomingCounter", i,
-					"reason", "duplicate",
-				)
+			if l.Level >= logrus.DebugLevel {
+				l.WithField("receiveWindow", m{"accepted": false, "currentCounter": b.current, "incomingCounter": i, "reason": "duplicate"}).
+					Debug("Receive window")
 			}
 			b.dupeCounter.Inc(1)
 			return false
@@ -110,13 +101,12 @@ func (b *Bits) Update(l *slog.Logger, i uint64) bool {
 
 	// In all other cases, fail and don't change current.
 	b.outOfWindowCounter.Inc(1)
-	if l.Enabled(context.Background(), slog.LevelDebug) {
-		l.Debug("Receive window",
-			"accepted", false,
-			"currentCounter", b.current,
-			"incomingCounter", i,
-			"reason", "nonsense",
-		)
+	if l.Level >= logrus.DebugLevel {
+		l.WithField("accepted", false).
+			WithField("currentCounter", b.current).
+			WithField("incomingCounter", i).
+			WithField("reason", "nonsense").
+			Debug("Receive window")
 	}
 	return false
 }

cert_test/cert.go

@@ -163,55 +163,3 @@ func P256Keypair() ([]byte, []byte) {
 	pubkey := privkey.PublicKey()
 	return pubkey.Bytes(), privkey.Bytes()
 }
-
-// DummyCert is a minimal cert.Certificate implementation for testing error paths.
-type DummyCert struct {
-	Version_        cert.Version
-	Curve_          cert.Curve
-	Groups_         []string
-	IsCA_           bool
-	Issuer_         string
-	Name_           string
-	Networks_       []netip.Prefix
-	NotAfter_       time.Time
-	NotBefore_      time.Time
-	PublicKey_      []byte
-	Signature_      []byte
-	UnsafeNetworks_ []netip.Prefix
-}
-
-func (d *DummyCert) Version() cert.Version                         { return d.Version_ }
-func (d *DummyCert) Curve() cert.Curve                             { return d.Curve_ }
-func (d *DummyCert) Groups() []string                              { return d.Groups_ }
-func (d *DummyCert) IsCA() bool                                    { return d.IsCA_ }
-func (d *DummyCert) Issuer() string                                { return d.Issuer_ }
-func (d *DummyCert) Name() string                                  { return d.Name_ }
-func (d *DummyCert) Networks() []netip.Prefix                      { return d.Networks_ }
-func (d *DummyCert) NotAfter() time.Time                           { return d.NotAfter_ }
-func (d *DummyCert) NotBefore() time.Time                          { return d.NotBefore_ }
-func (d *DummyCert) PublicKey() []byte                             { return d.PublicKey_ }
-func (d *DummyCert) Signature() []byte                             { return d.Signature_ }
-func (d *DummyCert) UnsafeNetworks() []netip.Prefix                { return d.UnsafeNetworks_ }
-func (d *DummyCert) Fingerprint() (string, error)                  { return "", nil }
-func (d *DummyCert) CheckSignature(key []byte) bool                { return false }
-func (d *DummyCert) MarshalForHandshakes() ([]byte, error)         { return nil, nil }
-func (d *DummyCert) MarshalPEM() ([]byte, error)                   { return nil, nil }
-func (d *DummyCert) MarshalJSON() ([]byte, error)                  { return nil, nil }
-func (d *DummyCert) Marshal() ([]byte, error)                      { return nil, nil }
-func (d *DummyCert) String() string                                { return "dummy" }
-func (d *DummyCert) Copy() cert.Certificate                        { return d }
-func (d *DummyCert) VerifyPrivateKey(c cert.Curve, k []byte) error { return nil }
-func (d *DummyCert) Expired(time.Time) bool                        { return false }
-func (d *DummyCert) MarshalPublicKeyPEM() []byte                   { return nil }
-func (d *DummyCert) PublicKeyPEM() []byte                          { return nil }
-
-// NewTestCAPool creates a CAPool from the given CA certificates, panicking on error.
-func NewTestCAPool(cas ...cert.Certificate) *cert.CAPool {
-	pool := cert.NewCAPool()
-	for _, ca := range cas {
-		if err := pool.AddCA(ca); err != nil {
-			panic(err)
-		}
-	}
-	return pool
-}

cmd/nebula-service/logs_generic.go

@@ -3,15 +3,8 @@
 
 package main
 
-import (
-	"log/slog"
-	"os"
+import "github.com/sirupsen/logrus"
 
-	"github.com/slackhq/nebula/logging"
-)
-
-// newPlatformLogger returns a *slog.Logger that writes to stdout. Non-Windows
-// platforms have no special sink to integrate with.
-func newPlatformLogger() *slog.Logger {
-	return logging.NewLogger(os.Stdout)
+func HookLogger(l *logrus.Logger) {
+	// Do nothing, let the logs flow to stdout/stderr
 }

cmd/nebula-service/logs_windows.go

@@ -1,86 +1,54 @@
 package main
 
 import (
-	"context"
-	"log/slog"
-	"strings"
-	"sync"
+	"fmt"
+	"io/ioutil"
+	"os"
 
-	"github.com/slackhq/nebula/logging"
+	"github.com/kardianos/service"
+	"github.com/sirupsen/logrus"
 )
 
-// newPlatformLogger returns a *slog.Logger that routes every log record
-// through the Windows service logger so records end up in the Windows
-// Event Log. All the heavy lifting (level management, format swap,
-// timestamp toggle, WithAttrs/WithGroup) comes from logging.NewHandler;
-// this file only contributes:
-//
-//   - an io.Writer that forwards each formatted line to the service
-//     logger at the current record's Event Log severity, and
-//   - a thin severityTag that embeds *logging.Handler and overrides
-//     only Handle / WithAttrs / WithGroup, so Event Viewer's severity
-//     column and severity-based filters keep working the way they did
-//     before the slog migration.
-//
-// Format (text vs json) is carried by the embedded *logging.Handler, so
-// logging.format: json in config still produces JSON lines in Event
-// Viewer, same as the pre-slog logrus setup.
-func newPlatformLogger() *slog.Logger {
-	w := &eventLogWriter{}
-	return slog.New(&severityTag{Handler: logging.NewHandler(w), w: w})
+// HookLogger routes the logrus logs through the service logger so that they end up in the Windows Event Viewer
+// logrus output will be discarded
+func HookLogger(l *logrus.Logger) {
+	l.AddHook(newLogHook(logger))
+	l.SetOutput(ioutil.Discard)
 }
 
-// eventLogWriter forwards slog-formatted lines to the Windows service
-// logger at the severity most recently stashed by severityTag.Handle.
-// The mutex serializes the stash + inner.Handle + Write cycle per record
-// across all concurrent goroutines; slog's builtin text/json handlers
-// each hold their own mutex around Write, but that only protects the
-// Write call itself, not our stash-then-handle sequence.
-type eventLogWriter struct {
-	mu    sync.Mutex
-	level slog.Level
+type logHook struct {
+	sl service.Logger
 }
 
-func (w *eventLogWriter) Write(p []byte) (int, error) {
-	line := strings.TrimRight(string(p), "\n")
-	switch {
-	case w.level >= slog.LevelError:
-		return len(p), logger.Error(line)
-	case w.level >= slog.LevelWarn:
-		return len(p), logger.Warning(line)
+func newLogHook(sl service.Logger) *logHook {
+	return &logHook{sl: sl}
+}
+
+func (h *logHook) Fire(entry *logrus.Entry) error {
+	line, err := entry.String()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err)
+		return err
+	}
+
+	switch entry.Level {
+	case logrus.PanicLevel:
+		return h.sl.Error(line)
+	case logrus.FatalLevel:
+		return h.sl.Error(line)
+	case logrus.ErrorLevel:
+		return h.sl.Error(line)
+	case logrus.WarnLevel:
+		return h.sl.Warning(line)
+	case logrus.InfoLevel:
+		return h.sl.Info(line)
+	case logrus.DebugLevel:
+		return h.sl.Info(line)
 	default:
-		return len(p), logger.Info(line)
+		return nil
 	}
 }
 
-// severityTag embeds *logging.Handler to pick up everything it does for
-// free (Enabled, SetLevel, GetLevel, SetFormat, GetFormat,
-// SetDisableTimestamp) and overrides only Handle / WithAttrs / WithGroup
-// so each record's slog.Level is stashed on the writer before formatting
-// and so derived handlers stay wrapped as severityTag rather than
-// downgrading to bare *logging.Handler.
-type severityTag struct {
-	*logging.Handler
-	w *eventLogWriter
-}
-
-func (s *severityTag) Handle(ctx context.Context, r slog.Record) error {
-	s.w.mu.Lock()
-	defer s.w.mu.Unlock()
-	s.w.level = r.Level
-	return s.Handler.Handle(ctx, r)
-}
-
-func (s *severityTag) WithAttrs(attrs []slog.Attr) slog.Handler {
-	if len(attrs) == 0 {
-		return s
-	}
-	return &severityTag{Handler: s.Handler.WithAttrs(attrs).(*logging.Handler), w: s.w}
-}
-
-func (s *severityTag) WithGroup(name string) slog.Handler {
-	if name == "" {
-		return s
-	}
-	return &severityTag{Handler: s.Handler.WithGroup(name).(*logging.Handler), w: s.w}
+func (h *logHook) Levels() []logrus.Level {
+	return logrus.AllLevels
 }

cmd/nebula-service/main.go

@@ -7,9 +7,9 @@ import (
 	"runtime/debug"
 	"strings"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/logging"
 	"github.com/slackhq/nebula/util"
 )
 
@@ -50,14 +50,9 @@ func main() {
 		os.Exit(0)
 	}
 
-	l := logging.NewLogger(os.Stdout)
-
 	if *serviceFlag != "" {
-		if err := doService(configPath, configTest, Build, serviceFlag); err != nil {
-			l.Error("Service command failed", "error", err)
-			os.Exit(1)
-		}
-		return
+		doService(configPath, configTest, Build, serviceFlag)
+		os.Exit(1)
 	}
 
 	if *configPath == "" {
@@ -66,6 +61,9 @@ func main() {
 		os.Exit(1)
 	}
 
+	l := logrus.New()
+	l.Out = os.Stdout
+
 	c := config.NewC(l)
 	err := c.Load(*configPath)
 	if err != nil {
@@ -73,16 +71,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	if err := logging.ApplyConfig(l, c); err != nil {
-		fmt.Printf("failed to apply logging config: %s", err)
-		os.Exit(1)
-	}
-	c.RegisterReloadCallback(func(c *config.C) {
-		if err := logging.ApplyConfig(l, c); err != nil {
-			l.Error("Failed to reconfigure logger on reload", "error", err)
-		}
-	})
-
 	ctrl, err := nebula.Main(c, *configTest, Build, l, nil)
 	if err != nil {
 		util.LogWithContextIfNeeded("Failed to start", err, l)
@@ -99,7 +87,7 @@ func main() {
 		go ctrl.ShutdownBlock()
 
 		if err := wait(); err != nil {
-			l.Error("Nebula stopped due to fatal error", "error", err)
+			l.WithError(err).Error("Nebula stopped due to fatal error")
 			os.Exit(2)
 		}
 

cmd/nebula-service/service.go

@@ -7,9 +7,9 @@ import (
 	"path/filepath"
 
 	"github.com/kardianos/service"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/logging"
 )
 
 var logger service.Logger
@@ -25,7 +25,8 @@ func (p *program) Start(s service.Service) error {
 	// Start should not block.
 	logger.Info("Nebula service starting.")
 
-	l := newPlatformLogger()
+	l := logrus.New()
+	HookLogger(l)
 
 	c := config.NewC(l)
 	err := c.Load(*p.configPath)
@@ -33,15 +34,6 @@ func (p *program) Start(s service.Service) error {
 		return fmt.Errorf("failed to load config: %s", err)
 	}
 
-	if err := logging.ApplyConfig(l, c); err != nil {
-		return fmt.Errorf("failed to apply logging config: %s", err)
-	}
-	c.RegisterReloadCallback(func(c *config.C) {
-		if err := logging.ApplyConfig(l, c); err != nil {
-			l.Error("Failed to reconfigure logger on reload", "error", err)
-		}
-	})
-
 	p.control, err = nebula.Main(c, *p.configTest, Build, l, nil)
 	if err != nil {
 		return err
@@ -65,11 +57,11 @@ func fileExists(filename string) bool {
 	return true
 }
 
-func doService(configPath *string, configTest *bool, build string, serviceFlag *string) error {
+func doService(configPath *string, configTest *bool, build string, serviceFlag *string) {
 	if *configPath == "" {
 		ex, err := os.Executable()
 		if err != nil {
-			return err
+			panic(err)
 		}
 		*configPath = filepath.Dir(ex) + "/config.yaml"
 		if !fileExists(*configPath) {
@@ -93,16 +85,16 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
 	// Here are what the different loggers are doing:
 	// - `log` is the standard go log utility, meant to be used while the process is still attached to stdout/stderr
 	// - `logger` is the service log utility that may be attached to a special place depending on OS (Windows will have it attached to the event log)
-	// - in program.Start we build a *slog.Logger via newPlatformLogger; on non-Windows that is a stdout-backed slog logger, on Windows it routes records through the service logger
+	// - above, in `Run` we create a `logrus.Logger` which is what nebula expects to use
 	s, err := service.New(prg, svcConfig)
 	if err != nil {
-		return err
+		log.Fatal(err)
 	}
 
 	errs := make(chan error, 5)
 	logger, err = s.Logger(errs)
 	if err != nil {
-		return err
+		log.Fatal(err)
 	}
 
 	go func() {
@@ -117,16 +109,18 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
 
 	switch *serviceFlag {
 	case "run":
-		if err := s.Run(); err != nil {
+		err = s.Run()
+		if err != nil {
 			// Route any errors to the system logger
 			logger.Error(err)
 		}
 	default:
-		if err := service.Control(s, *serviceFlag); err != nil {
+		err := service.Control(s, *serviceFlag)
+		if err != nil {
 			log.Printf("Valid actions: %q\n", service.ControlAction)
-			return err
+			log.Fatal(err)
 		}
+		return
 	}
 
-	return nil
 }

cmd/nebula/main.go

@@ -7,9 +7,9 @@ import (
 	"runtime/debug"
 	"strings"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/logging"
 	"github.com/slackhq/nebula/util"
 )
 
@@ -55,7 +55,8 @@ func main() {
 		os.Exit(1)
 	}
 
-	l := logging.NewLogger(os.Stdout)
+	l := logrus.New()
+	l.Out = os.Stdout
 
 	c := config.NewC(l)
 	err := c.Load(*configPath)
@@ -64,16 +65,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	if err := logging.ApplyConfig(l, c); err != nil {
-		fmt.Printf("failed to apply logging config: %s", err)
-		os.Exit(1)
-	}
-	c.RegisterReloadCallback(func(c *config.C) {
-		if err := logging.ApplyConfig(l, c); err != nil {
-			l.Error("Failed to reconfigure logger on reload", "error", err)
-		}
-	})
-
 	ctrl, err := nebula.Main(c, *configTest, Build, l, nil)
 	if err != nil {
 		util.LogWithContextIfNeeded("Failed to start", err, l)
@@ -91,7 +82,7 @@ func main() {
 		notifyReady(l)
 
 		if err := wait(); err != nil {
-			l.Error("Nebula stopped due to fatal error", "error", err)
+			l.WithError(err).Error("Nebula stopped due to fatal error")
 			os.Exit(2)
 		}
 

cmd/nebula/notify_linux.go

@@ -1,10 +1,11 @@
 package main
 
 import (
-	"log/slog"
 	"net"
 	"os"
 	"time"
+
+	"github.com/sirupsen/logrus"
 )
 
 // SdNotifyReady tells systemd the service is ready and dependent services can now be started
@@ -12,30 +13,30 @@ import (
 // https://www.freedesktop.org/software/systemd/man/systemd.service.html
 const SdNotifyReady = "READY=1"
 
-func notifyReady(l *slog.Logger) {
+func notifyReady(l *logrus.Logger) {
 	sockName := os.Getenv("NOTIFY_SOCKET")
 	if sockName == "" {
-		l.Debug("NOTIFY_SOCKET systemd env var not set, not sending ready signal")
+		l.Debugln("NOTIFY_SOCKET systemd env var not set, not sending ready signal")
 		return
 	}
 
 	conn, err := net.DialTimeout("unixgram", sockName, time.Second)
 	if err != nil {
-		l.Error("failed to connect to systemd notification socket", "error", err)
+		l.WithError(err).Error("failed to connect to systemd notification socket")
 		return
 	}
 	defer conn.Close()
 
 	err = conn.SetWriteDeadline(time.Now().Add(time.Second))
 	if err != nil {
-		l.Error("failed to set the write deadline for the systemd notification socket", "error", err)
+		l.WithError(err).Error("failed to set the write deadline for the systemd notification socket")
 		return
 	}
 
 	if _, err = conn.Write([]byte(SdNotifyReady)); err != nil {
-		l.Error("failed to signal the systemd notification socket", "error", err)
+		l.WithError(err).Error("failed to signal the systemd notification socket")
 		return
 	}
 
-	l.Debug("notified systemd the service is ready")
+	l.Debugln("notified systemd the service is ready")
 }

cmd/nebula/notify_notlinux.go

@@ -3,8 +3,8 @@
 
 package main
 
-import "log/slog"
+import "github.com/sirupsen/logrus"
 
-func notifyReady(_ *slog.Logger) {
+func notifyReady(_ *logrus.Logger) {
 	// No init service to notify
 }

config/config.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"log/slog"
 	"math"
 	"os"
 	"os/signal"
@@ -17,6 +16,7 @@ import (
 	"time"
 
 	"dario.cat/mergo"
+	"github.com/sirupsen/logrus"
 	"go.yaml.in/yaml/v3"
 )
 
@@ -26,11 +26,11 @@ type C struct {
 	Settings    map[string]any
 	oldSettings map[string]any
 	callbacks   []func(*C)
-	l           *slog.Logger
+	l           *logrus.Logger
 	reloadLock  sync.Mutex
 }
 
-func NewC(l *slog.Logger) *C {
+func NewC(l *logrus.Logger) *C {
 	return &C{
 		Settings: make(map[string]any),
 		l:        l,
@@ -107,18 +107,12 @@ func (c *C) HasChanged(k string) bool {
 
 	newVals, err := yaml.Marshal(nv)
 	if err != nil {
-		c.l.Error("Error while marshaling new config",
-			"config_path", k,
-			"error", err,
-		)
+		c.l.WithField("config_path", k).WithError(err).Error("Error while marshaling new config")
 	}
 
 	oldVals, err := yaml.Marshal(ov)
 	if err != nil {
-		c.l.Error("Error while marshaling old config",
-			"config_path", k,
-			"error", err,
-		)
+		c.l.WithField("config_path", k).WithError(err).Error("Error while marshaling old config")
 	}
 
 	return string(newVals) != string(oldVals)
@@ -160,10 +154,7 @@ func (c *C) ReloadConfig() {
 
 	err := c.Load(c.path)
 	if err != nil {
-		c.l.Error("Error occurred while reloading config",
-			"config_path", c.path,
-			"error", err,
-		)
+		c.l.WithField("config_path", c.path).WithError(err).Error("Error occurred while reloading config")
 		return
 	}
 

connection_manager.go

@@ -5,13 +5,13 @@ import (
 	"context"
 	"encoding/binary"
 	"fmt"
-	"log/slog"
 	"net/netip"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/rcrowley/go-metrics"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
@@ -47,10 +47,10 @@ type connectionManager struct {
 
 	metricsTxPunchy metrics.Counter
 
-	l *slog.Logger
+	l *logrus.Logger
 }
 
-func newConnectionManagerFromConfig(l *slog.Logger, c *config.C, hm *HostMap, p *Punchy) *connectionManager {
+func newConnectionManagerFromConfig(l *logrus.Logger, c *config.C, hm *HostMap, p *Punchy) *connectionManager {
 	cm := &connectionManager{
 		hostMap:         hm,
 		l:               l,
@@ -85,10 +85,9 @@ func (cm *connectionManager) reload(c *config.C, initial bool) {
 		old := cm.getInactivityTimeout()
 		cm.inactivityTimeout.Store((int64)(c.GetDuration("tunnels.inactivity_timeout", 10*time.Minute)))
 		if !initial {
-			cm.l.Info("Inactivity timeout has changed",
-				"oldDuration", old,
-				"newDuration", cm.getInactivityTimeout(),
-			)
+			cm.l.WithField("oldDuration", old).
+				WithField("newDuration", cm.getInactivityTimeout()).
+				Info("Inactivity timeout has changed")
 		}
 	}
 
@@ -96,10 +95,9 @@ func (cm *connectionManager) reload(c *config.C, initial bool) {
 		old := cm.dropInactive.Load()
 		cm.dropInactive.Store(c.GetBool("tunnels.drop_inactive", false))
 		if !initial {
-			cm.l.Info("Drop inactive setting has changed",
-				"oldBool", old,
-				"newBool", cm.dropInactive.Load(),
-			)
+			cm.l.WithField("oldBool", old).
+				WithField("newBool", cm.dropInactive.Load()).
+				Info("Drop inactive setting has changed")
 		}
 	}
 }
@@ -153,8 +151,8 @@ func (cm *connectionManager) Start(ctx context.Context) {
 	defer clockSource.Stop()
 
 	p := []byte("")
-	// Long-lived buf for the traffic-check goroutine; never released.
-	buf := cm.intf.bufAlloc.Acquire()
+	nb := make([]byte, 12, 12)
+	out := make([]byte, mtu)
 
 	for {
 		select {
@@ -169,13 +167,13 @@ func (cm *connectionManager) Start(ctx context.Context) {
 					break
 				}
 
-				cm.doTrafficCheck(localIndex, p, buf, now)
+				cm.doTrafficCheck(localIndex, p, nb, out, now)
 			}
 		}
 	}
 }
 
-func (cm *connectionManager) doTrafficCheck(localIndex uint32, p []byte, buf *WireBuffer, now time.Time) {
+func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
 	decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now)
 
 	switch decision {
@@ -199,7 +197,7 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p []byte, buf *Wi
 		cm.tryRehandshake(hostinfo)
 
 	case sendTestPacket:
-		cm.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, buf)
+		cm.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, nb, out)
 	}
 
 	cm.resetRelayTrafficCheck(hostinfo)
@@ -258,7 +256,7 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
 			var err error
 			index, err = AddRelay(cm.l, newhostinfo, cm.hostMap, r.PeerAddr, nil, r.Type, Requested)
 			if err != nil {
-				cm.l.Error("failed to migrate relay to new hostinfo", "error", err)
+				cm.l.WithError(err).Error("failed to migrate relay to new hostinfo")
 				continue
 			}
 			switch r.Type {
@@ -306,18 +304,16 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
 
 		msg, err := req.Marshal()
 		if err != nil {
-			cm.l.Error("failed to marshal Control message to migrate relay", "error", err)
+			cm.l.WithError(err).Error("failed to marshal Control message to migrate relay")
 		} else {
-			migBuf := cm.intf.bufAlloc.Acquire()
-			cm.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, migBuf)
-			cm.intf.bufAlloc.Release(migBuf)
-			cm.l.Info("send CreateRelayRequest",
-				"relayFrom", req.RelayFromAddr,
-				"relayTo", req.RelayToAddr,
-				"initiatorRelayIndex", req.InitiatorRelayIndex,
-				"responderRelayIndex", req.ResponderRelayIndex,
-				"vpnAddrs", newhostinfo.vpnAddrs,
-			)
+			cm.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, make([]byte, 12), make([]byte, mtu))
+			cm.l.WithFields(logrus.Fields{
+				"relayFrom":           req.RelayFromAddr,
+				"relayTo":             req.RelayToAddr,
+				"initiatorRelayIndex": req.InitiatorRelayIndex,
+				"responderRelayIndex": req.ResponderRelayIndex,
+				"vpnAddrs":            newhostinfo.vpnAddrs}).
+				Info("send CreateRelayRequest")
 		}
 	}
 }
@@ -329,7 +325,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 
 	hostinfo := cm.hostMap.Indexes[localIndex]
 	if hostinfo == nil {
-		cm.l.Debug("Not found in hostmap", "localIndex", localIndex)
+		cm.l.WithField("localIndex", localIndex).Debugln("Not found in hostmap")
 		return doNothing, nil, nil
 	}
 
@@ -349,10 +345,10 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 	// A hostinfo is determined alive if there is incoming traffic
 	if inTraffic {
 		decision := doNothing
-		if cm.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(cm.l).Debug("Tunnel status",
-				"tunnelCheck", m{"state": "alive", "method": "passive"},
-			)
+		if cm.l.Level >= logrus.DebugLevel {
+			hostinfo.logger(cm.l).
+				WithField("tunnelCheck", m{"state": "alive", "method": "passive"}).
+				Debug("Tunnel status")
 		}
 		hostinfo.pendingDeletion.Store(false)
 
@@ -379,9 +375,9 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 
 	if hostinfo.pendingDeletion.Load() {
 		// We have already sent a test packet and nothing was returned, this hostinfo is dead
-		hostinfo.logger(cm.l).Info("Tunnel status",
-			"tunnelCheck", m{"state": "dead", "method": "active"},
-		)
+		hostinfo.logger(cm.l).
+			WithField("tunnelCheck", m{"state": "dead", "method": "active"}).
+			Info("Tunnel status")
 
 		return deleteTunnel, hostinfo, nil
 	}
@@ -392,10 +388,10 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 			inactiveFor, isInactive := cm.isInactive(hostinfo, now)
 			if isInactive {
 				// Tunnel is inactive, tear it down
-				hostinfo.logger(cm.l).Info("Dropping tunnel due to inactivity",
-					"inactiveDuration", inactiveFor,
-					"primary", mainHostInfo,
-				)
+				hostinfo.logger(cm.l).
+					WithField("inactiveDuration", inactiveFor).
+					WithField("primary", mainHostInfo).
+					Info("Dropping tunnel due to inactivity")
 
 				return closeTunnel, hostinfo, primary
 			}
@@ -414,18 +410,18 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 			cm.sendPunch(hostinfo)
 		}
 
-		if cm.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(cm.l).Debug("Tunnel status",
-				"tunnelCheck", m{"state": "testing", "method": "active"},
-			)
+		if cm.l.Level >= logrus.DebugLevel {
+			hostinfo.logger(cm.l).
+				WithField("tunnelCheck", m{"state": "testing", "method": "active"}).
+				Debug("Tunnel status")
 		}
 
 		// Send a test packet to trigger an authenticated tunnel test, this should suss out any lingering tunnel issues
 		decision = sendTestPacket
 
 	} else {
-		if cm.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(cm.l).Debug("Hostinfo sadness")
+		if cm.l.Level >= logrus.DebugLevel {
+			hostinfo.logger(cm.l).Debugf("Hostinfo sadness")
 		}
 	}
 
@@ -497,16 +493,14 @@ func (cm *connectionManager) isInvalidCertificate(now time.Time, hostinfo *HostI
 		return false //cert is still valid! yay!
 	} else if err == cert.ErrBlockListed { //avoiding errors.Is for speed
 		// Block listed certificates should always be disconnected
-		hostinfo.logger(cm.l).Info("Remote certificate is blocked, tearing down the tunnel",
-			"error", err,
-			"fingerprint", remoteCert.Fingerprint,
-		)
+		hostinfo.logger(cm.l).WithError(err).
+			WithField("fingerprint", remoteCert.Fingerprint).
+			Info("Remote certificate is blocked, tearing down the tunnel")
 		return true
 	} else if cm.intf.disconnectInvalid.Load() {
-		hostinfo.logger(cm.l).Info("Remote certificate is no longer valid, tearing down the tunnel",
-			"error", err,
-			"fingerprint", remoteCert.Fingerprint,
-		)
+		hostinfo.logger(cm.l).WithError(err).
+			WithField("fingerprint", remoteCert.Fingerprint).
+			Info("Remote certificate is no longer valid, tearing down the tunnel")
 		return true
 	} else {
 		//if we reach here, the cert is no longer valid, but we're configured to keep tunnels from now-invalid certs open
@@ -545,11 +539,10 @@ func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
 	curCrtVersion := curCrt.Version()
 	myCrt := cs.getCertificate(curCrtVersion)
 	if myCrt == nil {
-		cm.l.Info("Re-handshaking with remote",
-			"vpnAddrs", hostinfo.vpnAddrs,
-			"version", curCrtVersion,
-			"reason", "local certificate removed",
-		)
+		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
+			WithField("version", curCrtVersion).
+			WithField("reason", "local certificate removed").
+			Info("Re-handshaking with remote")
 		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
 		return
 	}
@@ -557,12 +550,11 @@ func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
 	if peerCrt != nil && curCrtVersion < peerCrt.Certificate.Version() {
 		// if our certificate version is less than theirs, and we have a matching version available, rehandshake?
 		if cs.getCertificate(peerCrt.Certificate.Version()) != nil {
-			cm.l.Info("Re-handshaking with remote",
-				"vpnAddrs", hostinfo.vpnAddrs,
-				"version", curCrtVersion,
-				"peerVersion", peerCrt.Certificate.Version(),
-				"reason", "local certificate version lower than peer, attempting to correct",
-			)
+			cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
+				WithField("version", curCrtVersion).
+				WithField("peerVersion", peerCrt.Certificate.Version()).
+				WithField("reason", "local certificate version lower than peer, attempting to correct").
+				Info("Re-handshaking with remote")
 			cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], func(hh *HandshakeHostInfo) {
 				hh.initiatingVersionOverride = peerCrt.Certificate.Version()
 			})
@@ -570,19 +562,17 @@ func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
 		}
 	}
 	if !bytes.Equal(curCrt.Signature(), myCrt.Signature()) {
-		cm.l.Info("Re-handshaking with remote",
-			"vpnAddrs", hostinfo.vpnAddrs,
-			"reason", "local certificate is not current",
-		)
+		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
+			WithField("reason", "local certificate is not current").
+			Info("Re-handshaking with remote")
 
 		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
 		return
 	}
 	if curCrtVersion < cs.initiatingVersion {
-		cm.l.Info("Re-handshaking with remote",
-			"vpnAddrs", hostinfo.vpnAddrs,
-			"reason", "current cert version < pki.initiatingVersion",
-		)
+		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
+			WithField("reason", "current cert version < pki.initiatingVersion").
+			Info("Re-handshaking with remote")
 
 		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
 		return

connection_manager_test.go

@@ -7,9 +7,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/flynn/noise"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/overlaytest"
+	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/test"
 	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
@@ -46,13 +47,13 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 		initiatingVersion: cert.Version1,
 		privateKey:        []byte{},
 		v1Cert:            &dummyCert{version: cert.Version1},
-		v1Credential:      nil,
+		v1HandshakeBytes:  []byte{},
 	}
 
 	lh := newTestLighthouse()
 	ifce := &Interface{
 		hostMap:          hostMap,
-		inside:           &overlaytest.NoopTun{},
+		inside:           &overlay.NoopTun{},
 		outside:          &udp.NoopConn{},
 		firewall:         &Firewall{},
 		lightHouse:       lh,
@@ -63,13 +64,13 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	ifce.pki.cs.Store(cs)
 
 	// Create manager
-	conf := config.NewC(test.NewLogger())
-	punchy := NewPunchyFromConfig(test.NewLogger(), conf)
-	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
+	conf := config.NewC(l)
+	punchy := NewPunchyFromConfig(l, conf)
+	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
 	nc.intf = ifce
-
 	p := []byte("")
-	buf := NewWireBuffer(mtu, 0)
+	nb := make([]byte, 12, 12)
+	out := make([]byte, mtu)
 
 	// Add an ip we have established a connection w/ to hostmap
 	hostinfo := &HostInfo{
@@ -79,6 +80,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	}
 	hostinfo.ConnectionState = &ConnectionState{
 		myCert: &dummyCert{version: cert.Version1},
+		H:      &noise.HandshakeState{},
 	}
 	nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
 
@@ -92,7 +94,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	assert.True(t, hostinfo.in.Load())
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.False(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -100,7 +102,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	// Do another traffic check tick, this host should be pending deletion now
 	nc.Out(hostinfo)
 	assert.True(t, hostinfo.out.Load())
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.True(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -108,7 +110,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 
 	// Do a final traffic check tick, the host should now be removed
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.NotContains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs)
 	assert.NotContains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 }
@@ -128,13 +130,13 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 		initiatingVersion: cert.Version1,
 		privateKey:        []byte{},
 		v1Cert:            &dummyCert{version: cert.Version1},
-		v1Credential:      nil,
+		v1HandshakeBytes:  []byte{},
 	}
 
 	lh := newTestLighthouse()
 	ifce := &Interface{
 		hostMap:          hostMap,
-		inside:           &overlaytest.NoopTun{},
+		inside:           &overlay.NoopTun{},
 		outside:          &udp.NoopConn{},
 		firewall:         &Firewall{},
 		lightHouse:       lh,
@@ -145,13 +147,13 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	ifce.pki.cs.Store(cs)
 
 	// Create manager
-	conf := config.NewC(test.NewLogger())
-	punchy := NewPunchyFromConfig(test.NewLogger(), conf)
-	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
+	conf := config.NewC(l)
+	punchy := NewPunchyFromConfig(l, conf)
+	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
 	nc.intf = ifce
-
 	p := []byte("")
-	buf := NewWireBuffer(mtu, 0)
+	nb := make([]byte, 12, 12)
+	out := make([]byte, mtu)
 
 	// Add an ip we have established a connection w/ to hostmap
 	hostinfo := &HostInfo{
@@ -161,6 +163,7 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	}
 	hostinfo.ConnectionState = &ConnectionState{
 		myCert: &dummyCert{version: cert.Version1},
+		H:      &noise.HandshakeState{},
 	}
 	nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
 
@@ -174,14 +177,14 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.False(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
 
 	// Do another traffic check tick, this host should be pending deletion now
 	nc.Out(hostinfo)
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.True(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -190,7 +193,7 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 
 	// We saw traffic, should no longer be pending deletion
 	nc.In(hostinfo)
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.False(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -212,13 +215,13 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
 		initiatingVersion: cert.Version1,
 		privateKey:        []byte{},
 		v1Cert:            &dummyCert{version: cert.Version1},
-		v1Credential:      nil,
+		v1HandshakeBytes:  []byte{},
 	}
 
 	lh := newTestLighthouse()
 	ifce := &Interface{
 		hostMap:          hostMap,
-		inside:           &overlaytest.NoopTun{},
+		inside:           &overlay.NoopTun{},
 		outside:          &udp.NoopConn{},
 		firewall:         &Firewall{},
 		lightHouse:       lh,
@@ -229,12 +232,12 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
 	ifce.pki.cs.Store(cs)
 
 	// Create manager
-	conf := config.NewC(test.NewLogger())
+	conf := config.NewC(l)
 	conf.Settings["tunnels"] = map[string]any{
 		"drop_inactive": true,
 	}
-	punchy := NewPunchyFromConfig(test.NewLogger(), conf)
-	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
+	punchy := NewPunchyFromConfig(l, conf)
+	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
 	assert.True(t, nc.dropInactive.Load())
 	nc.intf = ifce
 
@@ -246,6 +249,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
 	}
 	hostinfo.ConnectionState = &ConnectionState{
 		myCert: &dummyCert{version: cert.Version1},
+		H:      &noise.HandshakeState{},
 	}
 	nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
 
@@ -336,15 +340,15 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 	cachedPeerCert, err := ncp.VerifyCertificate(now.Add(time.Second), peerCert)
 
 	cs := &CertState{
-		privateKey:   []byte{},
-		v1Cert:       &dummyCert{},
-		v1Credential: nil,
+		privateKey:       []byte{},
+		v1Cert:           &dummyCert{},
+		v1HandshakeBytes: []byte{},
 	}
 
 	lh := newTestLighthouse()
 	ifce := &Interface{
 		hostMap:          hostMap,
-		inside:           &overlaytest.NoopTun{},
+		inside:           &overlay.NoopTun{},
 		outside:          &udp.NoopConn{},
 		firewall:         &Firewall{},
 		lightHouse:       lh,
@@ -357,9 +361,9 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 	ifce.disconnectInvalid.Store(true)
 
 	// Create manager
-	conf := config.NewC(test.NewLogger())
-	punchy := NewPunchyFromConfig(test.NewLogger(), conf)
-	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
+	conf := config.NewC(l)
+	punchy := NewPunchyFromConfig(l, conf)
+	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
 	nc.intf = ifce
 	ifce.connectionManager = nc
 
@@ -368,6 +372,7 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 		ConnectionState: &ConnectionState{
 			myCert:   &dummyCert{},
 			peerCert: cachedPeerCert,
+			H:        &noise.HandshakeState{},
 		},
 	}
 	nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)

connection_state.go

@@ -1,12 +1,16 @@
 package nebula
 
 import (
+	"crypto/rand"
 	"encoding/json"
+	"fmt"
 	"sync"
 	"sync/atomic"
 
+	"github.com/flynn/noise"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/handshake"
+	"github.com/slackhq/nebula/noiseutil"
 )
 
 const ReplayWindow = 1024
@@ -14,6 +18,7 @@ const ReplayWindow = 1024
 type ConnectionState struct {
 	eKey           *NebulaCipherState
 	dKey           *NebulaCipherState
+	H              *noise.HandshakeState
 	myCert         cert.Certificate
 	peerCert       *cert.CachedCertificate
 	initiator      bool
@@ -22,24 +27,55 @@ type ConnectionState struct {
 	writeLock      sync.Mutex
 }
 
-// newConnectionStateFromResult builds a fully-populated ConnectionState from a
-// completed handshake.Result. It seeds messageCounter and the replay window so
-// that the post-handshake message indices already used on the wire don't count
-// as missed traffic in the data plane.
-func newConnectionStateFromResult(r *handshake.Result) *ConnectionState {
+func NewConnectionState(l *logrus.Logger, cs *CertState, crt cert.Certificate, initiator bool, pattern noise.HandshakePattern) (*ConnectionState, error) {
+	var dhFunc noise.DHFunc
+	switch crt.Curve() {
+	case cert.Curve_CURVE25519:
+		dhFunc = noise.DH25519
+	case cert.Curve_P256:
+		if cs.pkcs11Backed {
+			dhFunc = noiseutil.DHP256PKCS11
+		} else {
+			dhFunc = noiseutil.DHP256
+		}
+	default:
+		return nil, fmt.Errorf("invalid curve: %s", crt.Curve())
+	}
+
+	var ncs noise.CipherSuite
+	if cs.cipher == "chachapoly" {
+		ncs = noise.NewCipherSuite(dhFunc, noise.CipherChaChaPoly, noise.HashSHA256)
+	} else {
+		ncs = noise.NewCipherSuite(dhFunc, noiseutil.CipherAESGCM, noise.HashSHA256)
+	}
+
+	static := noise.DHKey{Private: cs.privateKey, Public: crt.PublicKey()}
+	hs, err := noise.NewHandshakeState(noise.Config{
+		CipherSuite:   ncs,
+		Random:        rand.Reader,
+		Pattern:       pattern,
+		Initiator:     initiator,
+		StaticKeypair: static,
+		//NOTE: These should come from CertState (pki.go) when we finally implement it
+		PresharedKey:          []byte{},
+		PresharedKeyPlacement: 0,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("NewConnectionState: %s", err)
+	}
+
+	// The queue and ready params prevent a counter race that would happen when
+	// sending stored packets and simultaneously accepting new traffic.
 	ci := &ConnectionState{
-		myCert:    r.MyCert,
-		initiator: r.Initiator,
-		peerCert:  r.RemoteCert,
-		eKey:      NewNebulaCipherState(r.EKey),
-		dKey:      NewNebulaCipherState(r.DKey),
+		H:         hs,
+		initiator: initiator,
 		window:    NewBits(ReplayWindow),
+		myCert:    crt,
 	}
-	ci.messageCounter.Add(r.MessageIndex)
-	for i := uint64(1); i <= r.MessageIndex; i++ {
-		ci.window.Update(nil, i)
-	}
-	return ci
+	// always start the counter from 2, as packet 1 and packet 2 are handshake packets.
+	ci.messageCounter.Add(2)
+
+	return ci, nil
 }
 
 func (cs *ConnectionState) MarshalJSON() ([]byte, error) {

connection_state_test.go

@@ -1,114 +0,0 @@
-package nebula
-
-import (
-	"net/netip"
-	"testing"
-	"time"
-
-	"github.com/flynn/noise"
-	"github.com/slackhq/nebula/cert"
-	ct "github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/handshake"
-	"github.com/slackhq/nebula/header"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// runTestHandshake runs a complete IX handshake between two freshly-built
-// peers and returns the initiator and responder Results. Used to produce
-// real cipher states for tests that need to exercise post-handshake glue.
-func runTestHandshake(t *testing.T) (initR, respR *handshake.Result) {
-	t.Helper()
-
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-
-	makeCreds := func(name string, networks []netip.Prefix) handshake.GetCredentialFunc {
-		c, _, rawKey, _ := ct.NewTestCert(
-			cert.Version2, cert.Curve_CURVE25519, ca, caKey,
-			name, ca.NotBefore(), ca.NotAfter(), networks, nil, nil,
-		)
-		priv, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey)
-		require.NoError(t, err)
-		hsBytes, err := c.MarshalForHandshakes()
-		require.NoError(t, err)
-		ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
-		cred := handshake.NewCredential(c, hsBytes, priv, ncs)
-		return func(v cert.Version) *handshake.Credential {
-			if v == cert.Version2 {
-				return cred
-			}
-			return nil
-		}
-	}
-
-	verifier := func(c cert.Certificate) (*cert.CachedCertificate, error) {
-		return caPool.VerifyCertificate(time.Now(), c)
-	}
-
-	initCreds := makeCreds("initiator", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	respCreds := makeCreds("responder", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")})
-
-	initM, err := handshake.NewMachine(
-		cert.Version2, initCreds, verifier,
-		func() (uint32, error) { return 1000, nil },
-		true, header.HandshakeIXPSK0,
-	)
-	require.NoError(t, err)
-
-	respM, err := handshake.NewMachine(
-		cert.Version2, respCreds, verifier,
-		func() (uint32, error) { return 2000, nil },
-		false, header.HandshakeIXPSK0,
-	)
-	require.NoError(t, err)
-
-	msg1, err := initM.Initiate(nil)
-	require.NoError(t, err)
-
-	resp, respR, err := respM.ProcessPacket(nil, msg1)
-	require.NoError(t, err)
-	require.NotNil(t, respR)
-
-	_, initR, err = initM.ProcessPacket(nil, resp)
-	require.NoError(t, err)
-	require.NotNil(t, initR)
-
-	return initR, respR
-}
-
-func TestNewConnectionStateFromResult(t *testing.T) {
-	initR, respR := runTestHandshake(t)
-
-	t.Run("initiator", func(t *testing.T) {
-		ci := newConnectionStateFromResult(initR)
-		assert.True(t, ci.initiator)
-		assert.Equal(t, initR.MyCert, ci.myCert)
-		assert.Equal(t, initR.RemoteCert, ci.peerCert)
-		assert.NotNil(t, ci.eKey)
-		assert.NotNil(t, ci.dKey)
-
-		// IX has 2 handshake messages; the next data-plane send is counter=3.
-		assert.Equal(t, uint64(2), ci.messageCounter.Load(),
-			"messageCounter must equal Result.MessageIndex so the next send is N+1")
-
-		// Both handshake counters must be marked seen so they don't appear lost.
-		// Check returns false if an index has already been recorded.
-		assert.False(t, ci.window.Check(nil, 1), "counter 1 must already be seen")
-		assert.False(t, ci.window.Check(nil, 2), "counter 2 must already be seen")
-		// Counter 3 is the next data-plane message and must NOT be pre-marked.
-		assert.True(t, ci.window.Check(nil, 3), "counter 3 must not be pre-seeded")
-	})
-
-	t.Run("responder", func(t *testing.T) {
-		ci := newConnectionStateFromResult(respR)
-		assert.False(t, ci.initiator)
-		assert.Equal(t, respR.MyCert, ci.myCert)
-		assert.Equal(t, respR.RemoteCert, ci.peerCert)
-		assert.NotNil(t, ci.eKey)
-		assert.NotNil(t, ci.dKey)
-		assert.Equal(t, uint64(2), ci.messageCounter.Load())
-	})
-}

control.go

@@ -3,13 +3,13 @@ package nebula
 import (
 	"context"
 	"errors"
-	"log/slog"
 	"net/netip"
 	"os"
 	"os/signal"
 	"sync"
 	"syscall"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
@@ -46,7 +46,7 @@ type Control struct {
 	state     RunState
 
 	f                      *Interface
-	l                      *slog.Logger
+	l                      *logrus.Logger
 	ctx                    context.Context
 	cancel                 context.CancelFunc
 	sshStart               func()
@@ -151,7 +151,7 @@ func (c *Control) Stop() {
 
 	c.CloseAllTunnels(false)
 	if err := c.f.Close(); err != nil {
-		c.l.Error("Close interface failed", "error", err)
+		c.l.WithError(err).Error("Close interface failed")
 	}
 	c.stateLock.Lock()
 	c.state = StateStopped
@@ -166,7 +166,7 @@ func (c *Control) ShutdownBlock() {
 
 	rawSig := <-sigChan
 	sig := rawSig.String()
-	c.l.Info("Caught signal, shutting down", "signal", sig)
+	c.l.WithField("signal", sig).Info("Caught signal, shutting down")
 	c.Stop()
 }
 
@@ -278,9 +278,15 @@ func (c *Control) CloseTunnel(vpnIp netip.Addr, localOnly bool) bool {
 	}
 
 	if !localOnly {
-		buf := c.f.bufAlloc.Acquire()
-		c.f.send(header.CloseTunnel, 0, hostInfo.ConnectionState, hostInfo, []byte{}, buf)
-		c.f.bufAlloc.Release(buf)
+		c.f.send(
+			header.CloseTunnel,
+			0,
+			hostInfo.ConnectionState,
+			hostInfo,
+			[]byte{},
+			make([]byte, 12, 12),
+			make([]byte, mtu),
+		)
 	}
 
 	c.f.closeTunnel(hostInfo)
@@ -290,20 +296,15 @@ func (c *Control) CloseTunnel(vpnIp netip.Addr, localOnly bool) bool {
 // CloseAllTunnels is just like CloseTunnel except it goes through and shuts them all down, optionally you can avoid shutting down lighthouse tunnels
 // the int returned is a count of tunnels closed
 func (c *Control) CloseAllTunnels(excludeLighthouses bool) (closed int) {
-	// One WireBuffer for the whole shutdown loop.
-	buf := c.f.bufAlloc.Acquire()
-	defer c.f.bufAlloc.Release(buf)
 	shutdown := func(h *HostInfo) {
 		if excludeLighthouses && c.f.lightHouse.IsAnyLighthouseAddr(h.vpnAddrs) {
 			return
 		}
-		c.f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, buf)
+		c.f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
 		c.f.closeTunnel(h)
 
-		c.l.Debug("Sending close tunnel message",
-			"vpnAddrs", h.vpnAddrs,
-			"udpAddr", h.remote,
-		)
+		c.l.WithField("vpnAddrs", h.vpnAddrs).WithField("udpAddr", h.remote).
+			Debug("Sending close tunnel message")
 		closed++
 	}
 

control_test.go

@@ -6,6 +6,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
@@ -82,7 +83,7 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 		f: &Interface{
 			hostMap: hm,
 		},
-		l: test.NewLogger(),
+		l: logrus.New(),
 	}
 
 	thi := c.GetHostInfoByVpnAddr(vpnIp, false)

control_tester.go

@@ -5,6 +5,8 @@ package nebula
 import (
 	"net/netip"
 
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/udp"
@@ -20,9 +22,7 @@ func (c *Control) WaitForType(msgType header.MessageType, subType header.Message
 			panic(err)
 		}
 		pipeTo.InjectUDPPacket(p)
-		match := h.Type == msgType && h.Subtype == subType
-		p.Release()
-		if match {
+		if h.Type == msgType && h.Subtype == subType {
 			return
 		}
 	}
@@ -38,9 +38,7 @@ func (c *Control) WaitForTypeByIndex(toIndex uint32, msgType header.MessageType,
 			panic(err)
 		}
 		pipeTo.InjectUDPPacket(p)
-		match := h.RemoteIndex == toIndex && h.Type == msgType && h.Subtype == subType
-		p.Release()
-		if match {
+		if h.RemoteIndex == toIndex && h.Type == msgType && h.Subtype == subType {
 			return
 		}
 	}
@@ -92,15 +90,65 @@ func (c *Control) GetTunTxChan() <-chan []byte {
 	return c.f.inside.(*overlay.TestTun).TxPackets
 }
 
-// InjectUDPPacket injects a packet into the udp side. We copy internally so the caller keeps ownership of p.
-// The copy comes from the freelist so steady-state alloc is zero.
+// InjectUDPPacket will inject a packet into the udp side of nebula
 func (c *Control) InjectUDPPacket(p *udp.Packet) {
-	c.f.outside.(*udp.TesterConn).Send(p.Copy())
+	c.f.outside.(*udp.TesterConn).Send(p)
 }
 
-// InjectTunPacket pushes an IP packet onto the tun interface.
-func (c *Control) InjectTunPacket(packet []byte) {
-	c.f.inside.(*overlay.TestTun).Send(packet)
+// InjectTunUDPPacket puts a udp packet on the tun interface. Using UDP here because it's a simpler protocol
+func (c *Control) InjectTunUDPPacket(toAddr netip.Addr, toPort uint16, fromAddr netip.Addr, fromPort uint16, data []byte) {
+	serialize := make([]gopacket.SerializableLayer, 0)
+	var netLayer gopacket.NetworkLayer
+	if toAddr.Is6() {
+		if !fromAddr.Is6() {
+			panic("Cant send ipv6 to ipv4")
+		}
+		ip := &layers.IPv6{
+			Version:    6,
+			NextHeader: layers.IPProtocolUDP,
+			SrcIP:      fromAddr.Unmap().AsSlice(),
+			DstIP:      toAddr.Unmap().AsSlice(),
+		}
+		serialize = append(serialize, ip)
+		netLayer = ip
+	} else {
+		if !fromAddr.Is4() {
+			panic("Cant send ipv4 to ipv6")
+		}
+
+		ip := &layers.IPv4{
+			Version:  4,
+			TTL:      64,
+			Protocol: layers.IPProtocolUDP,
+			SrcIP:    fromAddr.Unmap().AsSlice(),
+			DstIP:    toAddr.Unmap().AsSlice(),
+		}
+		serialize = append(serialize, ip)
+		netLayer = ip
+	}
+
+	udp := layers.UDP{
+		SrcPort: layers.UDPPort(fromPort),
+		DstPort: layers.UDPPort(toPort),
+	}
+	err := udp.SetNetworkLayerForChecksum(netLayer)
+	if err != nil {
+		panic(err)
+	}
+
+	buffer := gopacket.NewSerializeBuffer()
+	opt := gopacket.SerializeOptions{
+		ComputeChecksums: true,
+		FixLengths:       true,
+	}
+
+	serialize = append(serialize, &udp, gopacket.Payload(data))
+	err = gopacket.SerializeLayers(buffer, opt, serialize...)
+	if err != nil {
+		panic(err)
+	}
+
+	c.f.inside.(*overlay.TestTun).Send(buffer.Bytes())
 }
 
 func (c *Control) GetVpnAddrs() []netip.Addr {

dns_server.go

@@ -1,249 +1,63 @@
 package nebula
 
 import (
-	"context"
 	"fmt"
-	"log/slog"
 	"net"
 	"net/netip"
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 
 	"github.com/gaissmai/bart"
 	"github.com/miekg/dns"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 )
 
-type dnsServer struct {
+// This whole thing should be rewritten to use context
+
+var dnsR *dnsRecords
+var dnsServer *dns.Server
+var dnsAddr string
+
+type dnsRecords struct {
 	sync.RWMutex
-	l               *slog.Logger
-	ctx             context.Context
+	l               *logrus.Logger
 	dnsMap4         map[string]netip.Addr
 	dnsMap6         map[string]netip.Addr
 	hostMap         *HostMap
 	myVpnAddrsTable *bart.Lite
-
-	mux *dns.ServeMux
-
-	// enabled mirrors `lighthouse.serve_dns && lighthouse.am_lighthouse`.
-	// Start, Add, and reload consult it so callers don't need to know the
-	// gating rules. When it toggles off via reload, accumulated records are
-	// cleared so a later re-enable starts with a fresh map populated from
-	// new handshakes.
-	enabled atomic.Bool
-
-	serverMu sync.Mutex
-	server   *dns.Server
-	// started is closed once `server` has finished binding (or after
-	// ListenAndServe returns on a bind failure). Stop waits on it before
-	// calling Shutdown to avoid the miekg/dns "server not started" race
-	// where a Shutdown that arrives before bind completes is silently
-	// ignored, leaving the listener running forever.
-	started chan struct{}
-	addr    string
 }
 
-// newDnsServerFromConfig builds a dnsServer, applies the initial config, and
-// registers a reload callback. The reload callback is registered before the
-// initial config is applied, so a SIGHUP can later enable, fix, or disable
-// DNS even if the initial application failed.
-//
-// The dnsServer internally gates on `lighthouse.serve_dns &&
-// lighthouse.am_lighthouse`. Start and Add are safe to call unconditionally,
-// they no-op when DNS isn't enabled. Each Start invocation owns a ctx-cancel
-// watcher that tears the listener down on nebula shutdown. The returned
-// pointer is always non-nil, even on error.
-func newDnsServerFromConfig(ctx context.Context, l *slog.Logger, cs *CertState, hostMap *HostMap, c *config.C) (*dnsServer, error) {
-	ds := &dnsServer{
+func newDnsRecords(l *logrus.Logger, cs *CertState, hostMap *HostMap) *dnsRecords {
+	return &dnsRecords{
 		l:               l,
-		ctx:             ctx,
 		dnsMap4:         make(map[string]netip.Addr),
 		dnsMap6:         make(map[string]netip.Addr),
 		hostMap:         hostMap,
 		myVpnAddrsTable: cs.myVpnAddrsTable,
 	}
-	ds.mux = dns.NewServeMux()
-	ds.mux.HandleFunc(".", ds.handleDnsRequest)
-
-	c.RegisterReloadCallback(func(c *config.C) {
-		if err := ds.reload(c, false); err != nil {
-			ds.l.Error("Failed to reload DNS responder from config", "error", err)
-		}
-	})
-
-	if err := ds.reload(c, true); err != nil {
-		return ds, err
-	}
-	return ds, nil
 }
 
-// reload applies the latest config and reconciles the running state with it:
-//   - enabled toggled on  -> spawn a runner
-//   - enabled toggled off -> stop the runner
-//   - listen address changed (while running) -> restart on the new address
-//   - everything else     -> no-op
-//
-// On the initial call it only records configuration; Control.Start is what
-// launches the first runner via dnsStart.
-func (d *dnsServer) reload(c *config.C, initial bool) error {
-	wantsDns := c.GetBool("lighthouse.serve_dns", false)
-	amLighthouse := c.GetBool("lighthouse.am_lighthouse", false)
-	enabled := wantsDns && amLighthouse
-	newAddr := getDnsServerAddr(c)
-
-	d.serverMu.Lock()
-	running := d.server
-	runningStarted := d.started
-	sameAddr := d.addr == newAddr
-	d.addr = newAddr
-	d.enabled.Store(enabled)
-	d.serverMu.Unlock()
-
-	if initial {
-		if wantsDns && !amLighthouse {
-			d.l.Warn("DNS server refusing to run because this host is not a lighthouse.")
-		}
-		return nil
-	}
-
-	if !enabled {
-		if running != nil {
-			d.Stop()
-		}
-		// Drop any records that accumulated while enabled; a later re-enable
-		// will repopulate from fresh handshakes.
-		d.clearRecords()
-		return nil
-	}
-
-	if running == nil {
-		// Was disabled (or never started); bring it up now.
-		go d.Start()
-		return nil
-	}
-
-	if sameAddr {
-		return nil
-	}
-
-	d.shutdownServer(running, runningStarted, "reload")
-	// Old Start goroutine has now exited; bring up a fresh listener on the
-	// new address.
-	go d.Start()
-	return nil
-}
-
-// shutdownServer waits for the server to finish binding (so Shutdown actually
-// stops it rather than no-oping) and then shuts it down.
-func (d *dnsServer) shutdownServer(srv *dns.Server, started chan struct{}, reason string) {
-	if srv == nil {
-		return
-	}
-	if started != nil {
-		<-started
-	}
-	if err := srv.Shutdown(); err != nil {
-		d.l.Warn("Failed to shut down the DNS responder", "reason", reason, "error", err)
-	}
-}
-
-// Start binds and serves the DNS responder. Blocks until Stop is called or
-// the listener errors. Safe to call when DNS is disabled (returns
-// immediately). This is what Control.dnsStart points at.
-//
-// Must be invoked after the tun device is active so that lighthouse.dns.host
-// may bind to a nebula IP.
-func (d *dnsServer) Start() {
-	if !d.enabled.Load() {
-		return
-	}
-
-	started := make(chan struct{})
-	d.serverMu.Lock()
-	if d.ctx.Err() != nil {
-		d.serverMu.Unlock()
-		return
-	}
-	addr := d.addr
-	server := &dns.Server{
-		Addr:              addr,
-		Net:               "udp",
-		Handler:           d.mux,
-		NotifyStartedFunc: func() { close(started) },
-	}
-	d.server = server
-	d.started = started
-	d.serverMu.Unlock()
-
-	// Per-invocation ctx watcher. Exits when Start does, so we don't leak a
-	// watcher per reload-driven restart.
-	done := make(chan struct{})
-	go func() {
-		select {
-		case <-d.ctx.Done():
-			d.shutdownServer(server, started, "shutdown")
-		case <-done:
-		}
-	}()
-
-	d.l.Info("Starting DNS responder", "dnsListener", addr)
-	err := server.ListenAndServe()
-	close(done)
-
-	// If the listener never bound (bind error) NotifyStartedFunc never fires,
-	// so close started here to release any Stop caller waiting on it.
-	select {
-	case <-started:
-	default:
-		close(started)
-	}
-
-	if err != nil {
-		d.l.Warn("Failed to run the DNS responder", "error", err)
-	}
-}
-
-// Stop shuts down the active server, if any. Idempotent.
-func (d *dnsServer) Stop() {
-	d.serverMu.Lock()
-	srv := d.server
-	started := d.started
-	d.server = nil
-	d.started = nil
-	d.serverMu.Unlock()
-	d.shutdownServer(srv, started, "stop")
-}
-
-// Query returns the address for the given name and query type. The second
-// return value reports whether the name is known at all (in either A or AAAA),
-// which lets callers distinguish NODATA from NXDOMAIN.
-func (d *dnsServer) Query(q uint16, data string) (netip.Addr, bool) {
+func (d *dnsRecords) Query(q uint16, data string) netip.Addr {
 	data = strings.ToLower(data)
 	d.RLock()
 	defer d.RUnlock()
-	addr4, haveV4 := d.dnsMap4[data]
-	addr6, haveV6 := d.dnsMap6[data]
-	nameExists := haveV4 || haveV6
 	switch q {
 	case dns.TypeA:
-		if haveV4 {
-			return addr4, nameExists
+		if r, ok := d.dnsMap4[data]; ok {
+			return r
 		}
 	case dns.TypeAAAA:
-		if haveV6 {
-			return addr6, nameExists
+		if r, ok := d.dnsMap6[data]; ok {
+			return r
 		}
 	}
 
-	return netip.Addr{}, nameExists
+	return netip.Addr{}
 }
 
-func (d *dnsServer) QueryCert(data string) string {
-	if len(data) < 2 {
-		return ""
-	}
+func (d *dnsRecords) QueryCert(data string) string {
 	ip, err := netip.ParseAddr(data[:len(data)-1])
 	if err != nil {
 		return ""
@@ -266,19 +80,8 @@ func (d *dnsServer) QueryCert(data string) string {
 	return string(b)
 }
 
-// clearRecords drops all DNS records.
-func (d *dnsServer) clearRecords() {
-	d.Lock()
-	defer d.Unlock()
-	clear(d.dnsMap4)
-	clear(d.dnsMap6)
-}
-
 // Add adds the first IPv4 and IPv6 address that appears in `addresses` as the record for `host`
-func (d *dnsServer) Add(host string, addresses []netip.Addr) {
-	if !d.enabled.Load() {
-		return
-	}
+func (d *dnsRecords) Add(host string, addresses []netip.Addr) {
 	host = strings.ToLower(host)
 	d.Lock()
 	defer d.Unlock()
@@ -298,7 +101,7 @@ func (d *dnsServer) Add(host string, addresses []netip.Addr) {
 	}
 }
 
-func (d *dnsServer) isSelfNebulaOrLocalhost(addr string) bool {
+func (d *dnsRecords) isSelfNebulaOrLocalhost(addr string) bool {
 	a, _, _ := net.SplitHostPort(addr)
 	b, err := netip.ParseAddr(a)
 	if err != nil {
@@ -313,24 +116,13 @@ func (d *dnsServer) isSelfNebulaOrLocalhost(addr string) bool {
 	return d.myVpnAddrsTable.Contains(b)
 }
 
-func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
-	debugEnabled := d.l.Enabled(context.Background(), slog.LevelDebug)
-	// Per RFC 2308 §2.2, a name that exists but has no record of the requested
-	// type must be answered with NOERROR and an empty answer section (NODATA),
-	// not NXDOMAIN (RFC 2308 §2.1), which is reserved for names that do not
-	// exist at all.
-	anyNameExists := false
+func (d *dnsRecords) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 	for _, q := range m.Question {
 		switch q.Qtype {
 		case dns.TypeA, dns.TypeAAAA:
 			qType := dns.TypeToString[q.Qtype]
-			if debugEnabled {
-				d.l.Debug("DNS query", "type", qType, "name", q.Name)
-			}
-			ip, nameExists := d.Query(q.Qtype, q.Name)
-			if nameExists {
-				anyNameExists = true
-			}
+			d.l.Debugf("Query for %s %s", qType, q.Name)
+			ip := d.Query(q.Qtype, q.Name)
 			if ip.IsValid() {
 				rr, err := dns.NewRR(fmt.Sprintf("%s %s %s", q.Name, qType, ip))
 				if err == nil {
@@ -342,9 +134,7 @@ func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 			if !d.isSelfNebulaOrLocalhost(w.RemoteAddr().String()) {
 				return
 			}
-			if debugEnabled {
-				d.l.Debug("DNS query", "type", "TXT", "name", q.Name)
-			}
+			d.l.Debugf("Query for TXT %s", q.Name)
 			ip := d.QueryCert(q.Name)
 			if ip != "" {
 				rr, err := dns.NewRR(fmt.Sprintf("%s TXT %s", q.Name, ip))
@@ -355,12 +145,12 @@ func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 		}
 	}
 
-	if len(m.Answer) == 0 && !anyNameExists {
+	if len(m.Answer) == 0 {
 		m.Rcode = dns.RcodeNameError
 	}
 }
 
-func (d *dnsServer) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
+func (d *dnsRecords) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
 	m := new(dns.Msg)
 	m.SetReply(r)
 	m.Compress = false
@@ -373,6 +163,21 @@ func (d *dnsServer) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
 	w.WriteMsg(m)
 }
 
+func dnsMain(l *logrus.Logger, cs *CertState, hostMap *HostMap, c *config.C) func() {
+	dnsR = newDnsRecords(l, cs, hostMap)
+
+	// attach request handler func
+	dns.HandleFunc(".", dnsR.handleDnsRequest)
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		reloadDns(l, c)
+	})
+
+	return func() {
+		startDns(l, c)
+	}
+}
+
 func getDnsServerAddr(c *config.C) string {
 	dnsHost := strings.TrimSpace(c.GetString("lighthouse.dns.host", ""))
 	// Old guidance was to provide the literal `[::]` in `lighthouse.dns.host` but that won't resolve.
@@ -381,3 +186,25 @@ func getDnsServerAddr(c *config.C) string {
 	}
 	return net.JoinHostPort(dnsHost, strconv.Itoa(c.GetInt("lighthouse.dns.port", 53)))
 }
+
+func startDns(l *logrus.Logger, c *config.C) {
+	dnsAddr = getDnsServerAddr(c)
+	dnsServer = &dns.Server{Addr: dnsAddr, Net: "udp"}
+	l.WithField("dnsListener", dnsAddr).Info("Starting DNS responder")
+	err := dnsServer.ListenAndServe()
+	defer dnsServer.Shutdown()
+	if err != nil {
+		l.Errorf("Failed to start server: %s\n ", err.Error())
+	}
+}
+
+func reloadDns(l *logrus.Logger, c *config.C) {
+	if dnsAddr == getDnsServerAddr(c) {
+		l.Debug("No DNS server config change detected")
+		return
+	}
+
+	l.Debug("Restarting DNS server")
+	dnsServer.Shutdown()
+	go startDns(l, c)
+}

dns_server_test.go

@@ -1,43 +1,19 @@
 package nebula
 
 import (
-	"context"
-	"log/slog"
-	"net"
 	"net/netip"
-	"strconv"
 	"testing"
-	"time"
 
 	"github.com/miekg/dns"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
-type stubDNSWriter struct{}
-
-func (stubDNSWriter) LocalAddr() net.Addr { return &net.UDPAddr{} }
-func (stubDNSWriter) RemoteAddr() net.Addr {
-	return &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 5353}
-}
-func (stubDNSWriter) Write([]byte) (int, error) { return 0, nil }
-func (stubDNSWriter) WriteMsg(*dns.Msg) error   { return nil }
-func (stubDNSWriter) Close() error              { return nil }
-func (stubDNSWriter) TsigStatus() error         { return nil }
-func (stubDNSWriter) TsigTimersOnly(bool)       {}
-func (stubDNSWriter) Hijack()                   {}
-
 func TestParsequery(t *testing.T) {
-	l := slog.New(slog.DiscardHandler)
+	l := logrus.New()
 	hostMap := &HostMap{}
-	ds := &dnsServer{
-		l:       l,
-		dnsMap4: make(map[string]netip.Addr),
-		dnsMap6: make(map[string]netip.Addr),
-		hostMap: hostMap,
-	}
-	ds.enabled.Store(true)
+	ds := newDnsRecords(l, &CertState{}, hostMap)
 	addrs := []netip.Addr{
 		netip.MustParseAddr("1.2.3.4"),
 		netip.MustParseAddr("1.2.3.5"),
@@ -45,56 +21,18 @@ func TestParsequery(t *testing.T) {
 		netip.MustParseAddr("fd01::25"),
 	}
 	ds.Add("test.com.com", addrs)
-	ds.Add("v4only.com.com", []netip.Addr{netip.MustParseAddr("1.2.3.6")})
-	ds.Add("v6only.com.com", []netip.Addr{netip.MustParseAddr("fd01::26")})
 
 	m := &dns.Msg{}
 	m.SetQuestion("test.com.com", dns.TypeA)
 	ds.parseQuery(m, nil)
 	assert.NotNil(t, m.Answer)
 	assert.Equal(t, "1.2.3.4", m.Answer[0].(*dns.A).A.String())
-	assert.Equal(t, dns.RcodeSuccess, m.Rcode)
 
 	m = &dns.Msg{}
 	m.SetQuestion("test.com.com", dns.TypeAAAA)
 	ds.parseQuery(m, nil)
 	assert.NotNil(t, m.Answer)
 	assert.Equal(t, "fd01::24", m.Answer[0].(*dns.AAAA).AAAA.String())
-	assert.Equal(t, dns.RcodeSuccess, m.Rcode)
-
-	// A known name with no record of the requested type should return NODATA
-	// (NOERROR with empty answer), not NXDOMAIN.
-	m = &dns.Msg{}
-	m.SetQuestion("v4only.com.com", dns.TypeAAAA)
-	ds.parseQuery(m, nil)
-	assert.Empty(t, m.Answer)
-	assert.Equal(t, dns.RcodeSuccess, m.Rcode)
-
-	m = &dns.Msg{}
-	m.SetQuestion("v6only.com.com", dns.TypeA)
-	ds.parseQuery(m, nil)
-	assert.Empty(t, m.Answer)
-	assert.Equal(t, dns.RcodeSuccess, m.Rcode)
-
-	// An unknown name should still return NXDOMAIN.
-	m = &dns.Msg{}
-	m.SetQuestion("unknown.com.com", dns.TypeA)
-	ds.parseQuery(m, nil)
-	assert.Empty(t, m.Answer)
-	assert.Equal(t, dns.RcodeNameError, m.Rcode)
-
-	// short lookups should not fail
-	m = &dns.Msg{}
-	m.Question = []dns.Question{{Name: "", Qtype: dns.TypeTXT, Qclass: dns.ClassINET}}
-	ds.parseQuery(m, stubDNSWriter{})
-	assert.Empty(t, m.Answer)
-	assert.Equal(t, dns.RcodeNameError, m.Rcode)
-
-	m = &dns.Msg{}
-	m.Question = []dns.Question{{Name: ".", Qtype: dns.TypeTXT, Qclass: dns.ClassINET}}
-	ds.parseQuery(m, stubDNSWriter{})
-	assert.Empty(t, m.Answer)
-	assert.Equal(t, dns.RcodeNameError, m.Rcode)
 }
 
 func Test_getDnsServerAddr(t *testing.T) {
@@ -133,208 +71,3 @@ func Test_getDnsServerAddr(t *testing.T) {
 	}
 	assert.Equal(t, "[::]:1", getDnsServerAddr(c))
 }
-
-func newTestDnsServer(t *testing.T) (*dnsServer, *config.C) {
-	t.Helper()
-	sl := slog.New(slog.DiscardHandler)
-	ds := &dnsServer{
-		l:       sl,
-		ctx:     context.Background(),
-		dnsMap4: make(map[string]netip.Addr),
-		dnsMap6: make(map[string]netip.Addr),
-		hostMap: &HostMap{},
-	}
-	ds.mux = dns.NewServeMux()
-	ds.mux.HandleFunc(".", ds.handleDnsRequest)
-	return ds, config.NewC(nil)
-}
-
-func setDnsConfig(c *config.C, host string, port string, amLighthouse, serveDns bool) {
-	c.Settings["lighthouse"] = map[string]any{
-		"am_lighthouse": amLighthouse,
-		"serve_dns":     serveDns,
-		"dns": map[string]any{
-			"host": host,
-			"port": port,
-		},
-	}
-}
-
-func TestDnsServer_reload_initial_disabled(t *testing.T) {
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", "0", true, false)
-
-	require.NoError(t, ds.reload(c, true))
-	assert.False(t, ds.enabled.Load())
-	assert.Equal(t, "127.0.0.1:0", ds.addr)
-	assert.Nil(t, ds.server)
-}
-
-func TestDnsServer_reload_initial_enabled(t *testing.T) {
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", "0", true, true)
-
-	require.NoError(t, ds.reload(c, true))
-	assert.True(t, ds.enabled.Load())
-	assert.Equal(t, "127.0.0.1:0", ds.addr)
-	// initial never starts a runner; that's Control.Start's job
-	assert.Nil(t, ds.server)
-}
-
-func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) {
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", "0", false, true)
-
-	require.NoError(t, ds.reload(c, true))
-	// Wants DNS but isn't a lighthouse: gated off, no runner.
-	assert.False(t, ds.enabled.Load())
-}
-
-func TestDnsServer_reload_sameAddr_noOp(t *testing.T) {
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", "0", true, true)
-
-	require.NoError(t, ds.reload(c, true))
-	// No server running yet, no addr change. Reload should not spawn anything.
-	require.NoError(t, ds.reload(c, false))
-	assert.True(t, ds.enabled.Load())
-	assert.Nil(t, ds.server)
-}
-
-func TestDnsServer_StartStop_lifecycle(t *testing.T) {
-	// Bind to a real (random) UDP port so we exercise the actual
-	// ListenAndServe + Shutdown plumbing including the started-chan race fix.
-	port := freeUDPPort(t)
-
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", port, true, true)
-	require.NoError(t, ds.reload(c, true))
-
-	done := make(chan struct{})
-	go func() {
-		ds.Start()
-		close(done)
-	}()
-
-	waitFor(t, func() bool {
-		ds.serverMu.Lock()
-		started := ds.started
-		ds.serverMu.Unlock()
-		if started == nil {
-			return false
-		}
-		select {
-		case <-started:
-			return true
-		default:
-			return false
-		}
-	})
-
-	ds.Stop()
-	select {
-	case <-done:
-	case <-time.After(5 * time.Second):
-		t.Fatal("Start did not return after Stop")
-	}
-}
-
-func TestDnsServer_Stop_beforeBind_doesNotHang(t *testing.T) {
-	// Stop called immediately after Start should not deadlock even if bind
-	// hasn't completed yet. This exercises the started-chan close-on-bind-fail
-	// path: by binding to an obviously bad port (privileged) we get a fast
-	// bind error before NotifyStartedFunc fires.
-	ds, c := newTestDnsServer(t)
-	// Use a port that should fail to bind (negative would be invalid, use a
-	// host that won't resolve to ensure listenUDP fails quickly).
-	setDnsConfig(c, "256.256.256.256", "53", true, true)
-	require.NoError(t, ds.reload(c, true))
-
-	done := make(chan struct{})
-	go func() {
-		ds.Start()
-		close(done)
-	}()
-
-	// Give Start a moment to attempt the bind and fail.
-	select {
-	case <-done:
-		// Bind failed and Start returned; Stop should be a no-op.
-	case <-time.After(time.Second):
-		t.Fatal("Start did not return after a bad bind")
-	}
-
-	stopped := make(chan struct{})
-	go func() {
-		ds.Stop()
-		close(stopped)
-	}()
-	select {
-	case <-stopped:
-	case <-time.After(time.Second):
-		t.Fatal("Stop hung after a failed bind")
-	}
-}
-
-func TestDnsServer_reload_disable_stopsRunningServer(t *testing.T) {
-	port := freeUDPPort(t)
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", port, true, true)
-	require.NoError(t, ds.reload(c, true))
-
-	startReturned := make(chan struct{})
-	go func() {
-		ds.Start()
-		close(startReturned)
-	}()
-	waitForBind(t, ds)
-
-	// Toggle serve_dns off; reload should shut the running server down.
-	setDnsConfig(c, "127.0.0.1", port, true, false)
-	require.NoError(t, ds.reload(c, false))
-	select {
-	case <-startReturned:
-	case <-time.After(5 * time.Second):
-		t.Fatal("Start did not return after reload disabled DNS")
-	}
-	assert.False(t, ds.enabled.Load())
-}
-
-func freeUDPPort(t *testing.T) string {
-	t.Helper()
-	conn, err := net.ListenPacket("udp", "127.0.0.1:0")
-	require.NoError(t, err)
-	port := conn.LocalAddr().(*net.UDPAddr).Port
-	require.NoError(t, conn.Close())
-	return strconv.Itoa(port)
-}
-
-func waitForBind(t *testing.T, ds *dnsServer) {
-	t.Helper()
-	waitFor(t, func() bool {
-		ds.serverMu.Lock()
-		started := ds.started
-		ds.serverMu.Unlock()
-		if started == nil {
-			return false
-		}
-		select {
-		case <-started:
-			return true
-		default:
-			return false
-		}
-	})
-}
-
-func waitFor(t *testing.T, cond func() bool) {
-	t.Helper()
-	deadline := time.Now().Add(5 * time.Second)
-	for time.Now().Before(deadline) {
-		if cond() {
-			return
-		}
-		time.Sleep(5 * time.Millisecond)
-	}
-	t.Fatal("timed out waiting for condition")
-}

e2e/bench_test.go

@@ -1,68 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/e2e/router"
-)
-
-// BenchmarkHandshake measures end-to-end tunnel establishment time. The two
-// nodes and the router are constructed once before the loop so the timed window
-// is just the handshake itself: trigger packet -> handshake1 -> handshake2 ->
-// cached packet replay -> arrival on the remote TUN. Between iterations we
-// tear down both sides locally (no CloseTunnel notification on the wire) and
-// re-inject the lighthouse address that closeTunnel cleared, so the next
-// iteration runs through a fresh handshake against the same harness.
-func BenchmarkHandshake(b *testing.B) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	// Default try_interval is 100ms. The handshake manager schedules handshake1
-	// on its OutboundHandshakeTimer rather than firing immediately on trigger
-	// (the trigger channel only fast-paths static hosts), so a 100ms default
-	// drowns the actual handshake cost. Drop it to 1ms so the bench reflects
-	// the computation, not the wheel cadence.
-	bovr := m{"handshakes": m{"try_interval": "1ms"}}
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", bovr)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.2/24", bovr)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-	defer myControl.Stop()
-	defer theirControl.Stop()
-
-	r := router.NewR(b, myControl, theirControl)
-	r.CancelFlowLogs()
-	r.EnableFanIn()
-
-	trigger := BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-
-	b.ReportAllocs()
-	b.ResetTimer()
-
-	for n := 0; n < b.N; n++ {
-		myControl.InjectTunPacket(trigger)
-		// RouteForAllUntilTxTun returns the moment the cached packet arrives at
-		// the remote TUN, which is also when both sides are fully established.
-		_ = r.RouteForAllUntilTxTun(theirControl)
-
-		b.StopTimer()
-		// Local-only close removes hostmap state on both sides without putting a
-		// CloseTunnel packet on the wire that we'd then have to drain. The
-		// closeTunnel path also clears learned lighthouse state for the peer
-		// when the last hostinfo for that addr goes away, so we re-inject.
-		myControl.CloseTunnel(theirVpnIpNet[0].Addr(), true)
-		theirControl.CloseTunnel(myVpnIpNet[0].Addr(), true)
-		myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-		theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-		b.StartTimer()
-	}
-}

e2e/handshake_manager_test.go

@@ -28,7 +28,6 @@ func makeHandshakePacket(from, to netip.AddrPort, subtype header.MessageSubType,
 }
 
 func TestHandshakeRetransmitDuplicate(t *testing.T) {
-	t.Parallel()
 	// Verify the responder correctly handles receiving the same msg1 multiple times
 	// (retransmission). The duplicate goes through CheckAndComplete -> ErrAlreadySeen
 	// and the cached response is resent.
@@ -47,7 +46,7 @@ func TestHandshakeRetransmitDuplicate(t *testing.T) {
 	defer r.RenderFlow()
 
 	t.Log("Trigger handshake from me to them")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 
 	t.Log("Grab my msg1")
 	msg1 := myControl.GetFromUDP(true)
@@ -79,7 +78,6 @@ func TestHandshakeRetransmitDuplicate(t *testing.T) {
 }
 
 func TestHandshakeTruncatedPacketRecovery(t *testing.T) {
-	t.Parallel()
 	// Verify that a truncated handshake packet is ignored and the real
 	// packet can still complete the handshake.
 
@@ -97,7 +95,7 @@ func TestHandshakeTruncatedPacketRecovery(t *testing.T) {
 	defer r.RenderFlow()
 
 	t.Log("Trigger handshake")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 
 	t.Log("Get msg1 and deliver to responder")
 	msg1 := myControl.GetFromUDP(true)
@@ -128,7 +126,6 @@ func TestHandshakeTruncatedPacketRecovery(t *testing.T) {
 }
 
 func TestHandshakeOrphanedMsg2Dropped(t *testing.T) {
-	t.Parallel()
 	// A msg2 arriving with no matching pending index should be silently dropped
 	// with no response sent and no state changes.
 
@@ -146,7 +143,7 @@ func TestHandshakeOrphanedMsg2Dropped(t *testing.T) {
 	defer r.RenderFlow()
 
 	t.Log("Complete a normal handshake")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 	r.RouteForAllUntilTxTun(theirControl)
 	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
 
@@ -171,7 +168,6 @@ func TestHandshakeOrphanedMsg2Dropped(t *testing.T) {
 }
 
 func TestHandshakeUnknownMessageCounter(t *testing.T) {
-	t.Parallel()
 	// A handshake packet with an unexpected message counter should be silently
 	// dropped with no side effects and no UDP response.
 
@@ -203,7 +199,6 @@ func TestHandshakeUnknownMessageCounter(t *testing.T) {
 }
 
 func TestHandshakeUnknownSubtype(t *testing.T) {
-	t.Parallel()
 	// A handshake packet with an unknown subtype should be silently dropped.
 
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
@@ -229,7 +224,6 @@ func TestHandshakeUnknownSubtype(t *testing.T) {
 }
 
 func TestHandshakeLateResponse(t *testing.T) {
-	t.Parallel()
 	// After a handshake times out, a late response should be silently ignored
 	// with no new tunnels created.
 
@@ -248,7 +242,7 @@ func TestHandshakeLateResponse(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger handshake from me")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 
 	t.Log("Grab msg1 but don't deliver")
 	msg1 := myControl.GetFromUDP(true)
@@ -279,7 +273,6 @@ func TestHandshakeLateResponse(t *testing.T) {
 }
 
 func TestHandshakeSelfConnectionRejected(t *testing.T) {
-	t.Parallel()
 	// Verify that a node rejects a handshake containing its own VPN IP in the
 	// peer cert. We do this by sending the initiator's own msg1 back to itself.
 
@@ -292,7 +285,7 @@ func TestHandshakeSelfConnectionRejected(t *testing.T) {
 	myControl.Start()
 
 	t.Log("Trigger handshake from me")
-	myControl.InjectTunPacket(BuildTunUDPPacket(netip.MustParseAddr("10.128.0.2"), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(netip.MustParseAddr("10.128.0.2"), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 	msg1 := myControl.GetFromUDP(true)
 
 	t.Log("Drain any handshake retransmits before injecting")
@@ -328,7 +321,6 @@ func TestHandshakeSelfConnectionRejected(t *testing.T) {
 }
 
 func TestHandshakeMessageCounter0Dropped(t *testing.T) {
-	t.Parallel()
 	// MessageCounter=0 is not a valid handshake message and should be dropped.
 
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
@@ -349,7 +341,6 @@ func TestHandshakeMessageCounter0Dropped(t *testing.T) {
 }
 
 func TestHandshakeRemoteAllowList(t *testing.T) {
-	t.Parallel()
 	// Verify that a handshake from a blocked underlay IP is dropped with no
 	// response and no state changes. Then verify the same packet from an
 	// allowed IP succeeds.
@@ -375,7 +366,7 @@ func TestHandshakeRemoteAllowList(t *testing.T) {
 	defer r.RenderFlow()
 
 	t.Log("Trigger handshake from them")
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi"))
 	msg1 := theirControl.GetFromUDP(true)
 
 	t.Log("Rewrite the source to a blocked IP and inject")
@@ -408,7 +399,6 @@ func TestHandshakeRemoteAllowList(t *testing.T) {
 }
 
 func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) {
-	t.Parallel()
 	// When a duplicate msg1 arrives via ErrAlreadySeen, verify the tunnel
 	// remains functional and hostmap index count is stable.
 
@@ -426,7 +416,7 @@ func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) {
 	defer r.RenderFlow()
 
 	t.Log("Complete a normal handshake via the router")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 	r.RouteForAllUntilTxTun(theirControl)
 	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
 
@@ -437,7 +427,7 @@ func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) {
 	originalRemote := hi.CurrentRemote
 
 	t.Log("Re-trigger traffic to cause a new handshake attempt (ErrAlreadySeen)")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("roam")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("roam"))
 	r.RouteForAllUntilTxTun(theirControl)
 
 	t.Log("Verify tunnel still works")
@@ -455,7 +445,6 @@ func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) {
 }
 
 func TestHandshakeWrongResponderPacketStore(t *testing.T) {
-	t.Parallel()
 	// Verify that when the wrong host responds, the cached packets are
 	// transferred to the new handshake, the evil tunnel is closed, evil's
 	// address is blocked, and the correct tunnel is eventually established.
@@ -475,8 +464,8 @@ func TestHandshakeWrongResponderPacketStore(t *testing.T) {
 	evilControl.Start()
 
 	t.Log("Send multiple packets to them (cached during handshake)")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet1")))
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet2")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet1"))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet2"))
 
 	t.Log("Route until evil tunnel is closed")
 	h := &header.H{}
@@ -519,7 +508,6 @@ func TestHandshakeWrongResponderPacketStore(t *testing.T) {
 }
 
 func TestHandshakeRelayComplete(t *testing.T) {
-	t.Parallel()
 	// Verify that a relay handshake completes correctly and relay state is
 	// properly maintained on all three nodes.
 
@@ -540,7 +528,7 @@ func TestHandshakeRelayComplete(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger handshake via relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi via relay")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi via relay"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	assertUdpPacket(t, []byte("Hi via relay"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
@@ -568,7 +556,7 @@ func TestHandshakeRelayComplete(t *testing.T) {
 }
 
 // NOTE: Relay V1 cert + IPv6 rejection is not tested here because
-// BuildTunUDPPacket from a V4 node to a V6 address panics in the test
+// InjectTunUDPPacket from a V4 node to a V6 address panics in the test
 // framework. The check is in handshake_manager.go handleOutbound relay
 // logic (lines ~304-313): if the relay host has a V1 cert and either
 // address is IPv6, the relay is skipped.

e2e/handshakes_test.go

@@ -11,12 +11,12 @@ import (
 
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/e2e/router"
 	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -40,22 +40,11 @@ func BenchmarkHotPath(b *testing.B) {
 	r.CancelFlowLogs()
 
 	assertTunnel(b, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	// Pre-build the IP packet bytes once so the bench measures the data plane,
-	// not gopacket SerializeLayers overhead.
-	prebuilt := BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-
-	// EnableFanIn switches the router to a 0-alloc routing path. Required
-	// for hot-path benchmarks; would conflict with GetFromUDP-using tests.
-	r.EnableFanIn()
-
 	b.ResetTimer()
 
 	for n := 0; n < b.N; n++ {
-		myControl.InjectTunPacket(prebuilt)
-		// Release the TUN-side bytes back to the harness freelist; the bench
-		// just confirms a packet arrived, the contents aren't inspected.
-		overlay.ReleaseTunBuf(r.RouteForAllUntilTxTun(theirControl))
+		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
+		_ = r.RouteForAllUntilTxTun(theirControl)
 	}
 
 	myControl.Stop()
@@ -83,15 +72,11 @@ func BenchmarkHotPathRelay(b *testing.B) {
 	theirControl.Start()
 
 	assertTunnel(b, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
-
-	prebuilt := BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-	r.EnableFanIn()
-
 	b.ResetTimer()
 
 	for n := 0; n < b.N; n++ {
-		myControl.InjectTunPacket(prebuilt)
-		overlay.ReleaseTunBuf(r.RouteForAllUntilTxTun(theirControl))
+		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
+		_ = r.RouteForAllUntilTxTun(theirControl)
 	}
 
 	myControl.Stop()
@@ -100,7 +85,6 @@ func BenchmarkHotPathRelay(b *testing.B) {
 }
 
 func TestGoodHandshake(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
 	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
@@ -113,7 +97,7 @@ func TestGoodHandshake(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Send a udp packet through to begin standing up the tunnel, this should come out the other side")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
 	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
@@ -151,7 +135,6 @@ func TestGoodHandshake(t *testing.T) {
 }
 
 func TestGoodHandshakeNoOverlap(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", nil)
 	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "2001::69/24", nil) //look ma, cross-stack!
@@ -165,7 +148,7 @@ func TestGoodHandshakeNoOverlap(t *testing.T) {
 
 	empty := []byte{}
 	t.Log("do something to cause a handshake")
-	myControl.GetF().SendMessageToVpnAddr(header.Test, header.MessageNone, theirVpnIpNet[0].Addr(), empty, nebula.NewWireBuffer(9001, 0))
+	myControl.GetF().SendMessageToVpnAddr(header.Test, header.MessageNone, theirVpnIpNet[0].Addr(), empty, empty, empty)
 
 	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
 	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
@@ -187,7 +170,6 @@ func TestGoodHandshakeNoOverlap(t *testing.T) {
 }
 
 func TestWrongResponderHandshake(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.100/24", nil)
@@ -207,7 +189,7 @@ func TestWrongResponderHandshake(t *testing.T) {
 	evilControl.Start()
 
 	t.Log("Start the handshake process, we will route until we see the evil tunnel closed")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	h := &header.H{}
 	r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
@@ -264,7 +246,6 @@ func TestWrongResponderHandshake(t *testing.T) {
 }
 
 func TestWrongResponderHandshakeStaticHostMap(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 
 	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.99/24", nil)
@@ -289,7 +270,7 @@ func TestWrongResponderHandshakeStaticHostMap(t *testing.T) {
 	evilControl.Start()
 
 	t.Log("Start the handshake process, we will route until we see the evil tunnel closed")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	h := &header.H{}
 	r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
@@ -347,7 +328,6 @@ func TestWrongResponderHandshakeStaticHostMap(t *testing.T) {
 }
 
 func TestStage1Race(t *testing.T) {
-	t.Parallel()
 	// This tests ensures that two hosts handshaking with each other at the same time will allow traffic to flow
 	// But will eventually collapse down to a single tunnel
 
@@ -368,8 +348,8 @@ func TestStage1Race(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake to start on both me and them")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 	t.Log("Get both stage 1 handshake packets")
 	myHsForThem := myControl.GetFromUDP(true)
@@ -428,7 +408,6 @@ func TestStage1Race(t *testing.T) {
 }
 
 func TestUncleanShutdownRaceLoser(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me  ", "10.128.0.1/24", nil)
 	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
@@ -446,7 +425,7 @@ func TestUncleanShutdownRaceLoser(t *testing.T) {
 	theirControl.Start()
 
 	r.Log("Trigger a handshake from me to them")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
@@ -457,7 +436,7 @@ func TestUncleanShutdownRaceLoser(t *testing.T) {
 	myHostmap.Indexes = map[uint32]*nebula.HostInfo{}
 	myHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{}
 
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me again")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me again"))
 	p = r.RouteForAllUntilTxTun(theirControl)
 	assertUdpPacket(t, []byte("Hi from me again"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
 
@@ -478,7 +457,6 @@ func TestUncleanShutdownRaceLoser(t *testing.T) {
 }
 
 func TestUncleanShutdownRaceWinner(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me  ", "10.128.0.1/24", nil)
 	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
@@ -496,7 +474,7 @@ func TestUncleanShutdownRaceWinner(t *testing.T) {
 	theirControl.Start()
 
 	r.Log("Trigger a handshake from me to them")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
@@ -508,7 +486,7 @@ func TestUncleanShutdownRaceWinner(t *testing.T) {
 	theirHostmap.Indexes = map[uint32]*nebula.HostInfo{}
 	theirHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{}
 
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them again")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them again"))
 	p = r.RouteForAllUntilTxTun(myControl)
 	assertUdpPacket(t, []byte("Hi from them again"), p, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
 	r.RenderHostmaps("Derp hostmaps", myControl, theirControl)
@@ -530,7 +508,6 @@ func TestUncleanShutdownRaceWinner(t *testing.T) {
 }
 
 func TestRelays(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
 	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay  ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
@@ -551,7 +528,7 @@ func TestRelays(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
@@ -560,7 +537,6 @@ func TestRelays(t *testing.T) {
 }
 
 func TestRelaysDontCareAboutIps(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
 	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay  ", "2001::9999/24", m{"relay": m{"am_relay": true}})
@@ -581,7 +557,7 @@ func TestRelaysDontCareAboutIps(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
@@ -590,7 +566,6 @@ func TestRelaysDontCareAboutIps(t *testing.T) {
 }
 
 func TestReestablishRelays(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
 	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay  ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
@@ -611,14 +586,14 @@ func TestReestablishRelays(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
 	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
 
 	t.Log("Ensure packet traversal from them to me via the relay")
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 	p = r.RouteForAllUntilTxTun(myControl)
 	r.Log("Assert the tunnel works")
@@ -633,7 +608,7 @@ func TestReestablishRelays(t *testing.T) {
 	for curIndexes >= start {
 		curIndexes = len(myControl.GetHostmap().Indexes)
 		r.Logf("Wait for the dead index to go away:start=%v indexes, current=%v indexes", start, curIndexes)
-		myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me should fail")))
+		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me should fail"))
 
 		r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
 			return router.RouteAndExit
@@ -650,7 +625,7 @@ func TestReestablishRelays(t *testing.T) {
 		myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
 		myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
 		relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-		myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 		p = r.RouteForAllUntilTxTun(theirControl)
 		r.Log("Assert the tunnel works")
@@ -685,7 +660,7 @@ func TestReestablishRelays(t *testing.T) {
 	t.Log("Assert the tunnel works the other way, too")
 	for {
 		t.Log("RouteForAllUntilTxTun")
-		theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+		theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 		p = r.RouteForAllUntilTxTun(myControl)
 		r.Log("Assert the tunnel works")
@@ -722,7 +697,6 @@ func TestReestablishRelays(t *testing.T) {
 }
 
 func TestStage1RaceRelays(t *testing.T) {
-	t.Parallel()
 	//NOTE: this is a race between me and relay resulting in a full tunnel from me to them via relay
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
@@ -755,8 +729,8 @@ func TestStage1RaceRelays(t *testing.T) {
 	assertTunnel(t, theirVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), theirControl, relayControl, r)
 
 	r.Log("Trigger a handshake from both them and me via relay to them and me")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 	r.Log("Wait for a packet from them to me")
 	p := r.RouteForAllUntilTxTun(myControl)
@@ -770,12 +744,12 @@ func TestStage1RaceRelays(t *testing.T) {
 }
 
 func TestStage1RaceRelays2(t *testing.T) {
-	t.Parallel()
 	//NOTE: this is a race between me and relay resulting in a full tunnel from me to them via relay
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
 	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay  ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
 	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them   ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
+	l := NewTestLogger()
 
 	// Teach my how to get to the relay and that their can be reached via the relay
 	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
@@ -797,41 +771,49 @@ func TestStage1RaceRelays2(t *testing.T) {
 	theirControl.Start()
 
 	r.Log("Get a tunnel between me and relay")
+	l.Info("Get a tunnel between me and relay")
 	assertTunnel(t, myVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), myControl, relayControl, r)
 
 	r.Log("Get a tunnel between them and relay")
+	l.Info("Get a tunnel between them and relay")
 	assertTunnel(t, theirVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), theirControl, relayControl, r)
 
 	r.Log("Trigger a handshake from both them and me via relay to them and me")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	l.Info("Trigger a handshake from both them and me via relay to them and me")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 	//r.RouteUntilAfterMsgType(myControl, header.Control, header.MessageNone)
 	//r.RouteUntilAfterMsgType(theirControl, header.Control, header.MessageNone)
 
-	r.Log("Wait for a packet from them to me; myControl")
+	r.Log("Wait for a packet from them to me")
+	l.Info("Wait for a packet from them to me; myControl")
 	r.RouteForAllUntilTxTun(myControl)
-	r.Log("Wait for a packet from them to me; theirControl")
+	l.Info("Wait for a packet from them to me; theirControl")
 	r.RouteForAllUntilTxTun(theirControl)
 
 	r.Log("Assert the tunnel works")
+	l.Info("Assert the tunnel works")
 	assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
 
 	t.Log("Wait until we remove extra tunnels")
-	t.Logf("Waiting for hostinfos to be removed... myControl=%d theirControl=%d relayControl=%d",
-		len(myControl.GetHostmap().Indexes),
-		len(theirControl.GetHostmap().Indexes),
-		len(relayControl.GetHostmap().Indexes),
-	)
+	l.Info("Wait until we remove extra tunnels")
+	l.WithFields(
+		logrus.Fields{
+			"myControl":    len(myControl.GetHostmap().Indexes),
+			"theirControl": len(theirControl.GetHostmap().Indexes),
+			"relayControl": len(relayControl.GetHostmap().Indexes),
+		}).Info("Waiting for hostinfos to be removed...")
 	hostInfos := len(myControl.GetHostmap().Indexes) + len(theirControl.GetHostmap().Indexes) + len(relayControl.GetHostmap().Indexes)
 	retries := 60
 	for hostInfos > 6 && retries > 0 {
 		hostInfos = len(myControl.GetHostmap().Indexes) + len(theirControl.GetHostmap().Indexes) + len(relayControl.GetHostmap().Indexes)
-		t.Logf("Waiting for hostinfos to be removed... myControl=%d theirControl=%d relayControl=%d",
-			len(myControl.GetHostmap().Indexes),
-			len(theirControl.GetHostmap().Indexes),
-			len(relayControl.GetHostmap().Indexes),
-		)
+		l.WithFields(
+			logrus.Fields{
+				"myControl":    len(myControl.GetHostmap().Indexes),
+				"theirControl": len(theirControl.GetHostmap().Indexes),
+				"relayControl": len(relayControl.GetHostmap().Indexes),
+			}).Info("Waiting for hostinfos to be removed...")
 		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
 		t.Log("Connection manager hasn't ticked yet")
 		time.Sleep(time.Second)
@@ -839,6 +821,7 @@ func TestStage1RaceRelays2(t *testing.T) {
 	}
 
 	r.Log("Assert the tunnel works")
+	l.Info("Assert the tunnel works")
 	assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
 
 	myControl.Stop()
@@ -847,7 +830,6 @@ func TestStage1RaceRelays2(t *testing.T) {
 }
 
 func TestRehandshakingRelays(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
 	relayControl, relayVpnIpNet, relayUdpAddr, relayConfig := newSimpleServer(cert.Version1, ca, caKey, "relay  ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
@@ -868,7 +850,7 @@ func TestRehandshakingRelays(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
@@ -951,7 +933,6 @@ func TestRehandshakingRelays(t *testing.T) {
 }
 
 func TestRehandshakingRelaysPrimary(t *testing.T) {
-	t.Parallel()
 	// This test is the same as TestRehandshakingRelays but one of the terminal types is a primary swap winner
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.128/24", m{"relay": m{"use_relays": true}})
@@ -973,7 +954,7 @@ func TestRehandshakingRelaysPrimary(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
@@ -1056,7 +1037,6 @@ func TestRehandshakingRelaysPrimary(t *testing.T) {
 }
 
 func TestRehandshaking(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, myConfig := newSimpleServer(cert.Version1, ca, caKey, "me  ", "10.128.0.2/24", nil)
 	theirControl, theirVpnIpNet, theirUdpAddr, theirConfig := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.1/24", nil)
@@ -1152,7 +1132,6 @@ func TestRehandshaking(t *testing.T) {
 }
 
 func TestRehandshakingLoser(t *testing.T) {
-	t.Parallel()
 	// The purpose of this test is that the race loser renews their certificate and rehandshakes. The final tunnel
 	// Should be the one with the new certificate
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
@@ -1251,7 +1230,6 @@ func TestRehandshakingLoser(t *testing.T) {
 }
 
 func TestRaceRegression(t *testing.T) {
-	t.Parallel()
 	// This test forces stage 1, stage 2, stage 1 to be received by me from them
 	// We had a bug where we were not finding the duplicate handshake and responding to the final stage 1 which
 	// caused a cross-linked hostinfo
@@ -1275,8 +1253,8 @@ func TestRaceRegression(t *testing.T) {
 	//them rx stage:2 initiatorIndex=120607833 responderIndex=4209862089
 
 	t.Log("Start both handshakes")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 	t.Log("Get both stage 1")
 	myStage1ForThem := myControl.GetFromUDP(true)
@@ -1312,7 +1290,6 @@ func TestRaceRegression(t *testing.T) {
 }
 
 func TestV2NonPrimaryWithLighthouse(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh  ", "10.128.0.1/24, ff::1/64", m{"lighthouse": m{"am_lighthouse": true}})
 
@@ -1353,7 +1330,6 @@ func TestV2NonPrimaryWithLighthouse(t *testing.T) {
 }
 
 func TestV2NonPrimaryWithOffNetLighthouse(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh  ", "2001::1/64", m{"lighthouse": m{"am_lighthouse": true}})
 
@@ -1393,84 +1369,7 @@ func TestV2NonPrimaryWithOffNetLighthouse(t *testing.T) {
 	theirControl.Stop()
 }
 
-func TestLighthouseUpdateOnReload(t *testing.T) {
-	t.Parallel()
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	// Create the lighthouse
-	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{"lighthouse": m{"am_lighthouse": true}})
-
-	// Create a client with NO lighthouse configured and a long update interval.
-	// The initial SendUpdate at startup will be a no-op since no lighthouses are known.
-	myControl, myVpnIpNet, _, myConfig := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
-		"lighthouse": m{
-			"interval": 600,
-			"local_allow_list": m{
-				"10.0.0.0/24": true,
-				"::/0":        false,
-			},
-		},
-	})
-
-	r := router.NewR(t, lhControl, myControl)
-	defer r.RenderFlow()
-
-	lhControl.Start()
-	myControl.Start()
-
-	// Drain any startup packets (there should be none meaningful)
-	r.FlushAll()
-
-	// Verify lighthouse has no knowledge of the client
-	assert.Nil(t, lhControl.QueryLighthouse(myVpnIpNet[0].Addr()))
-
-	// Build a new config that adds the lighthouse
-	newSettings := make(m)
-	for k, v := range myConfig.Settings {
-		newSettings[k] = v
-	}
-	newSettings["static_host_map"] = m{
-		lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
-	}
-	newSettings["lighthouse"] = m{
-		"hosts":    []any{lhVpnIpNet[0].Addr().String()},
-		"interval": 600,
-		"local_allow_list": m{
-			"10.0.0.0/24": true,
-			"::/0":        false,
-		},
-	}
-	newCfg, err := yaml.Marshal(newSettings)
-	require.NoError(t, err)
-
-	// Reload the config. The lighthouse.hosts change triggers TriggerUpdate,
-	// which wakes the update worker. It calls SendUpdate, initiating a
-	// handshake to the new lighthouse and caching the HostUpdateNotification.
-	require.NoError(t, myConfig.ReloadConfigString(string(newCfg)))
-
-	// Route until the lighthouse receives the HostUpdateNotification.
-	// This covers: handshake stage 1, stage 2, then the cached update.
-	done := make(chan struct{})
-	go func() {
-		r.RouteForAllUntilAfterMsgTypeTo(lhControl, header.LightHouse, 0)
-		close(done)
-	}()
-	select {
-	case <-done:
-	case <-time.After(5 * time.Second):
-		t.Fatal("timed out waiting for lighthouse update after config reload")
-	}
-
-	// Verify lighthouse now has the client's addresses
-	assert.NotNil(t, lhControl.QueryLighthouse(myVpnIpNet[0].Addr()))
-
-	r.RenderHostmaps("Final hostmaps", lhControl, myControl)
-	lhControl.Stop()
-	myControl.Stop()
-}
-
 func TestGoodHandshakeUnsafeDest(t *testing.T) {
-	t.Parallel()
 	unsafePrefix := "192.168.6.0/24"
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdpAndUnsafeNetworks(cert.Version2, ca, caKey, "spooky", "10.128.0.2/24", netip.MustParseAddrPort("10.64.0.2:4242"), unsafePrefix, nil)
@@ -1492,7 +1391,7 @@ func TestGoodHandshakeUnsafeDest(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Send a udp packet through to begin standing up the tunnel, this should come out the other side")
-	myControl.InjectTunPacket(BuildTunUDPPacket(spookyDest, 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(spookyDest, 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
 	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
@@ -1520,7 +1419,7 @@ func TestGoodHandshakeUnsafeDest(t *testing.T) {
 	assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet[0].Addr(), spookyDest, 80, 80)
 
 	//reply
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, spookyDest, 80, []byte("Hi from the spookyman")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, spookyDest, 80, []byte("Hi from the spookyman"))
 	//wait for reply
 	theirControl.WaitForType(1, 0, myControl)
 	theirCachedPacket := myControl.GetFromTun(true)

e2e/helpers_test.go

@@ -4,6 +4,7 @@
 package e2e
 
 import (
+	"fmt"
 	"io"
 	"net/netip"
 	"os"
@@ -11,18 +12,15 @@ import (
 	"testing"
 	"time"
 
-	"log/slog"
-
 	"dario.cat/mergo"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
-
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/e2e/router"
-	"github.com/slackhq/nebula/logging"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.yaml.in/yaml/v3"
@@ -134,7 +132,8 @@ func newSimpleServerWithUdpAndUnsafeNetworks(v cert.Version, caCrt cert.Certific
 			"port": udpAddr.Port(),
 		},
 		"logging": m{
-			"level": testLogLevelName(),
+			"timestamp_format": fmt.Sprintf("%v 15:04:05.000000", name),
+			"level":            l.Level.String(),
 		},
 		"timers": m{
 			"pending_deletion_interval": 2,
@@ -235,7 +234,8 @@ func newServer(caCrt []cert.Certificate, certs []cert.Certificate, key []byte, o
 			"port": udpAddr.Port(),
 		},
 		"logging": m{
-			"level": testLogLevelName(),
+			"timestamp_format": fmt.Sprintf("%v 15:04:05.000000", certs[0].Name()),
+			"level":            l.Level.String(),
 		},
 		"timers": m{
 			"pending_deletion_interval": 2,
@@ -294,12 +294,12 @@ func deadline(t *testing.T, seconds time.Duration) doneCb {
 
 func assertTunnel(t testing.TB, vpnIpA, vpnIpB netip.Addr, controlA, controlB *nebula.Control, r *router.R) {
 	// Send a packet from them to me
-	controlB.InjectTunPacket(BuildTunUDPPacket(vpnIpA, 80, vpnIpB, 90, []byte("Hi from B")))
+	controlB.InjectTunUDPPacket(vpnIpA, 80, vpnIpB, 90, []byte("Hi from B"))
 	bPacket := r.RouteForAllUntilTxTun(controlA)
 	assertUdpPacket(t, []byte("Hi from B"), bPacket, vpnIpB, vpnIpA, 90, 80)
 
 	// And once more from me to them
-	controlA.InjectTunPacket(BuildTunUDPPacket(vpnIpB, 80, vpnIpA, 90, []byte("Hello from A")))
+	controlA.InjectTunUDPPacket(vpnIpB, 80, vpnIpA, 90, []byte("Hello from A"))
 	aPacket := r.RouteForAllUntilTxTun(controlB)
 	assertUdpPacket(t, []byte("Hello from A"), aPacket, vpnIpA, vpnIpB, 90, 80)
 }
@@ -379,87 +379,24 @@ func getAddrs(ns []netip.Prefix) []netip.Addr {
 	return a
 }
 
-func NewTestLogger() *slog.Logger {
+func NewTestLogger() *logrus.Logger {
+	l := logrus.New()
+
 	v := os.Getenv("TEST_LOGS")
 	if v == "" {
-		return slog.New(slog.NewTextHandler(io.Discard, nil))
+		l.SetOutput(io.Discard)
+		l.SetLevel(logrus.PanicLevel)
+		return l
 	}
 
-	level := slog.LevelInfo
 	switch v {
 	case "2":
-		level = slog.LevelDebug
+		l.SetLevel(logrus.DebugLevel)
 	case "3":
-		level = logging.LevelTrace
+		l.SetLevel(logrus.TraceLevel)
+	default:
+		l.SetLevel(logrus.InfoLevel)
 	}
-	return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level}))
-}
-
-// testLogLevelName returns the level name string accepted by logging.ApplyConfig
-// for the current TEST_LOGS setting. Kept in sync with NewTestLogger.
-func testLogLevelName() string {
-	switch os.Getenv("TEST_LOGS") {
-	case "2":
-		return "debug"
-	case "3":
-		return "trace"
-	case "":
-		return "info"
-	}
-	return "info"
-}
-
-// BuildTunUDPPacket assembles an IP+UDP packet suitable for Control.InjectTunPacket.
-// Using UDP here because it's a simpler protocol.
-func BuildTunUDPPacket(toAddr netip.Addr, toPort uint16, fromAddr netip.Addr, fromPort uint16, data []byte) []byte {
-	serialize := make([]gopacket.SerializableLayer, 0)
-	var netLayer gopacket.NetworkLayer
-	if toAddr.Is6() {
-		if !fromAddr.Is6() {
-			panic("Cant send ipv6 to ipv4")
-		}
-		ip := &layers.IPv6{
-			Version:    6,
-			NextHeader: layers.IPProtocolUDP,
-			SrcIP:      fromAddr.Unmap().AsSlice(),
-			DstIP:      toAddr.Unmap().AsSlice(),
-		}
-		serialize = append(serialize, ip)
-		netLayer = ip
-	} else {
-		if !fromAddr.Is4() {
-			panic("Cant send ipv4 to ipv6")
-		}
-
-		ip := &layers.IPv4{
-			Version:  4,
-			TTL:      64,
-			Protocol: layers.IPProtocolUDP,
-			SrcIP:    fromAddr.Unmap().AsSlice(),
-			DstIP:    toAddr.Unmap().AsSlice(),
-		}
-		serialize = append(serialize, ip)
-		netLayer = ip
-	}
-
-	udp := layers.UDP{
-		SrcPort: layers.UDPPort(fromPort),
-		DstPort: layers.UDPPort(toPort),
-	}
-	if err := udp.SetNetworkLayerForChecksum(netLayer); err != nil {
-		panic(err)
-	}
-
-	buffer := gopacket.NewSerializeBuffer()
-	opt := gopacket.SerializeOptions{
-		ComputeChecksums: true,
-		FixLengths:       true,
-	}
-
-	serialize = append(serialize, &udp, gopacket.Payload(data))
-	if err := gopacket.SerializeLayers(buffer, opt, serialize...); err != nil {
-		panic(err)
-	}
-
-	return buffer.Bytes()
+
+	return l
 }

e2e/leak_test.go

@@ -1,51 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/e2e/router"
-	"go.uber.org/goleak"
-)
-
-// TestNoGoroutineLeaks brings up two nebula instances, completes a tunnel,
-// stops both, and asserts no goroutines leak past the shutdown. goleak's
-// retry mechanism gives the wg.Wait()-driven goroutines a moment to drain
-// before failing the assertion.
-//
-// IgnoreCurrent is necessary in the parallelized suite: other tests can
-// leave goroutines mid-shutdown when this one runs (Stop is async, the
-// wg.Wait() drain is not blocking on test return). We're checking that
-// *this* test's setup tears down cleanly, not that the whole suite is
-// idle at this moment. Intentionally NOT t.Parallel()'d for the same
-// reason — concurrent test goroutines would always show up.
-func TestNoGoroutineLeaks(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	myControl.Stop()
-	theirControl.Stop()
-	r.RenderFlow()
-
-	// Settle period: Stop() is non-blocking; the wg-driven goroutines need
-	// a moment to drain. goleak retries internally too, but a short explicit
-	// settle reduces flakes when the suite is busy.
-	time.Sleep(50 * time.Millisecond)
-}

e2e/router/router.go

@@ -13,7 +13,6 @@ import (
 	"regexp"
 	"sort"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -25,19 +24,6 @@ import (
 	"golang.org/x/exp/maps"
 )
 
-// outNatKey is the (from, to) pair used by outNat. Comparable struct, so it works as a map key without the
-// allocation cost of a string-concat key.
-type outNatKey struct {
-	from, to netip.AddrPort
-}
-
-// fannedPacket pairs a UDP TX packet with its source control so the router can route it after popping from
-// the fan-in channel.
-type fannedPacket struct {
-	from *nebula.Control
-	pkt  *udp.Packet
-}
-
 type R struct {
 	// Simple map of the ip:port registered on a control to the control
 	// Basically a router, right?
@@ -48,28 +34,12 @@ type R struct {
 
 	// A last used map, if an inbound packet hit the inNat map then
 	// all return packets should use the same last used inbound address for the outbound sender
-	outNat map[outNatKey]netip.AddrPort
+	// map[from address + ":" + to address] => ip:port to rewrite in the udp packet to receiver
+	outNat map[string]netip.AddrPort
 
 	// A map of vpn ip to the nebula control it belongs to
 	vpnControls map[netip.Addr]*nebula.Control
 
-	// Cached select infrastructure for RouteForAllUntilTxTun.
-	// The controls map is immutable after NewR so the cases are good for the test lifetime.
-	// We only rebuild if a different receiver is asked.
-	selRecvCtl *nebula.Control
-	selCases   []reflect.SelectCase
-	selCtls    []*nebula.Control
-
-	// Optional fan-in mode for hot-path benchmarks: one forwarder goroutine per control drains UDP TX into udpFanIn,
-	// so RouteForAllUntilTxTun can do a fixed 2-way native select instead of paying reflect.Select per call.
-	// Off by default (would otherwise interleave with tests that use GetFromUDP directly on the same control).
-	// Enabled by EnableFanIn.
-	udpFanIn  chan fannedPacket
-	stopFanIn chan struct{}
-	fanInWG   sync.WaitGroup
-	fanInMu   sync.Mutex
-	fanInOn   atomic.Bool
-
 	ignoreFlows []ignoreFlow
 	flow        []flowEntry
 
@@ -149,7 +119,7 @@ func NewR(t testing.TB, controls ...*nebula.Control) *R {
 		controls:     make(map[netip.AddrPort]*nebula.Control),
 		vpnControls:  make(map[netip.Addr]*nebula.Control),
 		inNat:        make(map[netip.AddrPort]*nebula.Control),
-		outNat:       make(map[outNatKey]netip.AddrPort),
+		outNat:       make(map[string]netip.AddrPort),
 		flow:         []flowEntry{},
 		ignoreFlows:  []ignoreFlow{},
 		fn:           filepath.Join("mermaid", fmt.Sprintf("%s.md", t.Name())),
@@ -183,10 +153,8 @@ func NewR(t testing.TB, controls ...*nebula.Control) *R {
 			case <-ctx.Done():
 				return
 			case <-clockSource.C:
-				r.Lock()
 				r.renderHostmaps("clock tick")
 				r.renderFlow()
-				r.Unlock()
 			}
 		}
 	}()
@@ -212,21 +180,15 @@ func (r *R) AddRoute(ip netip.Addr, port uint16, c *nebula.Control) {
 // RenderFlow renders the packet flow seen up until now and stops further automatic renders from happening.
 func (r *R) RenderFlow() {
 	r.cancelRender()
-	r.Lock()
-	defer r.Unlock()
 	r.renderFlow()
 }
 
 // CancelFlowLogs stops flow logs from being tracked and destroys any logs already collected
 func (r *R) CancelFlowLogs() {
 	r.cancelRender()
-	r.Lock()
 	r.flow = nil
-	r.Unlock()
 }
 
-// renderFlow writes the flow log to disk. Caller must hold r.Lock. renderFlow reads r.flow / r.additionalGraphs and
-// the *packet pointers stashed inside, all of which are mutated under the same lock by routing paths.
 func (r *R) renderFlow() {
 	if r.flow == nil {
 		return
@@ -472,157 +434,68 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []
 				panic("No control for udp tx " + a.String())
 			}
 			fp := r.unlockedInjectFlow(sender, c, p, false)
-			c.InjectUDPPacket(p) // copies internally; original is ours to release
+			c.InjectUDPPacket(p)
 			fp.WasReceived()
 			r.Unlock()
-			p.Release()
 		}
 	}
 }
 
-// RouteForAllUntilTxTun will route for everyone and return when a packet is seen on the receiver's tun.
-// If a control's UDP TX address can't be matched to a registered control, we panic.
-//
-// For allocation-sensitive callers (hot-path benchmarks, in particular relay
-// benches with 3+ controls), call EnableFanIn() first.
+// RouteForAllUntilTxTun will route for everyone and return when a packet is seen on receivers tun
+// If the router doesn't have the nebula controller for that address, we panic
 func (r *R) RouteForAllUntilTxTun(receiver *nebula.Control) []byte {
-	if r.fanInOn.Load() {
-		return r.routeFanIn(receiver)
-	}
-	return r.routeReflect(receiver)
-}
-
-// routeFanIn is the alloc-free path used when EnableFanIn is in effect.
-func (r *R) routeFanIn(receiver *nebula.Control) []byte {
-	tunTx := receiver.GetTunTxChan()
-	for {
-		select {
-		case p := <-tunTx:
-			r.Lock()
-			if r.flow != nil {
-				np := udp.Packet{Data: make([]byte, len(p))}
-				copy(np.Data, p)
-				r.unlockedInjectFlow(receiver, receiver, &np, true)
-			}
-			r.Unlock()
-			return p
-		case fp := <-r.udpFanIn:
-			r.routeUDP(fp.from, fp.pkt)
-		}
-	}
-}
-
-// routeReflect is the default reflect.Select-based path. Pays the boxing allocation per call but doesn't interfere
-// with tests that pull packets directly from controls' UDP TX channels via GetFromUDP.
-func (r *R) routeReflect(receiver *nebula.Control) []byte {
-	sc, cm := r.selectCasesFor(receiver)
-	for {
-		x, rx, _ := reflect.Select(sc)
-		if x == 0 {
-			p := rx.Interface().([]byte)
-			r.Lock()
-			if r.flow != nil {
-				np := udp.Packet{Data: make([]byte, len(p))}
-				copy(np.Data, p)
-				r.unlockedInjectFlow(cm[x], cm[x], &np, true)
-			}
-			r.Unlock()
-			return p
-		}
-		r.routeUDP(cm[x], rx.Interface().(*udp.Packet))
-	}
-}
-
-// EnableFanIn switches RouteForAllUntilTxTun to the alloc-free fan-in path.
-// One forwarder goroutine per registered control drains UDP TX into a shared channel that RouteForAllUntilTxTun selects
-// on alongside the receiver's TUN TX channel.
-func (r *R) EnableFanIn() {
-	r.fanInMu.Lock()
-	defer r.fanInMu.Unlock()
-	if r.fanInOn.Load() {
-		return
-	}
-	r.udpFanIn = make(chan fannedPacket, 32)
-	r.stopFanIn = make(chan struct{})
-	for _, c := range r.controls {
-		r.startFanInWorker(c)
-	}
-	r.fanInOn.Store(true)
-	r.t.Cleanup(r.stopFanInWorkers)
-}
-
-// startFanInWorker spawns a goroutine that drains c's UDP TX into r.udpFanIn.
-func (r *R) startFanInWorker(c *nebula.Control) {
-	r.fanInWG.Add(1)
-	udpTx := c.GetUDPTxChan()
-	go func() {
-		defer r.fanInWG.Done()
-		for {
-			select {
-			case <-r.stopFanIn:
-				return
-			case p := <-udpTx:
-				select {
-				case <-r.stopFanIn:
-					p.Release()
-					return
-				case r.udpFanIn <- fannedPacket{from: c, pkt: p}:
-				}
-			}
-		}
-	}()
-}
-
-// stopFanInWorkers signals the fan-in goroutines to exit and waits for them.
-func (r *R) stopFanInWorkers() {
-	r.fanInMu.Lock()
-	wasOn := r.fanInOn.Swap(false)
-	r.fanInMu.Unlock()
-	if !wasOn {
-		return
-	}
-	close(r.stopFanIn)
-	r.fanInWG.Wait()
-}
-
-// routeUDP forwards a UDP TX packet from the named source control to the destination control derived from p.To,
-// releasing the source packet after InjectUDPPacket has copied its bytes into a fresh pool slot.
-func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
-	r.Lock()
-	defer r.Unlock()
-	a := from.GetUDPAddr()
-	c := r.getControl(a, p.To, p)
-	if c == nil {
-		panic(fmt.Sprintf("No control for udp tx %s", p.To))
-	}
-	fp := r.unlockedInjectFlow(from, c, p, false)
-	c.InjectUDPPacket(p) // copies internally; original is ours to release
-	fp.WasReceived()
-	p.Release()
-}
-
-// selectCasesFor returns the SelectCase array used by routeReflect: one slot for the receiver's TUN TX channel followed
-// by one per control's UDP TX channel. Cached for the test lifetime, only rebuilt if the receiver changes.
-func (r *R) selectCasesFor(receiver *nebula.Control) ([]reflect.SelectCase, []*nebula.Control) {
-	r.Lock()
-	defer r.Unlock()
-	if r.selRecvCtl == receiver && r.selCases != nil {
-		return r.selCases, r.selCtls
-	}
 	sc := make([]reflect.SelectCase, len(r.controls)+1)
 	cm := make([]*nebula.Control, len(r.controls)+1)
-	sc[0] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(receiver.GetTunTxChan())}
-	cm[0] = receiver
-	i := 1
+
+	i := 0
+	sc[i] = reflect.SelectCase{
+		Dir:  reflect.SelectRecv,
+		Chan: reflect.ValueOf(receiver.GetTunTxChan()),
+		Send: reflect.Value{},
+	}
+	cm[i] = receiver
+
+	i++
 	for _, c := range r.controls {
-		sc[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(c.GetUDPTxChan())}
+		sc[i] = reflect.SelectCase{
+			Dir:  reflect.SelectRecv,
+			Chan: reflect.ValueOf(c.GetUDPTxChan()),
+			Send: reflect.Value{},
+		}
+
 		cm[i] = c
 		i++
 	}
-	r.selRecvCtl = receiver
-	r.selCases = sc
-	r.selCtls = cm
-	return sc, cm
+
+	for {
+		x, rx, _ := reflect.Select(sc)
+		r.Lock()
+
+		if x == 0 {
+			// we are the tun tx, we can exit
+			p := rx.Interface().([]byte)
+			np := udp.Packet{Data: make([]byte, len(p))}
+			copy(np.Data, p)
+
+			r.unlockedInjectFlow(cm[x], cm[x], &np, true)
+			r.Unlock()
+			return p
+
+		} else {
+			// we are a udp tx, route and continue
+			p := rx.Interface().(*udp.Packet)
+			a := cm[x].GetUDPAddr()
+			c := r.getControl(a, p.To, p)
+			if c == nil {
+				r.Unlock()
+				panic(fmt.Sprintf("No control for udp tx %s", p.To))
+			}
+			fp := r.unlockedInjectFlow(cm[x], c, p, false)
+			c.InjectUDPPacket(p)
+			fp.WasReceived()
+		}
+		r.Unlock()
+	}
 }
 
 // RouteExitFunc will call the whatDo func with each udp packet from sender.
@@ -649,7 +522,6 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
 		switch e {
 		case ExitNow:
 			r.Unlock()
-			p.Release()
 			return
 
 		case RouteAndExit:
@@ -657,7 +529,6 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
 			receiver.InjectUDPPacket(p)
 			fp.WasReceived()
 			r.Unlock()
-			p.Release()
 			return
 
 		case KeepRouting:
@@ -670,7 +541,6 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
 		}
 
 		r.Unlock()
-		p.Release()
 	}
 }
 
@@ -771,7 +641,6 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
 		switch e {
 		case ExitNow:
 			r.Unlock()
-			p.Release()
 			return
 
 		case RouteAndExit:
@@ -779,7 +648,6 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
 			receiver.InjectUDPPacket(p)
 			fp.WasReceived()
 			r.Unlock()
-			p.Release()
 			return
 
 		case KeepRouting:
@@ -791,7 +659,6 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
 			panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
 		}
 		r.Unlock()
-		p.Release()
 	}
 }
 
@@ -835,20 +702,19 @@ func (r *R) FlushAll() {
 		}
 		receiver.InjectUDPPacket(p)
 		r.Unlock()
-		p.Release()
 	}
 }
 
 // getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change
 // This is an internal router function, the caller must hold the lock
 func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control {
-	if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
+	if newAddr, ok := r.outNat[fromAddr.String()+":"+toAddr.String()]; ok {
 		p.From = newAddr
 	}
 
 	c, ok := r.inNat[toAddr]
 	if ok {
-		r.outNat[outNatKey{from: c.GetUDPAddr(), to: fromAddr}] = toAddr
+		r.outNat[c.GetUDPAddr().String()+":"+fromAddr.String()] = toAddr
 		return c
 	}
 

e2e/tunnels_test.go

@@ -19,7 +19,6 @@ import (
 )
 
 func TestDropInactiveTunnels(t *testing.T) {
-	t.Parallel()
 	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
 	// under ideal conditions
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
@@ -64,7 +63,6 @@ func TestDropInactiveTunnels(t *testing.T) {
 }
 
 func TestCertUpgrade(t *testing.T) {
-	t.Parallel()
 	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
 	// under ideal conditions
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
@@ -159,7 +157,6 @@ func TestCertUpgrade(t *testing.T) {
 }
 
 func TestCertDowngrade(t *testing.T) {
-	t.Parallel()
 	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
 	// under ideal conditions
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
@@ -258,7 +255,6 @@ func TestCertDowngrade(t *testing.T) {
 }
 
 func TestCertMismatchCorrection(t *testing.T) {
-	t.Parallel()
 	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
 	// under ideal conditions
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
@@ -326,7 +322,6 @@ func TestCertMismatchCorrection(t *testing.T) {
 }
 
 func TestCrossStackRelaysWork(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24,fc00::1/64", m{"relay": m{"use_relays": true}})
 	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay  ", "10.128.0.128/24,fc00::128/64", m{"relay": m{"am_relay": true}})
@@ -355,14 +350,14 @@ func TestCrossStackRelaysWork(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
 	assertUdpPacket(t, []byte("Hi from me"), p, myVpnV6.Addr(), theirVpnV6.Addr(), 80, 80)
 
 	t.Log("reply?")
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them")))
+	theirControl.InjectTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them"))
 	p = r.RouteForAllUntilTxTun(myControl)
 	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnV6.Addr(), myVpnV6.Addr(), 80, 80)
 
@@ -374,7 +369,6 @@ func TestCrossStackRelaysWork(t *testing.T) {
 }
 
 func TestCloseTunnelAuthenticated(t *testing.T) {
-	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "5s"}})
 	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "10m"}})

examples/config.yml

@@ -292,21 +292,24 @@ tun:
 
 # Configure logging level
 logging:
-  # trace, debug, info, warn, or error. Default is info and is reloadable.
-  # fatal and panic are accepted for backwards compatibility and map to error.
-  #NOTE: Debug and trace modes can log remotely controlled/untrusted data which can quickly fill a disk in some
-  # scenarios. Debug and trace logging are also CPU intensive and will decrease performance overall.
-  # Only enable debug or trace logging while actively investigating an issue.
+  # panic, fatal, error, warning, info, or debug. Default is info and is reloadable.
+  #NOTE: Debug mode can log remotely controlled/untrusted data which can quickly fill a disk in some
+  # scenarios. Debug logging is also CPU intensive and will decrease performance overall.
+  # Only enable debug logging while actively investigating an issue.
   level: info
-  # json or text formats currently available. Default is text.
+  # json or text formats currently available. Default is text
   format: text
-  # Disable timestamp logging. Useful when output is redirected to a logging system that already adds timestamps. Default is false.
+  # Disable timestamp logging. useful when output is redirected to logging system that already adds timestamps. Default is false
   #disable_timestamp: true
-  # Timestamps use RFC3339Nano ("2006-01-02T15:04:05.999999999Z07:00") and are not configurable.
+  # timestamp format is specified in Go time format, see:
+  #     https://golang.org/pkg/time/#pkg-constants
+  # default when `format: json`: "2006-01-02T15:04:05Z07:00" (RFC3339)
+  # default when `format: text`:
+  #     when TTY attached: seconds since beginning of execution
+  #     otherwise: "2006-01-02T15:04:05Z07:00" (RFC3339)
+  # As an example, to log as RFC3339 with millisecond precision, set to:
+  #timestamp_format: "2006-01-02T15:04:05.000Z07:00"
 
-# The stats section is reloadable. A HUP may change the backend, toggle stats
-# on or off, switch the listen/host address, or pick up new DNS for the
-# configured graphite host.
 #stats:
   #type: graphite
   #prefix: nebula
@@ -324,12 +327,10 @@ logging:
   # enables counter metrics for meta packets
   #   e.g.: `messages.tx.handshake`
   # NOTE: `message.{tx,rx}.recv_error` is always emitted
-  # Not reloadable.
   #message_metrics: false
 
   # enables detailed counter metrics for lighthouse packets
   #   e.g.: `lighthouse.rx.HostQuery`
-  # Not reloadable.
   #lighthouse_metrics: false
 
 # Handshake Manager Settings

examples/go_service/main.go

@@ -7,9 +7,9 @@ import (
 	"net"
 	"os"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/logging"
 	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/service"
 )
@@ -64,7 +64,8 @@ pki:
 		return err
 	}
 
-	logger := logging.NewLogger(os.Stdout)
+	logger := logrus.New()
+	logger.Out = os.Stdout
 
 	ctrl, err := nebula.Main(&cfg, false, "custom-app", logger, overlay.NewUserDeviceFromConfig)
 	if err != nil {

firewall.go

@@ -1,13 +1,11 @@
 package nebula
 
 import (
-	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"errors"
 	"fmt"
 	"hash/fnv"
-	"log/slog"
 	"net/netip"
 	"reflect"
 	"slices"
@@ -18,6 +16,7 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/rcrowley/go-metrics"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
@@ -68,7 +67,7 @@ type Firewall struct {
 	incomingMetrics     firewallMetrics
 	outgoingMetrics     firewallMetrics
 
-	l *slog.Logger
+	l *logrus.Logger
 }
 
 type firewallMetrics struct {
@@ -132,7 +131,7 @@ type firewallLocalCIDR struct {
 
 // NewFirewall creates a new Firewall object. A TimerWheel is created for you from the provided timeouts.
 // The certificate provided should be the highest version loaded in memory.
-func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c cert.Certificate) *Firewall {
+func NewFirewall(l *logrus.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c cert.Certificate) *Firewall {
 	//TODO: error on 0 duration
 	var tmin, tmax time.Duration
 
@@ -192,7 +191,7 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
 	}
 }
 
-func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewall, error) {
+func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firewall, error) {
 	certificate := cs.getCertificate(cert.Version2)
 	if certificate == nil {
 		certificate = cs.getCertificate(cert.Version1)
@@ -220,7 +219,7 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
 	case "drop":
 		fw.InSendReject = false
 	default:
-		l.Warn("invalid firewall.inbound_action, defaulting to `drop`", "action", inboundAction)
+		l.WithField("action", inboundAction).Warn("invalid firewall.inbound_action, defaulting to `drop`")
 		fw.InSendReject = false
 	}
 
@@ -231,7 +230,7 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
 	case "drop":
 		fw.OutSendReject = false
 	default:
-		l.Warn("invalid firewall.outbound_action, defaulting to `drop`", "action", outboundAction)
+		l.WithField("action", outboundAction).Warn("invalid firewall.outbound_action, defaulting to `drop`")
 		fw.OutSendReject = false
 	}
 
@@ -269,7 +268,7 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 	case firewall.ProtoICMP, firewall.ProtoICMPv6:
 		//ICMP traffic doesn't have ports, so we always coerce to "any", even if a value is provided
 		if startPort != firewall.PortAny {
-			f.l.Warn("ignoring port specification for ICMP firewall rule", "startPort", startPort)
+			f.l.WithField("startPort", startPort).Warn("ignoring port specification for ICMP firewall rule")
 		}
 		startPort = firewall.PortAny
 		endPort = firewall.PortAny
@@ -291,9 +290,8 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 	if !incoming {
 		direction = "outgoing"
 	}
-	f.l.Info("Firewall rule added",
-		"firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha},
-	)
+	f.l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha}).
+		Info("Firewall rule added")
 
 	return fp.addRule(f, startPort, endPort, groups, host, cidr, localCidr, caName, caSha)
 }
@@ -316,7 +314,7 @@ func (f *Firewall) GetRuleHashes() string {
 	return "SHA:" + f.GetRuleHash() + ",FNV:" + strconv.FormatUint(uint64(f.GetRuleHashFNV()), 10)
 }
 
-func AddFirewallRulesFromConfig(l *slog.Logger, inbound bool, c *config.C, fw FirewallInterface) error {
+func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw FirewallInterface) error {
 	var table string
 	if inbound {
 		table = "firewall.inbound"
@@ -374,7 +372,7 @@ func AddFirewallRulesFromConfig(l *slog.Logger, inbound bool, c *config.C, fw Fi
 			startPort = firewall.PortAny
 			endPort = firewall.PortAny
 			if sPort != "" {
-				l.Warn("ignoring port specification for ICMP firewall rule", "port", sPort)
+				l.WithField("port", sPort).Warn("ignoring port specification for ICMP firewall rule")
 			}
 		default:
 			return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
@@ -398,11 +396,7 @@ func AddFirewallRulesFromConfig(l *slog.Logger, inbound bool, c *config.C, fw Fi
 		}
 
 		if warning := r.sanity(); warning != nil {
-			l.Warn("firewall rule sanity check",
-				"table", table,
-				"rule", i,
-				"warning", warning,
-			)
+			l.Warnf("%s rule #%v; %s", table, i, warning)
 		}
 
 		err = fw.AddRule(inbound, proto, startPort, endPort, r.Groups, r.Host, r.Cidr, r.LocalCidr, r.CAName, r.CASha)
@@ -534,26 +528,26 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 
 		// We now know which firewall table to check against
 		if !table.match(fp, c.incoming, h.ConnectionState.peerCert, caPool) {
-			if f.l.Enabled(context.Background(), slog.LevelDebug) {
-				h.logger(f.l).Debug("dropping old conntrack entry, does not match new ruleset",
-					"fwPacket", fp,
-					"incoming", c.incoming,
-					"rulesVersion", f.rulesVersion,
-					"oldRulesVersion", c.rulesVersion,
-				)
+			if f.l.Level >= logrus.DebugLevel {
+				h.logger(f.l).
+					WithField("fwPacket", fp).
+					WithField("incoming", c.incoming).
+					WithField("rulesVersion", f.rulesVersion).
+					WithField("oldRulesVersion", c.rulesVersion).
+					Debugln("dropping old conntrack entry, does not match new ruleset")
 			}
 			delete(conntrack.Conns, fp)
 			conntrack.Unlock()
 			return false
 		}
 
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			h.logger(f.l).Debug("keeping old conntrack entry, does match new ruleset",
-				"fwPacket", fp,
-				"incoming", c.incoming,
-				"rulesVersion", f.rulesVersion,
-				"oldRulesVersion", c.rulesVersion,
-			)
+		if f.l.Level >= logrus.DebugLevel {
+			h.logger(f.l).
+				WithField("fwPacket", fp).
+				WithField("incoming", c.incoming).
+				WithField("rulesVersion", f.rulesVersion).
+				WithField("oldRulesVersion", c.rulesVersion).
+				Debugln("keeping old conntrack entry, does match new ruleset")
 		}
 
 		c.rulesVersion = f.rulesVersion
@@ -941,7 +935,7 @@ type rule struct {
 	CASha     string
 }
 
-func convertRule(l *slog.Logger, p any, table string, i int) (rule, error) {
+func convertRule(l *logrus.Logger, p any, table string, i int) (rule, error) {
 	r := rule{}
 
 	m, ok := p.(map[string]any)
@@ -972,10 +966,7 @@ func convertRule(l *slog.Logger, p any, table string, i int) (rule, error) {
 			return r, errors.New("group should contain a single value, an array with more than one entry was provided")
 		}
 
-		l.Warn("group was an array with a single value, converting to simple value",
-			"table", table,
-			"rule", i,
-		)
+		l.Warnf("%s rule #%v; group was an array with a single value, converting to simple value", table, i)
 		m["group"] = v[0]
 	}
 

firewall/cache.go

@@ -1,10 +1,10 @@
 package firewall
 
 import (
-	"context"
-	"log/slog"
 	"sync/atomic"
 	"time"
+
+	"github.com/sirupsen/logrus"
 )
 
 // ConntrackCache is used as a local routine cache to know if a given flow
@@ -15,49 +15,41 @@ type ConntrackCacheTicker struct {
 	cacheV    uint64
 	cacheTick atomic.Uint64
 
-	l     *slog.Logger
 	cache ConntrackCache
 }
 
-func NewConntrackCacheTicker(ctx context.Context, l *slog.Logger, d time.Duration) *ConntrackCacheTicker {
+func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
 	if d == 0 {
 		return nil
 	}
 
 	c := &ConntrackCacheTicker{
-		l:     l,
 		cache: ConntrackCache{},
 	}
 
-	go c.tick(ctx, d)
+	go c.tick(d)
 
 	return c
 }
 
-func (c *ConntrackCacheTicker) tick(ctx context.Context, d time.Duration) {
-	t := time.NewTicker(d)
-	defer t.Stop()
+func (c *ConntrackCacheTicker) tick(d time.Duration) {
 	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-t.C:
-			c.cacheTick.Add(1)
-		}
+		time.Sleep(d)
+		c.cacheTick.Add(1)
 	}
 }
 
 // Get checks if the cache ticker has moved to the next version before returning
 // the map. If it has moved, we reset the map.
-func (c *ConntrackCacheTicker) Get() ConntrackCache {
+func (c *ConntrackCacheTicker) Get(l *logrus.Logger) ConntrackCache {
 	if c == nil {
 		return nil
 	}
 	if tick := c.cacheTick.Load(); tick != c.cacheV {
 		c.cacheV = tick
 		if ll := len(c.cache); ll > 0 {
-			if c.l.Enabled(context.Background(), slog.LevelDebug) {
-				c.l.Debug("resetting conntrack cache", "len", ll)
+			if l.Level == logrus.DebugLevel {
+				l.WithField("len", ll).Debug("resetting conntrack cache")
 			}
 			c.cache = make(ConntrackCache, ll)
 		}

firewall/cache_test.go

@@ -1,69 +0,0 @@
-package firewall
-
-import (
-	"bytes"
-	"log/slog"
-	"strings"
-	"testing"
-
-	"github.com/slackhq/nebula/test"
-	"github.com/stretchr/testify/assert"
-)
-
-// The tests below pin the log format produced by ConntrackCacheTicker.Get
-// so changes cannot silently break what operators are grepping for. The
-// ticker's internal state (cache + cacheTick) is poked directly to avoid
-// racing a goroutine-driven tick in tests.
-
-func newFixedTicker(t *testing.T, l *slog.Logger, cacheLen int) *ConntrackCacheTicker {
-	t.Helper()
-	c := &ConntrackCacheTicker{
-		l:     l,
-		cache: make(ConntrackCache, cacheLen),
-	}
-	for i := 0; i < cacheLen; i++ {
-		c.cache[Packet{LocalPort: uint16(i) + 1}] = struct{}{}
-	}
-	c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path
-	return c
-}
-
-func TestConntrackCacheTicker_Get_TextFormat(t *testing.T) {
-	buf := &bytes.Buffer{}
-	l := test.NewLoggerWithOutputAndLevel(buf, slog.LevelDebug)
-
-	c := newFixedTicker(t, l, 3)
-	c.Get()
-
-	assert.Equal(t, "level=DEBUG msg=\"resetting conntrack cache\" len=3\n", buf.String())
-}
-
-func TestConntrackCacheTicker_Get_JSONFormat(t *testing.T) {
-	buf := &bytes.Buffer{}
-	l := test.NewJSONLoggerWithOutput(buf, slog.LevelDebug)
-
-	c := newFixedTicker(t, l, 2)
-	c.Get()
-
-	assert.JSONEq(t, `{"level":"DEBUG","msg":"resetting conntrack cache","len":2}`, strings.TrimSpace(buf.String()))
-}
-
-func TestConntrackCacheTicker_Get_QuietBelowDebug(t *testing.T) {
-	buf := &bytes.Buffer{}
-	l := test.NewLoggerWithOutputAndLevel(buf, slog.LevelInfo)
-
-	c := newFixedTicker(t, l, 5)
-	c.Get()
-
-	assert.Empty(t, buf.String())
-}
-
-func TestConntrackCacheTicker_Get_QuietWhenCacheEmpty(t *testing.T) {
-	buf := &bytes.Buffer{}
-	l := test.NewLoggerWithOutputAndLevel(buf, slog.LevelDebug)
-
-	c := newFixedTicker(t, l, 0)
-	c.Get()
-
-	assert.Empty(t, buf.String())
-}

firewall_test.go

@@ -3,13 +3,13 @@ package nebula
 import (
 	"bytes"
 	"errors"
-	"log/slog"
 	"math"
 	"net/netip"
 	"testing"
 	"time"
 
 	"github.com/gaissmai/bart"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
@@ -58,8 +58,9 @@ func TestNewFirewall(t *testing.T) {
 }
 
 func TestFirewall_AddRule(t *testing.T) {
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 
 	c := &dummyCert{}
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c)
@@ -176,8 +177,9 @@ func TestFirewall_AddRule(t *testing.T) {
 }
 
 func TestFirewall_Drop(t *testing.T) {
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 	myVpnNetworksTable := new(bart.Lite)
 	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 	p := firewall.Packet{
@@ -252,8 +254,9 @@ func TestFirewall_Drop(t *testing.T) {
 }
 
 func TestFirewall_DropV6(t *testing.T) {
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 
 	myVpnNetworksTable := new(bart.Lite)
 	myVpnNetworksTable.Insert(netip.MustParsePrefix("fd00::/7"))
@@ -482,8 +485,9 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 }
 
 func TestFirewall_Drop2(t *testing.T) {
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 	myVpnNetworksTable := new(bart.Lite)
 	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 
@@ -540,8 +544,9 @@ func TestFirewall_Drop2(t *testing.T) {
 }
 
 func TestFirewall_Drop3(t *testing.T) {
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 	myVpnNetworksTable := new(bart.Lite)
 	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 
@@ -628,8 +633,9 @@ func TestFirewall_Drop3(t *testing.T) {
 }
 
 func TestFirewall_Drop3V6(t *testing.T) {
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 	myVpnNetworksTable := new(bart.Lite)
 	myVpnNetworksTable.Insert(netip.MustParsePrefix("fd00::/7"))
 
@@ -665,8 +671,9 @@ func TestFirewall_Drop3V6(t *testing.T) {
 }
 
 func TestFirewall_DropConntrackReload(t *testing.T) {
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 	myVpnNetworksTable := new(bart.Lite)
 	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 
@@ -729,8 +736,9 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 }
 
 func TestFirewall_ICMPPortBehavior(t *testing.T) {
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 	myVpnNetworksTable := new(bart.Lite)
 	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 
@@ -872,8 +880,9 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 }
 
 func TestFirewall_DropIPSpoofing(t *testing.T) {
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 	myVpnNetworksTable := new(bart.Lite)
 	myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
 
@@ -1033,28 +1042,28 @@ func TestNewFirewallFromConfig(t *testing.T) {
 	l := test.NewLogger()
 	// Test a bad rule definition
 	c := &dummyCert{}
-	cs, err := newCertState(cert.Version2, nil, c, false, cert.Curve_CURVE25519, nil, "aes")
+	cs, err := newCertState(cert.Version2, nil, c, false, cert.Curve_CURVE25519, nil)
 	require.NoError(t, err)
 
-	conf := config.NewC(test.NewLogger())
+	conf := config.NewC(l)
 	conf.Settings["firewall"] = map[string]any{"outbound": "asdf"}
 	_, err = NewFirewallFromConfig(l, cs, conf)
 	require.EqualError(t, err, "firewall.outbound failed to parse, should be an array of rules")
 
 	// Test both port and code
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "code": "2"}}}
 	_, err = NewFirewallFromConfig(l, cs, conf)
 	require.EqualError(t, err, "firewall.outbound rule #0; only one of port or code should be provided")
 
 	// Test missing host, group, cidr, ca_name and ca_sha
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{}}}
 	_, err = NewFirewallFromConfig(l, cs, conf)
 	require.EqualError(t, err, "firewall.outbound rule #0; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided")
 
 	// Test code/port error
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"code": "a", "host": "testh", "proto": "any"}}}
 	_, err = NewFirewallFromConfig(l, cs, conf)
 	require.EqualError(t, err, "firewall.outbound rule #0; code was not a number; `a`")
@@ -1064,25 +1073,25 @@ func TestNewFirewallFromConfig(t *testing.T) {
 	require.EqualError(t, err, "firewall.outbound rule #0; port was not a number; `a`")
 
 	// Test proto error
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"code": "1", "host": "testh"}}}
 	_, err = NewFirewallFromConfig(l, cs, conf)
 	require.EqualError(t, err, "firewall.outbound rule #0; proto was not understood; ``")
 
 	// Test cidr parse error
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"code": "1", "cidr": "testh", "proto": "any"}}}
 	_, err = NewFirewallFromConfig(l, cs, conf)
 	require.EqualError(t, err, "firewall.outbound rule #0; cidr did not parse; netip.ParsePrefix(\"testh\"): no '/'")
 
 	// Test local_cidr parse error
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"code": "1", "local_cidr": "testh", "proto": "any"}}}
 	_, err = NewFirewallFromConfig(l, cs, conf)
 	require.EqualError(t, err, "firewall.outbound rule #0; local_cidr did not parse; netip.ParsePrefix(\"testh\"): no '/'")
 
 	// Test both group and groups
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "group": "a", "groups": []string{"b", "c"}}}}
 	_, err = NewFirewallFromConfig(l, cs, conf)
 	require.EqualError(t, err, "firewall.inbound rule #0; only one of group or groups should be defined, both provided")
@@ -1091,35 +1100,35 @@ func TestNewFirewallFromConfig(t *testing.T) {
 func TestAddFirewallRulesFromConfig(t *testing.T) {
 	l := test.NewLogger()
 	// Test adding tcp rule
-	conf := config.NewC(test.NewLogger())
+	conf := config.NewC(l)
 	mf := &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "tcp", "host": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoTCP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
 
 	// Test adding udp rule
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "udp", "host": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoUDP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
 
 	// Test adding icmp rule
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "icmp", "host": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoICMP, startPort: firewall.PortAny, endPort: firewall.PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
 
 	// Test adding icmp rule no port
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"proto": "icmp", "host": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoICMP, startPort: firewall.PortAny, endPort: firewall.PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
 
 	// Test adding any rule
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "host": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
@@ -1127,14 +1136,14 @@ func TestAddFirewallRulesFromConfig(t *testing.T) {
 
 	// Test adding rule with cidr
 	cidr := netip.MustParsePrefix("10.0.0.0/8")
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr.String()}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr.String(), localIp: ""}, mf.lastCall)
 
 	// Test adding rule with local_cidr
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr.String()}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
@@ -1142,82 +1151,82 @@ func TestAddFirewallRulesFromConfig(t *testing.T) {
 
 	// Test adding rule with cidr ipv6
 	cidr6 := netip.MustParsePrefix("fd00::/8")
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr6.String()}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr6.String(), localIp: ""}, mf.lastCall)
 
 	// Test adding rule with any cidr
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "any"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "any", localIp: ""}, mf.lastCall)
 
 	// Test adding rule with junk cidr
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "junk/junk"}}}
 	require.EqualError(t, AddFirewallRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP")
 
 	// Test adding rule with local_cidr ipv6
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr6.String()}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: cidr6.String()}, mf.lastCall)
 
 	// Test adding rule with any local_cidr
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "any"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, localIp: "any"}, mf.lastCall)
 
 	// Test adding rule with junk local_cidr
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "junk/junk"}}}
 	require.EqualError(t, AddFirewallRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; local_cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP")
 
 	// Test adding rule with ca_sha
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_sha": "12312313123"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caSha: "12312313123"}, mf.lastCall)
 
 	// Test adding rule with ca_name
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_name": "root01"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caName: "root01"}, mf.lastCall)
 
 	// Test single group
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "group": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall)
 
 	// Test single groups
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall)
 
 	// Test multiple AND groups
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": []string{"a", "b"}}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
 	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a", "b"}, ip: "", localIp: ""}, mf.lastCall)
 
 	// Test Add error
-	conf = config.NewC(test.NewLogger())
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	mf.nextCallReturn = errors.New("test error")
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "host": "a"}}}
@@ -1225,8 +1234,9 @@ func TestAddFirewallRulesFromConfig(t *testing.T) {
 }
 
 func TestFirewall_convertRule(t *testing.T) {
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 
 	// Ensure group array of 1 is converted and a warning is printed
 	c := map[string]any{
@@ -1234,9 +1244,7 @@ func TestFirewall_convertRule(t *testing.T) {
 	}
 
 	r, err := convertRule(l, c, "test", 1)
-	assert.Contains(t, ob.String(), "group was an array with a single value, converting to simple value")
-	assert.Contains(t, ob.String(), "table=test")
-	assert.Contains(t, ob.String(), "rule=1")
+	assert.Contains(t, ob.String(), "test rule #1; group was an array with a single value, converting to simple value")
 	require.NoError(t, err)
 	assert.Equal(t, []string{"group1"}, r.Groups)
 
@@ -1262,8 +1270,9 @@ func TestFirewall_convertRule(t *testing.T) {
 }
 
 func TestFirewall_convertRuleSanity(t *testing.T) {
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 
 	noWarningPlease := []map[string]any{
 		{"group": "group1"},
@@ -1377,7 +1386,7 @@ type testsetup struct {
 	fw                 *Firewall
 }
 
-func newSetup(t *testing.T, l *slog.Logger, myPrefixes ...netip.Prefix) testsetup {
+func newSetup(t *testing.T, l *logrus.Logger, myPrefixes ...netip.Prefix) testsetup {
 	c := dummyCert{
 		name:     "me",
 		networks: myPrefixes,
@@ -1388,7 +1397,7 @@ func newSetup(t *testing.T, l *slog.Logger, myPrefixes ...netip.Prefix) testsetu
 	return newSetupFromCert(t, l, c)
 }
 
-func newSetupFromCert(t *testing.T, l *slog.Logger, c dummyCert) testsetup {
+func newSetupFromCert(t *testing.T, l *logrus.Logger, c dummyCert) testsetup {
 	myVpnNetworksTable := new(bart.Lite)
 	for _, prefix := range c.Networks() {
 		myVpnNetworksTable.Insert(prefix)
@@ -1405,8 +1414,9 @@ func newSetupFromCert(t *testing.T, l *slog.Logger, c dummyCert) testsetup {
 
 func TestFirewall_Drop_EnforceIPMatch(t *testing.T) {
 	t.Parallel()
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
-	l := test.NewLoggerWithOutput(ob)
+	l.SetOutput(ob)
 
 	myPrefix := netip.MustParsePrefix("1.1.1.1/8")
 	// for now, it's okay that these are all "incoming", the logic this test tries to check doesn't care about in/out

go.mod

@@ -18,11 +18,11 @@ require (
 	github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
 	github.com/prometheus/client_golang v1.23.2
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
+	github.com/sirupsen/logrus v1.9.4
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
 	github.com/stretchr/testify v1.11.1
 	github.com/vishvananda/netlink v1.3.1
-	go.uber.org/goleak v1.3.0
 	go.yaml.in/yaml/v3 v3.0.4
 	golang.org/x/crypto v0.50.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090

go.sum

@@ -133,6 +133,8 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw=

handshake/credential.go

@@ -1,57 +0,0 @@
-package handshake
-
-import (
-	"crypto/rand"
-
-	"github.com/flynn/noise"
-	"github.com/slackhq/nebula/cert"
-)
-
-// Credential holds everything needed to participate in a handshake
-// at a given cert version. Version and Curve are read from Cert; the public
-// half of the static keypair likewise comes from Cert.PublicKey().
-type Credential struct {
-	Cert        cert.Certificate  // the certificate
-	Bytes       []byte            // pre-marshaled certificate bytes
-	privateKey  []byte            // static private key (public half lives in Cert)
-	cipherSuite noise.CipherSuite // pre-built cipher suite (DH + cipher + hash)
-}
-
-// NewCredential creates a Credential with all material needed for handshake
-// participation. The cipherSuite should be pre-built by the caller with the
-// appropriate DH function, cipher, and hash.
-func NewCredential(
-	c cert.Certificate,
-	hsBytes []byte,
-	privateKey []byte,
-	cipherSuite noise.CipherSuite,
-) *Credential {
-	return &Credential{
-		Cert:        c,
-		Bytes:       hsBytes,
-		privateKey:  privateKey,
-		cipherSuite: cipherSuite,
-	}
-}
-
-// buildHandshakeState creates a noise.HandshakeState from this credential.
-func (hc *Credential) buildHandshakeState(initiator bool, pattern noise.HandshakePattern) (*noise.HandshakeState, error) {
-	return noise.NewHandshakeState(noise.Config{
-		CipherSuite:           hc.cipherSuite,
-		Random:                rand.Reader,
-		Pattern:               pattern,
-		Initiator:             initiator,
-		StaticKeypair:         noise.DHKey{Private: hc.privateKey, Public: hc.Cert.PublicKey()},
-		PresharedKey:          []byte{},
-		PresharedKeyPlacement: 0,
-	})
-}
-
-// GetCredentialFunc returns the handshake credential for the given version,
-// or nil if that version is not available.
-//
-// Implementations must return credentials drawn from a snapshot stable for
-// the lifetime of any single Machine. The Machine may call this multiple
-// times during a handshake (e.g. when negotiating to the peer's version)
-// and assumes the underlying static keypair is consistent across calls.
-type GetCredentialFunc func(v cert.Version) *Credential

handshake/errors.go

@@ -1,21 +0,0 @@
-package handshake
-
-import "errors"
-
-var (
-	ErrInitiateOnResponder     = errors.New("initiate called on responder")
-	ErrInitiateAlreadyCalled   = errors.New("initiate already called")
-	ErrInitiateNotCalled       = errors.New("initiate must be called before ProcessPacket for initiators")
-	ErrPacketTooShort          = errors.New("packet too short")
-	ErrPublicKeyMismatch       = errors.New("public key mismatch between certificate and handshake")
-	ErrIncompleteHandshake     = errors.New("handshake completed without receiving required content")
-	ErrMachineFailed           = errors.New("handshake machine has failed")
-	ErrUnknownSubtype          = errors.New("unknown handshake subtype")
-	ErrMissingContent          = errors.New("expected handshake content but message was empty")
-	ErrUnexpectedContent       = errors.New("received unexpected handshake content")
-	ErrIndexAllocation         = errors.New("failed to allocate local index")
-	ErrNoCredential            = errors.New("no handshake credential available for cert version")
-	ErrAsymmetricCipherKeys    = errors.New("noise produced only one cipher key")
-	ErrMultiMessageUnsupported = errors.New("multi-message handshake patterns are not yet supported by the manager")
-	ErrSubtypeMismatch         = errors.New("packet subtype does not match handshake machine subtype")
-)

handshake/handshake.proto

@@ -1,29 +0,0 @@
-// This file documents the wire format the nebula handshake speaks. It is
-// not run through protoc; the encoder/decoder in payload.go is hand-written
-// against this shape directly to keep the parser narrow and panic-free.
-//
-// Any change to the wire format must be reflected here, and adding a new
-// field requires updating MarshalPayload / unmarshalPayloadDetails together
-// with the field-uniqueness and wire-type checks in those functions.
-
-syntax = "proto3";
-package nebula.handshake;
-
-message NebulaHandshake {
-  NebulaHandshakeDetails Details = 1;
-  bytes Hmac = 2;
-}
-
-message NebulaHandshakeDetails {
-  bytes Cert = 1;
-  uint32 InitiatorIndex = 2;
-  uint32 ResponderIndex = 3;
-  // Cookie was reserved for an anti-DoS mechanism that was never
-  // implemented. No released version of nebula has ever populated it; the
-  // hand-written parser silently skips it on read.
-  uint64 Cookie = 4 [deprecated = true];
-  uint64 Time = 5;
-  uint32 CertVersion = 8;
-  // reserved for WIP multiport
-  reserved 6, 7;
-}

handshake/helpers_test.go

@@ -1,116 +0,0 @@
-package handshake
-
-import (
-	"net/netip"
-	"testing"
-	"time"
-
-	"github.com/flynn/noise"
-	"github.com/slackhq/nebula/cert"
-	ct "github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/header"
-	"github.com/stretchr/testify/require"
-)
-
-// testCertState holds cert material for a test peer.
-type testCertState struct {
-	version cert.Version
-	creds   map[cert.Version]*Credential
-}
-
-func (s *testCertState) getCredential(v cert.Version) *Credential {
-	return s.creds[v]
-}
-
-func newTestCertState(
-	t *testing.T, ca cert.Certificate, caKey []byte, name string, networks []netip.Prefix,
-) *testCertState {
-	return newTestCertStateWithCipher(t, ca, caKey, name, networks, noise.CipherChaChaPoly)
-}
-
-func newTestCertStateWithCipher(
-	t *testing.T, ca cert.Certificate, caKey []byte, name string, networks []netip.Prefix,
-	cipher noise.CipherFunc,
-) *testCertState {
-	t.Helper()
-	c, _, rawPrivKey, _ := ct.NewTestCert(
-		cert.Version2, cert.Curve_CURVE25519, ca, caKey,
-		name, ca.NotBefore(), ca.NotAfter(), networks, nil, nil,
-	)
-
-	priv, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawPrivKey)
-	require.NoError(t, err)
-
-	hsBytes, err := c.MarshalForHandshakes()
-	require.NoError(t, err)
-
-	ncs := noise.NewCipherSuite(noise.DH25519, cipher, noise.HashSHA256)
-	return &testCertState{
-		version: cert.Version2,
-		creds: map[cert.Version]*Credential{
-			cert.Version2: NewCredential(c, hsBytes, priv, ncs),
-		},
-	}
-}
-
-func testVerifier(pool *cert.CAPool) CertVerifier {
-	return func(c cert.Certificate) (*cert.CachedCertificate, error) {
-		return pool.VerifyCertificate(time.Now(), c)
-	}
-}
-
-func newTestMachine(
-	t *testing.T,
-	cs *testCertState,
-	verifier CertVerifier,
-	initiator bool,
-	localIndex uint32,
-) *Machine {
-	t.Helper()
-	m, err := NewMachine(
-		cs.version, cs.getCredential,
-		verifier, func() (uint32, error) { return localIndex, nil },
-		initiator, header.HandshakeIXPSK0,
-	)
-	require.NoError(t, err)
-	return m
-}
-
-func initiateHandshake(
-	t *testing.T,
-	initCS *testCertState, initVerifier CertVerifier,
-	respCS *testCertState, respVerifier CertVerifier,
-) (initM, respM *Machine, respResult *Result, resp []byte, err error) {
-	t.Helper()
-	initM = newTestMachine(t, initCS, initVerifier, true, 100)
-	msg1, merr := initM.Initiate(nil)
-	require.NoError(t, merr)
-
-	respM = newTestMachine(t, respCS, respVerifier, false, 200)
-	resp, respResult, err = respM.ProcessPacket(nil, msg1)
-	return
-}
-
-func doFullHandshake(
-	t *testing.T, initCS, respCS *testCertState, caPool *cert.CAPool,
-) (initResult, respResult *Result) {
-	t.Helper()
-	v := testVerifier(caPool)
-
-	initM := newTestMachine(t, initCS, v, true, 1000)
-	respM := newTestMachine(t, respCS, v, false, 2000)
-
-	msg1, err := initM.Initiate(nil)
-	require.NoError(t, err)
-
-	resp, respResult, err := respM.ProcessPacket(nil, msg1)
-	require.NoError(t, err)
-	require.NotNil(t, respResult)
-	require.NotEmpty(t, resp)
-
-	_, initResult, err = initM.ProcessPacket(nil, resp)
-	require.NoError(t, err)
-	require.NotNil(t, initResult)
-
-	return initResult, respResult
-}

handshake/machine.go

@@ -1,444 +0,0 @@
-package handshake
-
-import (
-	"bytes"
-	"fmt"
-	"slices"
-	"time"
-
-	"github.com/flynn/noise"
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/header"
-)
-
-// IndexAllocator is called by the Machine to allocate a local index for the
-// handshake. It is called at most once, when the first outgoing message that
-// carries a payload is built.
-//
-// Implementations MUST NOT return 0. Zero is reserved as a sentinel meaning
-// "no index assigned" on the wire and in the payload-presence checks. If an
-// allocator ever returned 0, a legitimate handshake's payload could be
-// indistinguishable from an empty one and would be rejected.
-type IndexAllocator func() (uint32, error)
-
-// CertVerifier is called by the Machine after reconstructing the peer's
-// certificate from the handshake. The verifier performs all validation
-// (CA trust, expiry, policy checks, allow lists).
-type CertVerifier func(cert.Certificate) (*cert.CachedCertificate, error)
-
-// Result contains the results of a successful handshake.
-// Returned by ProcessPacket when the handshake is complete.
-type Result struct {
-	EKey          *noise.CipherState
-	DKey          *noise.CipherState
-	MyCert        cert.Certificate
-	RemoteCert    *cert.CachedCertificate
-	RemoteIndex   uint32
-	LocalIndex    uint32
-	HandshakeTime uint64
-	MessageIndex  uint64 // number of messages exchanged during the handshake
-	Initiator     bool
-}
-
-// Machine drives a Noise handshake through N messages. It handles Noise
-// protocol operations, certificate reconstruction, and payload encoding.
-// Certificate validation is delegated to the caller via CertVerifier.
-//
-// A Machine is not safe for concurrent use. The caller must ensure that
-// Initiate and ProcessPacket are not called concurrently.
-//
-// Error contract: when ProcessPacket or Initiate returns an error, callers
-// must check Failed() to decide what to do next. If Failed() is false the
-// underlying noise state was not advanced (the packet was rejected before
-// ReadMessage took effect, or the rejection is non-fatal like a stale
-// retransmit) and the Machine can accept another packet. If Failed() is
-// true the Machine is unrecoverable and the caller must abandon it.
-type Machine struct {
-	hs             *noise.HandshakeState
-	getCred        GetCredentialFunc
-	allocIndex     IndexAllocator
-	verifier       CertVerifier
-	result         *Result
-	msgs           []msgFlags
-	myVersion      cert.Version
-	subtype        header.MessageSubType
-	indexAllocated bool
-	remoteCertSet  bool
-	payloadSet     bool
-	failed         bool
-}
-
-// NewMachine creates a handshake state machine. The subtype determines both
-// the noise pattern and the per-message content layout. The credential for
-// `version` is fetched via getCred and used to seed the noise.HandshakeState.
-// IndexAllocator is called lazily when the first outgoing payload is built.
-func NewMachine(
-	version cert.Version,
-	getCred GetCredentialFunc,
-	verifier CertVerifier,
-	allocIndex IndexAllocator,
-	initiator bool,
-	subtype header.MessageSubType,
-) (*Machine, error) {
-	info, err := subtypeInfoFor(subtype)
-	if err != nil {
-		return nil, err
-	}
-
-	cred := getCred(version)
-	if cred == nil {
-		return nil, fmt.Errorf("%w: %v", ErrNoCredential, version)
-	}
-
-	hs, err := cred.buildHandshakeState(initiator, info.pattern)
-	if err != nil {
-		return nil, fmt.Errorf("build noise state: %w", err)
-	}
-
-	return &Machine{
-		hs:         hs,
-		subtype:    subtype,
-		msgs:       info.msgs,
-		getCred:    getCred,
-		allocIndex: allocIndex,
-		verifier:   verifier,
-		myVersion:  version,
-		result: &Result{
-			Initiator: initiator,
-		},
-	}, nil
-}
-
-// Failed returns true if the Machine is in an unrecoverable state.
-func (m *Machine) Failed() bool {
-	return m.failed
-}
-
-// Subtype returns the handshake subtype this Machine was built for.
-func (m *Machine) Subtype() header.MessageSubType {
-	return m.subtype
-}
-
-// MessageIndex returns the noise handshake message index, which equals the
-// wire counter of the most recently sent or received message.
-func (m *Machine) MessageIndex() int {
-	return m.hs.MessageIndex()
-}
-
-// requireComplete checks that both a peer cert and payload have been received.
-// Marks the machine as failed if not.
-func (m *Machine) requireComplete() error {
-	if !m.payloadSet || !m.remoteCertSet {
-		m.failed = true
-		return ErrIncompleteHandshake
-	}
-	return nil
-}
-
-// myMsgFlags returns the flags for the current outgoing message.
-func (m *Machine) myMsgFlags() msgFlags {
-	idx := m.hs.MessageIndex()
-	if idx < len(m.msgs) {
-		return m.msgs[idx]
-	}
-	return msgFlags{}
-}
-
-// peerMsgFlags returns the flags for the message we just read.
-func (m *Machine) peerMsgFlags() msgFlags {
-	idx := m.hs.MessageIndex() - 1
-	if idx >= 0 && idx < len(m.msgs) {
-		return m.msgs[idx]
-	}
-	return msgFlags{}
-}
-
-// Initiate produces the first handshake message. Only valid for initiators,
-// and must be called exactly once before ProcessPacket.
-//
-// out is a destination buffer the message is appended to and returned. Pass
-// nil to allocate fresh, or pass a re-used buffer sliced to length 0 (e.g.
-// buf[:0]) with sufficient capacity to avoid allocation.
-//
-// An error return may not indicate a fatal condition, check Failed() to
-// determine if the Machine can still be used.
-func (m *Machine) Initiate(out []byte) ([]byte, error) {
-	if m.failed {
-		return nil, ErrMachineFailed
-	}
-	if !m.result.Initiator {
-		m.failed = true
-		return nil, ErrInitiateOnResponder
-	}
-	if m.hs.MessageIndex() != 0 {
-		m.failed = true
-		return nil, ErrInitiateAlreadyCalled
-	}
-
-	// At MessageIndex=0 with RemoteIndex still zero, buildResponse produces
-	// header counter 1 and remote index 0, which is what the initial message needs.
-	out, _, _, err := m.buildResponse(out)
-	if err != nil {
-		m.failed = true
-		return nil, err
-	}
-	return out, nil
-}
-
-// ProcessPacket handles an incoming handshake message. It advances the Noise
-// state, validates the peer certificate via the verifier, and optionally
-// produces a response.
-//
-// out is a destination buffer the response is appended to and returned. Pass
-// nil to allocate fresh, or pass a re-used buffer sliced to length 0 (e.g.
-// buf[:0]) with sufficient capacity to avoid allocation. The returned slice
-// is nil when no outgoing message is produced (handshake complete on this
-// side, or final message of a multi-message pattern).
-//
-// Returns a non-nil Result when the handshake is complete.
-// An error return may not indicate a fatal condition, check Failed() to
-// determine if the Machine can still be used.
-func (m *Machine) ProcessPacket(out, packet []byte) ([]byte, *Result, error) {
-	if m.failed {
-		return nil, nil, ErrMachineFailed
-	}
-	if len(packet) < header.Len {
-		return nil, nil, ErrPacketTooShort
-	}
-	// Reject packets whose subtype doesn't match the one this Machine was
-	// built for. A pending handshake that suddenly receives a different
-	// subtype on its index is either a stray packet that matched by chance
-	// or a peer protocol violation; drop it without failing the Machine so
-	// the legitimate retransmit can still complete.
-	if header.MessageSubType(packet[1]) != m.subtype {
-		return nil, nil, ErrSubtypeMismatch
-	}
-	if m.result.Initiator && m.hs.MessageIndex() == 0 {
-		m.failed = true
-		return nil, nil, ErrInitiateNotCalled
-	}
-
-	// The (eKey, dKey) ordering here is correct for IX, where the initiator
-	// completes the handshake by reading the responder's stage-2 message.
-	// noise returns (cs1, cs2) where cs1 is the initiator->responder cipher.
-	// For 3-message patterns where a responder finishes by reading the final
-	// message, this ordering would be wrong; revisit when XX/pqIX lands.
-	msg, eKey, dKey, err := m.hs.ReadMessage(nil, packet[header.Len:])
-	if err != nil {
-		// Noise ReadMessage failed. The noise library checkpoints and rolls back
-		// on failure, so the Machine is still alive. The caller can retry with
-		// a different packet.
-		return nil, nil, fmt.Errorf("noise ReadMessage: %w", err)
-	}
-
-	// From here on, noise state has advanced. Any error is fatal.
-	flags := m.peerMsgFlags()
-
-	if err := m.processPayload(msg, flags); err != nil {
-		return nil, nil, err
-	}
-
-	// If ReadMessage derived keys, the handshake is complete. Noise should
-	// always produce both keys together; asymmetry is a protocol invariant
-	// violation.
-	if eKey != nil || dKey != nil {
-		if eKey == nil || dKey == nil {
-			m.failed = true
-			return nil, nil, ErrAsymmetricCipherKeys
-		}
-		if err := m.requireComplete(); err != nil {
-			return nil, nil, err
-		}
-		return nil, m.completed(eKey, dKey), nil
-	}
-
-	// ReadMessage didn't complete, produce the next outgoing message
-	out, dk, ek, err := m.buildResponse(out)
-	if err != nil {
-		m.failed = true
-		return nil, nil, err
-	}
-
-	if ek != nil || dk != nil {
-		if ek == nil || dk == nil {
-			m.failed = true
-			return nil, nil, ErrAsymmetricCipherKeys
-		}
-		if err := m.requireComplete(); err != nil {
-			return nil, nil, err
-		}
-		return out, m.completed(ek, dk), nil
-	}
-
-	return out, nil, nil
-}
-
-func (m *Machine) completed(eKey, dKey *noise.CipherState) *Result {
-	m.result.EKey = eKey
-	m.result.DKey = dKey
-	m.result.MessageIndex = uint64(m.hs.MessageIndex())
-	return m.result
-}
-
-func (m *Machine) processPayload(msg []byte, flags msgFlags) error {
-	if len(msg) == 0 {
-		if flags.expectsPayload || flags.expectsCert {
-			m.failed = true
-			return ErrMissingContent
-		}
-		return nil
-	}
-
-	payload, err := UnmarshalPayload(msg)
-	if err != nil {
-		m.failed = true
-		return fmt.Errorf("unmarshal handshake: %w", err)
-	}
-
-	// Assert the payload contains exactly what we expect
-	hasPayloadData := payload.InitiatorIndex != 0 || payload.ResponderIndex != 0 || payload.Time != 0
-	if hasPayloadData != flags.expectsPayload {
-		m.failed = true
-		return ErrUnexpectedContent
-	}
-
-	hasCertData := len(payload.Cert) > 0
-	if hasCertData != flags.expectsCert {
-		m.failed = true
-		return ErrUnexpectedContent
-	}
-
-	// Process payload
-	if flags.expectsPayload {
-		if m.result.Initiator {
-			m.result.RemoteIndex = payload.ResponderIndex
-		} else {
-			m.result.RemoteIndex = payload.InitiatorIndex
-		}
-		m.result.HandshakeTime = payload.Time
-		m.payloadSet = true
-	}
-
-	// Process certificate
-	if flags.expectsCert {
-		if err := m.validateCert(payload); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (m *Machine) validateCert(payload Payload) error {
-	cred := m.getCred(m.myVersion)
-	if cred == nil {
-		m.failed = true
-		return fmt.Errorf("%w: %v", ErrNoCredential, m.myVersion)
-	}
-	rc, err := cert.Recombine(
-		cert.Version(payload.CertVersion),
-		payload.Cert,
-		m.hs.PeerStatic(),
-		cred.Cert.Curve(),
-	)
-	if err != nil {
-		m.failed = true
-		return fmt.Errorf("recombine cert: %w", err)
-	}
-
-	if !bytes.Equal(rc.PublicKey(), m.hs.PeerStatic()) {
-		m.failed = true
-		return ErrPublicKeyMismatch
-	}
-
-	// Version negotiation, if the peer sent a different version and we have it, switch
-	if rc.Version() != m.myVersion {
-		if m.getCred(rc.Version()) != nil {
-			m.myVersion = rc.Version()
-		}
-	}
-
-	verified, err := m.verifier(rc)
-	if err != nil {
-		m.failed = true
-		return fmt.Errorf("verify cert: %w", err)
-	}
-
-	m.result.RemoteCert = verified
-	m.remoteCertSet = true
-	return nil
-}
-
-func (m *Machine) marshalOutgoing(flags msgFlags) ([]byte, error) {
-	if !flags.expectsPayload && !flags.expectsCert {
-		return nil, nil
-	}
-
-	var p Payload
-	if flags.expectsPayload {
-		if !m.indexAllocated {
-			index, err := m.allocIndex()
-			if err != nil {
-				return nil, fmt.Errorf("%w: %w", ErrIndexAllocation, err)
-			}
-			m.result.LocalIndex = index
-			m.indexAllocated = true
-		}
-
-		if m.result.Initiator {
-			p.InitiatorIndex = m.result.LocalIndex
-		} else {
-			p.ResponderIndex = m.result.LocalIndex
-			p.InitiatorIndex = m.result.RemoteIndex
-		}
-		p.Time = uint64(time.Now().UnixNano())
-	}
-	if flags.expectsCert {
-		cred := m.getCred(m.myVersion)
-		if cred == nil {
-			return nil, fmt.Errorf("%w: %v", ErrNoCredential, m.myVersion)
-		}
-		p.Cert = cred.Bytes
-		p.CertVersion = uint32(cred.Cert.Version())
-		m.result.MyCert = cred.Cert
-	}
-
-	return MarshalPayload(nil, p), nil
-}
-
-func (m *Machine) buildResponse(out []byte) ([]byte, *noise.CipherState, *noise.CipherState, error) {
-	flags := m.myMsgFlags()
-	hsBytes, err := m.marshalOutgoing(flags)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	// Extend out by header.Len to make room for the header. slices.Grow is a
-	// no-op when the cap is already sufficient (the zero-copy case where the
-	// caller passed a pre-sized buffer). header.Encode overwrites the new
-	// bytes, so they don't need to be zeroed.
-	start := len(out)
-	out = slices.Grow(out, header.Len)[:start+header.Len]
-	header.Encode(
-		out[start:],
-		header.Version, header.Handshake, m.subtype,
-		m.result.RemoteIndex,
-		uint64(m.hs.MessageIndex()+1),
-	)
-
-	// noise.WriteMessage appends the encrypted handshake message to out,
-	// reusing capacity when present.
-	//
-	// The (dKey, eKey) ordering here is correct for IX, where the responder
-	// completes the handshake by writing the stage-2 message. noise returns
-	// (cs1, cs2) where cs1 is the initiator->responder cipher (which is the
-	// responder's decrypt key). For 3-message patterns where an initiator
-	// finishes by writing the final message, this ordering would be wrong;
-	// revisit when XX/pqIX lands.
-	out, dKey, eKey, err := m.hs.WriteMessage(out, hsBytes)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("noise WriteMessage: %w", err)
-	}
-
-	return out, dKey, eKey, nil
-}

handshake/machine_test.go

@@ -1,662 +0,0 @@
-package handshake
-
-import (
-	"net/netip"
-	"testing"
-	"time"
-
-	"github.com/flynn/noise"
-	"github.com/slackhq/nebula/cert"
-	ct "github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/noiseutil"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestMachineIXHappyPath(t *testing.T) {
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-
-	initCS := newTestCertState(t, ca, caKey, "initiator", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	respCS := newTestCertState(t, ca, caKey, "responder", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")})
-
-	initR, respR := doFullHandshake(t, initCS, respCS, caPool)
-
-	assert.Equal(t, "responder", initR.RemoteCert.Certificate.Name())
-	assert.Equal(t, "initiator", respR.RemoteCert.Certificate.Name())
-
-	assert.Equal(t, uint32(1000), initR.LocalIndex)
-	assert.Equal(t, uint32(2000), initR.RemoteIndex)
-	assert.Equal(t, uint32(2000), respR.LocalIndex)
-	assert.Equal(t, uint32(1000), respR.RemoteIndex)
-
-	assert.Equal(t, uint64(2), initR.MessageIndex, "IX has 2 messages")
-	assert.Equal(t, uint64(2), respR.MessageIndex, "IX has 2 messages")
-
-	ct1, err := initR.EKey.Encrypt(nil, nil, []byte("hello"))
-	require.NoError(t, err)
-	pt1, err := respR.DKey.Decrypt(nil, nil, ct1)
-	require.NoError(t, err)
-	assert.Equal(t, []byte("hello"), pt1)
-
-	ct2, err := respR.EKey.Encrypt(nil, nil, []byte("world"))
-	require.NoError(t, err)
-	pt2, err := initR.DKey.Decrypt(nil, nil, ct2)
-	require.NoError(t, err)
-	assert.Equal(t, []byte("world"), pt2)
-}
-
-func TestMachineInitiateErrors(t *testing.T) {
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-	cs := newTestCertState(t, ca, caKey, "test", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	v := testVerifier(caPool)
-
-	t.Run("initiate on responder", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		_, err := m.Initiate(nil)
-		require.ErrorIs(t, err, ErrInitiateOnResponder)
-		assert.True(t, m.Failed())
-	})
-
-	t.Run("initiate called twice", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, true, 100)
-		_, err := m.Initiate(nil)
-		require.NoError(t, err)
-		_, err = m.Initiate(nil)
-		require.ErrorIs(t, err, ErrInitiateAlreadyCalled)
-		assert.True(t, m.Failed())
-	})
-
-	t.Run("process packet before initiate on initiator", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, true, 100)
-		_, _, err := m.ProcessPacket(nil, make([]byte, 100))
-		require.ErrorIs(t, err, ErrInitiateNotCalled)
-		assert.True(t, m.Failed())
-	})
-
-	t.Run("calling failed machine", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		_, err := m.Initiate(nil) // fails: responder
-		require.Error(t, err)
-		_, err = m.Initiate(nil) // fails: already failed
-		require.ErrorIs(t, err, ErrMachineFailed)
-	})
-}
-
-func TestMachineProcessPacketErrors(t *testing.T) {
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-	cs := newTestCertState(t, ca, caKey, "test", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	v := testVerifier(caPool)
-
-	t.Run("packet too short", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		_, _, err := m.ProcessPacket(nil, []byte{1, 2, 3})
-		require.ErrorIs(t, err, ErrPacketTooShort)
-		assert.False(t, m.Failed(), "short packet should not kill machine")
-	})
-
-	t.Run("noise decryption failure is recoverable", func(t *testing.T) {
-		initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-		initM := newTestMachine(t, initCS, v, true, 100)
-		msg1, err := initM.Initiate(nil)
-		require.NoError(t, err)
-
-		respM := newTestMachine(t, cs, v, false, 200)
-		resp, _, err := respM.ProcessPacket(nil, msg1)
-		require.NoError(t, err)
-
-		corrupted := make([]byte, len(resp))
-		copy(corrupted, resp)
-		for i := header.Len; i < len(corrupted); i++ {
-			corrupted[i] ^= 0xff
-		}
-		_, _, err = initM.ProcessPacket(nil, corrupted)
-		require.Error(t, err)
-		assert.False(t, initM.Failed(), "noise failure should be recoverable")
-
-		// And the machine should still complete a real handshake afterward.
-		_, result, err := initM.ProcessPacket(nil, resp)
-		require.NoError(t, err)
-		require.NotNil(t, result, "initiator should complete on the legitimate response")
-	})
-
-	t.Run("invalid cert is fatal", func(t *testing.T) {
-		otherCA, _, otherCAKey, _ := ct.NewTestCaCert(
-			cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-		)
-		otherCS := newTestCertState(t, otherCA, otherCAKey, "other", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")})
-
-		initM := newTestMachine(t, otherCS, testVerifier(ct.NewTestCAPool(otherCA)), true, 100)
-		msg1, err := initM.Initiate(nil)
-		require.NoError(t, err)
-
-		respM := newTestMachine(t, cs, v, false, 200)
-		_, _, err = respM.ProcessPacket(nil, msg1)
-		require.Error(t, err)
-		assert.True(t, respM.Failed(), "cert validation failure should kill machine")
-	})
-
-	t.Run("subtype mismatch is recoverable", func(t *testing.T) {
-		initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-		initM := newTestMachine(t, initCS, v, true, 100)
-		msg1, err := initM.Initiate(nil)
-		require.NoError(t, err)
-
-		// Mutate the subtype byte (offset 1 in the header) to a value the
-		// responder Machine wasn't built for.
-		bad := make([]byte, len(msg1))
-		copy(bad, msg1)
-		bad[1] = 0xff
-
-		respM := newTestMachine(t, cs, v, false, 200)
-		_, _, err = respM.ProcessPacket(nil, bad)
-		require.ErrorIs(t, err, ErrSubtypeMismatch)
-		assert.False(t, respM.Failed(), "subtype mismatch should not kill the machine")
-
-		// And the machine should still complete a real handshake afterward.
-		resp, result, err := respM.ProcessPacket(nil, msg1)
-		require.NoError(t, err)
-		require.NotNil(t, result, "responder should complete on the legitimate stage-1 packet")
-		assert.NotEmpty(t, resp, "responder should produce a stage-2 reply")
-	})
-}
-
-// TestMachineProcessPayload exercises processPayload's internal validation
-// directly. Most of these failure modes can't be reached black-box once the
-// subtype check at the top of ProcessPacket gates external callers, so we
-// drive them by hand here for coverage.
-func TestMachineProcessPayload(t *testing.T) {
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-	cs := newTestCertState(t, ca, caKey, "test", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	v := testVerifier(caPool)
-
-	t.Run("empty message with expects fails", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		err := m.processPayload(nil, msgFlags{expectsPayload: true, expectsCert: true})
-		require.ErrorIs(t, err, ErrMissingContent)
-		assert.True(t, m.Failed())
-	})
-
-	t.Run("empty message with no expects passes", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		err := m.processPayload(nil, msgFlags{})
-		require.NoError(t, err)
-		assert.False(t, m.Failed())
-	})
-
-	t.Run("malformed protobuf is fatal", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		err := m.processPayload([]byte{0xff, 0xff, 0xff}, msgFlags{expectsPayload: true, expectsCert: true})
-		require.Error(t, err)
-		assert.True(t, m.Failed())
-	})
-
-	t.Run("unexpected payload data is fatal", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		// A payload with index data when none was expected.
-		bytes := MarshalPayload(nil, Payload{InitiatorIndex: 42, Time: 1})
-		err := m.processPayload(bytes, msgFlags{expectsPayload: false, expectsCert: false})
-		require.ErrorIs(t, err, ErrUnexpectedContent)
-		assert.True(t, m.Failed())
-	})
-
-	t.Run("unexpected cert data is fatal", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		// A payload with cert when none was expected.
-		bytes := MarshalPayload(nil, Payload{Cert: []byte{1, 2, 3}, CertVersion: 2})
-		err := m.processPayload(bytes, msgFlags{expectsPayload: false, expectsCert: false})
-		require.ErrorIs(t, err, ErrUnexpectedContent)
-		assert.True(t, m.Failed())
-	})
-
-	t.Run("missing payload data when expected is fatal", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		// Cert present, but no index/time fields.
-		bytes := MarshalPayload(nil, Payload{Cert: []byte{1, 2, 3}, CertVersion: 2})
-		err := m.processPayload(bytes, msgFlags{expectsPayload: true, expectsCert: true})
-		require.ErrorIs(t, err, ErrUnexpectedContent)
-		assert.True(t, m.Failed())
-	})
-}
-
-// TestMachineRequireComplete checks the fail-on-incomplete-handshake path
-// directly. Like processPayload above this isn't reachable from a normal IX
-// flow, so we drive it by hand.
-func TestMachineRequireComplete(t *testing.T) {
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-	cs := newTestCertState(t, ca, caKey, "test", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	v := testVerifier(caPool)
-
-	t.Run("missing both fails", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		err := m.requireComplete()
-		require.ErrorIs(t, err, ErrIncompleteHandshake)
-		assert.True(t, m.Failed())
-	})
-
-	t.Run("payload only fails", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		m.payloadSet = true
-		err := m.requireComplete()
-		require.ErrorIs(t, err, ErrIncompleteHandshake)
-		assert.True(t, m.Failed())
-	})
-
-	t.Run("cert only fails", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		m.remoteCertSet = true
-		err := m.requireComplete()
-		require.ErrorIs(t, err, ErrIncompleteHandshake)
-		assert.True(t, m.Failed())
-	})
-
-	t.Run("both set passes", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		m.payloadSet = true
-		m.remoteCertSet = true
-		err := m.requireComplete()
-		require.NoError(t, err)
-		assert.False(t, m.Failed())
-	})
-}
-
-func TestMachineAESCipher(t *testing.T) {
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-
-	initCS := newTestCertStateWithCipher(
-		t, ca, caKey, "init",
-		[]netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")},
-		noiseutil.CipherAESGCM,
-	)
-	respCS := newTestCertStateWithCipher(
-		t, ca, caKey, "resp",
-		[]netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")},
-		noiseutil.CipherAESGCM,
-	)
-
-	initR, respR := doFullHandshake(t, initCS, respCS, caPool)
-
-	ct1, err := initR.EKey.Encrypt(nil, nil, []byte("works"))
-	require.NoError(t, err)
-	pt1, err := respR.DKey.Decrypt(nil, nil, ct1)
-	require.NoError(t, err)
-	assert.Equal(t, []byte("works"), pt1)
-
-	ct2, err := respR.EKey.Encrypt(nil, nil, []byte("back"))
-	require.NoError(t, err)
-	pt2, err := initR.DKey.Decrypt(nil, nil, ct2)
-	require.NoError(t, err)
-	assert.Equal(t, []byte("back"), pt2)
-}
-
-func TestResultFields(t *testing.T) {
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-	initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	respCS := newTestCertState(t, ca, caKey, "resp", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")})
-
-	initR, respR := doFullHandshake(t, initCS, respCS, caPool)
-
-	assert.True(t, initR.Initiator)
-	assert.False(t, respR.Initiator)
-	assert.NotZero(t, initR.HandshakeTime)
-	assert.NotZero(t, respR.HandshakeTime)
-	assert.NotNil(t, initR.RemoteCert)
-	assert.NotNil(t, respR.RemoteCert)
-}
-
-func TestMachineBufferReuse(t *testing.T) {
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-	initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	respCS := newTestCertState(t, ca, caKey, "resp", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")})
-	v := testVerifier(caPool)
-
-	initM := newTestMachine(t, initCS, v, true, 1000)
-	respM := newTestMachine(t, respCS, v, false, 2000)
-
-	msg1, err := initM.Initiate(nil)
-	require.NoError(t, err)
-
-	t.Run("response writes into provided buffer", func(t *testing.T) {
-		buf := make([]byte, 0, 4096)
-		resp, result, err := respM.ProcessPacket(buf, msg1)
-		require.NoError(t, err)
-		require.NotNil(t, result)
-
-		assert.NotEmpty(t, resp, "response should have content")
-		assert.Equal(t, &buf[:1][0], &resp[:1][0],
-			"response should reuse the provided buffer's backing array")
-	})
-
-	t.Run("initiate writes into provided buffer", func(t *testing.T) {
-		initM2 := newTestMachine(t, initCS, v, true, 3000)
-		buf := make([]byte, 0, 4096)
-		msg, err := initM2.Initiate(buf)
-		require.NoError(t, err)
-
-		assert.NotEmpty(t, msg, "initiate should have content")
-		assert.Equal(t, &buf[:1][0], &msg[:1][0],
-			"initiate should reuse the provided buffer's backing array")
-	})
-
-	t.Run("nil out still works", func(t *testing.T) {
-		initM2 := newTestMachine(t, initCS, v, true, 4000)
-		respM2 := newTestMachine(t, respCS, v, false, 5000)
-
-		msg1, err := initM2.Initiate(nil)
-		require.NoError(t, err)
-
-		resp, _, err := respM2.ProcessPacket(nil, msg1)
-		require.NoError(t, err)
-
-		out, result, err := initM2.ProcessPacket(nil, resp)
-		require.NoError(t, err)
-		assert.NotNil(t, result)
-		assert.Nil(t, out, "initiator should have no response for IX msg2")
-	})
-}
-
-func TestMachineMsgIndexTracking(t *testing.T) {
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-	initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	respCS := newTestCertState(t, ca, caKey, "resp", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")})
-	v := testVerifier(caPool)
-
-	initM := newTestMachine(t, initCS, v, true, 100)
-	respM := newTestMachine(t, respCS, v, false, 200)
-
-	msg1, err := initM.Initiate(nil)
-	require.NoError(t, err)
-
-	resp1, result1, err := respM.ProcessPacket(nil, msg1)
-	require.NoError(t, err)
-	assert.NotNil(t, result1)
-
-	_, result2, err := initM.ProcessPacket(nil, resp1)
-	require.NoError(t, err)
-	assert.NotNil(t, result2)
-}
-
-func TestMachineThreeMessagePattern(t *testing.T) {
-	registerTestXXInfo(t)
-
-	// Use HandshakeXX (3 messages) to verify the Machine handles multi-message
-	// patterns correctly. XX flow:
-	//   msg1 (I->R): [E]           - payload only, no cert
-	//   msg2 (R->I): [E, ee, S, es] - payload + cert
-	//   msg3 (I->R): [S, se]       - cert only (no payload, not first two)
-
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-	v := testVerifier(caPool)
-
-	initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	respCS := newTestCertState(t, ca, caKey, "resp", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")})
-
-	initM, err := NewMachine(
-		cert.Version2,
-		initCS.getCredential, v,
-		func() (uint32, error) { return 1000, nil },
-		true, header.HandshakeXXPSK0,
-	)
-	require.NoError(t, err)
-
-	respM, err := NewMachine(
-		cert.Version2,
-		respCS.getCredential, v,
-		func() (uint32, error) { return 2000, nil },
-		false, header.HandshakeXXPSK0,
-	)
-	require.NoError(t, err)
-
-	// msg1: initiator -> responder (E only, no cert)
-	msg1, err := initM.Initiate(nil)
-	require.NoError(t, err)
-	assert.NotEmpty(t, msg1)
-
-	// Responder processes msg1, should not complete yet, should produce msg2
-	msg2, result, err := respM.ProcessPacket(nil, msg1)
-	require.NoError(t, err)
-	assert.Nil(t, result, "XX should not complete on msg1")
-	assert.NotEmpty(t, msg2, "responder should produce msg2")
-
-	// Initiator processes msg2: gets responder's cert, produces msg3, and
-	// completes (WriteMessage for msg3 derives keys)
-	msg3, initResult, err := initM.ProcessPacket(nil, msg2)
-	require.NoError(t, err)
-	require.NotNil(t, initResult, "XX initiator should complete after reading msg2 and writing msg3")
-	assert.NotEmpty(t, msg3, "initiator should produce msg3")
-	assert.Equal(t, "resp", initResult.RemoteCert.Certificate.Name())
-
-	// Responder processes msg3: gets initiator's cert and completes
-	_, respResult, err := respM.ProcessPacket(nil, msg3)
-	require.NoError(t, err)
-	require.NotNil(t, respResult, "XX responder should complete on msg3")
-	assert.Equal(t, "init", respResult.RemoteCert.Certificate.Name())
-
-	assert.Equal(t, uint64(3), initResult.MessageIndex, "XX has 3 messages")
-	assert.Equal(t, uint64(3), respResult.MessageIndex, "XX has 3 messages")
-
-	// Verify keys work
-	ct1, err := initResult.EKey.Encrypt(nil, nil, []byte("three messages"))
-	require.NoError(t, err)
-	pt1, err := respResult.DKey.Decrypt(nil, nil, ct1)
-	require.NoError(t, err)
-	assert.Equal(t, []byte("three messages"), pt1)
-}
-
-// NOTE: ErrIncompleteHandshake is tested implicitly. It can't be triggered with
-// IX since the cert is always in the payload. A 3-message pattern test (HybridIX)
-// should exercise the case where cert arrives in msg3 and verify that completing
-// without it fails.
-
-func TestMachineExpiredCert(t *testing.T) {
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519,
-		time.Now().Add(-24*time.Hour), time.Now().Add(24*time.Hour),
-		nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-
-	expCert, _, expKeyPEM, _ := ct.NewTestCert(
-		cert.Version2, cert.Curve_CURVE25519, ca, caKey,
-		"expired", time.Now().Add(-2*time.Hour), time.Now().Add(-1*time.Hour),
-		[]netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}, nil, nil,
-	)
-	expKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(expKeyPEM)
-	require.NoError(t, err)
-	expHsBytes, err := expCert.MarshalForHandshakes()
-	require.NoError(t, err)
-	ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
-
-	expiredCS := &testCertState{
-		version: cert.Version2,
-		creds: map[cert.Version]*Credential{
-			cert.Version2: NewCredential(expCert, expHsBytes, expKey, ncs),
-		},
-	}
-
-	respCS := newTestCertState(
-		t, ca, caKey, "responder",
-		[]netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")},
-	)
-
-	_, respM, _, _, err := initiateHandshake(
-		t, expiredCS, testVerifier(caPool),
-		respCS, testVerifier(caPool),
-	)
-	require.ErrorContains(t, err, "verify cert")
-	assert.True(t, respM.Failed())
-}
-
-func TestMachineNoCertNetworks(t *testing.T) {
-	ca, _, caKey, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca)
-
-	caHsBytes, err := ca.MarshalForHandshakes()
-	require.NoError(t, err)
-	ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
-
-	noNetCS := &testCertState{
-		version: cert.Version2,
-		creds: map[cert.Version]*Credential{
-			cert.Version2: NewCredential(ca, caHsBytes, caKey, ncs),
-		},
-	}
-
-	respCS := newTestCertState(
-		t, ca, caKey, "responder",
-		[]netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")},
-	)
-
-	_, respM, _, _, err := initiateHandshake(
-		t, noNetCS, testVerifier(caPool),
-		respCS, testVerifier(caPool),
-	)
-	require.Error(t, err)
-	assert.True(t, respM.Failed())
-}
-
-func TestMachineDifferentCAs(t *testing.T) {
-	ca1, _, caKey1, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	ca2, _, caKey2, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-
-	initCS := newTestCertState(
-		t, ca1, caKey1, "init",
-		[]netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")},
-	)
-	respCS := newTestCertState(
-		t, ca2, caKey2, "resp",
-		[]netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")},
-	)
-
-	_, respM, _, _, err := initiateHandshake(
-		t, initCS, testVerifier(ct.NewTestCAPool(ca1)),
-		respCS, testVerifier(ct.NewTestCAPool(ca2)),
-	)
-	require.ErrorContains(t, err, "verify cert")
-	assert.True(t, respM.Failed())
-}
-
-func TestMachineVersionNegotiation(t *testing.T) {
-	ca1, _, caKey1, _ := ct.NewTestCaCert(
-		cert.Version1, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	ca2, _, caKey2, _ := ct.NewTestCaCert(
-		cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
-	)
-	caPool := ct.NewTestCAPool(ca1, ca2)
-
-	makeMultiVersionResp := func(t *testing.T) *testCertState {
-		t.Helper()
-		respCertV1, _, respKeyPEM, _ := ct.NewTestCert(
-			cert.Version1, cert.Curve_CURVE25519, ca1, caKey1, "resp",
-			ca1.NotBefore(), ca1.NotAfter(),
-			[]netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil, nil,
-		)
-		respKey, _, _, _ := cert.UnmarshalPrivateKeyFromPEM(respKeyPEM)
-		respCertV2, _ := ct.NewTestCertDifferentVersion(respCertV1, cert.Version2, ca2, caKey2)
-		respHsV1, _ := respCertV1.MarshalForHandshakes()
-		respHsV2, _ := respCertV2.MarshalForHandshakes()
-		ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
-		return &testCertState{
-			version: cert.Version1,
-			creds: map[cert.Version]*Credential{
-				cert.Version1: NewCredential(respCertV1, respHsV1, respKey, ncs),
-				cert.Version2: NewCredential(respCertV2, respHsV2, respKey, ncs),
-			},
-		}
-	}
-
-	t.Run("responder matches initiator version", func(t *testing.T) {
-		initCS := newTestCertState(
-			t, ca2, caKey2, "init",
-			[]netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")},
-		)
-		respCS := makeMultiVersionResp(t)
-		v := testVerifier(caPool)
-
-		initM, _, respResult, resp, err := initiateHandshake(
-			t, initCS, v,
-			respCS, v,
-		)
-		require.NoError(t, err)
-		require.NotNil(t, respResult)
-
-		assert.Equal(t, cert.Version2, respResult.MyCert.Version(),
-			"responder should negotiate to initiator's version")
-
-		_, initResult, err := initM.ProcessPacket(nil, resp)
-		require.NoError(t, err)
-		require.NotNil(t, initResult)
-		assert.Equal(t, cert.Version2, initResult.RemoteCert.Certificate.Version(),
-			"initiator should see V2 cert from responder")
-	})
-
-	t.Run("responder keeps version when no match available", func(t *testing.T) {
-		initCS := newTestCertState(
-			t, ca2, caKey2, "init",
-			[]netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")},
-		)
-
-		respCert, _, respKeyPEM, _ := ct.NewTestCert(
-			cert.Version1, cert.Curve_CURVE25519, ca1, caKey1, "resp",
-			ca1.NotBefore(), ca1.NotAfter(),
-			[]netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil, nil,
-		)
-		respKey, _, _, _ := cert.UnmarshalPrivateKeyFromPEM(respKeyPEM)
-		respHs, _ := respCert.MarshalForHandshakes()
-		ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
-		respCS := &testCertState{
-			version: cert.Version1,
-			creds: map[cert.Version]*Credential{
-				cert.Version1: NewCredential(respCert, respHs, respKey, ncs),
-			},
-		}
-
-		v := testVerifier(caPool)
-		_, _, respResult, _, err := initiateHandshake(
-			t, initCS, v,
-			respCS, v,
-		)
-		require.NoError(t, err)
-		require.NotNil(t, respResult)
-
-		assert.Equal(t, cert.Version1, respResult.MyCert.Version(),
-			"responder should keep V1 when V2 not available")
-	})
-}

handshake/patterns.go

@@ -1,54 +0,0 @@
-package handshake
-
-import (
-	"fmt"
-
-	"github.com/flynn/noise"
-	"github.com/slackhq/nebula/header"
-)
-
-// msgFlags tracks what application data a handshake message carries.
-type msgFlags struct {
-	expectsPayload bool // message carries indexes and time
-	expectsCert    bool // message carries the certificate
-}
-
-// subtypeInfo bundles the noise pattern with the per-message flags for a
-// given handshake subtype.
-type subtypeInfo struct {
-	pattern noise.HandshakePattern
-	msgs    []msgFlags
-}
-
-// subtypeInfos defines the noise pattern and message content layout for each
-// handshake subtype.
-var subtypeInfos = map[header.MessageSubType]subtypeInfo{
-	// IX: 2 messages, both carry payload and cert
-	header.HandshakeIXPSK0: {
-		pattern: noise.HandshakeIX,
-		msgs: []msgFlags{
-			{expectsPayload: true, expectsCert: true},
-			{expectsPayload: true, expectsCert: true},
-		},
-	},
-
-	// XX: 3 messages
-	//   msg1 (I->R): payload only
-	//   msg2 (R->I): payload + cert
-	//   msg3 (I->R): cert only
-	//header.HandshakeXXPSK0: {
-	//	pattern: noise.HandshakeXX,
-	//	msgs: []msgFlags{
-	//		{expectsPayload: true, expectsCert: false},
-	//		{expectsPayload: true, expectsCert: true},
-	//		{expectsPayload: false, expectsCert: true},
-	//	},
-	//},
-}
-
-func subtypeInfoFor(subtype header.MessageSubType) (subtypeInfo, error) {
-	if info, ok := subtypeInfos[subtype]; ok {
-		return info, nil
-	}
-	return subtypeInfo{}, fmt.Errorf("%w: %d", ErrUnknownSubtype, subtype)
-}

handshake/patterns_test.go

@@ -1,63 +0,0 @@
-package handshake
-
-import (
-	"testing"
-
-	"github.com/flynn/noise"
-	"github.com/slackhq/nebula/header"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSubtypeInfo(t *testing.T) {
-	t.Run("IX", func(t *testing.T) {
-		info, err := subtypeInfoFor(header.HandshakeIXPSK0)
-		require.NoError(t, err)
-		assert.Equal(t, noise.HandshakeIX.Name, info.pattern.Name)
-		require.Len(t, info.msgs, 2)
-		// msg1: payload + cert
-		assert.True(t, info.msgs[0].expectsPayload)
-		assert.True(t, info.msgs[0].expectsCert)
-		// msg2: payload + cert
-		assert.True(t, info.msgs[1].expectsPayload)
-		assert.True(t, info.msgs[1].expectsCert)
-	})
-
-	t.Run("XX", func(t *testing.T) {
-		registerTestXXInfo(t)
-		info, err := subtypeInfoFor(header.HandshakeXXPSK0)
-		require.NoError(t, err)
-		assert.Equal(t, noise.HandshakeXX.Name, info.pattern.Name)
-		require.Len(t, info.msgs, 3)
-		// msg1: payload only
-		assert.True(t, info.msgs[0].expectsPayload)
-		assert.False(t, info.msgs[0].expectsCert)
-		// msg2: payload + cert
-		assert.True(t, info.msgs[1].expectsPayload)
-		assert.True(t, info.msgs[1].expectsCert)
-		// msg3: cert only
-		assert.False(t, info.msgs[2].expectsPayload)
-		assert.True(t, info.msgs[2].expectsCert)
-	})
-
-	t.Run("unknown subtype returns error", func(t *testing.T) {
-		_, err := subtypeInfoFor(99)
-		require.ErrorIs(t, err, ErrUnknownSubtype)
-	})
-}
-
-// registerTestXXInfo temporarily registers XX subtype info for testing.
-func registerTestXXInfo(t *testing.T) {
-	t.Helper()
-	subtypeInfos[header.HandshakeXXPSK0] = subtypeInfo{
-		pattern: noise.HandshakeXX,
-		msgs: []msgFlags{
-			{expectsPayload: true, expectsCert: false},
-			{expectsPayload: true, expectsCert: true},
-			{expectsPayload: false, expectsCert: true},
-		},
-	}
-	t.Cleanup(func() {
-		delete(subtypeInfos, header.HandshakeXXPSK0)
-	})
-}

handshake/payload.go

@@ -1,173 +0,0 @@
-package handshake
-
-import (
-	"errors"
-	"math"
-
-	"google.golang.org/protobuf/encoding/protowire"
-)
-
-var (
-	errInvalidHandshakeMessage = errors.New("invalid handshake message")
-	errInvalidHandshakeDetails = errors.New("invalid handshake details")
-)
-
-// Payload represents the decoded fields of a handshake message.
-// Wire format is protobuf-compatible with NebulaHandshake{Details: NebulaHandshakeDetails{...}}.
-type Payload struct {
-	Cert           []byte
-	InitiatorIndex uint32
-	ResponderIndex uint32
-	Time           uint64
-	CertVersion    uint32
-}
-
-// Proto field numbers for NebulaHandshakeDetails
-const (
-	fieldCert           = 1 // bytes
-	fieldInitiatorIndex = 2 // uint32
-	fieldResponderIndex = 3 // uint32
-	fieldTime           = 5 // uint64
-	fieldCertVersion    = 8 // uint32
-)
-
-// MarshalPayload encodes a handshake payload in protobuf wire format compatible
-// with NebulaHandshake{Details: NebulaHandshakeDetails{...}}.
-// Returns out (which may be nil), with the marshalled Payload appended to it.
-func MarshalPayload(out []byte, p Payload) []byte {
-	var details []byte
-
-	if len(p.Cert) > 0 {
-		details = protowire.AppendTag(details, fieldCert, protowire.BytesType)
-		details = protowire.AppendBytes(details, p.Cert)
-	}
-	if p.InitiatorIndex != 0 {
-		details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType)
-		details = protowire.AppendVarint(details, uint64(p.InitiatorIndex))
-	}
-	if p.ResponderIndex != 0 {
-		details = protowire.AppendTag(details, fieldResponderIndex, protowire.VarintType)
-		details = protowire.AppendVarint(details, uint64(p.ResponderIndex))
-	}
-	if p.Time != 0 {
-		details = protowire.AppendTag(details, fieldTime, protowire.VarintType)
-		details = protowire.AppendVarint(details, p.Time)
-	}
-	if p.CertVersion != 0 {
-		details = protowire.AppendTag(details, fieldCertVersion, protowire.VarintType)
-		details = protowire.AppendVarint(details, uint64(p.CertVersion))
-	}
-
-	out = protowire.AppendTag(out, 1, protowire.BytesType)
-	out = protowire.AppendBytes(out, details)
-
-	return out
-}
-
-// UnmarshalPayload decodes a protobuf-encoded NebulaHandshake message.
-func UnmarshalPayload(b []byte) (Payload, error) {
-	var p Payload
-
-	for len(b) > 0 {
-		num, typ, n := protowire.ConsumeTag(b)
-		if n < 0 {
-			return p, errInvalidHandshakeMessage
-		}
-		b = b[n:]
-
-		switch {
-		case num == 1 && typ == protowire.BytesType:
-			details, n := protowire.ConsumeBytes(b)
-			if n < 0 {
-				return p, errInvalidHandshakeMessage
-			}
-			b = b[n:]
-			if err := unmarshalPayloadDetails(&p, details); err != nil {
-				return p, err
-			}
-		default:
-			n := protowire.ConsumeFieldValue(num, typ, b)
-			if n < 0 {
-				return p, errInvalidHandshakeMessage
-			}
-			b = b[n:]
-		}
-	}
-
-	return p, nil
-}
-
-func unmarshalPayloadDetails(p *Payload, b []byte) error {
-	for len(b) > 0 {
-		num, typ, n := protowire.ConsumeTag(b)
-		if n < 0 {
-			return errInvalidHandshakeDetails
-		}
-		b = b[n:]
-
-		// For known field numbers, reject any non-matching wire type as a
-		// hard error rather than silently skipping. The caller will catch
-		// missing-field cases downstream, but a wire-type mismatch on a tag
-		// we know is a peer protocol violation worth flagging here.
-		// Repeated occurrences of a singular field follow proto3 last-wins.
-		switch num {
-		case fieldCert:
-			if typ != protowire.BytesType {
-				return errInvalidHandshakeDetails
-			}
-			v, n := protowire.ConsumeBytes(b)
-			if n < 0 {
-				return errInvalidHandshakeDetails
-			}
-			p.Cert = append([]byte(nil), v...)
-			b = b[n:]
-		case fieldInitiatorIndex:
-			if typ != protowire.VarintType {
-				return errInvalidHandshakeDetails
-			}
-			v, n := protowire.ConsumeVarint(b)
-			if n < 0 || v > math.MaxUint32 {
-				return errInvalidHandshakeDetails
-			}
-			p.InitiatorIndex = uint32(v)
-			b = b[n:]
-		case fieldResponderIndex:
-			if typ != protowire.VarintType {
-				return errInvalidHandshakeDetails
-			}
-			v, n := protowire.ConsumeVarint(b)
-			if n < 0 || v > math.MaxUint32 {
-				return errInvalidHandshakeDetails
-			}
-			p.ResponderIndex = uint32(v)
-			b = b[n:]
-		case fieldTime:
-			if typ != protowire.VarintType {
-				return errInvalidHandshakeDetails
-			}
-			v, n := protowire.ConsumeVarint(b)
-			if n < 0 {
-				return errInvalidHandshakeDetails
-			}
-			p.Time = v
-			b = b[n:]
-		case fieldCertVersion:
-			if typ != protowire.VarintType {
-				return errInvalidHandshakeDetails
-			}
-			v, n := protowire.ConsumeVarint(b)
-			if n < 0 || v > math.MaxUint32 {
-				return errInvalidHandshakeDetails
-			}
-			p.CertVersion = uint32(v)
-			b = b[n:]
-		default:
-			n := protowire.ConsumeFieldValue(num, typ, b)
-			if n < 0 {
-				return errInvalidHandshakeDetails
-			}
-			b = b[n:]
-		}
-	}
-	return nil
-}

handshake/payload_test.go

@@ -1,361 +0,0 @@
-package handshake
-
-import (
-	"bytes"
-	"math"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"google.golang.org/protobuf/encoding/protowire"
-)
-
-func TestPayloadRoundTrip(t *testing.T) {
-	t.Run("all fields set", func(t *testing.T) {
-		data := MarshalPayload(nil, Payload{
-			Cert:           []byte("test-cert-bytes"),
-			CertVersion:    2,
-			InitiatorIndex: 12345,
-			ResponderIndex: 67890,
-			Time:           1234567890,
-		})
-
-		got, err := UnmarshalPayload(data)
-		require.NoError(t, err)
-
-		assert.Equal(t, []byte("test-cert-bytes"), got.Cert)
-		assert.Equal(t, uint32(12345), got.InitiatorIndex)
-		assert.Equal(t, uint32(67890), got.ResponderIndex)
-		assert.Equal(t, uint64(1234567890), got.Time)
-		assert.Equal(t, uint32(2), got.CertVersion)
-	})
-
-	t.Run("minimal fields", func(t *testing.T) {
-		data := MarshalPayload(nil, Payload{InitiatorIndex: 1})
-
-		got, err := UnmarshalPayload(data)
-		require.NoError(t, err)
-
-		assert.Equal(t, uint32(1), got.InitiatorIndex)
-		assert.Equal(t, uint32(0), got.ResponderIndex)
-		assert.Equal(t, uint64(0), got.Time)
-		assert.Nil(t, got.Cert)
-	})
-
-	t.Run("empty payload", func(t *testing.T) {
-		data := MarshalPayload(nil, Payload{})
-
-		got, err := UnmarshalPayload(data)
-		require.NoError(t, err)
-
-		assert.Equal(t, uint32(0), got.InitiatorIndex)
-	})
-
-	t.Run("large cert bytes", func(t *testing.T) {
-		bigCert := make([]byte, 4096)
-		for i := range bigCert {
-			bigCert[i] = byte(i % 256)
-		}
-
-		data := MarshalPayload(nil, Payload{
-			Cert:           bigCert,
-			CertVersion:    2,
-			InitiatorIndex: 999,
-		})
-
-		got, err := UnmarshalPayload(data)
-		require.NoError(t, err)
-
-		assert.Equal(t, bigCert, got.Cert)
-		assert.Equal(t, uint32(999), got.InitiatorIndex)
-	})
-
-	t.Run("append to existing buffer", func(t *testing.T) {
-		prefix := []byte("prefix")
-		data := MarshalPayload(prefix, Payload{InitiatorIndex: 42})
-
-		assert.Equal(t, []byte("prefix"), data[:6])
-
-		got, err := UnmarshalPayload(data[6:])
-		require.NoError(t, err)
-		assert.Equal(t, uint32(42), got.InitiatorIndex)
-	})
-}
-
-func TestPayloadUnknownFields(t *testing.T) {
-	t.Run("unknown field in outer message is skipped", func(t *testing.T) {
-		// Marshal a normal payload then append an unknown field (field 99, varint)
-		data := MarshalPayload(nil, Payload{InitiatorIndex: 42})
-		data = protowire.AppendTag(data, 99, protowire.VarintType)
-		data = protowire.AppendVarint(data, 12345)
-
-		got, err := UnmarshalPayload(data)
-		require.NoError(t, err)
-		assert.Equal(t, uint32(42), got.InitiatorIndex)
-	})
-
-	t.Run("unknown field in details is skipped", func(t *testing.T) {
-		// Build details with a known field + unknown field
-		var details []byte
-		details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType)
-		details = protowire.AppendVarint(details, 77)
-		// Unknown field 50, varint
-		details = protowire.AppendTag(details, 50, protowire.VarintType)
-		details = protowire.AppendVarint(details, 9999)
-		// Another known field after the unknown one
-		details = protowire.AppendTag(details, fieldResponderIndex, protowire.VarintType)
-		details = protowire.AppendVarint(details, 88)
-
-		// Wrap in outer message
-		var data []byte
-		data = protowire.AppendTag(data, 1, protowire.BytesType)
-		data = protowire.AppendBytes(data, details)
-
-		got, err := UnmarshalPayload(data)
-		require.NoError(t, err)
-		assert.Equal(t, uint32(77), got.InitiatorIndex)
-		assert.Equal(t, uint32(88), got.ResponderIndex)
-	})
-
-	t.Run("reserved fields 6 and 7 are skipped", func(t *testing.T) {
-		// Fields 6 and 7 are reserved in the proto definition
-		var details []byte
-		details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType)
-		details = protowire.AppendVarint(details, 100)
-		details = protowire.AppendTag(details, 6, protowire.VarintType)
-		details = protowire.AppendVarint(details, 1)
-		details = protowire.AppendTag(details, 7, protowire.VarintType)
-		details = protowire.AppendVarint(details, 2)
-
-		var data []byte
-		data = protowire.AppendTag(data, 1, protowire.BytesType)
-		data = protowire.AppendBytes(data, details)
-
-		got, err := UnmarshalPayload(data)
-		require.NoError(t, err)
-		assert.Equal(t, uint32(100), got.InitiatorIndex)
-	})
-}
-
-func TestPayloadBytesConsumed(t *testing.T) {
-	t.Run("all bytes consumed on valid input", func(t *testing.T) {
-		original := Payload{
-			Cert:           []byte("cert"),
-			CertVersion:    2,
-			InitiatorIndex: 100,
-			ResponderIndex: 200,
-			Time:           999,
-		}
-		data := MarshalPayload(nil, original)
-
-		got, err := UnmarshalPayload(data)
-		require.NoError(t, err)
-
-		// Re-marshal and compare — proves we consumed and reproduced all fields
-		remarshaled := MarshalPayload(nil, got)
-		assert.Equal(t, data, remarshaled)
-	})
-}
-
-// wrapDetails wraps raw detail bytes in the outer NebulaHandshake envelope
-// so UnmarshalPayload can reach unmarshalPayloadDetails.
-func wrapDetails(details []byte) []byte {
-	var out []byte
-	out = protowire.AppendTag(out, 1, protowire.BytesType)
-	out = protowire.AppendBytes(out, details)
-	return out
-}
-
-func TestPayloadUnmarshalErrors(t *testing.T) {
-	t.Run("nil input", func(t *testing.T) {
-		got, err := UnmarshalPayload(nil)
-		require.NoError(t, err)
-		assert.Equal(t, uint32(0), got.InitiatorIndex)
-	})
-
-	t.Run("truncated outer tag", func(t *testing.T) {
-		_, err := UnmarshalPayload([]byte{0x80})
-		assert.Error(t, err)
-	})
-
-	t.Run("truncated outer details field", func(t *testing.T) {
-		_, err := UnmarshalPayload([]byte{0x0a, 0x64, 0x01, 0x02, 0x03, 0x04, 0x05})
-		assert.Error(t, err)
-	})
-
-	t.Run("truncated outer unknown field", func(t *testing.T) {
-		// Valid tag for unknown field 99 varint, but no value follows
-		var data []byte
-		data = protowire.AppendTag(data, 99, protowire.VarintType)
-		_, err := UnmarshalPayload(data)
-		assert.Error(t, err)
-	})
-
-	t.Run("truncated details tag", func(t *testing.T) {
-		_, err := UnmarshalPayload(wrapDetails([]byte{0x80}))
-		assert.Error(t, err)
-	})
-
-	t.Run("truncated cert bytes", func(t *testing.T) {
-		// Field 1 (cert), bytes type, length 10 but only 2 bytes
-		var details []byte
-		details = protowire.AppendTag(details, fieldCert, protowire.BytesType)
-		details = append(details, 0x0a, 0x01, 0x02) // length 10, only 2 bytes
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-	t.Run("truncated initiator index varint", func(t *testing.T) {
-		var details []byte
-		details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType)
-		details = append(details, 0x80) // incomplete varint
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-	t.Run("truncated responder index varint", func(t *testing.T) {
-		var details []byte
-		details = protowire.AppendTag(details, fieldResponderIndex, protowire.VarintType)
-		details = append(details, 0x80)
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-	t.Run("truncated time varint", func(t *testing.T) {
-		var details []byte
-		details = protowire.AppendTag(details, fieldTime, protowire.VarintType)
-		details = append(details, 0x80)
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-	t.Run("truncated cert version varint", func(t *testing.T) {
-		var details []byte
-		details = protowire.AppendTag(details, fieldCertVersion, protowire.VarintType)
-		details = append(details, 0x80)
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-	t.Run("truncated unknown field in details", func(t *testing.T) {
-		var details []byte
-		details = protowire.AppendTag(details, 50, protowire.VarintType)
-		details = append(details, 0x80) // incomplete varint
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-	t.Run("cert with wrong wire type rejected", func(t *testing.T) {
-		// fieldCert as Varint instead of Bytes.
-		var details []byte
-		details = protowire.AppendTag(details, fieldCert, protowire.VarintType)
-		details = protowire.AppendVarint(details, 42)
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-	t.Run("initiator index with wrong wire type rejected", func(t *testing.T) {
-		// fieldInitiatorIndex as Bytes instead of Varint.
-		var details []byte
-		details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.BytesType)
-		details = protowire.AppendBytes(details, []byte{1, 2, 3})
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-	t.Run("time with wrong wire type rejected", func(t *testing.T) {
-		var details []byte
-		details = protowire.AppendTag(details, fieldTime, protowire.BytesType)
-		details = protowire.AppendBytes(details, []byte{1, 2, 3})
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-	t.Run("cert version with wrong wire type rejected", func(t *testing.T) {
-		var details []byte
-		details = protowire.AppendTag(details, fieldCertVersion, protowire.BytesType)
-		details = protowire.AppendBytes(details, []byte{1, 2, 3})
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-	t.Run("repeated singular field follows proto3 last-wins", func(t *testing.T) {
-		// Per proto3, multiple instances of a singular field are accepted and
-		// the last value wins. We keep this behavior so that peers using
-		// alternative encoders aren't rejected.
-		var details []byte
-		details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType)
-		details = protowire.AppendVarint(details, 1)
-		details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType)
-		details = protowire.AppendVarint(details, 42)
-		got, err := UnmarshalPayload(wrapDetails(details))
-		require.NoError(t, err)
-		assert.Equal(t, uint32(42), got.InitiatorIndex)
-	})
-
-	t.Run("initiator index varint overflow rejected", func(t *testing.T) {
-		var details []byte
-		details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType)
-		details = protowire.AppendVarint(details, math.MaxUint32+1)
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-	t.Run("cert version varint overflow rejected", func(t *testing.T) {
-		var details []byte
-		details = protowire.AppendTag(details, fieldCertVersion, protowire.VarintType)
-		details = protowire.AppendVarint(details, math.MaxUint32+1)
-		_, err := UnmarshalPayload(wrapDetails(details))
-		assert.Error(t, err)
-	})
-
-}
-
-// FuzzPayload feeds arbitrary bytes through UnmarshalPayload to confirm it
-// never panics, and for any input that parses cleanly, that re-marshal +
-// re-parse is a fix-point. Inputs come from an authenticated peer (post-
-// noise-decrypt), so the threat model is "valid peer behaving arbitrarily,"
-// not "unauthenticated injection."
-func FuzzPayload(f *testing.F) {
-	// Seed corpus with a handful of known-good shapes.
-	f.Add(MarshalPayload(nil, Payload{}))
-	f.Add(MarshalPayload(nil, Payload{Cert: []byte{1, 2, 3}, CertVersion: 2}))
-	f.Add(MarshalPayload(nil, Payload{InitiatorIndex: 42, Time: 1}))
-	f.Add(MarshalPayload(nil, Payload{
-		Cert:           []byte("seed-cert"),
-		InitiatorIndex: 1,
-		ResponderIndex: 2,
-		Time:           3,
-		CertVersion:    2,
-	}))
-	f.Add([]byte{})
-	f.Add([]byte{0xff})
-
-	f.Fuzz(func(t *testing.T, data []byte) {
-		p1, err := UnmarshalPayload(data)
-		if err != nil {
-			return
-		}
-
-		// For any input that parses, re-marshaling and re-parsing must
-		// yield an equivalent Payload. This catches dispatch bugs (e.g.
-		// emitting a field on marshal that we don't accept on parse) and
-		// any non-idempotent parsing behavior.
-		b2 := MarshalPayload(nil, p1)
-		p2, err := UnmarshalPayload(b2)
-		if err != nil {
-			t.Fatalf("re-parse of self-marshaled payload failed: %v\nintermediate: %x\n", err, b2)
-		}
-		if !payloadsEqual(p1, p2) {
-			t.Fatalf("re-marshal not idempotent\nfirst:  %+v\nsecond: %+v", p1, p2)
-		}
-	})
-}
-
-func payloadsEqual(a, b Payload) bool {
-	return bytes.Equal(a.Cert, b.Cert) &&
-		a.InitiatorIndex == b.InitiatorIndex &&
-		a.ResponderIndex == b.ResponderIndex &&
-		a.Time == b.Time &&
-		a.CertVersion == b.CertVersion
-}

handshake_ix.go

@@ -0,0 +1,678 @@
+package nebula
+
+import (
+	"bytes"
+	"net/netip"
+	"time"
+
+	"github.com/flynn/noise"
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/header"
+)
+
+// NOISE IX Handshakes
+
+// This function constructs a handshake packet, but does not actually send it
+// Sending is done by the handshake manager
+func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
+	err := f.handshakeManager.allocateIndex(hh)
+	if err != nil {
+		f.l.WithError(err).WithField("vpnAddrs", hh.hostinfo.vpnAddrs).
+			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to generate index")
+		return false
+	}
+
+	cs := f.pki.getCertState()
+	v := cs.initiatingVersion
+	if hh.initiatingVersionOverride != cert.VersionPre1 {
+		v = hh.initiatingVersionOverride
+	} else if v < cert.Version2 {
+		// If we're connecting to a v6 address we should encourage use of a V2 cert
+		for _, a := range hh.hostinfo.vpnAddrs {
+			if a.Is6() {
+				v = cert.Version2
+				break
+			}
+		}
+	}
+
+	crt := cs.getCertificate(v)
+	if crt == nil {
+		f.l.WithField("vpnAddrs", hh.hostinfo.vpnAddrs).
+			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).
+			WithField("certVersion", v).
+			Error("Unable to handshake with host because no certificate is available")
+		return false
+	}
+
+	crtHs := cs.getHandshakeBytes(v)
+	if crtHs == nil {
+		f.l.WithField("vpnAddrs", hh.hostinfo.vpnAddrs).
+			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).
+			WithField("certVersion", v).
+			Error("Unable to handshake with host because no certificate handshake bytes is available")
+		return false
+	}
+
+	ci, err := NewConnectionState(f.l, cs, crt, true, noise.HandshakeIX)
+	if err != nil {
+		f.l.WithError(err).WithField("vpnAddrs", hh.hostinfo.vpnAddrs).
+			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).
+			WithField("certVersion", v).
+			Error("Failed to create connection state")
+		return false
+	}
+	hh.hostinfo.ConnectionState = ci
+
+	hs := &NebulaHandshake{
+		Details: &NebulaHandshakeDetails{
+			InitiatorIndex: hh.hostinfo.localIndexId,
+			Time:           uint64(time.Now().UnixNano()),
+			Cert:           crtHs,
+			CertVersion:    uint32(v),
+		},
+	}
+
+	hsBytes, err := hs.Marshal()
+	if err != nil {
+		f.l.WithError(err).WithField("vpnAddrs", hh.hostinfo.vpnAddrs).
+			WithField("certVersion", v).
+			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to marshal handshake message")
+		return false
+	}
+
+	h := header.Encode(make([]byte, header.Len), header.Version, header.Handshake, header.HandshakeIXPSK0, 0, 1)
+
+	msg, _, _, err := ci.H.WriteMessage(h, hsBytes)
+	if err != nil {
+		f.l.WithError(err).WithField("vpnAddrs", hh.hostinfo.vpnAddrs).
+			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage")
+		return false
+	}
+
+	// We are sending handshake packet 1, so we don't expect to receive
+	// handshake packet 1 from the responder
+	ci.window.Update(f.l, 1)
+
+	hh.hostinfo.HandshakePacket[0] = msg
+	hh.ready = true
+	return true
+}
+
+func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H) {
+	cs := f.pki.getCertState()
+	crt := cs.GetDefaultCertificate()
+	if crt == nil {
+		f.l.WithField("from", via).
+			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).
+			WithField("certVersion", cs.initiatingVersion).
+			Error("Unable to handshake with host because no certificate is available")
+		return
+	}
+
+	ci, err := NewConnectionState(f.l, cs, crt, false, noise.HandshakeIX)
+	if err != nil {
+		f.l.WithError(err).WithField("from", via).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			Error("Failed to create connection state")
+		return
+	}
+
+	// Mark packet 1 as seen so it doesn't show up as missed
+	ci.window.Update(f.l, 1)
+
+	msg, _, _, err := ci.H.ReadMessage(nil, packet[header.Len:])
+	if err != nil {
+		f.l.WithError(err).WithField("from", via).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			Error("Failed to call noise.ReadMessage")
+		return
+	}
+
+	hs := &NebulaHandshake{}
+	err = hs.Unmarshal(msg)
+	if err != nil || hs.Details == nil {
+		f.l.WithError(err).WithField("from", via).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			Error("Failed unmarshal handshake message")
+		return
+	}
+
+	rc, err := cert.Recombine(cert.Version(hs.Details.CertVersion), hs.Details.Cert, ci.H.PeerStatic(), ci.Curve())
+	if err != nil {
+		f.l.WithError(err).WithField("from", via).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			Info("Handshake did not contain a certificate")
+		return
+	}
+
+	remoteCert, err := f.pki.GetCAPool().VerifyCertificate(time.Now(), rc)
+	if err != nil {
+		fp, fperr := rc.Fingerprint()
+		if fperr != nil {
+			fp = "<error generating certificate fingerprint>"
+		}
+
+		e := f.l.WithError(err).WithField("from", via).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			WithField("certVpnNetworks", rc.Networks()).
+			WithField("certFingerprint", fp)
+
+		if f.l.Level >= logrus.DebugLevel {
+			e = e.WithField("cert", rc)
+		}
+
+		e.Info("Invalid certificate from host")
+		return
+	}
+
+	if !bytes.Equal(remoteCert.Certificate.PublicKey(), ci.H.PeerStatic()) {
+		f.l.WithField("from", via).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			WithField("cert", remoteCert).Info("public key mismatch between certificate and handshake")
+		return
+	}
+
+	if remoteCert.Certificate.Version() != ci.myCert.Version() {
+		// We started off using the wrong certificate version, lets see if we can match the version that was sent to us
+		myCertOtherVersion := cs.getCertificate(remoteCert.Certificate.Version())
+		if myCertOtherVersion == nil {
+			if f.l.Level >= logrus.DebugLevel {
+				f.l.WithError(err).WithFields(m{
+					"from":      via,
+					"handshake": m{"stage": 1, "style": "ix_psk0"},
+					"cert":      remoteCert,
+				}).Debug("Might be unable to handshake with host due to missing certificate version")
+			}
+		} else {
+			// Record the certificate we are actually using
+			ci.myCert = myCertOtherVersion
+		}
+	}
+
+	if len(remoteCert.Certificate.Networks()) == 0 {
+		f.l.WithError(err).WithField("from", via).
+			WithField("cert", remoteCert).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			Info("No networks in certificate")
+		return
+	}
+
+	certName := remoteCert.Certificate.Name()
+	certVersion := remoteCert.Certificate.Version()
+	fingerprint := remoteCert.Fingerprint
+	issuer := remoteCert.Certificate.Issuer()
+	vpnNetworks := remoteCert.Certificate.Networks()
+
+	anyVpnAddrsInCommon := false
+	vpnAddrs := make([]netip.Addr, len(vpnNetworks))
+	for i, network := range vpnNetworks {
+		if f.myVpnAddrsTable.Contains(network.Addr()) {
+			f.l.WithField("vpnNetworks", vpnNetworks).WithField("from", via).
+				WithField("certName", certName).
+				WithField("certVersion", certVersion).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Refusing to handshake with myself")
+			return
+		}
+		vpnAddrs[i] = network.Addr()
+		if f.myVpnNetworksTable.Contains(network.Addr()) {
+			anyVpnAddrsInCommon = true
+		}
+	}
+
+	if !via.IsRelayed {
+		// We only want to apply the remote allow list for direct tunnels here
+		if !f.lightHouse.GetRemoteAllowList().AllowAll(vpnAddrs, via.UdpAddr.Addr()) {
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+				Debug("lighthouse.remote_allow_list denied incoming handshake")
+			return
+		}
+	}
+
+	myIndex, err := generateIndex(f.l)
+	if err != nil {
+		f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to generate index")
+		return
+	}
+
+	hostinfo := &HostInfo{
+		ConnectionState:   ci,
+		localIndexId:      myIndex,
+		remoteIndexId:     hs.Details.InitiatorIndex,
+		vpnAddrs:          vpnAddrs,
+		HandshakePacket:   make(map[uint8][]byte, 0),
+		lastHandshakeTime: hs.Details.Time,
+		relayState: RelayState{
+			relays:         nil,
+			relayForByAddr: map[netip.Addr]*Relay{},
+			relayForByIdx:  map[uint32]*Relay{},
+		},
+	}
+
+	msgRxL := f.l.WithFields(m{
+		"vpnAddrs":       vpnAddrs,
+		"from":           via,
+		"certName":       certName,
+		"certVersion":    certVersion,
+		"fingerprint":    fingerprint,
+		"issuer":         issuer,
+		"initiatorIndex": hs.Details.InitiatorIndex,
+		"responderIndex": hs.Details.ResponderIndex,
+		"remoteIndex":    h.RemoteIndex,
+		"handshake":      m{"stage": 1, "style": "ix_psk0"},
+	})
+
+	if anyVpnAddrsInCommon {
+		msgRxL.Info("Handshake message received")
+	} else {
+		//todo warn if not lighthouse or relay?
+		msgRxL.Info("Handshake message received, but no vpnNetworks in common.")
+	}
+
+	hs.Details.ResponderIndex = myIndex
+	hs.Details.Cert = cs.getHandshakeBytes(ci.myCert.Version())
+	if hs.Details.Cert == nil {
+		msgRxL.WithField("myCertVersion", ci.myCert.Version()).
+			Error("Unable to handshake with host because no certificate handshake bytes is available")
+		return
+	}
+
+	hs.Details.CertVersion = uint32(ci.myCert.Version())
+	// Update the time in case their clock is way off from ours
+	hs.Details.Time = uint64(time.Now().UnixNano())
+
+	hsBytes, err := hs.Marshal()
+	if err != nil {
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to marshal handshake message")
+		return
+	}
+
+	nh := header.Encode(make([]byte, header.Len), header.Version, header.Handshake, header.HandshakeIXPSK0, hs.Details.InitiatorIndex, 2)
+	msg, dKey, eKey, err := ci.H.WriteMessage(nh, hsBytes)
+	if err != nil {
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage")
+		return
+	} else if dKey == nil || eKey == nil {
+		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Noise did not arrive at a key")
+		return
+	}
+
+	hostinfo.HandshakePacket[0] = make([]byte, len(packet[header.Len:]))
+	copy(hostinfo.HandshakePacket[0], packet[header.Len:])
+
+	// Regardless of whether you are the sender or receiver, you should arrive here
+	// and complete standing up the connection.
+	hostinfo.HandshakePacket[2] = make([]byte, len(msg))
+	copy(hostinfo.HandshakePacket[2], msg)
+
+	// We are sending handshake packet 2, so we don't expect to receive
+	// handshake packet 2 from the initiator.
+	ci.window.Update(f.l, 2)
+
+	ci.peerCert = remoteCert
+	ci.dKey = NewNebulaCipherState(dKey)
+	ci.eKey = NewNebulaCipherState(eKey)
+
+	hostinfo.remotes = f.lightHouse.QueryCache(vpnAddrs)
+	if !via.IsRelayed {
+		hostinfo.SetRemote(via.UdpAddr)
+	}
+	hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
+
+	existing, err := f.handshakeManager.CheckAndComplete(hostinfo, 0, f)
+	if err != nil {
+		switch err {
+		case ErrAlreadySeen:
+			// Update remote if preferred
+			if existing.SetRemoteIfPreferred(f.hostMap, via) {
+				// Send a test packet to ensure the other side has also switched to
+				// the preferred remote
+				f.SendMessageToVpnAddr(header.Test, header.TestRequest, vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+			}
+
+			msg = existing.HandshakePacket[2]
+			f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
+			if !via.IsRelayed {
+				err := f.outside.WriteTo(msg, via.UdpAddr)
+				if err != nil {
+					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("from", via).
+						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
+						WithError(err).Error("Failed to send handshake message")
+				} else {
+					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("from", via).
+						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
+						Info("Handshake message sent")
+				}
+				return
+			} else {
+				if via.relay == nil {
+					f.l.Error("Handshake send failed: both addr and via.relay are nil.")
+					return
+				}
+				hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
+				f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
+				f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("relay", via.relayHI.vpnAddrs[0]).
+					WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
+					Info("Handshake message sent")
+				return
+			}
+		case ErrExistingHostInfo:
+			// This means there was an existing tunnel and this handshake was older than the one we are currently based on
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+				WithField("certName", certName).
+				WithField("certVersion", certVersion).
+				WithField("oldHandshakeTime", existing.lastHandshakeTime).
+				WithField("newHandshakeTime", hostinfo.lastHandshakeTime).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+				Info("Handshake too old")
+
+			// Send a test packet to trigger an authenticated tunnel test, this should suss out any lingering tunnel issues
+			f.SendMessageToVpnAddr(header.Test, header.TestRequest, vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+			return
+		case ErrLocalIndexCollision:
+			// This means we failed to insert because of collision on localIndexId. Just let the next handshake packet retry
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+				WithField("certName", certName).
+				WithField("certVersion", certVersion).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+				WithField("localIndex", hostinfo.localIndexId).WithField("collision", existing.vpnAddrs).
+				Error("Failed to add HostInfo due to localIndex collision")
+			return
+		default:
+			// Shouldn't happen, but just in case someone adds a new error type to CheckAndComplete
+			// And we forget to update it here
+			f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+				WithField("certName", certName).
+				WithField("certVersion", certVersion).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+				Error("Failed to add HostInfo to HostMap")
+			return
+		}
+	}
+
+	// Do the send
+	f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
+	if !via.IsRelayed {
+		err = f.outside.WriteTo(msg, via.UdpAddr)
+		log := f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"})
+		if err != nil {
+			log.WithError(err).Error("Failed to send handshake")
+		} else {
+			log.Info("Handshake message sent")
+		}
+	} else {
+		if via.relay == nil {
+			f.l.Error("Handshake send failed: both addr and via.relay are nil.")
+			return
+		}
+		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
+		// I successfully received a handshake. Just in case I marked this tunnel as 'Disestablished', ensure
+		// it's correctly marked as working.
+		via.relayHI.relayState.UpdateRelayForByIdxState(via.remoteIdx, Established)
+		f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
+		f.l.WithField("vpnAddrs", vpnAddrs).WithField("relay", via.relayHI.vpnAddrs[0]).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			Info("Handshake message sent")
+	}
+
+	f.connectionManager.AddTrafficWatch(hostinfo)
+
+	hostinfo.remotes.RefreshFromHandshake(vpnAddrs)
+
+	return
+}
+
+func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packet []byte, h *header.H) bool {
+	if hh == nil {
+		// Nothing here to tear down, got a bogus stage 2 packet
+		return true
+	}
+
+	hh.Lock()
+	defer hh.Unlock()
+
+	hostinfo := hh.hostinfo
+	if !via.IsRelayed {
+		// The vpnAddr we know about is the one we tried to handshake with, use it to apply the remote allow list.
+		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
+			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).Debug("lighthouse.remote_allow_list denied incoming handshake")
+			return false
+		}
+	}
+
+	ci := hostinfo.ConnectionState
+	msg, eKey, dKey, err := ci.H.ReadMessage(nil, packet[header.Len:])
+	if err != nil {
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("header", h).
+			Error("Failed to call noise.ReadMessage")
+
+		// We don't want to tear down the connection on a bad ReadMessage because it could be an attacker trying
+		// to DOS us. Every other error condition after should to allow a possible good handshake to complete in the
+		// near future
+		return false
+	} else if dKey == nil || eKey == nil {
+		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			Error("Noise did not arrive at a key")
+
+		// This should be impossible in IX but just in case, if we get here then there is no chance to recover
+		// the handshake state machine. Tear it down
+		return true
+	}
+
+	hs := &NebulaHandshake{}
+	err = hs.Unmarshal(msg)
+	if err != nil || hs.Details == nil {
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).Error("Failed unmarshal handshake message")
+
+		// The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again
+		return true
+	}
+
+	rc, err := cert.Recombine(cert.Version(hs.Details.CertVersion), hs.Details.Cert, ci.H.PeerStatic(), ci.Curve())
+	if err != nil {
+		f.l.WithError(err).WithField("from", via).
+			WithField("vpnAddrs", hostinfo.vpnAddrs).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			Info("Handshake did not contain a certificate")
+		return true
+	}
+
+	remoteCert, err := f.pki.GetCAPool().VerifyCertificate(time.Now(), rc)
+	if err != nil {
+		fp, err := rc.Fingerprint()
+		if err != nil {
+			fp = "<error generating certificate fingerprint>"
+		}
+
+		e := f.l.WithError(err).WithField("from", via).
+			WithField("vpnAddrs", hostinfo.vpnAddrs).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			WithField("certFingerprint", fp).
+			WithField("certVpnNetworks", rc.Networks())
+
+		if f.l.Level >= logrus.DebugLevel {
+			e = e.WithField("cert", rc)
+		}
+
+		e.Info("Invalid certificate from host")
+		return true
+	}
+	if !bytes.Equal(remoteCert.Certificate.PublicKey(), ci.H.PeerStatic()) {
+		f.l.WithField("from", via).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			WithField("cert", remoteCert).Info("public key mismatch between certificate and handshake")
+		return true
+	}
+
+	if len(remoteCert.Certificate.Networks()) == 0 {
+		f.l.WithError(err).WithField("from", via).
+			WithField("vpnAddrs", hostinfo.vpnAddrs).
+			WithField("cert", remoteCert).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			Info("No networks in certificate")
+		return true
+	}
+
+	vpnNetworks := remoteCert.Certificate.Networks()
+	certName := remoteCert.Certificate.Name()
+	certVersion := remoteCert.Certificate.Version()
+	fingerprint := remoteCert.Fingerprint
+	issuer := remoteCert.Certificate.Issuer()
+
+	hostinfo.remoteIndexId = hs.Details.ResponderIndex
+	hostinfo.lastHandshakeTime = hs.Details.Time
+
+	// Store their cert and our symmetric keys
+	ci.peerCert = remoteCert
+	ci.dKey = NewNebulaCipherState(dKey)
+	ci.eKey = NewNebulaCipherState(eKey)
+
+	// Make sure the current udpAddr being used is set for responding
+	if !via.IsRelayed {
+		hostinfo.SetRemote(via.UdpAddr)
+	} else {
+		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
+	}
+
+	correctHostResponded := false
+	anyVpnAddrsInCommon := false
+	vpnAddrs := make([]netip.Addr, len(vpnNetworks))
+	for i, network := range vpnNetworks {
+		vpnAddrs[i] = network.Addr()
+		if f.myVpnNetworksTable.Contains(network.Addr()) {
+			anyVpnAddrsInCommon = true
+		}
+		if hostinfo.vpnAddrs[0] == network.Addr() {
+			// todo is it more correct to see if any of hostinfo.vpnAddrs are in the cert? it should have len==1, but one day it might not?
+			correctHostResponded = true
+		}
+	}
+
+	// Ensure the right host responded
+	if !correctHostResponded {
+		f.l.WithField("intendedVpnAddrs", hostinfo.vpnAddrs).WithField("haveVpnNetworks", vpnNetworks).
+			WithField("from", via).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			Info("Incorrect host responded to handshake")
+
+		// Release our old handshake from pending, it should not continue
+		f.handshakeManager.DeleteHostInfo(hostinfo)
+
+		// Create a new hostinfo/handshake for the intended vpn ip
+		//TODO is hostinfo.vpnAddrs[0] always the address to use?
+		f.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) {
+			// Block the current used address
+			newHH.hostinfo.remotes = hostinfo.remotes
+			newHH.hostinfo.remotes.BlockRemote(via)
+
+			f.l.WithField("blockedUdpAddrs", newHH.hostinfo.remotes.CopyBlockedRemotes()).
+				WithField("vpnNetworks", vpnNetworks).
+				WithField("remotes", newHH.hostinfo.remotes.CopyAddrs(f.hostMap.GetPreferredRanges())).
+				Info("Blocked addresses for handshakes")
+
+			// Swap the packet store to benefit the original intended recipient
+			newHH.packetStore = hh.packetStore
+			hh.packetStore = []*cachedPacket{}
+
+			// Finally, put the correct vpn addrs in the host info, tell them to close the tunnel, and return true to tear down
+			hostinfo.vpnAddrs = vpnAddrs
+			f.sendCloseTunnel(hostinfo)
+		})
+
+		return true
+	}
+
+	// Mark packet 2 as seen so it doesn't show up as missed
+	ci.window.Update(f.l, 2)
+
+	duration := time.Since(hh.startTime).Nanoseconds()
+	msgRxL := f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+		WithField("certName", certName).
+		WithField("certVersion", certVersion).
+		WithField("fingerprint", fingerprint).
+		WithField("issuer", issuer).
+		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+		WithField("durationNs", duration).
+		WithField("sentCachedPackets", len(hh.packetStore))
+	if anyVpnAddrsInCommon {
+		msgRxL.Info("Handshake message received")
+	} else {
+		//todo warn if not lighthouse or relay?
+		msgRxL.Info("Handshake message received, but no vpnNetworks in common.")
+	}
+
+	// Build up the radix for the firewall if we have subnets in the cert
+	hostinfo.vpnAddrs = vpnAddrs
+	hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
+
+	// Complete our handshake and update metrics, this will replace any existing tunnels for the vpnAddrs here
+	f.handshakeManager.Complete(hostinfo, f)
+	f.connectionManager.AddTrafficWatch(hostinfo)
+
+	if f.l.Level >= logrus.DebugLevel {
+		hostinfo.logger(f.l).Debugf("Sending %d stored packets", len(hh.packetStore))
+	}
+
+	if len(hh.packetStore) > 0 {
+		nb := make([]byte, 12, 12)
+		out := make([]byte, mtu)
+		for _, cp := range hh.packetStore {
+			cp.callback(cp.messageType, cp.messageSubType, hostinfo, cp.packet, nb, out)
+		}
+		f.cachedPacketMetrics.sent.Inc(int64(len(hh.packetStore)))
+	}
+
+	hostinfo.remotes.RefreshFromHandshake(vpnAddrs)
+	f.metricHandshakes.Update(duration)
+
+	return false
+}

handshake_manager_test.go

@@ -5,7 +5,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/test"
@@ -28,7 +27,7 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
 		initiatingVersion: cert.Version1,
 		privateKey:        []byte{},
 		v1Cert:            &dummyCert{version: cert.Version1},
-		v1Credential:      nil,
+		v1HandshakeBytes:  []byte{},
 	}
 
 	blah := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig)
@@ -80,15 +79,15 @@ func testCountTimerWheelEntries(tw *LockingTimerWheel[netip.Addr]) (c int) {
 type mockEncWriter struct {
 }
 
-func (mw *mockEncWriter) SendMessageToVpnAddr(_ header.MessageType, _ header.MessageSubType, _ netip.Addr, _ []byte, _ *WireBuffer) {
+func (mw *mockEncWriter) SendMessageToVpnAddr(_ header.MessageType, _ header.MessageSubType, _ netip.Addr, _, _, _ []byte) {
 	return
 }
 
-func (mw *mockEncWriter) SendVia(_ *HostInfo, _ *Relay, _ []byte, _ *WireBuffer) {
+func (mw *mockEncWriter) SendVia(_ *HostInfo, _ *Relay, _, _, _ []byte, _ bool) {
 	return
 }
 
-func (mw *mockEncWriter) SendMessageToHostInfo(_ header.MessageType, _ header.MessageSubType, _ *HostInfo, _ []byte, _ *WireBuffer) {
+func (mw *mockEncWriter) SendMessageToHostInfo(_ header.MessageType, _ header.MessageSubType, _ *HostInfo, _, _, _ []byte) {
 	return
 }
 
@@ -101,137 +100,3 @@ func (mw *mockEncWriter) GetHostInfo(_ netip.Addr) *HostInfo {
 func (mw *mockEncWriter) GetCertState() *CertState {
 	return &CertState{initiatingVersion: cert.Version2}
 }
-
-func TestValidatePeerCert(t *testing.T) {
-	l := test.NewLogger()
-
-	myNetwork := netip.MustParsePrefix("10.0.0.1/24")
-	myAddrTable := new(bart.Lite)
-	myAddrTable.Insert(netip.PrefixFrom(myNetwork.Addr(), myNetwork.Addr().BitLen()))
-	myNetTable := new(bart.Lite)
-	myNetTable.Insert(myNetwork.Masked())
-
-	newHM := func() *HandshakeManager {
-		hm := NewHandshakeManager(l, newHostMap(l), newTestLighthouse(), &udp.NoopConn{}, defaultHandshakeConfig)
-		hm.f = &Interface{
-			handshakeManager:   hm,
-			pki:                &PKI{},
-			l:                  l,
-			myVpnAddrsTable:    myAddrTable,
-			myVpnNetworksTable: myNetTable,
-			lightHouse:         hm.lightHouse,
-		}
-		return hm
-	}
-
-	cached := func(networks ...netip.Prefix) *cert.CachedCertificate {
-		return &cert.CachedCertificate{
-			Certificate: &dummyCert{name: "peer", networks: networks},
-		}
-	}
-
-	via := ViaSender{
-		UdpAddr:   netip.MustParseAddrPort("198.51.100.7:4242"),
-		IsRelayed: true, // skip the remote allow list (covered separately)
-	}
-
-	t.Run("addr inside our networks sets anyVpnAddrsInCommon", func(t *testing.T) {
-		hm := newHM()
-		// 10.0.0.2 falls inside our 10.0.0.0/24
-		addrs, common, ok := hm.validatePeerCert(via, cached(netip.MustParsePrefix("10.0.0.2/24")))
-		assert.True(t, ok)
-		assert.True(t, common)
-		assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.0.0.2")}, addrs)
-	})
-
-	t.Run("addr outside our networks leaves anyVpnAddrsInCommon false", func(t *testing.T) {
-		hm := newHM()
-		addrs, common, ok := hm.validatePeerCert(via, cached(netip.MustParsePrefix("192.168.1.5/24")))
-		assert.True(t, ok)
-		assert.False(t, common)
-		assert.Equal(t, []netip.Addr{netip.MustParseAddr("192.168.1.5")}, addrs)
-	})
-
-	t.Run("any matching network is enough", func(t *testing.T) {
-		hm := newHM()
-		addrs, common, ok := hm.validatePeerCert(via, cached(
-			netip.MustParsePrefix("192.168.1.5/24"),
-			netip.MustParsePrefix("10.0.0.42/24"),
-		))
-		assert.True(t, ok)
-		assert.True(t, common)
-		assert.Len(t, addrs, 2)
-	})
-
-	t.Run("self-handshake is rejected", func(t *testing.T) {
-		hm := newHM()
-		// 10.0.0.1 is in myVpnAddrsTable
-		addrs, common, ok := hm.validatePeerCert(via, cached(netip.MustParsePrefix("10.0.0.1/24")))
-		assert.False(t, ok)
-		assert.False(t, common)
-		assert.Nil(t, addrs)
-	})
-
-	t.Run("cert with no networks is rejected", func(t *testing.T) {
-		hm := newHM()
-		addrs, common, ok := hm.validatePeerCert(via, cached())
-		assert.False(t, ok)
-		assert.False(t, common)
-		assert.Nil(t, addrs)
-	})
-}
-
-func TestHandleIncomingDispatch(t *testing.T) {
-	l := test.NewLogger()
-
-	newHM := func() *HandshakeManager {
-		hm := NewHandshakeManager(l, newHostMap(l), newTestLighthouse(), &udp.NoopConn{}, defaultHandshakeConfig)
-		hm.f = &Interface{
-			handshakeManager: hm,
-			pki:              &PKI{},
-			l:                l,
-		}
-		return hm
-	}
-
-	via := ViaSender{
-		UdpAddr:   netip.MustParseAddrPort("198.51.100.7:4242"),
-		IsRelayed: true, // bypass remote allow list
-	}
-
-	// A packet body of zero length is fine for these tests: dispatch is
-	// gated on header fields, and we assert that we never reach noise/cert
-	// processing for any of the malformed shapes here.
-	pkt := make([]byte, header.Len)
-
-	t.Run("unsupported subtype dropped", func(t *testing.T) {
-		hm := newHM()
-		h := &header.H{Type: header.Handshake, Subtype: header.MessageSubType(99), MessageCounter: 1}
-		hm.HandleIncoming(via, pkt, h)
-		assert.Empty(t, hm.indexes, "no pending handshake should be created")
-	})
-
-	t.Run("stage-1 with non-zero RemoteIndex dropped", func(t *testing.T) {
-		hm := newHM()
-		h := &header.H{
-			Type:           header.Handshake,
-			Subtype:        header.HandshakeIXPSK0,
-			RemoteIndex:    0xdeadbeef,
-			MessageCounter: 1,
-		}
-		hm.HandleIncoming(via, pkt, h)
-		assert.Empty(t, hm.indexes, "spoofed stage-1 must not create a pending machine")
-	})
-
-	t.Run("continuation with no matching pending index dropped", func(t *testing.T) {
-		hm := newHM()
-		h := &header.H{
-			Type:           header.Handshake,
-			Subtype:        header.HandshakeIXPSK0,
-			RemoteIndex:    0xcafef00d,
-			MessageCounter: 2,
-		}
-		hm.HandleIncoming(via, pkt, h)
-		assert.Empty(t, hm.indexes, "orphan stage-2 must not create state")
-	})
-}

hostmap.go

@@ -1,11 +1,9 @@
 package nebula
 
 import (
-	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"log/slog"
 	"net"
 	"net/netip"
 	"slices"
@@ -15,10 +13,10 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/rcrowley/go-metrics"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/logging"
 )
 
 const defaultPromoteEvery = 1000       // Count of packets sent before we try moving a tunnel to a preferred underlay ip address
@@ -62,7 +60,7 @@ type HostMap struct {
 	RemoteIndexes   map[uint32]*HostInfo
 	Hosts           map[netip.Addr]*HostInfo
 	preferredRanges atomic.Pointer[[]netip.Prefix]
-	l               *slog.Logger
+	l               *logrus.Logger
 }
 
 // For synchronization, treat the pointed-to Relay struct as immutable. To edit the Relay
@@ -308,14 +306,14 @@ type cachedPacket struct {
 	packet         []byte
 }
 
-type packetCallback func(t header.MessageType, st header.MessageSubType, h *HostInfo, p []byte, buf *WireBuffer)
+type packetCallback func(t header.MessageType, st header.MessageSubType, h *HostInfo, p, nb, out []byte)
 
 type cachedPacketMetrics struct {
 	sent    metrics.Counter
 	dropped metrics.Counter
 }
 
-func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap {
+func NewHostMapFromConfig(l *logrus.Logger, c *config.C) *HostMap {
 	hm := newHostMap(l)
 
 	hm.reload(c, true)
@@ -323,12 +321,13 @@ func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap {
 		hm.reload(c, false)
 	})
 
-	l.Info("Main HostMap created", "preferredRanges", hm.GetPreferredRanges())
+	l.WithField("preferredRanges", hm.GetPreferredRanges()).
+		Info("Main HostMap created")
 
 	return hm
 }
 
-func newHostMap(l *slog.Logger) *HostMap {
+func newHostMap(l *logrus.Logger) *HostMap {
 	return &HostMap{
 		Indexes:       map[uint32]*HostInfo{},
 		Relays:        map[uint32]*HostInfo{},
@@ -347,10 +346,7 @@ func (hm *HostMap) reload(c *config.C, initial bool) {
 			preferredRange, err := netip.ParsePrefix(rawPreferredRange)
 
 			if err != nil {
-				hm.l.Warn("Failed to parse preferred ranges, ignoring",
-					"error", err,
-					"range", rawPreferredRanges,
-				)
+				hm.l.WithError(err).WithField("range", rawPreferredRanges).Warn("Failed to parse preferred ranges, ignoring")
 				continue
 			}
 
@@ -359,10 +355,7 @@ func (hm *HostMap) reload(c *config.C, initial bool) {
 
 		oldRanges := hm.preferredRanges.Swap(&preferredRanges)
 		if !initial {
-			hm.l.Info("preferred_ranges changed",
-				"oldPreferredRanges", *oldRanges,
-				"newPreferredRanges", preferredRanges,
-			)
+			hm.l.WithField("oldPreferredRanges", *oldRanges).WithField("newPreferredRanges", preferredRanges).Info("preferred_ranges changed")
 		}
 	}
 }
@@ -495,11 +488,10 @@ func (hm *HostMap) unlockedInnerDeleteHostInfo(hostinfo *HostInfo, addr netip.Ad
 		hm.Indexes = map[uint32]*HostInfo{}
 	}
 
-	if hm.l.Enabled(context.Background(), slog.LevelDebug) {
-		hm.l.Debug("Hostmap hostInfo deleted",
-			"hostMap", m{"mapTotalSize": len(hm.Hosts),
-				"vpnAddrs": hostinfo.vpnAddrs, "indexNumber": hostinfo.localIndexId, "remoteIndexNumber": hostinfo.remoteIndexId},
-		)
+	if hm.l.Level >= logrus.DebugLevel {
+		hm.l.WithField("hostMap", m{"mapTotalSize": len(hm.Hosts),
+			"vpnAddrs": hostinfo.vpnAddrs, "indexNumber": hostinfo.localIndexId, "remoteIndexNumber": hostinfo.remoteIndexId}).
+			Debug("Hostmap hostInfo deleted")
 	}
 
 	if isLastHostinfo {
@@ -612,9 +604,9 @@ func (hm *HostMap) queryVpnAddr(vpnIp netip.Addr, promoteIfce *Interface) *HostI
 // unlockedAddHostInfo assumes you have a write-lock and will add a hostinfo object to the hostmap Indexes and RemoteIndexes maps.
 // If an entry exists for the Hosts table (vpnIp -> hostinfo) then the provided hostinfo will be made primary
 func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
-	if f.dnsServer != nil {
+	if f.serveDns {
 		remoteCert := hostinfo.ConnectionState.peerCert
-		f.dnsServer.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
+		dnsR.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
 	}
 	for _, addr := range hostinfo.vpnAddrs {
 		hm.unlockedInnerAddHostInfo(addr, hostinfo, f)
@@ -623,11 +615,10 @@ func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
 	hm.Indexes[hostinfo.localIndexId] = hostinfo
 	hm.RemoteIndexes[hostinfo.remoteIndexId] = hostinfo
 
-	if hm.l.Enabled(context.Background(), slog.LevelDebug) {
-		hm.l.Debug("Hostmap vpnIp added",
-			"hostMap", m{"vpnAddrs": hostinfo.vpnAddrs, "mapTotalSize": len(hm.Hosts),
-				"hostinfo": m{"existing": true, "localIndexId": hostinfo.localIndexId, "vpnAddrs": hostinfo.vpnAddrs}},
-		)
+	if hm.l.Level >= logrus.DebugLevel {
+		hm.l.WithField("hostMap", m{"vpnAddrs": hostinfo.vpnAddrs, "mapTotalSize": len(hm.Hosts),
+			"hostinfo": m{"existing": true, "localIndexId": hostinfo.localIndexId, "vpnAddrs": hostinfo.vpnAddrs}}).
+			Debug("Hostmap vpnIp added")
 	}
 }
 
@@ -691,7 +682,6 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
 			}
 		}
 
-		buf := ifce.bufAlloc.Acquire()
 		i.remotes.ForEach(preferredRanges, func(addr netip.AddrPort, preferred bool) {
 			if remote.IsValid() && (!addr.IsValid() || !preferred) {
 				return
@@ -699,9 +689,8 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
 
 			// Try to send a test packet to that host, this should
 			// cause it to detect a roaming event and switch remotes
-			ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), buf)
+			ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
 		})
-		ifce.bufAlloc.Release(buf)
 	}
 
 	// Re query our lighthouses for new remotes occasionally
@@ -795,21 +784,18 @@ func (i *HostInfo) buildNetworks(myVpnNetworksTable *bart.Lite, c cert.Certifica
 	}
 }
 
-// logger returns a derived slog.Logger with per-hostinfo fields pre-bound.
-func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
+func (i *HostInfo) logger(l *logrus.Logger) *logrus.Entry {
 	if i == nil {
-		return l
+		return logrus.NewEntry(l)
 	}
 
-	li := l.With(
-		"vpnAddrs", i.vpnAddrs,
-		"localIndex", i.localIndexId,
-		"remoteIndex", i.remoteIndexId,
-	)
+	li := l.WithField("vpnAddrs", i.vpnAddrs).
+		WithField("localIndex", i.localIndexId).
+		WithField("remoteIndex", i.remoteIndexId)
 
 	if connState := i.ConnectionState; connState != nil {
 		if peerCert := connState.peerCert; peerCert != nil {
-			li = li.With("certName", peerCert.Certificate.Name())
+			li = li.WithField("certName", peerCert.Certificate.Name())
 		}
 	}
 
@@ -818,17 +804,14 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
 
 // Utility functions
 
-func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
+func localAddrs(l *logrus.Logger, allowList *LocalAllowList) []netip.Addr {
 	//FIXME: This function is pretty garbage
 	var finalAddrs []netip.Addr
 	ifaces, _ := net.Interfaces()
 	for _, i := range ifaces {
 		allow := allowList.AllowName(i.Name)
-		if l.Enabled(context.Background(), logging.LevelTrace) {
-			l.Log(context.Background(), logging.LevelTrace, "localAllowList.AllowName",
-				"interfaceName", i.Name,
-				"allow", allow,
-			)
+		if l.Level >= logrus.TraceLevel {
+			l.WithField("interfaceName", i.Name).WithField("allow", allow).Trace("localAllowList.AllowName")
 		}
 
 		if !allow {
@@ -846,8 +829,8 @@ func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
 			}
 
 			if !addr.IsValid() {
-				if l.Enabled(context.Background(), slog.LevelDebug) {
-					l.Debug("addr was invalid", "localAddr", rawAddr)
+				if l.Level >= logrus.DebugLevel {
+					l.WithField("localAddr", rawAddr).Debug("addr was invalid")
 				}
 				continue
 			}
@@ -855,11 +838,8 @@ func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
 
 			if addr.IsLoopback() == false && addr.IsLinkLocalUnicast() == false {
 				isAllowed := allowList.Allow(addr)
-				if l.Enabled(context.Background(), logging.LevelTrace) {
-					l.Log(context.Background(), logging.LevelTrace, "localAllowList.Allow",
-						"localAddr", addr,
-						"allowed", isAllowed,
-					)
+				if l.Level >= logrus.TraceLevel {
+					l.WithField("localAddr", addr).WithField("allowed", isAllowed).Trace("localAllowList.Allow")
 				}
 				if !isAllowed {
 					continue

hostmap_test.go

@@ -196,7 +196,7 @@ func TestHostMap_DeleteHostInfo(t *testing.T) {
 
 func TestHostMap_reload(t *testing.T) {
 	l := test.NewLogger()
-	c := config.NewC(test.NewLogger())
+	c := config.NewC(l)
 
 	hm := NewHostMapFromConfig(l, c)
 

inside.go

@@ -1,46 +1,41 @@
 package nebula
 
 import (
-	"context"
-	"log/slog"
 	"net/netip"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/noiseutil"
 	"github.com/slackhq/nebula/routing"
 )
 
-func (f *Interface) consumeInsidePacket(buf *WireBuffer, q int, localCache firewall.ConntrackCache) {
-	packet := buf.IPPacket()
-
-	err := newPacket(packet, false, buf.FwPacket)
+func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb []byte, batch *sendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
+	err := newPacket(packet, false, fwPacket)
 	if err != nil {
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			f.l.Debug("Error while validating outbound packet",
-				"packet", packet,
-				"error", err,
-			)
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
 		}
 		return
 	}
 
 	// Ignore local broadcast packets
 	if f.dropLocalBroadcast {
-		if f.myBroadcastAddrsTable.Contains(buf.FwPacket.RemoteAddr) {
+		if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
 			return
 		}
 	}
 
-	if f.myVpnAddrsTable.Contains(buf.FwPacket.RemoteAddr) {
+	if f.myVpnAddrsTable.Contains(fwPacket.RemoteAddr) {
 		// Immediately forward packets from self to self.
 		// This should only happen on Darwin-based and FreeBSD hosts, which
 		// routes packets from the Nebula addr to the Nebula addr through the Nebula
 		// TUN device.
 		if immediatelyForwardToSelf {
-			_, err := f.readers[q].Write(packet)
+			_, err := f.readers[q].WriteReject(packet)
 			if err != nil {
-				f.l.Error("Failed to forward to tun", "error", err)
+				f.l.WithError(err).Error("Failed to forward to tun")
 			}
 		}
 		// Otherwise, drop. On linux, we should never see these packets - Linux
@@ -49,21 +44,20 @@ func (f *Interface) consumeInsidePacket(buf *WireBuffer, q int, localCache firew
 	}
 
 	// Ignore multicast packets
-	if f.dropMulticast && buf.FwPacket.RemoteAddr.IsMulticast() {
+	if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() {
 		return
 	}
 
-	hostinfo, ready := f.getOrHandshakeConsiderRouting(buf.FwPacket, func(hh *HandshakeHostInfo) {
+	hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
 	})
 
 	if hostinfo == nil {
-		f.rejectInside(packet, buf.Out, q)
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			f.l.Debug("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks",
-				"vpnAddr", buf.FwPacket.RemoteAddr,
-				"fwPacket", buf.FwPacket,
-			)
+		f.rejectInside(packet, rejectBuf, q)
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("vpnAddr", fwPacket.RemoteAddr).
+				WithField("fwPacket", fwPacket).
+				Debugln("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks")
 		}
 		return
 	}
@@ -72,21 +66,78 @@ func (f *Interface) consumeInsidePacket(buf *WireBuffer, q int, localCache firew
 		return
 	}
 
-	dropReason := f.firewall.Drop(*buf.FwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
-		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, buf, q)
+		f.sendInsideMessage(hostinfo, packet, nb, batch, rejectBuf, q)
 
 	} else {
-		f.rejectInside(packet, buf.Out, q)
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("dropping outbound packet",
-				"fwPacket", buf.FwPacket,
-				"reason", dropReason,
-			)
+		f.rejectInside(packet, rejectBuf, q)
+		if f.l.Level >= logrus.DebugLevel {
+			hostinfo.logger(f.l).
+				WithField("fwPacket", fwPacket).
+				WithField("reason", dropReason).
+				Debugln("dropping outbound packet")
 		}
 	}
 }
 
+// sendInsideMessage encrypts a firewall-approved inside packet into the
+// caller's batch slot for later sendmmsg flush. When hostinfo.remote is not
+// valid we fall through to the relay slow path via the unbatched sendNoMetrics
+// so relay behavior is unchanged.
+func (f *Interface) sendInsideMessage(hostinfo *HostInfo, p, nb []byte, batch *sendBatch, rejectBuf []byte, q int) {
+	ci := hostinfo.ConnectionState
+	if ci.eKey == nil {
+		return
+	}
+
+	if !hostinfo.remote.IsValid() {
+		// Slow path: relay fallback. Reuse rejectBuf as the ciphertext
+		// scratch; sendNoMetrics arranges header space for SendVia.
+		f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, p, nb, rejectBuf, q)
+		return
+	}
+
+	scratch := batch.Next()
+	if scratch == nil {
+		// Batch full: bypass batching and send this packet directly so we
+		// never drop traffic on over-subscribed iterations.
+		f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, p, nb, rejectBuf, q)
+		return
+	}
+
+	if noiseutil.EncryptLockNeeded {
+		ci.writeLock.Lock()
+	}
+	c := ci.messageCounter.Add(1)
+
+	out := header.Encode(scratch, header.Version, header.Message, 0, hostinfo.remoteIndexId, c)
+	f.connectionManager.Out(hostinfo)
+
+	if hostinfo.lastRebindCount != f.rebindCount {
+		//NOTE: there is an update hole if a tunnel isn't used and exactly 256 rebinds occur before the tunnel is
+		// finally used again. This tunnel would eventually be torn down and recreated if this action didn't help.
+		f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
+		hostinfo.lastRebindCount = f.rebindCount
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).Debug("Lighthouse update triggered for punch due to rebind counter")
+		}
+	}
+
+	out, err := ci.eKey.EncryptDanger(out, out, p, c, nb)
+	if noiseutil.EncryptLockNeeded {
+		ci.writeLock.Unlock()
+	}
+	if err != nil {
+		hostinfo.logger(f.l).WithError(err).
+			WithField("udpAddr", hostinfo.remote).WithField("counter", c).
+			Error("Failed to encrypt outgoing packet")
+		return
+	}
+
+	batch.Commit(len(out), hostinfo.remote)
+}
+
 func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 	if !f.firewall.InSendReject {
 		return
@@ -97,33 +148,33 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 		return
 	}
 
-	_, err := f.readers[q].Write(out)
+	_, err := f.readers[q].WriteReject(out)
 	if err != nil {
-		f.l.Error("Failed to write to tun", "error", err)
+		f.l.WithError(err).Error("Failed to write to tun")
 	}
 }
 
-func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, scratch []byte, buf *WireBuffer, q int) {
+func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, out []byte, q int) {
 	if !f.firewall.OutSendReject {
 		return
 	}
 
-	rejectIP := iputil.CreateRejectPacket(packet, scratch)
-	if len(rejectIP) == 0 {
+	out = iputil.CreateRejectPacket(packet, out)
+	if len(out) == 0 {
 		return
 	}
 
-	if len(rejectIP) > iputil.MaxRejectPacketSize {
-		if f.l.Enabled(context.Background(), slog.LevelInfo) {
-			f.l.Info("rejectOutside: packet too big, not sending",
-				"packet", packet,
-				"outPacket", rejectIP,
-			)
+	if len(out) > iputil.MaxRejectPacketSize {
+		if f.l.GetLevel() >= logrus.InfoLevel {
+			f.l.
+				WithField("packet", packet).
+				WithField("outPacket", out).
+				Info("rejectOutside: packet too big, not sending")
 		}
 		return
 	}
 
-	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, rejectIP, buf, q)
+	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q)
 }
 
 // Handshake will attempt to initiate a tunnel with the provided vpn address. This is a no-op if the tunnel is already established or being established
@@ -190,11 +241,10 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac
 		// This would also need to interact with unsafe_route updates through reloading the config or
 		// use of the use_system_route_table option
 
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			f.l.Debug("Calculated gateway for ECMP not available, attempting other gateways",
-				"destination", destinationAddr,
-				"originalGateway", gatewayAddr,
-			)
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("destination", destinationAddr).
+				WithField("originalGateway", gatewayAddr).
+				Debugln("Calculated gateway for ECMP not available, attempting other gateways")
 		}
 
 		for i := range gateways {
@@ -216,41 +266,39 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac
 
 }
 
-func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p []byte, buf *WireBuffer) {
+func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
 	fp := &firewall.Packet{}
 	err := newPacket(p, false, fp)
 	if err != nil {
-		f.l.Warn("error while parsing outgoing packet for firewall check", "error", err)
+		f.l.Warnf("error while parsing outgoing packet for firewall check; %v", err)
 		return
 	}
 
 	// check if packet is in outbound fw rules
 	dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil)
 	if dropReason != nil {
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			f.l.Debug("dropping cached packet",
-				"fwPacket", fp,
-				"reason", dropReason,
-			)
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("fwPacket", fp).
+				WithField("reason", dropReason).
+				Debugln("dropping cached packet")
 		}
 		return
 	}
 
-	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, buf, 0)
+	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, 0)
 }
 
 // SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr.
 // This function ignores myVpnNetworksTable, and will always attempt to treat the address as a vpnAddr
-func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p []byte, buf *WireBuffer) {
+func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte) {
 	hostInfo, ready := f.handshakeManager.GetOrHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, t, st, p, f.SendMessageToHostInfo, f.cachedPacketMetrics)
 	})
 
 	if hostInfo == nil {
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			f.l.Debug("dropping SendMessageToVpnAddr, vpnAddr not in our vpn networks or in unsafe routes",
-				"vpnAddr", vpnAddr,
-			)
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("vpnAddr", vpnAddr).
+				Debugln("dropping SendMessageToVpnAddr, vpnAddr not in our vpn networks or in unsafe routes")
 		}
 		return
 	}
@@ -259,73 +307,113 @@ func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.Message
 		return
 	}
 
-	f.SendMessageToHostInfo(t, st, hostInfo, p, buf)
+	f.SendMessageToHostInfo(t, st, hostInfo, p, nb, out)
 }
 
-func (f *Interface) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hi *HostInfo, p []byte, buf *WireBuffer) {
-	f.send(t, st, hi.ConnectionState, hi, p, buf)
+func (f *Interface) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hi *HostInfo, p, nb, out []byte) {
+	f.send(t, st, hi.ConnectionState, hi, p, nb, out)
 }
 
-func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p []byte, buf *WireBuffer) {
+func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p, nb, out []byte) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, buf, 0)
+	f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, nb, out, 0)
 }
 
-func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p []byte, buf *WireBuffer) {
+func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, buf, 0)
+	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0)
 }
 
 // SendVia sends a payload through a Relay tunnel. No authentication or encryption is done
 // to the payload for the ultimate target host, making this a useful method for sending
 // handshake messages to peers through relay tunnels.
-//
-// via is the HostInfo through which the message is relayed. ad is staged into
-// the inner-payload slot of buf and then AAD-only sealed under via's key by
-// SealRelayInPlace. The sendNoMetrics relay-forward path skips this entry
-// point and calls sendViaInPlace directly because its inner ciphertext is
-// already in place from the encrypt step.
-func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad []byte, buf *WireBuffer) {
-	if header.Len+len(ad)+via.ConnectionState.eKey.Overhead() > cap(buf.Out) {
-		via.logger(f.l).Error("SendVia out buffer not large enough for relay",
-			"outCap", cap(buf.Out),
-			"payloadLen", len(ad),
-			"headerLen", header.Len,
-			"cipherOverhead", via.ConnectionState.eKey.Overhead(),
-		)
-		return
+// via is the HostInfo through which the message is relayed.
+// ad is the plaintext data to authenticate, but not encrypt
+// nb is a buffer used to store the nonce value, re-used for performance reasons.
+// out is a buffer used to store the result of the Encrypt operation
+// q indicates which writer to use to send the packet.
+func (f *Interface) SendVia(via *HostInfo,
+	relay *Relay,
+	ad,
+	nb,
+	out []byte,
+	nocopy bool,
+) {
+	if noiseutil.EncryptLockNeeded {
+		// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
+		via.ConnectionState.writeLock.Lock()
 	}
-	buf.StageRelayInner(ad)
-	f.sendViaInPlace(via, relay, len(ad), buf)
-}
+	c := via.ConnectionState.messageCounter.Add(1)
 
-// sendViaInPlace stamps the outer relay header, AAD-seals over the [outer
-// header | inner-already-staged] region, and writes the result to via.remote.
-// Called from SendVia (after staging ad) and from sendNoMetrics' relay-forward
-// path (where the inner ciphertext is already in place from SealForRelay).
-func (f *Interface) sendViaInPlace(via *HostInfo, relay *Relay, innerLen int, buf *WireBuffer) {
+	out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
 	f.connectionManager.Out(via)
-	out, err := buf.SealRelayInPlace(via.ConnectionState, relay.RemoteIndex, innerLen)
-	if err != nil {
-		via.logger(f.l).Info("Failed to EncryptDanger in sendVia", "error", err)
+
+	// Authenticate the header and payload, but do not encrypt for this message type.
+	// The payload consists of the inner, unencrypted Nebula header, as well as the end-to-end encrypted payload.
+	if len(out)+len(ad)+via.ConnectionState.eKey.Overhead() > cap(out) {
+		if noiseutil.EncryptLockNeeded {
+			via.ConnectionState.writeLock.Unlock()
+		}
+		via.logger(f.l).
+			WithField("outCap", cap(out)).
+			WithField("payloadLen", len(ad)).
+			WithField("headerLen", len(out)).
+			WithField("cipherOverhead", via.ConnectionState.eKey.Overhead()).
+			Error("SendVia out buffer not large enough for relay")
 		return
 	}
-	if err := f.writers[0].WriteTo(out, via.remote); err != nil {
-		via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err)
+
+	// The header bytes are written to the 'out' slice; Grow the slice to hold the header and associated data payload.
+	offset := len(out)
+	out = out[:offset+len(ad)]
+
+	// In one call path, the associated data _is_ already stored in out. In other call paths, the associated data must
+	// be copied into 'out'.
+	if !nocopy {
+		copy(out[offset:], ad)
+	}
+
+	var err error
+	out, err = via.ConnectionState.eKey.EncryptDanger(out, out, nil, c, nb)
+	if noiseutil.EncryptLockNeeded {
+		via.ConnectionState.writeLock.Unlock()
+	}
+	if err != nil {
+		via.logger(f.l).WithError(err).Info("Failed to EncryptDanger in sendVia")
+		return
+	}
+	err = f.writers[0].WriteTo(out, via.remote)
+	if err != nil {
+		via.logger(f.l).WithError(err).Info("Failed to WriteTo in sendVia")
 	}
 	f.connectionManager.RelayUsed(relay.LocalIndex)
 }
 
-// sendNoMetrics encrypts and writes one outbound nebula packet (data, control,
-// lighthouse, etc) using buf as the per-call wire scratch. When the hostinfo
-// has no direct remote we encrypt into the relay-reserved slot via
-// SealForRelay so sendViaInPlace can wrap it without an extra copy.
-func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p []byte, buf *WireBuffer, q int) {
+func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, q int) {
 	if ci.eKey == nil {
 		return
 	}
 	useRelay := !remote.IsValid() && !hostinfo.remote.IsValid()
+	fullOut := out
 
+	if useRelay {
+		if len(out) < header.Len {
+			// out always has a capacity of mtu, but not always a length greater than the header.Len.
+			// Grow it to make sure the next operation works.
+			out = out[:header.Len]
+		}
+		// Save a header's worth of data at the front of the 'out' buffer.
+		out = out[header.Len:]
+	}
+
+	if noiseutil.EncryptLockNeeded {
+		// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
+		ci.writeLock.Lock()
+	}
+	c := ci.messageCounter.Add(1)
+
+	//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
+	out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
 	f.connectionManager.Out(hostinfo)
 
 	// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against
@@ -335,49 +423,46 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 		// finally used again. This tunnel would eventually be torn down and recreated if this action didn't help.
 		f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
 		hostinfo.lastRebindCount = f.rebindCount
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			f.l.Debug("Lighthouse update triggered for punch due to rebind counter",
-				"vpnAddrs", hostinfo.vpnAddrs,
-			)
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).Debug("Lighthouse update triggered for punch due to rebind counter")
 		}
 	}
 
-	var out []byte
 	var err error
-	if useRelay {
-		out, err = buf.SealForRelay(ci, t, st, hostinfo.remoteIndexId, p)
-	} else {
-		out, err = buf.Seal(ci, t, st, hostinfo.remoteIndexId, p)
+	out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
+	if noiseutil.EncryptLockNeeded {
+		ci.writeLock.Unlock()
 	}
 	if err != nil {
-		hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet",
-			"error", err,
-			"udpAddr", remote,
-		)
+		hostinfo.logger(f.l).WithError(err).
+			WithField("udpAddr", remote).WithField("counter", c).
+			WithField("attemptedCounter", c).
+			Error("Failed to encrypt outgoing packet")
 		return
 	}
 
-	switch {
-	case remote.IsValid():
-		if err := f.writers[q].WriteTo(out, remote); err != nil {
-			hostinfo.logger(f.l).Error("Failed to write outgoing packet", "error", err, "udpAddr", remote)
+	if remote.IsValid() {
+		err = f.writers[q].WriteTo(out, remote)
+		if err != nil {
+			hostinfo.logger(f.l).WithError(err).
+				WithField("udpAddr", remote).Error("Failed to write outgoing packet")
 		}
-	case hostinfo.remote.IsValid():
-		if err := f.writers[q].WriteTo(out, hostinfo.remote); err != nil {
-			hostinfo.logger(f.l).Error("Failed to write outgoing packet", "error", err, "udpAddr", hostinfo.remote)
+	} else if hostinfo.remote.IsValid() {
+		err = f.writers[q].WriteTo(out, hostinfo.remote)
+		if err != nil {
+			hostinfo.logger(f.l).WithError(err).
+				WithField("udpAddr", remote).Error("Failed to write outgoing packet")
 		}
-	default:
-		// SealForRelay placed the inner ciphertext at buf.Out[header.Len:],
-		// so sendViaInPlace can wrap it with the outer relay header without
-		// an extra copy.
+	} else {
+		// Try to send via a relay
 		for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
 			relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
 			if err != nil {
 				hostinfo.relayState.DeleteRelay(relayIP)
-				hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo", "relay", relayIP, "error", err)
+				hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetrics failed to find HostInfo")
 				continue
 			}
-			f.sendViaInPlace(relayHostInfo, relay, len(out), buf)
+			f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
 			break
 		}
 	}

interface.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io"
-	"log/slog"
 	"net/netip"
 	"sync"
 	"sync/atomic"
@@ -13,11 +11,13 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/rcrowley/go-metrics"
-
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
+	"github.com/slackhq/nebula/overlay/coalesce"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/udp"
 )
 
@@ -30,7 +30,7 @@ type InterfaceConfig struct {
 	pki                *PKI
 	Cipher             string
 	Firewall           *Firewall
-	DnsServer          *dnsServer
+	ServeDns           bool
 	HandshakeManager   *HandshakeManager
 	lightHouse         *LightHouse
 	connectionManager  *connectionManager
@@ -47,7 +47,7 @@ type InterfaceConfig struct {
 	reQueryWait     time.Duration
 
 	ConntrackCacheTimeout time.Duration
-	l                     *slog.Logger
+	l                     *logrus.Logger
 }
 
 type Interface struct {
@@ -58,7 +58,7 @@ type Interface struct {
 	firewall              *Firewall
 	connectionManager     *connectionManager
 	handshakeManager      *HandshakeManager
-	dnsServer             *dnsServer
+	serveDns              bool
 	createTime            time.Time
 	lightHouse            *LightHouse
 	myBroadcastAddrsTable *bart.Lite
@@ -86,10 +86,13 @@ type Interface struct {
 
 	conntrackCacheTimeout time.Duration
 
-	ctx     context.Context
 	writers []udp.Conn
-	readers []io.ReadWriteCloser
-	wg      sync.WaitGroup
+	readers []tio.Queue
+	// tunCoalescers is one tcpCoalescer per tun queue, wrapping readers[i].
+	// decryptToTun sends plaintext into the coalescer; listenOut calls its
+	// Flush at the end of each UDP recvmmsg batch.
+	tunCoalescers []*coalesce.TCPCoalescer
+	wg            sync.WaitGroup
 
 	// fatalErr holds the first unexpected reader error that caused shutdown.
 	// nil means "no fatal error" (yet)
@@ -101,19 +104,19 @@ type Interface struct {
 	messageMetrics      *MessageMetrics
 	cachedPacketMetrics *cachedPacketMetrics
 
-	// bufAlloc hands out reusable WireBuffers sized for this interface's
-	// inside Device. All buf consumers (hot-path data-plane goroutines,
-	// long-lived workers, and cold callers) acquire from here so sizing
-	// is centralized and consistent. Long-lived owners just don't release.
-	bufAlloc WireBufferAllocator
-
-	l *slog.Logger
+	l *logrus.Logger
 }
 
 type EncWriter interface {
-	SendVia(via *HostInfo, relay *Relay, ad []byte, buf *WireBuffer)
-	SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p []byte, buf *WireBuffer)
-	SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p []byte, buf *WireBuffer)
+	SendVia(via *HostInfo,
+		relay *Relay,
+		ad,
+		nb,
+		out []byte,
+		nocopy bool,
+	)
+	SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte)
+	SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte)
 	Handshake(vpnAddr netip.Addr)
 	GetHostInfo(vpnAddr netip.Addr) *HostInfo
 	GetCertState() *CertState
@@ -172,13 +175,12 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 
 	cs := c.pki.getCertState()
 	ifce := &Interface{
-		ctx:                   ctx,
 		pki:                   c.pki,
 		hostMap:               c.HostMap,
 		outside:               c.Outside,
 		inside:                c.Inside,
 		firewall:              c.Firewall,
-		dnsServer:             c.DnsServer,
+		serveDns:              c.ServeDns,
 		handshakeManager:      c.HandshakeManager,
 		createTime:            time.Now(),
 		lightHouse:            c.lightHouse,
@@ -187,7 +189,8 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		routines:              c.routines,
 		version:               c.version,
 		writers:               make([]udp.Conn, c.routines),
-		readers:               make([]io.ReadWriteCloser, c.routines),
+		readers:               make([]tio.Queue, c.routines),
+		tunCoalescers:         make([]*coalesce.TCPCoalescer, c.routines),
 		myVpnNetworks:         cs.myVpnNetworks,
 		myVpnNetworksTable:    cs.myVpnNetworksTable,
 		myVpnAddrs:            cs.myVpnAddrs,
@@ -204,8 +207,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 			dropped: metrics.GetOrRegisterCounter("hostinfo.cached_packets.dropped", nil),
 		},
 
-		bufAlloc: NewWireBufferPool(mtu, c.Inside.TunPrefixLen()),
-
 		l: c.l,
 	}
 
@@ -226,16 +227,13 @@ func (f *Interface) activate() error {
 
 	addr, err := f.outside.LocalAddr()
 	if err != nil {
-		f.l.Error("Failed to get udp listen address", "error", err)
+		f.l.WithError(err).Error("Failed to get udp listen address")
 	}
 
-	f.l.Info("Nebula interface is active",
-		"interface", f.inside.Name(),
-		"networks", f.myVpnNetworks,
-		"build", f.version,
-		"udpAddr", addr,
-		"boringcrypto", boringEnabled(),
-	)
+	f.l.WithField("interface", f.inside.Name()).WithField("networks", f.myVpnNetworks).
+		WithField("build", f.version).WithField("udpAddr", addr).
+		WithField("boringcrypto", boringEnabled()).
+		Info("Nebula interface is active")
 
 	if f.routines > 1 {
 		if !f.inside.SupportsMultiqueue() || !f.outside.SupportsMultipleReaders() {
@@ -247,15 +245,17 @@ func (f *Interface) activate() error {
 	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
 
 	// Prepare n tun queues
-	var reader io.ReadWriteCloser = f.inside
 	for i := 0; i < f.routines; i++ {
 		if i > 0 {
-			reader, err = f.inside.NewMultiQueueReader()
+			err = f.inside.NewMultiQueueReader()
 			if err != nil {
 				return err
 			}
 		}
-		f.readers[i] = reader
+	}
+	f.readers = f.inside.Readers()
+	for i := range f.readers {
+		f.tunCoalescers[i] = coalesce.NewTCPCoalescer(f.readers[i]) //todo don't always do this
 	}
 
 	f.wg.Add(1) // for us to wait on Close() to return
@@ -311,42 +311,78 @@ func (f *Interface) listenOut(i int) {
 		li = f.outside
 	}
 
-	ctCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
+	ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
 	lhh := f.lightHouse.NewRequestHandler()
-	// Long-lived per-receive-goroutine buf; never released back to the pool.
-	buf := f.bufAlloc.Acquire()
+	h := &header.H{}
+	fwPacket := &firewall.Packet{}
+	nb := make([]byte, 12, 12)
 
+	// plaintexts is a ring of decrypt scratches, one per packet in a UDP
+	// recvmmsg batch. The coalescer borrows payload slices from here and
+	// requires they stay valid until Flush — so we rotate each packet and
+	// reset only in the batch-end flush callback.
+	var plaintexts [][]byte
+	idx := 0
+	coalescer := f.tunCoalescers[i]
 	err := li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
-		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, buf, payload, lhh, i, ctCache.Get())
+		if idx >= len(plaintexts) {
+			plaintexts = append(plaintexts, make([]byte, udp.MTU))
+		}
+		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintexts[idx][:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get(f.l))
+		idx++
+	}, func() {
+		if err := coalescer.Flush(); err != nil {
+			f.l.WithError(err).Error("Failed to flush tun coalescer")
+		}
+		idx = 0
 	})
 
 	if err != nil && !f.closed.Load() {
-		f.l.Error("Error while reading inbound packet, closing", "error", err)
+		f.l.WithError(err).Error("Error while reading inbound packet, closing")
 		f.onFatal(err)
 	}
 
-	f.l.Debug("underlay reader is done", "reader", i)
+	f.l.Debugf("underlay reader %v is done", i)
 }
 
-func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
-	// Long-lived per-tun-reader buf; never released back to the pool.
-	buf := f.bufAlloc.Acquire()
-	conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
+func (f *Interface) listenIn(reader tio.Queue, i int) {
+	rejectBuf := make([]byte, mtu)
+	batch := newSendBatch(sendBatchCap, udp.MTU+32)
+	fwPacket := &firewall.Packet{}
+	nb := make([]byte, 12, 12)
+
+	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
 
 	for {
-		_, err := buf.ReadIPFromTUN(reader)
+		pkts, err := reader.Read()
 		if err != nil {
 			if !f.closed.Load() {
-				f.l.Error("Error while reading outbound packet, closing", "error", err, "reader", i)
+				f.l.WithError(err).WithField("reader", i).Error("Error while reading outbound packet, closing")
 				f.onFatal(err)
 			}
 			break
 		}
 
-		f.consumeInsidePacket(buf, i, conntrackCache.Get())
+		batch.Reset()
+		for _, pkt := range pkts {
+			if batch.Len() >= batch.Cap() {
+				f.flushBatch(batch, i)
+				batch.Reset()
+			}
+			f.consumeInsidePacket(pkt, fwPacket, nb, batch, rejectBuf, i, conntrackCache.Get(f.l))
+		}
+		if batch.Len() > 0 {
+			f.flushBatch(batch, i)
+		}
 	}
 
-	f.l.Debug("overlay reader is done", "reader", i)
+	f.l.Debugf("overlay reader %v is done", i)
+}
+
+func (f *Interface) flushBatch(batch *sendBatch, q int) {
+	if err := f.writers[q].WriteBatch(batch.bufs, batch.dsts); err != nil {
+		f.l.WithError(err).WithField("writer", q).Error("Failed to write outgoing batch")
+	}
 }
 
 func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
@@ -366,7 +402,7 @@ func (f *Interface) reloadDisconnectInvalid(c *config.C) {
 	if initial || c.HasChanged("pki.disconnect_invalid") {
 		f.disconnectInvalid.Store(c.GetBool("pki.disconnect_invalid", true))
 		if !initial {
-			f.l.Info("pki.disconnect_invalid changed", "value", f.disconnectInvalid.Load())
+			f.l.Infof("pki.disconnect_invalid changed to %v", f.disconnectInvalid.Load())
 		}
 	}
 }
@@ -380,7 +416,7 @@ func (f *Interface) reloadFirewall(c *config.C) {
 
 	fw, err := NewFirewallFromConfig(f.l, f.pki.getCertState(), c)
 	if err != nil {
-		f.l.Error("Error while creating firewall during reload", "error", err)
+		f.l.WithError(err).Error("Error while creating firewall during reload")
 		return
 	}
 
@@ -393,11 +429,10 @@ func (f *Interface) reloadFirewall(c *config.C) {
 	// If rulesVersion is back to zero, we have wrapped all the way around. Be
 	// safe and just reset conntrack in this case.
 	if fw.rulesVersion == 0 {
-		f.l.Warn("firewall rulesVersion has overflowed, resetting conntrack",
-			"firewallHashes", fw.GetRuleHashes(),
-			"oldFirewallHashes", oldFw.GetRuleHashes(),
-			"rulesVersion", fw.rulesVersion,
-		)
+		f.l.WithField("firewallHashes", fw.GetRuleHashes()).
+			WithField("oldFirewallHashes", oldFw.GetRuleHashes()).
+			WithField("rulesVersion", fw.rulesVersion).
+			Warn("firewall rulesVersion has overflowed, resetting conntrack")
 	} else {
 		fw.Conntrack = conntrack
 	}
@@ -405,11 +440,10 @@ func (f *Interface) reloadFirewall(c *config.C) {
 	f.firewall = fw
 
 	oldFw.Destroy()
-	f.l.Info("New firewall has been installed",
-		"firewallHashes", fw.GetRuleHashes(),
-		"oldFirewallHashes", oldFw.GetRuleHashes(),
-		"rulesVersion", fw.rulesVersion,
-	)
+	f.l.WithField("firewallHashes", fw.GetRuleHashes()).
+		WithField("oldFirewallHashes", oldFw.GetRuleHashes()).
+		WithField("rulesVersion", fw.rulesVersion).
+		Info("New firewall has been installed")
 }
 
 func (f *Interface) reloadSendRecvError(c *config.C) {
@@ -431,7 +465,8 @@ func (f *Interface) reloadSendRecvError(c *config.C) {
 			}
 		}
 
-		f.l.Info("Loaded send_recv_error config", "sendRecvError", f.sendRecvErrorConfig.String())
+		f.l.WithField("sendRecvError", f.sendRecvErrorConfig.String()).
+			Info("Loaded send_recv_error config")
 	}
 }
 
@@ -454,7 +489,8 @@ func (f *Interface) reloadAcceptRecvError(c *config.C) {
 			}
 		}
 
-		f.l.Info("Loaded accept_recv_error config", "acceptRecvError", f.acceptRecvErrorConfig.String())
+		f.l.WithField("acceptRecvError", f.acceptRecvErrorConfig.String()).
+			Info("Loaded accept_recv_error config")
 	}
 }
 
@@ -528,7 +564,7 @@ func (f *Interface) Close() error {
 	for i, u := range f.writers {
 		err := u.Close()
 		if err != nil {
-			f.l.Error("Error while closing udp socket", "error", err, "writer", i)
+			f.l.WithError(err).WithField("writer", i).Error("Error while closing udp socket")
 			errs = append(errs, err)
 		}
 	}

lighthouse.go

@@ -5,7 +5,6 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
-	"log/slog"
 	"net"
 	"net/netip"
 	"slices"
@@ -16,10 +15,10 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/rcrowley/go-metrics"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/logging"
 	"github.com/slackhq/nebula/udp"
 	"github.com/slackhq/nebula/util"
 )
@@ -63,30 +62,25 @@ type LightHouse struct {
 	interval     atomic.Int64
 	updateCancel context.CancelFunc
 	ifce         EncWriter
-	// bufAlloc lets the lighthouse query/update workers, request handlers
-	// and punchback goroutines acquire correctly sized WireBuffers from
-	// the same pool as the data plane. Set by main.go alongside ifce.
-	bufAlloc   WireBufferAllocator
-	nebulaPort uint32 // 32 bits because protobuf does not have a uint16
+	nebulaPort   uint32 // 32 bits because protobuf does not have a uint16
 
 	advertiseAddrs atomic.Pointer[[]netip.AddrPort]
 
 	// Addr's of relays that can be used by peers to access me
 	relaysForMe atomic.Pointer[[]netip.Addr]
 
-	updateTrigger chan struct{}
-	queryChan     chan netip.Addr
+	queryChan chan netip.Addr
 
 	calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote
 
 	metrics           *MessageMetrics
 	metricHolepunchTx metrics.Counter
-	l                 *slog.Logger
+	l                 *logrus.Logger
 }
 
 // NewLightHouseFromConfig will build a Lighthouse struct from the values provided in the config object
 // addrMap should be nil unless this is during a config reload
-func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, cs *CertState, pc udp.Conn, p *Punchy) (*LightHouse, error) {
+func NewLightHouseFromConfig(ctx context.Context, l *logrus.Logger, c *config.C, cs *CertState, pc udp.Conn, p *Punchy) (*LightHouse, error) {
 	amLighthouse := c.GetBool("lighthouse.am_lighthouse", false)
 	nebulaPort := uint32(c.GetInt("listen.port", 0))
 	if amLighthouse && nebulaPort == 0 {
@@ -111,13 +105,8 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
 		nebulaPort:         nebulaPort,
 		punchConn:          pc,
 		punchy:             p,
-		updateTrigger:      make(chan struct{}, 1),
 		queryChan:          make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
-		// Default to a no-prefix pool so the query/update workers and
-		// request handlers have a working WireBufferAllocator before
-		// main.go wires up the real one from the Interface.
-		bufAlloc: NewWireBufferPool(mtu, 0),
-		l:        l,
+		l:                  l,
 	}
 	lighthouses := make([]netip.Addr, 0)
 	h.lighthouses.Store(&lighthouses)
@@ -142,7 +131,7 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
 		case *util.ContextualError:
 			v.Log(l)
 		case error:
-			l.Error("failed to reload lighthouse", "error", err)
+			l.WithError(err).Error("failed to reload lighthouse")
 		}
 	})
 
@@ -214,10 +203,8 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
 			//TODO: we could technically insert all returned addrs instead of just the first one if a dns lookup was used
 			addr := addrs[0].Unmap()
 			if lh.myVpnNetworksTable.Contains(addr) {
-				lh.l.Warn("Ignoring lighthouse.advertise_addrs report because it is within the nebula network range",
-					"addr", rawAddr,
-					"entry", i+1,
-				)
+				lh.l.WithField("addr", rawAddr).WithField("entry", i+1).
+					Warn("Ignoring lighthouse.advertise_addrs report because it is within the nebula network range")
 				continue
 			}
 
@@ -235,9 +222,7 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
 		lh.interval.Store(int64(c.GetInt("lighthouse.interval", 10)))
 
 		if !initial {
-			lh.l.Info("lighthouse.interval changed",
-				"interval", lh.interval.Load(),
-			)
+			lh.l.Infof("lighthouse.interval changed to %v", lh.interval.Load())
 
 			if lh.updateCancel != nil {
 				// May not always have a running routine
@@ -331,7 +316,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
 		if !initial {
 			//NOTE: we are not tearing down existing lighthouse connections because they might be used for non lighthouse traffic
 			lh.l.Info("lighthouse.hosts has changed")
-			lh.TriggerUpdate()
 		}
 	}
 
@@ -349,12 +333,9 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
 			for _, v := range c.GetStringSlice("relay.relays", nil) {
 				configRIP, err := netip.ParseAddr(v)
 				if err != nil {
-					lh.l.Warn("Parse relay from config failed",
-						"relay", v,
-						"error", err,
-					)
+					lh.l.WithField("relay", v).WithError(err).Warn("Parse relay from config failed")
 				} else {
-					lh.l.Info("Read relay from config", "relay", v)
+					lh.l.WithField("relay", v).Info("Read relay from config")
 					relaysForMe = append(relaysForMe, configRIP)
 				}
 			}
@@ -379,10 +360,8 @@ func (lh *LightHouse) parseLighthouses(c *config.C) ([]netip.Addr, error) {
 		}
 
 		if !lh.myVpnNetworksTable.Contains(addr) {
-			lh.l.Warn("lighthouse host is not within our networks, lighthouse functionality will work but layer 3 network traffic to the lighthouse will not",
-				"vpnAddr", addr,
-				"networks", lh.myVpnNetworks,
-			)
+			lh.l.WithFields(m{"vpnAddr": addr, "networks": lh.myVpnNetworks}).
+				Warn("lighthouse host is not within our networks, lighthouse functionality will work but layer 3 network traffic to the lighthouse will not")
 		}
 		out[i] = addr
 	}
@@ -453,11 +432,8 @@ func (lh *LightHouse) loadStaticMap(c *config.C, staticList map[netip.Addr]struc
 		}
 
 		if !lh.myVpnNetworksTable.Contains(vpnAddr) {
-			lh.l.Warn("static_host_map key is not within our networks, layer 3 network traffic to this host will not work",
-				"vpnAddr", vpnAddr,
-				"networks", lh.myVpnNetworks,
-				"entry", i+1,
-			)
+			lh.l.WithFields(m{"vpnAddr": vpnAddr, "networks": lh.myVpnNetworks, "entry": i + 1}).
+				Warn("static_host_map key is not within our networks, layer 3 network traffic to this host will not work")
 		}
 
 		vals, ok := v.([]any)
@@ -558,13 +534,12 @@ func (lh *LightHouse) DeleteVpnAddrs(allVpnAddrs []netip.Addr) {
 	lh.Lock()
 	rm, ok := lh.addrMap[allVpnAddrs[0]]
 	if ok {
-		debugEnabled := lh.l.Enabled(context.Background(), slog.LevelDebug)
 		for _, addr := range allVpnAddrs {
 			srm := lh.addrMap[addr]
 			if srm == rm {
 				delete(lh.addrMap, addr)
-				if debugEnabled {
-					lh.l.Debug("deleting from lighthouse", "vpnAddr", addr)
+				if lh.l.Level >= logrus.DebugLevel {
+					lh.l.Debugf("deleting %s from lighthouse.", addr)
 				}
 			}
 		}
@@ -681,12 +656,9 @@ func (lh *LightHouse) unlockedGetRemoteList(allAddrs []netip.Addr) *RemoteList {
 
 func (lh *LightHouse) shouldAdd(vpnAddrs []netip.Addr, to netip.Addr) bool {
 	allow := lh.GetRemoteAllowList().AllowAll(vpnAddrs, to)
-	if lh.l.Enabled(context.Background(), logging.LevelTrace) {
-		lh.l.Log(context.Background(), logging.LevelTrace, "remoteAllowList.Allow",
-			"vpnAddrs", vpnAddrs,
-			"udpAddr", to,
-			"allow", allow,
-		)
+	if lh.l.Level >= logrus.TraceLevel {
+		lh.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", to).WithField("allow", allow).
+			Trace("remoteAllowList.Allow")
 	}
 	if !allow {
 		return false
@@ -703,12 +675,9 @@ func (lh *LightHouse) shouldAdd(vpnAddrs []netip.Addr, to netip.Addr) bool {
 func (lh *LightHouse) unlockedShouldAddV4(vpnAddr netip.Addr, to *V4AddrPort) bool {
 	udpAddr := protoV4AddrPortToNetAddrPort(to)
 	allow := lh.GetRemoteAllowList().Allow(vpnAddr, udpAddr.Addr())
-	if lh.l.Enabled(context.Background(), logging.LevelTrace) {
-		lh.l.Log(context.Background(), logging.LevelTrace, "remoteAllowList.Allow",
-			"vpnAddr", vpnAddr,
-			"udpAddr", udpAddr,
-			"allow", allow,
-		)
+	if lh.l.Level >= logrus.TraceLevel {
+		lh.l.WithField("vpnAddr", vpnAddr).WithField("udpAddr", udpAddr).WithField("allow", allow).
+			Trace("remoteAllowList.Allow")
 	}
 
 	if !allow {
@@ -726,12 +695,9 @@ func (lh *LightHouse) unlockedShouldAddV4(vpnAddr netip.Addr, to *V4AddrPort) bo
 func (lh *LightHouse) unlockedShouldAddV6(vpnAddr netip.Addr, to *V6AddrPort) bool {
 	udpAddr := protoV6AddrPortToNetAddrPort(to)
 	allow := lh.GetRemoteAllowList().Allow(vpnAddr, udpAddr.Addr())
-	if lh.l.Enabled(context.Background(), logging.LevelTrace) {
-		lh.l.Log(context.Background(), logging.LevelTrace, "remoteAllowList.Allow",
-			"vpnAddr", vpnAddr,
-			"udpAddr", udpAddr,
-			"allow", allow,
-		)
+	if lh.l.Level >= logrus.TraceLevel {
+		lh.l.WithField("vpnAddr", vpnAddr).WithField("udpAddr", udpAddr).WithField("allow", allow).
+			Trace("remoteAllowList.Allow")
 	}
 
 	if !allow {
@@ -766,22 +732,21 @@ func (lh *LightHouse) startQueryWorker() {
 	}
 
 	go func() {
-		// Long-lived per-worker WireBuffer; reused for every lighthouse query
-		// this worker issues for the life of the goroutine.
-		buf := lh.bufAlloc.Acquire()
+		nb := make([]byte, 12, 12)
+		out := make([]byte, mtu)
 
 		for {
 			select {
 			case <-lh.ctx.Done():
 				return
 			case addr := <-lh.queryChan:
-				lh.innerQueryServer(addr, buf)
+				lh.innerQueryServer(addr, nb, out)
 			}
 		}
 	}()
 }
 
-func (lh *LightHouse) innerQueryServer(addr netip.Addr, buf *WireBuffer) {
+func (lh *LightHouse) innerQueryServer(addr netip.Addr, nb, out []byte) {
 	if lh.IsLighthouseAddr(addr) {
 		return
 	}
@@ -807,10 +772,8 @@ func (lh *LightHouse) innerQueryServer(addr netip.Addr, buf *WireBuffer) {
 
 		if v == cert.Version1 {
 			if !addr.Is4() {
-				lh.l.Error("Can't query lighthouse for v6 address using a v1 protocol",
-					"queryVpnAddr", addr,
-					"lighthouseAddr", lhVpnAddr,
-				)
+				lh.l.WithField("queryVpnAddr", addr).WithField("lighthouseAddr", lhVpnAddr).
+					Error("Can't query lighthouse for v6 address using a v1 protocol")
 				continue
 			}
 
@@ -821,16 +784,14 @@ func (lh *LightHouse) innerQueryServer(addr netip.Addr, buf *WireBuffer) {
 
 				v1Query, err = msg.Marshal()
 				if err != nil {
-					lh.l.Error("Failed to marshal lighthouse v1 query payload",
-						"error", err,
-						"queryVpnAddr", addr,
-						"lighthouseAddr", lhVpnAddr,
-					)
+					lh.l.WithError(err).WithField("queryVpnAddr", addr).
+						WithField("lighthouseAddr", lhVpnAddr).
+						Error("Failed to marshal lighthouse v1 query payload")
 					continue
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Query, buf)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Query, nb, out)
 			queried++
 
 		} else if v == cert.Version2 {
@@ -840,24 +801,18 @@ func (lh *LightHouse) innerQueryServer(addr netip.Addr, buf *WireBuffer) {
 
 				v2Query, err = msg.Marshal()
 				if err != nil {
-					lh.l.Error("Failed to marshal lighthouse v2 query payload",
-						"error", err,
-						"queryVpnAddr", addr,
-						"lighthouseAddr", lhVpnAddr,
-					)
+					lh.l.WithError(err).WithField("queryVpnAddr", addr).
+						WithField("lighthouseAddr", lhVpnAddr).
+						Error("Failed to marshal lighthouse v2 query payload")
 					continue
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Query, buf)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Query, nb, out)
 			queried++
 
 		} else {
-			lh.l.Debug("unsupported protocol version",
-				"op", "query",
-				"queryVpnAddr", addr,
-				"version", v,
-			)
+			lh.l.Debugf("Can not query lighthouse for %v using unknown protocol version: %v", addr, v)
 			continue
 		}
 	}
@@ -878,46 +833,20 @@ func (lh *LightHouse) StartUpdateWorker() {
 	go func() {
 		defer clockSource.Stop()
 
-		// Long-lived per-worker WireBuffer; reused across every periodic
-		// update for the life of this goroutine.
-		buf := lh.bufAlloc.Acquire()
-
 		for {
-			lh.sendUpdate(buf)
+			lh.SendUpdate()
 
 			select {
 			case <-updateCtx.Done():
 				return
 			case <-clockSource.C:
 				continue
-			case <-lh.updateTrigger:
-				continue
 			}
 		}
 	}()
 }
 
-// SendUpdate is the public entry point that triggers a one-shot lighthouse
-// update outside the worker loop (e.g. tests or reload paths). It allocates
-// its own WireBuffer since callers don't already own one.
 func (lh *LightHouse) SendUpdate() {
-	buf := lh.bufAlloc.Acquire()
-	defer lh.bufAlloc.Release(buf)
-	lh.sendUpdate(buf)
-}
-
-// TriggerUpdate requests an immediate lighthouse update. This is a non-blocking
-// operation intended to be called after a handshake completes with a lighthouse,
-// so the lighthouse has our current addresses without waiting for the next
-// periodic update.
-func (lh *LightHouse) TriggerUpdate() {
-	select {
-	case lh.updateTrigger <- struct{}{}:
-	default:
-	}
-}
-
-func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
 	var v4 []*V4AddrPort
 	var v6 []*V6AddrPort
 
@@ -943,6 +872,9 @@ func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
 		}
 	}
 
+	nb := make([]byte, 12, 12)
+	out := make([]byte, mtu)
+
 	var v1Update, v2Update []byte
 	var err error
 	updated := 0
@@ -959,9 +891,8 @@ func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
 		if v == cert.Version1 {
 			if v1Update == nil {
 				if !lh.myVpnNetworks[0].Addr().Is4() {
-					lh.l.Warn("cannot update lighthouse using v1 protocol without an IPv4 address",
-						"lighthouseAddr", lhVpnAddr,
-					)
+					lh.l.WithField("lighthouseAddr", lhVpnAddr).
+						Warn("cannot update lighthouse using v1 protocol without an IPv4 address")
 					continue
 				}
 				var relays []uint32
@@ -985,15 +916,13 @@ func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
 
 				v1Update, err = msg.Marshal()
 				if err != nil {
-					lh.l.Error("Error while marshaling for lighthouse v1 update",
-						"error", err,
-						"lighthouseAddr", lhVpnAddr,
-					)
+					lh.l.WithError(err).WithField("lighthouseAddr", lhVpnAddr).
+						Error("Error while marshaling for lighthouse v1 update")
 					continue
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Update, buf)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Update, nb, out)
 			updated++
 
 		} else if v == cert.Version2 {
@@ -1014,22 +943,17 @@ func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
 
 				v2Update, err = msg.Marshal()
 				if err != nil {
-					lh.l.Error("Error while marshaling for lighthouse v2 update",
-						"error", err,
-						"lighthouseAddr", lhVpnAddr,
-					)
+					lh.l.WithError(err).WithField("lighthouseAddr", lhVpnAddr).
+						Error("Error while marshaling for lighthouse v2 update")
 					continue
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Update, buf)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Update, nb, out)
 			updated++
 
 		} else {
-			lh.l.Debug("unsupported protocol version",
-				"op", "update",
-				"version", v,
-			)
+			lh.l.Debugf("Can not update lighthouse using unknown protocol version: %v", v)
 			continue
 		}
 	}
@@ -1038,20 +962,19 @@ func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
 }
 
 type LightHouseHandler struct {
-	lh *LightHouse
-	// buf is the long-lived per-handler wire scratch. NewRequestHandler is
-	// called once per data-plane receive goroutine, so buf is owned by that
-	// goroutine and reused for every lighthouse send the handler issues.
-	buf  *WireBuffer
+	lh   *LightHouse
+	nb   []byte
+	out  []byte
 	pb   []byte
 	meta *NebulaMeta
-	l    *slog.Logger
+	l    *logrus.Logger
 }
 
 func (lh *LightHouse) NewRequestHandler() *LightHouseHandler {
 	lhh := &LightHouseHandler{
 		lh:  lh,
-		buf: lh.bufAlloc.Acquire(),
+		nb:  make([]byte, 12, 12),
+		out: make([]byte, mtu),
 		l:   lh.l,
 		pb:  make([]byte, mtu),
 
@@ -1093,19 +1016,14 @@ func (lhh *LightHouseHandler) HandleRequest(rAddr netip.AddrPort, fromVpnAddrs [
 	n := lhh.resetMeta()
 	err := n.Unmarshal(p)
 	if err != nil {
-		lhh.l.Error("Failed to unmarshal lighthouse packet",
-			"error", err,
-			"vpnAddrs", fromVpnAddrs,
-			"udpAddr", rAddr,
-		)
+		lhh.l.WithError(err).WithField("vpnAddrs", fromVpnAddrs).WithField("udpAddr", rAddr).
+			Error("Failed to unmarshal lighthouse packet")
 		return
 	}
 
 	if n.Details == nil {
-		lhh.l.Error("Invalid lighthouse update",
-			"vpnAddrs", fromVpnAddrs,
-			"udpAddr", rAddr,
-		)
+		lhh.l.WithField("vpnAddrs", fromVpnAddrs).WithField("udpAddr", rAddr).
+			Error("Invalid lighthouse update")
 		return
 	}
 
@@ -1133,29 +1051,25 @@ func (lhh *LightHouseHandler) HandleRequest(rAddr netip.AddrPort, fromVpnAddrs [
 func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []netip.Addr, addr netip.AddrPort, w EncWriter) {
 	// Exit if we don't answer queries
 	if !lhh.lh.amLighthouse {
-		if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
-			lhh.l.Debug("I don't answer queries, but received one", "from", addr)
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.Debugln("I don't answer queries, but received from: ", addr)
 		}
 		return
 	}
 
 	queryVpnAddr, useVersion, err := n.Details.GetVpnAddrAndVersion()
 	if err != nil {
-		if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
-			lhh.l.Debug("Dropping malformed HostQuery",
-				"from", fromVpnAddrs,
-				"details", n.Details,
-			)
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.WithField("from", fromVpnAddrs).WithField("details", n.Details).
+				Debugln("Dropping malformed HostQuery")
 		}
 		return
 	}
 	if useVersion == cert.Version1 && queryVpnAddr.Is6() {
 		// this case really shouldn't be possible to represent, but reject it anyway.
-		if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
-			lhh.l.Debug("invalid vpn addr for v1 handleHostQuery",
-				"vpnAddrs", fromVpnAddrs,
-				"queryVpnAddr", queryVpnAddr,
-			)
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.WithField("vpnAddrs", fromVpnAddrs).WithField("queryVpnAddr", queryVpnAddr).
+				Debugln("invalid vpn addr for v1 handleHostQuery")
 		}
 		return
 	}
@@ -1180,15 +1094,12 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []neti
 	}
 
 	if err != nil {
-		lhh.l.Error("Failed to marshal lighthouse host query reply",
-			"error", err,
-			"vpnAddrs", fromVpnAddrs,
-		)
+		lhh.l.WithError(err).WithField("vpnAddrs", fromVpnAddrs).Error("Failed to marshal lighthouse host query reply")
 		return
 	}
 
 	lhh.lh.metricTx(NebulaMeta_HostQueryReply, 1)
-	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.buf)
+	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.nb, lhh.out[:0])
 
 	lhh.sendHostPunchNotification(n, fromVpnAddrs, queryVpnAddr, w)
 }
@@ -1211,10 +1122,8 @@ func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAd
 			if ok {
 				whereToPunch = newDest
 			} else {
-				if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
-					lhh.l.Debug("unable to punch to host, no addresses in common",
-						"to", crt.Networks(),
-					)
+				if lhh.l.Level >= logrus.DebugLevel {
+					lhh.l.WithField("to", crt.Networks()).Debugln("unable to punch to host, no addresses in common")
 				}
 			}
 		}
@@ -1240,15 +1149,12 @@ func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAd
 	}
 
 	if err != nil {
-		lhh.l.Error("Failed to marshal lighthouse host was queried for",
-			"error", err,
-			"vpnAddrs", fromVpnAddrs,
-		)
+		lhh.l.WithError(err).WithField("vpnAddrs", fromVpnAddrs).Error("Failed to marshal lighthouse host was queried for")
 		return
 	}
 
 	lhh.lh.metricTx(NebulaMeta_HostPunchNotification, 1)
-	w.SendMessageToVpnAddr(header.LightHouse, 0, punchNotifDest, lhh.pb[:ln], lhh.buf)
+	w.SendMessageToVpnAddr(header.LightHouse, 0, punchNotifDest, lhh.pb[:ln], lhh.nb, lhh.out[:0])
 }
 
 func (lhh *LightHouseHandler) coalesceAnswers(v cert.Version, c *cache, n *NebulaMeta) {
@@ -1285,11 +1191,8 @@ func (lhh *LightHouseHandler) coalesceAnswers(v cert.Version, c *cache, n *Nebul
 				n.Details.RelayVpnAddrs = append(n.Details.RelayVpnAddrs, netAddrToProtoAddr(r))
 			}
 		} else {
-			if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
-				lhh.l.Debug("unsupported protocol version",
-					"op", "coalesceAnswers",
-					"version", v,
-				)
+			if lhh.l.Level >= logrus.DebugLevel {
+				lhh.l.WithField("version", v).Debug("unsupported protocol version")
 			}
 		}
 	}
@@ -1302,11 +1205,8 @@ func (lhh *LightHouseHandler) handleHostQueryReply(n *NebulaMeta, fromVpnAddrs [
 
 	certVpnAddr, _, err := n.Details.GetVpnAddrAndVersion()
 	if err != nil {
-		if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
-			lhh.l.Error("dropping malformed HostQueryReply",
-				"error", err,
-				"vpnAddrs", fromVpnAddrs,
-			)
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.WithError(err).WithField("vpnAddrs", fromVpnAddrs).Error("dropping malformed HostQueryReply")
 		}
 		return
 	}
@@ -1331,8 +1231,8 @@ func (lhh *LightHouseHandler) handleHostQueryReply(n *NebulaMeta, fromVpnAddrs [
 
 func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, w EncWriter) {
 	if !lhh.lh.amLighthouse {
-		if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
-			lhh.l.Debug("I am not a lighthouse, do not take host updates", "from", fromVpnAddrs)
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.Debugln("I am not a lighthouse, do not take host updates: ", fromVpnAddrs)
 		}
 		return
 	}
@@ -1355,11 +1255,8 @@ func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, fromVp
 
 	//Simple check that the host sent this not someone else, if detailsVpnAddr is filled
 	if detailsVpnAddr.IsValid() && !slices.Contains(fromVpnAddrs, detailsVpnAddr) {
-		if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
-			lhh.l.Debug("Host sent invalid update",
-				"vpnAddrs", fromVpnAddrs,
-				"answer", detailsVpnAddr,
-			)
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.WithField("vpnAddrs", fromVpnAddrs).WithField("answer", detailsVpnAddr).Debugln("Host sent invalid update")
 		}
 		return
 	}
@@ -1381,9 +1278,7 @@ func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, fromVp
 	switch useVersion {
 	case cert.Version1:
 		if !fromVpnAddrs[0].Is4() {
-			lhh.l.Error("Can not send HostUpdateNotificationAck for a ipv6 vpn ip in a v1 message",
-				"vpnAddrs", fromVpnAddrs,
-			)
+			lhh.l.WithField("vpnAddrs", fromVpnAddrs).Error("Can not send HostUpdateNotificationAck for a ipv6 vpn ip in a v1 message")
 			return
 		}
 		vpnAddrB := fromVpnAddrs[0].As4()
@@ -1391,21 +1286,18 @@ func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, fromVp
 	case cert.Version2:
 		// do nothing, we want to send a blank message
 	default:
-		lhh.l.Error("invalid protocol version", "useVersion", useVersion)
+		lhh.l.WithField("useVersion", useVersion).Error("invalid protocol version")
 		return
 	}
 
 	ln, err := n.MarshalTo(lhh.pb)
 	if err != nil {
-		lhh.l.Error("Failed to marshal lighthouse host update ack",
-			"error", err,
-			"vpnAddrs", fromVpnAddrs,
-		)
+		lhh.l.WithError(err).WithField("vpnAddrs", fromVpnAddrs).Error("Failed to marshal lighthouse host update ack")
 		return
 	}
 
 	lhh.lh.metricTx(NebulaMeta_HostUpdateNotificationAck, 1)
-	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.buf)
+	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.nb, lhh.out[:0])
 }
 
 func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, w EncWriter) {
@@ -1417,11 +1309,8 @@ func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpn
 
 	detailsVpnAddr, _, err := n.Details.GetVpnAddrAndVersion()
 	if err != nil {
-		if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
-			lhh.l.Debug("dropping invalid HostPunchNotification",
-				"details", n.Details,
-				"error", err,
-			)
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.WithField("details", n.Details).WithError(err).Debugln("dropping invalid HostPunchNotification")
 		}
 		return
 	}
@@ -1438,11 +1327,8 @@ func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpn
 			lhh.lh.punchConn.WriteTo(empty, vpnPeer)
 		}()
 
-		if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
-			lhh.l.Debug("Punching",
-				"vpnPeer", vpnPeer,
-				"logVpnAddr", logVpnAddr,
-			)
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.Debugf("Punching on %v for %v", vpnPeer, logVpnAddr)
 		}
 	}
 
@@ -1467,18 +1353,13 @@ func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpn
 	if lhh.lh.punchy.GetRespond() {
 		go func() {
 			time.Sleep(lhh.lh.punchy.GetRespondDelay())
-			if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
-				lhh.l.Debug("Sending a nebula test packet",
-					"vpnAddr", detailsVpnAddr,
-				)
+			if lhh.l.Level >= logrus.DebugLevel {
+				lhh.l.Debugf("Sending a nebula test packet to vpn addr %s", detailsVpnAddr)
 			}
-			// We acquire and release a fresh buf within this goroutine so it
-			// returns to the pool once the punchback send completes. We
-			// should move this into a timerwheel or a single goroutine
+			//NOTE: we have to allocate a new output buffer here since we are spawning a new goroutine
+			// for each punchBack packet. We should move this into a timerwheel or a single goroutine
 			// managed by a channel.
-			pbuf := lhh.lh.bufAlloc.Acquire()
-			defer lhh.lh.bufAlloc.Release(pbuf)
-			w.SendMessageToVpnAddr(header.Test, header.TestRequest, detailsVpnAddr, []byte(""), pbuf)
+			w.SendMessageToVpnAddr(header.Test, header.TestRequest, detailsVpnAddr, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
 		}()
 	}
 }

lighthouse_test.go

@@ -372,12 +372,12 @@ type testEncWriter struct {
 	protocolVersion cert.Version
 }
 
-func (tw *testEncWriter) SendVia(via *HostInfo, relay *Relay, ad []byte, buf *WireBuffer) {
+func (tw *testEncWriter) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, nocopy bool) {
 }
 func (tw *testEncWriter) Handshake(vpnIp netip.Addr) {
 }
 
-func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p []byte, _ *WireBuffer) {
+func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, _, _ []byte) {
 	msg := &NebulaMeta{}
 	err := msg.Unmarshal(p)
 	if tw.metaFilter == nil || msg.Type == *tw.metaFilter {
@@ -394,7 +394,7 @@ func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.M
 	}
 }
 
-func (tw *testEncWriter) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnIp netip.Addr, p []byte, _ *WireBuffer) {
+func (tw *testEncWriter) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnIp netip.Addr, p, _, _ []byte) {
 	msg := &NebulaMeta{}
 	err := msg.Unmarshal(p)
 	if tw.metaFilter == nil || msg.Type == *tw.metaFilter {

logger.go

@@ -0,0 +1,45 @@
+package nebula
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/config"
+)
+
+func configLogger(l *logrus.Logger, c *config.C) error {
+	// set up our logging level
+	logLevel, err := logrus.ParseLevel(strings.ToLower(c.GetString("logging.level", "info")))
+	if err != nil {
+		return fmt.Errorf("%s; possible levels: %s", err, logrus.AllLevels)
+	}
+	l.SetLevel(logLevel)
+
+	disableTimestamp := c.GetBool("logging.disable_timestamp", false)
+	timestampFormat := c.GetString("logging.timestamp_format", "")
+	fullTimestamp := (timestampFormat != "")
+	if timestampFormat == "" {
+		timestampFormat = time.RFC3339
+	}
+
+	logFormat := strings.ToLower(c.GetString("logging.format", "text"))
+	switch logFormat {
+	case "text":
+		l.Formatter = &logrus.TextFormatter{
+			TimestampFormat:  timestampFormat,
+			FullTimestamp:    fullTimestamp,
+			DisableTimestamp: disableTimestamp,
+		}
+	case "json":
+		l.Formatter = &logrus.JSONFormatter{
+			TimestampFormat:  timestampFormat,
+			DisableTimestamp: disableTimestamp,
+		}
+	default:
+		return fmt.Errorf("unknown log format `%s`. possible formats: %s", logFormat, []string{"text", "json"})
+	}
+
+	return nil
+}

logging/logger.go

@@ -1,233 +0,0 @@
-// Package logging wires the nebula runtime-reconfigurable slog handler used
-// by nebula.Main and the nebula CLI binaries. Callers build a logger with
-// NewLogger, then call ApplyConfig at startup and from a config reload
-// callback to push logging.level, logging.format, and
-// logging.disable_timestamp changes onto the logger without rebuilding it.
-package logging
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"log/slog"
-	"strings"
-	"sync/atomic"
-	"time"
-)
-
-// Config is the subset of *config.C that ApplyConfig reads. Declaring it
-// here keeps the logging package from depending on config directly, which
-// would cycle through the shared test helpers (test.NewLogger imports
-// logging, and config's tests import test). *config.C satisfies this
-// interface structurally with no adapter.
-type Config interface {
-	GetString(key, def string) string
-	GetBool(key string, def bool) bool
-}
-
-// LevelTrace is a custom slog level below Debug, used when logging.level is
-// "trace". slog has no builtin trace level; the value is one step below
-// slog.LevelDebug in slog's 4-point spacing.
-const LevelTrace = slog.Level(-8)
-
-// NewLogger returns a *slog.Logger whose level, format, and timestamp
-// emission can be reconfigured at runtime via ApplyConfig and the SSH debug
-// commands. The default configuration is info-level text output so log
-// calls made before ApplyConfig runs still produce output. Timestamps
-// follow slog's default RFC3339Nano format; set logging.disable_timestamp
-// in config to suppress them.
-//
-// ApplyConfig and the SSH commands discover the reconfig surface via
-// structural type-assertion on l.Handler(), so replacement implementations
-// (tests, platform-specific sinks) need only implement the subset of
-// {SetLevel(slog.Level), SetFormat(string) error, SetDisableTimestamp(bool)}
-// they care about. Callers that pass a plain *slog.Logger without these
-// methods get a silent no-op; reconfiguration is always opt-in.
-func NewLogger(w io.Writer) *slog.Logger {
-	return slog.New(NewHandler(w))
-}
-
-// NewHandler builds the *Handler that NewLogger wraps. Exported for
-// platform-specific sinks (notably cmd/nebula-service/logs_windows.go)
-// that want to wrap the handler with extra behavior, such as tagging each
-// record with its Event Log severity, while still benefiting from all the
-// level / format / timestamp / WithAttrs machinery implemented here.
-func NewHandler(w io.Writer) *Handler {
-	root := &handlerRoot{}
-	root.level.Set(slog.LevelInfo)
-	opts := &slog.HandlerOptions{Level: &root.level}
-	return &Handler{
-		root: root,
-		text: slog.NewTextHandler(w, opts),
-		json: slog.NewJSONHandler(w, opts),
-	}
-}
-
-// handlerRoot carries the reconfiguration state shared by every logger
-// derived from a NewHandler call. All fields are consulted on the log
-// path and updated lock-free.
-type handlerRoot struct {
-	level            slog.LevelVar
-	disableTimestamp atomic.Bool
-	// jsonMode picks which of the pre-derived inner handlers Handler.Handle
-	// dispatches to. Flipping it propagates instantly to every derived logger
-	// without rebuilding or chain-replaying anything.
-	jsonMode atomic.Bool
-}
-
-// Handler is the slog.Handler returned by NewHandler. It holds two
-// pre-derived slog handlers -- one text, one json -- both built from the
-// same accumulated WithAttrs/WithGroup state. Handle picks which one to
-// dispatch to based on handlerRoot.jsonMode, so a SetFormat call takes
-// effect immediately across the whole process without having to rebuild
-// any derived loggers.
-type Handler struct {
-	root *handlerRoot
-	text slog.Handler
-	json slog.Handler
-}
-
-func (h *Handler) Enabled(_ context.Context, l slog.Level) bool {
-	return h.root.level.Level() <= l
-}
-
-func (h *Handler) Handle(ctx context.Context, r slog.Record) error {
-	if h.root.disableTimestamp.Load() {
-		r.Time = time.Time{}
-	}
-	if h.root.jsonMode.Load() {
-		return h.json.Handle(ctx, r)
-	}
-	return h.text.Handle(ctx, r)
-}
-
-func (h *Handler) WithAttrs(attrs []slog.Attr) slog.Handler {
-	if len(attrs) == 0 {
-		return h
-	}
-	return &Handler{
-		root: h.root,
-		text: h.text.WithAttrs(attrs),
-		json: h.json.WithAttrs(attrs),
-	}
-}
-
-func (h *Handler) WithGroup(name string) slog.Handler {
-	if name == "" {
-		return h
-	}
-	return &Handler{
-		root: h.root,
-		text: h.text.WithGroup(name),
-		json: h.json.WithGroup(name),
-	}
-}
-
-// SetLevel updates the effective log level. Propagates to every derived
-// logger via the shared LevelVar.
-func (h *Handler) SetLevel(level slog.Level) { h.root.level.Set(level) }
-
-// GetLevel reports the current log level.
-func (h *Handler) GetLevel() slog.Level { return h.root.level.Level() }
-
-// SetFormat flips the output format atomically. Valid formats are "text"
-// and "json". Every derived logger sees the new format on its next Handle
-// call; no rebuild or registration is required.
-func (h *Handler) SetFormat(format string) error {
-	switch format {
-	case "text":
-		h.root.jsonMode.Store(false)
-	case "json":
-		h.root.jsonMode.Store(true)
-	default:
-		return fmt.Errorf("unknown log format `%s`. possible formats: %s", format, []string{"text", "json"})
-	}
-	return nil
-}
-
-// GetFormat reports the currently selected format name.
-func (h *Handler) GetFormat() string {
-	if h.root.jsonMode.Load() {
-		return "json"
-	}
-	return "text"
-}
-
-// SetDisableTimestamp toggles whether Handle zeroes r.Time before
-// dispatching (slog's builtin text/json handlers skip emitting the time
-// attribute on a zero time).
-func (h *Handler) SetDisableTimestamp(v bool) { h.root.disableTimestamp.Store(v) }
-
-// ApplyConfig reads logging.level, logging.format, and (optionally)
-// logging.disable_timestamp from c and applies them to l. The reconfig
-// surface is discovered via structural type-assertion on l.Handler(), so
-// foreign handlers silently opt out of whichever capabilities they do not
-// implement.
-//
-// nebula.Main does NOT call this function on your behalf; callers that want
-// config-driven log level / format / timestamp updates invoke it at
-// startup and register it as a reload callback themselves. This keeps the
-// library from mutating an embedder's logger without their say-so.
-func ApplyConfig(l *slog.Logger, c Config) error {
-	h := l.Handler()
-
-	lvl, err := ParseLevel(strings.ToLower(c.GetString("logging.level", "info")))
-	if err != nil {
-		return err
-	}
-	if ls, ok := h.(interface{ SetLevel(slog.Level) }); ok {
-		ls.SetLevel(lvl)
-	}
-
-	format := strings.ToLower(c.GetString("logging.format", "text"))
-	if fs, ok := h.(interface{ SetFormat(string) error }); ok {
-		if err := fs.SetFormat(format); err != nil {
-			return err
-		}
-	}
-
-	if ts, ok := h.(interface{ SetDisableTimestamp(bool) }); ok {
-		ts.SetDisableTimestamp(c.GetBool("logging.disable_timestamp", false))
-	}
-	return nil
-}
-
-// ParseLevel converts a config-string level name ("trace", "debug", "info",
-// "warn"/"warning", "error", "fatal"/"panic") to a slog.Level. "fatal" and
-// "panic" are accepted for backwards compatibility with pre-slog configs
-// and both map to slog.LevelError.
-func ParseLevel(s string) (slog.Level, error) {
-	switch s {
-	case "trace":
-		return LevelTrace, nil
-	case "debug":
-		return slog.LevelDebug, nil
-	case "info":
-		return slog.LevelInfo, nil
-	case "warn", "warning":
-		return slog.LevelWarn, nil
-	case "error":
-		return slog.LevelError, nil
-	case "fatal", "panic":
-		return slog.LevelError, nil
-	default:
-		return 0, fmt.Errorf("not a valid logging level: %q", s)
-	}
-}
-
-// LevelName returns a human-readable name for a slog.Level matching the
-// strings accepted by ParseLevel.
-func LevelName(l slog.Level) string {
-	switch {
-	case l <= LevelTrace:
-		return "trace"
-	case l <= slog.LevelDebug:
-		return "debug"
-	case l <= slog.LevelInfo:
-		return "info"
-	case l <= slog.LevelWarn:
-		return "warn"
-	default:
-		return "error"
-	}
-}

logging/logger_bench_test.go

@@ -1,90 +0,0 @@
-package logging
-
-import (
-	"context"
-	"io"
-	"log/slog"
-	"testing"
-)
-
-// BenchmarkLogger_* compare the handler returned by NewLogger against a
-// stock slog text handler. The key thing we care about is the per-log
-// cost on a logger that has been derived via .With(), because that is the
-// shape subsystems store on their structs (HostInfo.logger(),
-// lh.l.With("subsystem", ...), etc.) and call from hot paths.
-
-func BenchmarkLogger_Stock_RootInfo(b *testing.B) {
-	l := slog.New(slog.DiscardHandler)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		l.Info("hello", "i", i)
-	}
-}
-
-func BenchmarkLogger_Nebula_RootInfo(b *testing.B) {
-	l := NewLogger(io.Discard)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		l.Info("hello", "i", i)
-	}
-}
-
-func BenchmarkLogger_Stock_DerivedInfo(b *testing.B) {
-	l := slog.New(slog.DiscardHandler).With(
-		"subsystem", "bench",
-		"localIndex", 1234,
-	)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		l.Info("hello", "i", i)
-	}
-}
-
-func BenchmarkLogger_Nebula_DerivedInfo(b *testing.B) {
-	l := NewLogger(io.Discard).With(
-		"subsystem", "bench",
-		"localIndex", 1234,
-	)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		l.Info("hello", "i", i)
-	}
-}
-
-// Gated-off-path benchmarks: mimic the typical hot-path shape
-// `if l.Enabled(ctx, slog.LevelDebug) { ... }` where the log is gated below
-// the active level. This is the dominant pattern in inside.go/outside.go and
-// what we pay on every packet.
-func BenchmarkLogger_Stock_DerivedEnabledGateMiss(b *testing.B) {
-	l := slog.New(slog.DiscardHandler).With(
-		"subsystem", "bench",
-		"localIndex", 1234,
-	)
-	ctx := context.Background()
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		if l.Enabled(ctx, slog.LevelDebug) {
-			l.Debug("hello", "i", i)
-		}
-	}
-}
-
-func BenchmarkLogger_Nebula_DerivedEnabledGateMiss(b *testing.B) {
-	l := NewLogger(io.Discard).With(
-		"subsystem", "bench",
-		"localIndex", 1234,
-	)
-	ctx := context.Background()
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		if l.Enabled(ctx, slog.LevelDebug) {
-			l.Debug("hello", "i", i)
-		}
-	}
-}

main.go

@@ -3,13 +3,16 @@ package nebula
 import (
 	"context"
 	"fmt"
-	"log/slog"
+	"log"
 	"net"
+	"net/http"
+	_ "net/http/pprof"
 	"net/netip"
 	"runtime/debug"
 	"strings"
 	"time"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/sshd"
@@ -20,7 +23,7 @@ import (
 
 type m = map[string]any
 
-func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
+func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
 	ctx, cancel := context.WithCancel(context.Background())
 	// Automatically cancel the context if Main returns an error, to signal all created goroutines to quit.
 	defer func() {
@@ -33,6 +36,11 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		buildVersion = moduleVersion()
 	}
 
+	l := logger
+	l.Formatter = &logrus.TextFormatter{
+		FullTimestamp: true,
+	}
+
 	// Print the config if in test, the exit comes later
 	if configTest {
 		b, err := yaml.Marshal(c.Settings)
@@ -41,9 +49,26 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		}
 
 		// Print the final config
-		l.Info(string(b))
+		l.Println(string(b))
 	}
 
+	//todo!!!
+	go func() {
+		log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
+	}()
+
+	err := configLogger(l, c)
+	if err != nil {
+		return nil, util.ContextualizeIfNeeded("Failed to configure the logger", err)
+	}
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		err := configLogger(l, c)
+		if err != nil {
+			l.WithError(err).Error("Failed to configure the logger")
+		}
+	})
+
 	pki, err := NewPKIFromConfig(l, c)
 	if err != nil {
 		return nil, util.ContextualizeIfNeeded("Failed to load PKI from config", err)
@@ -53,9 +78,9 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 	if err != nil {
 		return nil, util.ContextualizeIfNeeded("Error while loading firewall rules", err)
 	}
-	l.Info("Firewall started", "firewallHashes", fw.GetRuleHashes())
+	l.WithField("firewallHashes", fw.GetRuleHashes()).Info("Firewall started")
 
-	ssh, err := sshd.NewSSHServer(l.With("subsystem", "sshd"))
+	ssh, err := sshd.NewSSHServer(l.WithField("subsystem", "sshd"))
 	if err != nil {
 		return nil, util.ContextualizeIfNeeded("Error while creating SSH server", err)
 	}
@@ -64,7 +89,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 	if c.GetBool("sshd.enabled", false) {
 		sshStart, err = configSSH(l, ssh, c)
 		if err != nil {
-			l.Warn("Failed to configure sshd, ssh debugging will not be available", "error", err)
+			l.WithError(err).Warn("Failed to configure sshd, ssh debugging will not be available")
 			sshStart = nil
 		}
 	}
@@ -82,7 +107,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 			routines = 1
 		}
 		if routines > 1 {
-			l.Info("Using multiple routines", "routines", routines)
+			l.WithField("routines", routines).Info("Using multiple routines")
 		}
 	} else {
 		// deprecated and undocumented
@@ -90,7 +115,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		udpQueues := c.GetInt("listen.routines", 1)
 		routines = max(tunQueues, udpQueues)
 		if routines != 1 {
-			l.Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead", "routines", routines)
+			l.WithField("routines", routines).Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead")
 		}
 	}
 
@@ -103,7 +128,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		conntrackCacheTimeout = 1 * time.Second
 	}
 	if conntrackCacheTimeout > 0 {
-		l.Info("Using routine-local conntrack cache", "duration", conntrackCacheTimeout)
+		l.WithField("duration", conntrackCacheTimeout).Info("Using routine-local conntrack cache")
 	}
 
 	var tun overlay.Device
@@ -149,7 +174,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		}
 
 		for i := 0; i < routines; i++ {
-			l.Info("listening", "addr", netip.AddrPortFrom(listenHost, uint16(port)))
+			l.Infof("listening on %v", netip.AddrPortFrom(listenHost, uint16(port)))
 			udpServer, err := udp.NewListener(l, listenHost, port, routines > 1, c.GetInt("listen.batch", 64))
 			if err != nil {
 				return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err)
@@ -184,19 +209,27 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		messageMetrics = newMessageMetricsOnlyRecvError()
 	}
 
+	useRelays := c.GetBool("relay.use_relays", DefaultUseRelays) && !c.GetBool("relay.am_relay", false)
+
 	handshakeConfig := HandshakeConfig{
-		tryInterval:    c.GetDuration("handshakes.try_interval", DefaultHandshakeTryInterval),
-		retries:        int64(c.GetInt("handshakes.retries", DefaultHandshakeRetries)),
-		triggerBuffer:  c.GetInt("handshakes.trigger_buffer", DefaultHandshakeTriggerBuffer),
+		tryInterval:   c.GetDuration("handshakes.try_interval", DefaultHandshakeTryInterval),
+		retries:       int64(c.GetInt("handshakes.retries", DefaultHandshakeRetries)),
+		triggerBuffer: c.GetInt("handshakes.trigger_buffer", DefaultHandshakeTriggerBuffer),
+		useRelays:     useRelays,
+
 		messageMetrics: messageMetrics,
 	}
 
 	handshakeManager := NewHandshakeManager(l, hostMap, lightHouse, udpConns[0], handshakeConfig)
 	lightHouse.handshakeTrigger = handshakeManager.trigger
 
-	ds, err := newDnsServerFromConfig(ctx, l, pki.getCertState(), hostMap, c)
-	if err != nil {
-		l.Warn("Failed to start DNS responder", "error", err)
+	serveDns := false
+	if c.GetBool("lighthouse.serve_dns", false) {
+		if c.GetBool("lighthouse.am_lighthouse", false) {
+			serveDns = true
+		} else {
+			l.Warn("DNS server refusing to run because this host is not a lighthouse.")
+		}
 	}
 
 	ifConfig := &InterfaceConfig{
@@ -205,7 +238,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		Outside:               udpConns[0],
 		pki:                   pki,
 		Firewall:              fw,
-		DnsServer:             ds,
+		ServeDns:              serveDns,
 		HandshakeManager:      handshakeManager,
 		connectionManager:     connManager,
 		lightHouse:            lightHouse,
@@ -232,7 +265,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 
 		ifce.writers = udpConns
 		lightHouse.ifce = ifce
-		lightHouse.bufAlloc = ifce.bufAlloc
 
 		ifce.RegisterConfigChangeCallbacks(c)
 		ifce.reloadDisconnectInvalid(c)
@@ -243,7 +275,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		go handshakeManager.Run(ctx)
 	}
 
-	stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest)
+	statsStart, err := startStats(l, c, buildVersion, configTest)
 	if err != nil {
 		return nil, util.ContextualizeIfNeeded("Failed to start stats emitter", err)
 	}
@@ -256,6 +288,13 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 
 	attachCommands(l, c, ssh, ifce)
 
+	// Start DNS server last to allow using the nebula IP as lighthouse.dns.host
+	var dnsStart func()
+	if lightHouse.amLighthouse && serveDns {
+		l.Debugln("Starting dns server")
+		dnsStart = dnsMain(l, pki.getCertState(), hostMap, c)
+	}
+
 	return &Control{
 		state:                  StateReady,
 		f:                      ifce,
@@ -263,8 +302,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		ctx:                    ctx,
 		cancel:                 cancel,
 		sshStart:               sshStart,
-		statsStart:             stats.Start,
-		dnsStart:               ds.Start,
+		statsStart:             statsStart,
+		dnsStart:               dnsStart,
 		lighthouseStart:        lightHouse.StartUpdateWorker,
 		connectionManagerStart: connManager.Start,
 	}, nil

nebula.pb.go

@@ -124,7 +124,7 @@ func (x NebulaControl_MessageType) String() string {
 }
 
 func (NebulaControl_MessageType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_2d65afa7693df5ef, []int{6, 0}
+	return fileDescriptor_2d65afa7693df5ef, []int{8, 0}
 }
 
 type NebulaMeta struct {
@@ -489,6 +489,142 @@ func (m *NebulaPing) GetTime() uint64 {
 	return 0
 }
 
+type NebulaHandshake struct {
+	Details *NebulaHandshakeDetails `protobuf:"bytes,1,opt,name=Details,proto3" json:"Details,omitempty"`
+	Hmac    []byte                  `protobuf:"bytes,2,opt,name=Hmac,proto3" json:"Hmac,omitempty"`
+}
+
+func (m *NebulaHandshake) Reset()         { *m = NebulaHandshake{} }
+func (m *NebulaHandshake) String() string { return proto.CompactTextString(m) }
+func (*NebulaHandshake) ProtoMessage()    {}
+func (*NebulaHandshake) Descriptor() ([]byte, []int) {
+	return fileDescriptor_2d65afa7693df5ef, []int{6}
+}
+func (m *NebulaHandshake) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *NebulaHandshake) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_NebulaHandshake.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *NebulaHandshake) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_NebulaHandshake.Merge(m, src)
+}
+func (m *NebulaHandshake) XXX_Size() int {
+	return m.Size()
+}
+func (m *NebulaHandshake) XXX_DiscardUnknown() {
+	xxx_messageInfo_NebulaHandshake.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_NebulaHandshake proto.InternalMessageInfo
+
+func (m *NebulaHandshake) GetDetails() *NebulaHandshakeDetails {
+	if m != nil {
+		return m.Details
+	}
+	return nil
+}
+
+func (m *NebulaHandshake) GetHmac() []byte {
+	if m != nil {
+		return m.Hmac
+	}
+	return nil
+}
+
+type NebulaHandshakeDetails struct {
+	Cert           []byte `protobuf:"bytes,1,opt,name=Cert,proto3" json:"Cert,omitempty"`
+	InitiatorIndex uint32 `protobuf:"varint,2,opt,name=InitiatorIndex,proto3" json:"InitiatorIndex,omitempty"`
+	ResponderIndex uint32 `protobuf:"varint,3,opt,name=ResponderIndex,proto3" json:"ResponderIndex,omitempty"`
+	Cookie         uint64 `protobuf:"varint,4,opt,name=Cookie,proto3" json:"Cookie,omitempty"`
+	Time           uint64 `protobuf:"varint,5,opt,name=Time,proto3" json:"Time,omitempty"`
+	CertVersion    uint32 `protobuf:"varint,8,opt,name=CertVersion,proto3" json:"CertVersion,omitempty"`
+}
+
+func (m *NebulaHandshakeDetails) Reset()         { *m = NebulaHandshakeDetails{} }
+func (m *NebulaHandshakeDetails) String() string { return proto.CompactTextString(m) }
+func (*NebulaHandshakeDetails) ProtoMessage()    {}
+func (*NebulaHandshakeDetails) Descriptor() ([]byte, []int) {
+	return fileDescriptor_2d65afa7693df5ef, []int{7}
+}
+func (m *NebulaHandshakeDetails) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *NebulaHandshakeDetails) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_NebulaHandshakeDetails.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *NebulaHandshakeDetails) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_NebulaHandshakeDetails.Merge(m, src)
+}
+func (m *NebulaHandshakeDetails) XXX_Size() int {
+	return m.Size()
+}
+func (m *NebulaHandshakeDetails) XXX_DiscardUnknown() {
+	xxx_messageInfo_NebulaHandshakeDetails.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_NebulaHandshakeDetails proto.InternalMessageInfo
+
+func (m *NebulaHandshakeDetails) GetCert() []byte {
+	if m != nil {
+		return m.Cert
+	}
+	return nil
+}
+
+func (m *NebulaHandshakeDetails) GetInitiatorIndex() uint32 {
+	if m != nil {
+		return m.InitiatorIndex
+	}
+	return 0
+}
+
+func (m *NebulaHandshakeDetails) GetResponderIndex() uint32 {
+	if m != nil {
+		return m.ResponderIndex
+	}
+	return 0
+}
+
+func (m *NebulaHandshakeDetails) GetCookie() uint64 {
+	if m != nil {
+		return m.Cookie
+	}
+	return 0
+}
+
+func (m *NebulaHandshakeDetails) GetTime() uint64 {
+	if m != nil {
+		return m.Time
+	}
+	return 0
+}
+
+func (m *NebulaHandshakeDetails) GetCertVersion() uint32 {
+	if m != nil {
+		return m.CertVersion
+	}
+	return 0
+}
+
 type NebulaControl struct {
 	Type                NebulaControl_MessageType `protobuf:"varint,1,opt,name=Type,proto3,enum=nebula.NebulaControl_MessageType" json:"Type,omitempty"`
 	InitiatorRelayIndex uint32                    `protobuf:"varint,2,opt,name=InitiatorRelayIndex,proto3" json:"InitiatorRelayIndex,omitempty"`
@@ -503,7 +639,7 @@ func (m *NebulaControl) Reset()         { *m = NebulaControl{} }
 func (m *NebulaControl) String() string { return proto.CompactTextString(m) }
 func (*NebulaControl) ProtoMessage()    {}
 func (*NebulaControl) Descriptor() ([]byte, []int) {
-	return fileDescriptor_2d65afa7693df5ef, []int{6}
+	return fileDescriptor_2d65afa7693df5ef, []int{8}
 }
 func (m *NebulaControl) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -593,55 +729,65 @@ func init() {
 	proto.RegisterType((*V4AddrPort)(nil), "nebula.V4AddrPort")
 	proto.RegisterType((*V6AddrPort)(nil), "nebula.V6AddrPort")
 	proto.RegisterType((*NebulaPing)(nil), "nebula.NebulaPing")
+	proto.RegisterType((*NebulaHandshake)(nil), "nebula.NebulaHandshake")
+	proto.RegisterType((*NebulaHandshakeDetails)(nil), "nebula.NebulaHandshakeDetails")
 	proto.RegisterType((*NebulaControl)(nil), "nebula.NebulaControl")
 }
 
 func init() { proto.RegisterFile("nebula.proto", fileDescriptor_2d65afa7693df5ef) }
 
 var fileDescriptor_2d65afa7693df5ef = []byte{
-	// 665 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x54, 0xcd, 0x6e, 0xd3, 0x5c,
-	0x10, 0x8d, 0x1d, 0x27, 0x69, 0x27, 0x4d, 0x3e, 0x7f, 0x53, 0x51, 0x12, 0x24, 0xac, 0xe0, 0x45,
-	0x55, 0xb1, 0x48, 0x51, 0x5a, 0xba, 0xa6, 0x2d, 0x42, 0xa9, 0xd4, 0x9f, 0x70, 0x55, 0x8a, 0xc4,
-	0xce, 0xb5, 0x2f, 0x8d, 0x55, 0xc7, 0x37, 0xb5, 0x6f, 0x50, 0xf3, 0x16, 0x3c, 0x0c, 0x0f, 0x01,
-	0xbb, 0x2e, 0x59, 0xa2, 0x66, 0xc9, 0x92, 0x17, 0x40, 0xf7, 0xfa, 0xbf, 0x31, 0xb0, 0xbb, 0x33,
-	0xe7, 0x9c, 0x99, 0xc9, 0xc9, 0x8c, 0x61, 0xcd, 0xa7, 0x97, 0x33, 0xcf, 0xea, 0x4f, 0x03, 0xc6,
-	0x19, 0xd6, 0xa3, 0xc8, 0xfc, 0xa9, 0x02, 0x9c, 0xca, 0xe7, 0x09, 0xe5, 0x16, 0x0e, 0x40, 0x3b,
-	0x9f, 0x4f, 0x69, 0x47, 0xe9, 0x29, 0x5b, 0xed, 0x81, 0xd1, 0x8f, 0x35, 0x19, 0xa3, 0x7f, 0x42,
-	0xc3, 0xd0, 0xba, 0xa2, 0x82, 0x45, 0x24, 0x17, 0x77, 0xa0, 0xf1, 0x9a, 0x72, 0xcb, 0xf5, 0xc2,
-	0x8e, 0xda, 0x53, 0xb6, 0x9a, 0x83, 0xee, 0xb2, 0x2c, 0x26, 0x90, 0x84, 0x69, 0xfe, 0x52, 0xa0,
-	0x99, 0x2b, 0x85, 0x2b, 0xa0, 0x9d, 0x32, 0x9f, 0xea, 0x15, 0x6c, 0xc1, 0xea, 0x90, 0x85, 0xfc,
-	0xed, 0x8c, 0x06, 0x73, 0x5d, 0x41, 0x84, 0x76, 0x1a, 0x12, 0x3a, 0xf5, 0xe6, 0xba, 0x8a, 0x4f,
-	0x60, 0x43, 0xe4, 0xde, 0x4d, 0x1d, 0x8b, 0xd3, 0x53, 0xc6, 0xdd, 0x8f, 0xae, 0x6d, 0x71, 0x97,
-	0xf9, 0x7a, 0x15, 0xbb, 0xf0, 0x48, 0x60, 0x27, 0xec, 0x13, 0x75, 0x0a, 0x90, 0x96, 0x40, 0xa3,
-	0x99, 0x6f, 0x8f, 0x0b, 0x50, 0x0d, 0xdb, 0x00, 0x02, 0x7a, 0x3f, 0x66, 0xd6, 0xc4, 0xd5, 0xeb,
-	0xb8, 0x0e, 0xff, 0x65, 0x71, 0xd4, 0xb6, 0x21, 0x26, 0x1b, 0x59, 0x7c, 0x7c, 0x38, 0xa6, 0xf6,
-	0xb5, 0xbe, 0x22, 0x26, 0x4b, 0xc3, 0x88, 0xb2, 0x8a, 0x4f, 0xa1, 0x5b, 0x3e, 0xd9, 0xbe, 0x7d,
-	0xad, 0x83, 0xf9, 0x4d, 0x85, 0xff, 0x97, 0x4c, 0x41, 0x13, 0xe0, 0xcc, 0x73, 0x2e, 0xa6, 0xfe,
-	0xbe, 0xe3, 0x04, 0xd2, 0xfa, 0xd6, 0x81, 0xda, 0x51, 0x48, 0x2e, 0x8b, 0x9b, 0xd0, 0x48, 0x08,
-	0x75, 0x69, 0xf2, 0x5a, 0x62, 0xb2, 0xc8, 0x91, 0x04, 0xc4, 0x3e, 0xe8, 0x67, 0x9e, 0x43, 0xa8,
-	0x67, 0xcd, 0xe3, 0x54, 0xd8, 0xa9, 0xf5, 0xaa, 0x71, 0xc5, 0x25, 0x0c, 0x07, 0xd0, 0x2a, 0x92,
-	0x1b, 0xbd, 0xea, 0x52, 0xf5, 0x22, 0x05, 0x77, 0xa1, 0x79, 0xb1, 0x2b, 0x9e, 0x23, 0x16, 0x70,
-	0xf1, 0xa7, 0x0b, 0x05, 0x26, 0x8a, 0x0c, 0x22, 0x79, 0x9a, 0x54, 0xed, 0x65, 0x2a, 0xed, 0x81,
-	0x6a, 0x2f, 0xa7, 0xca, 0x68, 0xd8, 0x81, 0x86, 0xcd, 0x66, 0x3e, 0xa7, 0x41, 0xa7, 0x2a, 0x8c,
-	0x21, 0x49, 0x68, 0x6e, 0x82, 0x26, 0x7f, 0x71, 0x1b, 0xd4, 0xa1, 0x2b, 0x5d, 0xd3, 0x88, 0x3a,
-	0x74, 0x45, 0x7c, 0xcc, 0xe4, 0x26, 0x6a, 0x44, 0x3d, 0x66, 0xe6, 0x2e, 0x40, 0x36, 0x06, 0x62,
-	0xa4, 0x8a, 0x5c, 0x26, 0x51, 0x05, 0x04, 0x4d, 0x60, 0x52, 0xd3, 0x22, 0xf2, 0x6d, 0xbe, 0x02,
-	0xc8, 0xc6, 0xf8, 0x57, 0x8f, 0xb4, 0x42, 0x35, 0x57, 0xe1, 0x36, 0x39, 0xac, 0x91, 0xeb, 0x5f,
-	0xfd, 0xfd, 0xb0, 0x04, 0xa3, 0xe4, 0xb0, 0x10, 0xb4, 0x73, 0x77, 0x42, 0xe3, 0x3e, 0xf2, 0x6d,
-	0x9a, 0x4b, 0x67, 0x23, 0xc4, 0x7a, 0x05, 0x57, 0xa1, 0x16, 0x2d, 0xa1, 0x62, 0x7e, 0xa9, 0x42,
-	0x2b, 0x2a, 0x7c, 0xc8, 0x7c, 0x1e, 0x30, 0x0f, 0x5f, 0x16, 0xba, 0x3f, 0x2b, 0x76, 0x8f, 0x49,
-	0x25, 0x03, 0xbc, 0x80, 0xf5, 0x23, 0xdf, 0xe5, 0xae, 0xc5, 0x59, 0x20, 0x57, 0xe0, 0xc8, 0x77,
-	0xe8, 0x6d, 0xec, 0x53, 0x19, 0x24, 0x14, 0x84, 0x86, 0x53, 0xe6, 0x3b, 0x34, 0xaf, 0x88, 0x7c,
-	0x29, 0x83, 0xf0, 0x39, 0xb4, 0x93, 0xa5, 0x3c, 0x67, 0xf2, 0xaf, 0xd1, 0xd2, 0x03, 0x78, 0x80,
-	0xe4, 0x97, 0xfb, 0x4d, 0xc0, 0x26, 0x92, 0x5d, 0x4b, 0xd9, 0x4b, 0x18, 0xf6, 0xa1, 0x99, 0x2f,
-	0x5c, 0x76, 0x38, 0x79, 0x42, 0x7a, 0x0c, 0x69, 0xf1, 0x46, 0x89, 0xa2, 0x48, 0x31, 0x87, 0x7f,
-	0xfa, 0x8e, 0x6d, 0x00, 0x1e, 0x06, 0xd4, 0xe2, 0x54, 0xf2, 0x09, 0xbd, 0x99, 0xd1, 0x90, 0xeb,
-	0x0a, 0x3e, 0x86, 0xf5, 0x42, 0x5e, 0x58, 0x12, 0x52, 0x5d, 0x3d, 0xd8, 0xf9, 0x7a, 0x6f, 0x28,
-	0x77, 0xf7, 0x86, 0xf2, 0xe3, 0xde, 0x50, 0x3e, 0x2f, 0x8c, 0xca, 0xdd, 0xc2, 0xa8, 0x7c, 0x5f,
-	0x18, 0x95, 0x0f, 0xdd, 0x2b, 0x97, 0x8f, 0x67, 0x97, 0x7d, 0x9b, 0x4d, 0xb6, 0x43, 0xcf, 0xb2,
-	0xaf, 0xc7, 0x37, 0xdb, 0xd1, 0x48, 0x97, 0x75, 0xf9, 0x39, 0xdf, 0xf9, 0x1d, 0x00, 0x00, 0xff,
-	0xff, 0x51, 0x0a, 0xe3, 0xd7, 0xde, 0x05, 0x00, 0x00,
+	// 785 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x55, 0xcd, 0x6e, 0xeb, 0x44,
+	0x14, 0x8e, 0x1d, 0x27, 0x4e, 0x4f, 0x7e, 0xae, 0x39, 0x15, 0xc1, 0x41, 0x22, 0x0a, 0x5e, 0x54,
+	0x57, 0x2c, 0x72, 0x51, 0x5a, 0xae, 0x58, 0x72, 0x1b, 0x84, 0xd2, 0xaa, 0x3f, 0x61, 0x54, 0x8a,
+	0xc4, 0x06, 0xb9, 0xf6, 0xd0, 0x58, 0x71, 0x3c, 0xa9, 0x3d, 0x41, 0xcd, 0x5b, 0xf0, 0x30, 0x3c,
+	0x04, 0xec, 0xba, 0x42, 0x2c, 0x51, 0xbb, 0x64, 0xc9, 0x0b, 0xa0, 0x19, 0xff, 0x27, 0x86, 0xbb,
+	0x9b, 0x73, 0xbe, 0xef, 0x3b, 0x73, 0xe6, 0xf3, 0x9c, 0x31, 0x74, 0x02, 0x7a, 0xb7, 0xf1, 0xed,
+	0xf1, 0x3a, 0x64, 0x9c, 0x61, 0x33, 0x8e, 0xac, 0xbf, 0x55, 0x80, 0x2b, 0xb9, 0xbc, 0xa4, 0xdc,
+	0xc6, 0x09, 0x68, 0x37, 0xdb, 0x35, 0x35, 0x95, 0x91, 0xf2, 0xba, 0x37, 0x19, 0x8e, 0x13, 0x4d,
+	0xce, 0x18, 0x5f, 0xd2, 0x28, 0xb2, 0xef, 0xa9, 0x60, 0x11, 0xc9, 0xc5, 0x63, 0xd0, 0xbf, 0xa6,
+	0xdc, 0xf6, 0xfc, 0xc8, 0x54, 0x47, 0xca, 0xeb, 0xf6, 0x64, 0xb0, 0x2f, 0x4b, 0x08, 0x24, 0x65,
+	0x5a, 0xff, 0x28, 0xd0, 0x2e, 0x94, 0xc2, 0x16, 0x68, 0x57, 0x2c, 0xa0, 0x46, 0x0d, 0xbb, 0x70,
+	0x30, 0x63, 0x11, 0xff, 0x76, 0x43, 0xc3, 0xad, 0xa1, 0x20, 0x42, 0x2f, 0x0b, 0x09, 0x5d, 0xfb,
+	0x5b, 0x43, 0xc5, 0x8f, 0xa1, 0x2f, 0x72, 0xdf, 0xad, 0x5d, 0x9b, 0xd3, 0x2b, 0xc6, 0xbd, 0x9f,
+	0x3c, 0xc7, 0xe6, 0x1e, 0x0b, 0x8c, 0x3a, 0x0e, 0xe0, 0x43, 0x81, 0x5d, 0xb2, 0x9f, 0xa9, 0x5b,
+	0x82, 0xb4, 0x14, 0x9a, 0x6f, 0x02, 0x67, 0x51, 0x82, 0x1a, 0xd8, 0x03, 0x10, 0xd0, 0xf7, 0x0b,
+	0x66, 0xaf, 0x3c, 0xa3, 0x89, 0x87, 0xf0, 0x2a, 0x8f, 0xe3, 0x6d, 0x75, 0xd1, 0xd9, 0xdc, 0xe6,
+	0x8b, 0xe9, 0x82, 0x3a, 0x4b, 0xa3, 0x25, 0x3a, 0xcb, 0xc2, 0x98, 0x72, 0x80, 0x9f, 0xc0, 0xa0,
+	0xba, 0xb3, 0x77, 0xce, 0xd2, 0x00, 0xeb, 0x77, 0x15, 0x3e, 0xd8, 0x33, 0x05, 0x2d, 0x80, 0x6b,
+	0xdf, 0xbd, 0x5d, 0x07, 0xef, 0x5c, 0x37, 0x94, 0xd6, 0x77, 0x4f, 0x55, 0x53, 0x21, 0x85, 0x2c,
+	0x1e, 0x81, 0x9e, 0x12, 0x9a, 0xd2, 0xe4, 0x4e, 0x6a, 0xb2, 0xc8, 0x91, 0x14, 0xc4, 0x31, 0x18,
+	0xd7, 0xbe, 0x4b, 0xa8, 0x6f, 0x6f, 0x93, 0x54, 0x64, 0x36, 0x46, 0xf5, 0xa4, 0xe2, 0x1e, 0x86,
+	0x13, 0xe8, 0x96, 0xc9, 0xfa, 0xa8, 0xbe, 0x57, 0xbd, 0x4c, 0xc1, 0x13, 0x68, 0xdf, 0x9e, 0x88,
+	0xe5, 0x9c, 0x85, 0x5c, 0x7c, 0x74, 0xa1, 0xc0, 0x54, 0x91, 0x43, 0xa4, 0x48, 0x93, 0xaa, 0xb7,
+	0xb9, 0x4a, 0xdb, 0x51, 0xbd, 0x2d, 0xa8, 0x72, 0x1a, 0x9a, 0xa0, 0x3b, 0x6c, 0x13, 0x70, 0x1a,
+	0x9a, 0x75, 0x61, 0x0c, 0x49, 0x43, 0xeb, 0x08, 0x34, 0x79, 0xe2, 0x1e, 0xa8, 0x33, 0x4f, 0xba,
+	0xa6, 0x11, 0x75, 0xe6, 0x89, 0xf8, 0x82, 0xc9, 0x9b, 0xa8, 0x11, 0xf5, 0x82, 0x59, 0x27, 0x00,
+	0x79, 0x1b, 0x88, 0xb1, 0x2a, 0x76, 0x99, 0xc4, 0x15, 0x10, 0x34, 0x81, 0x49, 0x4d, 0x97, 0xc8,
+	0xb5, 0xf5, 0x15, 0x40, 0xde, 0xc6, 0xfb, 0xf6, 0xc8, 0x2a, 0xd4, 0x0b, 0x15, 0x1e, 0xd3, 0xc1,
+	0x9a, 0x7b, 0xc1, 0xfd, 0xff, 0x0f, 0x96, 0x60, 0x54, 0x0c, 0x16, 0x82, 0x76, 0xe3, 0xad, 0x68,
+	0xb2, 0x8f, 0x5c, 0x5b, 0xd6, 0xde, 0xd8, 0x08, 0xb1, 0x51, 0xc3, 0x03, 0x68, 0xc4, 0x97, 0x50,
+	0xb1, 0x7e, 0x84, 0x57, 0x71, 0xdd, 0x99, 0x1d, 0xb8, 0xd1, 0xc2, 0x5e, 0x52, 0xfc, 0x32, 0x9f,
+	0x51, 0x45, 0x5e, 0x9f, 0x9d, 0x0e, 0x32, 0xe6, 0xee, 0xa0, 0x8a, 0x26, 0x66, 0x2b, 0xdb, 0x91,
+	0x4d, 0x74, 0x88, 0x5c, 0x5b, 0x7f, 0x28, 0xd0, 0xaf, 0xd6, 0x09, 0xfa, 0x94, 0x86, 0x5c, 0xee,
+	0xd2, 0x21, 0x72, 0x8d, 0x47, 0xd0, 0x3b, 0x0b, 0x3c, 0xee, 0xd9, 0x9c, 0x85, 0x67, 0x81, 0x4b,
+	0x1f, 0x13, 0xa7, 0x77, 0xb2, 0x82, 0x47, 0x68, 0xb4, 0x66, 0x81, 0x4b, 0x13, 0x5e, 0xec, 0xe7,
+	0x4e, 0x16, 0xfb, 0xd0, 0x9c, 0x32, 0xb6, 0xf4, 0xa8, 0xa9, 0x49, 0x67, 0x92, 0x28, 0xf3, 0xab,
+	0x91, 0xfb, 0x85, 0x23, 0x68, 0x8b, 0x1e, 0x6e, 0x69, 0x18, 0x79, 0x2c, 0x30, 0x5b, 0xb2, 0x60,
+	0x31, 0x75, 0xae, 0xb5, 0x9a, 0x86, 0x7e, 0xae, 0xb5, 0x74, 0xa3, 0x65, 0xfd, 0x5a, 0x87, 0x6e,
+	0x7c, 0xb0, 0x29, 0x0b, 0x78, 0xc8, 0x7c, 0xfc, 0xa2, 0xf4, 0xdd, 0x3e, 0x2d, 0xbb, 0x96, 0x90,
+	0x2a, 0x3e, 0xdd, 0xe7, 0x70, 0x98, 0x1d, 0x4e, 0x0e, 0x4f, 0xf1, 0xdc, 0x55, 0x90, 0x50, 0x64,
+	0xc7, 0x2c, 0x28, 0x62, 0x07, 0xaa, 0x20, 0xfc, 0x0c, 0x7a, 0xe9, 0x38, 0xdf, 0x30, 0x79, 0xa9,
+	0xb5, 0xec, 0xe9, 0xd8, 0x41, 0x8a, 0xcf, 0xc2, 0x37, 0x21, 0x5b, 0x49, 0x76, 0x23, 0x63, 0xef,
+	0x61, 0x38, 0x86, 0x76, 0xb1, 0x70, 0xd5, 0x93, 0x53, 0x24, 0x64, 0xcf, 0x48, 0x56, 0x5c, 0xaf,
+	0x50, 0x94, 0x29, 0xd6, 0xec, 0xbf, 0xfe, 0x00, 0x7d, 0xc0, 0x69, 0x48, 0x6d, 0x4e, 0x25, 0x9f,
+	0xd0, 0x87, 0x0d, 0x8d, 0xb8, 0xa1, 0xe0, 0x47, 0x70, 0x58, 0xca, 0x0b, 0x4b, 0x22, 0x6a, 0xa8,
+	0xa7, 0xc7, 0xbf, 0x3d, 0x0f, 0x95, 0xa7, 0xe7, 0xa1, 0xf2, 0xd7, 0xf3, 0x50, 0xf9, 0xe5, 0x65,
+	0x58, 0x7b, 0x7a, 0x19, 0xd6, 0xfe, 0x7c, 0x19, 0xd6, 0x7e, 0x18, 0xdc, 0x7b, 0x7c, 0xb1, 0xb9,
+	0x1b, 0x3b, 0x6c, 0xf5, 0x26, 0xf2, 0x6d, 0x67, 0xb9, 0x78, 0x78, 0x13, 0xb7, 0x74, 0xd7, 0x94,
+	0x3f, 0xc2, 0xe3, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xea, 0x6f, 0xbc, 0x50, 0x18, 0x07, 0x00,
+	0x00,
 }
 
 func (m *NebulaMeta) Marshal() (dAtA []byte, err error) {
@@ -926,6 +1072,103 @@ func (m *NebulaPing) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *NebulaHandshake) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *NebulaHandshake) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *NebulaHandshake) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Hmac) > 0 {
+		i -= len(m.Hmac)
+		copy(dAtA[i:], m.Hmac)
+		i = encodeVarintNebula(dAtA, i, uint64(len(m.Hmac)))
+		i--
+		dAtA[i] = 0x12
+	}
+	if m.Details != nil {
+		{
+			size, err := m.Details.MarshalToSizedBuffer(dAtA[:i])
+			if err != nil {
+				return 0, err
+			}
+			i -= size
+			i = encodeVarintNebula(dAtA, i, uint64(size))
+		}
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *NebulaHandshakeDetails) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *NebulaHandshakeDetails) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *NebulaHandshakeDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.CertVersion != 0 {
+		i = encodeVarintNebula(dAtA, i, uint64(m.CertVersion))
+		i--
+		dAtA[i] = 0x40
+	}
+	if m.Time != 0 {
+		i = encodeVarintNebula(dAtA, i, uint64(m.Time))
+		i--
+		dAtA[i] = 0x28
+	}
+	if m.Cookie != 0 {
+		i = encodeVarintNebula(dAtA, i, uint64(m.Cookie))
+		i--
+		dAtA[i] = 0x20
+	}
+	if m.ResponderIndex != 0 {
+		i = encodeVarintNebula(dAtA, i, uint64(m.ResponderIndex))
+		i--
+		dAtA[i] = 0x18
+	}
+	if m.InitiatorIndex != 0 {
+		i = encodeVarintNebula(dAtA, i, uint64(m.InitiatorIndex))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.Cert) > 0 {
+		i -= len(m.Cert)
+		copy(dAtA[i:], m.Cert)
+		i = encodeVarintNebula(dAtA, i, uint64(len(m.Cert)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
 func (m *NebulaControl) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -1132,6 +1375,51 @@ func (m *NebulaPing) Size() (n int) {
 	return n
 }
 
+func (m *NebulaHandshake) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if m.Details != nil {
+		l = m.Details.Size()
+		n += 1 + l + sovNebula(uint64(l))
+	}
+	l = len(m.Hmac)
+	if l > 0 {
+		n += 1 + l + sovNebula(uint64(l))
+	}
+	return n
+}
+
+func (m *NebulaHandshakeDetails) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Cert)
+	if l > 0 {
+		n += 1 + l + sovNebula(uint64(l))
+	}
+	if m.InitiatorIndex != 0 {
+		n += 1 + sovNebula(uint64(m.InitiatorIndex))
+	}
+	if m.ResponderIndex != 0 {
+		n += 1 + sovNebula(uint64(m.ResponderIndex))
+	}
+	if m.Cookie != 0 {
+		n += 1 + sovNebula(uint64(m.Cookie))
+	}
+	if m.Time != 0 {
+		n += 1 + sovNebula(uint64(m.Time))
+	}
+	if m.CertVersion != 0 {
+		n += 1 + sovNebula(uint64(m.CertVersion))
+	}
+	return n
+}
+
 func (m *NebulaControl) Size() (n int) {
 	if m == nil {
 		return 0
@@ -1948,6 +2236,305 @@ func (m *NebulaPing) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *NebulaHandshake) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowNebula
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: NebulaHandshake: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: NebulaHandshake: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Details", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowNebula
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthNebula
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthNebula
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if m.Details == nil {
+				m.Details = &NebulaHandshakeDetails{}
+			}
+			if err := m.Details.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Hmac", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowNebula
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthNebula
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthNebula
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Hmac = append(m.Hmac[:0], dAtA[iNdEx:postIndex]...)
+			if m.Hmac == nil {
+				m.Hmac = []byte{}
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipNebula(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthNebula
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *NebulaHandshakeDetails) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowNebula
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: NebulaHandshakeDetails: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: NebulaHandshakeDetails: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Cert", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowNebula
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthNebula
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthNebula
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Cert = append(m.Cert[:0], dAtA[iNdEx:postIndex]...)
+			if m.Cert == nil {
+				m.Cert = []byte{}
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field InitiatorIndex", wireType)
+			}
+			m.InitiatorIndex = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowNebula
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.InitiatorIndex |= uint32(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ResponderIndex", wireType)
+			}
+			m.ResponderIndex = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowNebula
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.ResponderIndex |= uint32(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Cookie", wireType)
+			}
+			m.Cookie = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowNebula
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Cookie |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 5:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Time", wireType)
+			}
+			m.Time = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowNebula
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Time |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 8:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field CertVersion", wireType)
+			}
+			m.CertVersion = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowNebula
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.CertVersion |= uint32(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipNebula(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthNebula
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func (m *NebulaControl) Unmarshal(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0

nebula.proto

@@ -60,9 +60,21 @@ message NebulaPing {
 	uint64 Time = 2;
 }
 
-// NebulaHandshake / NebulaHandshakeDetails moved to
-// handshake/handshake.proto. The handshake package speaks that wire format
-// directly via a hand-written encoder/decoder.
+message NebulaHandshake {
+  NebulaHandshakeDetails Details = 1;
+  bytes Hmac = 2;
+}
+
+message NebulaHandshakeDetails {
+  bytes Cert = 1;
+  uint32 InitiatorIndex = 2;
+  uint32 ResponderIndex = 3;
+  uint64 Cookie = 4;
+  uint64 Time = 5;
+  uint32 CertVersion = 8;
+  // reserved for WIP multiport
+  reserved 6, 7;
+}
 
 message NebulaControl {
   enum MessageType {

noise.go

@@ -14,21 +14,10 @@ type endianness interface {
 
 var noiseEndianness endianness = binary.BigEndian
 
-// NonceSize is the AEAD nonce length used by all ciphers nebula supports
-// today (AES-GCM and ChaCha20-Poly1305 both use 96-bit nonces). Encrypt-
-// and DecryptDanger lay out the nonce as 4 zero bytes followed by an 8-byte
-// big-endian counter; if a future cipher with a different nonce size is
-// added, this constant and those layouts must change together.
-const NonceSize = 12
-
-// AEADOverhead is the AEAD authentication tag length the ciphers nebula
-// supports append to ciphertext. Both AES-GCM and ChaCha20-Poly1305 use
-// 128-bit tags. NebulaCipherState.Overhead() returns this dynamically from
-// the cipher; the constant is for sizing buffers at construction time.
-const AEADOverhead = 16
-
 type NebulaCipherState struct {
 	c cipher.AEAD
+	//k [32]byte
+	//n uint64
 }
 
 func NewNebulaCipherState(s *noise.CipherState) *NebulaCipherState {

outside.go

@@ -1,16 +1,15 @@
 package nebula
 
 import (
-	"context"
 	"encoding/binary"
 	"errors"
-	"log/slog"
 	"net/netip"
 	"time"
 
 	"github.com/google/gopacket/layers"
 	"golang.org/x/net/ipv6"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"golang.org/x/net/ipv4"
@@ -20,17 +19,12 @@ const (
 	minFwPacketLen = 4
 )
 
-func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []byte, lhf *LightHouseHandler, q int, localCache firewall.ConntrackCache) {
-	h := buf.H
+func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
 	err := h.Parse(packet)
 	if err != nil {
 		// Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
 		if len(packet) > 1 {
-			f.l.Info("Error while parsing inbound packet",
-				"from", via,
-				"error", err,
-				"packet", packet,
-			)
+			f.l.WithField("packet", packet).Infof("Error while parsing inbound packet from %s: %s", via, err)
 		}
 		return
 	}
@@ -38,8 +32,8 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 	//l.Error("in packet ", header, packet[HeaderLen:])
 	if !via.IsRelayed {
 		if f.myVpnNetworksTable.Contains(via.UdpAddr.Addr()) {
-			if f.l.Enabled(context.Background(), slog.LevelDebug) {
-				f.l.Debug("Refusing to process double encrypted packet", "from", via)
+			if f.l.Level >= logrus.DebugLevel {
+				f.l.WithField("from", via).Debug("Refusing to process double encrypted packet")
 			}
 			return
 		}
@@ -66,7 +60,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 
 		switch h.Subtype {
 		case header.MessageNone:
-			if !f.decryptToTun(hostinfo, h.MessageCounter, buf, packet, q, localCache) {
+			if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, nb, q, localCache) {
 				return
 			}
 		case header.MessageRelay:
@@ -77,9 +71,8 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 			// which will gracefully fail in the DecryptDanger call.
 			signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
 			signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
-			// AAD-only validation: passing dst=nil since there's no plaintext
-			// to recover (ciphertext is just the trailing AEAD tag).
-			if _, err = hostinfo.ConnectionState.dKey.DecryptDanger(nil, signedPayload, signatureValue, h.MessageCounter, buf.NB); err != nil {
+			out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
+			if err != nil {
 				return
 			}
 			// Successfully validated the thing. Get rid of the Relay header.
@@ -94,10 +87,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 			if !ok {
 				// The only way this happens is if hostmap has an index to the correct HostInfo, but the HostInfo is missing
 				// its internal mapping. This should never happen.
-				hostinfo.logger(f.l).Error("HostInfo missing remote relay index",
-					"vpnAddrs", hostinfo.vpnAddrs,
-					"remoteIndex", h.RemoteIndex,
-				)
+				hostinfo.logger(f.l).WithFields(logrus.Fields{"vpnAddrs": hostinfo.vpnAddrs, "remoteIndex": h.RemoteIndex}).Error("HostInfo missing remote relay index")
 				return
 			}
 
@@ -112,18 +102,13 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 					relay:     relay,
 					IsRelayed: true,
 				}
-				buf.Reset()
-				f.readOutsidePackets(via, buf, signedPayload, lhf, q, localCache)
+				f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
 				return
 			case ForwardingType:
 				// Find the target HostInfo relay object
 				targetHI, targetRelay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relay.PeerAddr)
 				if err != nil {
-					hostinfo.logger(f.l).Info("Failed to find target host info by ip",
-						"relayTo", relay.PeerAddr,
-						"error", err,
-						"hostinfo.vpnAddrs", hostinfo.vpnAddrs,
-					)
+					hostinfo.logger(f.l).WithField("relayTo", relay.PeerAddr).WithError(err).WithField("hostinfo.vpnAddrs", hostinfo.vpnAddrs).Info("Failed to find target host info by ip")
 					return
 				}
 
@@ -133,17 +118,13 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 					case ForwardingType:
 						// Forward this packet through the relay tunnel
 						// Find the target HostInfo
-						f.SendVia(targetHI, targetRelay, signedPayload, buf)
+						f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
 						return
 					case TerminalType:
 						hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
 					}
 				} else {
-					hostinfo.logger(f.l).Info("Unexpected target relay state",
-						"relayTo", relay.PeerAddr,
-						"relayFrom", hostinfo.vpnAddrs[0],
-						"targetRelayState", targetRelay.State,
-					)
+					hostinfo.logger(f.l).WithFields(logrus.Fields{"relayTo": relay.PeerAddr, "relayFrom": hostinfo.vpnAddrs[0], "targetRelayState": targetRelay.State}).Info("Unexpected target relay state")
 					return
 				}
 			}
@@ -155,13 +136,11 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).Error("Failed to decrypt lighthouse packet",
-				"error", err,
-				"from", via,
-				"packet", packet,
-			)
+			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+				WithField("packet", packet).
+				Error("Failed to decrypt lighthouse packet")
 			return
 		}
 
@@ -176,21 +155,19 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).Error("Failed to decrypt test packet",
-				"error", err,
-				"from", via,
-				"packet", packet,
-			)
+			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+				WithField("packet", packet).
+				Error("Failed to decrypt test packet")
 			return
 		}
 
 		if h.Subtype == header.TestRequest {
 			// This testRequest might be from TryPromoteBest, so we should roam
-			// to the new IP address before responding.
+			// to the new IP address before responding
 			f.handleHostRoaming(hostinfo, via)
-			f.send(header.Test, header.TestReply, ci, hostinfo, d, buf)
+			f.send(header.Test, header.TestReply, ci, hostinfo, d, nb, out)
 		}
 
 		// Fallthrough to the bottom to record incoming traffic
@@ -213,17 +190,16 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 		if !f.handleEncrypted(ci, via, h) {
 			return
 		}
-		_, err = f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
+		_, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).Error("Failed to decrypt CloseTunnel packet",
-				"error", err,
-				"from", via,
-				"packet", packet,
-			)
+			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+				WithField("packet", packet).
+				Error("Failed to decrypt CloseTunnel packet")
 			return
 		}
 
-		hostinfo.logger(f.l).Info("Close tunnel received, tearing down.", "from", via)
+		hostinfo.logger(f.l).WithField("from", via).
+			Info("Close tunnel received, tearing down.")
 
 		f.closeTunnel(hostinfo)
 		return
@@ -233,13 +209,11 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).Error("Failed to decrypt Control packet",
-				"error", err,
-				"from", via,
-				"packet", packet,
-			)
+			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+				WithField("packet", packet).
+				Error("Failed to decrypt Control packet")
 			return
 		}
 
@@ -247,9 +221,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 
 	default:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("Unexpected packet received", "from", via)
-		}
+		hostinfo.logger(f.l).Debugf("Unexpected packet received from %s", via)
 		return
 	}
 
@@ -269,35 +241,26 @@ func (f *Interface) closeTunnel(hostInfo *HostInfo) {
 
 // sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote
 func (f *Interface) sendCloseTunnel(h *HostInfo) {
-	buf := f.bufAlloc.Acquire()
-	defer f.bufAlloc.Release(buf)
-	f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, buf)
+	f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
 }
 
 func (f *Interface) handleHostRoaming(hostinfo *HostInfo, via ViaSender) {
 	if !via.IsRelayed && hostinfo.remote != via.UdpAddr {
 		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
-			if f.l.Enabled(context.Background(), slog.LevelDebug) {
-				hostinfo.logger(f.l).Debug("lighthouse.remote_allow_list denied roaming", "newAddr", via.UdpAddr)
-			}
+			hostinfo.logger(f.l).WithField("newAddr", via.UdpAddr).Debug("lighthouse.remote_allow_list denied roaming")
 			return
 		}
 
 		if !hostinfo.lastRoam.IsZero() && via.UdpAddr == hostinfo.lastRoamRemote && time.Since(hostinfo.lastRoam) < RoamingSuppressSeconds*time.Second {
-			if f.l.Enabled(context.Background(), slog.LevelDebug) {
-				hostinfo.logger(f.l).Debug("Suppressing roam back to previous remote",
-					"suppressSeconds", RoamingSuppressSeconds,
-					"udpAddr", hostinfo.remote,
-					"newAddr", via.UdpAddr,
-				)
+			if f.l.Level >= logrus.DebugLevel {
+				hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", via.UdpAddr).
+					Debugf("Suppressing roam back to previous remote for %d seconds", RoamingSuppressSeconds)
 			}
 			return
 		}
 
-		hostinfo.logger(f.l).Info("Host roamed to new udp ip/port.",
-			"udpAddr", hostinfo.remote,
-			"newAddr", via.UdpAddr,
-		)
+		hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", via.UdpAddr).
+			Info("Host roamed to new udp ip/port.")
 		hostinfo.lastRoam = time.Now()
 		hostinfo.lastRoamRemote = hostinfo.remote
 		hostinfo.SetRemote(via.UdpAddr)
@@ -520,63 +483,61 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
 	return nil
 }
 
-func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, buf *WireBuffer, packet []byte, h *header.H) ([]byte, error) {
-	plaintext, err := buf.DecryptForHandler(hostinfo.ConnectionState, packet, mc)
+func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []byte, h *header.H, nb []byte) ([]byte, error) {
+	var err error
+	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], mc, nb)
 	if err != nil {
 		return nil, err
 	}
 
 	if !hostinfo.ConnectionState.window.Update(f.l, mc) {
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("dropping out of window packet", "header", h)
-		}
+		hostinfo.logger(f.l).WithField("header", h).
+			Debugln("dropping out of window packet")
 		return nil, errors.New("out of window packet")
 	}
 
-	return plaintext, nil
+	return out, nil
 }
 
-func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, buf *WireBuffer, packet []byte, q int, localCache firewall.ConntrackCache) bool {
-	if err := buf.DecryptDatagram(hostinfo.ConnectionState, packet, messageCounter); err != nil {
-		hostinfo.logger(f.l).Error("Failed to decrypt packet", "error", err)
+func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) bool {
+	var err error
+
+	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
+	if err != nil {
+		hostinfo.logger(f.l).WithError(err).Error("Failed to decrypt packet")
 		return false
 	}
 
-	ipPacket := buf.IPPacket()
-	if err := newPacket(ipPacket, true, buf.FwPacket); err != nil {
-		hostinfo.logger(f.l).Warn("Error while validating inbound packet",
-			"error", err,
-			"packet", ipPacket,
-		)
+	err = newPacket(out, true, fwPacket)
+	if err != nil {
+		hostinfo.logger(f.l).WithError(err).WithField("packet", out).
+			Warnf("Error while validating inbound packet")
 		return false
 	}
 
 	if !hostinfo.ConnectionState.window.Update(f.l, messageCounter) {
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("dropping out of window packet", "fwPacket", buf.FwPacket)
-		}
+		hostinfo.logger(f.l).WithField("fwPacket", fwPacket).
+			Debugln("dropping out of window packet")
 		return false
 	}
 
-	dropReason := f.firewall.Drop(*buf.FwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason != nil {
-		// NOTE: We hand `packet` (the original UDP ciphertext we already
-		// decrypted from) as the reject-IP scratch since we no longer
-		// need its ciphertext, and it's disjoint from buf.Out where
-		// sendNoMetrics will encrypt the wire packet.
-		f.rejectOutside(ipPacket, hostinfo.ConnectionState, hostinfo, packet, buf, q)
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("dropping inbound packet",
-				"fwPacket", buf.FwPacket,
-				"reason", dropReason,
-			)
+		// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
+		// This gives us a buffer to build the reject packet in
+		f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, nb, packet, q)
+		if f.l.Level >= logrus.DebugLevel {
+			hostinfo.logger(f.l).WithField("fwPacket", fwPacket).
+				WithField("reason", dropReason).
+				Debugln("dropping inbound packet")
 		}
 		return false
 	}
 
 	f.connectionManager.In(hostinfo)
-	if _, err := buf.WriteIPToTUN(f.readers[q]); err != nil {
-		f.l.Error("Failed to write to tun", "error", err)
+	err = f.tunCoalescers[q].Add(out)
+	if err != nil {
+		f.l.WithError(err).Error("Failed to write to tun")
 	}
 	return true
 }
@@ -592,41 +553,35 @@ func (f *Interface) sendRecvError(endpoint netip.AddrPort, index uint32) {
 
 	b := header.Encode(make([]byte, header.Len), header.Version, header.RecvError, 0, index, 0)
 	_ = f.outside.WriteTo(b, endpoint)
-	if f.l.Enabled(context.Background(), slog.LevelDebug) {
-		f.l.Debug("Recv error sent",
-			"index", index,
-			"udpAddr", endpoint,
-		)
+	if f.l.Level >= logrus.DebugLevel {
+		f.l.WithField("index", index).
+			WithField("udpAddr", endpoint).
+			Debug("Recv error sent")
 	}
 }
 
 func (f *Interface) handleRecvError(addr netip.AddrPort, h *header.H) {
 	if !f.acceptRecvErrorConfig.ShouldRecvError(addr) {
-		f.l.Debug("Recv error received, ignoring",
-			"index", h.RemoteIndex,
-			"udpAddr", addr,
-		)
+		f.l.WithField("index", h.RemoteIndex).
+			WithField("udpAddr", addr).
+			Debug("Recv error received, ignoring")
 		return
 	}
 
-	if f.l.Enabled(context.Background(), slog.LevelDebug) {
-		f.l.Debug("Recv error received",
-			"index", h.RemoteIndex,
-			"udpAddr", addr,
-		)
+	if f.l.Level >= logrus.DebugLevel {
+		f.l.WithField("index", h.RemoteIndex).
+			WithField("udpAddr", addr).
+			Debug("Recv error received")
 	}
 
 	hostinfo := f.hostMap.QueryReverseIndex(h.RemoteIndex)
 	if hostinfo == nil {
-		f.l.Debug("Did not find remote index in main hostmap", "remoteIndex", h.RemoteIndex)
+		f.l.WithField("remoteIndex", h.RemoteIndex).Debugln("Did not find remote index in main hostmap")
 		return
 	}
 
 	if hostinfo.remote.IsValid() && hostinfo.remote != addr {
-		f.l.Info("Someone spoofing recv_errors?",
-			"addr", addr,
-			"hostinfoRemote", hostinfo.remote,
-		)
+		f.l.Infoln("Someone spoofing recv_errors? ", addr, hostinfo.remote)
 		return
 	}
 

overlay/coalesce/tcp_coalesce.go

@@ -0,0 +1,484 @@
+package coalesce
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+
+	"github.com/slackhq/nebula/overlay/tio"
+)
+
+// ipProtoTCP is the IANA protocol number for TCP. Hardcoded instead of
+// reaching for golang.org/x/sys/unix — that package doesn't define the
+// constant on Windows, which would break cross-compiles even though this
+// file runs unchanged on every platform.
+const ipProtoTCP = 6
+
+// tcpCoalesceBufSize caps total bytes per superpacket. Mirrors the kernel's
+// sk_gso_max_size of ~64KiB; anything beyond this would be rejected anyway.
+const tcpCoalesceBufSize = 65535
+
+// tcpCoalesceMaxSegs caps how many segments we'll coalesce into a single
+// superpacket. Keeping this well below the kernel's TSO ceiling bounds
+// latency.
+const tcpCoalesceMaxSegs = 64
+
+// tcpCoalesceHdrCap is the scratch space we copy a seed's IP+TCP header
+// into. IPv6 (40) + TCP with full options (60) = 100 bytes.
+const tcpCoalesceHdrCap = 100
+
+// initialSlots is the starting capacity of the slot pool. One flow per
+// packet is the worst case so this matches a typical UDP recvmmsg batch.
+const initialSlots = 64
+
+// flowKey identifies a TCP flow by {src, dst, sport, dport, family}.
+// Comparable, so linear scans over the slot list stay tight.
+type flowKey struct {
+	src, dst     [16]byte
+	sport, dport uint16
+	isV6         bool
+}
+
+// coalesceSlot is one entry in the coalescer's ordered event queue. When
+// passthrough is true the slot holds a single borrowed packet that must be
+// emitted verbatim (non-TCP, non-admissible TCP, or oversize seed). When
+// passthrough is false the slot is an in-progress coalesced superpacket:
+// hdrBuf is a mutable copy of the seed's IP+TCP header (we patch total
+// length and pseudo-header partial at flush), and payIovs are *borrowed*
+// slices from the caller's plaintext buffers — no payload is ever copied.
+// The caller (listenOut) must keep those buffers alive until Flush.
+type coalesceSlot struct {
+	passthrough bool
+	rawPkt      []byte // borrowed when passthrough
+
+	fk       flowKey
+	hdrBuf   [tcpCoalesceHdrCap]byte
+	hdrLen   int
+	ipHdrLen int
+	isV6     bool
+	gsoSize  int
+	numSeg   int
+	totalPay int
+	nextSeq  uint32
+	// psh closes the chain: set when the last-accepted segment had PSH or
+	// was sub-gsoSize. No further appends after that.
+	psh     bool
+	payIovs [][]byte
+}
+
+// TCPCoalescer accumulates adjacent in-flow TCP data segments across
+// multiple concurrent flows and emits each flow's run as a single TSO
+// superpacket via tio.GSOWriter. All output — coalesced or not — is
+// deferred until Flush so arrival order is preserved on the wire. Owns
+// no locks; one coalescer per TUN write queue.
+type TCPCoalescer struct {
+	plainW io.Writer
+	gsoW   tio.GSOWriter // nil when the queue doesn't support TSO
+
+	// slots is the ordered event queue. Flush walks it once and emits each
+	// entry as either a WriteGSO (coalesced) or a plainW.Write (passthrough).
+	slots []*coalesceSlot
+	// openSlots maps a flow key to its most recent non-sealed slot, so new
+	// segments can extend an in-progress superpacket in O(1). Slots are
+	// removed from this map when they close (PSH or short-last-segment),
+	// when a non-admissible packet for that flow arrives, or in Flush.
+	openSlots map[flowKey]*coalesceSlot
+	pool      []*coalesceSlot // free list for reuse
+}
+
+func NewTCPCoalescer(w io.Writer) *TCPCoalescer {
+	c := &TCPCoalescer{
+		plainW:    w,
+		slots:     make([]*coalesceSlot, 0, initialSlots),
+		openSlots: make(map[flowKey]*coalesceSlot, initialSlots),
+		pool:      make([]*coalesceSlot, 0, initialSlots),
+	}
+	if gw, ok := w.(tio.GSOWriter); ok && gw.GSOSupported() {
+		c.gsoW = gw
+	}
+	return c
+}
+
+// parsedTCP holds the fields extracted from a single parse so later steps
+// (admission, slot lookup, canAppend) don't re-walk the header.
+type parsedTCP struct {
+	fk        flowKey
+	ipHdrLen  int
+	tcpHdrLen int
+	hdrLen    int
+	payLen    int
+	seq       uint32
+	flags     byte
+}
+
+// parseTCPBase extracts the flow key and IP/TCP offsets for any TCP packet,
+// regardless of whether it's admissible for coalescing. Returns ok=false
+// for non-TCP or malformed input. Accepts IPv4 (no options, no fragmentation)
+// and IPv6 (no extension headers).
+func parseTCPBase(pkt []byte) (parsedTCP, bool) {
+	var p parsedTCP
+	if len(pkt) < 20 {
+		return p, false
+	}
+	v := pkt[0] >> 4
+	switch v {
+	case 4:
+		ihl := int(pkt[0]&0x0f) * 4
+		if ihl != 20 {
+			return p, false
+		}
+		if pkt[9] != ipProtoTCP {
+			return p, false
+		}
+		// Reject actual fragmentation (MF or non-zero frag offset).
+		if binary.BigEndian.Uint16(pkt[6:8])&0x3fff != 0 {
+			return p, false
+		}
+		totalLen := int(binary.BigEndian.Uint16(pkt[2:4]))
+		if totalLen > len(pkt) || totalLen < ihl {
+			return p, false
+		}
+		p.ipHdrLen = 20
+		p.fk.isV6 = false
+		copy(p.fk.src[:4], pkt[12:16])
+		copy(p.fk.dst[:4], pkt[16:20])
+		pkt = pkt[:totalLen]
+	case 6:
+		if len(pkt) < 40 {
+			return p, false
+		}
+		if pkt[6] != ipProtoTCP {
+			return p, false
+		}
+		payloadLen := int(binary.BigEndian.Uint16(pkt[4:6]))
+		if 40+payloadLen > len(pkt) {
+			return p, false
+		}
+		p.ipHdrLen = 40
+		p.fk.isV6 = true
+		copy(p.fk.src[:], pkt[8:24])
+		copy(p.fk.dst[:], pkt[24:40])
+		pkt = pkt[:40+payloadLen]
+	default:
+		return p, false
+	}
+
+	if len(pkt) < p.ipHdrLen+20 {
+		return p, false
+	}
+	tcpOff := int(pkt[p.ipHdrLen+12]>>4) * 4
+	if tcpOff < 20 || tcpOff > 60 {
+		return p, false
+	}
+	if len(pkt) < p.ipHdrLen+tcpOff {
+		return p, false
+	}
+	p.tcpHdrLen = tcpOff
+	p.hdrLen = p.ipHdrLen + tcpOff
+	p.payLen = len(pkt) - p.hdrLen
+	p.seq = binary.BigEndian.Uint32(pkt[p.ipHdrLen+4 : p.ipHdrLen+8])
+	p.flags = pkt[p.ipHdrLen+13]
+	p.fk.sport = binary.BigEndian.Uint16(pkt[p.ipHdrLen : p.ipHdrLen+2])
+	p.fk.dport = binary.BigEndian.Uint16(pkt[p.ipHdrLen+2 : p.ipHdrLen+4])
+	return p, true
+}
+
+// coalesceable reports whether a parsed TCP segment is eligible for
+// coalescing. Accepts only ACK or ACK|PSH with a non-empty payload.
+func (p parsedTCP) coalesceable() bool {
+	const ack = 0x10
+	const psh = 0x08
+	if p.flags&^(ack|psh) != 0 || p.flags&ack == 0 {
+		return false
+	}
+	return p.payLen > 0
+}
+
+// Add borrows pkt. The caller must keep pkt valid until the next Flush,
+// whether or not the packet was coalesced — passthrough (non-admissible)
+// packets are queued and written at Flush time, not synchronously.
+func (c *TCPCoalescer) Add(pkt []byte) error {
+	if c.gsoW == nil {
+		c.addPassthrough(pkt)
+		return nil
+	}
+
+	info, ok := parseTCPBase(pkt)
+	if !ok {
+		// Non-TCP or malformed — can't possibly collide with an open flow.
+		c.addPassthrough(pkt)
+		return nil
+	}
+	if !info.coalesceable() {
+		// TCP but not admissible (SYN/FIN/RST/URG/CWR/ECE or zero-payload).
+		// Seal this flow's open slot so later in-flow packets don't extend
+		// it and accidentally reorder past this passthrough.
+		delete(c.openSlots, info.fk)
+		c.addPassthrough(pkt)
+		return nil
+	}
+
+	if open := c.openSlots[info.fk]; open != nil {
+		if c.canAppend(open, pkt, info) {
+			c.appendPayload(open, pkt, info)
+			if open.psh {
+				delete(c.openSlots, info.fk)
+			}
+			return nil
+		}
+		// Can't extend — seal it and fall through to seed a fresh slot.
+		delete(c.openSlots, info.fk)
+	}
+	c.seed(pkt, info)
+	return nil
+}
+
+// Flush emits every queued event in arrival order. Coalesced slots go out
+// via WriteGSO; passthrough slots go out via plainW.Write. Returns the
+// first error observed; keeps draining so one bad packet doesn't hold up
+// the rest. After Flush returns, borrowed payload slices may be recycled.
+func (c *TCPCoalescer) Flush() error {
+	var first error
+	for _, s := range c.slots {
+		var err error
+		if s.passthrough {
+			_, err = c.plainW.Write(s.rawPkt)
+		} else {
+			err = c.flushSlot(s)
+		}
+		if err != nil && first == nil {
+			first = err
+		}
+		c.release(s)
+	}
+	for i := range c.slots {
+		c.slots[i] = nil
+	}
+	c.slots = c.slots[:0]
+	for k := range c.openSlots {
+		delete(c.openSlots, k)
+	}
+	return first
+}
+
+func (c *TCPCoalescer) addPassthrough(pkt []byte) {
+	s := c.take()
+	s.passthrough = true
+	s.rawPkt = pkt
+	c.slots = append(c.slots, s)
+}
+
+func (c *TCPCoalescer) seed(pkt []byte, info parsedTCP) {
+	if info.hdrLen > tcpCoalesceHdrCap || info.hdrLen+info.payLen > tcpCoalesceBufSize {
+		// Pathological shape — can't fit our scratch, emit as-is.
+		c.addPassthrough(pkt)
+		return
+	}
+	s := c.take()
+	s.passthrough = false
+	s.rawPkt = nil
+	copy(s.hdrBuf[:], pkt[:info.hdrLen])
+	s.hdrLen = info.hdrLen
+	s.ipHdrLen = info.ipHdrLen
+	s.isV6 = info.fk.isV6
+	s.fk = info.fk
+	s.gsoSize = info.payLen
+	s.numSeg = 1
+	s.totalPay = info.payLen
+	s.nextSeq = info.seq + uint32(info.payLen)
+	s.psh = info.flags&0x08 != 0
+	s.payIovs = append(s.payIovs[:0], pkt[info.hdrLen:info.hdrLen+info.payLen])
+	c.slots = append(c.slots, s)
+	if !s.psh {
+		c.openSlots[info.fk] = s
+	}
+}
+
+// canAppend reports whether info's packet extends the slot's seed: same
+// header shape and stable contents, adjacent seq, not oversized, chain not
+// closed.
+func (c *TCPCoalescer) canAppend(s *coalesceSlot, pkt []byte, info parsedTCP) bool {
+	if s.psh {
+		return false
+	}
+	if info.hdrLen != s.hdrLen {
+		return false
+	}
+	if info.seq != s.nextSeq {
+		return false
+	}
+	if s.numSeg >= tcpCoalesceMaxSegs {
+		return false
+	}
+	if info.payLen > s.gsoSize {
+		return false
+	}
+	if s.hdrLen+s.totalPay+info.payLen > tcpCoalesceBufSize {
+		return false
+	}
+	if !headersMatch(s.hdrBuf[:s.hdrLen], pkt[:info.hdrLen], s.isV6, s.ipHdrLen) {
+		return false
+	}
+	return true
+}
+
+func (c *TCPCoalescer) appendPayload(s *coalesceSlot, pkt []byte, info parsedTCP) {
+	s.payIovs = append(s.payIovs, pkt[info.hdrLen:info.hdrLen+info.payLen])
+	s.numSeg++
+	s.totalPay += info.payLen
+	s.nextSeq = info.seq + uint32(info.payLen)
+	if info.payLen < s.gsoSize || info.flags&0x08 != 0 {
+		s.psh = true
+	}
+}
+
+func (c *TCPCoalescer) take() *coalesceSlot {
+	if n := len(c.pool); n > 0 {
+		s := c.pool[n-1]
+		c.pool[n-1] = nil
+		c.pool = c.pool[:n-1]
+		return s
+	}
+	return &coalesceSlot{}
+}
+
+func (c *TCPCoalescer) release(s *coalesceSlot) {
+	s.passthrough = false
+	s.rawPkt = nil
+	for i := range s.payIovs {
+		s.payIovs[i] = nil
+	}
+	s.payIovs = s.payIovs[:0]
+	s.numSeg = 0
+	s.totalPay = 0
+	s.psh = false
+	c.pool = append(c.pool, s)
+}
+
+// flushSlot patches the header and calls WriteGSO. Does not remove the
+// slot from c.slots.
+func (c *TCPCoalescer) flushSlot(s *coalesceSlot) error {
+	total := s.hdrLen + s.totalPay
+	l4Len := total - s.ipHdrLen
+	hdr := s.hdrBuf[:s.hdrLen]
+
+	if s.isV6 {
+		binary.BigEndian.PutUint16(hdr[4:6], uint16(l4Len))
+	} else {
+		binary.BigEndian.PutUint16(hdr[2:4], uint16(total))
+		hdr[10] = 0
+		hdr[11] = 0
+		binary.BigEndian.PutUint16(hdr[10:12], ipv4HdrChecksum(hdr[:s.ipHdrLen]))
+	}
+
+	var psum uint32
+	if s.isV6 {
+		psum = pseudoSumIPv6(hdr[8:24], hdr[24:40], ipProtoTCP, l4Len)
+	} else {
+		psum = pseudoSumIPv4(hdr[12:16], hdr[16:20], ipProtoTCP, l4Len)
+	}
+	tcsum := s.ipHdrLen + 16
+	binary.BigEndian.PutUint16(hdr[tcsum:tcsum+2], foldOnceNoInvert(psum))
+
+	return c.gsoW.WriteGSO(hdr, s.payIovs, uint16(s.gsoSize), s.isV6, uint16(s.ipHdrLen))
+}
+
+// headersMatch compares two IP+TCP header prefixes for byte-for-byte
+// equality on every field that must be identical across coalesced
+// segments. Size/IPID/IPCsum/seq/flags/tcpCsum are masked out.
+func headersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	if isV6 {
+		// IPv6: bytes [0:4] = version/TC/flow-label, [6:8] = next_hdr/hop,
+		// [8:40] = src+dst. Skip [4:6] payload length.
+		if !bytes.Equal(a[0:4], b[0:4]) {
+			return false
+		}
+		if !bytes.Equal(a[6:40], b[6:40]) {
+			return false
+		}
+	} else {
+		// IPv4: [0:2] version/IHL/TOS, [6:10] flags/fragoff/TTL/proto,
+		// [12:20] src+dst. Skip [2:4] total len, [4:6] id, [10:12] csum.
+		if !bytes.Equal(a[0:2], b[0:2]) {
+			return false
+		}
+		if !bytes.Equal(a[6:10], b[6:10]) {
+			return false
+		}
+		if !bytes.Equal(a[12:20], b[12:20]) {
+			return false
+		}
+	}
+	// TCP: compare [0:4] ports, [8:13] ack+dataoff, [14:16] window,
+	// [18:tcpHdrLen] options (incl. urgent).
+	tcp := ipHdrLen
+	if !bytes.Equal(a[tcp:tcp+4], b[tcp:tcp+4]) {
+		return false
+	}
+	if !bytes.Equal(a[tcp+8:tcp+13], b[tcp+8:tcp+13]) {
+		return false
+	}
+	if !bytes.Equal(a[tcp+14:tcp+16], b[tcp+14:tcp+16]) {
+		return false
+	}
+	if !bytes.Equal(a[tcp+18:], b[tcp+18:]) {
+		return false
+	}
+	return true
+}
+
+// ipv4HdrChecksum computes the IPv4 header checksum over hdr (which must
+// already have its checksum field zeroed) and returns the folded/inverted
+// 16-bit value to store.
+func ipv4HdrChecksum(hdr []byte) uint16 {
+	var sum uint32
+	for i := 0; i+1 < len(hdr); i += 2 {
+		sum += uint32(binary.BigEndian.Uint16(hdr[i : i+2]))
+	}
+	if len(hdr)%2 == 1 {
+		sum += uint32(hdr[len(hdr)-1]) << 8
+	}
+	for sum>>16 != 0 {
+		sum = (sum & 0xffff) + (sum >> 16)
+	}
+	return ^uint16(sum)
+}
+
+// pseudoSumIPv4 / pseudoSumIPv6 build the TCP pseudo-header partial sum
+// expected by the virtio NEEDS_CSUM kernel path: the 32-bit accumulator
+// before folding.
+func pseudoSumIPv4(src, dst []byte, proto byte, l4Len int) uint32 {
+	var sum uint32
+	sum += uint32(binary.BigEndian.Uint16(src[0:2]))
+	sum += uint32(binary.BigEndian.Uint16(src[2:4]))
+	sum += uint32(binary.BigEndian.Uint16(dst[0:2]))
+	sum += uint32(binary.BigEndian.Uint16(dst[2:4]))
+	sum += uint32(proto)
+	sum += uint32(l4Len)
+	return sum
+}
+
+func pseudoSumIPv6(src, dst []byte, proto byte, l4Len int) uint32 {
+	var sum uint32
+	for i := 0; i < 16; i += 2 {
+		sum += uint32(binary.BigEndian.Uint16(src[i : i+2]))
+		sum += uint32(binary.BigEndian.Uint16(dst[i : i+2]))
+	}
+	sum += uint32(l4Len >> 16)
+	sum += uint32(l4Len & 0xffff)
+	sum += uint32(proto)
+	return sum
+}
+
+// foldOnceNoInvert folds the 32-bit accumulator to 16 bits and returns it
+// unchanged (no one's complement). This is what virtio NEEDS_CSUM wants in
+// the L4 checksum field — the kernel will add the payload sum and invert.
+func foldOnceNoInvert(sum uint32) uint16 {
+	for sum>>16 != 0 {
+		sum = (sum & 0xffff) + (sum >> 16)
+	}
+	return uint16(sum)
+}

overlay/coalesce/tcp_coalesce_test.go

@@ -0,0 +1,576 @@
+package coalesce
+
+import (
+	"encoding/binary"
+	"testing"
+)
+
+// fakeTunWriter records plain Writes and WriteGSO calls without touching a
+// real TUN fd. WriteGSO preserves the split between hdr and borrowed pays
+// so tests can inspect each independently.
+type fakeTunWriter struct {
+	gsoEnabled bool
+	writes     [][]byte
+	gsoWrites  []fakeGSOWrite
+}
+
+type fakeGSOWrite struct {
+	hdr       []byte
+	pays      [][]byte
+	gsoSize   uint16
+	isV6      bool
+	csumStart uint16
+}
+
+// total returns hdrLen + sum of pay lens.
+func (g fakeGSOWrite) total() int {
+	n := len(g.hdr)
+	for _, p := range g.pays {
+		n += len(p)
+	}
+	return n
+}
+
+// payLen sums the pays.
+func (g fakeGSOWrite) payLen() int {
+	var n int
+	for _, p := range g.pays {
+		n += len(p)
+	}
+	return n
+}
+
+func (w *fakeTunWriter) Write(p []byte) (int, error) {
+	buf := make([]byte, len(p))
+	copy(buf, p)
+	w.writes = append(w.writes, buf)
+	return len(p), nil
+}
+
+func (w *fakeTunWriter) WriteGSO(hdr []byte, pays [][]byte, gsoSize uint16, isV6 bool, csumStart uint16) error {
+	hcopy := make([]byte, len(hdr))
+	copy(hcopy, hdr)
+	paysCopy := make([][]byte, len(pays))
+	for i, p := range pays {
+		pc := make([]byte, len(p))
+		copy(pc, p)
+		paysCopy[i] = pc
+	}
+	w.gsoWrites = append(w.gsoWrites, fakeGSOWrite{
+		hdr:       hcopy,
+		pays:      paysCopy,
+		gsoSize:   gsoSize,
+		isV6:      isV6,
+		csumStart: csumStart,
+	})
+	return nil
+}
+
+func (w *fakeTunWriter) GSOSupported() bool { return w.gsoEnabled }
+
+// buildTCPv4 constructs a minimal IPv4+TCP packet with the given payload,
+// seq, and flags. Assumes no IP options and a 20-byte TCP header.
+func buildTCPv4(seq uint32, flags byte, payload []byte) []byte {
+	return buildTCPv4Ports(1000, 2000, seq, flags, payload)
+}
+
+// buildTCPv4Ports is buildTCPv4 with caller-specified ports so tests can
+// build distinct flows.
+func buildTCPv4Ports(sport, dport uint16, seq uint32, flags byte, payload []byte) []byte {
+	const ipHdrLen = 20
+	const tcpHdrLen = 20
+	total := ipHdrLen + tcpHdrLen + len(payload)
+	pkt := make([]byte, total)
+
+	pkt[0] = 0x45
+	pkt[1] = 0x00
+	binary.BigEndian.PutUint16(pkt[2:4], uint16(total))
+	binary.BigEndian.PutUint16(pkt[4:6], 0)
+	binary.BigEndian.PutUint16(pkt[6:8], 0x4000)
+	pkt[8] = 64
+	pkt[9] = ipProtoTCP
+	copy(pkt[12:16], []byte{10, 0, 0, 1})
+	copy(pkt[16:20], []byte{10, 0, 0, 2})
+
+	binary.BigEndian.PutUint16(pkt[20:22], sport)
+	binary.BigEndian.PutUint16(pkt[22:24], dport)
+	binary.BigEndian.PutUint32(pkt[24:28], seq)
+	binary.BigEndian.PutUint32(pkt[28:32], 12345)
+	pkt[32] = 0x50
+	pkt[33] = flags
+	binary.BigEndian.PutUint16(pkt[34:36], 0xffff)
+
+	copy(pkt[40:], payload)
+	return pkt
+}
+
+const (
+	tcpAck    = 0x10
+	tcpPsh    = 0x08
+	tcpSyn    = 0x02
+	tcpFin    = 0x01
+	tcpAckPsh = tcpAck | tcpPsh
+)
+
+func TestCoalescerPassthroughWhenGSOUnavailable(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: false}
+	c := NewTCPCoalescer(w)
+	pkt := buildTCPv4(1000, tcpAck, []byte("hello"))
+	if err := c.Add(pkt); err != nil {
+		t.Fatal(err)
+	}
+	// No sync write — passthrough is deferred to Flush.
+	if len(w.writes) != 0 || len(w.gsoWrites) != 0 {
+		t.Fatalf("no Add-time writes: got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
+		t.Fatalf("want single plain write, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
+	}
+}
+
+func TestCoalescerNonTCPPassthrough(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	pkt := make([]byte, 28)
+	pkt[0] = 0x45
+	binary.BigEndian.PutUint16(pkt[2:4], 28)
+	pkt[9] = 1
+	copy(pkt[12:16], []byte{10, 0, 0, 1})
+	copy(pkt[16:20], []byte{10, 0, 0, 2})
+	if err := c.Add(pkt); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
+		t.Fatalf("ICMP should pass through unchanged")
+	}
+}
+
+func TestCoalescerSeedThenFlushAlone(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	pkt := buildTCPv4(1000, tcpAck, make([]byte, 1000))
+	if err := c.Add(pkt); err != nil {
+		t.Fatal(err)
+	}
+	if len(w.writes) != 0 || len(w.gsoWrites) != 0 {
+		t.Fatalf("unexpected output before flush")
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	// Single-segment flush now goes through WriteGSO with GSO_NONE
+	// (virtio NEEDS_CSUM lets the kernel fill in the L4 csum).
+	if len(w.gsoWrites) != 1 || len(w.writes) != 0 {
+		t.Fatalf("single-seg flush: writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
+	}
+	g := w.gsoWrites[0]
+	if g.total() != 40+1000 {
+		t.Errorf("super total=%d want %d", g.total(), 40+1000)
+	}
+	if g.payLen() != 1000 {
+		t.Errorf("payLen=%d want 1000", g.payLen())
+	}
+}
+
+func TestCoalescerCoalescesAdjacentACKs(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	pay := make([]byte, 1200)
+	if err := c.Add(buildTCPv4(1000, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4(2200, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4(3400, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	if len(w.gsoWrites) != 1 {
+		t.Fatalf("want 1 gso write, got %d (plain=%d)", len(w.gsoWrites), len(w.writes))
+	}
+	g := w.gsoWrites[0]
+	if g.gsoSize != 1200 {
+		t.Errorf("gsoSize=%d want 1200", g.gsoSize)
+	}
+	if len(g.hdr) != 40 {
+		t.Errorf("hdrLen=%d want 40", len(g.hdr))
+	}
+	if g.csumStart != 20 {
+		t.Errorf("csumStart=%d want 20", g.csumStart)
+	}
+	if len(g.pays) != 3 {
+		t.Errorf("pay count=%d want 3", len(g.pays))
+	}
+	if g.total() != 40+3*1200 {
+		t.Errorf("superpacket len=%d want %d", g.total(), 40+3*1200)
+	}
+	if tot := binary.BigEndian.Uint16(g.hdr[2:4]); int(tot) != g.total() {
+		t.Errorf("ip total_length=%d want %d", tot, g.total())
+	}
+}
+
+func TestCoalescerRejectsSeqGap(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	pay := make([]byte, 1200)
+	if err := c.Add(buildTCPv4(1000, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4(3000, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	// Each packet flushes as its own single-segment WriteGSO now.
+	if len(w.gsoWrites) != 2 || len(w.writes) != 0 {
+		t.Fatalf("seq gap: want 2 gso writes got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
+	}
+}
+
+func TestCoalescerRejectsFlagMismatch(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	pay := make([]byte, 1200)
+	if err := c.Add(buildTCPv4(1000, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	// SYN|ACK is non-admissible. Must flush matching flow's slot (gso)
+	// and then plain-write the SYN packet itself.
+	syn := buildTCPv4(2200, tcpSyn|tcpAck, pay)
+	if err := c.Add(syn); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	if len(w.writes) != 1 || len(w.gsoWrites) != 1 {
+		t.Fatalf("flag mismatch: want 1 plain + 1 gso, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
+	}
+}
+
+func TestCoalescerRejectsFIN(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	fin := buildTCPv4(1000, tcpAck|tcpFin, []byte("x"))
+	if err := c.Add(fin); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	// FIN isn't admissible — passthrough as plain, no slot, no gso.
+	if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
+		t.Fatalf("FIN should be passthrough, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
+	}
+}
+
+func TestCoalescerShortLastSegmentClosesChain(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	full := make([]byte, 1200)
+	half := make([]byte, 500)
+	if err := c.Add(buildTCPv4(1000, tcpAck, full)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4(2200, tcpAck, half)); err != nil {
+		t.Fatal(err)
+	}
+	// Chain now closed; next packet seeds a new slot on the same flow
+	// after flushing the old one.
+	if err := c.Add(buildTCPv4(2700, tcpAck, full)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	// Expect two gso writes: the first two packets coalesced, then the
+	// third flushed alone (single-seg via GSO_NONE).
+	if len(w.gsoWrites) != 2 {
+		t.Fatalf("want 2 gso writes got %d", len(w.gsoWrites))
+	}
+	if len(w.writes) != 0 {
+		t.Fatalf("want 0 plain writes got %d", len(w.writes))
+	}
+	if w.gsoWrites[0].gsoSize != 1200 {
+		t.Errorf("gsoSize=%d want 1200", w.gsoWrites[0].gsoSize)
+	}
+	if got, want := w.gsoWrites[0].total(), 40+1200+500; got != want {
+		t.Errorf("super len=%d want %d", got, want)
+	}
+}
+
+func TestCoalescerPSHFinalizesChain(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	pay := make([]byte, 1200)
+	if err := c.Add(buildTCPv4(1000, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4(2200, tcpAckPsh, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4(3400, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	// First two coalesce; the third seeds a fresh slot that flushes alone.
+	if len(w.gsoWrites) != 2 {
+		t.Fatalf("want 2 gso writes got %d", len(w.gsoWrites))
+	}
+	if len(w.writes) != 0 {
+		t.Fatalf("want 0 plain writes got %d", len(w.writes))
+	}
+}
+
+func TestCoalescerRejectsDifferentFlow(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	pay := make([]byte, 1200)
+	p1 := buildTCPv4(1000, tcpAck, pay)
+	p2 := buildTCPv4(2200, tcpAck, pay)
+	binary.BigEndian.PutUint16(p2[20:22], 9999)
+	if err := c.Add(p1); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(p2); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	// Two independent flows, each flushes its own single-segment WriteGSO.
+	if len(w.gsoWrites) != 2 || len(w.writes) != 0 {
+		t.Fatalf("diff flow: want 2 gso writes got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
+	}
+}
+
+func TestCoalescerRejectsIPOptions(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	pay := make([]byte, 500)
+	pkt := buildTCPv4(1000, tcpAck, pay)
+	// Bump IHL to 6 to simulate 4 bytes of IP options. Don't actually add
+	// bytes — parser should bail before it matters.
+	pkt[0] = 0x46
+	if err := c.Add(pkt); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	// Non-admissible parse → passthrough as plain.
+	if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
+		t.Fatalf("IP options should passthrough, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
+	}
+}
+
+func TestCoalescerCapBySegments(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	pay := make([]byte, 512)
+	seq := uint32(1000)
+	for i := 0; i < tcpCoalesceMaxSegs+5; i++ {
+		if err := c.Add(buildTCPv4(seq, tcpAck, pay)); err != nil {
+			t.Fatal(err)
+		}
+		seq += uint32(len(pay))
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	for _, g := range w.gsoWrites {
+		segs := len(g.pays)
+		if segs > tcpCoalesceMaxSegs {
+			t.Fatalf("super exceeded seg cap: %d > %d", segs, tcpCoalesceMaxSegs)
+		}
+	}
+}
+
+// TestCoalescerMultipleFlowsInSameBatch proves two interleaved bulk TCP
+// flows coalesce independently in a single Flush.
+func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	pay := make([]byte, 1200)
+
+	// Flow A: sport 1000. Flow B: sport 3000.
+	if err := c.Add(buildTCPv4Ports(1000, 2000, 100, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4Ports(3000, 2000, 500, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4Ports(1000, 2000, 1300, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4Ports(3000, 2000, 1700, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4Ports(1000, 2000, 2500, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4Ports(3000, 2000, 2900, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+
+	if len(w.gsoWrites) != 2 {
+		t.Fatalf("want 2 gso writes (one per flow), got %d", len(w.gsoWrites))
+	}
+	if len(w.writes) != 0 {
+		t.Fatalf("want no plain writes, got %d", len(w.writes))
+	}
+	// Each superpacket should carry 3 segments.
+	for i, g := range w.gsoWrites {
+		if len(g.pays) != 3 {
+			t.Errorf("gso[%d]: segs=%d want 3", i, len(g.pays))
+		}
+		if g.gsoSize != 1200 {
+			t.Errorf("gso[%d]: gsoSize=%d want 1200", i, g.gsoSize)
+		}
+	}
+	// Verify each superpacket carries the source port it was seeded with.
+	seenSports := map[uint16]bool{}
+	for _, g := range w.gsoWrites {
+		sp := binary.BigEndian.Uint16(g.hdr[20:22])
+		seenSports[sp] = true
+	}
+	if !seenSports[1000] || !seenSports[3000] {
+		t.Errorf("expected superpackets for sports 1000 and 3000, got %v", seenSports)
+	}
+}
+
+// TestCoalescerPreservesArrivalOrder confirms that with passthrough and
+// coalesced events both queued, Flush emits them in Add order rather than
+// writing passthrough packets synchronously.
+func TestCoalescerPreservesArrivalOrder(t *testing.T) {
+	w := &orderedFakeWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	// Sequence: coalesceable TCP, ICMP (passthrough), coalesceable TCP on
+	// a different flow. Expected emit order: gso(X), plain(ICMP), gso(Y).
+	pay := make([]byte, 1200)
+	if err := c.Add(buildTCPv4Ports(1000, 2000, 100, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	icmp := make([]byte, 28)
+	icmp[0] = 0x45
+	binary.BigEndian.PutUint16(icmp[2:4], 28)
+	icmp[9] = 1
+	copy(icmp[12:16], []byte{10, 0, 0, 1})
+	copy(icmp[16:20], []byte{10, 0, 0, 3})
+	if err := c.Add(icmp); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4Ports(3000, 2000, 500, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	// Nothing should have hit the writer synchronously.
+	if len(w.events) != 0 {
+		t.Fatalf("Add emitted events synchronously: %v", w.events)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := w.events, []string{"gso", "plain", "gso"}; !stringSliceEq(got, want) {
+		t.Fatalf("flush order=%v want %v", got, want)
+	}
+}
+
+// orderedFakeWriter records only the sequence of call types so tests can
+// assert arrival order without inspecting bytes.
+type orderedFakeWriter struct {
+	gsoEnabled bool
+	events     []string
+}
+
+func (w *orderedFakeWriter) Write(p []byte) (int, error) {
+	w.events = append(w.events, "plain")
+	return len(p), nil
+}
+
+func (w *orderedFakeWriter) WriteGSO(hdr []byte, pays [][]byte, gsoSize uint16, isV6 bool, csumStart uint16) error {
+	w.events = append(w.events, "gso")
+	return nil
+}
+
+func (w *orderedFakeWriter) GSOSupported() bool { return w.gsoEnabled }
+
+func stringSliceEq(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// TestCoalescerInterleavedFlowsPreserveOrdering checks that a non-admissible
+// packet (SYN) mid-flow only flushes its own flow, not others.
+func TestCoalescerInterleavedFlowsPreserveOrdering(t *testing.T) {
+	w := &fakeTunWriter{gsoEnabled: true}
+	c := NewTCPCoalescer(w)
+	pay := make([]byte, 1200)
+
+	// Flow A two segments.
+	if err := c.Add(buildTCPv4Ports(1000, 2000, 100, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4Ports(1000, 2000, 1300, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	// Flow B two segments.
+	if err := c.Add(buildTCPv4Ports(3000, 2000, 500, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Add(buildTCPv4Ports(3000, 2000, 1700, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	// Flow A SYN (non-admissible) — must flush only flow A's slot.
+	syn := buildTCPv4Ports(1000, 2000, 9999, tcpSyn|tcpAck, pay)
+	if err := c.Add(syn); err != nil {
+		t.Fatal(err)
+	}
+	// Flow B continues — should still be coalesced with its seed.
+	if err := c.Add(buildTCPv4Ports(3000, 2000, 2900, tcpAck, pay)); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Expected:
+	//   - 1 gso for flow A (first 2 segments)
+	//   - 1 plain for flow A SYN
+	//   - 1 gso for flow B (3 segments)
+	if len(w.gsoWrites) != 2 {
+		t.Fatalf("want 2 gso writes, got %d", len(w.gsoWrites))
+	}
+	if len(w.writes) != 1 {
+		t.Fatalf("want 1 plain write (SYN), got %d", len(w.writes))
+	}
+	// Find the 3-segment gso (flow B) and the 2-segment gso (flow A).
+	var segCounts []int
+	for _, g := range w.gsoWrites {
+		segCounts = append(segCounts, len(g.pays))
+	}
+	if !(segCounts[0] == 2 && segCounts[1] == 3) && !(segCounts[0] == 3 && segCounts[1] == 2) {
+		t.Errorf("unexpected segment counts: %v (want 2 and 3)", segCounts)
+	}
+}

overlay/device.go

@@ -4,18 +4,21 @@ import (
 	"io"
 	"net/netip"
 
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
+// defaultBatchBufSize is the per-Queue scratch size for Read on backends
+// that don't do TSO segmentation. 65535 covers any single IP packet.
+const defaultBatchBufSize = 65535
+
 type Device interface {
-	io.ReadWriteCloser
+	io.Closer
 	Activate() error
 	Networks() []netip.Prefix
 	Name() string
 	RoutesFor(netip.Addr) routing.Gateways
-	SupportsMultiqueue() bool
-	NewMultiQueueReader() (io.ReadWriteCloser, error)
-	// TunPrefixLen reports the number of bytes the device prepends to every IP packet on the wire.
-	// Currently only non zero for the BSD tun devices.
-	TunPrefixLen() int
+	SupportsMultiqueue() bool //todo remove?
+	NewMultiQueueReader() error
+	Readers() []tio.Queue
 }

overlay/noop.go

@@ -1,18 +1,13 @@
-// Package overlaytest provides fakes of overlay.Device for tests that do
-// not want to touch a real tun device or route table.
-package overlaytest
+package overlay
 
 import (
 	"errors"
-	"io"
 	"net/netip"
 
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
-// NoopTun is an overlay.Device that silently discards every read and write.
-// Useful in tests that need to construct a nebula Interface but do not
-// exercise the datapath.
 type NoopTun struct{}
 
 func (NoopTun) RoutesFor(addr netip.Addr) routing.Gateways {
@@ -31,24 +26,30 @@ func (NoopTun) Name() string {
 	return "noop"
 }
 
-func (NoopTun) Read([]byte) (int, error) {
-	return 0, nil
+func (NoopTun) Read() ([][]byte, error) {
+	return nil, nil
 }
 
 func (NoopTun) Write([]byte) (int, error) {
 	return 0, nil
 }
 
+func (NoopTun) WriteReject(p []byte) (int, error) {
+	return 0, nil
+}
+
 func (NoopTun) SupportsMultiqueue() bool {
 	return false
 }
 
-func (NoopTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, errors.New("unsupported")
+func (NoopTun) NewMultiQueueReader() error {
+	return errors.New("unsupported")
+}
+
+func (NoopTun) Readers() []tio.Queue {
+	return []tio.Queue{NoopTun{}}
 }
 
 func (NoopTun) Close() error {
 	return nil
 }
-
-func (NoopTun) TunPrefixLen() int { return 0 }

overlay/route.go

@@ -2,7 +2,6 @@ package overlay
 
 import (
 	"fmt"
-	"log/slog"
 	"math"
 	"net"
 	"net/netip"
@@ -10,6 +9,7 @@ import (
 	"strconv"
 
 	"github.com/gaissmai/bart"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
 )
@@ -48,14 +48,11 @@ func (r Route) String() string {
 	return s
 }
 
-func makeRouteTree(l *slog.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) {
+func makeRouteTree(l *logrus.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) {
 	routeTree := new(bart.Table[routing.Gateways])
 	for _, r := range routes {
 		if !allowMTU && r.MTU > 0 {
-			l.Warn("route MTU is not supported on this platform",
-				"goos", runtime.GOOS,
-				"route", r,
-			)
+			l.WithField("route", r).Warnf("route MTU is not supported in %s", runtime.GOOS)
 		}
 
 		gateways := r.Via

overlay/route_test.go

@@ -295,7 +295,7 @@ func Test_makeRouteTree(t *testing.T) {
 	routes, err := parseUnsafeRoutes(c, []netip.Prefix{n})
 	require.NoError(t, err)
 	assert.Len(t, routes, 2)
-	routeTree, err := makeRouteTree(test.NewLogger(), routes, true)
+	routeTree, err := makeRouteTree(l, routes, true)
 	require.NoError(t, err)
 
 	ip, err := netip.ParseAddr("1.0.0.2")
@@ -367,7 +367,7 @@ func Test_makeMultipathUnsafeRouteTree(t *testing.T) {
 	routes, err := parseUnsafeRoutes(c, []netip.Prefix{n})
 	require.NoError(t, err)
 	assert.Len(t, routes, 3)
-	routeTree, err := makeRouteTree(test.NewLogger(), routes, true)
+	routeTree, err := makeRouteTree(l, routes, true)
 	require.NoError(t, err)
 
 	ip, err := netip.ParseAddr("192.168.86.1")

overlay/tio/container_gso_linux.go

@@ -0,0 +1,70 @@
+package tio
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+
+	"golang.org/x/sys/unix"
+)
+
+type offloadContainer struct {
+	pq []*Offload
+	// pqi is exactly the same as pq, but stored as the interface type
+	pqi        []Queue
+	shutdownFd int
+}
+
+func NewOffloadContainer() (Container, error) {
+	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create eventfd: %w", err)
+	}
+
+	out := &offloadContainer{
+		pq:         []*Offload{},
+		pqi:        []Queue{},
+		shutdownFd: shutdownFd,
+	}
+
+	return out, nil
+}
+
+func (c *offloadContainer) Queues() []Queue {
+	return c.pqi
+}
+
+func (c *offloadContainer) Add(fd int) error {
+	x, err := newOffload(fd, c.shutdownFd)
+	if err != nil {
+		return err
+	}
+	c.pq = append(c.pq, x)
+	c.pqi = append(c.pqi, x)
+
+	return nil
+}
+
+func (c *offloadContainer) wakeForShutdown() error {
+	var buf [8]byte
+	binary.NativeEndian.PutUint64(buf[:], 1)
+	_, err := unix.Write(c.shutdownFd, buf[:])
+	return err
+}
+
+func (c *offloadContainer) Close() error {
+	errs := []error{}
+
+	// Signal all readers blocked in poll to wake up and exit
+	if err := c.wakeForShutdown(); err != nil {
+		errs = append(errs, err)
+	}
+
+	for _, x := range c.pq {
+		if err := x.Close(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	return errors.Join(errs...)
+}

overlay/tio/container_poll_linux.go

@@ -0,0 +1,69 @@
+package tio
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+
+	"golang.org/x/sys/unix"
+)
+
+type pollContainer struct {
+	pq []*Poll
+	// pqi is exactly the same as pq, but stored as the interface type
+	pqi        []Queue
+	shutdownFd int
+}
+
+func NewPollContainer() (Container, error) {
+	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create eventfd: %w", err)
+	}
+
+	out := &pollContainer{
+		pq:         []*Poll{},
+		pqi:        []Queue{},
+		shutdownFd: shutdownFd,
+	}
+
+	return out, nil
+}
+
+func (c *pollContainer) Queues() []Queue {
+	return c.pqi
+}
+
+func (c *pollContainer) Add(fd int) error {
+	x, err := newPoll(fd, c.shutdownFd)
+	if err != nil {
+		return err
+	}
+	c.pq = append(c.pq, x)
+	c.pqi = append(c.pqi, x)
+
+	return nil
+}
+
+func (c *pollContainer) wakeForShutdown() error {
+	var buf [8]byte
+	binary.NativeEndian.PutUint64(buf[:], 1)
+	_, err := unix.Write(int(c.shutdownFd), buf[:])
+	return err
+}
+
+func (c *pollContainer) Close() error {
+	errs := []error{}
+
+	if err := c.wakeForShutdown(); err != nil {
+		errs = append(errs, err)
+	}
+
+	for _, x := range c.pq {
+		if err := x.Close(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	return errors.Join(errs...)
+}

overlay/tio/tio.go

@@ -0,0 +1,63 @@
+package tio
+
+import "io"
+
+// defaultBatchBufSize is the per-Queue scratch size for Read on backends
+// that don't do TSO segmentation. 65535 covers any single IP packet.
+const defaultBatchBufSize = 65535
+
+type Container interface {
+	Queues() []Queue
+	Add(fd int) error
+
+	io.Closer
+}
+
+// Queue is a readable/writable Poll queue. One Queue is driven by a single
+// read goroutine plus concurrent writers (see Write / WriteReject below).
+type Queue interface {
+	io.Closer
+
+	// Read returns one or more packets. The returned slices are borrowed
+	// from the Queue's internal buffer and are only valid until the next
+	// Read or Close on this Queue — callers must encrypt or copy each
+	// slice before the next call. Not safe for concurrent Reads; exactly
+	// one goroutine per Queue reads.
+	Read() ([][]byte, error)
+
+	// Write emits a single packet on the plaintext (outside→inside)
+	// delivery path. May run concurrently with WriteReject on the same
+	// Queue, but not with itself.
+	Write(p []byte) (int, error)
+
+	// WriteReject writes a single packet that originated from the inside
+	// path (reject replies or self-forward) using scratch state distinct
+	// from Write, so it can run concurrently with Write on the same Queue
+	// without a data race. On backends without a shared-scratch Write, a
+	// trivial delegation to Write is acceptable.
+	WriteReject(p []byte) (int, error)
+}
+
+// GSOWriter is implemented by Queues that can emit a TCP TSO superpacket
+// assembled from a header prefix plus one or more borrowed payload
+// fragments, in a single vectored write (writev with a leading
+// virtio_net_hdr). This lets the coalescer avoid copying payload bytes
+// between the caller's decrypt buffer and the TUN. Backends without GSO
+// support return false from GSOSupported and coalescing is skipped.
+//
+// hdr contains the IPv4/IPv6 + TCP header prefix (mutable — callers will
+// have filled in total length and pseudo-header partial). pays are
+// non-overlapping payload fragments whose concatenation is the full
+// superpacket payload; they are read-only from the writer's perspective
+// and must remain valid until the call returns. gsoSize is the MSS:
+// every segment except possibly the last is exactly that many bytes.
+// csumStart is the byte offset where the TCP header begins within hdr.
+//
+// # TODO fold into Queue
+//
+// hdr's TCP checksum field must already hold the pseudo-header partial
+// sum (single-fold, not inverted), per virtio NEEDS_CSUM semantics.
+type GSOWriter interface {
+	WriteGSO(hdr []byte, pays [][]byte, gsoSize uint16, isV6 bool, csumStart uint16) error
+	GSOSupported() bool
+}

overlay/tio/tio_gso_linux.go

@@ -0,0 +1,390 @@
+package tio
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"sync/atomic"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+// Space for segmented output. Worst case is many small segments, each paying
+// an IP+TCP header. Should be a multiple of 64KiB.
+// const tunSegBufSize = 0xffff * 8 TODO larger? config?
+const tunSegBufSize = 131072
+
+// tunSegBufCap is the total size we allocate for the per-reader segment
+// buffer. It is sized as one worst-case TSO superpacket (tunSegBufSize) plus
+// the same again as drain headroom so a Read wake can accumulate
+// additional packets after an initial big read without overflowing.
+const tunSegBufCap = tunSegBufSize * 2
+
+// tunDrainCap caps how many packets a single Read will accumulate via
+// the post-wake drain loop. Sized to soak up a burst of small ACKs while
+// bounding how much work a single caller holds before handing off.
+const tunDrainCap = 64 //256
+
+// gsoInitialPayIovs is the starting capacity (in payload fragments) of
+// Offload.gsoIovs. Sized to cover the default coalesce segment cap without
+// any reallocations.
+const gsoInitialPayIovs = 66
+
+// validVnetHdr is the 10-byte virtio_net_hdr we prepend to every non-GSO TUN
+// write. Only flag set is VIRTIO_NET_HDR_F_DATA_VALID, which marks the skb
+// CHECKSUM_UNNECESSARY so the receiving network stack skips L4 checksum
+// verification. All packets that reach the plain Write / WriteReject paths
+// already carry a valid L4 checksum (either supplied by a remote peer whose
+// ciphertext we AEAD-authenticated, or produced by finishChecksum during TSO
+// segmentation, or built locally by CreateRejectPacket), so trusting them is
+// safe.
+var validVnetHdr = [virtioNetHdrLen]byte{unix.VIRTIO_NET_HDR_F_DATA_VALID}
+
+// Offload wraps a TUN file descriptor with poll-based reads. The FD provided will be changed to non-blocking.
+// A shared eventfd allows Close to wake all readers blocked in poll.
+type Offload struct {
+	fd         int
+	shutdownFd int
+	readPoll   [2]unix.PollFd
+	writePoll  [2]unix.PollFd
+	closed     atomic.Bool
+	readBuf    []byte        // scratch for a single raw read (virtio hdr + superpacket)
+	segBuf     []byte        // backing store for segmented output
+	segOff     int           // cursor into segBuf for the current Read drain
+	pending    [][]byte      // segments returned from the most recent Read
+	writeIovs  [2]unix.Iovec // preallocated iovecs for Write (coalescer passthrough); iovs[0] is fixed to validVnetHdr
+	// rejectIovs is a second preallocated iovec scratch used exclusively by
+	// WriteReject (reject + self-forward from the inside path). It mirrors
+	// writeIovs but lets listenIn goroutines emit reject packets without
+	// racing with the listenOut coalescer that owns writeIovs.
+	rejectIovs [2]unix.Iovec
+
+	// gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted
+	// by WriteGSO. Separate from validVnetHdr so a concurrent non-GSO Write on
+	// another queue never observes a half-written header.
+	gsoHdrBuf [virtioNetHdrLen]byte
+	// gsoIovs is the writev iovec scratch for WriteGSO. Sized to hold the
+	// virtio header + IP/TCP header + up to gsoInitialPayIovs payload
+	// fragments; grown on demand if a coalescer pushes more.
+	gsoIovs []unix.Iovec
+}
+
+func newOffload(fd int, shutdownFd int) (*Offload, error) {
+	if err := unix.SetNonblock(fd, true); err != nil {
+		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
+	}
+
+	out := &Offload{
+		fd:         fd,
+		shutdownFd: shutdownFd,
+		closed:     atomic.Bool{},
+		readBuf:    make([]byte, tunReadBufSize),
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+
+		segBuf:  make([]byte, tunSegBufCap),
+		gsoIovs: make([]unix.Iovec, 2, 2+gsoInitialPayIovs),
+	}
+
+	out.writeIovs[0].Base = &validVnetHdr[0]
+	out.writeIovs[0].SetLen(virtioNetHdrLen)
+	out.rejectIovs[0].Base = &validVnetHdr[0]
+	out.rejectIovs[0].SetLen(virtioNetHdrLen)
+	out.gsoIovs[0].Base = &out.gsoHdrBuf[0]
+	out.gsoIovs[0].SetLen(virtioNetHdrLen)
+
+	return out, nil
+}
+
+func (r *Offload) blockOnRead() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(r.readPoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	//always reset these!
+	tunEvents := r.readPoll[0].Revents
+	shutdownEvents := r.readPoll[1].Revents
+	r.readPoll[0].Revents = 0
+	r.readPoll[1].Revents = 0
+	//do the err check before trusting the potentially bogus bits we just got
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	} else if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (r *Offload) blockOnWrite() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(r.writePoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	//always reset these!
+	tunEvents := r.writePoll[0].Revents
+	shutdownEvents := r.writePoll[1].Revents
+	r.writePoll[0].Revents = 0
+	r.writePoll[1].Revents = 0
+	//do the err check before trusting the potentially bogus bits we just got
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	} else if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (r *Offload) readRaw(buf []byte) (int, error) {
+	for {
+		if n, err := unix.Read(r.fd, buf); err == nil {
+			return n, nil
+		} else if err == unix.EAGAIN {
+			if err = r.blockOnRead(); err != nil {
+				return 0, err
+			}
+			continue
+		} else if err == unix.EINTR {
+			continue
+		} else if err == unix.EBADF {
+			return 0, os.ErrClosed
+		} else {
+			return 0, err
+		}
+	}
+}
+
+// Read reads one or more superpackets from the tun and returns the
+// resulting packets. The first read blocks via poll; once the fd is known
+// readable we drain additional packets non-blocking until the kernel queue
+// is empty (EAGAIN), we've collected tunDrainCap packets, or we're out of
+// segBuf headroom. This amortizes the poll wake over bursts of small
+// packets (e.g. TCP ACKs). Slices point into the Offload's internal buffers
+// and are only valid until the next Read or Close on this Queue.
+func (r *Offload) Read() ([][]byte, error) {
+	r.pending = r.pending[:0]
+	r.segOff = 0
+
+	// Initial (blocking) read. Retry on decode errors so a single bad
+	// packet does not stall the reader.
+	for {
+		n, err := r.readRaw(r.readBuf)
+		if err != nil {
+			return nil, err
+		}
+		if err := r.decodeRead(n); err != nil {
+			// Drop and read again — a bad packet should not kill the reader.
+			continue
+		}
+		break
+	}
+
+	// Drain: non-blocking reads until the kernel queue is empty, the drain
+	// cap is reached, or segBuf no longer has room for another worst-case
+	// superpacket.
+	for len(r.pending) < tunDrainCap && tunSegBufCap-r.segOff >= tunSegBufSize {
+		n, err := unix.Read(r.fd, r.readBuf)
+		if err != nil {
+			// EAGAIN / EINTR / anything else: stop draining. We already
+			// have a valid batch from the first read.
+			break
+		}
+		if n <= 0 {
+			break
+		}
+		if err := r.decodeRead(n); err != nil {
+			// Drop this packet and stop the drain; we'd rather hand off
+			// what we have than keep spinning here.
+			break
+		}
+	}
+
+	return r.pending, nil
+}
+
+// decodeRead decodes the virtio header plus payload in r.readBuf[:n], appends
+// the segments to r.pending, and advances r.segOff by the total scratch used.
+// Caller must have already ensured r.vnetHdr is true.
+func (r *Offload) decodeRead(n int) error {
+	if n < virtioNetHdrLen {
+		return fmt.Errorf("short tun read: %d < %d", n, virtioNetHdrLen)
+	}
+	var hdr VirtioNetHdr
+	hdr.decode(r.readBuf[:virtioNetHdrLen])
+	before := len(r.pending)
+	if err := segmentInto(r.readBuf[virtioNetHdrLen:n], hdr, &r.pending, r.segBuf[r.segOff:]); err != nil {
+		return err
+	}
+	for k := before; k < len(r.pending); k++ {
+		r.segOff += len(r.pending[k])
+	}
+	return nil
+}
+
+func (r *Offload) Write(buf []byte) (int, error) {
+	return r.writeWithScratch(buf, &r.writeIovs)
+}
+
+// WriteReject emits a packet using a dedicated iovec scratch (rejectIovs)
+// distinct from the one used by the coalescer's Write path. This avoids a
+// data race between the inside (listenIn) goroutine emitting reject or
+// self-forward packets and the outside (listenOut) goroutine flushing TCP
+// coalescer passthroughs on the same Offload.
+func (r *Offload) WriteReject(buf []byte) (int, error) {
+	return r.writeWithScratch(buf, &r.rejectIovs)
+}
+
+func (r *Offload) writeWithScratch(buf []byte, iovs *[2]unix.Iovec) (int, error) {
+	if len(buf) == 0 {
+		return 0, nil
+	}
+	// Point the payload iovec at the caller's buffer. iovs[0] is pre-wired
+	// to validVnetHdr during Offload construction so we don't rebuild it here.
+	iovs[1].Base = &buf[0]
+	iovs[1].SetLen(len(buf))
+	iovPtr := unsafe.Pointer(&iovs[0])
+	// Pin the caller's buffer AND the iovec array through the syscall.
+	return r.rawWrite(iovPtr, 2, buf, iovs)
+}
+
+func (r *Offload) rawWrite(iovs unsafe.Pointer, iovcnt int, keepAlive ...interface{}) (int, error) {
+	for {
+		n, _, errno := syscall.Syscall(unix.SYS_WRITEV, uintptr(r.fd), uintptr(iovs), uintptr(iovcnt))
+		// Anchor the iovec array + every user-supplied payload slice
+		// through the syscall return. Without these, Go's GC may move or
+		// collect the underlying backing arrays while the kernel is still
+		// reading them via DMA (we pass the iovec as uintptr, so the
+		// compiler does not keep it live). Observed in practice as a
+		// kernel refcount underflow on tun_chr_write_iter / sock_wfree.
+		runtime.KeepAlive(iovs)
+		for _, ka := range keepAlive {
+			runtime.KeepAlive(ka)
+		}
+		if errno == 0 {
+			if int(n) < virtioNetHdrLen {
+				return 0, io.ErrShortWrite
+			}
+			return int(n) - virtioNetHdrLen, nil
+		}
+		if errno == unix.EAGAIN {
+			if err := r.blockOnWrite(); err != nil {
+				return 0, err
+			}
+			continue
+		}
+		if errno == unix.EINTR {
+			continue
+		}
+		if errno == unix.EBADF {
+			return 0, os.ErrClosed
+		}
+		return 0, errno
+	}
+}
+
+// GSOSupported reports whether this queue was opened with IFF_VNET_HDR and
+// can accept WriteGSO. When false, callers should fall back to per-segment
+// Write calls.
+func (r *Offload) GSOSupported() bool { return true }
+
+// WriteGSO emits a TCP TSO superpacket in a single writev. hdr is the
+// IPv4/IPv6 + TCP header prefix (already finalized — total length, IP csum,
+// and TCP pseudo-header partial set by the caller). pays are payload
+// fragments whose concatenation forms the full coalesced payload; each
+// slice is read-only and must stay valid until return. gsoSize is the MSS;
+// every segment except possibly the last is exactly gsoSize bytes.
+// csumStart is the byte offset where the TCP header begins within hdr.
+func (r *Offload) WriteGSO(hdr []byte, pays [][]byte, gsoSize uint16, isV6 bool, csumStart uint16) error {
+	if len(hdr) == 0 || len(pays) == 0 {
+		return nil
+	}
+
+	// Build the virtio_net_hdr. When pays total to <= gsoSize the kernel
+	// would produce a single segment; keep NEEDS_CSUM semantics but skip
+	// the GSO type so the kernel doesn't spuriously mark this as TSO.
+	vhdr := VirtioNetHdr{
+		Flags:      unix.VIRTIO_NET_HDR_F_NEEDS_CSUM,
+		HdrLen:     uint16(len(hdr)),
+		GSOSize:    gsoSize,
+		CsumStart:  csumStart,
+		CsumOffset: 16, // TCP checksum field lives 16 bytes into the TCP header
+	}
+	var totalPay int
+	for _, p := range pays {
+		totalPay += len(p)
+	}
+	if totalPay > int(gsoSize) {
+		if isV6 {
+			vhdr.GSOType = unix.VIRTIO_NET_HDR_GSO_TCPV6
+		} else {
+			vhdr.GSOType = unix.VIRTIO_NET_HDR_GSO_TCPV4
+		}
+	} else {
+		vhdr.GSOType = unix.VIRTIO_NET_HDR_GSO_NONE
+		vhdr.GSOSize = 0
+	}
+	vhdr.encode(r.gsoHdrBuf[:])
+
+	// Build the iovec array: [virtio_hdr, hdr, pays...]. r.gsoIovs[0] is
+	// wired to gsoHdrBuf at construction and never changes.
+	need := 2 + len(pays)
+	if cap(r.gsoIovs) < need {
+		grown := make([]unix.Iovec, need)
+		grown[0] = r.gsoIovs[0]
+		r.gsoIovs = grown
+	} else {
+		r.gsoIovs = r.gsoIovs[:need]
+	}
+	r.gsoIovs[1].Base = &hdr[0]
+	r.gsoIovs[1].SetLen(len(hdr))
+	for i, p := range pays {
+		r.gsoIovs[2+i].Base = &p[0]
+		r.gsoIovs[2+i].SetLen(len(p))
+	}
+
+	iovPtr := unsafe.Pointer(&r.gsoIovs[0])
+	iovCnt := len(r.gsoIovs)
+	// Pin EVERYTHING the kernel might still read via DMA: the backing iovec
+	// slice, the IP/TCP header buffer, and every individual payload
+	// fragment. Skipping any of these risks a use-after-free in
+	// tun_chr_write_iter if GC runs mid-syscall.
+	_, err := r.rawWrite(iovPtr, iovCnt, r.gsoIovs, hdr, pays)
+	return err
+}
+
+func (r *Offload) Close() error {
+	if r.closed.Swap(true) {
+		return nil
+	}
+
+	//shutdownFd is owned by the container, so we should not close it
+	var err error
+	if r.fd >= 0 {
+		err = unix.Close(r.fd)
+		r.fd = -1
+	}
+
+	return err
+}

overlay/tio/tio_poll_linux.go

@@ -0,0 +1,217 @@
+package tio
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"sync/atomic"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+// Maximum size we accept for a single read from a TUN with IFF_VNET_HDR. A
+// TSO superpacket can be up to 64KiB of payload plus a single L2/L3/L4 header
+// prefix plus the virtio header.
+const tunReadBufSize = 65535
+
+type Poll struct {
+	fd int
+
+	readPoll  [2]unix.PollFd
+	writePoll [2]unix.PollFd
+	closed    atomic.Bool
+
+	readBuf  []byte
+	batchRet [1][]byte
+}
+
+func newPoll(fd int, shutdownFd int) (*Poll, error) {
+	if err := unix.SetNonblock(fd, true); err != nil {
+		_ = unix.Close(fd)
+		return nil, fmt.Errorf("failed to set Poll device as nonblocking: %w", err)
+	}
+
+	out := &Poll{
+		fd:      fd,
+		readBuf: make([]byte, tunReadBufSize),
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+	}
+	return out, nil
+}
+
+// blockOnRead waits until the Poll fd is readable or shutdown has been signaled.
+// Returns os.ErrClosed if Close was called.
+func (t *Poll) blockOnRead() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(t.readPoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	tunEvents := t.readPoll[0].Revents
+	shutdownEvents := t.readPoll[1].Revents
+	t.readPoll[0].Revents = 0
+	t.readPoll[1].Revents = 0
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	}
+	if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (t *Poll) blockOnWrite() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(t.writePoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	tunEvents := t.writePoll[0].Revents
+	shutdownEvents := t.writePoll[1].Revents
+	t.writePoll[0].Revents = 0
+	t.writePoll[1].Revents = 0
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	}
+	if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (t *Poll) Read() ([][]byte, error) {
+	if t.readBuf == nil {
+		t.readBuf = make([]byte, defaultBatchBufSize)
+	}
+	n, err := t.readOne(t.readBuf)
+	if err != nil {
+		return nil, err
+	}
+	t.batchRet[0] = t.readBuf[:n]
+	return t.batchRet[:], nil
+}
+
+func (t *Poll) readOne(to []byte) (int, error) {
+	// first 4 bytes is protocol family, in network byte order
+	var head [4]byte
+	iovecs := [2]syscall.Iovec{ //todo plat-specific
+		{&head[0], 4},
+		{&to[0], uint64(len(to))},
+	}
+	for {
+		n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		// Pin the iovec + destination buffer backing array across the syscall.
+		// Without these the Go runtime may move/GC them while the kernel is
+		// still writing via DMA (we pass the iovec as uintptr, which hides it
+		// from escape analysis). Same class of bug as rawWrite in the Offload
+		// path.
+		runtime.KeepAlive(iovecs)
+		runtime.KeepAlive(to)
+		if errno == 0 {
+			bytesRead := int(n)
+			if bytesRead < 4 {
+				return 0, nil
+			}
+			return bytesRead - 4, nil
+		}
+		switch errno {
+		case unix.EAGAIN:
+			if err := t.blockOnRead(); err != nil {
+				return 0, err
+			}
+		case unix.EINTR:
+			// retry
+		case unix.EBADF:
+			return 0, os.ErrClosed
+		default:
+			return 0, errno
+		}
+	}
+}
+
+// Write is only valid for single threaded use
+func (t *Poll) Write(from []byte) (int, error) {
+	if len(from) <= 1 {
+		return 0, syscall.EIO
+	}
+
+	ipVer := from[0] >> 4
+	var head [4]byte
+	// first 4 bytes is protocol family, in network byte order
+	switch ipVer {
+	case 4:
+		head[3] = syscall.AF_INET
+	case 6:
+		head[3] = syscall.AF_INET6
+	default:
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+
+	iovecs := [2]syscall.Iovec{ //todo plat specific
+		{&head[0], 4},
+		{&from[0], uint64(len(from))},
+	}
+	for {
+		n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		// Pin the iovec + source buffer backing array across the syscall.
+		// See readOne's KeepAlive comment for rationale.
+		runtime.KeepAlive(iovecs)
+		runtime.KeepAlive(from)
+		if errno == 0 {
+			return int(n) - 4, nil
+		}
+		switch errno {
+		case unix.EAGAIN:
+			if err := t.blockOnWrite(); err != nil {
+				return 0, err
+			}
+		case unix.EINTR:
+			// retry
+		case unix.EBADF:
+			return 0, os.ErrClosed
+		default:
+			return 0, errno
+		}
+	}
+}
+
+func (t *Poll) Close() error {
+	if t.closed.Swap(true) {
+		return nil
+	}
+
+	//shutdownFd is owned by the container, so we should not close it
+
+	var err error
+	if t.fd >= 0 {
+		err = unix.Close(t.fd)
+		t.fd = -1
+	}
+
+	return err
+}
+
+func (t *Poll) WriteReject(p []byte) (int, error) {
+	return t.Write(p)
+}

overlay/tio/tun_file_linux_test.go

@@ -0,0 +1,86 @@
+//go:build linux && !android && !e2e_testing
+// +build linux,!android,!e2e_testing
+
+package tio
+
+import (
+	"errors"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
+// The caller takes ownership of the read fd (pass it to newOffload / newFriend).
+func newReadPipe(t *testing.T) int {
+	t.Helper()
+	var fds [2]int
+	if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
+		t.Fatalf("pipe2: %v", err)
+	}
+	t.Cleanup(func() { _ = unix.Close(fds[1]) })
+	return fds[0]
+}
+
+func TestOffload_WakeForShutdown_WakesFriends(t *testing.T) {
+	pipe1 := newReadPipe(t)
+	pipe2 := newReadPipe(t)
+	parent, err := NewOffloadContainer()
+	if err != nil {
+		t.Fatalf("newOffload: %v", err)
+	}
+	require.NoError(t, parent.Add(pipe1))
+	require.NoError(t, parent.Add(pipe2))
+	t.Cleanup(func() {
+		_ = unix.Close(pipe1)
+		_ = unix.Close(pipe2)
+	})
+
+	readers := parent.Queues()
+	errs := make([]error, len(readers))
+	var wg sync.WaitGroup
+	for i, r := range readers {
+		wg.Add(1)
+		go func(i int, r Queue) {
+			defer wg.Done()
+			_, errs[i] = r.Read()
+		}(i, r)
+	}
+
+	time.Sleep(50 * time.Millisecond)
+
+	if err := parent.Close(); err != nil {
+		t.Fatalf("Close: %v", err)
+	}
+
+	done := make(chan struct{})
+	go func() { wg.Wait(); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("readers did not wake")
+	}
+
+	for i, err := range errs {
+		if !errors.Is(err, os.ErrClosed) {
+			t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
+		}
+	}
+}
+
+func TestTunFile_Close_Idempotent(t *testing.T) {
+	tf, err := newOffload(newReadPipe(t), 1)
+	if err != nil {
+		t.Fatalf("newOffload: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("first Close: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("second Close should be a no-op, got %v", err)
+	}
+}

overlay/tio/tun_linux_offload.go

@@ -0,0 +1,281 @@
+//go:build linux && !android && !e2e_testing
+// +build linux,!android,!e2e_testing
+
+package tio
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/tcpip/checksum"
+)
+
+// Protocol header size bounds used to validate / cap kernel-supplied offsets.
+const (
+	ipv4HeaderMinLen = 20 // IHL=5, no options
+	ipv4HeaderMaxLen = 60 // IHL=15, max options
+	ipv6FixedLen     = 40 // IPv6 base header; extensions would extend this
+	tcpHeaderMinLen  = 20 // data-offset=5, no options
+	tcpHeaderMaxLen  = 60 // data-offset=15, max options
+)
+
+// Byte offsets inside an IPv4 header.
+const (
+	ipv4TotalLenOff = 2
+	ipv4IDOff       = 4
+	ipv4ChecksumOff = 10
+	ipv4SrcOff      = 12
+	ipv4AddrsEnd    = 20 // end of dst address (ipv4SrcOff + 2*4)
+)
+
+// Byte offsets inside an IPv6 header.
+const (
+	ipv6PayloadLenOff = 4
+	ipv6SrcOff        = 8
+	ipv6AddrsEnd      = 40 // end of dst address (ipv6SrcOff + 2*16)
+)
+
+// Byte offsets inside a TCP header (relative to its start, i.e. csumStart).
+const (
+	tcpSeqOff      = 4
+	tcpDataOffOff  = 12 // upper nibble is header len in 32-bit words
+	tcpFlagsOff    = 13
+	tcpChecksumOff = 16
+)
+
+// tcpFinPshMask is cleared on every segment except the last of a TSO burst.
+const tcpFinPshMask = 0x09 // FIN(0x01) | PSH(0x08)
+
+// segmentInto splits a TUN-side packet described by hdr into one or more
+// IP packets, each appended to *out as a slice of scratch. scratch must be
+// sized to hold every segment (including replicated headers).
+func segmentInto(pkt []byte, hdr VirtioNetHdr, out *[][]byte, scratch []byte) error {
+	// When RSC_INFO is set the csum_start/csum_offset fields are repurposed to
+	// carry coalescing info rather than checksum offsets. A TUN writing via
+	// IFF_VNET_HDR should never emit this, but if it did we would silently
+	// miscompute the segment checksums — refuse the packet instead.
+	if hdr.Flags&unix.VIRTIO_NET_HDR_F_RSC_INFO != 0 {
+		return fmt.Errorf("virtio RSC_INFO flag not supported on TUN reads")
+	}
+
+	switch hdr.GSOType {
+	case unix.VIRTIO_NET_HDR_GSO_NONE:
+		if len(pkt) > len(scratch) {
+			return fmt.Errorf("packet larger than segment buffer: %d > %d", len(pkt), len(scratch))
+		}
+		copy(scratch, pkt)
+		seg := scratch[:len(pkt)]
+		if hdr.Flags&unix.VIRTIO_NET_HDR_F_NEEDS_CSUM != 0 {
+			if err := finishChecksum(seg, hdr); err != nil {
+				return err
+			}
+		}
+		*out = append(*out, seg)
+		return nil
+
+	case unix.VIRTIO_NET_HDR_GSO_TCPV4, unix.VIRTIO_NET_HDR_GSO_TCPV6:
+		return segmentTCP(pkt, hdr, out, scratch)
+
+	default:
+		return fmt.Errorf("unsupported virtio gso type: %d", hdr.GSOType)
+	}
+}
+
+// finishChecksum computes the L4 checksum for a non-GSO packet that the kernel
+// handed us with NEEDS_CSUM set. csum_start / csum_offset point at the 16-bit
+// checksum field; we zero it, fold a full sum (the field was pre-loaded with
+// the pseudo-header partial sum by the kernel), and store the result.
+func finishChecksum(seg []byte, hdr VirtioNetHdr) error {
+	cs := int(hdr.CsumStart)
+	co := int(hdr.CsumOffset)
+	if cs+co+2 > len(seg) {
+		return fmt.Errorf("csum offsets out of range: start=%d offset=%d len=%d", cs, co, len(seg))
+	}
+	// The kernel stores a partial pseudo-header sum at [cs+co:]; sum over the
+	// L4 region starting at cs, folding the prior partial in as the seed.
+	partial := binary.BigEndian.Uint16(seg[cs+co : cs+co+2])
+	seg[cs+co] = 0
+	seg[cs+co+1] = 0
+	binary.BigEndian.PutUint16(seg[cs+co:cs+co+2], ^checksum.Checksum(seg[cs:], partial))
+	return nil
+}
+
+// segmentTCP software-segments a TSO superpacket into one IP packet per MSS
+// chunk. The caller guarantees hdr.GSOType is TCPV4 or TCPV6.
+//
+// Hot-path shape: the per-segment loop only sums the payload chunk. The TCP
+// header, the IPv4 header, and the pseudo-header src/dst/proto contributions
+// are each summed once up front — every segment reuses those three pre-folded
+// uint32 values and combines them with small per-segment deltas (seq, flags,
+// tcpLen, ip_id, total_len) that are cheap to fold in.
+func segmentTCP(pkt []byte, hdr VirtioNetHdr, out *[][]byte, scratch []byte) error {
+	if hdr.GSOSize == 0 {
+		return fmt.Errorf("gso_size is zero")
+	}
+	if hdr.CsumStart == 0 {
+		return fmt.Errorf("csum_start is zero")
+	}
+
+	isV4 := hdr.GSOType == unix.VIRTIO_NET_HDR_GSO_TCPV4
+	csumStart := int(hdr.CsumStart)
+
+	if isV4 && csumStart < ipv4HeaderMinLen {
+		return fmt.Errorf("csum_start %d too small for IPv4", csumStart)
+	}
+	if !isV4 && csumStart < ipv6FixedLen {
+		return fmt.Errorf("csum_start %d too small for IPv6", csumStart)
+	}
+
+	// Don't trust hdr.HdrLen from the kernel: on some paths it can be set
+	// to the full length of the first packet rather than the true L3+L4 header length.
+	// Instead, read the TCP data-offset field from the packet itself and derive
+	// headerLen = csum_start + tcpHdrLen. Matches wireguard-go's approach.
+	if csumStart+tcpFlagsOff+1 > len(pkt) {
+		return fmt.Errorf("packet too short for tcp header at csum_start=%d (pkt %d)", csumStart, len(pkt))
+	}
+	tcpHdrLen := int(pkt[csumStart+tcpDataOffOff]>>4) * 4
+	if tcpHdrLen < tcpHeaderMinLen || tcpHdrLen > tcpHeaderMaxLen {
+		return fmt.Errorf("tcp data-offset out of range: %d", tcpHdrLen)
+	}
+	headerLen := csumStart + tcpHdrLen
+	if headerLen > len(pkt) {
+		return fmt.Errorf("derived hdr_len %d > pkt %d", headerLen, len(pkt))
+	}
+
+	payload := pkt[headerLen:]
+	payLen := len(payload)
+	gso := int(hdr.GSOSize)
+	numSeg := (payLen + gso - 1) / gso
+	if numSeg == 0 {
+		numSeg = 1
+	}
+
+	need := numSeg*headerLen + payLen
+	if need > len(scratch) {
+		return fmt.Errorf("scratch too small for %d segments: need %d have %d", numSeg, need, len(scratch))
+	}
+
+	origSeq := binary.BigEndian.Uint32(pkt[csumStart+tcpSeqOff : csumStart+tcpSeqOff+4])
+	origFlags := pkt[csumStart+tcpFlagsOff]
+
+	// Precompute the TCP header sum with seq/flags/csum zeroed. Copy onto
+	// the stack, zero the per-segment-varying fields, sum once.
+	var tmp [tcpHeaderMaxLen]byte
+	copy(tmp[:tcpHdrLen], pkt[csumStart:headerLen])
+	tmp[tcpSeqOff], tmp[tcpSeqOff+1], tmp[tcpSeqOff+2], tmp[tcpSeqOff+3] = 0, 0, 0, 0
+	tmp[tcpFlagsOff] = 0
+	tmp[tcpChecksumOff], tmp[tcpChecksumOff+1] = 0, 0
+	baseTcpHdrSum := uint32(checksum.Checksum(tmp[:tcpHdrLen], 0))
+
+	// Pseudo-header src+dst+proto contribution (tcpLen varies per segment).
+	var baseProtoSum uint32
+	if isV4 {
+		baseProtoSum = uint32(checksum.Checksum(pkt[ipv4SrcOff:ipv4AddrsEnd], 0))
+	} else {
+		baseProtoSum = uint32(checksum.Checksum(pkt[ipv6SrcOff:ipv6AddrsEnd], 0))
+	}
+	baseProtoSum += uint32(unix.IPPROTO_TCP)
+
+	// Precompute IPv4 header sum with total_len/id/csum zeroed.
+	var origIPID uint16
+	var ihl int
+	var baseIPHdrSum uint32
+	if isV4 {
+		origIPID = binary.BigEndian.Uint16(pkt[ipv4IDOff : ipv4IDOff+2])
+		ihl = int(pkt[0]&0x0f) * 4
+		if ihl < ipv4HeaderMinLen || ihl > csumStart {
+			return fmt.Errorf("bad IPv4 IHL: %d", ihl)
+		}
+		var ipTmp [ipv4HeaderMaxLen]byte
+		copy(ipTmp[:ihl], pkt[:ihl])
+		ipTmp[ipv4TotalLenOff], ipTmp[ipv4TotalLenOff+1] = 0, 0
+		ipTmp[ipv4IDOff], ipTmp[ipv4IDOff+1] = 0, 0
+		ipTmp[ipv4ChecksumOff], ipTmp[ipv4ChecksumOff+1] = 0, 0
+		baseIPHdrSum = uint32(checksum.Checksum(ipTmp[:ihl], 0))
+	}
+
+	off := 0
+	for i := 0; i < numSeg; i++ {
+		segStart := i * gso
+		segEnd := segStart + gso
+		if segEnd > payLen {
+			segEnd = payLen
+		}
+		segPayLen := segEnd - segStart
+
+		copy(scratch[off:], pkt[:headerLen])
+		copy(scratch[off+headerLen:], payload[segStart:segEnd])
+		seg := scratch[off : off+headerLen+segPayLen]
+		off += headerLen + segPayLen
+
+		segSeq := origSeq + uint32(segStart)
+		segFlags := origFlags
+		if i != numSeg-1 {
+			segFlags = origFlags &^ tcpFinPshMask
+		}
+		totalLen := headerLen + segPayLen
+
+		// Patch IP header and write the v4 header checksum from the precomputed base.
+		if isV4 {
+			segID := origIPID + uint16(i)
+			binary.BigEndian.PutUint16(seg[ipv4TotalLenOff:ipv4TotalLenOff+2], uint16(totalLen))
+			binary.BigEndian.PutUint16(seg[ipv4IDOff:ipv4IDOff+2], segID)
+			ipSum := baseIPHdrSum + uint32(totalLen) + uint32(segID)
+			binary.BigEndian.PutUint16(seg[ipv4ChecksumOff:ipv4ChecksumOff+2], foldComplement(ipSum))
+		} else {
+			// IPv6 payload length excludes the fixed header but includes any
+			// extension headers between [ipv6FixedLen:csumStart].
+			binary.BigEndian.PutUint16(seg[ipv6PayloadLenOff:ipv6PayloadLenOff+2], uint16(headerLen-ipv6FixedLen+segPayLen))
+		}
+
+		// Patch TCP header.
+		binary.BigEndian.PutUint32(seg[csumStart+tcpSeqOff:csumStart+tcpSeqOff+4], segSeq)
+		seg[csumStart+tcpFlagsOff] = segFlags
+		// (csum is written below; its prior contents in `seg` don't affect the
+		// computation since we never sum over the segment's own header.)
+
+		tcpLen := tcpHdrLen + segPayLen
+		paySum := uint32(checksum.Checksum(payload[segStart:segEnd], 0))
+
+		// Combine pre-folded uint32s into a wider accumulator, then fold. Using
+		// uint64 guards against overflow when segSeq's high bits set.
+		wide := uint64(baseTcpHdrSum) + uint64(paySum) + uint64(baseProtoSum)
+		wide += uint64(segSeq) + uint64(segFlags) + uint64(tcpLen)
+		wide = (wide & 0xffffffff) + (wide >> 32)
+		wide = (wide & 0xffffffff) + (wide >> 32)
+		binary.BigEndian.PutUint16(seg[csumStart+tcpChecksumOff:csumStart+tcpChecksumOff+2], foldComplement(uint32(wide)))
+
+		*out = append(*out, seg)
+	}
+
+	return nil
+}
+
+// foldComplement folds a 32-bit one's-complement partial sum to 16 bits and
+// complements it, yielding the on-wire Internet checksum value.
+func foldComplement(sum uint32) uint16 {
+	sum = (sum & 0xffff) + (sum >> 16)
+	sum = (sum & 0xffff) + (sum >> 16)
+	return ^uint16(sum)
+}
+
+// pseudoHeaderIPv4 returns the folded pseudo-header sum used to verify a TCP
+// segment's checksum in tests. src/dst are 4 bytes each.
+func pseudoHeaderIPv4(src, dst []byte, proto byte, tcpLen int) uint16 {
+	s := uint32(checksum.Checksum(src, 0)) + uint32(checksum.Checksum(dst, 0))
+	s += uint32(proto) + uint32(tcpLen)
+	s = (s & 0xffff) + (s >> 16)
+	s = (s & 0xffff) + (s >> 16)
+	return uint16(s)
+}
+
+// pseudoHeaderIPv6 returns the folded pseudo-header sum used to verify a TCP
+// segment's checksum in tests. src/dst are 16 bytes each.
+func pseudoHeaderIPv6(src, dst []byte, proto byte, tcpLen int) uint16 {
+	s := uint32(checksum.Checksum(src, 0)) + uint32(checksum.Checksum(dst, 0))
+	s += uint32(tcpLen>>16) + uint32(tcpLen&0xffff) + uint32(proto)
+	s = (s & 0xffff) + (s >> 16)
+	s = (s & 0xffff) + (s >> 16)
+	return uint16(s)
+}

overlay/tio/tun_linux_offload_test.go

@@ -0,0 +1,330 @@
+//go:build linux && !android && !e2e_testing
+// +build linux,!android,!e2e_testing
+
+package tio
+
+import (
+	"encoding/binary"
+	"os"
+	"testing"
+
+	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/tcpip/checksum"
+)
+
+// verifyChecksum confirms that the one's-complement sum across `b`, seeded
+// with a folded pseudo-header sum, equals all-ones (valid).
+func verifyChecksum(b []byte, pseudo uint16) bool {
+	return checksum.Checksum(b, pseudo) == 0xffff
+}
+
+// buildTSOv4 builds a synthetic IPv4/TCP TSO superpacket with a payload of
+// `payLen` bytes split at `mss`.
+func buildTSOv4(t *testing.T, payLen, mss int) ([]byte, VirtioNetHdr) {
+	t.Helper()
+	const ipLen = 20
+	const tcpLen = 20
+	pkt := make([]byte, ipLen+tcpLen+payLen)
+
+	// IPv4 header
+	pkt[0] = 0x45 // version 4, IHL 5
+	// total length is meaningless for TSO but set it anyway
+	binary.BigEndian.PutUint16(pkt[2:4], uint16(ipLen+tcpLen+payLen))
+	binary.BigEndian.PutUint16(pkt[4:6], 0x4242) // original ID
+	pkt[8] = 64                                  // TTL
+	pkt[9] = unix.IPPROTO_TCP
+	copy(pkt[12:16], []byte{10, 0, 0, 1}) // src
+	copy(pkt[16:20], []byte{10, 0, 0, 2}) // dst
+
+	// TCP header
+	binary.BigEndian.PutUint16(pkt[20:22], 12345) // sport
+	binary.BigEndian.PutUint16(pkt[22:24], 80)    // dport
+	binary.BigEndian.PutUint32(pkt[24:28], 10000) // seq
+	binary.BigEndian.PutUint32(pkt[28:32], 20000) // ack
+	pkt[32] = 0x50                                // data offset 5 words
+	pkt[33] = 0x18                                // ACK | PSH
+	binary.BigEndian.PutUint16(pkt[34:36], 65535) // window
+
+	// payload
+	for i := 0; i < payLen; i++ {
+		pkt[ipLen+tcpLen+i] = byte(i & 0xff)
+	}
+
+	return pkt, VirtioNetHdr{
+		Flags:      unix.VIRTIO_NET_HDR_F_NEEDS_CSUM,
+		GSOType:    unix.VIRTIO_NET_HDR_GSO_TCPV4,
+		HdrLen:     uint16(ipLen + tcpLen),
+		GSOSize:    uint16(mss),
+		CsumStart:  uint16(ipLen),
+		CsumOffset: 16,
+	}
+}
+
+func TestSegmentTCPv4(t *testing.T) {
+	const mss = 100
+	const numSeg = 3
+	pkt, hdr := buildTSOv4(t, mss*numSeg, mss)
+
+	scratch := make([]byte, tunSegBufSize)
+	var out [][]byte
+	if err := segmentTCP(pkt, hdr, &out, scratch); err != nil {
+		t.Fatalf("segmentTCP: %v", err)
+	}
+	if len(out) != numSeg {
+		t.Fatalf("expected %d segments, got %d", numSeg, len(out))
+	}
+
+	for i, seg := range out {
+		if len(seg) != 40+mss {
+			t.Errorf("seg %d: unexpected len %d", i, len(seg))
+		}
+		totalLen := binary.BigEndian.Uint16(seg[2:4])
+		if totalLen != uint16(40+mss) {
+			t.Errorf("seg %d: total_len=%d want %d", i, totalLen, 40+mss)
+		}
+		id := binary.BigEndian.Uint16(seg[4:6])
+		if id != 0x4242+uint16(i) {
+			t.Errorf("seg %d: ip id=%#x want %#x", i, id, 0x4242+uint16(i))
+		}
+		seq := binary.BigEndian.Uint32(seg[24:28])
+		wantSeq := uint32(10000 + i*mss)
+		if seq != wantSeq {
+			t.Errorf("seg %d: seq=%d want %d", i, seq, wantSeq)
+		}
+		flags := seg[33]
+		wantFlags := byte(0x10) // ACK only, PSH cleared
+		if i == numSeg-1 {
+			wantFlags = 0x18 // ACK | PSH preserved on last
+		}
+		if flags != wantFlags {
+			t.Errorf("seg %d: flags=%#x want %#x", i, flags, wantFlags)
+		}
+		// IPv4 header checksum must verify against itself.
+		if !verifyChecksum(seg[:20], 0) {
+			t.Errorf("seg %d: bad IPv4 header checksum", i)
+		}
+		// TCP checksum must verify against the pseudo-header.
+		psum := pseudoHeaderIPv4(seg[12:16], seg[16:20], unix.IPPROTO_TCP, 20+mss)
+		if !verifyChecksum(seg[20:], psum) {
+			t.Errorf("seg %d: bad TCP checksum", i)
+		}
+	}
+}
+
+func TestSegmentTCPv4OddTail(t *testing.T) {
+	// Payload of 250 bytes with MSS 100 → segments of 100, 100, 50.
+	pkt, hdr := buildTSOv4(t, 250, 100)
+	scratch := make([]byte, tunSegBufSize)
+	var out [][]byte
+	if err := segmentTCP(pkt, hdr, &out, scratch); err != nil {
+		t.Fatalf("segmentTCP: %v", err)
+	}
+	if len(out) != 3 {
+		t.Fatalf("want 3 segments, got %d", len(out))
+	}
+	wantPayLens := []int{100, 100, 50}
+	for i, seg := range out {
+		if len(seg)-40 != wantPayLens[i] {
+			t.Errorf("seg %d: pay len %d want %d", i, len(seg)-40, wantPayLens[i])
+		}
+		if !verifyChecksum(seg[:20], 0) {
+			t.Errorf("seg %d: bad IPv4 header checksum", i)
+		}
+		psum := pseudoHeaderIPv4(seg[12:16], seg[16:20], unix.IPPROTO_TCP, 20+wantPayLens[i])
+		if !verifyChecksum(seg[20:], psum) {
+			t.Errorf("seg %d: bad TCP checksum", i)
+		}
+	}
+}
+
+func TestSegmentTCPv6(t *testing.T) {
+	const ipLen = 40
+	const tcpLen = 20
+	const mss = 120
+	const numSeg = 2
+	payLen := mss * numSeg
+	pkt := make([]byte, ipLen+tcpLen+payLen)
+
+	// IPv6 header
+	pkt[0] = 0x60 // version 6
+	binary.BigEndian.PutUint16(pkt[4:6], uint16(tcpLen+payLen))
+	pkt[6] = unix.IPPROTO_TCP
+	pkt[7] = 64
+	// src/dst fe80::1 / fe80::2
+	pkt[8] = 0xfe
+	pkt[9] = 0x80
+	pkt[23] = 1
+	pkt[24] = 0xfe
+	pkt[25] = 0x80
+	pkt[39] = 2
+
+	// TCP header
+	binary.BigEndian.PutUint16(pkt[40:42], 12345)
+	binary.BigEndian.PutUint16(pkt[42:44], 80)
+	binary.BigEndian.PutUint32(pkt[44:48], 7)
+	binary.BigEndian.PutUint32(pkt[48:52], 99)
+	pkt[52] = 0x50
+	pkt[53] = 0x19 // FIN | ACK | PSH — exercise FIN clearing too
+	binary.BigEndian.PutUint16(pkt[54:56], 65535)
+
+	for i := 0; i < payLen; i++ {
+		pkt[ipLen+tcpLen+i] = byte(i)
+	}
+
+	hdr := VirtioNetHdr{
+		Flags:      unix.VIRTIO_NET_HDR_F_NEEDS_CSUM,
+		GSOType:    unix.VIRTIO_NET_HDR_GSO_TCPV6,
+		HdrLen:     uint16(ipLen + tcpLen),
+		GSOSize:    uint16(mss),
+		CsumStart:  uint16(ipLen),
+		CsumOffset: 16,
+	}
+
+	scratch := make([]byte, tunSegBufSize)
+	var out [][]byte
+	if err := segmentTCP(pkt, hdr, &out, scratch); err != nil {
+		t.Fatalf("segmentTCP: %v", err)
+	}
+	if len(out) != numSeg {
+		t.Fatalf("want %d segments, got %d", numSeg, len(out))
+	}
+
+	for i, seg := range out {
+		if len(seg) != ipLen+tcpLen+mss {
+			t.Errorf("seg %d: len %d want %d", i, len(seg), ipLen+tcpLen+mss)
+		}
+		pl := binary.BigEndian.Uint16(seg[4:6])
+		if pl != uint16(tcpLen+mss) {
+			t.Errorf("seg %d: payload_length=%d want %d", i, pl, tcpLen+mss)
+		}
+		seq := binary.BigEndian.Uint32(seg[44:48])
+		if seq != uint32(7+i*mss) {
+			t.Errorf("seg %d: seq=%d want %d", i, seq, 7+i*mss)
+		}
+		flags := seg[53]
+		// Original flags = 0x19 (FIN|ACK|PSH). FIN(0x01)+PSH(0x08) should be
+		// cleared on all but the last; ACK(0x10) always preserved.
+		wantFlags := byte(0x10)
+		if i == numSeg-1 {
+			wantFlags = 0x19
+		}
+		if flags != wantFlags {
+			t.Errorf("seg %d: flags=%#x want %#x", i, flags, wantFlags)
+		}
+		psum := pseudoHeaderIPv6(seg[8:24], seg[24:40], unix.IPPROTO_TCP, tcpLen+mss)
+		if !verifyChecksum(seg[ipLen:], psum) {
+			t.Errorf("seg %d: bad TCP checksum", i)
+		}
+	}
+}
+
+func TestSegmentGSONonePassesThrough(t *testing.T) {
+	pkt, hdr := buildTSOv4(t, 100, 100)
+	hdr.GSOType = unix.VIRTIO_NET_HDR_GSO_NONE
+	hdr.Flags = 0 // no NEEDS_CSUM, leave packet untouched
+
+	scratch := make([]byte, tunSegBufSize)
+	var out [][]byte
+	if err := segmentInto(pkt, hdr, &out, scratch); err != nil {
+		t.Fatalf("segmentInto: %v", err)
+	}
+	if len(out) != 1 {
+		t.Fatalf("want 1 segment, got %d", len(out))
+	}
+	if len(out[0]) != len(pkt) {
+		t.Fatalf("unexpected length: %d vs %d", len(out[0]), len(pkt))
+	}
+}
+
+func TestSegmentRejectsUDP(t *testing.T) {
+	hdr := VirtioNetHdr{GSOType: unix.VIRTIO_NET_HDR_GSO_UDP}
+	var out [][]byte
+	if err := segmentInto(nil, hdr, &out, nil); err == nil {
+		t.Fatalf("expected rejection for UDP GSO")
+	}
+}
+
+func BenchmarkSegmentTCPv4(b *testing.B) {
+	sizes := []struct {
+		name   string
+		payLen int
+		mss    int
+	}{
+		{"64KiB_MSS1460", 65000, 1460},
+		{"16KiB_MSS1460", 16384, 1460},
+		{"4KiB_MSS1460", 4096, 1460},
+	}
+	for _, sz := range sizes {
+		b.Run(sz.name, func(b *testing.B) {
+			const ipLen = 20
+			const tcpLen = 20
+			pkt := make([]byte, ipLen+tcpLen+sz.payLen)
+			pkt[0] = 0x45
+			binary.BigEndian.PutUint16(pkt[2:4], uint16(ipLen+tcpLen+sz.payLen))
+			binary.BigEndian.PutUint16(pkt[4:6], 0x4242)
+			pkt[8] = 64
+			pkt[9] = unix.IPPROTO_TCP
+			copy(pkt[12:16], []byte{10, 0, 0, 1})
+			copy(pkt[16:20], []byte{10, 0, 0, 2})
+			binary.BigEndian.PutUint16(pkt[20:22], 12345)
+			binary.BigEndian.PutUint16(pkt[22:24], 80)
+			binary.BigEndian.PutUint32(pkt[24:28], 10000)
+			binary.BigEndian.PutUint32(pkt[28:32], 20000)
+			pkt[32] = 0x50
+			pkt[33] = 0x18
+			binary.BigEndian.PutUint16(pkt[34:36], 65535)
+			for i := 0; i < sz.payLen; i++ {
+				pkt[ipLen+tcpLen+i] = byte(i)
+			}
+			hdr := VirtioNetHdr{
+				Flags:      unix.VIRTIO_NET_HDR_F_NEEDS_CSUM,
+				GSOType:    unix.VIRTIO_NET_HDR_GSO_TCPV4,
+				HdrLen:     uint16(ipLen + tcpLen),
+				GSOSize:    uint16(sz.mss),
+				CsumStart:  uint16(ipLen),
+				CsumOffset: 16,
+			}
+
+			scratch := make([]byte, tunSegBufSize)
+			out := make([][]byte, 0, 64)
+
+			b.SetBytes(int64(len(pkt)))
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				out = out[:0]
+				if err := segmentTCP(pkt, hdr, &out, scratch); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+// TestTunFileWriteVnetHdrNoAlloc verifies the IFF_VNET_HDR fast-path write is
+// allocation-free. We write to /dev/null so every call succeeds synchronously.
+func TestTunFileWriteVnetHdrNoAlloc(t *testing.T) {
+	fd, err := unix.Open("/dev/null", os.O_WRONLY, 0)
+	if err != nil {
+		t.Fatalf("open /dev/null: %v", err)
+	}
+	t.Cleanup(func() { _ = unix.Close(fd) })
+
+	tf := &Offload{fd: fd}
+	tf.writeIovs[0].Base = &validVnetHdr[0]
+	tf.writeIovs[0].SetLen(virtioNetHdrLen)
+
+	payload := make([]byte, 1400)
+	// Warm up (first call may trigger one-time internal allocations elsewhere).
+	if _, err := tf.Write(payload); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+
+	allocs := testing.AllocsPerRun(1000, func() {
+		if _, err := tf.Write(payload); err != nil {
+			t.Fatalf("Write: %v", err)
+		}
+	})
+	if allocs != 0 {
+		t.Fatalf("Write allocated %.1f times per call, want 0", allocs)
+	}
+}

overlay/tio/vnethdr_linux.go

@@ -0,0 +1,39 @@
+package tio
+
+import "encoding/binary"
+
+// Size of the legacy struct virtio_net_hdr that the kernel prepends/expects on
+// a TUN opened with IFF_VNET_HDR (TUNSETVNETHDRSZ not set).
+const virtioNetHdrLen = 10
+
+type VirtioNetHdr struct {
+	Flags      uint8
+	GSOType    uint8
+	HdrLen     uint16
+	GSOSize    uint16
+	CsumStart  uint16
+	CsumOffset uint16
+}
+
+// decode reads a virtio_net_hdr in host byte order (TUN default; we never
+// call TUNSETVNETLE so the kernel matches our endianness).
+func (h *VirtioNetHdr) decode(b []byte) {
+	h.Flags = b[0]
+	h.GSOType = b[1]
+	h.HdrLen = binary.NativeEndian.Uint16(b[2:4])
+	h.GSOSize = binary.NativeEndian.Uint16(b[4:6])
+	h.CsumStart = binary.NativeEndian.Uint16(b[6:8])
+	h.CsumOffset = binary.NativeEndian.Uint16(b[8:10])
+}
+
+// encode is the inverse of decode: writes the virtio_net_hdr fields into b
+// (must be at least virtioNetHdrLen bytes). Used to emit a TSO superpacket
+// on egress.
+func (h *VirtioNetHdr) encode(b []byte) {
+	b[0] = h.Flags
+	b[1] = h.GSOType
+	binary.NativeEndian.PutUint16(b[2:4], h.HdrLen)
+	binary.NativeEndian.PutUint16(b[4:6], h.GSOSize)
+	binary.NativeEndian.PutUint16(b[6:8], h.CsumStart)
+	binary.NativeEndian.PutUint16(b[8:10], h.CsumOffset)
+}

overlay/tun.go

@@ -2,10 +2,10 @@ package overlay
 
 import (
 	"fmt"
-	"log/slog"
 	"net"
 	"net/netip"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/util"
 )
@@ -22,9 +22,9 @@ func (e *NameError) Error() string {
 }
 
 // TODO: We may be able to remove routines
-type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error)
+type DeviceFactory func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error)
 
-func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
+func NewDeviceFromConfig(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
 	switch {
 	case c.GetBool("tun.disabled", false):
 		tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
@@ -36,7 +36,7 @@ func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix
 }
 
 func NewFdDeviceFromConfig(fd *int) DeviceFactory {
-	return func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
+	return func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
 		return newTunFromFd(c, l, *fd, vpnNetworks)
 	}
 }

overlay/tun_android.go

@@ -6,36 +6,64 @@ package overlay
 import (
 	"fmt"
 	"io"
-	"log/slog"
 	"net/netip"
 	"os"
 	"sync/atomic"
 
 	"github.com/gaissmai/bart"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 )
 
 type tun struct {
-	io.ReadWriteCloser
+	rwc         io.ReadWriteCloser
 	fd          int
 	vpnNetworks []netip.Prefix
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	l           *slog.Logger
+	l           *logrus.Logger
+
+	readBuf  []byte
+	batchRet [1][]byte
 }
 
-func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
+func (t *tun) Read() ([][]byte, error) {
+	if t.readBuf == nil {
+		t.readBuf = make([]byte, defaultBatchBufSize)
+	}
+	n, err := t.rwc.Read(t.readBuf)
+	if err != nil {
+		return nil, err
+	}
+	t.batchRet[0] = t.readBuf[:n]
+	return t.batchRet[:], nil
+}
+
+func (t *tun) Write(p []byte) (int, error) {
+	return t.rwc.Write(p)
+}
+
+func (t *tun) WriteReject(p []byte) (int, error) {
+	return t.rwc.Write(p)
+}
+
+func (t *tun) Close() error {
+	return t.rwc.Close()
+}
+
+func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
 	// XXX Android returns an fd in non-blocking mode which is necessary for shutdown to work properly.
 	// Be sure not to call file.Fd() as it will set the fd to blocking mode.
 	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
 
 	t := &tun{
-		ReadWriteCloser: file,
-		fd:              deviceFd,
-		vpnNetworks:     vpnNetworks,
-		l:               l,
+		rwc:         file,
+		fd:          deviceFd,
+		vpnNetworks: vpnNetworks,
+		l:           l,
 	}
 
 	err := t.reload(c, true)
@@ -53,7 +81,7 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 	return t, nil
 }
 
-func newTun(_ *config.C, _ *slog.Logger, _ []netip.Prefix, _ bool) (*tun, error) {
+func newTun(_ *config.C, _ *logrus.Logger, _ []netip.Prefix, _ bool) (*tun, error) {
 	return nil, fmt.Errorf("newTun not supported in Android")
 }
 
@@ -99,8 +127,6 @@ func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
 
-func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+func (t *tun) NewMultiQueueReader() (tio.Queue, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
 }
-
-func (t *tun) TunPrefixLen() int { return 0 }

overlay/tun_bsd.go

@@ -1,29 +0,0 @@
-//go:build (darwin || ios || freebsd || openbsd || netbsd) && !e2e_testing
-
-package overlay
-
-import (
-	"fmt"
-	"syscall"
-)
-
-// StampTunPrefix writes the 4-byte AF_INET / AF_INET6 protocol-family marker into buf[0:4] in place,
-// picking the family from the first byte of the IP packet at buf[4].
-func StampTunPrefix(buf []byte) error {
-	if len(buf) < 5 {
-		return fmt.Errorf("tun write buffer too small for prefix")
-	}
-	ipVer := buf[4] >> 4
-	buf[0] = 0
-	buf[1] = 0
-	buf[2] = 0
-	switch ipVer {
-	case 4:
-		buf[3] = syscall.AF_INET
-	case 6:
-		buf[3] = syscall.AF_INET6
-	default:
-		return fmt.Errorf("unable to determine IP version from packet")
-	}
-	return nil
-}

overlay/tun_darwin.go

@@ -7,14 +7,16 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log/slog"
 	"net/netip"
 	"os"
 	"sync/atomic"
+	"syscall"
 	"unsafe"
 
 	"github.com/gaissmai/bart"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -22,14 +24,20 @@ import (
 )
 
 type tun struct {
-	io.ReadWriteCloser
+	rwc         io.ReadWriteCloser
 	Device      string
 	vpnNetworks []netip.Prefix
 	DefaultMTU  int
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	linkAddr    *netroute.LinkAddr
-	l           *slog.Logger
+	l           *logrus.Logger
+
+	// cache out buffer since we need to prepend 4 bytes for tun metadata
+	out []byte
+
+	readBuf  []byte
+	batchRet [1][]byte
 }
 
 type ifReq struct {
@@ -75,7 +83,7 @@ type ifreqAlias6 struct {
 	Lifetime   addrLifetime
 }
 
-func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	name := c.GetString("tun.dev", "")
 	ifIndex := -1
 	if name != "" && name != "utun" {
@@ -120,11 +128,11 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 	}
 
 	t := &tun{
-		ReadWriteCloser: os.NewFile(uintptr(fd), ""),
-		Device:          name,
-		vpnNetworks:     vpnNetworks,
-		DefaultMTU:      c.GetInt("tun.mtu", DefaultMTU),
-		l:               l,
+		rwc:         os.NewFile(uintptr(fd), ""),
+		Device:      name,
+		vpnNetworks: vpnNetworks,
+		DefaultMTU:  c.GetInt("tun.mtu", DefaultMTU),
+		l:           l,
 	}
 
 	err = t.reload(c, true)
@@ -149,13 +157,13 @@ func (t *tun) deviceBytes() (o [16]byte) {
 	return
 }
 
-func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in Darwin")
 }
 
 func (t *tun) Close() error {
-	if t.ReadWriteCloser != nil {
-		return t.ReadWriteCloser.Close()
+	if t.rwc != nil {
+		return t.rwc.Close()
 	}
 	return nil
 }
@@ -385,7 +393,8 @@ func (t *tun) addRoutes(logErrors bool) error {
 		err := addRoute(r.Cidr, t.linkAddr)
 		if err != nil {
 			if errors.Is(err, unix.EEXIST) {
-				t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr)
+				t.l.WithField("route", r.Cidr).
+					Warnf("unable to add unsafe_route, identical route already exists")
 			} else {
 				retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
 				if logErrors {
@@ -395,7 +404,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 				}
 			}
 		} else {
-			t.l.Info("Added route", "route", r)
+			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -410,9 +419,9 @@ func (t *tun) removeRoutes(routes []Route) error {
 
 		err := delRoute(r.Cidr, t.linkAddr)
 		if err != nil {
-			t.l.Error("Failed to remove route", "error", err, "route", r)
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
-			t.l.Info("Removed route", "route", r)
+			t.l.WithField("route", r).Info("Removed route")
 		}
 	}
 	return nil
@@ -498,6 +507,60 @@ func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
 	return nil
 }
 
+func (t *tun) readOne(to []byte) (int, error) {
+	buf := make([]byte, len(to)+4)
+
+	n, err := t.rwc.Read(buf)
+
+	copy(to, buf[4:])
+	return n - 4, err
+}
+
+func (t *tun) Read() ([][]byte, error) {
+	if t.readBuf == nil {
+		t.readBuf = make([]byte, defaultBatchBufSize)
+	}
+	n, err := t.readOne(t.readBuf)
+	if err != nil {
+		return nil, err
+	}
+	t.batchRet[0] = t.readBuf[:n]
+	return t.batchRet[:], nil
+}
+
+func (t *tun) WriteReject(p []byte) (int, error) {
+	return t.Write(p)
+}
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	buf := t.out
+	if cap(buf) < len(from)+4 {
+		buf = make([]byte, len(from)+4)
+		t.out = buf
+	}
+	buf = buf[:len(from)+4]
+
+	if len(from) == 0 {
+		return 0, syscall.EIO
+	}
+
+	// Determine the IP Family for the NULL L2 Header
+	ipVer := from[0] >> 4
+	if ipVer == 4 {
+		buf[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		buf[3] = syscall.AF_INET6
+	} else {
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+
+	copy(buf[4:], from)
+
+	n, err := t.rwc.Write(buf)
+	return n - 4, err
+}
+
 func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
@@ -510,10 +573,6 @@ func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
 
-func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+func (t *tun) NewMultiQueueReader() (tio.Queue, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
 }
-
-// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
-// marker the kernel prepends on read and expects on write.
-func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_disabled.go

@@ -1,15 +1,15 @@
 package overlay
 
 import (
-	"context"
 	"fmt"
 	"io"
-	"log/slog"
 	"net/netip"
 	"strings"
 
 	"github.com/rcrowley/go-metrics"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -18,16 +18,35 @@ type disabledTun struct {
 	vpnNetworks []netip.Prefix
 
 	// Track these metrics since we don't have the tun device to do it for us
-	tx metrics.Counter
-	rx metrics.Counter
-	l  *slog.Logger
+	tx         metrics.Counter
+	rx         metrics.Counter
+	l          *logrus.Logger
+	numReaders int
+
+	batchRet [1][]byte
 }
 
-func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled bool, l *slog.Logger) *disabledTun {
+func (t *disabledTun) Read() ([][]byte, error) {
+	r, ok := <-t.read
+	if !ok {
+		return nil, io.EOF
+	}
+
+	t.tx.Inc(1)
+	if t.l.Level >= logrus.DebugLevel {
+		t.l.WithField("raw", prettyPacket(r)).Debugf("Write payload")
+	}
+
+	t.batchRet[0] = r
+	return t.batchRet[:], nil
+}
+
+func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled bool, l *logrus.Logger) *disabledTun {
 	tun := &disabledTun{
 		vpnNetworks: vpnNetworks,
 		read:        make(chan []byte, queueLen),
 		l:           l,
+		numReaders:  1,
 	}
 
 	if metricsEnabled {
@@ -57,24 +76,6 @@ func (*disabledTun) Name() string {
 	return "disabled"
 }
 
-func (t *disabledTun) Read(b []byte) (int, error) {
-	r, ok := <-t.read
-	if !ok {
-		return 0, io.EOF
-	}
-
-	if len(r) > len(b) {
-		return 0, fmt.Errorf("packet larger than mtu: %d > %d bytes", len(r), len(b))
-	}
-
-	t.tx.Inc(1)
-	if t.l.Enabled(context.Background(), slog.LevelDebug) {
-		t.l.Debug("Write payload", "raw", prettyPacket(r))
-	}
-
-	return copy(b, r), nil
-}
-
 func (t *disabledTun) handleICMPEchoRequest(b []byte) bool {
 	out := make([]byte, len(b))
 	out = iputil.CreateICMPEchoResponse(b, out)
@@ -86,7 +87,7 @@ func (t *disabledTun) handleICMPEchoRequest(b []byte) bool {
 	select {
 	case t.read <- out:
 	default:
-		t.l.Debug("tun_disabled: dropped ICMP Echo Reply response")
+		t.l.Debugf("tun_disabled: dropped ICMP Echo Reply response")
 	}
 
 	return true
@@ -97,21 +98,34 @@ func (t *disabledTun) Write(b []byte) (int, error) {
 
 	// Check for ICMP Echo Request before spending time doing the full parsing
 	if t.handleICMPEchoRequest(b) {
-		if t.l.Enabled(context.Background(), slog.LevelDebug) {
-			t.l.Debug("Disabled tun responded to ICMP Echo Request", "raw", prettyPacket(b))
+		if t.l.Level >= logrus.DebugLevel {
+			t.l.WithField("raw", prettyPacket(b)).Debugf("Disabled tun responded to ICMP Echo Request")
 		}
-	} else if t.l.Enabled(context.Background(), slog.LevelDebug) {
-		t.l.Debug("Disabled tun received unexpected payload", "raw", prettyPacket(b))
+	} else if t.l.Level >= logrus.DebugLevel {
+		t.l.WithField("raw", prettyPacket(b)).Debugf("Disabled tun received unexpected payload")
 	}
 	return len(b), nil
 }
 
+func (t *disabledTun) WriteReject(b []byte) (int, error) {
+	return t.Write(b)
+}
+
 func (t *disabledTun) SupportsMultiqueue() bool {
 	return true
 }
 
-func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return t, nil
+func (t *disabledTun) NewMultiQueueReader() error {
+	t.numReaders++
+	return nil
+}
+
+func (t *disabledTun) Readers() []tio.Queue {
+	out := make([]tio.Queue, t.numReaders)
+	for i := range t.numReaders {
+		out[i] = t
+	}
+	return out
 }
 
 func (t *disabledTun) Close() error {
@@ -136,5 +150,3 @@ func (p prettyPacket) String() string {
 
 	return s.String()
 }
-
-func (t *disabledTun) TunPrefixLen() int { return 0 }

overlay/tun_file_linux_test.go

@@ -1,120 +0,0 @@
-//go:build linux && !android && !e2e_testing
-// +build linux,!android,!e2e_testing
-
-package overlay
-
-import (
-	"errors"
-	"os"
-	"sync"
-	"testing"
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
-// The caller takes ownership of the read fd (pass it to newTunFd / newFriend).
-func newReadPipe(t *testing.T) int {
-	t.Helper()
-	var fds [2]int
-	if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
-		t.Fatalf("pipe2: %v", err)
-	}
-	t.Cleanup(func() { _ = unix.Close(fds[1]) })
-	return fds[0]
-}
-
-func TestTunFile_WakeForShutdown_UnblocksRead(t *testing.T) {
-	tf, err := newTunFd(newReadPipe(t))
-	if err != nil {
-		t.Fatalf("newTunFd: %v", err)
-	}
-	t.Cleanup(func() { _ = tf.Close() })
-
-	done := make(chan error, 1)
-	go func() {
-		_, err := tf.Read(make([]byte, 64))
-		done <- err
-	}()
-
-	// Verify Read is actually blocked in poll.
-	select {
-	case err := <-done:
-		t.Fatalf("Read returned before shutdown signal: %v", err)
-	case <-time.After(50 * time.Millisecond):
-	}
-
-	if err := tf.wakeForShutdown(); err != nil {
-		t.Fatalf("wakeForShutdown: %v", err)
-	}
-
-	select {
-	case err := <-done:
-		if !errors.Is(err, os.ErrClosed) {
-			t.Fatalf("expected os.ErrClosed, got %v", err)
-		}
-	case <-time.After(2 * time.Second):
-		t.Fatal("Read did not wake on shutdown")
-	}
-}
-
-func TestTunFile_WakeForShutdown_WakesFriends(t *testing.T) {
-	parent, err := newTunFd(newReadPipe(t))
-	if err != nil {
-		t.Fatalf("newTunFd: %v", err)
-	}
-	friend, err := parent.newFriend(newReadPipe(t))
-	if err != nil {
-		_ = parent.Close()
-		t.Fatalf("newFriend: %v", err)
-	}
-	t.Cleanup(func() {
-		_ = friend.Close()
-		_ = parent.Close()
-	})
-
-	readers := []*tunFile{parent, friend}
-	errs := make([]error, len(readers))
-	var wg sync.WaitGroup
-	for i, r := range readers {
-		wg.Add(1)
-		go func(i int, r *tunFile) {
-			defer wg.Done()
-			_, errs[i] = r.Read(make([]byte, 64))
-		}(i, r)
-	}
-
-	time.Sleep(50 * time.Millisecond)
-
-	if err := parent.wakeForShutdown(); err != nil {
-		t.Fatalf("wakeForShutdown: %v", err)
-	}
-
-	done := make(chan struct{})
-	go func() { wg.Wait(); close(done) }()
-	select {
-	case <-done:
-	case <-time.After(2 * time.Second):
-		t.Fatal("readers did not wake")
-	}
-
-	for i, err := range errs {
-		if !errors.Is(err, os.ErrClosed) {
-			t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
-		}
-	}
-}
-
-func TestTunFile_Close_Idempotent(t *testing.T) {
-	tf, err := newTunFd(newReadPipe(t))
-	if err != nil {
-		t.Fatalf("newTunFd: %v", err)
-	}
-	if err := tf.Close(); err != nil {
-		t.Fatalf("first Close: %v", err)
-	}
-	if err := tf.Close(); err != nil {
-		t.Fatalf("second Close should be a no-op, got %v", err)
-	}
-}

overlay/tun_freebsd.go

@@ -7,9 +7,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"io/fs"
-	"log/slog"
 	"net/netip"
 	"os"
 	"sync/atomic"
@@ -18,9 +16,9 @@ import (
 	"unsafe"
 
 	"github.com/gaissmai/bart"
-
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
-
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -95,7 +93,7 @@ type tun struct {
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	linkAddr    *netroute.LinkAddr
-	l           *slog.Logger
+	l           *logrus.Logger
 
 	fd        int
 	shutdownR int // read end of the shutdown pipe; closing the write end wakes blocked polls
@@ -103,6 +101,9 @@ type tun struct {
 	readPoll  [2]unix.PollFd
 	writePoll [2]unix.PollFd
 	closed    atomic.Bool
+
+	readBuf  []byte
+	batchRet [1][]byte
 }
 
 // blockOnRead waits until the tun fd is readable or shutdown has been signaled.
@@ -157,44 +158,91 @@ func (t *tun) blockOnWrite() error {
 	return nil
 }
 
-func (t *tun) Read(to []byte) (int, error) {
+func (t *tun) Read() ([][]byte, error) {
+	if t.readBuf == nil {
+		t.readBuf = make([]byte, defaultBatchBufSize)
+	}
+	n, err := t.readOne(t.readBuf)
+	if err != nil {
+		return nil, err
+	}
+	t.batchRet[0] = t.readBuf[:n]
+	return t.batchRet[:], nil
+}
+
+func (t *tun) WriteReject(p []byte) (int, error) {
+	return t.Write(p)
+}
+
+func (t *tun) readOne(to []byte) (int, error) {
+	// first 4 bytes is protocol family, in network byte order
+	var head [4]byte
+	iovecs := [2]syscall.Iovec{
+		{&head[0], 4},
+		{&to[0], uint64(len(to))},
+	}
 	for {
-		n, err := unix.Read(t.fd, to)
-		if err == nil {
-			return n, nil
+		n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		if errno == 0 {
+			bytesRead := int(n)
+			if bytesRead < 4 {
+				return 0, nil
+			}
+			return bytesRead - 4, nil
 		}
-		switch err {
+		switch errno {
 		case unix.EAGAIN:
-			if berr := t.blockOnRead(); berr != nil {
-				return 0, berr
+			if err := t.blockOnRead(); err != nil {
+				return 0, err
 			}
 		case unix.EINTR:
 			// retry
 		case unix.EBADF:
 			return 0, os.ErrClosed
 		default:
-			return 0, err
+			return 0, errno
 		}
 	}
 }
 
+// Write is only valid for single threaded use
 func (t *tun) Write(from []byte) (int, error) {
+	if len(from) <= 1 {
+		return 0, syscall.EIO
+	}
+
+	ipVer := from[0] >> 4
+	var head [4]byte
+	// first 4 bytes is protocol family, in network byte order
+	switch ipVer {
+	case 4:
+		head[3] = syscall.AF_INET
+	case 6:
+		head[3] = syscall.AF_INET6
+	default:
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+
+	iovecs := [2]syscall.Iovec{
+		{&head[0], 4},
+		{&from[0], uint64(len(from))},
+	}
 	for {
-		n, err := unix.Write(t.fd, from)
-		if err == nil {
-			return n, nil
+		n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		if errno == 0 {
+			return int(n) - 4, nil
 		}
-		switch err {
+		switch errno {
 		case unix.EAGAIN:
-			if berr := t.blockOnWrite(); berr != nil {
-				return 0, berr
+			if err := t.blockOnWrite(); err != nil {
+				return 0, err
 			}
 		case unix.EINTR:
 			// retry
 		case unix.EBADF:
 			return 0, os.ErrClosed
 		default:
-			return 0, err
+			return 0, errno
 		}
 	}
 }
@@ -214,7 +262,7 @@ func (t *tun) Close() error {
 
 	if t.fd >= 0 {
 		if err := unix.Close(t.fd); err != nil {
-			t.l.Error("Error closing device", "error", err)
+			t.l.WithError(err).Error("Error closing device")
 		}
 		t.fd = -1
 	}
@@ -235,7 +283,7 @@ func (t *tun) Close() error {
 			err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
 		}
 		if err != nil {
-			t.l.Error("Error destroying tunnel", "error", err)
+			t.l.WithError(err).Error("Error destroying tunnel")
 		}
 	}()
 
@@ -248,11 +296,11 @@ func (t *tun) Close() error {
 	return nil
 }
 
-func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in FreeBSD")
 }
 
-func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	// Try to open existing tun device
 	var fd int
 	var err error
@@ -534,7 +582,7 @@ func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
 
-func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+func (t *tun) NewMultiQueueReader() (tio.Queue, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
 }
 
@@ -555,7 +603,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 				return retErr
 			}
 		} else {
-			t.l.Info("Added route", "route", r)
+			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -570,9 +618,9 @@ func (t *tun) removeRoutes(routes []Route) error {
 
 		err := delRoute(r.Cidr, t.linkAddr)
 		if err != nil {
-			t.l.Error("Failed to remove route", "error", err, "route", r)
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
-			t.l.Info("Removed route", "route", r)
+			t.l.WithField("route", r).Info("Removed route")
 		}
 	}
 	return nil
@@ -701,7 +749,3 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
 
 	return nil, nil
 }
-
-// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
-// marker the kernel prepends on read and expects on write.
-func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_ios.go

@@ -4,37 +4,68 @@
 package overlay
 
 import (
+	"errors"
 	"fmt"
 	"io"
-	"log/slog"
 	"net/netip"
 	"os"
+	"sync"
 	"sync/atomic"
+	"syscall"
 
 	"github.com/gaissmai/bart"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 )
 
 type tun struct {
-	io.ReadWriteCloser
+	rwc         io.ReadWriteCloser
 	vpnNetworks []netip.Prefix
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	l           *slog.Logger
+	l           *logrus.Logger
+
+	readBuf  []byte
+	batchRet [1][]byte
 }
 
-func newTun(_ *config.C, _ *slog.Logger, _ []netip.Prefix, _ bool) (*tun, error) {
+func (t *tun) Read() ([][]byte, error) {
+	if t.readBuf == nil {
+		t.readBuf = make([]byte, defaultBatchBufSize)
+	}
+	n, err := t.rwc.Read(t.readBuf)
+	if err != nil {
+		return nil, err
+	}
+	t.batchRet[0] = t.readBuf[:n]
+	return t.batchRet[:], nil
+}
+
+func (t *tun) Write(p []byte) (int, error) {
+	return t.rwc.Write(p)
+}
+
+func (t *tun) WriteReject(p []byte) (int, error) {
+	return t.rwc.Write(p)
+}
+
+func (t *tun) Close() error {
+	return t.rwc.Close()
+}
+
+func newTun(_ *config.C, _ *logrus.Logger, _ []netip.Prefix, _ bool) (*tun, error) {
 	return nil, fmt.Errorf("newTun not supported in iOS")
 }
 
-func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
+func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
 	file := os.NewFile(uintptr(deviceFd), "/dev/tun")
 	t := &tun{
-		vpnNetworks:     vpnNetworks,
-		ReadWriteCloser: file,
-		l:               l,
+		vpnNetworks: vpnNetworks,
+		rwc:         &tunReadCloser{f: file},
+		l:           l,
 	}
 
 	err := t.reload(c, true)
@@ -82,6 +113,64 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	return r
 }
 
+// The following is hoisted up from water, we do this so we can inject our own fd on iOS
+type tunReadCloser struct {
+	f io.ReadWriteCloser
+
+	rMu  sync.Mutex
+	rBuf []byte
+
+	wMu  sync.Mutex
+	wBuf []byte
+}
+
+func (tr *tunReadCloser) Read(to []byte) (int, error) {
+	tr.rMu.Lock()
+	defer tr.rMu.Unlock()
+
+	if cap(tr.rBuf) < len(to)+4 {
+		tr.rBuf = make([]byte, len(to)+4)
+	}
+	tr.rBuf = tr.rBuf[:len(to)+4]
+
+	n, err := tr.f.Read(tr.rBuf)
+	copy(to, tr.rBuf[4:])
+	return n - 4, err
+}
+
+func (tr *tunReadCloser) Write(from []byte) (int, error) {
+	if len(from) == 0 {
+		return 0, syscall.EIO
+	}
+
+	tr.wMu.Lock()
+	defer tr.wMu.Unlock()
+
+	if cap(tr.wBuf) < len(from)+4 {
+		tr.wBuf = make([]byte, len(from)+4)
+	}
+	tr.wBuf = tr.wBuf[:len(from)+4]
+
+	// Determine the IP Family for the NULL L2 Header
+	ipVer := from[0] >> 4
+	if ipVer == 4 {
+		tr.wBuf[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		tr.wBuf[3] = syscall.AF_INET6
+	} else {
+		return 0, errors.New("unable to determine IP version from packet")
+	}
+
+	copy(tr.wBuf[4:], from)
+
+	n, err := tr.f.Write(tr.wBuf)
+	return n - 4, err
+}
+
+func (tr *tunReadCloser) Close() error {
+	return tr.f.Close()
+}
+
 func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
@@ -94,10 +183,6 @@ func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
 
-func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+func (t *tun) NewMultiQueueReader() (tio.Queue, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
 }
-
-// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
-// marker the kernel prepends on read and expects on write.
-func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_linux.go

@@ -4,10 +4,7 @@
 package overlay
 
 import (
-	"encoding/binary"
 	"fmt"
-	"io"
-	"log/slog"
 	"net"
 	"net/netip"
 	"os"
@@ -18,181 +15,17 @@ import (
 	"unsafe"
 
 	"github.com/gaissmai/bart"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
 )
 
-// tunFile wraps a TUN file descriptor with poll-based reads. The FD provided will be changed to non-blocking.
-// A shared eventfd allows Close to wake all readers blocked in poll.
-type tunFile struct {
-	fd         int
-	shutdownFd int
-	lastOne    bool
-	readPoll   [2]unix.PollFd
-	writePoll  [2]unix.PollFd
-	closed     bool
-}
-
-// newFriend makes a tunFile for a MultiQueueReader that copies the shutdown eventfd from the parent tun
-func (r *tunFile) newFriend(fd int) (*tunFile, error) {
-	if err := unix.SetNonblock(fd, true); err != nil {
-		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
-	}
-	return &tunFile{
-		fd:         fd,
-		shutdownFd: r.shutdownFd,
-		readPoll: [2]unix.PollFd{
-			{Fd: int32(fd), Events: unix.POLLIN},
-			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
-		},
-		writePoll: [2]unix.PollFd{
-			{Fd: int32(fd), Events: unix.POLLOUT},
-			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
-		},
-	}, nil
-}
-
-func newTunFd(fd int) (*tunFile, error) {
-	if err := unix.SetNonblock(fd, true); err != nil {
-		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
-	}
-
-	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create eventfd: %w", err)
-	}
-
-	out := &tunFile{
-		fd:         fd,
-		shutdownFd: shutdownFd,
-		lastOne:    true,
-		readPoll: [2]unix.PollFd{
-			{Fd: int32(fd), Events: unix.POLLIN},
-			{Fd: int32(shutdownFd), Events: unix.POLLIN},
-		},
-		writePoll: [2]unix.PollFd{
-			{Fd: int32(fd), Events: unix.POLLOUT},
-			{Fd: int32(shutdownFd), Events: unix.POLLIN},
-		},
-	}
-
-	return out, nil
-}
-
-func (r *tunFile) blockOnRead() error {
-	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
-	var err error
-	for {
-		_, err = unix.Poll(r.readPoll[:], -1)
-		if err != unix.EINTR {
-			break
-		}
-	}
-	//always reset these!
-	tunEvents := r.readPoll[0].Revents
-	shutdownEvents := r.readPoll[1].Revents
-	r.readPoll[0].Revents = 0
-	r.readPoll[1].Revents = 0
-	//do the err check before trusting the potentially bogus bits we just got
-	if err != nil {
-		return err
-	}
-	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
-		return os.ErrClosed
-	} else if tunEvents&problemFlags != 0 {
-		return os.ErrClosed
-	}
-	return nil
-}
-
-func (r *tunFile) blockOnWrite() error {
-	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
-	var err error
-	for {
-		_, err = unix.Poll(r.writePoll[:], -1)
-		if err != unix.EINTR {
-			break
-		}
-	}
-	//always reset these!
-	tunEvents := r.writePoll[0].Revents
-	shutdownEvents := r.writePoll[1].Revents
-	r.writePoll[0].Revents = 0
-	r.writePoll[1].Revents = 0
-	//do the err check before trusting the potentially bogus bits we just got
-	if err != nil {
-		return err
-	}
-	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
-		return os.ErrClosed
-	} else if tunEvents&problemFlags != 0 {
-		return os.ErrClosed
-	}
-	return nil
-}
-
-func (r *tunFile) Read(buf []byte) (int, error) {
-	for {
-		if n, err := unix.Read(r.fd, buf); err == nil {
-			return n, nil
-		} else if err == unix.EAGAIN {
-			if err = r.blockOnRead(); err != nil {
-				return 0, err
-			}
-			continue
-		} else if err == unix.EINTR {
-			continue
-		} else if err == unix.EBADF {
-			return 0, os.ErrClosed
-		} else {
-			return 0, err
-		}
-	}
-}
-
-func (r *tunFile) Write(buf []byte) (int, error) {
-	for {
-		if n, err := unix.Write(r.fd, buf); err == nil {
-			return n, nil
-		} else if err == unix.EAGAIN {
-			if err = r.blockOnWrite(); err != nil {
-				return 0, err
-			}
-			continue
-		} else if err == unix.EINTR {
-			continue
-		} else if err == unix.EBADF {
-			return 0, os.ErrClosed
-		} else {
-			return 0, err
-		}
-	}
-}
-
-func (r *tunFile) wakeForShutdown() error {
-	var buf [8]byte
-	binary.NativeEndian.PutUint64(buf[:], 1)
-	_, err := unix.Write(int(r.readPoll[1].Fd), buf[:])
-	return err
-}
-
-func (r *tunFile) Close() error {
-	if r.closed { // avoid closing more than once. Technically a fd could get re-used, which would be a problem
-		return nil
-	}
-	r.closed = true
-	if r.lastOne {
-		_ = unix.Close(r.shutdownFd)
-	}
-	return unix.Close(r.fd)
-}
-
 type tun struct {
-	*tunFile
-	readers     []*tunFile
+	readers     tio.Container
 	closeLock   sync.Mutex
 	Device      string
 	vpnNetworks []netip.Prefix
@@ -201,6 +34,7 @@ type tun struct {
 	TXQueueLen  int
 	deviceIndex int
 	ioctlFd     uintptr
+	vnetHdr     bool
 
 	Routes                    atomic.Pointer[[]Route]
 	routeTree                 atomic.Pointer[bart.Table[routing.Gateways]]
@@ -213,7 +47,7 @@ type tun struct {
 	routesFromSystem     map[netip.Prefix]routing.Gateways
 	routesFromSystemLock sync.Mutex
 
-	l *slog.Logger
+	l *logrus.Logger
 }
 
 func (t *tun) Networks() []netip.Prefix {
@@ -238,8 +72,10 @@ type ifreqQLEN struct {
 	pad   [8]byte
 }
 
-func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
-	t, err := newTunGeneric(c, l, deviceFd, vpnNetworks)
+func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
+	// We don't know what flags the caller opened this fd with and can't turn
+	// on IFF_VNET_HDR after TUNSETIFF, so skip offload on inherited fds.
+	t, err := newTunGeneric(c, l, deviceFd, false, vpnNetworks)
 	if err != nil {
 		return nil, err
 	}
@@ -249,46 +85,83 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 	return t, nil
 }
 
-func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
+// openTunDev opens /dev/net/tun, creating the device node first if it's
+// missing (docker containers occasionally omit it).
+func openTunDev() (int, error) {
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
-	if err != nil {
-		// If /dev/net/tun doesn't exist, try to create it (will happen in docker)
-		if os.IsNotExist(err) {
-			err = os.MkdirAll("/dev/net", 0755)
-			if err != nil {
-				return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
-			}
-			err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200)))
-			if err != nil {
-				return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err)
-			}
-
-			fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0)
-			if err != nil {
-				return nil, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
-			}
-		} else {
-			return nil, err
-		}
+	if err == nil {
+		return fd, nil
 	}
+	if !os.IsNotExist(err) {
+		return -1, err
+	}
+	if err = os.MkdirAll("/dev/net", 0755); err != nil {
+		return -1, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
+	}
+	if err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200))); err != nil {
+		return -1, fmt.Errorf("failed to create /dev/net/tun: %w", err)
+	}
+	fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0)
+	if err != nil {
+		return -1, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
+	}
+	return fd, nil
+}
 
+// tunSetIff runs TUNSETIFF with the given flags and returns the kernel-chosen
+// device name on success.
+func tunSetIff(fd int, name string, flags uint16) (string, error) {
 	var req ifReq
-	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
+	req.Flags = flags
+	copy(req.Name[:], name)
+	if err := ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
+		return "", err
+	}
+	return strings.Trim(string(req.Name[:]), "\x00"), nil
+}
+
+// tsoOffloadFlags are the TUN_F_* bits we ask the kernel to enable when a
+// TSO-capable TUN is available. CSUM is required as a prerequisite for TSO.
+const tsoOffloadFlags = unix.TUN_F_CSUM | unix.TUN_F_TSO4 | unix.TUN_F_TSO6
+
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
+	baseFlags := uint16(unix.IFF_TUN | unix.IFF_NO_PI)
 	if multiqueue {
-		req.Flags |= unix.IFF_MULTI_QUEUE
+		baseFlags |= unix.IFF_MULTI_QUEUE
 	}
 	nameStr := c.GetString("tun.dev", "")
-	copy(req.Name[:], nameStr)
-	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
+
+	// First try to open with IFF_VNET_HDR + TUNSETOFFLOAD so we can receive
+	// TSO superpackets. If either step fails (older kernel, unprivileged
+	// container, etc.) we close and fall back to a plain TUN.
+	fd, err := openTunDev()
+	if err != nil {
+		return nil, err
+	}
+	vnetHdr := true
+	name, err := tunSetIff(fd, nameStr, baseFlags|unix.IFF_VNET_HDR|unix.IFF_NAPI)
+	if err != nil {
 		_ = unix.Close(fd)
-		return nil, &NameError{
-			Name:       nameStr,
-			Underlying: err,
+		vnetHdr = false
+	} else if err = ioctl(uintptr(fd), unix.TUNSETOFFLOAD, uintptr(tsoOffloadFlags)); err != nil {
+		l.WithError(err).Warn("Failed to enable TUN offload (TSO); proceeding without virtio headers")
+		_ = unix.Close(fd)
+		vnetHdr = false
+	}
+
+	if !vnetHdr {
+		fd, err = openTunDev()
+		if err != nil {
+			return nil, err
+		}
+		name, err = tunSetIff(fd, nameStr, baseFlags)
+		if err != nil {
+			_ = unix.Close(fd)
+			return nil, &NameError{Name: nameStr, Underlying: err}
 		}
 	}
-	name := strings.Trim(string(req.Name[:]), "\x00")
 
-	t, err := newTunGeneric(c, l, fd, vpnNetworks)
+	t, err := newTunGeneric(c, l, fd, vnetHdr, vpnNetworks)
 	if err != nil {
 		return nil, err
 	}
@@ -299,16 +172,29 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
 }
 
 // newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error.
-func newTunGeneric(c *config.C, l *slog.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) {
-	tfd, err := newTunFd(fd)
+func newTunGeneric(c *config.C, l *logrus.Logger, fd int, vnetHdr bool, vpnNetworks []netip.Prefix) (*tun, error) {
+	var container tio.Container
+	var err error
+	if vnetHdr {
+		container, err = tio.NewOffloadContainer()
+	} else {
+		container, err = tio.NewPollContainer()
+	}
+
 	if err != nil {
 		_ = unix.Close(fd)
 		return nil, err
 	}
+	err = container.Add(fd)
+	if err != nil {
+		_ = unix.Close(fd)
+		return nil, err
+	}
+
 	t := &tun{
-		tunFile:                   tfd,
-		readers:                   []*tunFile{tfd},
+		readers:                   container,
 		closeLock:                 sync.Mutex{},
+		vnetHdr:                   vnetHdr,
 		vpnNetworks:               vpnNetworks,
 		TXQueueLen:                c.GetInt("tun.tx_queue", 500),
 		useSystemRoutes:           c.GetBool("tun.use_system_route_table", false),
@@ -378,16 +264,16 @@ func (t *tun) reload(c *config.C, initial bool) error {
 	if !initial {
 		if oldMaxMTU != newMaxMTU {
 			t.setMTU()
-			t.l.Info("Set max MTU", "mtu", t.MaxMTU, "oldMTU", oldMaxMTU)
+			t.l.Infof("Set max MTU to %v was %v", t.MaxMTU, oldMaxMTU)
 		}
 
 		if oldDefaultMTU != newDefaultMTU {
 			for i := range t.vpnNetworks {
 				err := t.setDefaultRoute(t.vpnNetworks[i])
 				if err != nil {
-					t.l.Warn(err.Error())
+					t.l.Warn(err)
 				} else {
-					t.l.Info("Set default MTU", "mtu", t.DefaultMTU, "oldMTU", oldDefaultMTU)
+					t.l.Infof("Set default MTU to %v was %v", t.DefaultMTU, oldDefaultMTU)
 				}
 			}
 		}
@@ -410,32 +296,38 @@ func (t *tun) SupportsMultiqueue() bool {
 	return true
 }
 
-func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+func (t *tun) NewMultiQueueReader() error {
 	t.closeLock.Lock()
 	defer t.closeLock.Unlock()
 
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	var req ifReq
-	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
-	copy(req.Name[:], t.Device)
-	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
+	flags := uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
+	if t.vnetHdr {
+		flags |= unix.IFF_VNET_HDR | unix.IFF_NAPI
+	}
+	if _, err = tunSetIff(fd, t.Device, flags); err != nil {
 		_ = unix.Close(fd)
-		return nil, err
+		return err
 	}
 
-	out, err := t.tunFile.newFriend(fd)
+	if t.vnetHdr {
+		if err = ioctl(uintptr(fd), unix.TUNSETOFFLOAD, uintptr(tsoOffloadFlags)); err != nil {
+			_ = unix.Close(fd)
+			return fmt.Errorf("failed to enable offload on multiqueue tun fd: %w", err)
+		}
+	}
+
+	err = t.readers.Add(fd)
 	if err != nil {
 		_ = unix.Close(fd)
-		return nil, err
+		return err
 	}
 
-	t.readers = append(t.readers, out)
-
-	return out, nil
+	return nil
 }
 
 func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
@@ -492,9 +384,9 @@ func (t *tun) addIPs(link netlink.Link) error {
 		}
 		err = netlink.AddrDel(link, &al[i])
 		if err != nil {
-			t.l.Error("failed to remove address from tun address list", "error", err)
+			t.l.WithError(err).Error("failed to remove address from tun address list")
 		} else {
-			t.l.Info("removed address not listed in cert(s)", "removed", al[i].String())
+			t.l.WithField("removed", al[i].String()).Info("removed address not listed in cert(s)")
 		}
 	}
 
@@ -538,12 +430,12 @@ func (t *tun) Activate() error {
 	ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)}
 	if err = ioctl(t.ioctlFd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil {
 		// If we can't set the queue length nebula will still work but it may lead to packet loss
-		t.l.Error("Failed to set tun tx queue length", "error", err)
+		t.l.WithError(err).Error("Failed to set tun tx queue length")
 	}
 
 	const modeNone = 1
 	if err = netlink.LinkSetIP6AddrGenMode(link, modeNone); err != nil {
-		t.l.Warn("Failed to disable link local address generation", "error", err)
+		t.l.WithError(err).Warn("Failed to disable link local address generation")
 	}
 
 	if err = t.addIPs(link); err != nil {
@@ -582,7 +474,7 @@ func (t *tun) setMTU() {
 	ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)}
 	if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
 		// This is currently a non fatal condition because the route table must have the MTU set appropriately as well
-		t.l.Error("Failed to set tun mtu", "error", err)
+		t.l.WithError(err).Error("Failed to set tun mtu")
 	}
 }
 
@@ -605,7 +497,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
 	}
 	err := netlink.RouteReplace(&nr)
 	if err != nil {
-		t.l.Warn("Failed to set default route MTU, retrying", "error", err, "cidr", cidr)
+		t.l.WithError(err).WithField("cidr", cidr).Warn("Failed to set default route MTU, retrying")
 		//retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument`
 		for i := 0; i < 2; i++ {
 			time.Sleep(100 * time.Millisecond)
@@ -613,11 +505,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
 			if err == nil {
 				break
 			} else {
-				t.l.Warn("Failed to set default route MTU, retrying",
-					"error", err,
-					"cidr", cidr,
-					"mtu", t.DefaultMTU,
-				)
+				t.l.WithError(err).WithField("cidr", cidr).WithField("mtu", t.DefaultMTU).Warn("Failed to set default route MTU, retrying")
 			}
 		}
 		if err != nil {
@@ -662,7 +550,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 				return retErr
 			}
 		} else {
-			t.l.Info("Added route", "route", r)
+			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -694,9 +582,9 @@ func (t *tun) removeRoutes(routes []Route) {
 
 		err := netlink.RouteDel(&nr)
 		if err != nil {
-			t.l.Error("Failed to remove route", "error", err, "route", r)
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
-			t.l.Info("Removed route", "route", r)
+			t.l.WithField("route", r).Info("Removed route")
 		}
 	}
 }
@@ -725,11 +613,11 @@ func (t *tun) watchRoutes() {
 	netlinkOptions := netlink.RouteSubscribeOptions{
 		ReceiveBufferSize:      t.useSystemRoutesBufferSize,
 		ReceiveBufferForceSize: t.useSystemRoutesBufferSize != 0,
-		ErrorCallback:          func(e error) { t.l.Error("netlink error", "error", e) },
+		ErrorCallback:          func(e error) { t.l.WithError(e).Errorf("netlink error") },
 	}
 
 	if err := netlink.RouteSubscribeWithOptions(rch, doneChan, netlinkOptions); err != nil {
-		t.l.Error("failed to subscribe to system route changes", "error", err)
+		t.l.WithError(err).Errorf("failed to subscribe to system route changes")
 		return
 	}
 
@@ -771,7 +659,7 @@ func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways {
 
 	link, err := netlink.LinkByName(t.Device)
 	if err != nil {
-		t.l.Error("Ignoring route update: failed to get link by name", "deviceName", t.Device)
+		t.l.WithField("deviceName", t.Device).Error("Ignoring route update: failed to get link by name")
 		return gateways
 	}
 
@@ -783,10 +671,10 @@ func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways {
 				gateways = append(gateways, routing.NewGateway(gwAddr, 1))
 			} else {
 				// Gateway isn't in our overlay network, ignore
-				t.l.Debug("Ignoring route update, gateway is not in our network", "route", r)
+				t.l.WithField("route", r).Debug("Ignoring route update, gateway is not in our network")
 			}
 		} else {
-			t.l.Debug("Ignoring route update, invalid gateway or via address", "route", r)
+			t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway or via address")
 		}
 	}
 
@@ -799,10 +687,10 @@ func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways {
 					gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1))
 				} else {
 					// Gateway isn't in our overlay network, ignore
-					t.l.Debug("Ignoring route update, gateway is not in our network", "route", r)
+					t.l.WithField("route", r).Debug("Ignoring route update, gateway is not in our network")
 				}
 			} else {
-				t.l.Debug("Ignoring route update, invalid gateway or via address", "route", r)
+				t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway or via address")
 			}
 		}
 	}
@@ -834,18 +722,18 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
 	gateways := t.getGatewaysFromRoute(&r.Route)
 	if len(gateways) == 0 {
 		// No gateways relevant to our network, no routing changes required.
-		t.l.Debug("Ignoring route update, no gateways", "route", r)
+		t.l.WithField("route", r).Debug("Ignoring route update, no gateways")
 		return
 	}
 
 	if r.Dst == nil {
-		t.l.Debug("Ignoring route update, no destination address", "route", r)
+		t.l.WithField("route", r).Debug("Ignoring route update, no destination address")
 		return
 	}
 
 	dstAddr, ok := netip.AddrFromSlice(r.Dst.IP)
 	if !ok {
-		t.l.Debug("Ignoring route update, invalid destination address", "route", r)
+		t.l.WithField("route", r).Debug("Ignoring route update, invalid destination address")
 		return
 	}
 
@@ -856,12 +744,12 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
 
 	t.routesFromSystemLock.Lock()
 	if r.Type == unix.RTM_NEWROUTE {
-		t.l.Info("Adding route", "destination", dst, "via", gateways)
+		t.l.WithField("destination", dst).WithField("via", gateways).Info("Adding route")
 		t.routesFromSystem[dst] = gateways
 		newTree.Insert(dst, gateways)
 
 	} else {
-		t.l.Info("Removing route", "destination", dst, "via", gateways)
+		t.l.WithField("destination", dst).WithField("via", gateways).Info("Removing route")
 		delete(t.routesFromSystem, dst)
 		newTree.Delete(dst)
 	}
@@ -869,6 +757,10 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
 	t.routeTree.Store(newTree)
 }
 
+func (t *tun) Readers() []tio.Queue {
+	return t.readers.Queues()
+}
+
 func (t *tun) Close() error {
 	t.closeLock.Lock()
 	defer t.closeLock.Unlock()
@@ -878,34 +770,10 @@ func (t *tun) Close() error {
 		t.routeChan = nil
 	}
 
-	// Signal all readers blocked in poll to wake up and exit
-	_ = t.tunFile.wakeForShutdown()
-
 	if t.ioctlFd > 0 {
 		_ = unix.Close(int(t.ioctlFd))
 		t.ioctlFd = 0
 	}
 
-	for i := range t.readers {
-		if i == 0 {
-			continue //we want to close the zeroth reader last
-		}
-		err := t.readers[i].Close()
-		if err != nil {
-			t.l.Error("error closing tun reader", "reader", i, "error", err)
-		} else {
-			t.l.Info("closed tun reader", "reader", i)
-		}
-	}
-
-	//this is t.readers[0] too
-	err := t.tunFile.Close()
-	if err != nil {
-		t.l.Error("error closing tun reader", "reader", 0, "error", err)
-	} else {
-		t.l.Info("closed tun reader", "reader", 0)
-	}
-	return err
+	return t.readers.Close()
 }
-
-func (t *tun) TunPrefixLen() int { return 0 }

overlay/tun_linux_test.go

@@ -3,7 +3,9 @@
 
 package overlay
 
-import "testing"
+import (
+	"testing"
+)
 
 var runAdvMSSTests = []struct {
 	name     string

overlay/tun_netbsd.go

@@ -6,8 +6,6 @@ package overlay
 import (
 	"errors"
 	"fmt"
-	"io"
-	"log/slog"
 	"net/netip"
 	"os"
 	"regexp"
@@ -16,7 +14,9 @@ import (
 	"unsafe"
 
 	"github.com/gaissmai/bart"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -58,23 +58,42 @@ type addrLifetime struct {
 }
 
 type tun struct {
-	io.ReadWriteCloser
 	Device      string
 	vpnNetworks []netip.Prefix
 	MTU         int
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	l           *slog.Logger
+	l           *logrus.Logger
+	f           *os.File
 	fd          int
+
+	readBuf  []byte
+	batchRet [1][]byte
+}
+
+func (t *tun) Read() ([][]byte, error) {
+	if t.readBuf == nil {
+		t.readBuf = make([]byte, defaultBatchBufSize)
+	}
+	n, err := t.readOne(t.readBuf)
+	if err != nil {
+		return nil, err
+	}
+	t.batchRet[0] = t.readBuf[:n]
+	return t.batchRet[:], nil
+}
+
+func (t *tun) WriteReject(p []byte) (int, error) {
+	return t.Write(p)
 }
 
 var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
 
-func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in NetBSD")
 }
 
-func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	// Try to open tun device
 	var err error
 	deviceName := c.GetString("tun.dev", "")
@@ -92,16 +111,16 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 
 	err = unix.SetNonblock(fd, true)
 	if err != nil {
-		l.Warn("Failed to set the tun device as nonblocking", "error", err)
+		l.WithError(err).Warn("Failed to set the tun device as nonblocking")
 	}
 
 	t := &tun{
-		ReadWriteCloser: os.NewFile(uintptr(fd), ""),
-		fd:              fd,
-		Device:          deviceName,
-		vpnNetworks:     vpnNetworks,
-		MTU:             c.GetInt("tun.mtu", DefaultMTU),
-		l:               l,
+		f:           os.NewFile(uintptr(fd), ""),
+		fd:          fd,
+		Device:      deviceName,
+		vpnNetworks: vpnNetworks,
+		MTU:         c.GetInt("tun.mtu", DefaultMTU),
+		l:           l,
 	}
 
 	err = t.reload(c, true)
@@ -120,12 +139,12 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 }
 
 func (t *tun) Close() error {
-	if t.ReadWriteCloser != nil {
-		if err := t.ReadWriteCloser.Close(); err != nil {
+	if t.f != nil {
+		if err := t.f.Close(); err != nil {
 			return fmt.Errorf("error closing tun file: %w", err)
 		}
 
-		// Close on the os.File should have handled the fd for us but let's be extra sure
+		// t.f.Close should have handled it for us but let's be extra sure
 		_ = unix.Close(t.fd)
 
 		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
@@ -141,6 +160,99 @@ func (t *tun) Close() error {
 	return nil
 }
 
+func (t *tun) readOne(to []byte) (int, error) {
+	rc, err := t.f.SyscallConn()
+	if err != nil {
+		return 0, fmt.Errorf("failed to get syscall conn for tun: %w", err)
+	}
+
+	var errno syscall.Errno
+	var n uintptr
+	err = rc.Read(func(fd uintptr) bool {
+		// first 4 bytes is protocol family, in network byte order
+		head := [4]byte{}
+		iovecs := []syscall.Iovec{
+			{&head[0], 4},
+			{&to[0], uint64(len(to))},
+		}
+
+		n, _, errno = syscall.Syscall(syscall.SYS_READV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+		if errno.Temporary() {
+			// We got an EAGAIN, EINTR, or EWOULDBLOCK, go again
+			return false
+		}
+		return true
+	})
+	if err != nil {
+		if err == syscall.EBADF || err.Error() == "use of closed file" {
+			// Go doesn't export poll.ErrFileClosing but happily reports it to us so here we are
+			// https://github.com/golang/go/blob/master/src/internal/poll/fd_poll_runtime.go#L121
+			return 0, os.ErrClosed
+		}
+		return 0, fmt.Errorf("failed to make read call for tun: %w", err)
+	}
+
+	if errno != 0 {
+		return 0, fmt.Errorf("failed to make inner read call for tun: %w", errno)
+	}
+
+	// fix bytes read number to exclude header
+	bytesRead := int(n)
+	if bytesRead < 0 {
+		return bytesRead, nil
+	} else if bytesRead < 4 {
+		return 0, nil
+	} else {
+		return bytesRead - 4, nil
+	}
+}
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	if len(from) <= 1 {
+		return 0, syscall.EIO
+	}
+
+	ipVer := from[0] >> 4
+	var head [4]byte
+	// first 4 bytes is protocol family, in network byte order
+	if ipVer == 4 {
+		head[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		head[3] = syscall.AF_INET6
+	} else {
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+
+	rc, err := t.f.SyscallConn()
+	if err != nil {
+		return 0, err
+	}
+
+	var errno syscall.Errno
+	var n uintptr
+	err = rc.Write(func(fd uintptr) bool {
+		iovecs := []syscall.Iovec{
+			{&head[0], 4},
+			{&from[0], uint64(len(from))},
+		}
+
+		n, _, errno = syscall.Syscall(syscall.SYS_WRITEV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+		// According to NetBSD documentation for TUN, writes will only return errors in which
+		// this packet will never be delivered so just go on living life.
+		return true
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	if errno != 0 {
+		return 0, errno
+	}
+
+	return int(n) - 4, err
+}
+
 func (t *tun) addIp(cidr netip.Prefix) error {
 	if cidr.Addr().Is4() {
 		var req ifreqAlias4
@@ -301,7 +413,7 @@ func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
 
-func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+func (t *tun) NewMultiQueueReader() (tio.Queue, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd")
 }
 
@@ -323,7 +435,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 				return retErr
 			}
 		} else {
-			t.l.Info("Added route", "route", r)
+			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -338,9 +450,9 @@ func (t *tun) removeRoutes(routes []Route) error {
 
 		err := delRoute(r.Cidr, t.vpnNetworks)
 		if err != nil {
-			t.l.Error("Failed to remove route", "error", err, "route", r)
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
-			t.l.Info("Removed route", "route", r)
+			t.l.WithField("route", r).Info("Removed route")
 		}
 	}
 	return nil
@@ -458,7 +570,3 @@ func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
 
 	return nil
 }
-
-// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
-// marker the kernel prepends on read and expects on write.
-func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_no_prefix.go

@@ -1,10 +0,0 @@
-//go:build (!darwin && !ios && !freebsd && !openbsd && !netbsd) || e2e_testing
-
-package overlay
-
-// StampTunPrefix is a no-op on platforms whose tun devices have no
-// protocol-family marker. WireBuffer only invokes it when its prefixLen
-// is non-zero, so this should never be reached on these platforms.
-func StampTunPrefix(buf []byte) error {
-	return nil
-}

overlay/tun_openbsd.go

@@ -6,8 +6,6 @@ package overlay
 import (
 	"errors"
 	"fmt"
-	"io"
-	"log/slog"
 	"net/netip"
 	"os"
 	"regexp"
@@ -16,7 +14,9 @@ import (
 	"unsafe"
 
 	"github.com/gaissmai/bart"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -49,23 +49,44 @@ type ifreq struct {
 }
 
 type tun struct {
-	io.ReadWriteCloser
 	Device      string
 	vpnNetworks []netip.Prefix
 	MTU         int
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	l           *slog.Logger
+	l           *logrus.Logger
+	f           *os.File
 	fd          int
+	// cache out buffer since we need to prepend 4 bytes for tun metadata
+	out []byte
+
+	readBuf  []byte
+	batchRet [1][]byte
+}
+
+func (t *tun) Read() ([][]byte, error) {
+	if t.readBuf == nil {
+		t.readBuf = make([]byte, defaultBatchBufSize)
+	}
+	n, err := t.readOne(t.readBuf)
+	if err != nil {
+		return nil, err
+	}
+	t.batchRet[0] = t.readBuf[:n]
+	return t.batchRet[:], nil
+}
+
+func (t *tun) WriteReject(p []byte) (int, error) {
+	return t.Write(p)
 }
 
 var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
 
-func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in openbsd")
 }
 
-func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	// Try to open tun device
 	var err error
 	deviceName := c.GetString("tun.dev", "")
@@ -83,16 +104,16 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 
 	err = unix.SetNonblock(fd, true)
 	if err != nil {
-		l.Warn("Failed to set the tun device as nonblocking", "error", err)
+		l.WithError(err).Warn("Failed to set the tun device as nonblocking")
 	}
 
 	t := &tun{
-		ReadWriteCloser: os.NewFile(uintptr(fd), ""),
-		fd:              fd,
-		Device:          deviceName,
-		vpnNetworks:     vpnNetworks,
-		MTU:             c.GetInt("tun.mtu", DefaultMTU),
-		l:               l,
+		f:           os.NewFile(uintptr(fd), ""),
+		fd:          fd,
+		Device:      deviceName,
+		vpnNetworks: vpnNetworks,
+		MTU:         c.GetInt("tun.mtu", DefaultMTU),
+		l:           l,
 	}
 
 	err = t.reload(c, true)
@@ -111,17 +132,55 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 }
 
 func (t *tun) Close() error {
-	if t.ReadWriteCloser != nil {
-		if err := t.ReadWriteCloser.Close(); err != nil {
+	if t.f != nil {
+		if err := t.f.Close(); err != nil {
 			return fmt.Errorf("error closing tun file: %w", err)
 		}
 
-		// Close on the os.File should have handled the fd for us but let's be extra sure
+		// t.f.Close should have handled it for us but let's be extra sure
 		_ = unix.Close(t.fd)
 	}
 	return nil
 }
 
+func (t *tun) readOne(to []byte) (int, error) {
+	buf := make([]byte, len(to)+4)
+
+	n, err := t.f.Read(buf)
+
+	copy(to, buf[4:])
+	return n - 4, err
+}
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	buf := t.out
+	if cap(buf) < len(from)+4 {
+		buf = make([]byte, len(from)+4)
+		t.out = buf
+	}
+	buf = buf[:len(from)+4]
+
+	if len(from) == 0 {
+		return 0, syscall.EIO
+	}
+
+	// Determine the IP Family for the NULL L2 Header
+	ipVer := from[0] >> 4
+	if ipVer == 4 {
+		buf[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		buf[3] = syscall.AF_INET6
+	} else {
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+
+	copy(buf[4:], from)
+
+	n, err := t.f.Write(buf)
+	return n - 4, err
+}
+
 func (t *tun) addIp(cidr netip.Prefix) error {
 	if cidr.Addr().Is4() {
 		var req ifreqAlias4
@@ -274,7 +333,7 @@ func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
 
-func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+func (t *tun) NewMultiQueueReader() (tio.Queue, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd")
 }
 
@@ -296,7 +355,7 @@ func (t *tun) addRoutes(logErrors bool) error {
 				return retErr
 			}
 		} else {
-			t.l.Info("Added route", "route", r)
+			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
@@ -311,9 +370,9 @@ func (t *tun) removeRoutes(routes []Route) error {
 
 		err := delRoute(r.Cidr, t.vpnNetworks)
 		if err != nil {
-			t.l.Error("Failed to remove route", "error", err, "route", r)
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
-			t.l.Info("Removed route", "route", r)
+			t.l.WithField("route", r).Info("Removed route")
 		}
 	}
 	return nil
@@ -431,7 +490,3 @@ func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
 
 	return nil
 }
-
-// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
-// marker the kernel prepends on read and expects on write.
-func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_tester.go

@@ -4,18 +4,17 @@
 package overlay
 
 import (
-	"context"
 	"fmt"
 	"io"
-	"log/slog"
 	"net/netip"
 	"os"
 	"sync/atomic"
 
 	"github.com/gaissmai/bart"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
-	"github.com/slackhq/nebula/udp"
 )
 
 type TestTun struct {
@@ -23,14 +22,25 @@ type TestTun struct {
 	vpnNetworks []netip.Prefix
 	Routes      []Route
 	routeTree   *bart.Table[routing.Gateways]
-	l           *slog.Logger
+	l           *logrus.Logger
 
 	closed    atomic.Bool
 	rxPackets chan []byte // Packets to receive into nebula
 	TxPackets chan []byte // Packets transmitted outside by nebula
+
+	batchRet [1][]byte
 }
 
-func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) {
+func (t *TestTun) Read() ([][]byte, error) {
+	p, ok := <-t.rxPackets
+	if !ok {
+		return nil, os.ErrClosed
+	}
+	t.batchRet[0] = p
+	return t.batchRet[:], nil
+}
+
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) {
 	_, routes, err := getAllRoutesFromConfig(c, vpnNetworks, true)
 	if err != nil {
 		return nil, err
@@ -51,27 +61,22 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*T
 	}, nil
 }
 
-func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*TestTun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*TestTun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported")
 }
 
-// Send will place a byte array onto the receive queue for nebula to consume.
+// Send will place a byte array onto the receive queue for nebula to consume
 // These are unencrypted ip layer frames destined for another nebula node.
-// packets should exit the udp side, capture them with udpConn.Get.
-//
-// Send copies the input via the freelist, so the caller is free to mutate
-// or reuse it after the call returns.
+// packets should exit the udp side, capture them with udpConn.Get
 func (t *TestTun) Send(packet []byte) {
 	if t.closed.Load() {
 		return
 	}
 
-	if t.l.Enabled(context.Background(), slog.LevelDebug) {
-		t.l.Debug("Tun receiving injected packet", "dataLen", len(packet))
+	if t.l.Level >= logrus.DebugLevel {
+		t.l.WithField("dataLen", len(packet)).Debug("Tun receiving injected packet")
 	}
-	buf := acquireTunBuf(len(packet))
-	copy(buf, packet)
-	t.rxPackets <- buf
+	t.rxPackets <- packet
 }
 
 // Get will pull an unencrypted ip layer frame from the transmit queue
@@ -116,42 +121,14 @@ func (t *TestTun) Write(b []byte) (n int, err error) {
 		return 0, io.ErrClosedPipe
 	}
 
-	packet := acquireTunBuf(len(b))
+	packet := make([]byte, len(b), len(b))
 	copy(packet, b)
 	t.TxPackets <- packet
 	return len(b), nil
 }
 
-// ReleaseTunBuf returns a slice from TxPackets to the harness freelist, don't use the bytes after the call.
-// Channel-backed instead of sync.Pool because putting a []byte in a sync.Pool escapes the slice header to heap.
-func ReleaseTunBuf(b []byte) {
-	if b == nil {
-		return
-	}
-	select {
-	case tunBufFreelist <- b:
-	default:
-		// Freelist full; drop the buffer for the GC.
-	}
-}
-
-// tunBufFreelist retains the backing arrays for TestTun.Write so steady-state allocation drops to zero once the
-// freelist has saturated for the current MTU.
-var tunBufFreelist = make(chan []byte, 64)
-
-func acquireTunBuf(n int) []byte {
-	var b []byte
-	select {
-	case b = <-tunBufFreelist:
-	default:
-		b = make([]byte, 0, udp.MTU)
-	}
-	if cap(b) < n {
-		b = make([]byte, n)
-	} else {
-		b = b[:n]
-	}
-	return b
+func (t *TestTun) WriteReject(b []byte) (int, error) {
+	return t.Write(b)
 }
 
 func (t *TestTun) Close() error {
@@ -162,27 +139,10 @@ func (t *TestTun) Close() error {
 	return nil
 }
 
-func (t *TestTun) Read(b []byte) (int, error) {
-	p, ok := <-t.rxPackets
-	if !ok {
-		return 0, os.ErrClosed
-	}
-	n := len(p)
-	copy(b, p)
-	// Send always pushes a freelist-acquired slice, return it once we've copied the bytes into the caller's buffer.
-	select {
-	case tunBufFreelist <- p:
-	default:
-	}
-	return n, nil
-}
-
 func (t *TestTun) SupportsMultiqueue() bool {
 	return false
 }
 
-func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+func (t *TestTun) NewMultiQueueReader() (tio.Queue, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented")
 }
-
-func (t *TestTun) TunPrefixLen() int { return 0 }

overlay/tun_windows.go

@@ -6,8 +6,6 @@ package overlay
 import (
 	"crypto"
 	"fmt"
-	"io"
-	"log/slog"
 	"net/netip"
 	"os"
 	"path/filepath"
@@ -17,7 +15,9 @@ import (
 	"unsafe"
 
 	"github.com/gaissmai/bart"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	"github.com/slackhq/nebula/wintun"
@@ -33,16 +33,35 @@ type winTun struct {
 	MTU         int
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	l           *slog.Logger
+	l           *logrus.Logger
 
 	tun *wintun.NativeTun
+
+	readBuf  []byte
+	batchRet [1][]byte
 }
 
-func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (Device, error) {
+func (t *winTun) Read() ([][]byte, error) {
+	if t.readBuf == nil {
+		t.readBuf = make([]byte, defaultBatchBufSize)
+	}
+	n, err := t.tun.Read(t.readBuf, 0)
+	if err != nil {
+		return nil, err
+	}
+	t.batchRet[0] = t.readBuf[:n]
+	return t.batchRet[:], nil
+}
+
+func (t *winTun) WriteReject(p []byte) (int, error) {
+	return t.Write(p)
+}
+
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (Device, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in Windows")
 }
 
-func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*winTun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*winTun, error) {
 	err := checkWinTunExists()
 	if err != nil {
 		return nil, fmt.Errorf("can not load the wintun driver: %w", err)
@@ -71,7 +90,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
 	if err != nil {
 		// Windows 10 has an issue with unclean shutdowns not fully cleaning up the wintun device.
 		// Trying a second time resolves the issue.
-		l.Debug("Failed to create wintun device, retrying", "error", err)
+		l.WithError(err).Debug("Failed to create wintun device, retrying")
 		tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU)
 		if err != nil {
 			return nil, &NameError{
@@ -170,7 +189,7 @@ func (t *winTun) addRoutes(logErrors bool) error {
 				return retErr
 			}
 		} else {
-			t.l.Info("Added route", "route", r)
+			t.l.WithField("route", r).Info("Added route")
 		}
 
 		if !foundDefault4 {
@@ -208,9 +227,9 @@ func (t *winTun) removeRoutes(routes []Route) error {
 		// See comment on luid.AddRoute
 		err := luid.DeleteRoute(r.Cidr, r.Via[0].Addr())
 		if err != nil {
-			t.l.Error("Failed to remove route", "error", err, "route", r)
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
 		} else {
-			t.l.Info("Removed route", "route", r)
+			t.l.WithField("route", r).Info("Removed route")
 		}
 	}
 	return nil
@@ -229,10 +248,6 @@ func (t *winTun) Name() string {
 	return t.Device
 }
 
-func (t *winTun) Read(b []byte) (int, error) {
-	return t.tun.Read(b, 0)
-}
-
 func (t *winTun) Write(b []byte) (int, error) {
 	return t.tun.Write(b, 0)
 }
@@ -241,7 +256,7 @@ func (t *winTun) SupportsMultiqueue() bool {
 	return false
 }
 
-func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+func (t *winTun) NewMultiQueueReader() (tio.Queue, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
 }
 
@@ -296,5 +311,3 @@ func checkWinTunExists() error {
 	_, err = syscall.LoadDLL(filepath.Join(filepath.Dir(myPath), "dist", "windows", "wintun", "bin", arch, "wintun.dll"))
 	return err
 }
-
-func (t *winTun) TunPrefixLen() int { return 0 }

overlay/user.go

@@ -2,14 +2,15 @@ package overlay
 
 import (
 	"io"
-	"log/slog"
 	"net/netip"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
-func NewUserDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
+func NewUserDeviceFromConfig(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
 	return NewUserDevice(vpnNetworks)
 }
 
@@ -23,17 +24,34 @@ func NewUserDevice(vpnNetworks []netip.Prefix) (Device, error) {
 		outboundWriter: ow,
 		inboundReader:  ir,
 		inboundWriter:  iw,
+		numReaders:     1,
 	}, nil
 }
 
 type UserDevice struct {
 	vpnNetworks []netip.Prefix
+	numReaders  int
 
 	outboundReader *io.PipeReader
 	outboundWriter *io.PipeWriter
 
 	inboundReader *io.PipeReader
 	inboundWriter *io.PipeWriter
+
+	readBuf  []byte
+	batchRet [1][]byte
+}
+
+func (d *UserDevice) Read() ([][]byte, error) {
+	if d.readBuf == nil {
+		d.readBuf = make([]byte, defaultBatchBufSize)
+	}
+	n, err := d.outboundReader.Read(d.readBuf)
+	if err != nil {
+		return nil, err
+	}
+	d.batchRet[0] = d.readBuf[:n]
+	return d.batchRet[:], nil
 }
 
 func (d *UserDevice) Activate() error {
@@ -50,24 +68,31 @@ func (d *UserDevice) SupportsMultiqueue() bool {
 	return true
 }
 
-func (d *UserDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return d, nil
+func (d *UserDevice) NewMultiQueueReader() error {
+	d.numReaders++
+	return nil
+}
+
+func (d *UserDevice) Readers() []tio.Queue {
+	out := make([]tio.Queue, d.numReaders)
+	for i := range d.numReaders {
+		out[i] = d
+	}
+	return out
 }
 
 func (d *UserDevice) Pipe() (*io.PipeReader, *io.PipeWriter) {
 	return d.inboundReader, d.outboundWriter
 }
 
-func (d *UserDevice) Read(p []byte) (n int, err error) {
-	return d.outboundReader.Read(p)
-}
 func (d *UserDevice) Write(p []byte) (n int, err error) {
 	return d.inboundWriter.Write(p)
 }
+func (d *UserDevice) WriteReject(p []byte) (n int, err error) {
+	return d.Write(p)
+}
 func (d *UserDevice) Close() error {
 	d.inboundWriter.Close()
 	d.outboundWriter.Close()
 	return nil
 }
-
-func (d *UserDevice) TunPrefixLen() int { return 0 }

pki.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log/slog"
 	"net"
 	"net/netip"
 	"os"
@@ -15,27 +14,25 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/flynn/noise"
 	"github.com/gaissmai/bart"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/handshake"
-	"github.com/slackhq/nebula/noiseutil"
 	"github.com/slackhq/nebula/util"
 )
 
 type PKI struct {
 	cs     atomic.Pointer[CertState]
 	caPool atomic.Pointer[cert.CAPool]
-	l      *slog.Logger
+	l      *logrus.Logger
 }
 
 type CertState struct {
-	v1Cert       cert.Certificate
-	v1Credential *handshake.Credential
+	v1Cert           cert.Certificate
+	v1HandshakeBytes []byte
 
-	v2Cert       cert.Certificate
-	v2Credential *handshake.Credential
+	v2Cert           cert.Certificate
+	v2HandshakeBytes []byte
 
 	initiatingVersion cert.Version
 	privateKey        []byte
@@ -49,7 +46,7 @@ type CertState struct {
 	myVpnBroadcastAddrsTable *bart.Lite
 }
 
-func NewPKIFromConfig(l *slog.Logger, c *config.C) (*PKI, error) {
+func NewPKIFromConfig(l *logrus.Logger, c *config.C) (*PKI, error) {
 	pki := &PKI{l: l}
 	err := pki.reload(c, true)
 	if err != nil {
@@ -95,35 +92,13 @@ func (p *PKI) reload(c *config.C, initial bool) error {
 }
 
 func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
-	var cipher string
-	var currentState *CertState
-	if initial {
-		cipher = c.GetString("cipher", "aes")
-		//TODO: this sucks and we should make it not a global
-		switch cipher {
-		case "aes":
-			noiseEndianness = binary.BigEndian
-		case "chachapoly":
-			noiseEndianness = binary.LittleEndian
-		default:
-			return util.NewContextualError(
-				"unknown cipher",
-				m{"cipher": cipher},
-				nil,
-			)
-		}
-	} else {
-		// Cipher cant be hot swapped so just leave it at what it was before
-		currentState = p.cs.Load()
-		cipher = currentState.cipher
-	}
-
-	newState, err := newCertStateFromConfig(c, cipher)
+	newState, err := newCertStateFromConfig(c)
 	if err != nil {
 		return util.NewContextualError("Could not load client cert", nil, err)
 	}
 
-	if currentState != nil {
+	if !initial {
+		currentState := p.cs.Load()
 		if newState.v1Cert != nil {
 			if currentState.v1Cert == nil {
 				//adding certs is fine, actually. Networks-in-common confirmed in newCertState().
@@ -183,14 +158,33 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
 				)
 			}
 		}
+
+		// Cipher cant be hot swapped so just leave it at what it was before
+		newState.cipher = currentState.cipher
+
+	} else {
+		newState.cipher = c.GetString("cipher", "aes")
+		//TODO: this sucks and we should make it not a global
+		switch newState.cipher {
+		case "aes":
+			noiseEndianness = binary.BigEndian
+		case "chachapoly":
+			noiseEndianness = binary.LittleEndian
+		default:
+			return util.NewContextualError(
+				"unknown cipher",
+				m{"cipher": newState.cipher},
+				nil,
+			)
+		}
 	}
 
 	p.cs.Store(newState)
 
 	if initial {
-		p.l.Debug("Client nebula certificate(s)", "cert", newState)
+		p.l.WithField("cert", newState).Debug("Client nebula certificate(s)")
 	} else {
-		p.l.Info("Client certificate(s) refreshed from disk", "cert", newState)
+		p.l.WithField("cert", newState).Info("Client certificate(s) refreshed from disk")
 	}
 	return nil
 }
@@ -202,7 +196,7 @@ func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError {
 	}
 
 	p.caPool.Store(caPool)
-	p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints())
+	p.l.WithField("fingerprints", caPool.GetFingerprints()).Debug("Trusted CA fingerprints")
 	return nil
 }
 
@@ -214,20 +208,6 @@ func (cs *CertState) GetDefaultCertificate() cert.Certificate {
 	return c
 }
 
-// DefaultVersion returns the preferred cert version for initiating handshakes.
-func (cs *CertState) DefaultVersion() cert.Version { return cs.initiatingVersion }
-
-// GetCredential returns the pre-computed handshake credential for the given version, or nil.
-func (cs *CertState) GetCredential(v cert.Version) *handshake.Credential {
-	switch v {
-	case cert.Version1:
-		return cs.v1Credential
-	case cert.Version2:
-		return cs.v2Credential
-	}
-	return nil
-}
-
 func (cs *CertState) getCertificate(v cert.Version) cert.Certificate {
 	switch v {
 	case cert.Version1:
@@ -239,25 +219,17 @@ func (cs *CertState) getCertificate(v cert.Version) cert.Certificate {
 	return nil
 }
 
-func newCipherSuite(curve cert.Curve, pkcs11backed bool, cipher string) (noise.CipherSuite, error) {
-	var dhFunc noise.DHFunc
-	switch curve {
-	case cert.Curve_CURVE25519:
-		dhFunc = noise.DH25519
-	case cert.Curve_P256:
-		if pkcs11backed {
-			dhFunc = noiseutil.DHP256PKCS11
-		} else {
-			dhFunc = noiseutil.DHP256
-		}
+// getHandshakeBytes returns the cached bytes to be used in a handshake message for the requested version.
+// Callers must check if the return []byte is nil.
+func (cs *CertState) getHandshakeBytes(v cert.Version) []byte {
+	switch v {
+	case cert.Version1:
+		return cs.v1HandshakeBytes
+	case cert.Version2:
+		return cs.v2HandshakeBytes
 	default:
-		return nil, fmt.Errorf("unsupported curve: %s", curve)
+		return nil
 	}
-
-	if cipher == "chachapoly" {
-		return noise.NewCipherSuite(dhFunc, noise.CipherChaChaPoly, noise.HashSHA256), nil
-	}
-	return noise.NewCipherSuite(dhFunc, noiseutil.CipherAESGCM, noise.HashSHA256), nil
 }
 
 func (cs *CertState) String() string {
@@ -289,7 +261,7 @@ func (cs *CertState) MarshalJSON() ([]byte, error) {
 	return json.Marshal(msg)
 }
 
-func newCertStateFromConfig(c *config.C, cipher string) (*CertState, error) {
+func newCertStateFromConfig(c *config.C) (*CertState, error) {
 	var err error
 
 	privPathOrPEM := c.GetString("pki.key", "")
@@ -373,14 +345,13 @@ func newCertStateFromConfig(c *config.C, cipher string) (*CertState, error) {
 		return nil, fmt.Errorf("unknown pki.initiating_version: %v", rawInitiatingVersion)
 	}
 
-	return newCertState(initiatingVersion, v1, v2, isPkcs11, curve, rawKey, cipher)
+	return newCertState(initiatingVersion, v1, v2, isPkcs11, curve, rawKey)
 }
 
-func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, privateKeyCurve cert.Curve, privateKey []byte, cipher string) (*CertState, error) {
+func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, privateKeyCurve cert.Curve, privateKey []byte) (*CertState, error) {
 	cs := CertState{
 		privateKey:               privateKey,
 		pkcs11Backed:             pkcs11backed,
-		cipher:                   cipher,
 		myVpnNetworksTable:       new(bart.Lite),
 		myVpnAddrsTable:          new(bart.Lite),
 		myVpnBroadcastAddrsTable: new(bart.Lite),
@@ -413,14 +384,10 @@ func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, p
 
 		v1hs, err := v1.MarshalForHandshakes()
 		if err != nil {
-			return nil, fmt.Errorf("error marshalling v1 certificate for handshake: %w", err)
-		}
-		ncs, err := newCipherSuite(v1.Curve(), pkcs11backed, cipher)
-		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("error marshalling certificate for handshake: %w", err)
 		}
 		cs.v1Cert = v1
-		cs.v1Credential = handshake.NewCredential(v1, v1hs, privateKey, ncs)
+		cs.v1HandshakeBytes = v1hs
 
 		if cs.initiatingVersion == 0 {
 			cs.initiatingVersion = cert.Version1
@@ -438,14 +405,10 @@ func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, p
 
 		v2hs, err := v2.MarshalForHandshakes()
 		if err != nil {
-			return nil, fmt.Errorf("error marshalling v2 certificate for handshake: %w", err)
-		}
-		ncs, err := newCipherSuite(v2.Curve(), pkcs11backed, cipher)
-		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("error marshalling certificate for handshake: %w", err)
 		}
 		cs.v2Cert = v2
-		cs.v2Credential = handshake.NewCredential(v2, v2hs, privateKey, ncs)
+		cs.v2HandshakeBytes = v2hs
 
 		if cs.initiatingVersion == 0 {
 			cs.initiatingVersion = cert.Version2
@@ -524,7 +487,7 @@ func loadCertificate(b []byte) (cert.Certificate, []byte, error) {
 	return c, b, nil
 }
 
-func loadCAPoolFromConfig(l *slog.Logger, c *config.C) (*cert.CAPool, error) {
+func loadCAPoolFromConfig(l *logrus.Logger, c *config.C) (*cert.CAPool, error) {
 	caPathOrPEM := c.GetString("pki.ca", "")
 	if caPathOrPEM == "" {
 		return nil, errors.New("no pki.ca path or PEM data provided")
@@ -549,7 +512,7 @@ func loadCAPoolFromConfig(l *slog.Logger, c *config.C) (*cert.CAPool, error) {
 		for _, crt := range caPool.CAs {
 			if crt.Certificate.Expired(time.Now()) {
 				expired++
-				l.Warn("expired certificate present in CA pool", "cert", crt)
+				l.WithField("cert", crt).Warn("expired certificate present in CA pool")
 			}
 		}
 
@@ -567,7 +530,7 @@ func loadCAPoolFromConfig(l *slog.Logger, c *config.C) (*cert.CAPool, error) {
 			caPool.BlocklistFingerprint(fp)
 		}
 
-		l.Info("Blocklisted certificates", "fingerprintCount", len(bl))
+		l.WithField("fingerprintCount", len(bl)).Info("Blocklisted certificates")
 	}
 
 	return caPool, nil

pki_hup_benchmark_test.go

@@ -41,7 +41,7 @@ func BenchmarkReloadConfigWithCAs(b *testing.B) {
 			c := config.NewC(l)
 			require.NoError(b, c.Load(dir))
 
-			_, err := NewPKIFromConfig(test.NewLogger(), c)
+			_, err := NewPKIFromConfig(l, c)
 			require.NoError(b, err)
 
 			b.ReportAllocs()

punchy.go

@@ -1,10 +1,10 @@
 package nebula
 
 import (
-	"log/slog"
 	"sync/atomic"
 	"time"
 
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 )
 
@@ -14,10 +14,10 @@ type Punchy struct {
 	delay           atomic.Int64
 	respondDelay    atomic.Int64
 	punchEverything atomic.Bool
-	l               *slog.Logger
+	l               *logrus.Logger
 }
 
-func NewPunchyFromConfig(l *slog.Logger, c *config.C) *Punchy {
+func NewPunchyFromConfig(l *logrus.Logger, c *config.C) *Punchy {
 	p := &Punchy{l: l}
 
 	p.reload(c, true)
@@ -62,7 +62,7 @@ func (p *Punchy) reload(c *config.C, initial bool) {
 		p.respond.Store(yes)
 
 		if !initial {
-			p.l.Info("punchy.respond changed", "respond", p.GetRespond())
+			p.l.Infof("punchy.respond changed to %v", p.GetRespond())
 		}
 	}
 
@@ -70,21 +70,21 @@ func (p *Punchy) reload(c *config.C, initial bool) {
 	if initial || c.HasChanged("punchy.delay") {
 		p.delay.Store((int64)(c.GetDuration("punchy.delay", time.Second)))
 		if !initial {
-			p.l.Info("punchy.delay changed", "delay", p.GetDelay())
+			p.l.Infof("punchy.delay changed to %s", p.GetDelay())
 		}
 	}
 
 	if initial || c.HasChanged("punchy.target_all_remotes") {
 		p.punchEverything.Store(c.GetBool("punchy.target_all_remotes", false))
 		if !initial {
-			p.l.Info("punchy.target_all_remotes changed", "target_all_remotes", p.GetTargetEverything())
+			p.l.WithField("target_all_remotes", p.GetTargetEverything()).Info("punchy.target_all_remotes changed")
 		}
 	}
 
 	if initial || c.HasChanged("punchy.respond_delay") {
 		p.respondDelay.Store((int64)(c.GetDuration("punchy.respond_delay", 5*time.Second)))
 		if !initial {
-			p.l.Info("punchy.respond_delay changed", "respond_delay", p.GetRespondDelay())
+			p.l.Infof("punchy.respond_delay changed to %s", p.GetRespondDelay())
 		}
 	}
 }

punchy_test.go

@@ -1,8 +1,6 @@
 package nebula
 
 import (
-	"context"
-	"log/slog"
 	"testing"
 	"time"
 
@@ -17,7 +15,7 @@ func TestNewPunchyFromConfig(t *testing.T) {
 	c := config.NewC(l)
 
 	// Test defaults
-	p := NewPunchyFromConfig(test.NewLogger(), c)
+	p := NewPunchyFromConfig(l, c)
 	assert.False(t, p.GetPunch())
 	assert.False(t, p.GetRespond())
 	assert.Equal(t, time.Second, p.GetDelay())
@@ -25,33 +23,33 @@ func TestNewPunchyFromConfig(t *testing.T) {
 
 	// punchy deprecation
 	c.Settings["punchy"] = true
-	p = NewPunchyFromConfig(test.NewLogger(), c)
+	p = NewPunchyFromConfig(l, c)
 	assert.True(t, p.GetPunch())
 
 	// punchy.punch
 	c.Settings["punchy"] = map[string]any{"punch": true}
-	p = NewPunchyFromConfig(test.NewLogger(), c)
+	p = NewPunchyFromConfig(l, c)
 	assert.True(t, p.GetPunch())
 
 	// punch_back deprecation
 	c.Settings["punch_back"] = true
-	p = NewPunchyFromConfig(test.NewLogger(), c)
+	p = NewPunchyFromConfig(l, c)
 	assert.True(t, p.GetRespond())
 
 	// punchy.respond
 	c.Settings["punchy"] = map[string]any{"respond": true}
 	c.Settings["punch_back"] = false
-	p = NewPunchyFromConfig(test.NewLogger(), c)
+	p = NewPunchyFromConfig(l, c)
 	assert.True(t, p.GetRespond())
 
 	// punchy.delay
 	c.Settings["punchy"] = map[string]any{"delay": "1m"}
-	p = NewPunchyFromConfig(test.NewLogger(), c)
+	p = NewPunchyFromConfig(l, c)
 	assert.Equal(t, time.Minute, p.GetDelay())
 
 	// punchy.respond_delay
 	c.Settings["punchy"] = map[string]any{"respond_delay": "1m"}
-	p = NewPunchyFromConfig(test.NewLogger(), c)
+	p = NewPunchyFromConfig(l, c)
 	assert.Equal(t, time.Minute, p.GetRespondDelay())
 }
 
@@ -64,7 +62,7 @@ punchy:
   delay: 1m
   respond: false
 `))
-	p := NewPunchyFromConfig(test.NewLogger(), c)
+	p := NewPunchyFromConfig(l, c)
 	assert.Equal(t, delay, p.GetDelay())
 	assert.False(t, p.GetRespond())
 
@@ -78,158 +76,3 @@ punchy:
 	assert.Equal(t, newDelay, p.GetDelay())
 	assert.True(t, p.GetRespond())
 }
-
-// The tests below pin the shape of each log line Punchy produces so changes
-// cannot silently break whatever operators are grepping for. The assertions
-// are on the structured message + attrs (e.g. "punchy.respond changed" with
-// a respond=true field) rather than a formatted string.
-//
-// Punchy.reload also emits a spurious "Changing punchy.punch with reload is
-// not supported" warning whenever any key under punchy changes, because of
-// the c.HasChanged("punchy") fallback kept for the deprecated top-level
-// punchy form. The tests filter by message rather than asserting total
-// entry counts so that warning is tolerated without being locked into
-// the format.
-
-type capturedEntry struct {
-	Level slog.Level
-	Msg   string
-	Attrs map[string]any
-}
-
-// capturingHandler is a slog.Handler that records each Record it receives so
-// tests can assert on the level, message, and attribute map of individual log
-// lines without coupling to any specific text format.
-type capturingHandler struct {
-	entries []capturedEntry
-}
-
-func (h *capturingHandler) Enabled(_ context.Context, _ slog.Level) bool { return true }
-
-func (h *capturingHandler) Handle(_ context.Context, r slog.Record) error {
-	e := capturedEntry{
-		Level: r.Level,
-		Msg:   r.Message,
-		Attrs: make(map[string]any),
-	}
-	r.Attrs(func(a slog.Attr) bool {
-		e.Attrs[a.Key] = a.Value.Resolve().Any()
-		return true
-	})
-	h.entries = append(h.entries, e)
-	return nil
-}
-
-func (h *capturingHandler) WithAttrs(_ []slog.Attr) slog.Handler { return h }
-func (h *capturingHandler) WithGroup(_ string) slog.Handler      { return h }
-
-func newCapturingPunchyLogger(t *testing.T) (*slog.Logger, *capturingHandler) {
-	t.Helper()
-	hook := &capturingHandler{}
-	return slog.New(hook), hook
-}
-
-func findEntry(t *testing.T, entries []capturedEntry, msg string) capturedEntry {
-	t.Helper()
-	for _, e := range entries {
-		if e.Msg == msg {
-			return e
-		}
-	}
-	t.Fatalf("no entry with message %q among %d entries", msg, len(entries))
-	return capturedEntry{}
-}
-
-func TestPunchy_LogFormat_InitialEnabled(t *testing.T) {
-	l, hook := newCapturingPunchyLogger(t)
-	c := config.NewC(test.NewLogger())
-	require.NoError(t, c.LoadString(`punchy: {punch: true}`))
-
-	NewPunchyFromConfig(l, c)
-
-	entry := findEntry(t, hook.entries, "punchy enabled")
-	assert.Equal(t, slog.LevelInfo, entry.Level)
-	assert.Empty(t, entry.Attrs)
-}
-
-func TestPunchy_LogFormat_InitialDisabled(t *testing.T) {
-	l, hook := newCapturingPunchyLogger(t)
-	c := config.NewC(test.NewLogger())
-	require.NoError(t, c.LoadString(`punchy: {punch: false}`))
-
-	NewPunchyFromConfig(l, c)
-
-	entry := findEntry(t, hook.entries, "punchy disabled")
-	assert.Equal(t, slog.LevelInfo, entry.Level)
-	assert.Empty(t, entry.Attrs)
-}
-
-func TestPunchy_LogFormat_ReloadPunchUnsupported(t *testing.T) {
-	l, hook := newCapturingPunchyLogger(t)
-	c := config.NewC(test.NewLogger())
-	require.NoError(t, c.LoadString(`punchy: {punch: false}`))
-	NewPunchyFromConfig(l, c)
-	hook.entries = nil
-
-	require.NoError(t, c.ReloadConfigString(`punchy: {punch: true}`))
-
-	entry := findEntry(t, hook.entries, "Changing punchy.punch with reload is not supported, ignoring.")
-	assert.Equal(t, slog.LevelWarn, entry.Level)
-	assert.Empty(t, entry.Attrs)
-}
-
-func TestPunchy_LogFormat_ReloadRespond(t *testing.T) {
-	l, hook := newCapturingPunchyLogger(t)
-	c := config.NewC(test.NewLogger())
-	require.NoError(t, c.LoadString(`punchy: {respond: false}`))
-	NewPunchyFromConfig(l, c)
-	hook.entries = nil
-
-	require.NoError(t, c.ReloadConfigString(`punchy: {respond: true}`))
-
-	entry := findEntry(t, hook.entries, "punchy.respond changed")
-	assert.Equal(t, slog.LevelInfo, entry.Level)
-	assert.Equal(t, map[string]any{"respond": true}, entry.Attrs)
-}
-
-func TestPunchy_LogFormat_ReloadDelay(t *testing.T) {
-	l, hook := newCapturingPunchyLogger(t)
-	c := config.NewC(test.NewLogger())
-	require.NoError(t, c.LoadString(`punchy: {delay: 1s}`))
-	NewPunchyFromConfig(l, c)
-	hook.entries = nil
-
-	require.NoError(t, c.ReloadConfigString(`punchy: {delay: 10s}`))
-
-	entry := findEntry(t, hook.entries, "punchy.delay changed")
-	assert.Equal(t, slog.LevelInfo, entry.Level)
-	assert.Equal(t, map[string]any{"delay": 10 * time.Second}, entry.Attrs)
-}
-
-func TestPunchy_LogFormat_ReloadTargetAllRemotes(t *testing.T) {
-	l, hook := newCapturingPunchyLogger(t)
-	c := config.NewC(test.NewLogger())
-	require.NoError(t, c.LoadString(`punchy: {target_all_remotes: false}`))
-	NewPunchyFromConfig(l, c)
-	hook.entries = nil
-
-	require.NoError(t, c.ReloadConfigString(`punchy: {target_all_remotes: true}`))
-
-	entry := findEntry(t, hook.entries, "punchy.target_all_remotes changed")
-	assert.Equal(t, slog.LevelInfo, entry.Level)
-	assert.Equal(t, map[string]any{"target_all_remotes": true}, entry.Attrs)
-}
-
-func TestPunchy_LogFormat_ReloadRespondDelay(t *testing.T) {
-	l, hook := newCapturingPunchyLogger(t)
-	c := config.NewC(test.NewLogger())
-	require.NoError(t, c.LoadString(`punchy: {respond_delay: 5s}`))
-	NewPunchyFromConfig(l, c)
-	hook.entries = nil
-
-	require.NoError(t, c.ReloadConfigString(`punchy: {respond_delay: 15s}`))
-
-	entry := findEntry(t, hook.entries, "punchy.respond_delay changed")
-	assert.Equal(t, slog.LevelInfo, entry.Level)
-	assert.Equal(t, map[string]any{"respond_delay": 15 * time.Second}, entry.Attrs)
-}