update synctrace

* add netlink options

* force use buffer

* fix namings and add config examples

* fix linter

.github/ISSUE_TEMPLATE/config.yml

@@ -1,13 +1,21 @@
 blank_issues_enabled: true
 contact_links:
+  - name: 💨 Performance Issues
+    url: https://github.com/slackhq/nebula/discussions/new/choose
+    about: 'We ask that you create a discussion instead of an issue for performance-related questions. This allows us to have a more open conversation about the issue and helps us to better understand the problem.'
+
+  - name: 📄 Documentation Issues
+    url: https://github.com/definednet/nebula-docs
+    about: "If you've found an issue with the website documentation, please file it in the nebula-docs repository."
+
+  - name: 📱 Mobile Nebula Issues
+    url: https://github.com/definednet/mobile_nebula
+    about: "If you're using the mobile Nebula app and have found an issue, please file it in the mobile_nebula repository."
+
   - name: 📘 Documentation
     url: https://nebula.defined.net/docs/
-    about: Review documentation.
+    about: 'The documentation is the best place to start if you are new to Nebula.'
 
   - name: 💁 Support/Chat
-    url: https://join.slack.com/t/nebulaoss/shared_invite/enQtOTA5MDI4NDg3MTg4LTkwY2EwNTI4NzQyMzc0M2ZlODBjNWI3NTY1MzhiOThiMmZlZjVkMTI0NGY4YTMyNjUwMWEyNzNkZTJmYzQxOGU
-    about: 'This issue tracker is not for support questions. Join us on Slack for assistance!'
-
-  - name: 📱 Mobile Nebula
-    url: https://github.com/definednet/mobile_nebula
-    about: 'This issue tracker is not for mobile support. Try the Mobile Nebula repo instead!'
+    url: https://join.slack.com/t/nebulaoss/shared_invite/zt-2xqe6e7vn-k_KGi8s13nsr7cvHVvHvuQ
+    about: 'For faster support, join us on Slack for assistance!'

.github/pull_request_template.md

@@ -0,0 +1,11 @@
+<!--
+Thank you for taking the time to submit a pull request!
+
+Please be sure to provide a clear description of what you're trying to achieve with the change.
+
+- If you're submitting a new feature, please explain how to use it and document any new config options in the example config.
+- If you're submitting a bugfix, please link the related issue or describe the circumstances surrounding the issue.
+- If you're changing a default, explain why you believe the new default is appropriate for most users.
+
+P.S. If you're only updating the README or other docs, please file a pull request here instead: https://github.com/DefinedNet/nebula-docs
+-->

Makefile

@@ -61,7 +61,7 @@ ALL = $(ALL_LINUX) \
 	windows-arm64
 
 e2e:
-	$(TEST_ENV) go test -tags=e2e_testing -count=1 $(TEST_FLAGS) ./e2e
+	$(TEST_ENV) go test -tags=synctrace,e2e_testing -count=1 $(TEST_FLAGS) ./e2e
 
 e2ev: TEST_FLAGS += -v
 e2ev: e2e
@@ -215,6 +215,7 @@ ifeq ($(words $(MAKECMDGOALS)),1)
 	@$(MAKE) service ${.DEFAULT_GOAL} --no-print-directory
 endif
 
+bin-docker: BUILD_ARGS = -tags=synctrace
 bin-docker: bin build/linux-amd64/nebula build/linux-amd64/nebula-cert
 
 smoke-docker: bin-docker

README.md

@@ -4,7 +4,7 @@ It lets you seamlessly connect computers anywhere in the world. Nebula is portab
 It can be used to connect a small number of computers, but is also able to connect tens of thousands of computers.
 
 Nebula incorporates a number of existing concepts like encryption, security groups, certificates,
-and tunneling, and each of those individual pieces existed before Nebula in various forms.
+and tunneling.
 What makes Nebula different to existing offerings is that it brings all of these ideas together,
 resulting in a sum that is greater than its individual parts.
 
@@ -28,33 +28,33 @@ Check the [releases](https://github.com/slackhq/nebula/releases/latest) page for
 #### Distribution Packages
 
 - [Arch Linux](https://archlinux.org/packages/extra/x86_64/nebula/)
-    ```
-    $ sudo pacman -S nebula
+    ```sh
+    sudo pacman -S nebula
     ```
 
 - [Fedora Linux](https://src.fedoraproject.org/rpms/nebula)
-    ```
-    $ sudo dnf install nebula
+    ```sh
+    sudo dnf install nebula
     ```
 
 - [Debian Linux](https://packages.debian.org/source/stable/nebula)
-    ```
-    $ sudo apt install nebula
+    ```sh
+    sudo apt install nebula
     ```
 
 - [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=nebula)
-    ```
-    $ sudo apk add nebula
+    ```sh
+    sudo apk add nebula
     ```
 
 - [macOS Homebrew](https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/n/nebula.rb)
-    ```
-    $ brew install nebula
+    ```sh
+    brew install nebula
     ```
 
 - [Docker](https://hub.docker.com/r/nebulaoss/nebula)
-    ```
-    $ docker pull nebulaoss/nebula
+    ```sh
+    docker pull nebulaoss/nebula
     ```
 
 #### Mobile
@@ -64,10 +64,10 @@ Check the [releases](https://github.com/slackhq/nebula/releases/latest) page for
 
 ## Technical Overview
 
-Nebula is a mutually authenticated peer-to-peer software defined network based on the [Noise Protocol Framework](https://noiseprotocol.org/).
+Nebula is a mutually authenticated peer-to-peer software-defined network based on the [Noise Protocol Framework](https://noiseprotocol.org/).
 Nebula uses certificates to assert a node's IP address, name, and membership within user-defined groups.
 Nebula's user-defined groups allow for provider agnostic traffic filtering between nodes.
-Discovery nodes allow individual peers to find each other and optionally use UDP hole punching to establish connections from behind most firewalls or NATs.
+Discovery nodes (aka lighthouses) allow individual peers to find each other and optionally use UDP hole punching to establish connections from behind most firewalls or NATs.
 Users can move data between nodes in any number of cloud service providers, datacenters, and endpoints, without needing to maintain a particular addressing scheme.
 
 Nebula uses Elliptic-curve Diffie-Hellman (`ECDH`) key exchange and `AES-256-GCM` in its default configuration.
@@ -82,28 +82,34 @@ To set up a Nebula network, you'll need:
 
 #### 2. (Optional, but you really should..) At least one discovery node with a routable IP address, which we call a lighthouse.
 
-Nebula lighthouses allow nodes to find each other, anywhere in the world. A lighthouse is the only node in a Nebula network whose IP should not change. Running a lighthouse requires very few compute resources, and you can easily use the least expensive option from a cloud hosting provider. If you're not sure which provider to use, a number of us have used $5/mo [DigitalOcean](https://digitalocean.com) droplets as lighthouses.
+Nebula lighthouses allow nodes to find each other, anywhere in the world. A lighthouse is the only node in a Nebula network whose IP should not change. Running a lighthouse requires very few compute resources, and you can easily use the least expensive option from a cloud hosting provider. If you're not sure which provider to use, a number of us have used $6/mo [DigitalOcean](https://digitalocean.com) droplets as lighthouses.
 
 Once you have launched an instance, ensure that Nebula udp traffic (default port udp/4242) can reach it over the internet.
 
-
 #### 3. A Nebula certificate authority, which will be the root of trust for a particular Nebula network.
 
-  ```
+```sh
 ./nebula-cert ca -name "Myorganization, Inc"
 ```
+
 This will create files named `ca.key` and `ca.cert` in the current directory. The `ca.key` file is the most sensitive file you'll create, because it is the key used to sign the certificates for individual nebula nodes/hosts. Please store this file somewhere safe, preferably with strong encryption.
 
+**Be aware!** By default, certificate authorities have a 1-year lifetime before expiration. See [this guide](https://nebula.defined.net/docs/guides/rotating-certificate-authority/) for details on rotating a CA.
+
 #### 4. Nebula host keys and certificates generated from that certificate authority
+
 This assumes you have four nodes, named lighthouse1, laptop, server1, host3. You can name the nodes any way you'd like, including FQDN. You'll also need to choose IP addresses and the associated subnet. In this example, we are creating a nebula network that will use 192.168.100.x/24 as its network range. This example also demonstrates nebula groups, which can later be used to define traffic rules in a nebula network.
-```
+```sh
 ./nebula-cert sign -name "lighthouse1" -ip "192.168.100.1/24"
 ./nebula-cert sign -name "laptop" -ip "192.168.100.2/24" -groups "laptop,home,ssh"
 ./nebula-cert sign -name "server1" -ip "192.168.100.9/24" -groups "servers"
 ./nebula-cert sign -name "host3" -ip "192.168.100.10/24"
 ```
 
+By default, host certificates will expire 1 second before the CA expires. Use the `-duration` flag to specify a shorter lifetime.
+
 #### 5. Configuration files for each host
+
 Download a copy of the nebula [example configuration](https://github.com/slackhq/nebula/blob/master/examples/config.yml).
 
 * On the lighthouse node, you'll need to ensure `am_lighthouse: true` is set.
@@ -118,10 +124,13 @@ For each host, copy the nebula binary to the host, along with `config.yml` from
 **DO NOT COPY `ca.key` TO INDIVIDUAL NODES.**
 
 #### 7. Run nebula on each host
-```
+
+```sh
 ./nebula -config /path/to/config.yml
 ```
 
+For more detailed instructions, [find the full documentation here](https://nebula.defined.net/docs/).
+
 ## Building Nebula from source
 
 Make sure you have [go](https://go.dev/doc/install) installed and clone this repo. Change to the nebula directory.
@@ -140,8 +149,10 @@ The default curve used for cryptographic handshakes and signatures is Curve25519
 
 In addition, Nebula can be built using the [BoringCrypto GOEXPERIMENT](https://github.com/golang/go/blob/go1.20/src/crypto/internal/boring/README.md) by running either of the following make targets:
 
+```sh
 make bin-boringcrypto
 make release-boringcrypto
+```
 
 This is not the recommended default deployment, but may be useful based on your compliance requirements.
 
@@ -149,5 +160,3 @@ This is not the recommended default deployment, but may be useful based on your
 
 Nebula was created at Slack Technologies, Inc by Nate Brown and Ryan Huber, with contributions from Oliver Fross, Alan Lam, Wade Simmons, and Lining Wang.
 
-
-

cert/crypto_test.go

@@ -26,21 +26,21 @@ func TestNewArgon2Parameters(t *testing.T) {
 }
 
 func TestDecryptAndUnmarshalSigningPrivateKey(t *testing.T) {
-	passphrase := []byte("DO NOT USE THIS KEY")
+	passphrase := []byte("DO NOT USE")
 	privKey := []byte(`# A good key
 -----BEGIN NEBULA ED25519 ENCRYPTED PRIVATE KEY-----
-CjwKC0FFUy0yNTYtR0NNEi0IExCAgIABGAEgBCognnjujd67Vsv99p22wfAjQaDT
-oCMW1mdjkU3gACKNW4MSXOWR9Sts4C81yk1RUku2gvGKs3TB9LYoklLsIizSYOLl
-+Vs//O1T0I1Xbml2XBAROsb/VSoDln/6LMqR4B6fn6B3GOsLBBqRI8daDl9lRMPB
-qrlJ69wer3ZUHFXA
+CjsKC0FFUy0yNTYtR0NNEiwIExCAgAQYAyAEKiCPoDfGQiosxNPTbPn5EsMlc2MI
+c0Bt4oz6gTrFQhX3aBJcimhHKeAuhyTGvllD0Z19fe+DFPcLH3h5VrdjVfIAajg0
+KrbV3n9UHif/Au5skWmquNJzoW1E4MTdRbvpti6o+WdQ49DxjBFhx0YH8LBqrbPU
+0BGkUHmIO7daP24=
 -----END NEBULA ED25519 ENCRYPTED PRIVATE KEY-----
 `)
 	shortKey := []byte(`# A key which, once decrypted, is too short
 -----BEGIN NEBULA ED25519 ENCRYPTED PRIVATE KEY-----
-CjwKC0FFUy0yNTYtR0NNEi0IExCAgIABGAEgBCoga5h8owMEBWRSMMJKzuUvWce7
-k0qlBkQmCxiuLh80MuASW70YcKt8jeEIS2axo2V6zAKA9TSMcCsJW1kDDXEtL/xe
-GLF5T7sDl5COp4LU3pGxpV+KoeQ/S3gQCAAcnaOtnJQX+aSDnbO3jCHyP7U9CHbs
-rQr3bdH3Oy/WiYU=
+CjsKC0FFUy0yNTYtR0NNEiwIExCAgAQYAyAEKiAVJwdfl3r+eqi/vF6S7OMdpjfo
+hAzmTCRnr58Su4AqmBJbCv3zleYCEKYJP6UI3S8ekLMGISsgO4hm5leukCCyqT0Z
+cQ76yrberpzkJKoPLGisX8f+xdy4aXSZl7oEYWQte1+vqbtl/eY9PGZhxUQdcyq7
+hqzIyrRqfUgVuA==
 -----END NEBULA ED25519 ENCRYPTED PRIVATE KEY-----
 `)
 	invalidBanner := []byte(`# Invalid banner (not encrypted)

config/config.go

@@ -11,12 +11,12 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
 	"dario.cat/mergo"
 	"github.com/sirupsen/logrus"
+	"github.com/wadey/synctrace"
 	"gopkg.in/yaml.v3"
 )
 
@@ -27,13 +27,14 @@ type C struct {
 	oldSettings map[string]any
 	callbacks   []func(*C)
 	l           *logrus.Logger
-	reloadLock  sync.Mutex
+	reloadLock  synctrace.Mutex
 }
 
 func NewC(l *logrus.Logger) *C {
 	return &C{
 		Settings:   make(map[string]any),
 		l:          l,
+		reloadLock: synctrace.NewMutex("config-reload"),
 	}
 }
 

connection_manager.go

@@ -5,13 +5,13 @@ import (
 	"context"
 	"encoding/binary"
 	"net/netip"
-	"sync"
 	"time"
 
 	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/header"
+	"github.com/wadey/synctrace"
 )
 
 type trafficDecision int
@@ -28,14 +28,14 @@ const (
 
 type connectionManager struct {
 	in     map[uint32]struct{}
-	inLock *sync.RWMutex
+	inLock synctrace.RWMutex
 
 	out     map[uint32]struct{}
-	outLock *sync.RWMutex
+	outLock synctrace.RWMutex
 
 	// relayUsed holds which relay localIndexs are in use
 	relayUsed     map[uint32]struct{}
-	relayUsedLock *sync.RWMutex
+	relayUsedLock synctrace.RWMutex
 
 	hostMap                 *HostMap
 	trafficTimer            *LockingTimerWheel[uint32]
@@ -60,12 +60,12 @@ func newConnectionManager(ctx context.Context, l *logrus.Logger, intf *Interface
 	nc := &connectionManager{
 		hostMap:                 intf.hostMap,
 		in:                      make(map[uint32]struct{}),
-		inLock:                  &sync.RWMutex{},
+		inLock:                  synctrace.NewRWMutex("connection-manager-in"),
 		out:                     make(map[uint32]struct{}),
-		outLock:                 &sync.RWMutex{},
+		outLock:                 synctrace.NewRWMutex("connection-manager-out"),
 		relayUsed:               make(map[uint32]struct{}),
-		relayUsedLock:           &sync.RWMutex{},
-		trafficTimer:            NewLockingTimerWheel[uint32](time.Millisecond*500, max),
+		relayUsedLock:           synctrace.NewRWMutex("connection-manager-relay-used"),
+		trafficTimer:            NewLockingTimerWheel[uint32]("traffic-timer", time.Millisecond*500, max),
 		intf:                    intf,
 		pendingDeletion:         make(map[uint32]struct{}),
 		checkInterval:           checkInterval,

connection_state.go

@@ -4,13 +4,13 @@ import (
 	"crypto/rand"
 	"encoding/json"
 	"fmt"
-	"sync"
 	"sync/atomic"
 
 	"github.com/flynn/noise"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/noiseutil"
+	"github.com/wadey/synctrace"
 )
 
 const ReplayWindow = 1024
@@ -24,7 +24,7 @@ type ConnectionState struct {
 	initiator      bool
 	messageCounter atomic.Uint64
 	window         *Bits
-	writeLock      sync.Mutex
+	writeLock      synctrace.Mutex
 }
 
 func NewConnectionState(l *logrus.Logger, cs *CertState, crt cert.Certificate, initiator bool, pattern noise.HandshakePattern) (*ConnectionState, error) {
@@ -76,6 +76,7 @@ func NewConnectionState(l *logrus.Logger, cs *CertState, crt cert.Certificate, i
 		initiator: initiator,
 		window:    b,
 		myCert:    crt,
+		writeLock: synctrace.NewMutex("connection-state"),
 	}
 	// always start the counter from 2, as packet 1 and packet 2 are handshake packets.
 	ci.messageCounter.Add(2)

dns_server.go

@@ -6,12 +6,12 @@ import (
 	"net/netip"
 	"strconv"
 	"strings"
-	"sync"
 
 	"github.com/gaissmai/bart"
 	"github.com/miekg/dns"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/wadey/synctrace"
 )
 
 // This whole thing should be rewritten to use context
@@ -21,7 +21,7 @@ var dnsServer *dns.Server
 var dnsAddr string
 
 type dnsRecords struct {
-	sync.RWMutex
+	synctrace.RWMutex
 	l               *logrus.Logger
 	dnsMap4         map[string]netip.Addr
 	dnsMap6         map[string]netip.Addr
@@ -31,6 +31,7 @@ type dnsRecords struct {
 
 func newDnsRecords(l *logrus.Logger, cs *CertState, hostMap *HostMap) *dnsRecords {
 	return &dnsRecords{
+		RWMutex:         synctrace.NewRWMutex("dns-records"),
 		l:               l,
 		dnsMap4:         make(map[string]netip.Addr),
 		dnsMap6:         make(map[string]netip.Addr),

examples/config.yml

@@ -275,6 +275,10 @@ tun:
   # On linux only, set to true to manage unsafe routes directly on the system route table with gateway routes instead of
   # in nebula configuration files. Default false, not reloadable.
   #use_system_route_table: false
+  # Buffer size for reading routes updates. 0 means default system buffer size. (/proc/sys/net/core/rmem_default).
+  # If using massive routes updates, for example BGP, you may need to increase this value to avoid packet loss.
+  # SO_RCVBUFFORCE is used to avoid having to raise the system wide max
+  #use_system_route_table_buffer_size: 0
 
 # Configure logging level
 logging:

examples/go_service/main.go

@@ -5,8 +5,12 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"os"
 
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/service"
 )
 
@@ -59,7 +63,16 @@ pki:
 	if err := cfg.LoadString(configStr); err != nil {
 		return err
 	}
-	svc, err := service.New(&cfg)
+
+	logger := logrus.New()
+	logger.Out = os.Stdout
+
+	ctrl, err := nebula.Main(&cfg, false, "custom-app", logger, overlay.NewUserDeviceFromConfig)
+	if err != nil {
+		return err
+	}
+
+	svc, err := service.New(ctrl)
 	if err != nil {
 		return err
 	}

firewall.go

@@ -10,7 +10,6 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/gaissmai/bart"
@@ -19,6 +18,7 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
+	"github.com/wadey/synctrace"
 )
 
 type FirewallInterface interface {
@@ -76,7 +76,7 @@ type firewallMetrics struct {
 }
 
 type FirewallConntrack struct {
-	sync.Mutex
+	synctrace.Mutex
 
 	Conns      map[firewall.Packet]*conn
 	TimerWheel *TimerWheel[firewall.Packet]
@@ -164,6 +164,7 @@ func NewFirewall(l *logrus.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.D
 
 	return &Firewall{
 		Conntrack: &FirewallConntrack{
+			Mutex:      synctrace.NewMutex("firewall-conntrack"),
 			Conns:      make(map[firewall.Packet]*conn),
 			TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax),
 		},

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
 	github.com/stretchr/testify v1.10.0
 	github.com/vishvananda/netlink v1.3.0
+	github.com/wadey/synctrace v0.0.0-20250612192159-94547ef50dfe
 	golang.org/x/crypto v0.37.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
 	golang.org/x/net v0.39.0
@@ -42,12 +43,16 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/google/btree v1.1.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/heimdalr/dag v1.4.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/timandy/routine v1.1.5 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
 	golang.org/x/mod v0.23.0 // indirect
 	golang.org/x/time v0.5.0 // indirect

go.sum

@@ -22,6 +22,8 @@ github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432/go.
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
 github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
 github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U=
@@ -33,6 +35,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
+github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -58,6 +62,10 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/heimdalr/dag v1.4.0 h1:zG3JA4RDVLc55k3AXAgfwa+EgBNZ0TkfOO3C29Ucpmg=
+github.com/heimdalr/dag v1.4.0/go.mod h1:OCh6ghKmU0hPjtwMqWBoNxPmtRioKd1xSu7Zs4sbIqM=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -145,10 +153,14 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/timandy/routine v1.1.5 h1:LSpm7Iijwb9imIPlucl4krpr2EeCeAUvifiQ9Uf5X+M=
+github.com/timandy/routine v1.1.5/go.mod h1:kXslgIosdY8LW0byTyPnenDgn4/azt2euufAq9rK51w=
 github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk=
 github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs=
 github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
 github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
+github.com/wadey/synctrace v0.0.0-20250612192159-94547ef50dfe h1:dc8Q42VsX+ABr0drJw27f3smvGfcz7eB8rJx+IkVMAo=
+github.com/wadey/synctrace v0.0.0-20250612192159-94547ef50dfe/go.mod h1:F2VCml4UxGPgAAqqm9T0ZfnVRWITrQS1EMZM+KCAm/Q=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

handshake_ix.go

@@ -9,6 +9,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/header"
+	"github.com/wadey/synctrace"
 )
 
 // NOISE IX Handshakes
@@ -249,6 +250,7 @@ func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet
 		HandshakePacket:   make(map[uint8][]byte, 0),
 		lastHandshakeTime: hs.Details.Time,
 		relayState: RelayState{
+			RWMutex:        synctrace.NewRWMutex("relay-state"),
 			relays:         map[netip.Addr]struct{}{},
 			relayForByAddr: map[netip.Addr]*Relay{},
 			relayForByIdx:  map[uint32]*Relay{},

handshake_manager.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"net/netip"
 	"slices"
-	"sync"
 	"time"
 
 	"github.com/rcrowley/go-metrics"
@@ -16,6 +15,7 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/udp"
+	"github.com/wadey/synctrace"
 )
 
 const (
@@ -45,7 +45,7 @@ type HandshakeConfig struct {
 
 type HandshakeManager struct {
 	// Mutex for interacting with the vpnIps and indexes maps
-	sync.RWMutex
+	synctrace.RWMutex
 
 	vpnIps  map[netip.Addr]*HandshakeHostInfo
 	indexes map[uint32]*HandshakeHostInfo
@@ -66,7 +66,7 @@ type HandshakeManager struct {
 }
 
 type HandshakeHostInfo struct {
-	sync.Mutex
+	synctrace.Mutex
 
 	startTime   time.Time        // Time that we first started trying with this handshake
 	ready       bool             // Is the handshake ready
@@ -104,6 +104,7 @@ func (hh *HandshakeHostInfo) cachePacket(l *logrus.Logger, t header.MessageType,
 
 func NewHandshakeManager(l *logrus.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager {
 	return &HandshakeManager{
+		RWMutex:                synctrace.NewRWMutex("handshake-manager"),
 		vpnIps:                 map[netip.Addr]*HandshakeHostInfo{},
 		indexes:                map[uint32]*HandshakeHostInfo{},
 		mainHostMap:            mainHostMap,
@@ -111,7 +112,7 @@ func NewHandshakeManager(l *logrus.Logger, mainHostMap *HostMap, lightHouse *Lig
 		outside:                outside,
 		config:                 config,
 		trigger:                make(chan netip.Addr, config.triggerBuffer),
-		OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
+		OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr]("outbound-handshake-timer", config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
 		messageMetrics:         config.messageMetrics,
 		metricInitiated:        metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
 		metricTimedOut:         metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
@@ -450,6 +451,7 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
 		vpnAddrs:        []netip.Addr{vpnAddr},
 		HandshakePacket: make(map[uint8][]byte, 0),
 		relayState: RelayState{
+			RWMutex:        synctrace.NewRWMutex("relay-state"),
 			relays:         map[netip.Addr]struct{}{},
 			relayForByAddr: map[netip.Addr]*Relay{},
 			relayForByIdx:  map[uint32]*Relay{},
@@ -457,6 +459,7 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
 	}
 
 	hh := &HandshakeHostInfo{
+		Mutex:     synctrace.NewMutex("handshake-hostinfo"),
 		hostinfo:  hostinfo,
 		startTime: time.Now(),
 	}

hostmap.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"net"
 	"net/netip"
-	"sync"
 	"sync/atomic"
 	"time"
 
@@ -14,6 +13,7 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
+	"github.com/wadey/synctrace"
 )
 
 // const ProbeLen = 100
@@ -53,7 +53,7 @@ type Relay struct {
 }
 
 type HostMap struct {
-	sync.RWMutex    //Because we concurrently read and write to our maps
+	synctrace.RWMutex //Because we concurrently read and write to our maps
 	Indexes           map[uint32]*HostInfo
 	Relays            map[uint32]*HostInfo // Maps a Relay IDX to a Relay HostInfo object
 	RemoteIndexes     map[uint32]*HostInfo
@@ -66,7 +66,7 @@ type HostMap struct {
 // struct, make a copy of an existing value, edit the fileds in the copy, and
 // then store a pointer to the new copy in both realyForBy* maps.
 type RelayState struct {
-	sync.RWMutex
+	synctrace.RWMutex
 
 	relays map[netip.Addr]struct{} // Set of vpnAddr's of Hosts to use as relays to access this peer
 	// For data race avoidance, the contents of a *Relay are treated immutably. To update a *Relay, copy the existing data,
@@ -288,6 +288,7 @@ func NewHostMapFromConfig(l *logrus.Logger, c *config.C) *HostMap {
 
 func newHostMap(l *logrus.Logger) *HostMap {
 	return &HostMap{
+		RWMutex:       synctrace.NewRWMutex("hostmap"),
 		Indexes:       map[uint32]*HostInfo{},
 		Relays:        map[uint32]*HostInfo{},
 		RemoteIndexes: map[uint32]*HostInfo{},

lighthouse.go

@@ -9,7 +9,6 @@ import (
 	"net/netip"
 	"slices"
 	"strconv"
-	"sync"
 	"sync/atomic"
 	"time"
 
@@ -21,13 +20,14 @@ import (
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/udp"
 	"github.com/slackhq/nebula/util"
+	"github.com/wadey/synctrace"
 )
 
 var ErrHostNotKnown = errors.New("host not known")
 
 type LightHouse struct {
 	//TODO: We need a timer wheel to kick out vpnAddrs that haven't reported in a long time
-	sync.RWMutex //Because we concurrently read and write to our maps
+	synctrace.RWMutex //Because we concurrently read and write to our maps
 	ctx               context.Context
 	amLighthouse      bool
 
@@ -96,6 +96,7 @@ func NewLightHouseFromConfig(ctx context.Context, l *logrus.Logger, c *config.C,
 	}
 
 	h := LightHouse{
+		RWMutex:            synctrace.NewRWMutex("lighthouse"),
 		ctx:                ctx,
 		amLighthouse:       amLighthouse,
 		myVpnNetworks:      cs.myVpnNetworks,
@@ -472,6 +473,7 @@ func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) {
 		return
 	}
 
+	synctrace.ChanDebugSend("lighthouse-querychan")
 	lh.queryChan <- vpnAddr
 }
 
@@ -725,9 +727,11 @@ func (lh *LightHouse) startQueryWorker() {
 		nb := make([]byte, 12, 12)
 		out := make([]byte, mtu)
 
+		synctrace.ChanDebugRecvLock("lighthouse-querychan")
 		for {
 			select {
 			case <-lh.ctx.Done():
+				synctrace.ChanDebugRecvUnlock("lighthouse-querychan")
 				return
 			case addr := <-lh.queryChan:
 				lh.innerQueryServer(addr, nb, out)

outside.go

@@ -312,12 +312,11 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 	offset := ipv6.HeaderLen // Start at the end of the ipv6 header
 	next := 0
 	for {
-		if dataLen < offset {
+		if protoAt >= dataLen {
 			break
 		}
-
 		proto := layers.IPProtocol(data[protoAt])
-		//fmt.Println(proto, protoAt)
+
 		switch proto {
 		case layers.IPProtocolICMPv6, layers.IPProtocolESP, layers.IPProtocolNoNextHeader:
 			fp.Protocol = uint8(proto)
@@ -365,7 +364,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 
 		case layers.IPProtocolAH:
 			// Auth headers, used by IPSec, have a different meaning for header length
-			if dataLen < offset+1 {
+			if dataLen <= offset+1 {
 				break
 			}
 
@@ -373,7 +372,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 
 		default:
 			// Normal ipv6 header length processing
-			if dataLen < offset+1 {
+			if dataLen <= offset+1 {
 				break
 			}
 

outside_test.go

@@ -117,6 +117,45 @@ func Test_newPacket_v6(t *testing.T) {
 	err = newPacket(buffer.Bytes(), true, p)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
+	// A v6 packet with a hop-by-hop extension
+	// ICMPv6 Payload (Echo Request)
+	icmpLayer := layers.ICMPv6{
+		TypeCode: layers.ICMPv6TypeEchoRequest,
+	}
+	// Hop-by-Hop Extension Header
+	hopOption := layers.IPv6HopByHopOption{}
+	hopOption.OptionData = []byte{0, 0, 0, 0}
+	hopByHop := layers.IPv6HopByHop{}
+	hopByHop.Options = append(hopByHop.Options, &hopOption)
+
+	ip = layers.IPv6{
+		Version:    6,
+		HopLimit:   128,
+		NextHeader: layers.IPProtocolIPv6Destination,
+		SrcIP:      net.IPv6linklocalallrouters,
+		DstIP:      net.IPv6linklocalallnodes,
+	}
+
+	buffer.Clear()
+	err = gopacket.SerializeLayers(buffer, gopacket.SerializeOptions{
+		ComputeChecksums: false,
+		FixLengths:       true,
+	}, &ip, &hopByHop, &icmpLayer)
+	if err != nil {
+		panic(err)
+	}
+	// Ensure buffer length checks during parsing with the next 2 tests.
+
+	// A full IPv6 header and 1 byte in the first extension, but missing
+	// the length byte.
+	err = newPacket(buffer.Bytes()[:41], true, p)
+	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
+
+	// A full IPv6 header plus 1 full extension, but only 1 byte of the
+	// next layer, missing length byte
+	err = newPacket(buffer.Bytes()[:49], true, p)
+	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
+
 	// A good ICMP packet
 	ip = layers.IPv6{
 		Version:    6,
@@ -288,6 +327,10 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.Equal(t, uint16(22), p.LocalPort)
 	assert.False(t, p.Fragment)
 
+	// Ensure buffer bounds checking during processing
+	err = newPacket(b[:41], true, p)
+	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
+
 	// Invalid AH header
 	b = buffer.Bytes()
 	err = newPacket(b, true, p)

overlay/tun_linux.go

@@ -38,6 +38,7 @@ type tun struct {
 	routeTree                 atomic.Pointer[bart.Table[routing.Gateways]]
 	routeChan                 chan struct{}
 	useSystemRoutes           bool
+	useSystemRoutesBufferSize int
 
 	l *logrus.Logger
 }
@@ -129,6 +130,7 @@ func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []n
 		vpnNetworks:               vpnNetworks,
 		TXQueueLen:                c.GetInt("tun.tx_queue", 500),
 		useSystemRoutes:           c.GetBool("tun.use_system_route_table", false),
+		useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
 		l:                         l,
 	}
 
@@ -531,7 +533,13 @@ func (t *tun) watchRoutes() {
 	rch := make(chan netlink.RouteUpdate)
 	doneChan := make(chan struct{})
 
-	if err := netlink.RouteSubscribe(rch, doneChan); err != nil {
+	netlinkOptions := netlink.RouteSubscribeOptions{
+		ReceiveBufferSize:      t.useSystemRoutesBufferSize,
+		ReceiveBufferForceSize: t.useSystemRoutesBufferSize != 0,
+		ErrorCallback:          func(e error) { t.l.WithError(e).Errorf("netlink error") },
+	}
+
+	if err := netlink.RouteSubscribeWithOptions(rch, doneChan, netlinkOptions); err != nil {
 		t.l.WithError(err).Errorf("failed to subscribe to system route changes")
 		return
 	}
@@ -541,8 +549,14 @@ func (t *tun) watchRoutes() {
 	go func() {
 		for {
 			select {
-			case r := <-rch:
+			case r, ok := <-rch:
+				if ok {
 					t.updateRoutes(r)
+				} else {
+					// may be should do something here as
+					// netlink stops sending updates
+					return
+				}
 			case <-doneChan:
 				// netlink.RouteSubscriber will close the rch for us
 				return

remote_list.go

@@ -7,11 +7,11 @@ import (
 	"slices"
 	"sort"
 	"strconv"
-	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/sirupsen/logrus"
+	"github.com/wadey/synctrace"
 )
 
 // forEachFunc is used to benefit folks that want to do work inside the lock
@@ -185,7 +185,7 @@ func (hr *hostnamesResults) GetAddrs() []netip.AddrPort {
 // It serves as a local cache of query replies, host update notifications, and locally learned addresses
 type RemoteList struct {
 	// Every interaction with internals requires a lock!
-	sync.RWMutex
+	synctrace.RWMutex
 
 	// The full list of vpn addresses assigned to this host
 	vpnAddrs []netip.Addr
@@ -215,6 +215,7 @@ type RemoteList struct {
 // NewRemoteList creates a new empty RemoteList
 func NewRemoteList(vpnAddrs []netip.Addr, shouldAdd func(netip.Addr) bool) *RemoteList {
 	r := &RemoteList{
+		RWMutex:   synctrace.NewRWMutex("remote-list"),
 		vpnAddrs:  make([]netip.Addr, len(vpnAddrs)),
 		addrs:     make([]netip.AddrPort, 0),
 		relays:    make([]netip.Addr, 0),

service/service.go

@@ -9,13 +9,10 @@ import (
 	"math"
 	"net"
 	"net/netip"
-	"os"
 	"strings"
 	"sync"
 
-	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
-	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/overlay"
 	"golang.org/x/sync/errgroup"
 	"gvisor.dev/gvisor/pkg/buffer"
@@ -46,14 +43,7 @@ type Service struct {
 	}
 }
 
-func New(config *config.C) (*Service, error) {
-	logger := logrus.New()
-	logger.Out = os.Stdout
-
-	control, err := nebula.Main(config, false, "custom-app", logger, overlay.NewUserDeviceFromConfig)
-	if err != nil {
-		return nil, err
-	}
+func New(control *nebula.Control) (*Service, error) {
 	control.Start()
 
 	ctx := control.Context()

service/service_test.go

@@ -5,13 +5,17 @@ import (
 	"context"
 	"errors"
 	"net/netip"
+	"os"
 	"testing"
 	"time"
 
 	"dario.cat/mergo"
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/overlay"
 	"golang.org/x/sync/errgroup"
 	"gopkg.in/yaml.v3"
 )
@@ -71,7 +75,15 @@ func newSimpleService(caCrt cert.Certificate, caKey []byte, name string, udpIp n
 		panic(err)
 	}
 
-	s, err := New(&c)
+	logger := logrus.New()
+	logger.Out = os.Stdout
+
+	control, err := nebula.Main(&c, false, "custom-app", logger, overlay.NewUserDeviceFromConfig)
+	if err != nil {
+		panic(err)
+	}
+
+	s, err := New(control)
 	if err != nil {
 		panic(err)
 	}

sshd/server.go

@@ -5,10 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"sync"
 
 	"github.com/armon/go-radix"
 	"github.com/sirupsen/logrus"
+	"github.com/wadey/synctrace"
 	"golang.org/x/crypto/ssh"
 )
 
@@ -28,7 +28,7 @@ type SSHServer struct {
 	listener    net.Listener
 
 	// Locks the conns/counter to avoid concurrent map access
-	connsLock sync.Mutex
+	connsLock synctrace.Mutex
 	conns     map[int]*session
 	counter   int
 }
@@ -41,6 +41,7 @@ func NewSSHServer(l *logrus.Entry) (*SSHServer, error) {
 		l:           l,
 		commands:    radix.New(),
 		conns:       make(map[int]*session),
+		connsLock:   synctrace.NewMutex("ssh-server-conns"),
 	}
 
 	cc := ssh.CertChecker{

timeout.go

@@ -1,8 +1,9 @@
 package nebula
 
 import (
-	"sync"
 	"time"
+
+	"github.com/wadey/synctrace"
 )
 
 // How many timer objects should be cached
@@ -34,7 +35,7 @@ type TimerWheel[T any] struct {
 }
 
 type LockingTimerWheel[T any] struct {
-	m sync.Mutex
+	m synctrace.Mutex
 	t *TimerWheel[T]
 }
 
@@ -81,8 +82,9 @@ func NewTimerWheel[T any](min, max time.Duration) *TimerWheel[T] {
 }
 
 // NewLockingTimerWheel is version of TimerWheel that is safe for concurrent use with a small performance penalty
-func NewLockingTimerWheel[T any](min, max time.Duration) *LockingTimerWheel[T] {
+func NewLockingTimerWheel[T any](name string, min, max time.Duration) *LockingTimerWheel[T] {
 	return &LockingTimerWheel[T]{
+		m: synctrace.NewMutex(name),
 		t: NewTimerWheel[T](min, max),
 	}
 }

udp/udp_rio_windows.go

@@ -11,13 +11,13 @@ import (
 	"io"
 	"net"
 	"net/netip"
-	"sync"
 	"sync/atomic"
 	"syscall"
 	"unsafe"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
+	"github.com/wadey/synctrace"
 	"golang.org/x/sys/windows"
 	"golang.zx2c4.com/wireguard/conn/winrio"
 )
@@ -46,7 +46,7 @@ type ringBuffer struct {
 	iocp       windows.Handle
 	isFull     bool
 	cq         winrio.Cq
-	mu         sync.Mutex
+	mu         synctrace.Mutex
 	overlapped windows.Overlapped
 }
 
@@ -64,7 +64,11 @@ func NewRIOListener(l *logrus.Logger, addr netip.Addr, port int) (*RIOConn, erro
 		return nil, errors.New("could not initialize winrio")
 	}
 
-	u := &RIOConn{l: l}
+	u := &RIOConn{
+		l:  l,
+		rx: ringBuffer{mu: synctrace.NewMutex("rio-rx")},
+		tx: ringBuffer{mu: synctrace.NewMutex("rio-tx")},
+	}
 
 	err := u.bind(&windows.SockaddrInet6{Addr: addr.As16(), Port: port})
 	if err != nil {