mutex_debug

experimental test to see if we can have a test mode that verifies
mutexes lock in the order we want, while having no hit on production
performance. Since this uses a build tag, it should all compile out
during the build process and be a no-op unless the tag is set.

Makefile

@@ -54,6 +54,9 @@ ALL = $(ALL_LINUX) \
 e2e:
 	$(TEST_ENV) go test -tags=e2e_testing -count=1 $(TEST_FLAGS) ./e2e
 
+e2e-mutex-debug:
+	$(TEST_ENV) go test -tags=mutex_debug,e2e_testing -count=1 $(TEST_FLAGS) ./e2e
+
 e2ev: TEST_FLAGS = -v
 e2ev: e2e
 

go.mod

@@ -42,6 +42,7 @@ require (
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
+	github.com/timandy/routine v1.1.1 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
 	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/tools v0.8.0 // indirect

go.sum

@@ -140,6 +140,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/timandy/routine v1.1.1 h1:6/Z7qLFZj3GrzuRksBFzIG8YGUh8CLhjnnMePBQTrEI=
+github.com/timandy/routine v1.1.1/go.mod h1:OZHPOKSvqL/ZvqXFkNZyit0xIVelERptYXdAHH00adQ=
 github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=

hostmap.go

@@ -51,7 +51,7 @@ type Relay struct {
 }
 
 type HostMap struct {
-	sync.RWMutex    //Because we concurrently read and write to our maps
+	syncRWMutex     //Because we concurrently read and write to our maps
 	name            string
 	Indexes         map[uint32]*HostInfo
 	Relays          map[uint32]*HostInfo // Maps a Relay IDX to a Relay HostInfo object
@@ -197,7 +197,7 @@ func (rs *RelayState) InsertRelay(ip iputil.VpnIp, idx uint32, r *Relay) {
 }
 
 type HostInfo struct {
-	sync.RWMutex
+	syncRWMutex
 
 	remote               *udp.Addr
 	remotes              *RemoteList
@@ -261,6 +261,7 @@ func NewHostMap(l *logrus.Logger, name string, vpnCIDR *net.IPNet, preferredRang
 	r := map[uint32]*HostInfo{}
 	relays := map[uint32]*HostInfo{}
 	m := HostMap{
+		syncRWMutex:     newSyncRWMutex("hostmap", name),
 		name:            name,
 		Indexes:         i,
 		Relays:          relays,
@@ -321,6 +322,7 @@ func (hm *HostMap) AddVpnIp(vpnIp iputil.VpnIp, init func(hostinfo *HostInfo)) (
 	if h, ok := hm.Hosts[vpnIp]; !ok {
 		hm.RUnlock()
 		h = &HostInfo{
+			syncRWMutex:     newSyncRWMutex("hostinfo"),
 			vpnIp:           vpnIp,
 			HandshakePacket: make(map[uint8][]byte, 0),
 			relayState: RelayState{

mutex.go

@@ -0,0 +1,14 @@
+//go:build !mutex_debug
+// +build !mutex_debug
+
+package nebula
+
+import (
+	"sync"
+)
+
+type syncRWMutex = sync.RWMutex
+
+func newSyncRWMutex(t ...string) syncRWMutex {
+	return sync.RWMutex{}
+}

mutex_debug.go

@@ -0,0 +1,66 @@
+//go:build mutex_debug
+// +build mutex_debug
+
+package nebula
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/timandy/routine"
+)
+
+var threadLocal routine.ThreadLocal = routine.NewThreadLocalWithInitial(func() any { return map[string]bool{} })
+
+type syncRWMutex struct {
+	sync.RWMutex
+	mutexType string
+}
+
+func newSyncRWMutex(t ...string) syncRWMutex {
+	return syncRWMutex{
+		mutexType: strings.Join(t, "-"),
+	}
+}
+
+func checkMutex(state map[string]bool, add string) {
+	if add == "hostinfo" {
+		if state["hostmap-main"] {
+			panic("grabbing hostinfo lock and already have hostmap-main")
+		}
+		if state["hostmap-pending"] {
+			panic("grabbing hostinfo lock and already have hostmap-pending")
+		}
+	}
+	if add == "hostmap-pending" {
+		if state["hostmap-main"] {
+			panic("grabbing hostmap-pending lock and already have hostmap-main")
+		}
+	}
+}
+
+func (s *syncRWMutex) Lock() {
+	m := threadLocal.Get().(map[string]bool)
+	checkMutex(m, s.mutexType)
+	m[s.mutexType] = true
+	s.RWMutex.Lock()
+}
+
+func (s *syncRWMutex) Unlock() {
+	m := threadLocal.Get().(map[string]bool)
+	m[s.mutexType] = false
+	s.RWMutex.Unlock()
+}
+
+func (s *syncRWMutex) RLock() {
+	m := threadLocal.Get().(map[string]bool)
+	checkMutex(m, s.mutexType)
+	m[s.mutexType] = true
+	s.RWMutex.RLock()
+}
+
+func (s *syncRWMutex) RUnlock() {
+	m := threadLocal.Get().(map[string]bool)
+	m[s.mutexType] = false
+	s.RWMutex.RUnlock()
+}