diff --git a/Makefile b/Makefile index 512fdc2..8e771b0 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,9 @@ ALL = $(ALL_LINUX) \ e2e: $(TEST_ENV) go test -tags=e2e_testing -count=1 $(TEST_FLAGS) ./e2e +e2e-mutex-debug: + $(TEST_ENV) go test -tags=mutex_debug,e2e_testing -count=1 $(TEST_FLAGS) ./e2e + e2ev: TEST_FLAGS = -v e2ev: e2e diff --git a/go.mod b/go.mod index 52c2e92..93fb85d 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/timandy/routine v1.1.1 // indirect github.com/vishvananda/netns v0.0.4 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/tools v0.8.0 // indirect diff --git a/go.sum b/go.sum index 452a1d2..6758c3d 100644 --- a/go.sum +++ b/go.sum @@ -140,6 +140,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/timandy/routine v1.1.1 h1:6/Z7qLFZj3GrzuRksBFzIG8YGUh8CLhjnnMePBQTrEI= +github.com/timandy/routine v1.1.1/go.mod h1:OZHPOKSvqL/ZvqXFkNZyit0xIVelERptYXdAHH00adQ= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= diff --git a/hostmap.go b/hostmap.go index e5949ad..597d7e4 100644 --- a/hostmap.go +++ b/hostmap.go @@ -51,7 +51,7 @@ type Relay struct { } type HostMap struct { - sync.RWMutex //Because we concurrently read and write to our maps + syncRWMutex //Because we concurrently read and write to our maps name string Indexes map[uint32]*HostInfo Relays map[uint32]*HostInfo // Maps a Relay IDX to a Relay HostInfo object @@ -197,7 +197,7 @@ func (rs *RelayState) InsertRelay(ip iputil.VpnIp, idx uint32, r *Relay) { } type HostInfo struct { - sync.RWMutex + syncRWMutex remote *udp.Addr remotes *RemoteList @@ -261,6 +261,7 @@ func NewHostMap(l *logrus.Logger, name string, vpnCIDR *net.IPNet, preferredRang r := map[uint32]*HostInfo{} relays := map[uint32]*HostInfo{} m := HostMap{ + syncRWMutex: newSyncRWMutex("hostmap", name), name: name, Indexes: i, Relays: relays, @@ -321,6 +322,7 @@ func (hm *HostMap) AddVpnIp(vpnIp iputil.VpnIp, init func(hostinfo *HostInfo)) ( if h, ok := hm.Hosts[vpnIp]; !ok { hm.RUnlock() h = &HostInfo{ + syncRWMutex: newSyncRWMutex("hostinfo"), vpnIp: vpnIp, HandshakePacket: make(map[uint8][]byte, 0), relayState: RelayState{ diff --git a/mutex.go b/mutex.go new file mode 100644 index 0000000..b7ff88a --- /dev/null +++ b/mutex.go @@ -0,0 +1,14 @@ +//go:build !mutex_debug +// +build !mutex_debug + +package nebula + +import ( + "sync" +) + +type syncRWMutex = sync.RWMutex + +func newSyncRWMutex(t ...string) syncRWMutex { + return sync.RWMutex{} +} diff --git a/mutex_debug.go b/mutex_debug.go new file mode 100644 index 0000000..5f20092 --- /dev/null +++ b/mutex_debug.go @@ -0,0 +1,66 @@ +//go:build mutex_debug +// +build mutex_debug + +package nebula + +import ( + "strings" + "sync" + + "github.com/timandy/routine" +) + +var threadLocal routine.ThreadLocal = routine.NewThreadLocalWithInitial(func() any { return map[string]bool{} }) + +type syncRWMutex struct { + sync.RWMutex + mutexType string +} + +func newSyncRWMutex(t ...string) syncRWMutex { + return syncRWMutex{ + mutexType: strings.Join(t, "-"), + } +} + +func checkMutex(state map[string]bool, add string) { + if add == "hostinfo" { + if state["hostmap-main"] { + panic("grabbing hostinfo lock and already have hostmap-main") + } + if state["hostmap-pending"] { + panic("grabbing hostinfo lock and already have hostmap-pending") + } + } + if add == "hostmap-pending" { + if state["hostmap-main"] { + panic("grabbing hostmap-pending lock and already have hostmap-main") + } + } +} + +func (s *syncRWMutex) Lock() { + m := threadLocal.Get().(map[string]bool) + checkMutex(m, s.mutexType) + m[s.mutexType] = true + s.RWMutex.Lock() +} + +func (s *syncRWMutex) Unlock() { + m := threadLocal.Get().(map[string]bool) + m[s.mutexType] = false + s.RWMutex.Unlock() +} + +func (s *syncRWMutex) RLock() { + m := threadLocal.Get().(map[string]bool) + checkMutex(m, s.mutexType) + m[s.mutexType] = true + s.RWMutex.RLock() +} + +func (s *syncRWMutex) RUnlock() { + m := threadLocal.Get().(map[string]bool) + m[s.mutexType] = false + s.RWMutex.RUnlock() +}