Revert "smoke test"

This reverts commit fa034a6d83.

.github/workflows/gofmt.yml

@@ -14,10 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.16
+    - name: Set up Go 1.15
       uses: actions/setup-go@v1
       with:
-        go-version: 1.16
+        go-version: 1.15
       id: go
 
     - name: Check out code into the Go module directory
@@ -26,9 +26,9 @@ jobs:
     - uses: actions/cache@v1
       with:
         path: ~/go/pkg/mod
-        key: ${{ runner.os }}-gofmt1.16-${{ hashFiles('**/go.sum') }}
+        key: ${{ runner.os }}-gofmt-${{ hashFiles('**/go.sum') }}
         restore-keys: |
-          ${{ runner.os }}-gofmt1.16-
+          ${{ runner.os }}-gofmt-
 
     - name: Install goimports
       run: |

.github/workflows/release.yml

@@ -10,10 +10,10 @@ jobs:
     name: Build Linux All
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.16
+      - name: Set up Go 1.15
         uses: actions/setup-go@v1
         with:
-          go-version: 1.16
+          go-version: 1.15
 
       - name: Checkout code
         uses: actions/checkout@v2
@@ -34,10 +34,10 @@ jobs:
     name: Build Windows amd64
     runs-on: windows-latest
     steps:
-      - name: Set up Go 1.16
+      - name: Set up Go 1.15
         uses: actions/setup-go@v1
         with:
-          go-version: 1.16
+          go-version: 1.15
 
       - name: Checkout code
         uses: actions/checkout@v2
@@ -58,10 +58,10 @@ jobs:
     name: Build Darwin amd64
     runs-on: macOS-latest
     steps:
-      - name: Set up Go 1.16
+      - name: Set up Go 1.15
         uses: actions/setup-go@v1
         with:
-          go-version: 1.16
+          go-version: 1.15
 
       - name: Checkout code
         uses: actions/checkout@v2
@@ -69,7 +69,6 @@ jobs:
       - name: Build
         run: |
           make BUILD_NUMBER="${GITHUB_REF#refs/tags/v}" service build/nebula-darwin-amd64.tar.gz
-          make BUILD_NUMBER="${GITHUB_REF#refs/tags/v}" service build/nebula-darwin-arm64.tar.gz
           mkdir release
           mv build/*.tar.gz release
 
@@ -160,16 +159,6 @@ jobs:
           asset_name: nebula-darwin-amd64.tar.gz
           asset_content_type: application/gzip
 
-      - name: Upload darwin-arm64
-        uses: actions/upload-release-asset@v1.0.1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./darwin-latest/nebula-darwin-arm64.tar.gz
-          asset_name: nebula-darwin-arm64.tar.gz
-          asset_content_type: application/gzip
-
       - name: Upload windows-amd64
         uses: actions/upload-release-asset@v1.0.1
         env:

.github/workflows/smoke.yml

@@ -18,10 +18,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.16
+    - name: Set up Go 1.15
       uses: actions/setup-go@v1
       with:
-        go-version: 1.16
+        go-version: 1.15
       id: go
 
     - name: Check out code into the Go module directory
@@ -30,12 +30,12 @@ jobs:
     - uses: actions/cache@v1
       with:
         path: ~/go/pkg/mod
-        key: ${{ runner.os }}-go1.16-${{ hashFiles('**/go.sum') }}
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
-          ${{ runner.os }}-go1.16-
+          ${{ runner.os }}-go-
 
     - name: build
-      run: make bin-docker
+      run: make
 
     - name: setup docker image
       working-directory: ./.github/workflows/smoke

.github/workflows/smoke/Dockerfile

@@ -1,7 +1,5 @@
 FROM debian:buster
 
-ADD ./build /nebula
+ADD ./build /
 
-WORKDIR /nebula
-
-ENTRYPOINT ["/nebula/nebula"]
+ENTRYPOINT ["/nebula"]

.github/workflows/smoke/build.sh

@@ -8,8 +8,8 @@ mkdir ./build
 (
     cd build
 
-    cp ../../../../build/linux-amd64/nebula .
-    cp ../../../../build/linux-amd64/nebula-cert .
+    cp ../../../../nebula .
+    cp ../../../../nebula-cert .
 
     HOST="lighthouse1" \
         AM_LIGHTHOUSE=true \
@@ -29,11 +29,11 @@ mkdir ./build
         OUTBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host4.yml
 
-    ../../../../nebula-cert ca -name "Smoke Test"
-    ../../../../nebula-cert sign -name "lighthouse1" -groups "lighthouse,lighthouse1" -ip "192.168.100.1/24"
-    ../../../../nebula-cert sign -name "host2" -groups "host,host2" -ip "192.168.100.2/24"
-    ../../../../nebula-cert sign -name "host3" -groups "host,host3" -ip "192.168.100.3/24"
-    ../../../../nebula-cert sign -name "host4" -groups "host,host4" -ip "192.168.100.4/24"
+    ./nebula-cert ca -name "Smoke Test"
+    ./nebula-cert sign -name "lighthouse1" -groups "lighthouse,lighthouse1" -ip "192.168.100.1/24"
+    ./nebula-cert sign -name "host2" -groups "host,host2" -ip "192.168.100.2/24"
+    ./nebula-cert sign -name "host3" -groups "host,host3" -ip "192.168.100.3/24"
+    ./nebula-cert sign -name "host4" -groups "host,host4" -ip "192.168.100.4/24"
 )
 
-sudo docker build -t nebula:smoke .
+docker build -t nebula:smoke .

.github/workflows/smoke/genconfig.sh

@@ -33,9 +33,9 @@ lighthouse_hosts() {
 
 cat <<EOF
 pki:
-  ca: ca.crt
-  cert: ${HOST}.crt
-  key: ${HOST}.key
+  ca: /ca.crt
+  cert: /${HOST}.crt
+  key: /${HOST}.key
 
 lighthouse:
   am_lighthouse: ${AM_LIGHTHOUSE:-false}

.github/workflows/smoke/smoke.sh

@@ -1,33 +1,19 @@
-#!/bin/bash
+#!/bin/sh
 
 set -e -x
 
-set -o pipefail
+docker run --name lighthouse1 --rm nebula:smoke -config lighthouse1.yml -test
+docker run --name host2 --rm nebula:smoke -config host2.yml -test
+docker run --name host3 --rm nebula:smoke -config host3.yml -test
+docker run --name host4 --rm nebula:smoke -config host4.yml -test
 
-mkdir -p logs
-
-cleanup() {
-    set +e
-    if [ "$(jobs -r)" ]
-    then
-        sudo docker kill lighthouse1 host2 host3 host4
-    fi
-}
-
-trap cleanup EXIT
-
-sudo docker run --name lighthouse1 --rm nebula:smoke -config lighthouse1.yml -test
-sudo docker run --name host2 --rm nebula:smoke -config host2.yml -test
-sudo docker run --name host3 --rm nebula:smoke -config host3.yml -test
-sudo docker run --name host4 --rm nebula:smoke -config host4.yml -test
-
-sudo docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 &
+docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config lighthouse1.yml &
 sleep 1
-sudo docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host2.yml 2>&1 | tee logs/host2 &
+docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host2.yml &
 sleep 1
-sudo docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host3.yml 2>&1 | tee logs/host3 &
+docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host3.yml &
 sleep 1
-sudo docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host4.yml 2>&1 | tee logs/host4 &
+docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host4.yml &
 sleep 1
 
 set +x
@@ -35,35 +21,35 @@ echo
 echo " *** Testing ping from lighthouse1"
 echo
 set -x
-sudo docker exec lighthouse1 ping -c1 192.168.100.2
-sudo docker exec lighthouse1 ping -c1 192.168.100.3
+docker exec lighthouse1 ping -c1 192.168.100.2
+docker exec lighthouse1 ping -c1 192.168.100.3
 
 set +x
 echo
 echo " *** Testing ping from host2"
 echo
 set -x
-sudo docker exec host2 ping -c1 192.168.100.1
+docker exec host2 ping -c1 192.168.100.1
 # Should fail because not allowed by host3 inbound firewall
-! sudo docker exec host2 ping -c1 192.168.100.3 -w5 || exit 1
+! docker exec host2 ping -c1 192.168.100.3 -w5 || exit 1
 
 set +x
 echo
 echo " *** Testing ping from host3"
 echo
 set -x
-sudo docker exec host3 ping -c1 192.168.100.1
-sudo docker exec host3 ping -c1 192.168.100.2
+docker exec host3 ping -c1 192.168.100.1
+docker exec host3 ping -c1 192.168.100.2
 
 set +x
 echo
 echo " *** Testing ping from host4"
 echo
 set -x
-sudo docker exec host4 ping -c1 192.168.100.1
+docker exec host4 ping -c1 192.168.100.1
 # Should fail because not allowed by host4 outbound firewall
-! sudo docker exec host4 ping -c1 192.168.100.2 -w5 || exit 1
-! sudo docker exec host4 ping -c1 192.168.100.3 -w5 || exit 1
+! docker exec host4 ping -c1 192.168.100.2 -w5 || exit 1
+! docker exec host4 ping -c1 192.168.100.3 -w5 || exit 1
 
 set +x
 echo
@@ -71,13 +57,7 @@ echo " *** Testing conntrack"
 echo
 set -x
 # host2 can ping host3 now that host3 pinged it first
-sudo docker exec host2 ping -c1 192.168.100.3
+docker exec host2 ping -c1 192.168.100.3
 # host4 can ping host2 once conntrack established
-sudo docker exec host2 ping -c1 192.168.100.4
-sudo docker exec host4 ping -c1 192.168.100.2
-
-sudo docker exec host4 sh -c 'kill 1'
-sudo docker exec host3 sh -c 'kill 1'
-sudo docker exec host2 sh -c 'kill 1'
-sudo docker exec lighthouse1 sh -c 'kill 1'
-sleep 1
+docker exec host2 ping -c1 192.168.100.4
+docker exec host4 ping -c1 192.168.100.2

.github/workflows/test.yml

@@ -18,10 +18,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.16
+    - name: Set up Go 1.15
       uses: actions/setup-go@v1
       with:
-        go-version: 1.16
+        go-version: 1.15
       id: go
 
     - name: Check out code into the Go module directory
@@ -30,9 +30,9 @@ jobs:
     - uses: actions/cache@v1
       with:
         path: ~/go/pkg/mod
-        key: ${{ runner.os }}-go1.16-${{ hashFiles('**/go.sum') }}
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
-          ${{ runner.os }}-go1.16-
+          ${{ runner.os }}-go-
 
     - name: Build
       run: make all
@@ -48,10 +48,10 @@ jobs:
         os: [windows-latest, macOS-latest]
     steps:
 
-    - name: Set up Go 1.16
+    - name: Set up Go 1.15
       uses: actions/setup-go@v1
       with:
-        go-version: 1.16
+        go-version: 1.15
       id: go
 
     - name: Check out code into the Go module directory
@@ -60,9 +60,9 @@ jobs:
     - uses: actions/cache@v1
       with:
         path: ~/go/pkg/mod
-        key: ${{ runner.os }}-go1.16-${{ hashFiles('**/go.sum') }}
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
-          ${{ runner.os }}-go1.16-
+          ${{ runner.os }}-go-
 
     - name: Build nebula
       run: go build ./cmd/nebula

Makefile

@@ -1,16 +1,8 @@
-GOMINVERSION = 1.16
 NEBULA_CMD_PATH = "./cmd/nebula"
 BUILD_NUMBER ?= dev+$(shell date -u '+%Y%m%d%H%M%S')
 GO111MODULE = on
 export GO111MODULE
 
-# Ensure the version of go we are using is at least what is defined in GOMINVERSION at the top of this file
-GOVERSION := $(shell go version | awk '{print substr($$3, 3)}')
-GOISMIN := $(shell expr "$(GOVERSION)" ">=" "$(GOMINVERSION)")
-ifneq "$(GOISMIN)" "1"
-$(error "go version $(GOVERSION) is not supported, upgrade to $(GOMINVERSION) or above")
-endif
-
 LDFLAGS = -X main.Build=$(BUILD_NUMBER)
 
 ALL_LINUX = linux-amd64 \
@@ -28,12 +20,9 @@ ALL_LINUX = linux-amd64 \
 
 ALL = $(ALL_LINUX) \
 	darwin-amd64 \
-	darwin-arm64 \
 	freebsd-amd64 \
 	windows-amd64
 
-
-
 all: $(ALL:%=build/%/nebula) $(ALL:%=build/%/nebula-cert)
 
 release: $(ALL:%=build/nebula-%.tar.gz)
@@ -42,8 +31,6 @@ release-linux: $(ALL_LINUX:%=build/nebula-%.tar.gz)
 
 release-freebsd: build/nebula-freebsd-amd64.tar.gz
 
-BUILD_ARGS = -trimpath
-
 bin-windows: build/windows-amd64/nebula.exe build/windows-amd64/nebula-cert.exe
 	mv $? .
 
@@ -54,12 +41,12 @@ bin-freebsd: build/freebsd-amd64/nebula build/freebsd-amd64/nebula-cert
 	mv $? .
 
 bin:
-	go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula ${NEBULA_CMD_PATH}
-	go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula-cert ./cmd/nebula-cert
+	go build -trimpath -ldflags "$(LDFLAGS)" -o ./nebula ${NEBULA_CMD_PATH}
+	go build -trimpath -ldflags "$(LDFLAGS)" -o ./nebula-cert ./cmd/nebula-cert
 
 install:
-	go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ${NEBULA_CMD_PATH}
-	go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ./cmd/nebula-cert
+	go install -trimpath -ldflags "$(LDFLAGS)" ${NEBULA_CMD_PATH}
+	go install -trimpath -ldflags "$(LDFLAGS)" ./cmd/nebula-cert
 
 build/linux-arm-%: GOENV += GOARM=$(word 3, $(subst -, ,$*))
 build/linux-mips-%: GOENV += GOMIPS=$(word 3, $(subst -, ,$*))
@@ -70,12 +57,12 @@ build/linux-mips-softfloat/%: LDFLAGS += -s -w
 build/%/nebula: .FORCE
 	GOOS=$(firstword $(subst -, , $*)) \
 		GOARCH=$(word 2, $(subst -, ,$*)) $(GOENV) \
-		go build $(BUILD_ARGS) -o $@ -ldflags "$(LDFLAGS)" ${NEBULA_CMD_PATH}
+		go build -trimpath -o $@ -ldflags "$(LDFLAGS)" ${NEBULA_CMD_PATH}
 
 build/%/nebula-cert: .FORCE
 	GOOS=$(firstword $(subst -, , $*)) \
 		GOARCH=$(word 2, $(subst -, ,$*)) $(GOENV) \
-		go build $(BUILD_ARGS) -o $@ -ldflags "$(LDFLAGS)" ./cmd/nebula-cert
+		go build -trimpath -o $@ -ldflags "$(LDFLAGS)" ./cmd/nebula-cert
 
 build/%/nebula.exe: build/%/nebula
 	mv $< $@
@@ -127,15 +114,6 @@ ifeq ($(words $(MAKECMDGOALS)),1)
 	$(MAKE) service ${.DEFAULT_GOAL} --no-print-directory
 endif
 
-bin-docker: bin build/linux-amd64/nebula build/linux-amd64/nebula-cert
-
-smoke-docker: bin-docker
-	cd .github/workflows/smoke/ && ./build.sh
-	cd .github/workflows/smoke/ && ./smoke.sh
-
-smoke-docker-race: BUILD_ARGS = -race
-smoke-docker-race: smoke-docker
-
 .FORCE:
-.PHONY: test test-cov-html bench bench-cpu bench-cpu-long bin proto release service smoke-docker smoke-docker-race
+.PHONY: test test-cov-html bench bench-cpu bench-cpu-long bin proto release service
 .DEFAULT_GOAL := bin

README.md

@@ -1,6 +1,7 @@
 ## What is Nebula?
 Nebula is a scalable overlay networking tool with a focus on performance, simplicity and security.
-It lets you seamlessly connect computers anywhere in the world. Nebula is portable, and runs on Linux, OSX, Windows, iOS, and Android.
+It lets you seamlessly connect computers anywhere in the world. Nebula is portable, and runs on Linux, OSX, and Windows.
+(Also: keep this quiet, but we have an early prototype running on iOS).
 It can be used to connect a small number of computers, but is also able to connect tens of thousands of computers.
 
 Nebula incorporates a number of existing concepts like encryption, security groups, certificates,
@@ -12,22 +13,6 @@ You can read more about Nebula [here](https://medium.com/p/884110a5579).
 
 You can also join the NebulaOSS Slack group [here](https://join.slack.com/t/nebulaoss/shared_invite/enQtOTA5MDI4NDg3MTg4LTkwY2EwNTI4NzQyMzc0M2ZlODBjNWI3NTY1MzhiOThiMmZlZjVkMTI0NGY4YTMyNjUwMWEyNzNkZTJmYzQxOGU)
 
-## Supported Platforms
-
-#### Desktop and Server
-
-Check the [releases](https://github.com/slackhq/nebula/releases/latest) page for downloads
-
-- Linux - 64 and 32 bit, arm, and others 
-- Windows
-- MacOS
-- Freebsd
-
-#### Mobile
-
-- [iOS](https://apps.apple.com/us/app/mobile-nebula/id1509587936?itsct=apps_box&itscg=30200)
-- [Android](https://play.google.com/store/apps/details?id=net.defined.mobile_nebula&pcampaignid=pcampaignidMKT-Other-global-all-co-prtnr-py-PartBadge-Mar2515-1)
-
 ## Technical Overview
 
 Nebula is a mutually authenticated peer-to-peer software defined network based on the [Noise Protocol Framework](https://noiseprotocol.org/).

cert/cert.go

@@ -61,10 +61,6 @@ func UnmarshalNebulaCertificate(b []byte) (*NebulaCertificate, error) {
 		return nil, err
 	}
 
-	if rc.Details == nil {
-		return nil, fmt.Errorf("encoded Details was nil")
-	}
-
 	if len(rc.Details.Ips)%2 != 0 {
 		return nil, fmt.Errorf("encoded IPs should be in pairs, an odd number was found")
 	}
@@ -127,9 +123,6 @@ func UnmarshalNebulaCertificateFromPEM(b []byte) (*NebulaCertificate, []byte, er
 	if p == nil {
 		return nil, r, fmt.Errorf("input did not contain a valid PEM encoded block")
 	}
-	if p.Type != CertBanner {
-		return nil, r, fmt.Errorf("bytes did not contain a proper nebula certificate banner")
-	}
 	nc, err := UnmarshalNebulaCertificate(p.Bytes)
 	return nc, r, err
 }

cert/cert_test.go

@@ -447,255 +447,6 @@ BVG+oJpAoqokUBbI4U0N8CSfpUABEkB/Pm5A2xyH/nc8mg/wvGUWG3pZ7nHzaDMf
 	assert.Equal(t, pp.CAs[string("5c9c3f23e7ee7fe97637cbd3a0a5b854154d1d9aaaf7b566a51f4a88f76b64cd")].Details.Name, rootCA01.Details.Name)
 }
 
-func appendByteSlices(b ...[]byte) []byte {
-	retSlice := []byte{}
-	for _, v := range b {
-		retSlice = append(retSlice, v...)
-	}
-	return retSlice
-}
-
-func TestUnmrshalCertPEM(t *testing.T) {
-	goodCert := []byte(`
-# A good cert
------BEGIN NEBULA CERTIFICATE-----
-CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL
-vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv
-bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB
------END NEBULA CERTIFICATE-----
-`)
-	badBanner := []byte(`# A bad banner
------BEGIN NOT A NEBULA CERTIFICATE-----
-CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL
-vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv
-bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB
------END NOT A NEBULA CERTIFICATE-----
-`)
-	invalidPem := []byte(`# Not a valid PEM format
--BEGIN NEBULA CERTIFICATE-----
-CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL
-vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv
-bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB
--END NEBULA CERTIFICATE----`)
-
-	certBundle := appendByteSlices(goodCert, badBanner, invalidPem)
-
-	// Success test case
-	cert, rest, err := UnmarshalNebulaCertificateFromPEM(certBundle)
-	assert.NotNil(t, cert)
-	assert.Equal(t, rest, append(badBanner, invalidPem...))
-	assert.Nil(t, err)
-
-	// Fail due to invalid banner.
-	cert, rest, err = UnmarshalNebulaCertificateFromPEM(rest)
-	assert.Nil(t, cert)
-	assert.Equal(t, rest, invalidPem)
-	assert.EqualError(t, err, "bytes did not contain a proper nebula certificate banner")
-
-	// Fail due to ivalid PEM format, because
-	// it's missing the requisite pre-encapsulation boundary.
-	cert, rest, err = UnmarshalNebulaCertificateFromPEM(rest)
-	assert.Nil(t, cert)
-	assert.Equal(t, rest, invalidPem)
-	assert.EqualError(t, err, "input did not contain a valid PEM encoded block")
-}
-
-func TestUnmarshalEd25519PrivateKey(t *testing.T) {
-	privKey := []byte(`# A good key
------BEGIN NEBULA ED25519 PRIVATE KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
------END NEBULA ED25519 PRIVATE KEY-----
-`)
-	shortKey := []byte(`# A short key
------BEGIN NEBULA ED25519 PRIVATE KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
------END NEBULA ED25519 PRIVATE KEY-----
-`)
-	invalidBanner := []byte(`# Invalid banner
------BEGIN NOT A NEBULA PRIVATE KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
------END NOT A NEBULA PRIVATE KEY-----
-`)
-	invalidPem := []byte(`# Not a valid PEM format
--BEGIN NEBULA ED25519 PRIVATE KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
--END NEBULA ED25519 PRIVATE KEY-----`)
-
-	keyBundle := appendByteSlices(privKey, shortKey, invalidBanner, invalidPem)
-
-	// Success test case
-	k, rest, err := UnmarshalEd25519PrivateKey(keyBundle)
-	assert.Len(t, k, 64)
-	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
-	assert.Nil(t, err)
-
-	// Fail due to short key
-	k, rest, err = UnmarshalEd25519PrivateKey(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem))
-	assert.EqualError(t, err, "key was not 64 bytes, is invalid ed25519 private key")
-
-	// Fail due to invalid banner
-	k, rest, err = UnmarshalEd25519PrivateKey(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, invalidPem)
-	assert.EqualError(t, err, "bytes did not contain a proper nebula Ed25519 private key banner")
-
-	// Fail due to ivalid PEM format, because
-	// it's missing the requisite pre-encapsulation boundary.
-	k, rest, err = UnmarshalEd25519PrivateKey(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, invalidPem)
-	assert.EqualError(t, err, "input did not contain a valid PEM encoded block")
-}
-
-func TestUnmarshalX25519PrivateKey(t *testing.T) {
-	privKey := []byte(`# A good key
------BEGIN NEBULA X25519 PRIVATE KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA X25519 PRIVATE KEY-----
-`)
-	shortKey := []byte(`# A short key
------BEGIN NEBULA X25519 PRIVATE KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
------END NEBULA X25519 PRIVATE KEY-----
-`)
-	invalidBanner := []byte(`# Invalid banner
------BEGIN NOT A NEBULA PRIVATE KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
------END NOT A NEBULA PRIVATE KEY-----
-`)
-	invalidPem := []byte(`# Not a valid PEM format
--BEGIN NEBULA X25519 PRIVATE KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
--END NEBULA X25519 PRIVATE KEY-----`)
-
-	keyBundle := appendByteSlices(privKey, shortKey, invalidBanner, invalidPem)
-
-	// Success test case
-	k, rest, err := UnmarshalX25519PrivateKey(keyBundle)
-	assert.Len(t, k, 32)
-	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
-	assert.Nil(t, err)
-
-	// Fail due to short key
-	k, rest, err = UnmarshalX25519PrivateKey(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem))
-	assert.EqualError(t, err, "key was not 32 bytes, is invalid X25519 private key")
-
-	// Fail due to invalid banner
-	k, rest, err = UnmarshalX25519PrivateKey(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, invalidPem)
-	assert.EqualError(t, err, "bytes did not contain a proper nebula X25519 private key banner")
-
-	// Fail due to ivalid PEM format, because
-	// it's missing the requisite pre-encapsulation boundary.
-	k, rest, err = UnmarshalX25519PrivateKey(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, invalidPem)
-	assert.EqualError(t, err, "input did not contain a valid PEM encoded block")
-}
-
-func TestUnmarshalEd25519PublicKey(t *testing.T) {
-	pubKey := []byte(`# A good key
------BEGIN NEBULA ED25519 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA ED25519 PUBLIC KEY-----
-`)
-	shortKey := []byte(`# A short key
------BEGIN NEBULA ED25519 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
------END NEBULA ED25519 PUBLIC KEY-----
-`)
-	invalidBanner := []byte(`# Invalid banner
------BEGIN NOT A NEBULA PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
------END NOT A NEBULA PUBLIC KEY-----
-`)
-	invalidPem := []byte(`# Not a valid PEM format
--BEGIN NEBULA ED25519 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
--END NEBULA ED25519 PUBLIC KEY-----`)
-
-	keyBundle := appendByteSlices(pubKey, shortKey, invalidBanner, invalidPem)
-
-	// Success test case
-	k, rest, err := UnmarshalEd25519PublicKey(keyBundle)
-	assert.Equal(t, len(k), 32)
-	assert.Nil(t, err)
-	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
-
-	// Fail due to short key
-	k, rest, err = UnmarshalEd25519PublicKey(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem))
-	assert.EqualError(t, err, "key was not 32 bytes, is invalid ed25519 public key")
-
-	// Fail due to invalid banner
-	k, rest, err = UnmarshalEd25519PublicKey(rest)
-	assert.Nil(t, k)
-	assert.EqualError(t, err, "bytes did not contain a proper nebula Ed25519 public key banner")
-	assert.Equal(t, rest, invalidPem)
-
-	// Fail due to ivalid PEM format, because
-	// it's missing the requisite pre-encapsulation boundary.
-	k, rest, err = UnmarshalEd25519PublicKey(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, invalidPem)
-	assert.EqualError(t, err, "input did not contain a valid PEM encoded block")
-}
-
-func TestUnmarshalX25519PublicKey(t *testing.T) {
-	pubKey := []byte(`# A good key
------BEGIN NEBULA X25519 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA X25519 PUBLIC KEY-----
-`)
-	shortKey := []byte(`# A short key
------BEGIN NEBULA X25519 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
------END NEBULA X25519 PUBLIC KEY-----
-`)
-	invalidBanner := []byte(`# Invalid banner
------BEGIN NOT A NEBULA PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
------END NOT A NEBULA PUBLIC KEY-----
-`)
-	invalidPem := []byte(`# Not a valid PEM format
--BEGIN NEBULA X25519 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
--END NEBULA X25519 PUBLIC KEY-----`)
-
-	keyBundle := appendByteSlices(pubKey, shortKey, invalidBanner, invalidPem)
-
-	// Success test case
-	k, rest, err := UnmarshalX25519PublicKey(keyBundle)
-	assert.Equal(t, len(k), 32)
-	assert.Nil(t, err)
-	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
-
-	// Fail due to short key
-	k, rest, err = UnmarshalX25519PublicKey(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem))
-	assert.EqualError(t, err, "key was not 32 bytes, is invalid X25519 public key")
-
-	// Fail due to invalid banner
-	k, rest, err = UnmarshalX25519PublicKey(rest)
-	assert.Nil(t, k)
-	assert.EqualError(t, err, "bytes did not contain a proper nebula X25519 public key banner")
-	assert.Equal(t, rest, invalidPem)
-
-	// Fail due to ivalid PEM format, because
-	// it's missing the requisite pre-encapsulation boundary.
-	k, rest, err = UnmarshalX25519PublicKey(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, invalidPem)
-	assert.EqualError(t, err, "input did not contain a valid PEM encoded block")
-}
-
 // Ensure that upgrading the protobuf library does not change how certificates
 // are marshalled, since this would break signature verification
 func TestMarshalingNebulaCertificateConsistency(t *testing.T) {
@@ -748,13 +499,6 @@ func TestNebulaCertificate_Copy(t *testing.T) {
 	util.AssertDeepCopyEqual(t, c, cc)
 }
 
-func TestUnmarshalNebulaCertificate(t *testing.T) {
-	// Test that we don't panic with an invalid certificate (#332)
-	data := []byte("\x98\x00\x00")
-	_, err := UnmarshalNebulaCertificate(data)
-	assert.EqualError(t, err, "encoded Details was nil")
-}
-
 func newTestCaCert(before, after time.Time, ips, subnets []*net.IPNet, groups []string) (*NebulaCertificate, []byte, []byte, error) {
 	pub, priv, err := ed25519.GenerateKey(rand.Reader)
 	if before.IsZero() {

cmd/nebula-cert/ca.go

@@ -11,7 +11,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/skip2/go-qrcode"
 	"github.com/slackhq/nebula/cert"
 	"golang.org/x/crypto/ed25519"
 )
@@ -22,7 +21,6 @@ type caFlags struct {
 	duration    *time.Duration
 	outKeyPath  *string
 	outCertPath *string
-	outQRPath   *string
 	groups      *string
 	ips         *string
 	subnets     *string
@@ -35,7 +33,6 @@ func newCaFlags() *caFlags {
 	cf.duration = cf.set.Duration("duration", time.Duration(time.Hour*8760), "Optional: amount of time the certificate should be valid for. Valid time units are seconds: \"s\", minutes: \"m\", hours: \"h\"")
 	cf.outKeyPath = cf.set.String("out-key", "ca.key", "Optional: path to write the private key to")
 	cf.outCertPath = cf.set.String("out-crt", "ca.crt", "Optional: path to write the certificate to")
-	cf.outQRPath = cf.set.String("out-qr", "", "Optional: output a qr code image (png) of the certificate")
 	cf.groups = cf.set.String("groups", "", "Optional: comma separated list of groups. This will limit which groups subordinate certs can use")
 	cf.ips = cf.set.String("ips", "", "Optional: comma separated list of ip and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use")
 	cf.subnets = cf.set.String("subnets", "", "Optional: comma separated list of ip and network in CIDR notation. This will limit which subnet addresses and networks subordinate certs can use")
@@ -149,18 +146,6 @@ func ca(args []string, out io.Writer, errOut io.Writer) error {
 		return fmt.Errorf("error while writing out-crt: %s", err)
 	}
 
-	if *cf.outQRPath != "" {
-		b, err = qrcode.Encode(string(b), qrcode.Medium, -5)
-		if err != nil {
-			return fmt.Errorf("error while generating qr code: %s", err)
-		}
-
-		err = ioutil.WriteFile(*cf.outQRPath, b, 0600)
-		if err != nil {
-			return fmt.Errorf("error while writing out-qr: %s", err)
-		}
-	}
-
 	return nil
 }
 

cmd/nebula-cert/ca_test.go

@@ -37,8 +37,6 @@ func Test_caHelp(t *testing.T) {
 			"    \tOptional: path to write the certificate to (default \"ca.crt\")\n"+
 			"  -out-key string\n"+
 			"    \tOptional: path to write the private key to (default \"ca.key\")\n"+
-			"  -out-qr string\n"+
-			"    \tOptional: output a qr code image (png) of the certificate\n"+
 			"  -subnets string\n"+
 			"    \tOptional: comma separated list of ip and network in CIDR notation. This will limit which subnet addresses and networks subordinate certs can use\n",
 		ob.String(),

cmd/nebula-cert/print.go

@@ -9,22 +9,19 @@ import (
 	"os"
 	"strings"
 
-	"github.com/skip2/go-qrcode"
 	"github.com/slackhq/nebula/cert"
 )
 
 type printFlags struct {
-	set       *flag.FlagSet
-	json      *bool
-	outQRPath *string
-	path      *string
+	set  *flag.FlagSet
+	json *bool
+	path *string
 }
 
 func newPrintFlags() *printFlags {
 	pf := printFlags{set: flag.NewFlagSet("print", flag.ContinueOnError)}
 	pf.set.Usage = func() {}
 	pf.json = pf.set.Bool("json", false, "Optional: outputs certificates in json format")
-	pf.outQRPath = pf.set.String("out-qr", "", "Optional: output a qr code image (png) of the certificate")
 	pf.path = pf.set.String("path", "", "Required: path to the certificate")
 
 	return &pf
@@ -47,8 +44,6 @@ func printCert(args []string, out io.Writer, errOut io.Writer) error {
 	}
 
 	var c *cert.NebulaCertificate
-	var qrBytes []byte
-	part := 0
 
 	for {
 		c, rawCert, err = cert.UnmarshalNebulaCertificateFromPEM(rawCert)
@@ -66,31 +61,9 @@ func printCert(args []string, out io.Writer, errOut io.Writer) error {
 			out.Write([]byte("\n"))
 		}
 
-		if *pf.outQRPath != "" {
-			b, err := c.MarshalToPEM()
-			if err != nil {
-				return fmt.Errorf("error while marshalling cert to PEM: %s", err)
-			}
-			qrBytes = append(qrBytes, b...)
-		}
-
 		if rawCert == nil || len(rawCert) == 0 || strings.TrimSpace(string(rawCert)) == "" {
 			break
 		}
-
-		part++
-	}
-
-	if *pf.outQRPath != "" {
-		b, err := qrcode.Encode(string(qrBytes), qrcode.Medium, -5)
-		if err != nil {
-			return fmt.Errorf("error while generating qr code: %s", err)
-		}
-
-		err = ioutil.WriteFile(*pf.outQRPath, b, 0600)
-		if err != nil {
-			return fmt.Errorf("error while writing out-qr: %s", err)
-		}
 	}
 
 	return nil

cmd/nebula-cert/print_test.go

@@ -23,8 +23,6 @@ func Test_printHelp(t *testing.T) {
 		"Usage of "+os.Args[0]+" print <flags>: prints details about a certificate\n"+
 			"  -json\n"+
 			"    \tOptional: outputs certificates in json format\n"+
-			"  -out-qr string\n"+
-			"    \tOptional: output a qr code image (png) of the certificate\n"+
 			"  -path string\n"+
 			"    \tRequired: path to the certificate\n",
 		ob.String(),

cmd/nebula-cert/sign.go

@@ -11,7 +11,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/skip2/go-qrcode"
 	"github.com/slackhq/nebula/cert"
 	"golang.org/x/crypto/curve25519"
 )
@@ -26,7 +25,6 @@ type signFlags struct {
 	inPubPath   *string
 	outKeyPath  *string
 	outCertPath *string
-	outQRPath   *string
 	groups      *string
 	subnets     *string
 }
@@ -42,9 +40,8 @@ func newSignFlags() *signFlags {
 	sf.inPubPath = sf.set.String("in-pub", "", "Optional (if out-key not set): path to read a previously generated public key")
 	sf.outKeyPath = sf.set.String("out-key", "", "Optional (if in-pub not set): path to write the private key to")
 	sf.outCertPath = sf.set.String("out-crt", "", "Optional: path to write the certificate to")
-	sf.outQRPath = sf.set.String("out-qr", "", "Optional: output a qr code image (png) of the certificate")
 	sf.groups = sf.set.String("groups", "", "Optional: comma separated list of groups")
-	sf.subnets = sf.set.String("subnets", "", "Optional: comma separated list of subnet this cert can serve for")
+	sf.subnets = sf.set.String("subnets", "", "Optional: comma seperated list of subnet this cert can serve for")
 	return &sf
 
 }
@@ -206,18 +203,6 @@ func signCert(args []string, out io.Writer, errOut io.Writer) error {
 		return fmt.Errorf("error while writing out-crt: %s", err)
 	}
 
-	if *sf.outQRPath != "" {
-		b, err = qrcode.Encode(string(b), qrcode.Medium, -5)
-		if err != nil {
-			return fmt.Errorf("error while generating qr code: %s", err)
-		}
-
-		err = ioutil.WriteFile(*sf.outQRPath, b, 0600)
-		if err != nil {
-			return fmt.Errorf("error while writing out-qr: %s", err)
-		}
-	}
-
 	return nil
 }
 

cmd/nebula-cert/sign_test.go

@@ -45,10 +45,8 @@ func Test_signHelp(t *testing.T) {
 			"    \tOptional: path to write the certificate to\n"+
 			"  -out-key string\n"+
 			"    \tOptional (if in-pub not set): path to write the private key to\n"+
-			"  -out-qr string\n"+
-			"    \tOptional: output a qr code image (png) of the certificate\n"+
 			"  -subnets string\n"+
-			"    \tOptional: comma separated list of subnet this cert can serve for\n",
+			"    \tOptional: comma seperated list of subnet this cert can serve for\n",
 		ob.String(),
 	)
 }
@@ -288,4 +286,5 @@ func Test_signCert(t *testing.T) {
 	assert.EqualError(t, signCert(args, ob, eb), "refusing to overwrite existing cert: "+crtF.Name())
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
+
 }

connection_manager.go

@@ -141,17 +141,14 @@ func (n *connectionManager) Start() {
 
 func (n *connectionManager) Run() {
 	clockSource := time.Tick(500 * time.Millisecond)
-	p := []byte("")
-	nb := make([]byte, 12, 12)
-	out := make([]byte, mtu)
 
 	for now := range clockSource {
-		n.HandleMonitorTick(now, p, nb, out)
+		n.HandleMonitorTick(now)
 		n.HandleDeletionTick(now)
 	}
 }
 
-func (n *connectionManager) HandleMonitorTick(now time.Time, p, nb, out []byte) {
+func (n *connectionManager) HandleMonitorTick(now time.Time) {
 	n.TrafficTimer.advance(now)
 	for {
 		ep := n.TrafficTimer.Purge()
@@ -191,7 +188,7 @@ func (n *connectionManager) HandleMonitorTick(now time.Time, p, nb, out []byte)
 
 		if hostinfo != nil && hostinfo.ConnectionState != nil {
 			// Send a test packet to trigger an authenticated tunnel test, this should suss out any lingering tunnel issues
-			n.intf.SendMessageToVpnIp(test, testRequest, vpnIP, p, nb, out)
+			n.intf.SendMessageToVpnIp(test, testRequest, vpnIP, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
 
 		} else {
 			hostinfo.logger().Debugf("Hostinfo sadness: %s", IntIp(vpnIP))
@@ -247,7 +244,8 @@ func (n *connectionManager) HandleDeletionTick(now time.Time) {
 			if n.intf.lightHouse != nil {
 				n.intf.lightHouse.DeleteVpnIP(vpnIP)
 			}
-			n.hostMap.DeleteHostInfo(hostinfo)
+			n.hostMap.DeleteVpnIP(vpnIP)
+			n.hostMap.DeleteIndex(hostinfo.localIndexId)
 		} else {
 			n.ClearIP(vpnIP)
 			n.ClearPendingDeletion(vpnIP)

connection_manager_test.go

@@ -42,15 +42,13 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 
 	// Create manager
 	nc := newConnectionManager(ifce, 5, 10)
-	p := []byte("")
-	nb := make([]byte, 12, 12)
-	out := make([]byte, mtu)
-	nc.HandleMonitorTick(now, p, nb, out)
+	nc.HandleMonitorTick(now)
 	// Add an ip we have established a connection w/ to hostmap
 	hostinfo := nc.hostMap.AddVpnIP(vpnIP)
 	hostinfo.ConnectionState = &ConnectionState{
-		certState: cs,
-		H:         &noise.HandshakeState{},
+		certState:      cs,
+		H:              &noise.HandshakeState{},
+		messageCounter: new(uint64),
 	}
 
 	// We saw traffic out to vpnIP
@@ -59,18 +57,18 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	assert.Contains(t, nc.hostMap.Hosts, vpnIP)
 	// Move ahead 5s. Nothing should happen
 	next_tick := now.Add(5 * time.Second)
-	nc.HandleMonitorTick(next_tick, p, nb, out)
+	nc.HandleMonitorTick(next_tick)
 	nc.HandleDeletionTick(next_tick)
 	// Move ahead 6s. We haven't heard back
 	next_tick = now.Add(6 * time.Second)
-	nc.HandleMonitorTick(next_tick, p, nb, out)
+	nc.HandleMonitorTick(next_tick)
 	nc.HandleDeletionTick(next_tick)
 	// This host should now be up for deletion
 	assert.Contains(t, nc.pendingDeletion, vpnIP)
 	assert.Contains(t, nc.hostMap.Hosts, vpnIP)
 	// Move ahead some more
 	next_tick = now.Add(45 * time.Second)
-	nc.HandleMonitorTick(next_tick, p, nb, out)
+	nc.HandleMonitorTick(next_tick)
 	nc.HandleDeletionTick(next_tick)
 	// The host should be evicted
 	assert.NotContains(t, nc.pendingDeletion, vpnIP)
@@ -107,15 +105,13 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 
 	// Create manager
 	nc := newConnectionManager(ifce, 5, 10)
-	p := []byte("")
-	nb := make([]byte, 12, 12)
-	out := make([]byte, mtu)
-	nc.HandleMonitorTick(now, p, nb, out)
+	nc.HandleMonitorTick(now)
 	// Add an ip we have established a connection w/ to hostmap
 	hostinfo := nc.hostMap.AddVpnIP(vpnIP)
 	hostinfo.ConnectionState = &ConnectionState{
-		certState: cs,
-		H:         &noise.HandshakeState{},
+		certState:      cs,
+		H:              &noise.HandshakeState{},
+		messageCounter: new(uint64),
 	}
 
 	// We saw traffic out to vpnIP
@@ -124,11 +120,11 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	assert.Contains(t, nc.hostMap.Hosts, vpnIP)
 	// Move ahead 5s. Nothing should happen
 	next_tick := now.Add(5 * time.Second)
-	nc.HandleMonitorTick(next_tick, p, nb, out)
+	nc.HandleMonitorTick(next_tick)
 	nc.HandleDeletionTick(next_tick)
 	// Move ahead 6s. We haven't heard back
 	next_tick = now.Add(6 * time.Second)
-	nc.HandleMonitorTick(next_tick, p, nb, out)
+	nc.HandleMonitorTick(next_tick)
 	nc.HandleDeletionTick(next_tick)
 	// This host should now be up for deletion
 	assert.Contains(t, nc.pendingDeletion, vpnIP)
@@ -137,7 +133,7 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	nc.In(vpnIP)
 	// Move ahead some more
 	next_tick = now.Add(45 * time.Second)
-	nc.HandleMonitorTick(next_tick, p, nb, out)
+	nc.HandleMonitorTick(next_tick)
 	nc.HandleDeletionTick(next_tick)
 	// The host should be evicted
 	assert.NotContains(t, nc.pendingDeletion, vpnIP)

connection_state.go

@@ -4,7 +4,6 @@ import (
 	"crypto/rand"
 	"encoding/json"
 	"sync"
-	"sync/atomic"
 
 	"github.com/flynn/noise"
 	"github.com/slackhq/nebula/cert"
@@ -13,17 +12,17 @@ import (
 const ReplayWindow = 1024
 
 type ConnectionState struct {
-	eKey                 *NebulaCipherState
-	dKey                 *NebulaCipherState
-	H                    *noise.HandshakeState
-	certState            *CertState
-	peerCert             *cert.NebulaCertificate
-	initiator            bool
-	atomicMessageCounter uint64
-	window               *Bits
-	queueLock            sync.Mutex
-	writeLock            sync.Mutex
-	ready                bool
+	eKey           *NebulaCipherState
+	dKey           *NebulaCipherState
+	H              *noise.HandshakeState
+	certState      *CertState
+	peerCert       *cert.NebulaCertificate
+	initiator      bool
+	messageCounter *uint64
+	window         *Bits
+	queueLock      sync.Mutex
+	writeLock      sync.Mutex
+	ready          bool
 }
 
 func (f *Interface) newConnectionState(initiator bool, pattern noise.HandshakePattern, psk []byte, pskStage int) *ConnectionState {
@@ -55,11 +54,12 @@ func (f *Interface) newConnectionState(initiator bool, pattern noise.HandshakePa
 	// The queue and ready params prevent a counter race that would happen when
 	// sending stored packets and simultaneously accepting new traffic.
 	ci := &ConnectionState{
-		H:         hs,
-		initiator: initiator,
-		window:    b,
-		ready:     false,
-		certState: curCertState,
+		H:              hs,
+		initiator:      initiator,
+		window:         b,
+		ready:          false,
+		certState:      curCertState,
+		messageCounter: new(uint64),
 	}
 
 	return ci
@@ -69,7 +69,7 @@ func (cs *ConnectionState) MarshalJSON() ([]byte, error) {
 	return json.Marshal(m{
 		"certificate":     cs.peerCert,
 		"initiator":       cs.initiator,
-		"message_counter": atomic.LoadUint64(&cs.atomicMessageCounter),
+		"message_counter": cs.messageCounter,
 		"ready":           cs.ready,
 	})
 }

control.go

@@ -1,14 +1,16 @@
 package nebula
 
 import (
+	"encoding/binary"
+	"fmt"
 	"net"
 	"os"
 	"os/signal"
-	"sync/atomic"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
+	"golang.org/x/net/ipv4"
 )
 
 // Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching
@@ -66,12 +68,6 @@ func (c *Control) ShutdownBlock() {
 // RebindUDPServer asks the UDP listener to rebind it's listener. Mainly used on mobile clients when interfaces change
 func (c *Control) RebindUDPServer() {
 	_ = c.f.outside.Rebind()
-
-	// Trigger a lighthouse update, useful for mobile clients that should have an update interval of 0
-	c.f.lightHouse.SendUpdate(c.f)
-
-	// Let the main interface know that we rebound so that underlying tunnels know to trigger punches from their remotes
-	c.f.rebindCount++
 }
 
 // ListHostmap returns details about the actual or pending (handshaking) hostmap
@@ -157,7 +153,7 @@ func copyHostInfo(h *HostInfo) ControlHostInfo {
 		RemoteIndex:    h.remoteIndexId,
 		RemoteAddrs:    make([]udpAddr, len(addrs), len(addrs)),
 		CachedPackets:  len(h.packetStore),
-		MessageCounter: atomic.LoadUint64(&h.ConnectionState.atomicMessageCounter),
+		MessageCounter: *h.ConnectionState.messageCounter,
 	}
 
 	if c := h.GetCert(); c != nil {
@@ -174,3 +170,47 @@ func copyHostInfo(h *HostInfo) ControlHostInfo {
 
 	return chi
 }
+
+// Hook provides the ability to hook into the network path for a particular
+// message sub type. Any received message of that subtype that is allowed by
+// the firewall will be written to the provided write func instead of the
+// inside interface.
+// TODO: make this an io.Writer
+func (c *Control) Hook(t NebulaMessageSubType, w func([]byte) error) error {
+	if t == 0 {
+		return fmt.Errorf("non-default message subtype must be specified")
+	}
+	if _, ok := c.f.handlers[Version][message][t]; ok {
+		return fmt.Errorf("message subtype %d already hooked", t)
+	}
+
+	c.f.handlers[Version][message][t] = c.f.newHook(w)
+	return nil
+}
+
+// Send provides the ability to send arbitrary message packets to peer nodes.
+// The provided payload will be encapsulated in a Nebula Firewall packet
+// (IPv4 plus ports) from the node IP to the provided destination nebula IP.
+// Any protocol handling above layer 3 (IP) must be managed by the caller.
+func (c *Control) Send(ip uint32, port uint16, st NebulaMessageSubType, payload []byte) {
+	headerLen := ipv4.HeaderLen + minFwPacketLen
+	length := headerLen + len(payload)
+	packet := make([]byte, length)
+	packet[0] = 0x45 // IPv4 HL=20
+	packet[9] = 114  // Declare as arbitrary 0-hop protocol
+	binary.BigEndian.PutUint16(packet[2:4], uint16(length))
+	binary.BigEndian.PutUint32(packet[12:16], ip2int(c.f.inside.CidrNet().IP.To4()))
+	binary.BigEndian.PutUint32(packet[16:20], ip)
+
+	// Set identical values for src and dst port as they're only
+	// used for nebula firewall rule/conntrack matching.
+	binary.BigEndian.PutUint16(packet[20:22], port)
+	binary.BigEndian.PutUint16(packet[22:24], port)
+
+	copy(packet[headerLen:], payload)
+
+	fp := &FirewallPacket{}
+	nb := make([]byte, 12)
+	out := make([]byte, mtu)
+	c.f.consumeInsidePacket(st, packet, fp, nb, out)
+}

control_test.go

@@ -43,13 +43,15 @@ func TestControl_GetHostInfoByVpnIP(t *testing.T) {
 		},
 		Signature: []byte{1, 2, 1, 2, 1, 3},
 	}
+	counter := uint64(0)
 
 	remotes := []*HostInfoDest{NewHostInfoDest(remote1), NewHostInfoDest(remote2)}
 	hm.Add(ip2int(ipNet.IP), &HostInfo{
 		remote:  remote1,
 		Remotes: remotes,
 		ConnectionState: &ConnectionState{
-			peerCert: crt,
+			peerCert:       crt,
+			messageCounter: &counter,
 		},
 		remoteIndexId: 200,
 		localIndexId:  201,
@@ -60,7 +62,8 @@ func TestControl_GetHostInfoByVpnIP(t *testing.T) {
 		remote:  remote1,
 		Remotes: remotes,
 		ConnectionState: &ConnectionState{
-			peerCert: nil,
+			peerCert:       nil,
+			messageCounter: &counter,
 		},
 		remoteIndexId: 200,
 		localIndexId:  201,

examples/config.yml

@@ -86,15 +86,6 @@ listen:
   #read_buffer: 10485760
   #write_buffer: 10485760
 
-# EXPERIMENTAL: This option is currently only supported on linux and may
-# change in future minor releases.
-#
-# Routines is the number of thread pairs to run that consume from the tun and UDP queues.
-# Currently, this defaults to 1 which means we have 1 tun queue reader and 1
-# UDP queue reader. Setting this above one will set IFF_MULTI_QUEUE on the tun
-# device and SO_REUSEPORT on the UDP socket to allow multiple queues.
-#routines: 1
-
 punchy:
   # Continues to punch inbound/outbound at a regular interval to avoid expiration of firewall nat mappings
   punch: true
@@ -107,7 +98,7 @@ punchy:
   # delays a punch response for misbehaving NATs, default is 1 second, respond must be true to take effect
   #delay: 1s
 
-# Cipher allows you to choose between the available ciphers for your network. Options are chachapoly or aes
+# Cipher allows you to choose between the available ciphers for your network.
 # IMPORTANT: this value must be identical on ALL NODES/LIGHTHOUSES. We do not/will not support use of different ciphers simultaneously!
 #cipher: chachapoly
 

examples/service_scripts/nebula.service

@@ -2,7 +2,6 @@
 Description=nebula
 Wants=basic.target
 After=basic.target network.target
-Before=sshd.service
 
 [Service]
 SyslogIdentifier=nebula

firewall.go

@@ -12,7 +12,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/rcrowley/go-metrics"
@@ -373,9 +372,9 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
 
 // Drop returns an error if the packet should be dropped, explaining why. It
 // returns nil if the packet should not be dropped.
-func (f *Firewall) Drop(packet []byte, fp FirewallPacket, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache ConntrackCache) error {
+func (f *Firewall) Drop(packet []byte, fp FirewallPacket, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool) error {
 	// Check if we spoke to this tuple, if we did then allow this packet
-	if f.inConns(packet, fp, incoming, h, caPool, localCache) {
+	if f.inConns(packet, fp, incoming, h, caPool) {
 		return nil
 	}
 
@@ -427,12 +426,7 @@ func (f *Firewall) EmitStats() {
 	metrics.GetOrRegisterGauge("firewall.rules.version", nil).Update(int64(f.rulesVersion))
 }
 
-func (f *Firewall) inConns(packet []byte, fp FirewallPacket, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache ConntrackCache) bool {
-	if localCache != nil {
-		if _, ok := localCache[fp]; ok {
-			return true
-		}
-	}
+func (f *Firewall) inConns(packet []byte, fp FirewallPacket, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool) bool {
 	conntrack := f.Conntrack
 	conntrack.Lock()
 
@@ -500,10 +494,6 @@ func (f *Firewall) inConns(packet []byte, fp FirewallPacket, incoming bool, h *H
 
 	conntrack.Unlock()
 
-	if localCache != nil {
-		localCache[fp] = struct{}{}
-	}
-
 	return true
 }
 
@@ -933,54 +923,3 @@ func (f *Firewall) checkTCPRTT(c *conn, p []byte) bool {
 	c.Seq = 0
 	return true
 }
-
-// ConntrackCache is used as a local routine cache to know if a given flow
-// has been seen in the conntrack table.
-type ConntrackCache map[FirewallPacket]struct{}
-
-type ConntrackCacheTicker struct {
-	cacheV    uint64
-	cacheTick uint64
-
-	cache ConntrackCache
-}
-
-func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
-	if d == 0 {
-		return nil
-	}
-
-	c := &ConntrackCacheTicker{
-		cache: ConntrackCache{},
-	}
-
-	go c.tick(d)
-
-	return c
-}
-
-func (c *ConntrackCacheTicker) tick(d time.Duration) {
-	for {
-		time.Sleep(d)
-		atomic.AddUint64(&c.cacheTick, 1)
-	}
-}
-
-// Get checks if the cache ticker has moved to the next version before returning
-// the map. If it has moved, we reset the map.
-func (c *ConntrackCacheTicker) Get() ConntrackCache {
-	if c == nil {
-		return nil
-	}
-	if tick := atomic.LoadUint64(&c.cacheTick); tick != c.cacheV {
-		c.cacheV = tick
-		if ll := len(c.cache); ll > 0 {
-			if l.GetLevel() == logrus.DebugLevel {
-				l.WithField("len", ll).Debug("resetting conntrack cache")
-			}
-			c.cache = make(ConntrackCache, ll)
-		}
-	}
-
-	return c.cache
-}

firewall_test.go

@@ -182,44 +182,44 @@ func TestFirewall_Drop(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp), ErrNoMatchingRule)
 	// Allow inbound
 	resetConntrack(fw)
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp))
 	// Allow outbound because conntrack
-	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil))
+	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp))
 
 	// test remote mismatch
 	oldRemote := p.RemoteIP
 	p.RemoteIP = ip2int(net.IPv4(1, 2, 3, 10))
-	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp), ErrInvalidRemoteIP)
 	p.RemoteIP = oldRemote
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(time.Second, time.Minute, time.Hour, &c)
 	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"nope"}, "", nil, "", "signer-shasum"))
 	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"default-group"}, "", nil, "", "signer-shasum-bad"))
-	assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(time.Second, time.Minute, time.Hour, &c)
 	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"nope"}, "", nil, "", "signer-shasum-bad"))
 	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"default-group"}, "", nil, "", "signer-shasum"))
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.NebulaCertificate{Details: cert.NebulaCertificateDetails{Name: "ca-good"}}
 	fw = NewFirewall(time.Second, time.Minute, time.Hour, &c)
 	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"nope"}, "", nil, "ca-good", ""))
 	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"default-group"}, "", nil, "ca-good-bad", ""))
-	assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.NebulaCertificate{Details: cert.NebulaCertificateDetails{Name: "ca-good"}}
 	fw = NewFirewall(time.Second, time.Minute, time.Hour, &c)
 	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"nope"}, "", nil, "ca-good-bad", ""))
 	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"default-group"}, "", nil, "ca-good", ""))
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp))
 }
 
 func BenchmarkFirewallTable_match(b *testing.B) {
@@ -370,10 +370,10 @@ func TestFirewall_Drop2(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// h1/c1 lacks the proper groups
-	assert.Error(t, fw.Drop([]byte{}, p, true, &h1, cp, nil), ErrNoMatchingRule)
+	assert.Error(t, fw.Drop([]byte{}, p, true, &h1, cp), ErrNoMatchingRule)
 	// c has the proper groups
 	resetConntrack(fw)
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp))
 }
 
 func TestFirewall_Drop3(t *testing.T) {
@@ -454,13 +454,13 @@ func TestFirewall_Drop3(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// c1 should pass because host match
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h1, cp, nil))
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h1, cp))
 	// c2 should pass because ca sha match
 	resetConntrack(fw)
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h2, cp, nil))
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h2, cp))
 	// c3 should fail because no match
 	resetConntrack(fw)
-	assert.Equal(t, fw.Drop([]byte{}, p, true, &h3, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop([]byte{}, p, true, &h3, cp), ErrNoMatchingRule)
 }
 
 func TestFirewall_DropConntrackReload(t *testing.T) {
@@ -505,12 +505,12 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp), ErrNoMatchingRule)
 	// Allow inbound
 	resetConntrack(fw)
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp))
 	// Allow outbound because conntrack
-	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil))
+	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp))
 
 	oldFw := fw
 	fw = NewFirewall(time.Second, time.Minute, time.Hour, &c)
@@ -519,7 +519,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Allow outbound because conntrack and new rules allow port 10
-	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil))
+	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp))
 
 	oldFw = fw
 	fw = NewFirewall(time.Second, time.Minute, time.Hour, &c)
@@ -528,7 +528,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Drop outbound because conntrack doesn't match new ruleset
-	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp), ErrNoMatchingRule)
 }
 
 func BenchmarkLookup(b *testing.B) {

go.mod

@@ -1,6 +1,6 @@
 module github.com/slackhq/nebula
 
-go 1.16
+go 1.12
 
 require (
 	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239
@@ -21,7 +21,6 @@ require (
 	github.com/prometheus/procfs v0.0.8 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563
 	github.com/sirupsen/logrus v1.4.2
-	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/songgao/water v0.0.0-20190725173103-fd331bda3f4b
 	github.com/stretchr/testify v1.6.1
 	github.com/vishvananda/netlink v1.0.1-0.20190522153524-00009fb8606a

go.sum

@@ -98,8 +98,6 @@ github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqn
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
-github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/songgao/water v0.0.0-20190725173103-fd331bda3f4b h1:+y4hCMc/WKsDbAPsOQZgBSaSZ26uh2afyaWeVg/3s/c=
 github.com/songgao/water v0.0.0-20190725173103-fd331bda3f4b/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

handler.go

@@ -0,0 +1,83 @@
+package nebula
+
+func (f *Interface) newHook(w func([]byte) error) InsideHandler {
+	fn := func(hostInfo *HostInfo, ci *ConnectionState, addr *udpAddr, header *Header, out []byte, packet []byte, fwPacket *FirewallPacket, nb []byte) {
+		f.decryptTo(w, hostInfo, header.MessageCounter, out, packet, fwPacket, nb)
+	}
+	return f.encrypted(fn)
+}
+
+func (f *Interface) encrypted(h InsideHandler) InsideHandler {
+	return func(hostInfo *HostInfo, ci *ConnectionState, addr *udpAddr, header *Header, out []byte, packet []byte, fwPacket *FirewallPacket, nb []byte) {
+		if !f.handleEncrypted(ci, addr, header) {
+			return
+		}
+
+		h(hostInfo, ci, addr, header, out, packet, fwPacket, nb)
+
+		f.handleHostRoaming(hostInfo, addr)
+		f.connectionManager.In(hostInfo.hostId)
+	}
+}
+
+func (f *Interface) rxMetrics(h InsideHandler) InsideHandler {
+	return func(hostInfo *HostInfo, ci *ConnectionState, addr *udpAddr, header *Header, out []byte, packet []byte, fwPacket *FirewallPacket, nb []byte) {
+		f.messageMetrics.Rx(header.Type, header.Subtype, 1)
+		h(hostInfo, ci, addr, header, out, packet, fwPacket, nb)
+	}
+}
+
+func (f *Interface) handleMessagePacket(hostInfo *HostInfo, ci *ConnectionState, addr *udpAddr, header *Header, out []byte, packet []byte, fwPacket *FirewallPacket, nb []byte) {
+	f.decryptTo(f.inside.WriteRaw, hostInfo, header.MessageCounter, out, packet, fwPacket, nb)
+}
+
+func (f *Interface) handleLighthousePacket(hostInfo *HostInfo, ci *ConnectionState, addr *udpAddr, header *Header, out []byte, packet []byte, fwPacket *FirewallPacket, nb []byte) {
+	d, err := f.decrypt(hostInfo, header.MessageCounter, out, packet, header, nb)
+	if err != nil {
+		hostInfo.logger().WithError(err).WithField("udpAddr", addr).
+			WithField("packet", packet).
+			Error("Failed to decrypt lighthouse packet")
+
+		//TODO: maybe after build 64 is out? 06/14/2018 - NB
+		//f.sendRecvError(net.Addr(addr), header.RemoteIndex)
+		return
+	}
+
+	f.lightHouse.HandleRequest(addr, hostInfo.hostId, d, hostInfo.GetCert(), f)
+}
+
+func (f *Interface) handleTestPacket(hostInfo *HostInfo, ci *ConnectionState, addr *udpAddr, header *Header, out []byte, packet []byte, fwPacket *FirewallPacket, nb []byte) {
+	d, err := f.decrypt(hostInfo, header.MessageCounter, out, packet, header, nb)
+	if err != nil {
+		hostInfo.logger().WithError(err).WithField("udpAddr", addr).
+			WithField("packet", packet).
+			Error("Failed to decrypt test packet")
+
+		//TODO: maybe after build 64 is out? 06/14/2018 - NB
+		//f.sendRecvError(net.Addr(addr), header.RemoteIndex)
+		return
+	}
+
+	if header.Subtype == testRequest {
+		// This testRequest might be from TryPromoteBest, so we should roam
+		// to the new IP address before responding
+		f.handleHostRoaming(hostInfo, addr)
+		f.send(test, testReply, ci, hostInfo, hostInfo.remote, d, nb, out)
+	}
+}
+
+func (f *Interface) handleHandshakePacket(hostInfo *HostInfo, ci *ConnectionState, addr *udpAddr, header *Header, out []byte, packet []byte, fwPacket *FirewallPacket, nb []byte) {
+	HandleIncomingHandshake(f, addr, packet, header, hostInfo)
+}
+
+func (f *Interface) handleRecvErrorPacket(hostInfo *HostInfo, ci *ConnectionState, addr *udpAddr, header *Header, out []byte, packet []byte, fwPacket *FirewallPacket, nb []byte) {
+	// TODO: Remove this with recv_error deprecation
+	f.handleRecvError(addr, header)
+}
+
+func (f *Interface) handleCloseTunnelPacket(hostInfo *HostInfo, ci *ConnectionState, addr *udpAddr, header *Header, out []byte, packet []byte, fwPacket *FirewallPacket, nb []byte) {
+	hostInfo.logger().WithField("udpAddr", addr).
+		Info("Close tunnel received, tearing down.")
+
+	f.closeTunnel(hostInfo)
+}

handshake.go

@@ -6,23 +6,31 @@ const (
 )
 
 func HandleIncomingHandshake(f *Interface, addr *udpAddr, packet []byte, h *Header, hostinfo *HostInfo) {
+	newHostinfo, _ := f.handshakeManager.QueryIndex(h.RemoteIndex)
+	//TODO: For stage 1 we won't have hostinfo yet but stage 2 and above would require it, this check may be helpful in those cases
+	//if err != nil {
+	//	l.WithError(err).WithField("udpAddr", addr).Error("Error while finding host info for handshake message")
+	//	return
+	//}
+
 	if !f.lightHouse.remoteAllowList.Allow(udp2ipInt(addr)) {
 		l.WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
 		return
 	}
 
+	tearDown := false
 	switch h.Subtype {
 	case handshakeIXPSK0:
 		switch h.MessageCounter {
 		case 1:
-			ixHandshakeStage1(f, addr, packet, h)
+			tearDown = ixHandshakeStage1(f, addr, newHostinfo, packet, h)
 		case 2:
-			newHostinfo, _ := f.handshakeManager.QueryIndex(h.RemoteIndex)
-			tearDown := ixHandshakeStage2(f, addr, newHostinfo, packet, h)
-			if tearDown && newHostinfo != nil {
-				f.handshakeManager.DeleteHostInfo(newHostinfo)
-			}
+			tearDown = ixHandshakeStage2(f, addr, newHostinfo, packet, h)
 		}
 	}
 
+	if tearDown && newHostinfo != nil {
+		f.handshakeManager.DeleteIndex(newHostinfo.localIndexId)
+		f.handshakeManager.DeleteVpnIP(newHostinfo.hostId)
+	}
 }

handshake_ix.go

@@ -25,17 +25,17 @@ func ixHandshakeStage0(f *Interface, vpnIp uint32, hostinfo *HostInfo) {
 		}
 	}
 
-	err := f.handshakeManager.AddIndexHostInfo(hostinfo)
+	myIndex, err := generateIndex()
 	if err != nil {
 		l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to generate index")
 		return
 	}
-
 	ci := hostinfo.ConnectionState
+	f.handshakeManager.AddIndexHostInfo(myIndex, hostinfo)
 
 	hsProto := &NebulaHandshakeDetails{
-		InitiatorIndex: hostinfo.localIndexId,
+		InitiatorIndex: myIndex,
 		Time:           uint64(time.Now().Unix()),
 		Cert:           ci.certState.rawCertificateNoKey,
 	}
@@ -54,7 +54,7 @@ func ixHandshakeStage0(f *Interface, vpnIp uint32, hostinfo *HostInfo) {
 	}
 
 	header := HeaderEncode(make([]byte, HeaderLen), Version, uint8(handshake), handshakeIXPSK0, 0, 1)
-	atomic.AddUint64(&ci.atomicMessageCounter, 1)
+	atomic.AddUint64(ci.messageCounter, 1)
 
 	msg, _, _, err := ci.H.WriteMessage(header, hsBytes)
 	if err != nil {
@@ -63,158 +63,120 @@ func ixHandshakeStage0(f *Interface, vpnIp uint32, hostinfo *HostInfo) {
 		return
 	}
 
-	// We are sending handshake packet 1, so we don't expect to receive
-	// handshake packet 1 from the responder
-	ci.window.Update(1)
-
 	hostinfo.HandshakePacket[0] = msg
 	hostinfo.HandshakeReady = true
 	hostinfo.handshakeStart = time.Now()
 
 }
 
-func ixHandshakeStage1(f *Interface, addr *udpAddr, packet []byte, h *Header) {
-	ci := f.newConnectionState(false, noise.HandshakeIX, []byte{}, 0)
-	// Mark packet 1 as seen so it doesn't show up as missed
-	ci.window.Update(1)
+func ixHandshakeStage1(f *Interface, addr *udpAddr, hostinfo *HostInfo, packet []byte, h *Header) bool {
+	var ip uint32
+	if h.RemoteIndex == 0 {
+		ci := f.newConnectionState(false, noise.HandshakeIX, []byte{}, 0)
+		// Mark packet 1 as seen so it doesn't show up as missed
+		ci.window.Update(1)
 
-	msg, _, _, err := ci.H.ReadMessage(nil, packet[HeaderLen:])
-	if err != nil {
-		l.WithError(err).WithField("udpAddr", addr).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.ReadMessage")
-		return
-	}
+		msg, _, _, err := ci.H.ReadMessage(nil, packet[HeaderLen:])
+		if err != nil {
+			l.WithError(err).WithField("udpAddr", addr).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.ReadMessage")
+			return true
+		}
 
-	hs := &NebulaHandshake{}
-	err = proto.Unmarshal(msg, hs)
-	/*
-		l.Debugln("GOT INDEX: ", hs.Details.InitiatorIndex)
-	*/
-	if err != nil || hs.Details == nil {
-		l.WithError(err).WithField("udpAddr", addr).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed unmarshal handshake message")
-		return
-	}
+		hs := &NebulaHandshake{}
+		err = proto.Unmarshal(msg, hs)
+		/*
+			l.Debugln("GOT INDEX: ", hs.Details.InitiatorIndex)
+		*/
+		if err != nil || hs.Details == nil {
+			l.WithError(err).WithField("udpAddr", addr).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed unmarshal handshake message")
+			return true
+		}
 
-	remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert)
-	if err != nil {
-		l.WithError(err).WithField("udpAddr", addr).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cert", remoteCert).
-			Info("Invalid certificate from host")
-		return
-	}
-	vpnIP := ip2int(remoteCert.Details.Ips[0].IP)
-	certName := remoteCert.Details.Name
-	fingerprint, _ := remoteCert.Sha256Sum()
+		hostinfo, _ := f.handshakeManager.pendingHostMap.QueryReverseIndex(hs.Details.InitiatorIndex)
+		if hostinfo != nil && bytes.Equal(hostinfo.HandshakePacket[0], packet[HeaderLen:]) {
+			if msg, ok := hostinfo.HandshakePacket[2]; ok {
+				f.messageMetrics.Tx(handshake, NebulaMessageSubType(msg[1]), 1)
+				err := f.outside.WriteTo(msg, addr)
+				if err != nil {
+					l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
+						WithError(err).Error("Failed to send handshake message")
+				} else {
+					l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
+						Info("Handshake message sent")
+				}
+				return false
+			}
 
-	if vpnIP == ip2int(f.certState.certificate.Details.Ips[0].IP) {
+			l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cached", true).
+				WithField("packets", hostinfo.HandshakePacket).
+				Error("Seen this handshake packet already but don't have a cached packet to return")
+		}
+
+		remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert)
+		if err != nil {
+			l.WithError(err).WithField("udpAddr", addr).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cert", remoteCert).
+				Info("Invalid certificate from host")
+			return true
+		}
+		vpnIP := ip2int(remoteCert.Details.Ips[0].IP)
+		certName := remoteCert.Details.Name
+		fingerprint, _ := remoteCert.Sha256Sum()
+
+		myIndex, err := generateIndex()
+		if err != nil {
+			l.WithError(err).WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("fingerprint", fingerprint).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to generate index")
+			return true
+		}
+
+		hostinfo, err = f.handshakeManager.AddIndex(myIndex, ci)
+		if err != nil {
+			l.WithError(err).WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("fingerprint", fingerprint).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Error adding index to connection manager")
+
+			return true
+		}
 		l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
 			WithField("certName", certName).
 			WithField("fingerprint", fingerprint).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Refusing to handshake with myself")
-		return
-	}
+			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			Info("Handshake message received")
 
-	myIndex, err := generateIndex()
-	if err != nil {
-		l.WithError(err).WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
-			WithField("certName", certName).
-			WithField("fingerprint", fingerprint).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to generate index")
-		return
-	}
+		hostinfo.remoteIndexId = hs.Details.InitiatorIndex
+		hs.Details.ResponderIndex = myIndex
+		hs.Details.Cert = ci.certState.rawCertificateNoKey
 
-	hostinfo := &HostInfo{
-		ConnectionState: ci,
-		Remotes:         []*HostInfoDest{},
-		localIndexId:    myIndex,
-		remoteIndexId:   hs.Details.InitiatorIndex,
-		hostId:          vpnIP,
-		HandshakePacket: make(map[uint8][]byte, 0),
-	}
+		hsBytes, err := proto.Marshal(hs)
+		if err != nil {
+			l.WithError(err).WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("fingerprint", fingerprint).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to marshal handshake message")
+			return true
+		}
 
-	l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
-		WithField("certName", certName).
-		WithField("fingerprint", fingerprint).
-		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-		Info("Handshake message received")
+		header := HeaderEncode(make([]byte, HeaderLen), Version, uint8(handshake), handshakeIXPSK0, hs.Details.InitiatorIndex, 2)
+		msg, dKey, eKey, err := ci.H.WriteMessage(header, hsBytes)
+		if err != nil {
+			l.WithError(err).WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("fingerprint", fingerprint).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage")
+			return true
+		}
 
-	hs.Details.ResponderIndex = myIndex
-	hs.Details.Cert = ci.certState.rawCertificateNoKey
-
-	hsBytes, err := proto.Marshal(hs)
-	if err != nil {
-		l.WithError(err).WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
-			WithField("certName", certName).
-			WithField("fingerprint", fingerprint).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to marshal handshake message")
-		return
-	}
-
-	header := HeaderEncode(make([]byte, HeaderLen), Version, uint8(handshake), handshakeIXPSK0, hs.Details.InitiatorIndex, 2)
-	msg, dKey, eKey, err := ci.H.WriteMessage(header, hsBytes)
-	if err != nil {
-		l.WithError(err).WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
-			WithField("certName", certName).
-			WithField("fingerprint", fingerprint).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage")
-		return
-	} else if dKey == nil || eKey == nil {
-		l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
-			WithField("certName", certName).
-			WithField("fingerprint", fingerprint).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Noise did not arrive at a key")
-		return
-	}
-
-	hostinfo.HandshakePacket[0] = make([]byte, len(packet[HeaderLen:]))
-	copy(hostinfo.HandshakePacket[0], packet[HeaderLen:])
-
-	// Regardless of whether you are the sender or receiver, you should arrive here
-	// and complete standing up the connection.
-	hostinfo.HandshakePacket[2] = make([]byte, len(msg))
-	copy(hostinfo.HandshakePacket[2], msg)
-
-	// We are sending handshake packet 2, so we don't expect to receive
-	// handshake packet 2 from the initiator.
-	ci.window.Update(2)
-
-	ci.peerCert = remoteCert
-	ci.dKey = NewNebulaCipherState(dKey)
-	ci.eKey = NewNebulaCipherState(eKey)
-	//l.Debugln("got symmetric pairs")
-
-	//hostinfo.ClearRemotes()
-	hostinfo.AddRemote(*addr)
-	hostinfo.ForcePromoteBest(f.hostMap.preferredRanges)
-	hostinfo.CreateRemoteCIDR(remoteCert)
-
-	hostinfo.Lock()
-	defer hostinfo.Unlock()
-
-	// Only overwrite existing record if we should win the handshake race
-	overwrite := vpnIP > ip2int(f.certState.certificate.Details.Ips[0].IP)
-	existing, err := f.handshakeManager.CheckAndComplete(hostinfo, 0, overwrite, f)
-	if err != nil {
-		switch err {
-		case ErrAlreadySeen:
-			msg = existing.HandshakePacket[2]
-			f.messageMetrics.Tx(handshake, NebulaMessageSubType(msg[1]), 1)
-			err := f.outside.WriteTo(msg, addr)
-			if err != nil {
-				l.WithField("vpnIp", IntIp(existing.hostId)).WithField("udpAddr", addr).
-					WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
-					WithError(err).Error("Failed to send handshake message")
-			} else {
-				l.WithField("vpnIp", IntIp(existing.hostId)).WithField("udpAddr", addr).
-					WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
-					Info("Handshake message sent")
-			}
-			return
-		case ErrExistingHostInfo:
-			// This means there was an existing tunnel and we didn't win
-			// handshake avoidance
+		if f.hostMap.CheckHandshakeCompleteIP(vpnIP) && vpnIP < ip2int(f.certState.certificate.Details.Ips[0].IP) {
 			l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
 				WithField("certName", certName).
 				WithField("fingerprint", fingerprint).
@@ -224,65 +186,89 @@ func ixHandshakeStage1(f *Interface, addr *udpAddr, packet []byte, h *Header) {
 
 			// Send a test packet to trigger an authenticated tunnel test, this should suss out any lingering tunnel issues
 			f.SendMessageToVpnIp(test, testRequest, vpnIP, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
-			return
-		case ErrLocalIndexCollision:
-			// This means we failed to insert because of collision on localIndexId. Just let the next handshake packet retry
-			l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
-				WithField("certName", certName).
-				WithField("fingerprint", fingerprint).
-				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-				WithField("localIndex", hostinfo.localIndexId).WithField("collision", IntIp(existing.hostId)).
-				Error("Failed to add HostInfo due to localIndex collision")
-			return
-		default:
-			// Shouldn't happen, but just in case someone adds a new error type to CheckAndComplete
-			// And we forget to update it here
-			l.WithError(err).WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
-				WithField("certName", certName).
-				WithField("fingerprint", fingerprint).
-				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-				Error("Failed to add HostInfo to HostMap")
-			return
+			return true
 		}
+
+		hostinfo.HandshakePacket[0] = make([]byte, len(packet[HeaderLen:]))
+		copy(hostinfo.HandshakePacket[0], packet[HeaderLen:])
+
+		// Regardless of whether you are the sender or receiver, you should arrive here
+		// and complete standing up the connection.
+		if dKey != nil && eKey != nil {
+			hostinfo.HandshakePacket[2] = make([]byte, len(msg))
+			copy(hostinfo.HandshakePacket[2], msg)
+
+			f.messageMetrics.Tx(handshake, NebulaMessageSubType(msg[1]), 1)
+			err := f.outside.WriteTo(msg, addr)
+			if err != nil {
+				l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
+					WithField("certName", certName).
+					WithField("fingerprint", fingerprint).
+					WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+					WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+					WithError(err).Error("Failed to send handshake")
+			} else {
+				l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
+					WithField("certName", certName).
+					WithField("fingerprint", fingerprint).
+					WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+					WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+					Info("Handshake message sent")
+			}
+
+			ip = ip2int(remoteCert.Details.Ips[0].IP)
+			ci.peerCert = remoteCert
+			ci.dKey = NewNebulaCipherState(dKey)
+			ci.eKey = NewNebulaCipherState(eKey)
+			//l.Debugln("got symmetric pairs")
+
+			//hostinfo.ClearRemotes()
+			hostinfo.AddRemote(*addr)
+			hostinfo.CreateRemoteCIDR(remoteCert)
+			f.lightHouse.AddRemoteAndReset(ip, addr)
+			if f.serveDns {
+				dnsR.Add(remoteCert.Details.Name+".", remoteCert.Details.Ips[0].IP.String())
+			}
+
+			ho, err := f.hostMap.QueryVpnIP(vpnIP)
+			if err == nil && ho.localIndexId != 0 {
+				l.WithField("vpnIp", vpnIP).
+					WithField("certName", certName).
+					WithField("fingerprint", fingerprint).
+					WithField("action", "removing stale index").
+					WithField("index", ho.localIndexId).
+					Debug("Handshake processing")
+				f.hostMap.DeleteIndex(ho.localIndexId)
+			}
+
+			f.hostMap.AddIndexHostInfo(hostinfo.localIndexId, hostinfo)
+			f.hostMap.AddVpnIPHostInfo(vpnIP, hostinfo)
+
+			hostinfo.handshakeComplete()
+		} else {
+			l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("fingerprint", fingerprint).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+				Error("Noise did not arrive at a key")
+			return true
+		}
+
 	}
 
-	// Do the send
-	f.messageMetrics.Tx(handshake, NebulaMessageSubType(msg[1]), 1)
-	err = f.outside.WriteTo(msg, addr)
-	if err != nil {
-		l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
-			WithField("certName", certName).
-			WithField("fingerprint", fingerprint).
-			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
-			WithError(err).Error("Failed to send handshake")
-	} else {
-		l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
-			WithField("certName", certName).
-			WithField("fingerprint", fingerprint).
-			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
-			Info("Handshake message sent")
-	}
-
-	hostinfo.handshakeComplete()
-
-	return
+	f.hostMap.AddRemote(ip, addr)
+	return false
 }
 
 func ixHandshakeStage2(f *Interface, addr *udpAddr, hostinfo *HostInfo, packet []byte, h *Header) bool {
 	if hostinfo == nil {
 		return true
 	}
-	hostinfo.Lock()
-	defer hostinfo.Unlock()
 
 	if bytes.Equal(hostinfo.HandshakePacket[2], packet[HeaderLen:]) {
 		l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("header", h).
-			Info("Already seen this handshake packet")
+			Error("Already seen this handshake packet")
 		return false
 	}
 
@@ -303,11 +289,6 @@ func ixHandshakeStage2(f *Interface, addr *udpAddr, hostinfo *HostInfo, packet [
 		// to DOS us. Every other error condition after should to allow a possible good handshake to complete in the
 		// near future
 		return false
-	} else if dKey == nil || eKey == nil {
-		l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
-			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
-			Error("Noise did not arrive at a key")
-		return true
 	}
 
 	hs := &NebulaHandshake{}
@@ -352,18 +333,45 @@ func ixHandshakeStage2(f *Interface, addr *udpAddr, hostinfo *HostInfo, packet [
 
 	// Regardless of whether you are the sender or receiver, you should arrive here
 	// and complete standing up the connection.
+	if dKey != nil && eKey != nil {
+		ip := ip2int(remoteCert.Details.Ips[0].IP)
+		ci.peerCert = remoteCert
+		ci.dKey = NewNebulaCipherState(dKey)
+		ci.eKey = NewNebulaCipherState(eKey)
+		//l.Debugln("got symmetric pairs")
 
-	ci.peerCert = remoteCert
-	ci.dKey = NewNebulaCipherState(dKey)
-	ci.eKey = NewNebulaCipherState(eKey)
-	//l.Debugln("got symmetric pairs")
+		//hostinfo.ClearRemotes()
+		f.hostMap.AddRemote(ip, addr)
+		hostinfo.CreateRemoteCIDR(remoteCert)
+		f.lightHouse.AddRemoteAndReset(ip, addr)
+		if f.serveDns {
+			dnsR.Add(remoteCert.Details.Name+".", remoteCert.Details.Ips[0].IP.String())
+		}
 
-	hostinfo.SetRemote(*addr)
-	hostinfo.CreateRemoteCIDR(remoteCert)
+		ho, err := f.hostMap.QueryVpnIP(vpnIP)
+		if err == nil && ho.localIndexId != 0 {
+			l.WithField("vpnIp", vpnIP).
+				WithField("certName", certName).
+				WithField("fingerprint", fingerprint).
+				WithField("action", "removing stale index").
+				WithField("index", ho.localIndexId).
+				Debug("Handshake processing")
+			f.hostMap.DeleteIndex(ho.localIndexId)
+		}
 
-	f.handshakeManager.Complete(hostinfo, f)
-	hostinfo.handshakeComplete()
-	f.metricHandshakes.Update(duration)
+		f.hostMap.AddVpnIPHostInfo(vpnIP, hostinfo)
+		f.hostMap.AddIndexHostInfo(hostinfo.localIndexId, hostinfo)
+
+		hostinfo.handshakeComplete()
+		f.metricHandshakes.Update(duration)
+	} else {
+		l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("fingerprint", fingerprint).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			Error("Noise did not arrive at a key")
+		return true
+	}
 
 	return false
 }

handshake_manager.go

@@ -1,10 +1,9 @@
 package nebula
 
 import (
-	"bytes"
 	"crypto/rand"
 	"encoding/binary"
-	"errors"
+	"fmt"
 	"net"
 	"time"
 
@@ -100,12 +99,14 @@ func (c *HandshakeManager) NextOutboundHandshakeTimerTick(now time.Time, f EncWr
 }
 
 func (c *HandshakeManager) handleOutbound(vpnIP uint32, f EncWriter, lighthouseTriggered bool) {
+	index, err := c.pendingHostMap.GetIndexByVpnIP(vpnIP)
+	if err != nil {
+		return
+	}
 	hostinfo, err := c.pendingHostMap.QueryVpnIP(vpnIP)
 	if err != nil {
 		return
 	}
-	hostinfo.Lock()
-	defer hostinfo.Unlock()
 
 	// If we haven't finished the handshake and we haven't hit max retries, query
 	// lighthouse and then send the handshake packet again.
@@ -171,7 +172,8 @@ func (c *HandshakeManager) handleOutbound(vpnIP uint32, f EncWriter, lighthouseT
 			c.OutboundHandshakeTimer.Add(vpnIP, c.config.tryInterval*time.Duration(hostinfo.HandshakeCounter))
 		}
 	} else {
-		c.pendingHostMap.DeleteHostInfo(hostinfo)
+		c.pendingHostMap.DeleteVpnIP(vpnIP)
+		c.pendingHostMap.DeleteIndex(index)
 	}
 }
 
@@ -184,7 +186,12 @@ func (c *HandshakeManager) NextInboundHandshakeTimerTick(now time.Time) {
 		}
 		index := ep.(uint32)
 
+		vpnIP, err := c.pendingHostMap.GetVpnIPByIndex(index)
+		if err != nil {
+			continue
+		}
 		c.pendingHostMap.DeleteIndex(index)
+		c.pendingHostMap.DeleteVpnIP(vpnIP)
 	}
 }
 
@@ -197,132 +204,28 @@ func (c *HandshakeManager) AddVpnIP(vpnIP uint32) *HostInfo {
 	return hostinfo
 }
 
-var (
-	ErrExistingHostInfo    = errors.New("existing hostinfo")
-	ErrAlreadySeen         = errors.New("already seen")
-	ErrLocalIndexCollision = errors.New("local index collision")
-)
-
-// CheckAndComplete checks for any conflicts in the main and pending hostmap
-// before adding hostinfo to main. If err is nil, it was added. Otherwise err will be:
-
-// ErrAlreadySeen if we already have an entry in the hostmap that has seen the
-// exact same handshake packet
-//
-// ErrExistingHostInfo if we already have an entry in the hostmap for this
-// VpnIP and overwrite was false.
-//
-// ErrLocalIndexCollision if we already have an entry in the main or pending
-// hostmap for the hostinfo.localIndexId.
-func (c *HandshakeManager) CheckAndComplete(hostinfo *HostInfo, handshakePacket uint8, overwrite bool, f *Interface) (*HostInfo, error) {
-	c.pendingHostMap.RLock()
-	defer c.pendingHostMap.RUnlock()
-	c.mainHostMap.Lock()
-	defer c.mainHostMap.Unlock()
-
-	existingHostInfo, found := c.mainHostMap.Hosts[hostinfo.hostId]
-	if found && existingHostInfo != nil {
-		if bytes.Equal(hostinfo.HandshakePacket[handshakePacket], existingHostInfo.HandshakePacket[handshakePacket]) {
-			return existingHostInfo, ErrAlreadySeen
-		}
-		if !overwrite {
-			return existingHostInfo, ErrExistingHostInfo
-		}
-	}
-
-	existingIndex, found := c.mainHostMap.Indexes[hostinfo.localIndexId]
-	if found {
-		// We have a collision, but for a different hostinfo
-		return existingIndex, ErrLocalIndexCollision
-	}
-	existingIndex, found = c.pendingHostMap.Indexes[hostinfo.localIndexId]
-	if found && existingIndex != hostinfo {
-		// We have a collision, but for a different hostinfo
-		return existingIndex, ErrLocalIndexCollision
-	}
-
-	existingRemoteIndex, found := c.mainHostMap.RemoteIndexes[hostinfo.remoteIndexId]
-	if found && existingRemoteIndex != nil && existingRemoteIndex.hostId != hostinfo.hostId {
-		// We have a collision, but this can happen since we can't control
-		// the remote ID. Just log about the situation as a note.
-		hostinfo.logger().
-			WithField("remoteIndex", hostinfo.remoteIndexId).WithField("collision", IntIp(existingRemoteIndex.hostId)).
-			Info("New host shadows existing host remoteIndex")
-	}
-
-	if existingHostInfo != nil {
-		// We are going to overwrite this entry, so remove the old references
-		delete(c.mainHostMap.Hosts, existingHostInfo.hostId)
-		delete(c.mainHostMap.Indexes, existingHostInfo.localIndexId)
-		delete(c.mainHostMap.RemoteIndexes, existingHostInfo.remoteIndexId)
-	}
-
-	c.mainHostMap.addHostInfo(hostinfo, f)
-	return existingHostInfo, nil
+func (c *HandshakeManager) DeleteVpnIP(vpnIP uint32) {
+	//l.Debugln("Deleting pending vpn ip :", IntIp(vpnIP))
+	c.pendingHostMap.DeleteVpnIP(vpnIP)
 }
 
-// Complete is a simpler version of CheckAndComplete when we already know we
-// won't have a localIndexId collision because we already have an entry in the
-// pendingHostMap
-func (c *HandshakeManager) Complete(hostinfo *HostInfo, f *Interface) {
-	c.mainHostMap.Lock()
-	defer c.mainHostMap.Unlock()
-
-	existingHostInfo, found := c.mainHostMap.Hosts[hostinfo.hostId]
-	if found && existingHostInfo != nil {
-		// We are going to overwrite this entry, so remove the old references
-		delete(c.mainHostMap.Hosts, existingHostInfo.hostId)
-		delete(c.mainHostMap.Indexes, existingHostInfo.localIndexId)
-		delete(c.mainHostMap.RemoteIndexes, existingHostInfo.remoteIndexId)
+func (c *HandshakeManager) AddIndex(index uint32, ci *ConnectionState) (*HostInfo, error) {
+	hostinfo, err := c.pendingHostMap.AddIndex(index, ci)
+	if err != nil {
+		return nil, fmt.Errorf("Issue adding index: %d", index)
 	}
-
-	existingRemoteIndex, found := c.mainHostMap.RemoteIndexes[hostinfo.remoteIndexId]
-	if found && existingRemoteIndex != nil {
-		// We have a collision, but this can happen since we can't control
-		// the remote ID. Just log about the situation as a note.
-		hostinfo.logger().
-			WithField("remoteIndex", hostinfo.remoteIndexId).WithField("collision", IntIp(existingRemoteIndex.hostId)).
-			Info("New host shadows existing host remoteIndex")
-	}
-
-	c.mainHostMap.addHostInfo(hostinfo, f)
+	//c.mainHostMap.AddIndexHostInfo(index, hostinfo)
+	c.InboundHandshakeTimer.Add(index, time.Second*10)
+	return hostinfo, nil
 }
 
-// AddIndexHostInfo generates a unique localIndexId for this HostInfo
-// and adds it to the pendingHostMap. Will error if we are unable to generate
-// a unique localIndexId
-func (c *HandshakeManager) AddIndexHostInfo(h *HostInfo) error {
-	c.pendingHostMap.Lock()
-	defer c.pendingHostMap.Unlock()
-	c.mainHostMap.RLock()
-	defer c.mainHostMap.RUnlock()
-
-	for i := 0; i < 32; i++ {
-		index, err := generateIndex()
-		if err != nil {
-			return err
-		}
-
-		_, inPending := c.pendingHostMap.Indexes[index]
-		_, inMain := c.mainHostMap.Indexes[index]
-
-		if !inMain && !inPending {
-			h.localIndexId = index
-			c.pendingHostMap.Indexes[index] = h
-			return nil
-		}
-	}
-
-	return errors.New("failed to generate unique localIndexId")
+func (c *HandshakeManager) AddIndexHostInfo(index uint32, h *HostInfo) {
+	c.pendingHostMap.AddIndexHostInfo(index, h)
 }
 
-func (c *HandshakeManager) addRemoteIndexHostInfo(index uint32, h *HostInfo) {
-	c.pendingHostMap.addRemoteIndexHostInfo(index, h)
-}
-
-func (c *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) {
-	//l.Debugln("Deleting pending hostinfo :", hostinfo)
-	c.pendingHostMap.DeleteHostInfo(hostinfo)
+func (c *HandshakeManager) DeleteIndex(index uint32) {
+	//l.Debugln("Deleting pending index :", index)
+	c.pendingHostMap.DeleteIndex(index)
 }
 
 func (c *HandshakeManager) QueryIndex(index uint32) (*HostInfo, error) {
@@ -338,19 +241,13 @@ func (c *HandshakeManager) EmitStats() {
 
 func generateIndex() (uint32, error) {
 	b := make([]byte, 4)
-
-	// Let zero mean we don't know the ID, so don't generate zero
-	var index uint32
-	for index == 0 {
-		_, err := rand.Read(b)
-		if err != nil {
-			l.Errorln(err)
-			return 0, err
-		}
-
-		index = binary.BigEndian.Uint32(b)
+	_, err := rand.Read(b)
+	if err != nil {
+		l.Errorln(err)
+		return 0, err
 	}
 
+	index := binary.BigEndian.Uint32(b)
 	if l.Level >= logrus.DebugLevel {
 		l.WithField("index", index).
 			Debug("Generated index")

handshake_manager_test.go

@@ -8,11 +8,12 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+var indexes []uint32 = []uint32{1000, 2000, 3000, 4000}
+
 //var ips []uint32 = []uint32{9000, 9999999, 3, 292394923}
 var ips []uint32
 
 func Test_NewHandshakeManagerIndex(t *testing.T) {
-
 	_, tuncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
@@ -25,18 +26,9 @@ func Test_NewHandshakeManagerIndex(t *testing.T) {
 	now := time.Now()
 	blah.NextInboundHandshakeTimerTick(now)
 
-	var indexes = make([]uint32, 4)
-	var hostinfo = make([]*HostInfo, len(indexes))
-	for i := range indexes {
-		hostinfo[i] = &HostInfo{ConnectionState: &ConnectionState{}}
-	}
-
 	// Add four indexes
-	for i := range indexes {
-		err := blah.AddIndexHostInfo(hostinfo[i])
-		assert.NoError(t, err)
-		indexes[i] = hostinfo[i].localIndexId
-		blah.InboundHandshakeTimer.Add(indexes[i], time.Second*10)
+	for _, v := range indexes {
+		blah.AddIndex(v, &ConnectionState{})
 	}
 	// Confirm they are in the pending index list
 	for _, v := range indexes {
@@ -177,11 +169,8 @@ func Test_NewHandshakeManagerVpnIPcleanup(t *testing.T) {
 
 	hostinfo := blah.AddVpnIP(vpnIP)
 	// Pretned we have an index too
-	err := blah.AddIndexHostInfo(hostinfo)
-	assert.NoError(t, err)
-	blah.InboundHandshakeTimer.Add(hostinfo.localIndexId, time.Second*10)
-	assert.NotZero(t, hostinfo.localIndexId)
-	assert.Contains(t, blah.pendingHostMap.Indexes, hostinfo.localIndexId)
+	blah.AddIndexHostInfo(12341234, hostinfo)
+	assert.Contains(t, blah.pendingHostMap.Indexes, uint32(12341234))
 
 	// Jump ahead `HandshakeRetries` ticks. Eviction should happen in pending
 	// but not main hostmap
@@ -227,10 +216,7 @@ func Test_NewHandshakeManagerIndexcleanup(t *testing.T) {
 	now := time.Now()
 	blah.NextInboundHandshakeTimerTick(now)
 
-	hostinfo := &HostInfo{ConnectionState: &ConnectionState{}}
-	err := blah.AddIndexHostInfo(hostinfo)
-	assert.NoError(t, err)
-	blah.InboundHandshakeTimer.Add(hostinfo.localIndexId, time.Second*10)
+	hostinfo, _ := blah.AddIndex(12341234, &ConnectionState{})
 	// Pretned we have an index too
 	blah.pendingHostMap.AddVpnIPHostInfo(101010, hostinfo)
 	assert.Contains(t, blah.pendingHostMap.Hosts, uint32(101010))
@@ -243,7 +229,7 @@ func Test_NewHandshakeManagerIndexcleanup(t *testing.T) {
 	next_tick := now.Add(DefaultHandshakeTryInterval*DefaultHandshakeRetries + 3)
 	blah.NextInboundHandshakeTimerTick(next_tick)
 	assert.NotContains(t, blah.pendingHostMap.Hosts, uint32(101010))
-	assert.NotContains(t, blah.pendingHostMap.Indexes, uint32(hostinfo.localIndexId))
+	assert.NotContains(t, blah.pendingHostMap.Indexes, uint32(12341234))
 }
 
 type mockEncWriter struct {

header.go

@@ -54,6 +54,7 @@ var typeMap = map[NebulaMessageType]string{
 }
 
 const (
+	subTypeNone NebulaMessageSubType = 0
 	testRequest NebulaMessageSubType = 0
 	testReply   NebulaMessageSubType = 1
 )

hostmap.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/rcrowley/go-metrics"
@@ -20,13 +19,12 @@ const MaxRemotes = 10
 
 // How long we should prevent roaming back to the previous IP.
 // This helps prevent flapping due to packets already in flight
-const RoamingSuppressSeconds = 2
+const RoamingSupressSeconds = 2
 
 type HostMap struct {
 	sync.RWMutex    //Because we concurrently read and write to our maps
 	name            string
 	Indexes         map[uint32]*HostInfo
-	RemoteIndexes   map[uint32]*HostInfo
 	Hosts           map[uint32]*HostInfo
 	preferredRanges []*net.IPNet
 	vpnCIDR         *net.IPNet
@@ -36,8 +34,6 @@ type HostMap struct {
 }
 
 type HostInfo struct {
-	sync.RWMutex
-
 	remote            *udpAddr
 	Remotes           []*HostInfoDest
 	promoteCounter    uint32
@@ -54,11 +50,6 @@ type HostInfo struct {
 	recvError         int
 	remoteCidr        *CIDRTree
 
-	// lastRebindCount is the other side of Interface.rebindCount, if these values don't match then we need to ask LH
-	// for a punch from the remote end of this tunnel. The goal being to prime their conntrack for our traffic just like
-	// with a handshake
-	lastRebindCount int8
-
 	lastRoam       time.Time
 	lastRoamRemote *udpAddr
 }
@@ -73,7 +64,8 @@ type cachedPacket struct {
 type packetCallback func(t NebulaMessageType, st NebulaMessageSubType, h *HostInfo, p, nb, out []byte)
 
 type HostInfoDest struct {
-	addr *udpAddr
+	active bool
+	addr   *udpAddr
 	//probes       [ProbeLen]bool
 	probeCounter int
 }
@@ -86,11 +78,9 @@ type Probe struct {
 func NewHostMap(name string, vpnCIDR *net.IPNet, preferredRanges []*net.IPNet) *HostMap {
 	h := map[uint32]*HostInfo{}
 	i := map[uint32]*HostInfo{}
-	r := map[uint32]*HostInfo{}
 	m := HostMap{
 		name:            name,
 		Indexes:         i,
-		RemoteIndexes:   r,
 		Hosts:           h,
 		preferredRanges: preferredRanges,
 		vpnCIDR:         vpnCIDR,
@@ -105,12 +95,10 @@ func (hm *HostMap) EmitStats(name string) {
 	hm.RLock()
 	hostLen := len(hm.Hosts)
 	indexLen := len(hm.Indexes)
-	remoteIndexLen := len(hm.RemoteIndexes)
 	hm.RUnlock()
 
 	metrics.GetOrRegisterGauge("hostmap."+name+".hosts", nil).Update(int64(hostLen))
 	metrics.GetOrRegisterGauge("hostmap."+name+".indexes", nil).Update(int64(indexLen))
-	metrics.GetOrRegisterGauge("hostmap."+name+".remoteIndexes", nil).Update(int64(remoteIndexLen))
 }
 
 func (hm *HostMap) GetIndexByVpnIP(vpnIP uint32) (uint32, error) {
@@ -124,6 +112,17 @@ func (hm *HostMap) GetIndexByVpnIP(vpnIP uint32) (uint32, error) {
 	return 0, errors.New("vpn IP not found")
 }
 
+func (hm *HostMap) GetVpnIPByIndex(index uint32) (uint32, error) {
+	hm.RLock()
+	if i, ok := hm.Indexes[index]; ok {
+		vpnIP := i.hostId
+		hm.RUnlock()
+		return vpnIP, nil
+	}
+	hm.RUnlock()
+	return 0, errors.New("vpn IP not found")
+}
+
 func (hm *HostMap) Add(ip uint32, hostinfo *HostInfo) {
 	hm.Lock()
 	hm.Hosts[ip] = hostinfo
@@ -166,17 +165,37 @@ func (hm *HostMap) DeleteVpnIP(vpnIP uint32) {
 	}
 }
 
-// Only used by pendingHostMap when the remote index is not initially known
-func (hm *HostMap) addRemoteIndexHostInfo(index uint32, h *HostInfo) {
+func (hm *HostMap) AddIndex(index uint32, ci *ConnectionState) (*HostInfo, error) {
 	hm.Lock()
-	h.remoteIndexId = index
-	hm.RemoteIndexes[index] = h
+	if _, ok := hm.Indexes[index]; !ok {
+		h := &HostInfo{
+			ConnectionState: ci,
+			Remotes:         []*HostInfoDest{},
+			localIndexId:    index,
+			HandshakePacket: make(map[uint8][]byte, 0),
+		}
+		hm.Indexes[index] = h
+		l.WithField("hostMap", m{"mapName": hm.name, "indexNumber": index, "mapTotalSize": len(hm.Indexes),
+			"hostinfo": m{"existing": false, "localIndexId": h.localIndexId, "hostId": IntIp(h.hostId)}}).
+			Debug("Hostmap index added")
+
+		hm.Unlock()
+		return h, nil
+	}
+	hm.Unlock()
+	return nil, fmt.Errorf("refusing to overwrite existing index: %d", index)
+}
+
+func (hm *HostMap) AddIndexHostInfo(index uint32, h *HostInfo) {
+	hm.Lock()
+	h.localIndexId = index
+	hm.Indexes[index] = h
 	hm.Unlock()
 
 	if l.Level > logrus.DebugLevel {
 		l.WithField("hostMap", m{"mapName": hm.name, "indexNumber": index, "mapTotalSize": len(hm.Indexes),
 			"hostinfo": m{"existing": true, "localIndexId": h.localIndexId, "hostId": IntIp(h.hostId)}}).
-			Debug("Hostmap remoteIndex added")
+			Debug("Hostmap index added")
 	}
 }
 
@@ -184,8 +203,6 @@ func (hm *HostMap) AddVpnIPHostInfo(vpnIP uint32, h *HostInfo) {
 	hm.Lock()
 	h.hostId = vpnIP
 	hm.Hosts[vpnIP] = h
-	hm.Indexes[h.localIndexId] = h
-	hm.RemoteIndexes[h.remoteIndexId] = h
 	hm.Unlock()
 
 	if l.Level > logrus.DebugLevel {
@@ -195,20 +212,11 @@ func (hm *HostMap) AddVpnIPHostInfo(vpnIP uint32, h *HostInfo) {
 	}
 }
 
-// This is only called in pendingHostmap, to cleanup an inbound handshake
 func (hm *HostMap) DeleteIndex(index uint32) {
 	hm.Lock()
-	hostinfo, ok := hm.Indexes[index]
-	if ok {
-		delete(hm.Indexes, index)
-		delete(hm.RemoteIndexes, hostinfo.remoteIndexId)
-
-		// Check if we have an entry under hostId that matches the same hostinfo
-		// instance. Clean it up as well if we do.
-		hostinfo2, ok := hm.Hosts[hostinfo.hostId]
-		if ok && hostinfo2 == hostinfo {
-			delete(hm.Hosts, hostinfo.hostId)
-		}
+	delete(hm.Indexes, index)
+	if len(hm.Indexes) == 0 {
+		hm.Indexes = map[uint32]*HostInfo{}
 	}
 	hm.Unlock()
 
@@ -218,64 +226,6 @@ func (hm *HostMap) DeleteIndex(index uint32) {
 	}
 }
 
-// This is used to cleanup on recv_error
-func (hm *HostMap) DeleteReverseIndex(index uint32) {
-	hm.Lock()
-	hostinfo, ok := hm.RemoteIndexes[index]
-	if ok {
-		delete(hm.Indexes, hostinfo.localIndexId)
-		delete(hm.RemoteIndexes, index)
-
-		// Check if we have an entry under hostId that matches the same hostinfo
-		// instance. Clean it up as well if we do (they might not match in pendingHostmap)
-		var hostinfo2 *HostInfo
-		hostinfo2, ok = hm.Hosts[hostinfo.hostId]
-		if ok && hostinfo2 == hostinfo {
-			delete(hm.Hosts, hostinfo.hostId)
-		}
-	}
-	hm.Unlock()
-
-	if l.Level >= logrus.DebugLevel {
-		l.WithField("hostMap", m{"mapName": hm.name, "indexNumber": index, "mapTotalSize": len(hm.Indexes)}).
-			Debug("Hostmap remote index deleted")
-	}
-}
-
-func (hm *HostMap) DeleteHostInfo(hostinfo *HostInfo) {
-	hm.Lock()
-
-	// Check if this same hostId is in the hostmap with a different instance.
-	// This could happen if we have an entry in the pending hostmap with different
-	// index values than the one in the main hostmap.
-	hostinfo2, ok := hm.Hosts[hostinfo.hostId]
-	if ok && hostinfo2 != hostinfo {
-		delete(hm.Hosts, hostinfo2.hostId)
-		delete(hm.Indexes, hostinfo2.localIndexId)
-		delete(hm.RemoteIndexes, hostinfo2.remoteIndexId)
-	}
-
-	delete(hm.Hosts, hostinfo.hostId)
-	if len(hm.Hosts) == 0 {
-		hm.Hosts = map[uint32]*HostInfo{}
-	}
-	delete(hm.Indexes, hostinfo.localIndexId)
-	if len(hm.Indexes) == 0 {
-		hm.Indexes = map[uint32]*HostInfo{}
-	}
-	delete(hm.RemoteIndexes, hostinfo.remoteIndexId)
-	if len(hm.RemoteIndexes) == 0 {
-		hm.RemoteIndexes = map[uint32]*HostInfo{}
-	}
-	hm.Unlock()
-
-	if l.Level >= logrus.DebugLevel {
-		l.WithField("hostMap", m{"mapName": hm.name, "mapTotalSize": len(hm.Hosts),
-			"vpnIp": IntIp(hostinfo.hostId), "indexNumber": hostinfo.localIndexId, "remoteIndexNumber": hostinfo.remoteIndexId}).
-			Debug("Hostmap hostInfo deleted")
-	}
-}
-
 func (hm *HostMap) QueryIndex(index uint32) (*HostInfo, error) {
 	//TODO: we probably just want ot return bool instead of error, or at least a static error
 	hm.RLock()
@@ -288,15 +238,23 @@ func (hm *HostMap) QueryIndex(index uint32) (*HostInfo, error) {
 	}
 }
 
+// This function needs to range because we don't keep a map of remote indexes.
 func (hm *HostMap) QueryReverseIndex(index uint32) (*HostInfo, error) {
 	hm.RLock()
-	if h, ok := hm.RemoteIndexes[index]; ok {
-		hm.RUnlock()
-		return h, nil
-	} else {
-		hm.RUnlock()
-		return nil, fmt.Errorf("unable to find reverse index or connectionstate nil in %s hostmap", hm.name)
+	for _, h := range hm.Indexes {
+		if h.ConnectionState != nil && h.remoteIndexId == index {
+			hm.RUnlock()
+			return h, nil
+		}
 	}
+	for _, h := range hm.Hosts {
+		if h.ConnectionState != nil && h.remoteIndexId == index {
+			hm.RUnlock()
+			return h, nil
+		}
+	}
+	hm.RUnlock()
+	return nil, fmt.Errorf("unable to find reverse index or connectionstate nil in %s hostmap", hm.name)
 }
 
 func (hm *HostMap) AddRemote(vpnIp uint32, remote *udpAddr) *HostInfo {
@@ -362,26 +320,36 @@ func (hm *HostMap) queryUnsafeRoute(ip uint32) uint32 {
 	}
 }
 
-// We already have the hm Lock when this is called, so make sure to not call
-// any other methods that might try to grab it again
-func (hm *HostMap) addHostInfo(hostinfo *HostInfo, f *Interface) {
-	remoteCert := hostinfo.ConnectionState.peerCert
-	ip := ip2int(remoteCert.Details.Ips[0].IP)
+func (hm *HostMap) CheckHandshakeCompleteIP(vpnIP uint32) bool {
+	hm.RLock()
+	if i, ok := hm.Hosts[vpnIP]; ok {
+		if i == nil {
+			hm.RUnlock()
+			return false
+		}
+		complete := i.HandshakeComplete
+		hm.RUnlock()
+		return complete
 
-	f.lightHouse.AddRemoteAndReset(ip, hostinfo.remote)
-	if f.serveDns {
-		dnsR.Add(remoteCert.Details.Name+".", remoteCert.Details.Ips[0].IP.String())
 	}
+	hm.RUnlock()
+	return false
+}
 
-	hm.Hosts[hostinfo.hostId] = hostinfo
-	hm.Indexes[hostinfo.localIndexId] = hostinfo
-	hm.RemoteIndexes[hostinfo.remoteIndexId] = hostinfo
+func (hm *HostMap) CheckHandshakeCompleteIndex(index uint32) bool {
+	hm.RLock()
+	if i, ok := hm.Indexes[index]; ok {
+		if i == nil {
+			hm.RUnlock()
+			return false
+		}
+		complete := i.HandshakeComplete
+		hm.RUnlock()
+		return complete
 
-	if l.Level >= logrus.DebugLevel {
-		l.WithField("hostMap", m{"mapName": hm.name, "vpnIp": IntIp(hostinfo.hostId), "mapTotalSize": len(hm.Hosts),
-			"hostinfo": m{"existing": true, "localIndexId": hostinfo.localIndexId, "hostId": IntIp(hostinfo.hostId)}}).
-			Debug("Hostmap vpnIp added")
 	}
+	hm.RUnlock()
+	return false
 }
 
 func (hm *HostMap) ClearRemotes(vpnIP uint32) {
@@ -471,7 +439,8 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []*net.IPNet, ifce *Interface)
 		return
 	}
 
-	if atomic.AddUint32(&i.promoteCounter, 1)&PromoteEvery == 0 {
+	i.promoteCounter++
+	if i.promoteCounter%PromoteEvery == 0 {
 		// return early if we are already on a preferred remote
 		rIP := udp2ip(i.remote)
 		for _, l := range preferredRanges {
@@ -572,12 +541,10 @@ func (i *HostInfo) cachePacket(t NebulaMessageType, st NebulaMessageSubType, pac
 		copy(tempPacket, packet)
 		//l.WithField("trace", string(debug.Stack())).Error("Caching packet", tempPacket)
 		i.packetStore = append(i.packetStore, &cachedPacket{t, st, f, tempPacket})
-		if l.Level >= logrus.DebugLevel {
-			i.logger().
-				WithField("length", len(i.packetStore)).
-				WithField("stored", true).
-				Debugf("Packet store")
-		}
+		i.logger().
+			WithField("length", len(i.packetStore)).
+			WithField("stored", true).
+			Debugf("Packet store")
 
 	} else if l.Level >= logrus.DebugLevel {
 		i.logger().
@@ -597,7 +564,7 @@ func (i *HostInfo) handshakeComplete() {
 	i.HandshakeComplete = true
 	//TODO: this should be managed by the handshake state machine to set it based on how many handshake were seen.
 	// Clamping it to 2 gets us out of the woods for now
-	atomic.StoreUint64(&i.ConnectionState.atomicMessageCounter, 2)
+	*i.ConnectionState.messageCounter = 2
 	i.logger().Debugf("Sending %d stored packets", len(i.packetStore))
 	nb := make([]byte, 12, 12)
 	out := make([]byte, mtu)
@@ -708,6 +675,7 @@ func NewHostInfoDest(addr *udpAddr) *HostInfoDest {
 
 func (hid *HostInfoDest) MarshalJSON() ([]byte, error) {
 	return json.Marshal(m{
+		"active":      hid.active,
 		"address":     hid.addr,
 		"probe_count": hid.probeCounter,
 	})

inside.go

@@ -7,7 +7,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *FirewallPacket, nb, out []byte, q int, localCache ConntrackCache) {
+func (f *Interface) consumeInsidePacket(st NebulaMessageSubType, packet []byte, fwPacket *FirewallPacket, nb, out []byte) {
 	err := newPacket(packet, false, fwPacket)
 	if err != nil {
 		l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
@@ -45,16 +45,16 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *FirewallPacket,
 		// the packet queue.
 		ci.queueLock.Lock()
 		if !ci.ready {
-			hostinfo.cachePacket(message, 0, packet, f.sendMessageNow)
+			hostinfo.cachePacket(message, st, packet, f.sendMessageNow)
 			ci.queueLock.Unlock()
 			return
 		}
 		ci.queueLock.Unlock()
 	}
 
-	dropReason := f.firewall.Drop(packet, *fwPacket, false, hostinfo, trustedCAs, localCache)
+	dropReason := f.firewall.Drop(packet, *fwPacket, false, hostinfo, trustedCAs)
 	if dropReason == nil {
-		mc := f.sendNoMetrics(message, 0, ci, hostinfo, hostinfo.remote, packet, nb, out, q)
+		mc := f.sendNoMetrics(message, st, ci, hostinfo, hostinfo.remote, packet, nb, out)
 		if f.lightHouse != nil && mc%5000 == 0 {
 			f.lightHouse.Query(fwPacket.RemoteIP, f)
 		}
@@ -91,17 +91,6 @@ func (f *Interface) getOrHandshake(vpnIp uint32) *HostInfo {
 		return hostinfo
 	}
 
-	// Handshake is not ready, we need to grab the lock now before we start
-	// the handshake process
-	hostinfo.Lock()
-	defer hostinfo.Unlock()
-
-	// Double check, now that we have the lock
-	ci = hostinfo.ConnectionState
-	if ci != nil && ci.eKey != nil && ci.ready {
-		return hostinfo
-	}
-
 	if ci == nil {
 		// if we don't have a connection state, then send a handshake initiation
 		ci = f.newConnectionState(true, noise.HandshakeIX, []byte{}, 0)
@@ -140,7 +129,7 @@ func (f *Interface) sendMessageNow(t NebulaMessageType, st NebulaMessageSubType,
 	}
 
 	// check if packet is in outbound fw rules
-	dropReason := f.firewall.Drop(p, *fp, false, hostInfo, trustedCAs, nil)
+	dropReason := f.firewall.Drop(p, *fp, false, hostInfo, trustedCAs)
 	if dropReason != nil {
 		if l.Level >= logrus.DebugLevel {
 			l.WithField("fwPacket", fp).
@@ -150,8 +139,8 @@ func (f *Interface) sendMessageNow(t NebulaMessageType, st NebulaMessageSubType,
 		return
 	}
 
-	messageCounter := f.sendNoMetrics(message, st, hostInfo.ConnectionState, hostInfo, hostInfo.remote, p, nb, out, 0)
-	if f.lightHouse != nil && messageCounter%5000 == 0 {
+	f.sendNoMetrics(message, st, hostInfo.ConnectionState, hostInfo, hostInfo.remote, p, nb, out)
+	if f.lightHouse != nil && *hostInfo.ConnectionState.messageCounter%5000 == 0 {
 		f.lightHouse.Query(fp.RemoteIP, f)
 	}
 }
@@ -222,10 +211,10 @@ func (f *Interface) sendMessageToAll(t NebulaMessageType, st NebulaMessageSubTyp
 
 func (f *Interface) send(t NebulaMessageType, st NebulaMessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote *udpAddr, p, nb, out []byte) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0)
+	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out)
 }
 
-func (f *Interface) sendNoMetrics(t NebulaMessageType, st NebulaMessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote *udpAddr, p, nb, out []byte, q int) uint64 {
+func (f *Interface) sendNoMetrics(t NebulaMessageType, st NebulaMessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote *udpAddr, p, nb, out []byte) uint64 {
 	if ci.eKey == nil {
 		//TODO: log warning
 		return 0
@@ -234,36 +223,24 @@ func (f *Interface) sendNoMetrics(t NebulaMessageType, st NebulaMessageSubType,
 	var err error
 	//TODO: enable if we do more than 1 tun queue
 	//ci.writeLock.Lock()
-	c := atomic.AddUint64(&ci.atomicMessageCounter, 1)
+	c := atomic.AddUint64(ci.messageCounter, 1)
 
 	//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
 	out = HeaderEncode(out, Version, uint8(t), uint8(st), hostinfo.remoteIndexId, c)
 	f.connectionManager.Out(hostinfo.hostId)
 
-	// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against
-	// all our IPs and enable a faster roaming.
-	if hostinfo.lastRebindCount != f.rebindCount {
-		//NOTE: there is an update hole if a tunnel isn't used and exactly 256 rebinds occur before the tunnel is
-		// finally used again. This tunnel would eventually be torn down and recreated if this action didn't help.
-		f.lightHouse.Query(hostinfo.hostId, f)
-		hostinfo.lastRebindCount = f.rebindCount
-		if l.Level >= logrus.DebugLevel {
-			l.WithField("vpnIp", hostinfo.hostId).Debug("Lighthouse update triggered for punch due to rebind counter")
-		}
-	}
-
 	out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
 	//TODO: see above note on lock
 	//ci.writeLock.Unlock()
 	if err != nil {
 		hostinfo.logger().WithError(err).
 			WithField("udpAddr", remote).WithField("counter", c).
-			WithField("attemptedCounter", c).
+			WithField("attemptedCounter", ci.messageCounter).
 			Error("Failed to encrypt outgoing packet")
 		return c
 	}
 
-	err = f.writers[q].WriteTo(out, remote)
+	err = f.outside.WriteTo(out, remote)
 	if err != nil {
 		hostinfo.logger().WithError(err).
 			WithField("udpAddr", remote).Error("Failed to write outgoing packet")

interface.go

@@ -5,7 +5,6 @@ import (
 	"io"
 	"net"
 	"os"
-	"runtime"
 	"time"
 
 	"github.com/rcrowley/go-metrics"
@@ -19,9 +18,10 @@ type Inside interface {
 	CidrNet() *net.IPNet
 	DeviceName() string
 	WriteRaw([]byte) error
-	NewMultiQueueReader() (io.ReadWriteCloser, error)
 }
 
+type InsideHandler func(hostInfo *HostInfo, ci *ConnectionState, addr *udpAddr, header *Header, out []byte, packet []byte, fp *FirewallPacket, nb []byte)
+
 type InterfaceConfig struct {
 	HostMap                 *HostMap
 	Outside                 *udpConn
@@ -37,11 +37,10 @@ type InterfaceConfig struct {
 	DropLocalBroadcast      bool
 	DropMulticast           bool
 	UDPBatchSize            int
-	routines                int
+	udpQueues               int
+	tunQueues               int
 	MessageMetrics          *MessageMetrics
 	version                 string
-
-	ConntrackCacheTimeout time.Duration
 }
 
 type Interface struct {
@@ -60,16 +59,12 @@ type Interface struct {
 	dropLocalBroadcast bool
 	dropMulticast      bool
 	udpBatchSize       int
-	routines           int
+	udpQueues          int
+	tunQueues          int
+	version            string
 
-	// rebindCount is used to decide if an active tunnel should trigger a punch notification through a lighthouse
-	rebindCount int8
-	version     string
-
-	conntrackCacheTimeout time.Duration
-
-	writers []*udpConn
-	readers []io.ReadWriteCloser
+	// handlers are mapped by protocol version -> type -> subtype
+	handlers map[uint8]map[NebulaMessageType]map[NebulaMessageSubType]InsideHandler
 
 	metricHandshakes metrics.Histogram
 	messageMetrics   *MessageMetrics
@@ -104,24 +99,47 @@ func NewInterface(c *InterfaceConfig) (*Interface, error) {
 		dropLocalBroadcast: c.DropLocalBroadcast,
 		dropMulticast:      c.DropMulticast,
 		udpBatchSize:       c.UDPBatchSize,
-		routines:           c.routines,
+		udpQueues:          c.udpQueues,
+		tunQueues:          c.tunQueues,
 		version:            c.version,
-		writers:            make([]*udpConn, c.routines),
-		readers:            make([]io.ReadWriteCloser, c.routines),
-
-		conntrackCacheTimeout: c.ConntrackCacheTimeout,
 
 		metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
 		messageMetrics:   c.MessageMetrics,
 	}
 
 	ifce.connectionManager = newConnectionManager(ifce, c.checkInterval, c.pendingDeletionInterval)
+	ifce.handlers = map[uint8]map[NebulaMessageType]map[NebulaMessageSubType]InsideHandler{
+		Version: {
+			handshake: {
+				handshakeIXPSK0: ifce.rxMetrics(ifce.handleHandshakePacket),
+			},
+			message: {
+				subTypeNone: ifce.encrypted(ifce.handleMessagePacket),
+			},
+			recvError: {
+				subTypeNone: ifce.rxMetrics(ifce.handleRecvErrorPacket),
+			},
+			lightHouse: {
+				subTypeNone: ifce.rxMetrics(ifce.encrypted(ifce.handleLighthousePacket)),
+			},
+			test: {
+				testRequest: ifce.rxMetrics(ifce.encrypted(ifce.handleTestPacket)),
+				testReply:   ifce.rxMetrics(ifce.encrypted(ifce.handleTestPacket)),
+			},
+			closeTunnel: {
+				subTypeNone: ifce.rxMetrics(ifce.encrypted(ifce.handleCloseTunnelPacket)),
+			},
+		},
+	}
 
 	return ifce, nil
 }
 
 func (f *Interface) run() {
 	// actually turn on tun dev
+	if err := f.inside.Activate(); err != nil {
+		l.Fatal(err)
+	}
 
 	addr, err := f.outside.LocalAddr()
 	if err != nil {
@@ -132,67 +150,53 @@ func (f *Interface) run() {
 		WithField("build", f.version).WithField("udpAddr", addr).
 		Info("Nebula interface is active")
 
-	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
-
-	// Prepare n tun queues
-	var reader io.ReadWriteCloser = f.inside
-	for i := 0; i < f.routines; i++ {
-		if i > 0 {
-			reader, err = f.inside.NewMultiQueueReader()
-			if err != nil {
-				l.Fatal(err)
-			}
-		}
-		f.readers[i] = reader
-	}
-
-	if err := f.inside.Activate(); err != nil {
-		l.Fatal(err)
-	}
-
 	// Launch n queues to read packets from udp
-	for i := 0; i < f.routines; i++ {
+	for i := 0; i < f.udpQueues; i++ {
 		go f.listenOut(i)
 	}
 
 	// Launch n queues to read packets from tun dev
-	for i := 0; i < f.routines; i++ {
-		go f.listenIn(f.readers[i], i)
+	for i := 0; i < f.tunQueues; i++ {
+		go f.listenIn(i)
 	}
 }
 
 func (f *Interface) listenOut(i int) {
-	runtime.LockOSThread()
+	//TODO: handle error
+	addr, err := f.outside.LocalAddr()
+	if err != nil {
+		l.WithError(err).Error("failed to discover udp listening address")
+	}
 
 	var li *udpConn
-	// TODO clean this up with a coherent interface for each outside connection
 	if i > 0 {
-		li = f.writers[i]
+		//TODO: handle error
+		li, err = NewListener(udp2ip(addr).String(), int(addr.Port), i > 0)
+		if err != nil {
+			l.WithError(err).Error("failed to make a new udp listener")
+		}
 	} else {
 		li = f.outside
 	}
-	li.ListenOut(f, i)
+
+	li.ListenOut(f)
 }
 
-func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
-	runtime.LockOSThread()
-
+func (f *Interface) listenIn(i int) {
 	packet := make([]byte, mtu)
 	out := make([]byte, mtu)
 	fwPacket := &FirewallPacket{}
 	nb := make([]byte, 12, 12)
 
-	conntrackCache := NewConntrackCacheTicker(f.conntrackCacheTimeout)
-
 	for {
-		n, err := reader.Read(packet)
+		n, err := f.inside.Read(packet)
 		if err != nil {
 			l.WithError(err).Error("Error while reading outbound packet")
 			// This only seems to happen when something fatal happens to the fd, so exit.
 			os.Exit(2)
 		}
 
-		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get())
+		f.consumeInsidePacket(subTypeNone, packet[:n], fwPacket, nb, out)
 	}
 }
 
@@ -200,9 +204,7 @@ func (f *Interface) RegisterConfigChangeCallbacks(c *Config) {
 	c.RegisterReloadCallback(f.reloadCA)
 	c.RegisterReloadCallback(f.reloadCertKey)
 	c.RegisterReloadCallback(f.reloadFirewall)
-	for _, udpConn := range f.writers {
-		c.RegisterReloadCallback(udpConn.reloadConfig)
-	}
+	c.RegisterReloadCallback(f.outside.reloadConfig)
 }
 
 func (f *Interface) reloadCA(c *Config) {
@@ -279,13 +281,8 @@ func (f *Interface) reloadFirewall(c *Config) {
 
 func (f *Interface) emitStats(i time.Duration) {
 	ticker := time.NewTicker(i)
-
-	udpStats := NewUDPStatsEmitter(f.writers)
-
 	for range ticker.C {
 		f.firewall.EmitStats()
 		f.handshakeManager.EmitStats()
-
-		udpStats()
 	}
 }

lighthouse.go

@@ -1,7 +1,6 @@
 package nebula
 
 import (
-	"errors"
 	"fmt"
 	"net"
 	"sync"
@@ -12,8 +11,6 @@ import (
 	"github.com/slackhq/nebula/cert"
 )
 
-var ErrHostNotKnown = errors.New("host not known")
-
 type LightHouse struct {
 	sync.RWMutex //Because we concurrently read and write to our maps
 	amLighthouse bool
@@ -41,7 +38,7 @@ type LightHouse struct {
 	staticList  map[uint32]struct{}
 	lighthouses map[uint32]struct{}
 	interval    int
-	nebulaPort  uint32
+	nebulaPort  int
 	punchBack   bool
 	punchDelay  time.Duration
 
@@ -54,7 +51,7 @@ type EncWriter interface {
 	SendMessageToAll(t NebulaMessageType, st NebulaMessageSubType, vpnIp uint32, p, nb, out []byte)
 }
 
-func NewLightHouse(amLighthouse bool, myIp uint32, ips []uint32, interval int, nebulaPort uint32, pc *udpConn, punchBack bool, punchDelay time.Duration, metricsEnabled bool) *LightHouse {
+func NewLightHouse(amLighthouse bool, myIp uint32, ips []uint32, interval int, nebulaPort int, pc *udpConn, punchBack bool, punchDelay time.Duration, metricsEnabled bool) *LightHouse {
 	h := LightHouse{
 		amLighthouse: amLighthouse,
 		myIp:         myIp,
@@ -116,7 +113,7 @@ func (lh *LightHouse) Query(ip uint32, f EncWriter) ([]udpAddr, error) {
 		return v, nil
 	}
 	lh.RUnlock()
-	return nil, ErrHostNotKnown
+	return nil, fmt.Errorf("host %s not known, queries sent to lighthouses", IntIp(ip))
 }
 
 // This is asynchronous so no reply should be expected
@@ -172,9 +169,9 @@ func (lh *LightHouse) AddRemote(vpnIP uint32, toIp *udpAddr, static bool) {
 	}
 
 	lh.Lock()
-	defer lh.Unlock()
 	for _, v := range lh.addrMap[vpnIP] {
 		if v.Equals(toIp) {
+			lh.Unlock()
 			return
 		}
 	}
@@ -190,6 +187,7 @@ func (lh *LightHouse) AddRemote(vpnIP uint32, toIp *udpAddr, static bool) {
 		lh.staticList[vpnIP] = struct{}{}
 	}
 	lh.addrMap[vpnIP] = append(lh.addrMap[vpnIP], *toIp)
+	lh.Unlock()
 }
 
 func (lh *LightHouse) AddRemoteAndReset(vpnIP uint32, toIp *udpAddr) {
@@ -207,6 +205,12 @@ func (lh *LightHouse) IsLighthouseIP(vpnIP uint32) bool {
 	return false
 }
 
+// Quick generators for protobuf
+
+func NewLhQueryByIpString(VpnIp string) *NebulaMeta {
+	return NewLhQueryByInt(ip2int(net.ParseIP(VpnIp)))
+}
+
 func NewLhQueryByInt(VpnIp uint32) *NebulaMeta {
 	return &NebulaMeta{
 		Type: NebulaMeta_HostQuery,
@@ -216,12 +220,26 @@ func NewLhQueryByInt(VpnIp uint32) *NebulaMeta {
 	}
 }
 
-func NewIpAndPort(ip net.IP, port uint32) IpAndPort {
-	return IpAndPort{Ip: ip2int(ip), Port: port}
+func NewLhWhoami() *NebulaMeta {
+	return &NebulaMeta{
+		Type:    NebulaMeta_HostWhoami,
+		Details: &NebulaMetaDetails{},
+	}
 }
 
-func NewIpAndPortFromUDPAddr(addr udpAddr) IpAndPort {
-	return IpAndPort{Ip: udp2ipInt(&addr), Port: uint32(addr.Port)}
+// End Quick generators for protobuf
+
+func NewIpAndPortFromUDPAddr(addr udpAddr) *IpAndPort {
+	return &IpAndPort{Ip: udp2ipInt(&addr), Port: uint32(addr.Port)}
+}
+
+func NewIpAndPortsFromNetIps(ips []udpAddr) *[]*IpAndPort {
+	var iap []*IpAndPort
+	for _, e := range ips {
+		// Only add IPs that aren't my VPN/tun IP
+		iap = append(iap, NewIpAndPortFromUDPAddr(e))
+	}
+	return &iap
 }
 
 func (lh *LightHouse) LhUpdateWorker(f EncWriter) {
@@ -230,105 +248,42 @@ func (lh *LightHouse) LhUpdateWorker(f EncWriter) {
 	}
 
 	for {
-		lh.SendUpdate(f)
+		ipp := []*IpAndPort{}
+
+		for _, e := range *localIps(lh.localAllowList) {
+			// Only add IPs that aren't my VPN/tun IP
+			if ip2int(e) != lh.myIp {
+				ipp = append(ipp, &IpAndPort{Ip: ip2int(e), Port: uint32(lh.nebulaPort)})
+				//fmt.Println(e)
+			}
+		}
+		m := &NebulaMeta{
+			Type: NebulaMeta_HostUpdateNotification,
+			Details: &NebulaMetaDetails{
+				VpnIp:      lh.myIp,
+				IpAndPorts: ipp,
+			},
+		}
+
+		lh.metricTx(NebulaMeta_HostUpdateNotification, int64(len(lh.lighthouses)))
+		nb := make([]byte, 12, 12)
+		out := make([]byte, mtu)
+		for vpnIp := range lh.lighthouses {
+			mm, err := proto.Marshal(m)
+			if err != nil {
+				l.Debugf("Invalid marshal to update")
+			}
+			//l.Error("LIGHTHOUSE PACKET SEND", mm)
+			f.SendMessageToVpnIp(lightHouse, 0, vpnIp, mm, nb, out)
+
+		}
 		time.Sleep(time.Second * time.Duration(lh.interval))
 	}
 }
 
-func (lh *LightHouse) SendUpdate(f EncWriter) {
-	var ipps []*IpAndPort
-
-	for _, e := range *localIps(lh.localAllowList) {
-		// Only add IPs that aren't my VPN/tun IP
-		if ip2int(e) != lh.myIp {
-			ipp := NewIpAndPort(e, lh.nebulaPort)
-			ipps = append(ipps, &ipp)
-		}
-	}
-	m := &NebulaMeta{
-		Type: NebulaMeta_HostUpdateNotification,
-		Details: &NebulaMetaDetails{
-			VpnIp:      lh.myIp,
-			IpAndPorts: ipps,
-		},
-	}
-
-	lh.metricTx(NebulaMeta_HostUpdateNotification, int64(len(lh.lighthouses)))
-	nb := make([]byte, 12, 12)
-	out := make([]byte, mtu)
-	for vpnIp := range lh.lighthouses {
-		mm, err := proto.Marshal(m)
-		if err != nil {
-			l.Debugf("Invalid marshal to update")
-		}
-		//l.Error("LIGHTHOUSE PACKET SEND", mm)
-		f.SendMessageToVpnIp(lightHouse, 0, vpnIp, mm, nb, out)
-
-	}
-}
-
-type LightHouseHandler struct {
-	lh   *LightHouse
-	nb   []byte
-	out  []byte
-	meta *NebulaMeta
-	iap  []IpAndPort
-	iapp []*IpAndPort
-}
-
-func (lh *LightHouse) NewRequestHandler() *LightHouseHandler {
-	lhh := &LightHouseHandler{
-		lh:  lh,
-		nb:  make([]byte, 12, 12),
-		out: make([]byte, mtu),
-
-		meta: &NebulaMeta{
-			Details: &NebulaMetaDetails{},
-		},
-	}
-
-	lhh.resizeIpAndPorts(10)
-
-	return lhh
-}
-
-// This method is similar to Reset(), but it re-uses the pointer structs
-// so that we don't have to re-allocate them
-func (lhh *LightHouseHandler) resetMeta() *NebulaMeta {
-	details := lhh.meta.Details
-
-	details.Reset()
-	lhh.meta.Reset()
-	lhh.meta.Details = details
-
-	return lhh.meta
-}
-
-func (lhh *LightHouseHandler) resizeIpAndPorts(n int) {
-	if cap(lhh.iap) < n {
-		lhh.iap = make([]IpAndPort, n)
-		lhh.iapp = make([]*IpAndPort, n)
-
-		for i := range lhh.iap {
-			lhh.iapp[i] = &lhh.iap[i]
-		}
-	}
-	lhh.iap = lhh.iap[:n]
-	lhh.iapp = lhh.iapp[:n]
-}
-
-func (lhh *LightHouseHandler) setIpAndPortsFromNetIps(ips []udpAddr) []*IpAndPort {
-	lhh.resizeIpAndPorts(len(ips))
-	for i, e := range ips {
-		lhh.iap[i] = NewIpAndPortFromUDPAddr(e)
-	}
-	return lhh.iapp
-}
-
-func (lhh *LightHouseHandler) HandleRequest(rAddr *udpAddr, vpnIp uint32, p []byte, c *cert.NebulaCertificate, f EncWriter) {
-	lh := lhh.lh
-	n := lhh.resetMeta()
-	err := proto.UnmarshalMerge(p, n)
+func (lh *LightHouse) HandleRequest(rAddr *udpAddr, vpnIp uint32, p []byte, c *cert.NebulaCertificate, f EncWriter) {
+	n := &NebulaMeta{}
+	err := proto.Unmarshal(p, n)
 	if err != nil {
 		l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).WithField("udpAddr", rAddr).
 			Error("Failed to unmarshal lighthouse packet")
@@ -359,18 +314,21 @@ func (lhh *LightHouseHandler) HandleRequest(rAddr *udpAddr, vpnIp uint32, p []by
 			//l.Debugf("Can't answer query %s from %s because error: %s", IntIp(n.Details.VpnIp), rAddr, err)
 			return
 		} else {
-			reqVpnIP := n.Details.VpnIp
-			n = lhh.resetMeta()
-			n.Type = NebulaMeta_HostQueryReply
-			n.Details.VpnIp = reqVpnIP
-			n.Details.IpAndPorts = lhh.setIpAndPortsFromNetIps(ips)
-			reply, err := proto.Marshal(n)
+			iap := NewIpAndPortsFromNetIps(ips)
+			answer := &NebulaMeta{
+				Type: NebulaMeta_HostQueryReply,
+				Details: &NebulaMetaDetails{
+					VpnIp:      n.Details.VpnIp,
+					IpAndPorts: *iap,
+				},
+			}
+			reply, err := proto.Marshal(answer)
 			if err != nil {
 				l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).Error("Failed to marshal lighthouse host query reply")
 				return
 			}
 			lh.metricTx(NebulaMeta_HostQueryReply, 1)
-			f.SendMessageToVpnIp(lightHouse, 0, vpnIp, reply, lhh.nb, lhh.out[:0])
+			f.SendMessageToVpnIp(lightHouse, 0, vpnIp, reply, make([]byte, 12, 12), make([]byte, mtu))
 
 			// This signals the other side to punch some zero byte udp packets
 			ips, err = lh.Query(vpnIp, f)
@@ -379,13 +337,17 @@ func (lhh *LightHouseHandler) HandleRequest(rAddr *udpAddr, vpnIp uint32, p []by
 				return
 			} else {
 				//l.Debugln("Notify host to punch", iap)
-				n = lhh.resetMeta()
-				n.Type = NebulaMeta_HostPunchNotification
-				n.Details.VpnIp = vpnIp
-				n.Details.IpAndPorts = lhh.setIpAndPortsFromNetIps(ips)
-				reply, _ := proto.Marshal(n)
+				iap = NewIpAndPortsFromNetIps(ips)
+				answer = &NebulaMeta{
+					Type: NebulaMeta_HostPunchNotification,
+					Details: &NebulaMetaDetails{
+						VpnIp:      vpnIp,
+						IpAndPorts: *iap,
+					},
+				}
+				reply, _ := proto.Marshal(answer)
 				lh.metricTx(NebulaMeta_HostPunchNotification, 1)
-				f.SendMessageToVpnIp(lightHouse, 0, reqVpnIP, reply, lhh.nb, lhh.out[:0])
+				f.SendMessageToVpnIp(lightHouse, 0, n.Details.VpnIp, reply, make([]byte, 12, 12), make([]byte, mtu))
 			}
 			//fmt.Println(reply, remoteaddr)
 		}
@@ -439,9 +401,6 @@ func (lhh *LightHouseHandler) HandleRequest(rAddr *udpAddr, vpnIp uint32, p []by
 			go func() {
 				time.Sleep(time.Second * 5)
 				l.Debugf("Sending a nebula test packet to vpn ip %s", IntIp(n.Details.VpnIp))
-				// TODO we have to allocate a new output buffer here since we are spawning a new goroutine
-				// for each punchBack packet. We should move this into a timerwheel or a single goroutine
-				// managed by a channel.
 				f.SendMessageToVpnIp(test, testRequest, n.Details.VpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
 			}()
 		}

lighthouse_test.go

@@ -36,19 +36,12 @@ func TestNewipandportfromudpaddr(t *testing.T) {
 	assert.Equal(t, uint32(12345), meh.Port)
 }
 
-func TestSetipandportsfromudpaddrs(t *testing.T) {
+func TestNewipandportsfromudpaddrs(t *testing.T) {
 	blah := NewUDPAddrFromString("1.2.2.3:12345")
 	blah2 := NewUDPAddrFromString("9.9.9.9:47828")
 	group := []udpAddr{*blah, *blah2}
-	var lh *LightHouse
-	lhh := lh.NewRequestHandler()
-	result := lhh.setIpAndPortsFromNetIps(group)
-	assert.IsType(t, []*IpAndPort{}, result)
-	assert.Len(t, result, 2)
-	assert.Equal(t, uint32(0x01020203), result[0].Ip)
-	assert.Equal(t, uint32(12345), result[0].Port)
-	assert.Equal(t, uint32(0x09090909), result[1].Ip)
-	assert.Equal(t, uint32(47828), result[1].Port)
+	hah := NewIpAndPortsFromNetIps(group)
+	assert.IsType(t, &[]*IpAndPort{}, hah)
 	//t.Error(reflect.TypeOf(hah))
 
 }
@@ -73,86 +66,6 @@ func Test_lhStaticMapping(t *testing.T) {
 	assert.EqualError(t, err, "Lighthouse 10.128.0.3 does not have a static_host_map entry")
 }
 
-func BenchmarkLighthouseHandleRequest(b *testing.B) {
-	lh1 := "10.128.0.2"
-	lh1IP := net.ParseIP(lh1)
-
-	udpServer, _ := NewListener("0.0.0.0", 0, true)
-
-	lh := NewLightHouse(true, 1, []uint32{ip2int(lh1IP)}, 10, 10003, udpServer, false, 1, false)
-
-	hAddr := NewUDPAddrFromString("4.5.6.7:12345")
-	hAddr2 := NewUDPAddrFromString("4.5.6.7:12346")
-	lh.addrMap[3] = []udpAddr{*hAddr, *hAddr2}
-
-	rAddr := NewUDPAddrFromString("1.2.2.3:12345")
-	rAddr2 := NewUDPAddrFromString("1.2.2.3:12346")
-	lh.addrMap[2] = []udpAddr{*rAddr, *rAddr2}
-
-	mw := &mockEncWriter{}
-
-	b.Run("notfound", func(b *testing.B) {
-		lhh := lh.NewRequestHandler()
-		req := &NebulaMeta{
-			Type: NebulaMeta_HostQuery,
-			Details: &NebulaMetaDetails{
-				VpnIp:      4,
-				IpAndPorts: nil,
-			},
-		}
-		p, err := proto.Marshal(req)
-		assert.NoError(b, err)
-		for n := 0; n < b.N; n++ {
-			lhh.HandleRequest(rAddr, 2, p, nil, mw)
-		}
-	})
-	b.Run("found", func(b *testing.B) {
-		lhh := lh.NewRequestHandler()
-		req := &NebulaMeta{
-			Type: NebulaMeta_HostQuery,
-			Details: &NebulaMetaDetails{
-				VpnIp:      3,
-				IpAndPorts: nil,
-			},
-		}
-		p, err := proto.Marshal(req)
-		assert.NoError(b, err)
-
-		for n := 0; n < b.N; n++ {
-			lhh.HandleRequest(rAddr, 2, p, nil, mw)
-		}
-	})
-}
-
-func Test_lhRemoteAllowList(t *testing.T) {
-	c := NewConfig()
-	c.Settings["remoteallowlist"] = map[interface{}]interface{}{
-		"10.20.0.0/12": false,
-	}
-	allowList, err := c.GetAllowList("remoteallowlist", false)
-	assert.Nil(t, err)
-
-	lh1 := "10.128.0.2"
-	lh1IP := net.ParseIP(lh1)
-
-	udpServer, _ := NewListener("0.0.0.0", 0, true)
-
-	lh := NewLightHouse(true, 1, []uint32{ip2int(lh1IP)}, 10, 10003, udpServer, false, 1, false)
-	lh.SetRemoteAllowList(allowList)
-
-	remote1 := "10.20.0.3"
-	remote1IP := net.ParseIP(remote1)
-	lh.AddRemote(ip2int(remote1IP), NewUDPAddr(ip2int(remote1IP), uint16(4242)), true)
-	assert.Nil(t, lh.addrMap[ip2int(remote1IP)])
-
-	remote2 := "10.128.0.3"
-	remote2IP := net.ParseIP(remote2)
-	remote2UDPAddr := NewUDPAddr(ip2int(remote2IP), uint16(4242))
-
-	lh.AddRemote(ip2int(remote2IP), remote2UDPAddr, true)
-	assert.Equal(t, remote2UDPAddr, &lh.addrMap[ip2int(remote2IP)][0])
-}
-
 //func NewLightHouse(amLighthouse bool, myIp uint32, ips []string, interval int, nebulaPort int, pc *udpConn, punchBack bool) *LightHouse {
 
 /*

main.go

@@ -93,49 +93,13 @@ func Main(config *Config, configTest bool, buildVersion string, logger *logrus.L
 	// tun config, listeners, anything modifying the computer should be below
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-	var routines int
-
-	// If `routines` is set, use that and ignore the specific values
-	if routines = config.GetInt("routines", 0); routines != 0 {
-		if routines < 1 {
-			routines = 1
-		}
-		if routines > 1 {
-			l.WithField("routines", routines).Info("Using multiple routines")
-		}
-	} else {
-		// deprecated and undocumented
-		tunQueues := config.GetInt("tun.routines", 1)
-		udpQueues := config.GetInt("listen.routines", 1)
-		if tunQueues > udpQueues {
-			routines = tunQueues
-		} else {
-			routines = udpQueues
-		}
-		if routines != 1 {
-			l.WithField("routines", routines).Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead")
-		}
-	}
-
-	// EXPERIMENTAL
-	// Intentionally not documented yet while we do more testing and determine
-	// a good default value.
-	conntrackCacheTimeout := config.GetDuration("firewall.conntrack.routine_cache_timeout", 0)
-	if routines > 1 && !config.IsSet("firewall.conntrack.routine_cache_timeout") {
-		// Use a different default if we are running with multiple routines
-		conntrackCacheTimeout = 1 * time.Second
-	}
-	if conntrackCacheTimeout > 0 {
-		l.WithField("duration", conntrackCacheTimeout).Info("Using routine-local conntrack cache")
-	}
-
 	var tun Inside
 	if !configTest {
 		config.CatchHUP()
 
 		switch {
 		case config.GetBool("tun.disabled", false):
-			tun = newDisabledTun(tunCidr, config.GetInt("tun.tx_queue", 500), config.GetBool("stats.message_metrics", false), l)
+			tun = newDisabledTun(tunCidr, l)
 		case tunFd != nil:
 			tun, err = newTunFromFd(
 				*tunFd,
@@ -153,7 +117,6 @@ func Main(config *Config, configTest bool, buildVersion string, logger *logrus.L
 				routes,
 				unsafeRoutes,
 				config.GetInt("tun.tx_queue", 500),
-				routines > 1,
 			)
 		}
 
@@ -163,27 +126,15 @@ func Main(config *Config, configTest bool, buildVersion string, logger *logrus.L
 	}
 
 	// set up our UDP listener
-	udpConns := make([]*udpConn, routines)
-	port := config.GetInt("listen.port", 0)
+	udpQueues := config.GetInt("listen.routines", 1)
+	var udpServer *udpConn
 
 	if !configTest {
-		for i := 0; i < routines; i++ {
-			udpServer, err := NewListener(config.GetString("listen.host", "0.0.0.0"), port, routines > 1)
-			if err != nil {
-				return nil, NewContextualError("Failed to open udp listener", m{"queue": i}, err)
-			}
-			udpServer.reloadConfig(config)
-			udpConns[i] = udpServer
-
-			// If port is dynamic, discover it
-			if port == 0 {
-				uPort, err := udpServer.LocalAddr()
-				if err != nil {
-					return nil, NewContextualError("Failed to get listening port", nil, err)
-				}
-				port = int(uPort.Port)
-			}
+		udpServer, err = NewListener(config.GetString("listen.host", "0.0.0.0"), config.GetInt("listen.port", 0), udpQueues > 1)
+		if err != nil {
+			return nil, NewContextualError("Failed to open udp listener", nil, err)
 		}
+		udpServer.reloadConfig(config)
 	}
 
 	// Set up my internal host map
@@ -239,16 +190,21 @@ func Main(config *Config, configTest bool, buildVersion string, logger *logrus.L
 	punchy := NewPunchyFromConfig(config)
 	if punchy.Punch && !configTest {
 		l.Info("UDP hole punching enabled")
-		go hostMap.Punchy(udpConns[0])
+		go hostMap.Punchy(udpServer)
+	}
+
+	port := config.GetInt("listen.port", 0)
+	// If port is dynamic, discover it
+	if port == 0 && !configTest {
+		uPort, err := udpServer.LocalAddr()
+		if err != nil {
+			return nil, NewContextualError("Failed to get listening port", nil, err)
+		}
+		port = int(uPort.Port)
 	}
 
 	amLighthouse := config.GetBool("lighthouse.am_lighthouse", false)
 
-	// fatal if am_lighthouse is enabled but we are using an ephemeral port
-	if amLighthouse && (config.GetInt("listen.port", 0) == 0) {
-		return nil, NewContextualError("lighthouse.am_lighthouse enabled on node but no port number is set in config", nil, nil)
-	}
-
 	// warn if am_lighthouse is enabled but upstream lighthouses exists
 	rawLighthouseHosts := config.GetStringSlice("lighthouse.hosts", []string{})
 	if amLighthouse && len(rawLighthouseHosts) != 0 {
@@ -273,8 +229,8 @@ func Main(config *Config, configTest bool, buildVersion string, logger *logrus.L
 		lighthouseHosts,
 		//TODO: change to a duration
 		config.GetInt("lighthouse.interval", 10),
-		uint32(port),
-		udpConns[0],
+		port,
+		udpServer,
 		punchy.Respond,
 		punchy.Delay,
 		config.GetBool("stats.lighthouse_metrics", false),
@@ -348,7 +304,7 @@ func Main(config *Config, configTest bool, buildVersion string, logger *logrus.L
 		messageMetrics: messageMetrics,
 	}
 
-	handshakeManager := NewHandshakeManager(tunCidr, preferredRanges, hostMap, lightHouse, udpConns[0], handshakeConfig)
+	handshakeManager := NewHandshakeManager(tunCidr, preferredRanges, hostMap, lightHouse, udpServer, handshakeConfig)
 	lightHouse.handshakeTrigger = handshakeManager.trigger
 
 	//TODO: These will be reused for psk
@@ -361,7 +317,7 @@ func Main(config *Config, configTest bool, buildVersion string, logger *logrus.L
 	ifConfig := &InterfaceConfig{
 		HostMap:                 hostMap,
 		Inside:                  tun,
-		Outside:                 udpConns[0],
+		Outside:                 udpServer,
 		certState:               cs,
 		Cipher:                  config.GetString("cipher", "aes"),
 		Firewall:                fw,
@@ -373,11 +329,10 @@ func Main(config *Config, configTest bool, buildVersion string, logger *logrus.L
 		DropLocalBroadcast:      config.GetBool("tun.drop_local_broadcast", false),
 		DropMulticast:           config.GetBool("tun.drop_multicast", false),
 		UDPBatchSize:            config.GetInt("listen.batch", 64),
-		routines:                routines,
+		udpQueues:               udpQueues,
+		tunQueues:               config.GetInt("tun.routines", 1),
 		MessageMetrics:          messageMetrics,
 		version:                 buildVersion,
-
-		ConntrackCacheTimeout: conntrackCacheTimeout,
 	}
 
 	switch ifConfig.Cipher {
@@ -396,10 +351,6 @@ func Main(config *Config, configTest bool, buildVersion string, logger *logrus.L
 			return nil, fmt.Errorf("failed to initialize interface: %s", err)
 		}
 
-		// TODO: Better way to attach these, probably want a new interface in InterfaceConfig
-		// I don't want to make this initial commit too far-reaching though
-		ifce.writers = udpConns
-
 		ifce.RegisterConfigChangeCallbacks(config)
 
 		go handshakeManager.Run(ifce)

outside.go

@@ -17,7 +17,7 @@ const (
 	minFwPacketLen = 4
 )
 
-func (f *Interface) readOutsidePackets(addr *udpAddr, out []byte, packet []byte, header *Header, fwPacket *FirewallPacket, lhh *LightHouseHandler, nb []byte, q int, localCache ConntrackCache) {
+func (f *Interface) readOutsidePackets(addr *udpAddr, out []byte, packet []byte, header *Header, fwPacket *FirewallPacket, nb []byte) {
 	err := header.Parse(packet)
 	if err != nil {
 		// TODO: best if we return this and let caller log
@@ -39,97 +39,15 @@ func (f *Interface) readOutsidePackets(addr *udpAddr, out []byte, packet []byte,
 		ci = hostinfo.ConnectionState
 	}
 
-	switch header.Type {
-	case message:
-		if !f.handleEncrypted(ci, addr, header) {
-			return
-		}
+	handle := f.handlers[header.Version][header.Type][header.Subtype]
 
-		f.decryptToTun(hostinfo, header.MessageCounter, out, packet, fwPacket, nb, q, localCache)
-
-		// Fallthrough to the bottom to record incoming traffic
-
-	case lightHouse:
-		f.messageMetrics.Rx(header.Type, header.Subtype, 1)
-		if !f.handleEncrypted(ci, addr, header) {
-			return
-		}
-
-		d, err := f.decrypt(hostinfo, header.MessageCounter, out, packet, header, nb)
-		if err != nil {
-			hostinfo.logger().WithError(err).WithField("udpAddr", addr).
-				WithField("packet", packet).
-				Error("Failed to decrypt lighthouse packet")
-
-			//TODO: maybe after build 64 is out? 06/14/2018 - NB
-			//f.sendRecvError(net.Addr(addr), header.RemoteIndex)
-			return
-		}
-
-		lhh.HandleRequest(addr, hostinfo.hostId, d, hostinfo.GetCert(), f)
-
-		// Fallthrough to the bottom to record incoming traffic
-
-	case test:
-		f.messageMetrics.Rx(header.Type, header.Subtype, 1)
-		if !f.handleEncrypted(ci, addr, header) {
-			return
-		}
-
-		d, err := f.decrypt(hostinfo, header.MessageCounter, out, packet, header, nb)
-		if err != nil {
-			hostinfo.logger().WithError(err).WithField("udpAddr", addr).
-				WithField("packet", packet).
-				Error("Failed to decrypt test packet")
-
-			//TODO: maybe after build 64 is out? 06/14/2018 - NB
-			//f.sendRecvError(net.Addr(addr), header.RemoteIndex)
-			return
-		}
-
-		if header.Subtype == testRequest {
-			// This testRequest might be from TryPromoteBest, so we should roam
-			// to the new IP address before responding
-			f.handleHostRoaming(hostinfo, addr)
-			f.send(test, testReply, ci, hostinfo, hostinfo.remote, d, nb, out)
-		}
-
-		// Fallthrough to the bottom to record incoming traffic
-
-		// Non encrypted messages below here, they should not fall through to avoid tracking incoming traffic since they
-		// are unauthenticated
-
-	case handshake:
-		f.messageMetrics.Rx(header.Type, header.Subtype, 1)
-		HandleIncomingHandshake(f, addr, packet, header, hostinfo)
-		return
-
-	case recvError:
-		f.messageMetrics.Rx(header.Type, header.Subtype, 1)
-		f.handleRecvError(addr, header)
-		return
-
-	case closeTunnel:
-		f.messageMetrics.Rx(header.Type, header.Subtype, 1)
-		if !f.handleEncrypted(ci, addr, header) {
-			return
-		}
-
-		hostinfo.logger().WithField("udpAddr", addr).
-			Info("Close tunnel received, tearing down.")
-
-		f.closeTunnel(hostinfo)
-		return
-
-	default:
+	if handle == nil {
 		f.messageMetrics.Rx(header.Type, header.Subtype, 1)
 		hostinfo.logger().Debugf("Unexpected packet received from %s", addr)
 		return
 	}
 
-	f.handleHostRoaming(hostinfo, addr)
-
-	f.connectionManager.In(hostinfo.hostId)
+	handle(hostinfo, ci, addr, header, out, packet, fwPacket, nb)
 }
 
 func (f *Interface) closeTunnel(hostInfo *HostInfo) {
@@ -137,7 +55,8 @@ func (f *Interface) closeTunnel(hostInfo *HostInfo) {
 	f.connectionManager.ClearIP(hostInfo.hostId)
 	f.connectionManager.ClearPendingDeletion(hostInfo.hostId)
 	f.lightHouse.DeleteVpnIP(hostInfo.hostId)
-	f.hostMap.DeleteHostInfo(hostInfo)
+	f.hostMap.DeleteVpnIP(hostInfo.hostId)
+	f.hostMap.DeleteIndex(hostInfo.localIndexId)
 }
 
 func (f *Interface) handleHostRoaming(hostinfo *HostInfo, addr *udpAddr) {
@@ -146,10 +65,10 @@ func (f *Interface) handleHostRoaming(hostinfo *HostInfo, addr *udpAddr) {
 			hostinfo.logger().WithField("newAddr", addr).Debug("lighthouse.remote_allow_list denied roaming")
 			return
 		}
-		if !hostinfo.lastRoam.IsZero() && addr.Equals(hostinfo.lastRoamRemote) && time.Since(hostinfo.lastRoam) < RoamingSuppressSeconds*time.Second {
+		if !hostinfo.lastRoam.IsZero() && addr.Equals(hostinfo.lastRoamRemote) && time.Since(hostinfo.lastRoam) < RoamingSupressSeconds*time.Second {
 			if l.Level >= logrus.DebugLevel {
 				hostinfo.logger().WithField("udpAddr", hostinfo.remote).WithField("newAddr", addr).
-					Debugf("Suppressing roam back to previous remote for %d seconds", RoamingSuppressSeconds)
+					Debugf("Supressing roam back to previous remote for %d seconds", RoamingSupressSeconds)
 			}
 			return
 		}
@@ -256,7 +175,7 @@ func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []
 	return out, nil
 }
 
-func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *FirewallPacket, nb []byte, q int, localCache ConntrackCache) {
+func (f *Interface) decryptTo(write func([]byte) error, hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *FirewallPacket, nb []byte) {
 	var err error
 
 	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:HeaderLen], packet[HeaderLen:], messageCounter, nb)
@@ -280,7 +199,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 		return
 	}
 
-	dropReason := f.firewall.Drop(out, *fwPacket, true, hostinfo, trustedCAs, localCache)
+	dropReason := f.firewall.Drop(out, *fwPacket, true, hostinfo, trustedCAs)
 	if dropReason != nil {
 		if l.Level >= logrus.DebugLevel {
 			hostinfo.logger().WithField("fwPacket", fwPacket).
@@ -291,7 +210,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 	}
 
 	f.connectionManager.In(hostinfo.hostId)
-	_, err = f.readers[q].Write(out)
+	err = write(out)
 	if err != nil {
 		l.WithError(err).Error("Failed to write to tun")
 	}
@@ -311,24 +230,20 @@ func (f *Interface) sendRecvError(endpoint *udpAddr, index uint32) {
 }
 
 func (f *Interface) handleRecvError(addr *udpAddr, h *Header) {
+	// This flag is to stop caring about recv_error from old versions
+	// This should go away when the old version is gone from prod
 	if l.Level >= logrus.DebugLevel {
 		l.WithField("index", h.RemoteIndex).
 			WithField("udpAddr", addr).
 			Debug("Recv error received")
 	}
 
-	// First, clean up in the pending hostmap
-	f.handshakeManager.pendingHostMap.DeleteReverseIndex(h.RemoteIndex)
-
 	hostinfo, err := f.hostMap.QueryReverseIndex(h.RemoteIndex)
 	if err != nil {
 		l.Debugln(err, ": ", h.RemoteIndex)
 		return
 	}
 
-	hostinfo.Lock()
-	defer hostinfo.Unlock()
-
 	if !hostinfo.RecvErrorExceeded() {
 		return
 	}
@@ -337,13 +252,17 @@ func (f *Interface) handleRecvError(addr *udpAddr, h *Header) {
 		return
 	}
 
+	id := hostinfo.localIndexId
+	host := hostinfo.hostId
 	// We delete this host from the main hostmap
-	f.hostMap.DeleteHostInfo(hostinfo)
+	f.hostMap.DeleteIndex(id)
+	f.hostMap.DeleteVpnIP(host)
 	// We also delete it from pending to allow for
 	// fast reconnect. We must null the connectionstate
 	// or a counter reuse may happen
 	hostinfo.ConnectionState = nil
-	f.handshakeManager.DeleteHostInfo(hostinfo)
+	f.handshakeManager.DeleteIndex(id)
+	f.handshakeManager.DeleteVpnIP(host)
 }
 
 /*

ssh.go

@@ -11,7 +11,6 @@ import (
 	"reflect"
 	"runtime/pprof"
 	"strings"
-	"sync/atomic"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
@@ -354,7 +353,7 @@ func sshListHostMap(hostMap *HostMap, a interface{}, w sshd.StringWriter) error
 			}
 
 			if v.ConnectionState != nil {
-				h["messageCounter"] = atomic.LoadUint64(&v.ConnectionState.atomicMessageCounter)
+				h["messageCounter"] = v.ConnectionState.messageCounter
 			}
 
 			d[x] = h

tun_android.go

@@ -37,7 +37,7 @@ func newTunFromFd(deviceFd int, cidr *net.IPNet, defaultMTU int, routes []route,
 	return
 }
 
-func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
+func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
 	return nil, fmt.Errorf("newTun not supported in Android")
 }
 
@@ -74,7 +74,3 @@ func (c *Tun) CidrNet() *net.IPNet {
 func (c *Tun) DeviceName() string {
 	return c.Device
 }
-
-func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
-}

tun_darwin.go

@@ -4,7 +4,6 @@ package nebula
 
 import (
 	"fmt"
-	"io"
 	"net"
 	"os/exec"
 	"strconv"
@@ -21,7 +20,7 @@ type Tun struct {
 	*water.Interface
 }
 
-func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
+func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
 	if len(routes) > 0 {
 		return nil, fmt.Errorf("route MTU not supported in Darwin")
 	}
@@ -81,7 +80,3 @@ func (c *Tun) WriteRaw(b []byte) error {
 	_, err := c.Write(b)
 	return err
 }
-
-func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
-}

tun_disabled.go

@@ -1,43 +1,26 @@
 package nebula
 
 import (
-	"encoding/binary"
 	"fmt"
 	"io"
 	"net"
 	"strings"
 
-	"github.com/rcrowley/go-metrics"
-	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 )
 
 type disabledTun struct {
-	read   chan []byte
+	block  chan struct{}
 	cidr   *net.IPNet
 	logger *log.Logger
-
-	// Track these metrics since we don't have the tun device to do it for us
-	tx metrics.Counter
-	rx metrics.Counter
 }
 
-func newDisabledTun(cidr *net.IPNet, queueLen int, metricsEnabled bool, l *log.Logger) *disabledTun {
-	tun := &disabledTun{
+func newDisabledTun(cidr *net.IPNet, l *log.Logger) *disabledTun {
+	return &disabledTun{
 		cidr:   cidr,
-		read:   make(chan []byte, queueLen),
+		block:  make(chan struct{}),
 		logger: l,
 	}
-
-	if metricsEnabled {
-		tun.tx = metrics.GetOrRegisterCounter("messages.tx.message", nil)
-		tun.rx = metrics.GetOrRegisterCounter("messages.rx.message", nil)
-	} else {
-		tun.tx = &metrics.NilCounter{}
-		tun.rx = &metrics.NilCounter{}
-	}
-
-	return tun
 }
 
 func (*disabledTun) Activate() error {
@@ -53,73 +36,12 @@ func (*disabledTun) DeviceName() string {
 }
 
 func (t *disabledTun) Read(b []byte) (int, error) {
-	r, ok := <-t.read
-	if !ok {
-		return 0, io.EOF
-	}
-
-	if len(r) > len(b) {
-		return 0, fmt.Errorf("packet larger than mtu: %d > %d bytes", len(r), len(b))
-	}
-
-	t.tx.Inc(1)
-	if l.Level >= logrus.DebugLevel {
-		t.logger.WithField("raw", prettyPacket(r)).Debugf("Write payload")
-	}
-
-	return copy(b, r), nil
-}
-
-func (t *disabledTun) handleICMPEchoRequest(b []byte) bool {
-	// Return early if this is not a simple ICMP Echo Request
-	if !(len(b) >= 28 && len(b) <= mtu && b[0] == 0x45 && b[9] == 0x01 && b[20] == 0x08) {
-		return false
-	}
-
-	// We don't support fragmented packets
-	if b[7] != 0 || (b[6]&0x2F != 0) {
-		return false
-	}
-
-	buf := make([]byte, len(b))
-	copy(buf, b)
-
-	// Swap dest / src IPs and recalculate checksum
-	ipv4 := buf[0:20]
-	copy(ipv4[12:16], b[16:20])
-	copy(ipv4[16:20], b[12:16])
-	ipv4[10] = 0
-	ipv4[11] = 0
-	binary.BigEndian.PutUint16(ipv4[10:], ipChecksum(ipv4))
-
-	// Change type to ICMP Echo Reply and recalculate checksum
-	icmp := buf[20:]
-	icmp[0] = 0
-	icmp[2] = 0
-	icmp[3] = 0
-	binary.BigEndian.PutUint16(icmp[2:], ipChecksum(icmp))
-
-	// attempt to write it, but don't block
-	select {
-	case t.read <- buf:
-	default:
-		t.logger.Debugf("tun_disabled: dropped ICMP Echo Reply response")
-	}
-
-	return true
+	<-t.block
+	return 0, io.EOF
 }
 
 func (t *disabledTun) Write(b []byte) (int, error) {
-	t.rx.Inc(1)
-
-	// Check for ICMP Echo Request before spending time doing the full parsing
-	if t.handleICMPEchoRequest(b) {
-		if l.Level >= logrus.DebugLevel {
-			t.logger.WithField("raw", prettyPacket(b)).Debugf("Disabled tun responded to ICMP Echo Request")
-		}
-	} else if l.Level >= logrus.DebugLevel {
-		t.logger.WithField("raw", prettyPacket(b)).Debugf("Disabled tun received unexpected payload")
-	}
+	t.logger.WithField("raw", prettyPacket(b)).Debugf("Disabled tun received unexpected payload")
 	return len(b), nil
 }
 
@@ -128,14 +50,10 @@ func (t *disabledTun) WriteRaw(b []byte) error {
 	return err
 }
 
-func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return t, nil
-}
-
 func (t *disabledTun) Close() error {
-	if t.read != nil {
-		close(t.read)
-		t.read = nil
+	if t.block != nil {
+		close(t.block)
+		t.block = nil
 	}
 	return nil
 }
@@ -154,22 +72,3 @@ func (p prettyPacket) String() string {
 
 	return s.String()
 }
-
-func ipChecksum(b []byte) uint16 {
-	var c uint32
-	sz := len(b) - 1
-
-	for i := 0; i < sz; i += 2 {
-		c += uint32(b[i]) << 8
-		c += uint32(b[i+1])
-	}
-	if sz%2 == 0 {
-		c += uint32(b[sz]) << 8
-	}
-
-	for (c >> 16) > 0 {
-		c = (c & 0xffff) + (c >> 16)
-	}
-
-	return ^uint16(c)
-}

tun_freebsd.go

@@ -26,7 +26,7 @@ func newTunFromFd(deviceFd int, cidr *net.IPNet, defaultMTU int, routes []route,
 	return nil, fmt.Errorf("newTunFromFd not supported in FreeBSD")
 }
 
-func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
+func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
 	if len(routes) > 0 {
 		return nil, fmt.Errorf("Route MTU not supported in FreeBSD")
 	}
@@ -87,7 +87,3 @@ func (c *Tun) WriteRaw(b []byte) error {
 	_, err := c.Write(b)
 	return err
 }
-
-func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
-}

tun_ios.go

@@ -18,7 +18,7 @@ type Tun struct {
 	Cidr   *net.IPNet
 }
 
-func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
+func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
 	return nil, fmt.Errorf("newTun not supported in iOS")
 }
 
@@ -111,7 +111,3 @@ func (c *Tun) CidrNet() *net.IPNet {
 func (c *Tun) DeviceName() string {
 	return c.Device
 }
-
-func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
-}

tun_linux.go

@@ -55,9 +55,8 @@ func ipv4(addr string) (o [4]byte, err error) {
 */
 
 const (
-	cIFF_TUN         = 0x0001
-	cIFF_NO_PI       = 0x1000
-	cIFF_MULTI_QUEUE = 0x0100
+	cIFF_TUN   = 0x0001
+	cIFF_NO_PI = 0x1000
 )
 
 type ifreqAddr struct {
@@ -95,7 +94,7 @@ func newTunFromFd(deviceFd int, cidr *net.IPNet, defaultMTU int, routes []route,
 	return
 }
 
-func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
+func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
@@ -103,12 +102,9 @@ func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route,
 
 	var req ifReq
 	req.Flags = uint16(cIFF_TUN | cIFF_NO_PI)
-	if multiqueue {
-		req.Flags |= cIFF_MULTI_QUEUE
-	}
 	copy(req.Name[:], deviceName)
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
-		return nil, err
+		return
 	}
 	name := strings.Trim(string(req.Name[:]), "\x00")
 
@@ -135,24 +131,6 @@ func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route,
 	return
 }
 
-func (c *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
-	if err != nil {
-		return nil, err
-	}
-
-	var req ifReq
-	req.Flags = uint16(cIFF_TUN | cIFF_NO_PI | cIFF_MULTI_QUEUE)
-	copy(req.Name[:], c.Device)
-	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
-		return nil, err
-	}
-
-	file := os.NewFile(uintptr(fd), "/dev/net/tun")
-
-	return file, nil
-}
-
 func (c *Tun) WriteRaw(b []byte) error {
 	var nn int
 	for {
@@ -175,10 +153,6 @@ func (c *Tun) WriteRaw(b []byte) error {
 	}
 }
 
-func (c *Tun) Write(b []byte) (int, error) {
-	return len(b), c.WriteRaw(b)
-}
-
 func (c Tun) deviceBytes() (o [16]byte) {
 	for i, c := range c.Device {
 		o[i] = byte(c)

tun_windows.go

@@ -2,7 +2,6 @@ package nebula
 
 import (
 	"fmt"
-	"io"
 	"net"
 	"os/exec"
 	"strconv"
@@ -23,7 +22,7 @@ func newTunFromFd(deviceFd int, cidr *net.IPNet, defaultMTU int, routes []route,
 	return nil, fmt.Errorf("newTunFromFd not supported in Windows")
 }
 
-func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
+func newTun(deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
 	if len(routes) > 0 {
 		return nil, fmt.Errorf("route MTU not supported in Windows")
 	}
@@ -101,7 +100,3 @@ func (c *Tun) WriteRaw(b []byte) error {
 	_, err := c.Write(b)
 	return err
 }
-
-func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
-}

udp_darwin.go

@@ -13,32 +13,6 @@ import (
 func NewListenConfig(multi bool) net.ListenConfig {
 	return net.ListenConfig{
 		Control: func(network, address string, c syscall.RawConn) error {
-			var controlErr error
-			err := c.Control(func(fd uintptr) {
-				if err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, 999999); err != nil {
-					controlErr = fmt.Errorf("SO_SNDBUF failed: %v", err)
-					return
-				}
-			})
-			if err != nil {
-				return err
-			}
-			if controlErr != nil {
-				return controlErr
-			}
-			err = c.Control(func(fd uintptr) {
-				if err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, 999999); err != nil {
-					controlErr = fmt.Errorf("SO_RCVBUF failed: %v", err)
-					return
-				}
-			})
-			if err != nil {
-				return err
-			}
-			if controlErr != nil {
-				return controlErr
-			}
-
 			if multi {
 				var controlErr error
 				err := c.Control(func(fd uintptr) {

udp_generic.go

@@ -96,16 +96,11 @@ func (u *udpConn) reloadConfig(c *Config) {
 	// TODO
 }
 
-func NewUDPStatsEmitter(udpConns []*udpConn) func() {
-	// No UDP stats for non-linux
-	return func() {}
-}
-
 type rawMessage struct {
 	Len uint32
 }
 
-func (u *udpConn) ListenOut(f *Interface, q int) {
+func (u *udpConn) ListenOut(f *Interface) {
 	plaintext := make([]byte, mtu)
 	buffer := make([]byte, mtu)
 	header := &Header{}
@@ -113,10 +108,6 @@ func (u *udpConn) ListenOut(f *Interface, q int) {
 	udpAddr := &udpAddr{}
 	nb := make([]byte, 12, 12)
 
-	lhh := f.lightHouse.NewRequestHandler()
-
-	conntrackCache := NewConntrackCacheTicker(f.conntrackCacheTimeout)
-
 	for {
 		// Just read one packet at a time
 		n, rua, err := u.ReadFromUDP(buffer)
@@ -126,7 +117,7 @@ func (u *udpConn) ListenOut(f *Interface, q int) {
 		}
 
 		udpAddr.UDPAddr = *rua
-		f.readOutsidePackets(udpAddr, plaintext[:0], buffer[:n], header, fwPacket, lhh, nb, q, conntrackCache.Get())
+		f.readOutsidePackets(udpAddr, plaintext[:0], buffer[:n], header, fwPacket, nb)
 	}
 }
 

udp_linux.go

@@ -12,7 +12,6 @@ import (
 	"syscall"
 	"unsafe"
 
-	"github.com/rcrowley/go-metrics"
 	"golang.org/x/sys/unix"
 )
 
@@ -56,23 +55,6 @@ type rawSockaddrAny struct {
 
 var x int
 
-// From linux/sock_diag.h
-const (
-	_SK_MEMINFO_RMEM_ALLOC = iota
-	_SK_MEMINFO_RCVBUF
-	_SK_MEMINFO_WMEM_ALLOC
-	_SK_MEMINFO_SNDBUF
-	_SK_MEMINFO_FWD_ALLOC
-	_SK_MEMINFO_WMEM_QUEUED
-	_SK_MEMINFO_OPTMEM
-	_SK_MEMINFO_BACKLOG
-	_SK_MEMINFO_DROPS
-
-	_SK_MEMINFO_VARS
-)
-
-type _SK_MEMINFO [_SK_MEMINFO_VARS]uint32
-
 func NewListener(ip string, port int, multi bool) (*udpConn, error) {
 	syscall.ForkLock.RLock()
 	fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
@@ -157,15 +139,13 @@ func (u *udpConn) LocalAddr() (*udpAddr, error) {
 	return addr, nil
 }
 
-func (u *udpConn) ListenOut(f *Interface, q int) {
+func (u *udpConn) ListenOut(f *Interface) {
 	plaintext := make([]byte, mtu)
 	header := &Header{}
 	fwPacket := &FirewallPacket{}
 	udpAddr := &udpAddr{}
 	nb := make([]byte, 12, 12)
 
-	lhh := f.lightHouse.NewRequestHandler()
-
 	//TODO: should we track this?
 	//metric := metrics.GetOrRegisterHistogram("test.batch_read", nil, metrics.NewExpDecaySample(1028, 0.015))
 	msgs, buffers, names := u.PrepareRawMessages(f.udpBatchSize)
@@ -174,8 +154,6 @@ func (u *udpConn) ListenOut(f *Interface, q int) {
 		read = u.ReadSingle
 	}
 
-	conntrackCache := NewConntrackCacheTicker(f.conntrackCacheTimeout)
-
 	for {
 		n, err := read(msgs)
 		if err != nil {
@@ -188,7 +166,7 @@ func (u *udpConn) ListenOut(f *Interface, q int) {
 			udpAddr.IP = binary.BigEndian.Uint32(names[i][4:8])
 			udpAddr.Port = binary.BigEndian.Uint16(names[i][2:4])
 
-			f.readOutsidePackets(udpAddr, plaintext[:0], buffers[i][:msgs[i].Len], header, fwPacket, lhh, nb, q, conntrackCache.Get())
+			f.readOutsidePackets(udpAddr, plaintext[:0], buffers[i][:msgs[i].Len], header, fwPacket, nb)
 		}
 	}
 }
@@ -301,47 +279,6 @@ func (u *udpConn) reloadConfig(c *Config) {
 	}
 }
 
-func (u *udpConn) getMemInfo(meminfo *_SK_MEMINFO) error {
-	var vallen uint32 = 4 * _SK_MEMINFO_VARS
-	_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
-	if err != 0 {
-		return err
-	}
-	return nil
-}
-
-func NewUDPStatsEmitter(udpConns []*udpConn) func() {
-	// Check if our kernel supports SO_MEMINFO before registering the gauges
-	var udpGauges [][_SK_MEMINFO_VARS]metrics.Gauge
-	var meminfo _SK_MEMINFO
-	if err := udpConns[0].getMemInfo(&meminfo); err == nil {
-		udpGauges = make([][_SK_MEMINFO_VARS]metrics.Gauge, len(udpConns))
-		for i := range udpConns {
-			udpGauges[i] = [_SK_MEMINFO_VARS]metrics.Gauge{
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.rmem_alloc", i), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.rcvbuf", i), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.wmem_alloc", i), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.sndbuf", i), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.fwd_alloc", i), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.wmem_queued", i), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.optmem", i), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.backlog", i), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.drops", i), nil),
-			}
-		}
-	}
-
-	return func() {
-		for i, gauges := range udpGauges {
-			if err := udpConns[i].getMemInfo(&meminfo); err == nil {
-				for j := 0; j < _SK_MEMINFO_VARS; j++ {
-					gauges[j].Update(int64(meminfo[j]))
-				}
-			}
-		}
-	}
-}
-
 func (ua *udpAddr) Equals(t *udpAddr) bool {
 	if t == nil || ua == nil {
 		return t == nil && ua == nil

udp_windows.go

@@ -11,32 +11,6 @@ import (
 func NewListenConfig(multi bool) net.ListenConfig {
 	return net.ListenConfig{
 		Control: func(network, address string, c syscall.RawConn) error {
-			var controlErr error
-			err := c.Control(func(fd uintptr) {
-				if err := syscall.SetsockoptInt(syscall.Handle(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, 999999); err != nil {
-					controlErr = fmt.Errorf("SO_SNDBUF failed: %v", err)
-					return
-				}
-			})
-			if err != nil {
-				return err
-			}
-			if controlErr != nil {
-				return controlErr
-			}
-			err = c.Control(func(fd uintptr) {
-				if err := syscall.SetsockoptInt(syscall.Handle(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, 999999); err != nil {
-					controlErr = fmt.Errorf("SO_RCVBUF failed: %v", err)
-					return
-				}
-			})
-			if err != nil {
-				return err
-			}
-			if controlErr != nil {
-				return controlErr
-			}
-
 			if multi {
 				// There is no way to support multiple listeners safely on Windows:
 				// https://docs.microsoft.com/en-us/windows/desktop/winsock/using-so-reuseaddr-and-so-exclusiveaddruse