Fix typos

Add an additional transitional mode to get us to enforced safely
Add tests for the successful psk mode matrix
2025-11-22 08:24:25 +01:00 · 2021-11-04 10:45:25 -05:00 · 2021-11-04 10:45:25 -05:00 · 2021-11-04 10:44:25 -05:00 · 2021-11-04 10:41:48 -05:00 · 2021-11-04 10:25:13 -04:00
145 changed files with 15699 additions and 4431 deletions
--- a/.github/workflows/gofmt.yml
+++ b/.github/workflows/gofmt.yml
@@ -0,0 +1,44 @@
+name: gofmt
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - '.github/workflows/gofmt.yml'
+      - '**.go'
+jobs:
+
+  gofmt:
+    name: Run gofmt
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.17
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - uses: actions/cache@v2
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-gofmt1.17-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-gofmt1.17-
+
+    - name: Install goimports
+      run: |
+        go get golang.org/x/tools/cmd/goimports
+        go build golang.org/x/tools/cmd/goimports
+
+    - name: gofmt
+      run: |
+        if [ "$(find . -iname '*.go' | grep -v '\.pb\.go$' | xargs ./goimports -l)" ]
+        then
+          find . -iname '*.go' | grep -v '\.pb\.go$' | xargs ./goimports -d
+          exit 1
+        fi
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -0,0 +1,321 @@
+on:
+  push:
+    tags:
+    - 'v[0-9]+.[0-9]+.[0-9]*'
+
+name: Create release and upload binaries
+
+jobs:
+  build-linux:
+    name: Build Linux All
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Build
+        run: |
+          make BUILD_NUMBER="${GITHUB_REF#refs/tags/v}" release-linux release-freebsd
+          mkdir release
+          mv build/*.tar.gz release
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: linux-latest
+          path: release
+
+  build-windows:
+    name: Build Windows amd64
+    runs-on: windows-latest
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Build
+        run: |
+          echo $Env:GITHUB_REF.Substring(11)
+          go build -trimpath -ldflags "-X main.Build=$($Env:GITHUB_REF.Substring(11))" -o build\nebula.exe ./cmd/nebula-service
+          go build -trimpath -ldflags "-X main.Build=$($Env:GITHUB_REF.Substring(11))" -o build\nebula-cert.exe ./cmd/nebula-cert
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: windows-latest
+          path: build
+
+  build-darwin:
+    name: Build Darwin amd64
+    runs-on: macOS-latest
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Build
+        run: |
+          make BUILD_NUMBER="${GITHUB_REF#refs/tags/v}" service build/nebula-darwin-amd64.tar.gz
+          make BUILD_NUMBER="${GITHUB_REF#refs/tags/v}" service build/nebula-darwin-arm64.tar.gz
+          mkdir release
+          mv build/*.tar.gz release
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: darwin-latest
+          path: release
+
+  release:
+    name: Create and Upload Release
+    needs: [build-linux, build-darwin, build-windows]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download Linux artifacts
+        uses: actions/download-artifact@v1
+        with:
+          name: linux-latest
+
+      - name: Download Darwin artifacts
+        uses: actions/download-artifact@v1
+        with:
+          name: darwin-latest
+
+      - name: Download Windows artifacts
+        uses: actions/download-artifact@v1
+        with:
+          name: windows-latest
+
+      - name: Zip Windows
+        run: |
+          cd windows-latest
+          zip nebula-windows-amd64.zip nebula.exe nebula-cert.exe
+
+      - name: Create sha256sum
+        run: |
+          for dir in linux-latest darwin-latest windows-latest
+          do
+            (
+              cd $dir
+              if [ "$dir" = windows-latest ]
+              then
+                sha256sum <nebula.exe | sed 's=-$=nebula-windows-amd64.zip/nebula.exe='
+                sha256sum <nebula-cert.exe | sed 's=-$=nebula-windows-amd64.zip/nebula-cert.exe='
+                sha256sum nebula-windows-amd64.zip
+              else
+                  for v in *.tar.gz
+                  do
+                    sha256sum $v
+                    tar zxf $v --to-command='sh -c "sha256sum | sed s=-$='$v'/$TAR_FILENAME="'
+                  done
+              fi
+            )
+          done | sort -k 2 >SHASUM256.txt
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Release ${{ github.ref }}
+          draft: false
+          prerelease: false
+
+      ##
+      ## Upload assets (I wish we could just upload the whole folder at once...
+      ##
+
+      - name: Upload SHASUM256.txt
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./SHASUM256.txt
+          asset_name: SHASUM256.txt
+          asset_content_type: text/plain
+
+      - name: Upload darwin-amd64
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./darwin-latest/nebula-darwin-amd64.tar.gz
+          asset_name: nebula-darwin-amd64.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload darwin-arm64
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./darwin-latest/nebula-darwin-arm64.tar.gz
+          asset_name: nebula-darwin-arm64.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload windows-amd64
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./windows-latest/nebula-windows-amd64.zip
+          asset_name: nebula-windows-amd64.zip
+          asset_content_type: application/zip
+
+      - name: Upload linux-amd64
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-amd64.tar.gz
+          asset_name: nebula-linux-amd64.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-386
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-386.tar.gz
+          asset_name: nebula-linux-386.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-ppc64le
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-ppc64le.tar.gz
+          asset_name: nebula-linux-ppc64le.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-arm-5
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-arm-5.tar.gz
+          asset_name: nebula-linux-arm-5.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-arm-6
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-arm-6.tar.gz
+          asset_name: nebula-linux-arm-6.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-arm-7
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-arm-7.tar.gz
+          asset_name: nebula-linux-arm-7.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-arm64
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-arm64.tar.gz
+          asset_name: nebula-linux-arm64.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-mips
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-mips.tar.gz
+          asset_name: nebula-linux-mips.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-mipsle
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-mipsle.tar.gz
+          asset_name: nebula-linux-mipsle.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-mips64
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-mips64.tar.gz
+          asset_name: nebula-linux-mips64.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-mips64le
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-mips64le.tar.gz
+          asset_name: nebula-linux-mips64le.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-mips-softfloat
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-mips-softfloat.tar.gz
+          asset_name: nebula-linux-mips-softfloat.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload linux-riscv64
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-linux-riscv64.tar.gz
+          asset_name: nebula-linux-riscv64.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload freebsd-amd64
+        uses: actions/upload-release-asset@v1.0.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./linux-latest/nebula-freebsd-amd64.tar.gz
+          asset_name: nebula-freebsd-amd64.tar.gz
+          asset_content_type: application/gzip
--- a/.github/workflows/smoke.yml
+++ b/.github/workflows/smoke.yml
@@ -0,0 +1,48 @@
+name: smoke
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - '.github/workflows/smoke**'
+      - '**Makefile'
+      - '**.go'
+      - '**.proto'
+      - 'go.mod'
+      - 'go.sum'
+jobs:
+
+  smoke:
+    name: Run multi node smoke test
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.17
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - uses: actions/cache@v2
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go1.17-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go1.17-
+
+    - name: build
+      run: make bin-docker
+
+    - name: setup docker image
+      working-directory: ./.github/workflows/smoke
+      run: ./build.sh
+
+    - name: run smoke
+      working-directory: ./.github/workflows/smoke
+      run: ./smoke.sh
+
+    timeout-minutes: 10
--- a/.github/workflows/smoke/.gitignore
+++ b/.github/workflows/smoke/.gitignore
@@ -0,0 +1 @@
+/build
--- a/.github/workflows/smoke/Dockerfile
+++ b/.github/workflows/smoke/Dockerfile
@@ -0,0 +1,7 @@
+FROM debian:buster
+
+ADD ./build /nebula
+
+WORKDIR /nebula
+
+ENTRYPOINT ["/nebula/nebula"]
--- a/.github/workflows/smoke/build.sh
+++ b/.github/workflows/smoke/build.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+set -e -x
+
+rm -rf ./build
+mkdir ./build
+
+(
+    cd build
+
+    cp ../../../../build/linux-amd64/nebula .
+    cp ../../../../build/linux-amd64/nebula-cert .
+
+    HOST="lighthouse1" \
+        AM_LIGHTHOUSE=true \
+        ../genconfig.sh >lighthouse1.yml
+
+    HOST="host2" \
+        LIGHTHOUSES="192.168.100.1 172.17.0.2:4242" \
+        ../genconfig.sh >host2.yml
+
+    HOST="host3" \
+        LIGHTHOUSES="192.168.100.1 172.17.0.2:4242" \
+        INBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
+        ../genconfig.sh >host3.yml
+
+    HOST="host4" \
+        LIGHTHOUSES="192.168.100.1 172.17.0.2:4242" \
+        OUTBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
+        ../genconfig.sh >host4.yml
+
+    ../../../../nebula-cert ca -name "Smoke Test"
+    ../../../../nebula-cert sign -name "lighthouse1" -groups "lighthouse,lighthouse1" -ip "192.168.100.1/24"
+    ../../../../nebula-cert sign -name "host2" -groups "host,host2" -ip "192.168.100.2/24"
+    ../../../../nebula-cert sign -name "host3" -groups "host,host3" -ip "192.168.100.3/24"
+    ../../../../nebula-cert sign -name "host4" -groups "host,host4" -ip "192.168.100.4/24"
+)
+
+sudo docker build -t nebula:smoke .
--- a/.github/workflows/smoke/genconfig.sh
+++ b/.github/workflows/smoke/genconfig.sh
@@ -0,0 +1,54 @@
+#!/bin/sh
+
+set -e
+
+FIREWALL_ALL='[{"port": "any", "proto": "any", "host": "any"}]'
+
+if [ "$STATIC_HOSTS" ] || [ "$LIGHTHOUSES" ]
+then
+  echo "static_host_map:"
+  echo "$STATIC_HOSTS" | while read -r NEBULA_IP STATIC
+  do
+    [ -z "$NEBULA_IP" ] || echo "  '$NEBULA_IP': ['$STATIC']"
+  done
+  echo "$LIGHTHOUSES" | while read -r NEBULA_IP STATIC
+  do
+    [ -z "$NEBULA_IP" ] || echo "  '$NEBULA_IP': ['$STATIC']"
+  done
+  echo
+fi
+
+lighthouse_hosts() {
+  if [ "$LIGHTHOUSES" ]
+  then
+    echo
+    echo "$LIGHTHOUSES" | while read -r NEBULA_IP STATIC
+    do
+      echo "    - '$NEBULA_IP'"
+    done
+  else
+    echo "[]"
+  fi
+}
+
+cat <<EOF
+pki:
+  ca: ca.crt
+  cert: ${HOST}.crt
+  key: ${HOST}.key
+
+lighthouse:
+  am_lighthouse: ${AM_LIGHTHOUSE:-false}
+  hosts: $(lighthouse_hosts)
+
+listen:
+  host: 0.0.0.0
+  port: ${LISTEN_PORT:-4242}
+
+tun:
+  dev: ${TUN_DEV:-nebula1}
+
+firewall:
+  outbound: ${OUTBOUND:-$FIREWALL_ALL}
+  inbound: ${INBOUND:-$FIREWALL_ALL}
+EOF
--- a/.github/workflows/smoke/smoke.sh
+++ b/.github/workflows/smoke/smoke.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+set -e -x
+
+set -o pipefail
+
+mkdir -p logs
+
+cleanup() {
+    set +e
+    if [ "$(jobs -r)" ]
+    then
+        sudo docker kill lighthouse1 host2 host3 host4
+    fi
+}
+
+trap cleanup EXIT
+
+sudo docker run --name lighthouse1 --rm nebula:smoke -config lighthouse1.yml -test
+sudo docker run --name host2 --rm nebula:smoke -config host2.yml -test
+sudo docker run --name host3 --rm nebula:smoke -config host3.yml -test
+sudo docker run --name host4 --rm nebula:smoke -config host4.yml -test
+
+sudo docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 &
+sleep 1
+sudo docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host2.yml 2>&1 | tee logs/host2 &
+sleep 1
+sudo docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host3.yml 2>&1 | tee logs/host3 &
+sleep 1
+sudo docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host4.yml 2>&1 | tee logs/host4 &
+sleep 1
+
+set +x
+echo
+echo " *** Testing ping from lighthouse1"
+echo
+set -x
+sudo docker exec lighthouse1 ping -c1 192.168.100.2
+sudo docker exec lighthouse1 ping -c1 192.168.100.3
+
+set +x
+echo
+echo " *** Testing ping from host2"
+echo
+set -x
+sudo docker exec host2 ping -c1 192.168.100.1
+# Should fail because not allowed by host3 inbound firewall
+! sudo docker exec host2 ping -c1 192.168.100.3 -w5 || exit 1
+
+set +x
+echo
+echo " *** Testing ping from host3"
+echo
+set -x
+sudo docker exec host3 ping -c1 192.168.100.1
+sudo docker exec host3 ping -c1 192.168.100.2
+
+set +x
+echo
+echo " *** Testing ping from host4"
+echo
+set -x
+sudo docker exec host4 ping -c1 192.168.100.1
+# Should fail because not allowed by host4 outbound firewall
+! sudo docker exec host4 ping -c1 192.168.100.2 -w5 || exit 1
+! sudo docker exec host4 ping -c1 192.168.100.3 -w5 || exit 1
+
+set +x
+echo
+echo " *** Testing conntrack"
+echo
+set -x
+# host2 can ping host3 now that host3 pinged it first
+sudo docker exec host2 ping -c1 192.168.100.3
+# host4 can ping host2 once conntrack established
+sudo docker exec host2 ping -c1 192.168.100.4
+sudo docker exec host4 ping -c1 192.168.100.2
+
+sudo docker exec host4 sh -c 'kill 1'
+sudo docker exec host3 sh -c 'kill 1'
+sudo docker exec host2 sh -c 'kill 1'
+sudo docker exec lighthouse1 sh -c 'kill 1'
+sleep 1
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -0,0 +1,80 @@
+name: Build and test
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - '.github/workflows/test.yml'
+      - '**Makefile'
+      - '**.go'
+      - '**.proto'
+      - 'go.mod'
+      - 'go.sum'
+jobs:
+
+  test-linux:
+    name: Build all and test on ubuntu-linux
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.17
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - uses: actions/cache@v2
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go1.17-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go1.17-
+
+    - name: Build
+      run: make all
+
+    - name: Test
+      run: make test
+
+    - name: End 2 end
+      run: make e2evv
+
+  test:
+    name: Build and test on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, macOS-latest]
+    steps:
+
+    - name: Set up Go 1.17
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - uses: actions/cache@v2
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go1.17-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go1.17-
+
+    - name: Build nebula
+      run: go build ./cmd/nebula
+
+    - name: Build nebula-cert
+      run: go build ./cmd/nebula-cert
+
+    - name: Test
+      run: go test -v ./...
+
+    - name: End 2 end
+      run: make e2evv
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -0,0 +1,295 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- SSH `print-cert` has a new `-raw` flag to get the PEM representation of a certificate. (#483)
+
+- New build architecture: Linux `riscv64`. (#542)
+
+- New experimental config option `remote_allow_ranges`. (#540)
+
+- New config option `pki.disconnect_invalid` that will tear down tunnels when they become invalid (through expiry or
+  removal of root trust). Default is `false`. Note, this will not currently recognize if a remote has changed
+  certificates since the last handshake. (#370)
+
+- New config option `unsafe_routes.<route>.metric` will set a metric for a specific unsafe route. It's useful if you have
+  more than one identical route and want to prefer one against the other.
+
+### Changed
+
+- Build against go 1.17. (#553)
+
+### Deprecated
+
+- The `preferred_ranges` option has been supported as a replacement for
+  `local_range` since v1.0.0. It has now been documented and `local_range`
+  has been officially deprecated. (#541)
+
+### Fixed
+
+- Valid recv_error packets were incorrectly marked as "spoofing" and ignored. (#482)
+
+- SSH server handles single `exec` requests correctly. (#483)
+
+- Signing a certificate with `nebula-cert sign` now verifies that the supplied
+  ca-key matches the ca-crt. (#503)
+
+- If `preferred_ranges` (or the deprecated `local_range`) is configured, we
+  will immediately switch to a preferred remote address after the reception of
+  a handshake packet (instead of waiting until 1,000 packets have been sent).
+  (#532)
+  
+- A race condition when `punchy.respond` is enabled and ensures the correct
+  vpn ip is sent a punch back response in highly queried node. (#566)
+
+## [1.4.0] - 2021-05-11
+
+### Added
+
+- Ability to output qr code images in `print`, `ca`, and `sign` modes for `nebula-cert`.
+  This is useful when configuring mobile clients. (#297)
+
+- Experimental: Nebula can now do work on more than 2 cpu cores in send and receive paths via
+  the new `routines` config option. (#382, #391, #395)
+
+- ICMP ping requests can be responded to when the `tun.disabled` is `true`.
+  This is useful so that you can "ping" a lighthouse running in this mode. (#342)
+
+- Run smoke tests via `make smoke-docker`. (#287)
+
+- More reported stats, udp memory use on linux, build version (when using Prometheus), firewall,
+  handshake, and cached packet stats. (#390, #405, #450, #453)
+
+- IPv6 support for the underlay network. (#369)
+
+- End to end testing, run with `make e2e`. (#425, #427, #428)
+
+### Changed
+
+- Darwin will now log stdout/stderr to a file when using `-service` mode. (#303)
+
+- Example systemd unit file now better arranged startup order when using `sshd`
+  and other fixes. (#317, #412, #438)
+
+- Reduced memory utilization/garbage collection. (#320, #323, #340)
+
+- Reduced CPU utilization. (#329)
+
+- Build against go 1.16. (#381)
+
+- Refactored handshakes to improve performance and correctness. (#401, #402, #404, #416, #451)
+
+- Improved roaming support for mobile clients. (#394, #457)
+
+- Lighthouse performance and correctness improvements. (#406, #418, #429, #433, #437, #442, #449)
+
+- Better ordered startup to enable `sshd`, `stats`, and `dns` subsystems to listen on
+  the nebula interface. (#375)
+
+### Fixed
+
+- No longer report handshake packets as `lost` in stats. (#331)
+
+- Error handling in the `cert` package. (#339, #373)
+
+- Orphaned pending hostmap entries are cleaned up. (#344)
+
+- Most known data races are now resolved. (#396, #400, #424)
+
+- Refuse to run a lighthouse on an ephemeral port. (#399)
+
+- Removed the global references. (#423, #426, #446)
+
+- Reloading via ssh command avoids a panic. (#447)
+
+- Shutdown is now performed in a cleaner way. (#448)
+
+- Logs will now find their way to Windows event viewer when running under `-service` mode
+  in Windows. (#443)
+
+## [1.3.0] - 2020-09-22
+
+### Added
+
+- You can emit statistics about non-message packets by setting the option
+  `stats.message_metrics`. You can similarly emit detailed statistics about
+  lighthouse packets by setting the option `stats.lighthouse_metrics`. See
+  the example config for more details. (#230)
+
+- We now support freebsd/amd64. This is experimental, please give us feedback.
+  (#103)
+
+- We now release a binary for `linux/mips-softfloat` which has also been
+  stripped to reduce filesize and hopefully have a better chance on running on
+  small mips devices. (#231)
+
+- You can set `tun.disabled` to true to run a standalone lighthouse without a
+  tun device (and thus, without root). (#269)
+
+- You can set `logging.disable_timestamp` to remove timestamps from log lines,
+  which is useful when output is redirected to a logging system that already
+  adds timestamps. (#288)
+
+### Changed
+
+- Handshakes should now trigger faster, as we try to be proactive with sending
+  them instead of waiting for the next timer tick in most cases. (#246, #265)
+
+- Previously, we would drop the conntrack table whenever firewall rules were
+  changed during a SIGHUP. Now, we will maintain the table and just validate
+  that an entry still matches with the new rule set. (#233)
+
+- Debug logs for firewall drops now include the reason. (#220, #239)
+
+- Logs for handshakes now include the fingerprint of the remote host. (#262)
+
+- Config item `pki.blacklist` is now `pki.blocklist`. (#272)
+
+- Better support for older Linux kernels. We now only set `SO_REUSEPORT` if
+  `tun.routines` is greater than 1 (default is 1). We also only use the
+  `recvmmsg` syscall if `listen.batch` is greater than 1 (default is 64).
+  (#275)
+
+- It is possible to run Nebula as a library inside of another process now.
+  Note that this is still experimental and the internal APIs around this might
+  change in minor version releases. (#279)
+
+### Deprecated
+
+- `pki.blacklist` is deprecated in favor of `pki.blocklist` with the same
+   functionality. Existing configs will continue to load for this release to
+   allow for migrations. (#272)
+
+### Fixed
+
+- `advmss` is now set correctly for each route table entry when `tun.routes`
+  is configured to have some routes with higher MTU. (#245)
+
+- Packets that arrive on the tun device with an unroutable destination IP are
+  now dropped correctly, instead of wasting time making queries to the
+  lighthouses for IP `0.0.0.0` (#267)
+
+## [1.2.0] - 2020-04-08
+
+### Added
+
+- Add `logging.timestamp_format` config option. The primary purpose of this
+  change is to allow logging timestamps with millisecond precision. (#187)
+
+- Support `unsafe_routes` on Windows. (#184)
+
+- Add `lighthouse.remote_allow_list` to filter which subnets we will use to
+  handshake with other hosts. See the example config for more details. (#217)
+
+- Add `lighthouse.local_allow_list` to filter which local IP addresses and/or
+  interfaces we advertise to the lighthouses. See the example config for more
+  details. (#217)
+
+- Wireshark dissector plugin. Add this file in `dist/wireshark` to your
+  Wireshark plugins folder to see Nebula packet headers decoded. (#216)
+
+- systemd unit for Arch, so it can be built entirely from this repo. (#216)
+
+### Changed
+
+- Added a delay to punching via lighthouse signal to deal with race conditions
+  in some linux conntrack implementations. (#210)
+
+  See deprecated, this also adds a new `punchy.delay` option that defaults to `1s`.
+
+- Validate all `lighthouse.hosts` and `static_host_map` VPN IPs are in the
+  subnet defined in our cert. Exit with a fatal error if they are not in our
+  subnet, as this is an invalid configuration (we will not have the proper
+  routes set up to communicate with these hosts). (#170)
+
+- Use absolute paths to system binaries on macOS and Windows. (#191)
+
+- Add configuration options for `handshakes`. This includes options to tweak
+  `try_interval`, `retries` and `wait_rotation`. See example config for
+  descriptions. (#179)
+
+- Allow `-config` file to not end in `.yaml` or `yml`. Useful when using
+  `-test` and automated tools like Ansible that create temporary files without
+  suffixes. (#189)
+
+- The config test mode, `-test`, is now more thorough and catches more parsing
+  issues. (#177)
+
+- Various documentation and example fixes. (#196)
+
+- Improved log messages. (#181, #200)
+
+- Dependencies updated. (#188)
+
+### Deprecated
+
+- `punchy`, `punch_back` configuration options have been collapsed under the
+  now top level `punchy` config directive. (#210)
+
+  `punchy.punch` - This is the old `punchy` option. Should we perform NAT hole
+  punching (default false)?
+
+  `punchy.respond` - This is the old `punch_back` option. Should we respond to
+  hole punching by hole punching back (default false)?
+
+### Fixed
+
+- Reduce memory allocations when not using `unsafe_routes`. (#198)
+
+- Ignore packets from self to self. (#192)
+
+- MTU fixed for `unsafe_routes`. (#209)
+
+## [1.1.0] - 2020-01-17
+
+### Added
+
+- For macOS and Windows, build a special version of the binary that can install
+  and manage its own service configuration. You can use this with `nebula
+  -service`.  If you are building from source, use `make service` to build this feature.
+- Support for `mips`, `mips64`, `386` and `ppc64le` processors on Linux.
+- You can now configure the DNS listen host and port with `lighthouse.dns.host`
+  and `lighthouse.dns.port`.
+- Subnet and routing support. You can now add a `unsafe_routes` section to your
+  config to allow hosts to act as gateways to other subnets. Read the example
+  config for more details. This is supported on Linux and macOS.
+
+### Changed
+
+- Certificates now have more verifications performed, including making sure
+  the certificate lifespan does not exceed the lifespan of the root CA. This
+  could cause issues if you have signed certificates with expirations beyond
+  the expiration of your CA, and you will need to reissue your certificates.
+- If lighthouse interval is set to `0`, never update the lighthouse (mobile
+  optimization).
+- Various documentation and example fixes.
+- Improved error messages.
+- Dependencies updated.
+
+### Fixed
+
+- If you have a firewall rule with `group: ["one-group"]`, this will
+  now be accepted, with a warning to use `group: "one-group"` instead.
+- The `listen.host` configuration option was previously ignored (the bind host
+  was always 0.0.0.0). This option will now be honored.
+- The `ca_sha` and `ca_name` firewall rule options should now work correctly.
+
+## [1.0.0] - 2019-11-19
+
+### Added
+
+- Initial public release.
+
+[Unreleased]: https://github.com/slackhq/nebula/compare/v1.4.0...HEAD
+[1.4.0]: https://github.com/slackhq/nebula/releases/tag/v1.4.0
+[1.3.0]: https://github.com/slackhq/nebula/releases/tag/v1.3.0
+[1.2.0]: https://github.com/slackhq/nebula/releases/tag/v1.2.0
+[1.1.0]: https://github.com/slackhq/nebula/releases/tag/v1.1.0
+[1.0.0]: https://github.com/slackhq/nebula/releases/tag/v1.0.0
--- a/174
+++ b/174
@@ -1,64 +1,122 @@
-BUILD_NUMBER ?= dev+$(shell date -u '+%Y%m%d%H%M%S')
+GOMINVERSION = 1.17
+NEBULA_CMD_PATH = "./cmd/nebula"
 GO111MODULE = on
 export GO111MODULE

-all:
-	make bin-linux
-	make bin-arm
-	make bin-arm6
-	make bin-arm64
-	make bin-darwin
-	make bin-windows
+# Set up OS specific bits
+ifeq ($(OS),Windows_NT)
+	#TODO: we should be able to ditch awk as well
+	GOVERSION := $(shell go version | awk "{print substr($$3, 3)}")
+	GOISMIN := $(shell IF "$(GOVERSION)" GEQ "$(GOMINVERSION)" ECHO 1)
+	NEBULA_CMD_SUFFIX = .exe
+	NULL_FILE = nul
+else
+	GOVERSION := $(shell go version | awk '{print substr($$3, 3)}')
+	GOISMIN := $(shell expr "$(GOVERSION)" ">=" "$(GOMINVERSION)")
+	NEBULA_CMD_SUFFIX =
+	NULL_FILE = /dev/null
+endif
+
+# Only defined the build number if we haven't already
+ifndef BUILD_NUMBER
+	ifeq ($(shell git describe --exact-match 2>$(NULL_FILE)),)
+		BUILD_NUMBER = $(shell git describe --abbrev=0 --match "v*" | cut -dv -f2)-$(shell git branch --show-current)-$(shell git describe --long --dirty | cut -d- -f2-)
+	else
+		BUILD_NUMBER = $(shell git describe --exact-match --dirty | cut -dv -f2)
+	endif
+endif
+
+LDFLAGS = -X main.Build=$(BUILD_NUMBER)
+
+ALL_LINUX = linux-amd64 \
+	linux-386 \
+	linux-ppc64le \
+	linux-arm-5 \
+	linux-arm-6 \
+	linux-arm-7 \
+	linux-arm64 \
+	linux-mips \
+	linux-mipsle \
+	linux-mips64 \
+	linux-mips64le \
+	linux-mips-softfloat \
+	linux-riscv64
+
+ALL = $(ALL_LINUX) \
+	darwin-amd64 \
+	darwin-arm64 \
+	freebsd-amd64 \
+	windows-amd64
+
+e2e:
+	$(TEST_ENV) go test -tags=e2e_testing -count=1 $(TEST_FLAGS) ./e2e
+
+e2ev: TEST_FLAGS = -v
+e2ev: e2e
+
+e2evv: TEST_ENV += TEST_LOGS=1
+e2evv: e2ev
+
+e2evvv: TEST_ENV += TEST_LOGS=2
+e2evvv: e2ev
+
+e2evvvv: TEST_ENV += TEST_LOGS=3
+e2evvvv: e2ev
+
+all: $(ALL:%=build/%/nebula) $(ALL:%=build/%/nebula-cert)
+
+release: $(ALL:%=build/nebula-%.tar.gz)
+
+release-linux: $(ALL_LINUX:%=build/nebula-%.tar.gz)
+
+release-freebsd: build/nebula-freebsd-amd64.tar.gz
+
+BUILD_ARGS = -trimpath
+
+bin-windows: build/windows-amd64/nebula.exe build/windows-amd64/nebula-cert.exe
+	mv $? .
+
+bin-darwin: build/darwin-amd64/nebula build/darwin-amd64/nebula-cert
+	mv $? .
+
+bin-freebsd: build/freebsd-amd64/nebula build/freebsd-amd64/nebula-cert
+	mv $? .

 bin:
-	go build -ldflags "-X main.Build=$(BUILD_NUMBER)" -o ./nebula ./cmd/nebula
-	go build -ldflags "-X main.Build=$(BUILD_NUMBER)" -o ./nebula-cert ./cmd/nebula-cert
+	go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula${NEBULA_CMD_SUFFIX} ${NEBULA_CMD_PATH}
+	go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula-cert${NEBULA_CMD_SUFFIX} ./cmd/nebula-cert

 install:
-	go install -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula
-	go install -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula-cert
+	go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ${NEBULA_CMD_PATH}
+	go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ./cmd/nebula-cert

-bin-arm:
-	mkdir -p build/arm
-	GOARCH=arm GOOS=linux go build -o build/arm/nebula -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula
-	GOARCH=arm GOOS=linux go build -o build/arm/nebula-cert -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula-cert
+build/linux-arm-%: GOENV += GOARM=$(word 3, $(subst -, ,$*))
+build/linux-mips-%: GOENV += GOMIPS=$(word 3, $(subst -, ,$*))

-bin-arm6:
-	mkdir -p build/arm6
-	GOARCH=arm GOARM=6 GOOS=linux go build -o build/arm6/nebula -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula
-	GOARCH=arm GOARM=6 GOOS=linux go build -o build/arm6/nebula-cert -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula-cert
+# Build an extra small binary for mips-softfloat
+build/linux-mips-softfloat/%: LDFLAGS += -s -w

-bin-arm64:
-	mkdir -p build/arm64
-	GOARCH=arm64 GOOS=linux go build -o build/arm64/nebula -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula
-	GOARCH=arm64 GOOS=linux go build -o build/arm64/nebula-cert -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula-cert
+build/%/nebula: .FORCE
+	GOOS=$(firstword $(subst -, , $*)) \
+		GOARCH=$(word 2, $(subst -, ,$*)) $(GOENV) \
+		go build $(BUILD_ARGS) -o $@ -ldflags "$(LDFLAGS)" ${NEBULA_CMD_PATH}

-bin-vagrant:
-	GOARCH=amd64 GOOS=linux go build -o nebula -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula
-	GOARCH=amd64 GOOS=linux go build -ldflags "-X main.Build=$(BUILD_NUMBER)" -o ./nebula-cert ./cmd/nebula-cert
+build/%/nebula-cert: .FORCE
+	GOOS=$(firstword $(subst -, , $*)) \
+		GOARCH=$(word 2, $(subst -, ,$*)) $(GOENV) \
+		go build $(BUILD_ARGS) -o $@ -ldflags "$(LDFLAGS)" ./cmd/nebula-cert

-bin-darwin:
-	mkdir -p build/darwin
-	GOARCH=amd64 GOOS=darwin go build -o build/darwin/nebula -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula
-	GOARCH=amd64 GOOS=darwin go build -o build/darwin/nebula-cert -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula-cert
+build/%/nebula.exe: build/%/nebula
+	mv $< $@

-bin-windows:
-	mkdir -p build/windows
-	GOARCH=amd64 GOOS=windows go build -o build/windows/nebula.exe -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula
-	GOARCH=amd64 GOOS=windows go build -o build/windows/nebula-cert.exe -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula-cert
+build/%/nebula-cert.exe: build/%/nebula-cert
+	mv $< $@

-bin-linux:
-	mkdir -p build/linux
-	GOARCH=amd64 GOOS=linux go build -o build/linux/nebula -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula
-	GOARCH=amd64 GOOS=linux go build -o build/linux/nebula-cert -ldflags "-X main.Build=$(BUILD_NUMBER)" ./cmd/nebula-cert
+build/nebula-%.tar.gz: build/%/nebula build/%/nebula-cert
+	tar -zcv -C build/$* -f $@ nebula nebula-cert

-release: all
-	tar -zcv -C build/arm/ -f nebula-linux-arm.tar.gz nebula nebula-cert
-	tar -zcv -C build/arm6/ -f nebula-linux-arm6.tar.gz nebula nebula-cert
-	tar -zcv -C build/arm64/ -f nebula-linux-arm64.tar.gz nebula nebula-cert
-	tar -zcv -C build/darwin/ -f nebula-darwin-amd64.tar.gz nebula nebula-cert
-	tar -zcv -C build/windows/ -f nebula-windows-amd64.tar.gz nebula.exe nebula-cert.exe
-	tar -zcv -C build/linux/ -f nebula-linux-amd64.tar.gz nebula nebula-cert
+build/nebula-%.zip: build/%/nebula.exe build/%/nebula-cert.exe
+	cd build/$* && zip ../nebula-$*.zip nebula.exe nebula-cert.exe

 vet:
 	go vet -v ./...
@@ -84,13 +142,29 @@ bench-cpu-long:
 proto: nebula.pb.go cert/cert.pb.go

 nebula.pb.go: nebula.proto .FORCE
-	go build github.com/golang/protobuf/protoc-gen-go
-	PATH="$(PWD):$(PATH)" protoc --go_out=. $<
-	rm protoc-gen-go
+	go build github.com/gogo/protobuf/protoc-gen-gogofaster
+	PATH="$(CURDIR):$(PATH)" protoc --gogofaster_out=paths=source_relative:. $<
+	rm protoc-gen-gogofaster

 cert/cert.pb.go: cert/cert.proto .FORCE
 	$(MAKE) -C cert cert.pb.go

+service:
+	@echo > $(NULL_FILE)
+	$(eval NEBULA_CMD_PATH := "./cmd/nebula-service")
+ifeq ($(words $(MAKECMDGOALS)),1)
+	@$(MAKE) service ${.DEFAULT_GOAL} --no-print-directory
+endif
+
+bin-docker: bin build/linux-amd64/nebula build/linux-amd64/nebula-cert
+
+smoke-docker: bin-docker
+	cd .github/workflows/smoke/ && ./build.sh
+	cd .github/workflows/smoke/ && ./smoke.sh
+
+smoke-docker-race: BUILD_ARGS = -race
+smoke-docker-race: smoke-docker
+
 .FORCE:
-.PHONY: test test-cov-html bench bench-cpu bench-cpu-long bin proto release
+.PHONY: e2e e2ev e2evv e2evvv e2evvvv test test-cov-html bench bench-cpu bench-cpu-long bin proto release service smoke-docker smoke-docker-race
 .DEFAULT_GOAL := bin
--- a/README.md
+++ b/README.md
@@ -1,7 +1,6 @@
-## What is Nebula?
+## What is Nebula?
 Nebula is a scalable overlay networking tool with a focus on performance, simplicity and security.
-It lets you seamlessly connect computers anywhere in the world. Nebula is portable, and runs on Linux, OSX, and Windows.
-(Also: keep this quiet, but we have an early prototype running on iOS).
+It lets you seamlessly connect computers anywhere in the world. Nebula is portable, and runs on Linux, OSX, Windows, iOS, and Android.
 It can be used to connect a small number of computers, but is also able to connect tens of thousands of computers.

 Nebula incorporates a number of existing concepts like encryption, security groups, certificates,
@@ -9,8 +8,40 @@ and tunneling, and each of those individual pieces existed before Nebula in vari
 What makes Nebula different to existing offerings is that it brings all of these ideas together,
 resulting in a sum that is greater than its individual parts.

+Further documentation can be found [here](https://www.defined.net/nebula/introduction/).
+
 You can read more about Nebula [here](https://medium.com/p/884110a5579).

+You can also join the NebulaOSS Slack group [here](https://join.slack.com/t/nebulaoss/shared_invite/enQtOTA5MDI4NDg3MTg4LTkwY2EwNTI4NzQyMzc0M2ZlODBjNWI3NTY1MzhiOThiMmZlZjVkMTI0NGY4YTMyNjUwMWEyNzNkZTJmYzQxOGU).
+
+## Supported Platforms
+
+#### Desktop and Server
+
+Check the [releases](https://github.com/slackhq/nebula/releases/latest) page for downloads or see the [Distribution Packages](https://github.com/slackhq/nebula#distribution-packages) section.
+
+- Linux - 64 and 32 bit, arm, and others
+- Windows
+- MacOS
+- Freebsd
+
+#### Distribution Packages
+
+- [Arch Linux](https://archlinux.org/packages/community/x86_64/nebula/)
+    ```
+    $ sudo pacman -S nebula
+    ```
+- [Fedora Linux](https://copr.fedorainfracloud.org/coprs/jdoss/nebula/)
+    ```
+    $ sudo dnf copr enable jdoss/nebula
+    $ sudo dnf install nebula
+    ```
+
+#### Mobile
+
+- [iOS](https://apps.apple.com/us/app/mobile-nebula/id1509587936?itsct=apps_box&amp;itscg=30200)
+- [Android](https://play.google.com/store/apps/details?id=net.defined.mobile_nebula&pcampaignid=pcampaignidMKT-Other-global-all-co-prtnr-py-PartBadge-Mar2515-1)
+
 ## Technical Overview

 Nebula is a mutually authenticated peer-to-peer software defined network based on the [Noise Protocol Framework](https://noiseprotocol.org/).
@@ -27,7 +58,7 @@ Nebula was created to provide a mechanism for groups hosts to communicate secure

 To set up a Nebula network, you'll need:

-#### 1. The [Nebula binaries](https://github.com/slackhq/nebula/releases) for your specific platform. Specifically you'll need `nebula-cert` and the specific nebula binary for each platform you use.
+#### 1. The [Nebula binaries](https://github.com/slackhq/nebula/releases) or [Distribution Packages](https://github.com/slackhq/nebula#distribution-packages) for your specific platform. Specifically you'll need `nebula-cert` and the specific nebula binary for each platform you use.

 #### 2. (Optional, but you really should..) At least one discovery node with a routable IP address, which we call a lighthouse.

@@ -44,16 +75,16 @@ Nebula lighthouses allow nodes to find each other, anywhere in the world. A ligh
  This will create files named `ca.key` and `ca.cert` in the current directory. The `ca.key` file is the most sensitive file you'll create, because it is the key used to sign the certificates for individual nebula nodes/hosts. Please store this file somewhere safe, preferably with strong encryption.

 #### 4. Nebula host keys and certificates generated from that certificate authority
-This assumes you have three nodes, named lighthouse1, host1, host3. You can name the nodes any way you'd like, including FQDN. You'll also need to choose IP addresses and the associated subnet. In this example, we are creating a nebula network that will use 192.168.100.x/24 as its network range. This example also demonstrates nebula groups, which can later be used to define traffic rules in a nebula network.
+This assumes you have four nodes, named lighthouse1, laptop, server1, host3. You can name the nodes any way you'd like, including FQDN. You'll also need to choose IP addresses and the associated subnet. In this example, we are creating a nebula network that will use 192.168.100.x/24 as its network range. This example also demonstrates nebula groups, which can later be used to define traffic rules in a nebula network.
 ```
 ./nebula-cert sign -name "lighthouse1" -ip "192.168.100.1/24"
 ./nebula-cert sign -name "laptop" -ip "192.168.100.2/24" -groups "laptop,home,ssh"
 ./nebula-cert sign -name "server1" -ip "192.168.100.9/24" -groups "servers"
-./nebula-cert sign -name "host3" -ip "192.168.100.9/24"
+./nebula-cert sign -name "host3" -ip "192.168.100.10/24"
 ```

 #### 5. Configuration files for each host
-Download a copy of the nebula [example configuration](https://github.com/slackhq/nebula/blob/master/examples/config.yaml).
+Download a copy of the nebula [example configuration](https://github.com/slackhq/nebula/blob/master/examples/config.yml).

 * On the lighthouse node, you'll need to ensure `am_lighthouse: true` is set.

@@ -62,7 +93,7 @@ Download a copy of the nebula [example configuration](https://github.com/slackhq

 #### 6. Copy nebula credentials, configuration, and binaries to each host

-For each host, copy the nebula binary to the host, along with `config.yaml` from step 5, and the files `ca.crt`, `{host}.crt`, and `{host}.key` from step 2.
+For each host, copy the nebula binary to the host, along with `config.yaml` from step 5, and the files `ca.crt`, `{host}.crt`, and `{host}.key` from step 4.

 **DO NOT COPY `ca.key` TO INDIVIDUAL NODES.**

--- a/allow_list.go
+++ b/allow_list.go
@@ -0,0 +1,361 @@
+package nebula
+
+import (
+	"fmt"
+	"net"
+	"regexp"
+
+	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/iputil"
+)
+
+type AllowList struct {
+	// The values of this cidrTree are `bool`, signifying allow/deny
+	cidrTree *cidr.Tree6
+}
+
+type RemoteAllowList struct {
+	AllowList *AllowList
+
+	// Inside Range Specific, keys of this tree are inside CIDRs and values
+	// are *AllowList
+	insideAllowLists *cidr.Tree6
+}
+
+type LocalAllowList struct {
+	AllowList *AllowList
+
+	// To avoid ambiguity, all rules must be true, or all rules must be false.
+	nameRules []AllowListNameRule
+}
+
+type AllowListNameRule struct {
+	Name  *regexp.Regexp
+	Allow bool
+}
+
+func NewLocalAllowListFromConfig(c *config.C, k string) (*LocalAllowList, error) {
+	var nameRules []AllowListNameRule
+	handleKey := func(key string, value interface{}) (bool, error) {
+		if key == "interfaces" {
+			var err error
+			nameRules, err = getAllowListInterfaces(k, value)
+			if err != nil {
+				return false, err
+			}
+
+			return true, nil
+		}
+		return false, nil
+	}
+
+	al, err := newAllowListFromConfig(c, k, handleKey)
+	if err != nil {
+		return nil, err
+	}
+	return &LocalAllowList{AllowList: al, nameRules: nameRules}, nil
+}
+
+func NewRemoteAllowListFromConfig(c *config.C, k, rangesKey string) (*RemoteAllowList, error) {
+	al, err := newAllowListFromConfig(c, k, nil)
+	if err != nil {
+		return nil, err
+	}
+	remoteAllowRanges, err := getRemoteAllowRanges(c, rangesKey)
+	if err != nil {
+		return nil, err
+	}
+	return &RemoteAllowList{AllowList: al, insideAllowLists: remoteAllowRanges}, nil
+}
+
+// If the handleKey func returns true, the rest of the parsing is skipped
+// for this key. This allows parsing of special values like `interfaces`.
+func newAllowListFromConfig(c *config.C, k string, handleKey func(key string, value interface{}) (bool, error)) (*AllowList, error) {
+	r := c.Get(k)
+	if r == nil {
+		return nil, nil
+	}
+
+	return newAllowList(k, r, handleKey)
+}
+
+// If the handleKey func returns true, the rest of the parsing is skipped
+// for this key. This allows parsing of special values like `interfaces`.
+func newAllowList(k string, raw interface{}, handleKey func(key string, value interface{}) (bool, error)) (*AllowList, error) {
+	rawMap, ok := raw.(map[interface{}]interface{})
+	if !ok {
+		return nil, fmt.Errorf("config `%s` has invalid type: %T", k, raw)
+	}
+
+	tree := cidr.NewTree6()
+
+	// Keep track of the rules we have added for both ipv4 and ipv6
+	type allowListRules struct {
+		firstValue     bool
+		allValuesMatch bool
+		defaultSet     bool
+		allValues      bool
+	}
+
+	rules4 := allowListRules{firstValue: true, allValuesMatch: true, defaultSet: false}
+	rules6 := allowListRules{firstValue: true, allValuesMatch: true, defaultSet: false}
+
+	for rawKey, rawValue := range rawMap {
+		rawCIDR, ok := rawKey.(string)
+		if !ok {
+			return nil, fmt.Errorf("config `%s` has invalid key (type %T): %v", k, rawKey, rawKey)
+		}
+
+		if handleKey != nil {
+			handled, err := handleKey(rawCIDR, rawValue)
+			if err != nil {
+				return nil, err
+			}
+			if handled {
+				continue
+			}
+		}
+
+		value, ok := rawValue.(bool)
+		if !ok {
+			return nil, fmt.Errorf("config `%s` has invalid value (type %T): %v", k, rawValue, rawValue)
+		}
+
+		_, ipNet, err := net.ParseCIDR(rawCIDR)
+		if err != nil {
+			return nil, fmt.Errorf("config `%s` has invalid CIDR: %s", k, rawCIDR)
+		}
+
+		// TODO: should we error on duplicate CIDRs in the config?
+		tree.AddCIDR(ipNet, value)
+
+		maskBits, maskSize := ipNet.Mask.Size()
+
+		var rules *allowListRules
+		if maskSize == 32 {
+			rules = &rules4
+		} else {
+			rules = &rules6
+		}
+
+		if rules.firstValue {
+			rules.allValues = value
+			rules.firstValue = false
+		} else {
+			if value != rules.allValues {
+				rules.allValuesMatch = false
+			}
+		}
+
+		// Check if this is 0.0.0.0/0 or ::/0
+		if maskBits == 0 {
+			rules.defaultSet = true
+		}
+	}
+
+	if !rules4.defaultSet {
+		if rules4.allValuesMatch {
+			_, zeroCIDR, _ := net.ParseCIDR("0.0.0.0/0")
+			tree.AddCIDR(zeroCIDR, !rules4.allValues)
+		} else {
+			return nil, fmt.Errorf("config `%s` contains both true and false rules, but no default set for 0.0.0.0/0", k)
+		}
+	}
+
+	if !rules6.defaultSet {
+		if rules6.allValuesMatch {
+			_, zeroCIDR, _ := net.ParseCIDR("::/0")
+			tree.AddCIDR(zeroCIDR, !rules6.allValues)
+		} else {
+			return nil, fmt.Errorf("config `%s` contains both true and false rules, but no default set for ::/0", k)
+		}
+	}
+
+	return &AllowList{cidrTree: tree}, nil
+}
+
+func getAllowListInterfaces(k string, v interface{}) ([]AllowListNameRule, error) {
+	var nameRules []AllowListNameRule
+
+	rawRules, ok := v.(map[interface{}]interface{})
+	if !ok {
+		return nil, fmt.Errorf("config `%s.interfaces` is invalid (type %T): %v", k, v, v)
+	}
+
+	firstEntry := true
+	var allValues bool
+	for rawName, rawAllow := range rawRules {
+		name, ok := rawName.(string)
+		if !ok {
+			return nil, fmt.Errorf("config `%s.interfaces` has invalid key (type %T): %v", k, rawName, rawName)
+		}
+		allow, ok := rawAllow.(bool)
+		if !ok {
+			return nil, fmt.Errorf("config `%s.interfaces` has invalid value (type %T): %v", k, rawAllow, rawAllow)
+		}
+
+		nameRE, err := regexp.Compile("^" + name + "$")
+		if err != nil {
+			return nil, fmt.Errorf("config `%s.interfaces` has invalid key: %s: %v", k, name, err)
+		}
+
+		nameRules = append(nameRules, AllowListNameRule{
+			Name:  nameRE,
+			Allow: allow,
+		})
+
+		if firstEntry {
+			allValues = allow
+			firstEntry = false
+		} else {
+			if allow != allValues {
+				return nil, fmt.Errorf("config `%s.interfaces` values must all be the same true/false value", k)
+			}
+		}
+	}
+
+	return nameRules, nil
+}
+
+func getRemoteAllowRanges(c *config.C, k string) (*cidr.Tree6, error) {
+	value := c.Get(k)
+	if value == nil {
+		return nil, nil
+	}
+
+	remoteAllowRanges := cidr.NewTree6()
+
+	rawMap, ok := value.(map[interface{}]interface{})
+	if !ok {
+		return nil, fmt.Errorf("config `%s` has invalid type: %T", k, value)
+	}
+	for rawKey, rawValue := range rawMap {
+		rawCIDR, ok := rawKey.(string)
+		if !ok {
+			return nil, fmt.Errorf("config `%s` has invalid key (type %T): %v", k, rawKey, rawKey)
+		}
+
+		allowList, err := newAllowList(fmt.Sprintf("%s.%s", k, rawCIDR), rawValue, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		_, ipNet, err := net.ParseCIDR(rawCIDR)
+		if err != nil {
+			return nil, fmt.Errorf("config `%s` has invalid CIDR: %s", k, rawCIDR)
+		}
+
+		remoteAllowRanges.AddCIDR(ipNet, allowList)
+	}
+
+	return remoteAllowRanges, nil
+}
+
+func (al *AllowList) Allow(ip net.IP) bool {
+	if al == nil {
+		return true
+	}
+
+	result := al.cidrTree.MostSpecificContains(ip)
+	switch v := result.(type) {
+	case bool:
+		return v
+	default:
+		panic(fmt.Errorf("invalid state, allowlist returned: %T %v", result, result))
+	}
+}
+
+func (al *AllowList) AllowIpV4(ip iputil.VpnIp) bool {
+	if al == nil {
+		return true
+	}
+
+	result := al.cidrTree.MostSpecificContainsIpV4(ip)
+	switch v := result.(type) {
+	case bool:
+		return v
+	default:
+		panic(fmt.Errorf("invalid state, allowlist returned: %T %v", result, result))
+	}
+}
+
+func (al *AllowList) AllowIpV6(hi, lo uint64) bool {
+	if al == nil {
+		return true
+	}
+
+	result := al.cidrTree.MostSpecificContainsIpV6(hi, lo)
+	switch v := result.(type) {
+	case bool:
+		return v
+	default:
+		panic(fmt.Errorf("invalid state, allowlist returned: %T %v", result, result))
+	}
+}
+
+func (al *LocalAllowList) Allow(ip net.IP) bool {
+	if al == nil {
+		return true
+	}
+	return al.AllowList.Allow(ip)
+}
+
+func (al *LocalAllowList) AllowName(name string) bool {
+	if al == nil || len(al.nameRules) == 0 {
+		return true
+	}
+
+	for _, rule := range al.nameRules {
+		if rule.Name.MatchString(name) {
+			return rule.Allow
+		}
+	}
+
+	// If no rules match, return the default, which is the inverse of the rules
+	return !al.nameRules[0].Allow
+}
+
+func (al *RemoteAllowList) AllowUnknownVpnIp(ip net.IP) bool {
+	if al == nil {
+		return true
+	}
+	return al.AllowList.Allow(ip)
+}
+
+func (al *RemoteAllowList) Allow(vpnIp iputil.VpnIp, ip net.IP) bool {
+	if !al.getInsideAllowList(vpnIp).Allow(ip) {
+		return false
+	}
+	return al.AllowList.Allow(ip)
+}
+
+func (al *RemoteAllowList) AllowIpV4(vpnIp iputil.VpnIp, ip iputil.VpnIp) bool {
+	if al == nil {
+		return true
+	}
+	if !al.getInsideAllowList(vpnIp).AllowIpV4(ip) {
+		return false
+	}
+	return al.AllowList.AllowIpV4(ip)
+}
+
+func (al *RemoteAllowList) AllowIpV6(vpnIp iputil.VpnIp, hi, lo uint64) bool {
+	if al == nil {
+		return true
+	}
+	if !al.getInsideAllowList(vpnIp).AllowIpV6(hi, lo) {
+		return false
+	}
+	return al.AllowList.AllowIpV6(hi, lo)
+}
+
+func (al *RemoteAllowList) getInsideAllowList(vpnIp iputil.VpnIp) *AllowList {
+	if al.insideAllowLists != nil {
+		inside := al.insideAllowLists.MostSpecificContainsIpV4(vpnIp)
+		if inside != nil {
+			return inside.(*AllowList)
+		}
+	}
+	return nil
+}
--- a/allow_list_test.go
+++ b/allow_list_test.go
@@ -0,0 +1,145 @@
+package nebula
+
+import (
+	"net"
+	"regexp"
+	"testing"
+
+	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewAllowListFromConfig(t *testing.T) {
+	l := util.NewTestLogger()
+	c := config.NewC(l)
+	c.Settings["allowlist"] = map[interface{}]interface{}{
+		"192.168.0.0": true,
+	}
+	r, err := newAllowListFromConfig(c, "allowlist", nil)
+	assert.EqualError(t, err, "config `allowlist` has invalid CIDR: 192.168.0.0")
+	assert.Nil(t, r)
+
+	c.Settings["allowlist"] = map[interface{}]interface{}{
+		"192.168.0.0/16": "abc",
+	}
+	r, err = newAllowListFromConfig(c, "allowlist", nil)
+	assert.EqualError(t, err, "config `allowlist` has invalid value (type string): abc")
+
+	c.Settings["allowlist"] = map[interface{}]interface{}{
+		"192.168.0.0/16": true,
+		"10.0.0.0/8":     false,
+	}
+	r, err = newAllowListFromConfig(c, "allowlist", nil)
+	assert.EqualError(t, err, "config `allowlist` contains both true and false rules, but no default set for 0.0.0.0/0")
+
+	c.Settings["allowlist"] = map[interface{}]interface{}{
+		"0.0.0.0/0":      true,
+		"10.0.0.0/8":     false,
+		"10.42.42.0/24":  true,
+		"fd00::/8":       true,
+		"fd00:fd00::/16": false,
+	}
+	r, err = newAllowListFromConfig(c, "allowlist", nil)
+	assert.EqualError(t, err, "config `allowlist` contains both true and false rules, but no default set for ::/0")
+
+	c.Settings["allowlist"] = map[interface{}]interface{}{
+		"0.0.0.0/0":     true,
+		"10.0.0.0/8":    false,
+		"10.42.42.0/24": true,
+	}
+	r, err = newAllowListFromConfig(c, "allowlist", nil)
+	if assert.NoError(t, err) {
+		assert.NotNil(t, r)
+	}
+
+	c.Settings["allowlist"] = map[interface{}]interface{}{
+		"0.0.0.0/0":      true,
+		"10.0.0.0/8":     false,
+		"10.42.42.0/24":  true,
+		"::/0":           false,
+		"fd00::/8":       true,
+		"fd00:fd00::/16": false,
+	}
+	r, err = newAllowListFromConfig(c, "allowlist", nil)
+	if assert.NoError(t, err) {
+		assert.NotNil(t, r)
+	}
+
+	// Test interface names
+
+	c.Settings["allowlist"] = map[interface{}]interface{}{
+		"interfaces": map[interface{}]interface{}{
+			`docker.*`: "foo",
+		},
+	}
+	lr, err := NewLocalAllowListFromConfig(c, "allowlist")
+	assert.EqualError(t, err, "config `allowlist.interfaces` has invalid value (type string): foo")
+
+	c.Settings["allowlist"] = map[interface{}]interface{}{
+		"interfaces": map[interface{}]interface{}{
+			`docker.*`: false,
+			`eth.*`:    true,
+		},
+	}
+	lr, err = NewLocalAllowListFromConfig(c, "allowlist")
+	assert.EqualError(t, err, "config `allowlist.interfaces` values must all be the same true/false value")
+
+	c.Settings["allowlist"] = map[interface{}]interface{}{
+		"interfaces": map[interface{}]interface{}{
+			`docker.*`: false,
+		},
+	}
+	lr, err = NewLocalAllowListFromConfig(c, "allowlist")
+	if assert.NoError(t, err) {
+		assert.NotNil(t, lr)
+	}
+}
+
+func TestAllowList_Allow(t *testing.T) {
+	assert.Equal(t, true, ((*AllowList)(nil)).Allow(net.ParseIP("1.1.1.1")))
+
+	tree := cidr.NewTree6()
+	tree.AddCIDR(cidr.Parse("0.0.0.0/0"), true)
+	tree.AddCIDR(cidr.Parse("10.0.0.0/8"), false)
+	tree.AddCIDR(cidr.Parse("10.42.42.42/32"), true)
+	tree.AddCIDR(cidr.Parse("10.42.0.0/16"), true)
+	tree.AddCIDR(cidr.Parse("10.42.42.0/24"), true)
+	tree.AddCIDR(cidr.Parse("10.42.42.0/24"), false)
+	tree.AddCIDR(cidr.Parse("::1/128"), true)
+	tree.AddCIDR(cidr.Parse("::2/128"), false)
+	al := &AllowList{cidrTree: tree}
+
+	assert.Equal(t, true, al.Allow(net.ParseIP("1.1.1.1")))
+	assert.Equal(t, false, al.Allow(net.ParseIP("10.0.0.4")))
+	assert.Equal(t, true, al.Allow(net.ParseIP("10.42.42.42")))
+	assert.Equal(t, false, al.Allow(net.ParseIP("10.42.42.41")))
+	assert.Equal(t, true, al.Allow(net.ParseIP("10.42.0.1")))
+	assert.Equal(t, true, al.Allow(net.ParseIP("::1")))
+	assert.Equal(t, false, al.Allow(net.ParseIP("::2")))
+}
+
+func TestLocalAllowList_AllowName(t *testing.T) {
+	assert.Equal(t, true, ((*LocalAllowList)(nil)).AllowName("docker0"))
+
+	rules := []AllowListNameRule{
+		{Name: regexp.MustCompile("^docker.*$"), Allow: false},
+		{Name: regexp.MustCompile("^tun.*$"), Allow: false},
+	}
+	al := &LocalAllowList{nameRules: rules}
+
+	assert.Equal(t, false, al.AllowName("docker0"))
+	assert.Equal(t, false, al.AllowName("tun0"))
+	assert.Equal(t, true, al.AllowName("eth0"))
+
+	rules = []AllowListNameRule{
+		{Name: regexp.MustCompile("^eth.*$"), Allow: true},
+		{Name: regexp.MustCompile("^ens.*$"), Allow: true},
+	}
+	al = &LocalAllowList{nameRules: rules}
+
+	assert.Equal(t, false, al.AllowName("docker0"))
+	assert.Equal(t, true, al.AllowName("eth0"))
+	assert.Equal(t, true, al.AllowName("ens5"))
+}
--- a/bits.go
+++ b/bits.go
@@ -26,7 +26,7 @@ func NewBits(bits uint64) *Bits {
 	}
 }

-func (b *Bits) Check(i uint64) bool {
+func (b *Bits) Check(l logrus.FieldLogger, i uint64) bool {
 	// If i is the next number, return true.
 	if i > b.current || (i == 0 && b.firstSeen == false && b.current < b.length) {
 		return true
@@ -47,7 +47,7 @@ func (b *Bits) Check(i uint64) bool {
 	return false
 }

-func (b *Bits) Update(i uint64) bool {
+func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 	// If i is the next number, return true and update current.
 	if i == b.current+1 {
 		// Report missed packets, we can only understand what was missed after the first window has been gone through
--- a/bits_test.go
+++ b/bits_test.go
@@ -3,10 +3,12 @@ package nebula
 import (
 	"testing"

+	"github.com/slackhq/nebula/util"
 	"github.com/stretchr/testify/assert"
 )

 func TestBits(t *testing.T) {
+	l := util.NewTestLogger()
 	b := NewBits(10)

 	// make sure it is the right size
@@ -14,46 +16,46 @@ func TestBits(t *testing.T) {

 	// This is initialized to zero - receive one. This should work.

-	assert.True(t, b.Check(1))
-	u := b.Update(1)
+	assert.True(t, b.Check(l, 1))
+	u := b.Update(l, 1)
 	assert.True(t, u)
 	assert.EqualValues(t, 1, b.current)
 	g := []bool{false, true, false, false, false, false, false, false, false, false}
 	assert.Equal(t, g, b.bits)

 	// Receive two
-	assert.True(t, b.Check(2))
-	u = b.Update(2)
+	assert.True(t, b.Check(l, 2))
+	u = b.Update(l, 2)
 	assert.True(t, u)
 	assert.EqualValues(t, 2, b.current)
 	g = []bool{false, true, true, false, false, false, false, false, false, false}
 	assert.Equal(t, g, b.bits)

 	// Receive two again - it will fail
-	assert.False(t, b.Check(2))
-	u = b.Update(2)
+	assert.False(t, b.Check(l, 2))
+	u = b.Update(l, 2)
 	assert.False(t, u)
 	assert.EqualValues(t, 2, b.current)

 	// Jump ahead to 15, which should clear everything and set the 6th element
-	assert.True(t, b.Check(15))
-	u = b.Update(15)
+	assert.True(t, b.Check(l, 15))
+	u = b.Update(l, 15)
 	assert.True(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, false, true, false, false, false, false}
 	assert.Equal(t, g, b.bits)

 	// Mark 14, which is allowed because it is in the window
-	assert.True(t, b.Check(14))
-	u = b.Update(14)
+	assert.True(t, b.Check(l, 14))
+	u = b.Update(l, 14)
 	assert.True(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, true, true, false, false, false, false}
 	assert.Equal(t, g, b.bits)

 	// Mark 5, which is not allowed because it is not in the window
-	assert.False(t, b.Check(5))
-	u = b.Update(5)
+	assert.False(t, b.Check(l, 5))
+	u = b.Update(l, 5)
 	assert.False(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, true, true, false, false, false, false}
@@ -61,63 +63,65 @@ func TestBits(t *testing.T) {

 	// make sure we handle wrapping around once to the current position
 	b = NewBits(10)
-	assert.True(t, b.Update(1))
-	assert.True(t, b.Update(11))
+	assert.True(t, b.Update(l, 1))
+	assert.True(t, b.Update(l, 11))
 	assert.Equal(t, []bool{false, true, false, false, false, false, false, false, false, false}, b.bits)

 	// Walk through a few windows in order
 	b = NewBits(10)
 	for i := uint64(0); i <= 100; i++ {
-		assert.True(t, b.Check(i), "Error while checking %v", i)
-		assert.True(t, b.Update(i), "Error while updating %v", i)
+		assert.True(t, b.Check(l, i), "Error while checking %v", i)
+		assert.True(t, b.Update(l, i), "Error while updating %v", i)
 	}
 }

 func TestBitsDupeCounter(t *testing.T) {
+	l := util.NewTestLogger()
 	b := NewBits(10)
 	b.lostCounter.Clear()
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()

-	assert.True(t, b.Update(1))
+	assert.True(t, b.Update(l, 1))
 	assert.Equal(t, int64(0), b.dupeCounter.Count())

-	assert.False(t, b.Update(1))
+	assert.False(t, b.Update(l, 1))
 	assert.Equal(t, int64(1), b.dupeCounter.Count())

-	assert.True(t, b.Update(2))
+	assert.True(t, b.Update(l, 2))
 	assert.Equal(t, int64(1), b.dupeCounter.Count())

-	assert.True(t, b.Update(3))
+	assert.True(t, b.Update(l, 3))
 	assert.Equal(t, int64(1), b.dupeCounter.Count())

-	assert.False(t, b.Update(1))
+	assert.False(t, b.Update(l, 1))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
 	assert.Equal(t, int64(2), b.dupeCounter.Count())
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 }

 func TestBitsOutOfWindowCounter(t *testing.T) {
+	l := util.NewTestLogger()
 	b := NewBits(10)
 	b.lostCounter.Clear()
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()

-	assert.True(t, b.Update(20))
+	assert.True(t, b.Update(l, 20))
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())

-	assert.True(t, b.Update(21))
-	assert.True(t, b.Update(22))
-	assert.True(t, b.Update(23))
-	assert.True(t, b.Update(24))
-	assert.True(t, b.Update(25))
-	assert.True(t, b.Update(26))
-	assert.True(t, b.Update(27))
-	assert.True(t, b.Update(28))
-	assert.True(t, b.Update(29))
+	assert.True(t, b.Update(l, 21))
+	assert.True(t, b.Update(l, 22))
+	assert.True(t, b.Update(l, 23))
+	assert.True(t, b.Update(l, 24))
+	assert.True(t, b.Update(l, 25))
+	assert.True(t, b.Update(l, 26))
+	assert.True(t, b.Update(l, 27))
+	assert.True(t, b.Update(l, 28))
+	assert.True(t, b.Update(l, 29))
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())

-	assert.False(t, b.Update(0))
+	assert.False(t, b.Update(l, 0))
 	assert.Equal(t, int64(1), b.outOfWindowCounter.Count())

 	//tODO: make sure lostcounter doesn't increase in orderly increment
@@ -127,23 +131,24 @@ func TestBitsOutOfWindowCounter(t *testing.T) {
 }

 func TestBitsLostCounter(t *testing.T) {
+	l := util.NewTestLogger()
 	b := NewBits(10)
 	b.lostCounter.Clear()
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()

 	//assert.True(t, b.Update(0))
-	assert.True(t, b.Update(0))
-	assert.True(t, b.Update(20))
-	assert.True(t, b.Update(21))
-	assert.True(t, b.Update(22))
-	assert.True(t, b.Update(23))
-	assert.True(t, b.Update(24))
-	assert.True(t, b.Update(25))
-	assert.True(t, b.Update(26))
-	assert.True(t, b.Update(27))
-	assert.True(t, b.Update(28))
-	assert.True(t, b.Update(29))
+	assert.True(t, b.Update(l, 0))
+	assert.True(t, b.Update(l, 20))
+	assert.True(t, b.Update(l, 21))
+	assert.True(t, b.Update(l, 22))
+	assert.True(t, b.Update(l, 23))
+	assert.True(t, b.Update(l, 24))
+	assert.True(t, b.Update(l, 25))
+	assert.True(t, b.Update(l, 26))
+	assert.True(t, b.Update(l, 27))
+	assert.True(t, b.Update(l, 28))
+	assert.True(t, b.Update(l, 29))
 	assert.Equal(t, int64(20), b.lostCounter.Count())
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
@@ -153,56 +158,56 @@ func TestBitsLostCounter(t *testing.T) {
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()

-	assert.True(t, b.Update(0))
+	assert.True(t, b.Update(l, 0))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(9))
+	assert.True(t, b.Update(l, 9))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
 	// 10 will set 0 index, 0 was already set, no lost packets
-	assert.True(t, b.Update(10))
+	assert.True(t, b.Update(l, 10))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
 	// 11 will set 1 index, 1 was missed, we should see 1 packet lost
-	assert.True(t, b.Update(11))
+	assert.True(t, b.Update(l, 11))
 	assert.Equal(t, int64(1), b.lostCounter.Count())
 	// Now let's fill in the window, should end up with 8 lost packets
-	assert.True(t, b.Update(12))
-	assert.True(t, b.Update(13))
-	assert.True(t, b.Update(14))
-	assert.True(t, b.Update(15))
-	assert.True(t, b.Update(16))
-	assert.True(t, b.Update(17))
-	assert.True(t, b.Update(18))
-	assert.True(t, b.Update(19))
+	assert.True(t, b.Update(l, 12))
+	assert.True(t, b.Update(l, 13))
+	assert.True(t, b.Update(l, 14))
+	assert.True(t, b.Update(l, 15))
+	assert.True(t, b.Update(l, 16))
+	assert.True(t, b.Update(l, 17))
+	assert.True(t, b.Update(l, 18))
+	assert.True(t, b.Update(l, 19))
 	assert.Equal(t, int64(8), b.lostCounter.Count())

 	// Jump ahead by a window size
-	assert.True(t, b.Update(29))
+	assert.True(t, b.Update(l, 29))
 	assert.Equal(t, int64(8), b.lostCounter.Count())
 	// Now lets walk ahead normally through the window, the missed packets should fill in
-	assert.True(t, b.Update(30))
-	assert.True(t, b.Update(31))
-	assert.True(t, b.Update(32))
-	assert.True(t, b.Update(33))
-	assert.True(t, b.Update(34))
-	assert.True(t, b.Update(35))
-	assert.True(t, b.Update(36))
-	assert.True(t, b.Update(37))
-	assert.True(t, b.Update(38))
+	assert.True(t, b.Update(l, 30))
+	assert.True(t, b.Update(l, 31))
+	assert.True(t, b.Update(l, 32))
+	assert.True(t, b.Update(l, 33))
+	assert.True(t, b.Update(l, 34))
+	assert.True(t, b.Update(l, 35))
+	assert.True(t, b.Update(l, 36))
+	assert.True(t, b.Update(l, 37))
+	assert.True(t, b.Update(l, 38))
 	// 39 packets tracked, 22 seen, 17 lost
 	assert.Equal(t, int64(17), b.lostCounter.Count())

 	// Jump ahead by 2 windows, should have recording 1 full window missing
-	assert.True(t, b.Update(58))
+	assert.True(t, b.Update(l, 58))
 	assert.Equal(t, int64(27), b.lostCounter.Count())
 	// Now lets walk ahead normally through the window, the missed packets should fill in from this window
-	assert.True(t, b.Update(59))
-	assert.True(t, b.Update(60))
-	assert.True(t, b.Update(61))
-	assert.True(t, b.Update(62))
-	assert.True(t, b.Update(63))
-	assert.True(t, b.Update(64))
-	assert.True(t, b.Update(65))
-	assert.True(t, b.Update(66))
-	assert.True(t, b.Update(67))
+	assert.True(t, b.Update(l, 59))
+	assert.True(t, b.Update(l, 60))
+	assert.True(t, b.Update(l, 61))
+	assert.True(t, b.Update(l, 62))
+	assert.True(t, b.Update(l, 63))
+	assert.True(t, b.Update(l, 64))
+	assert.True(t, b.Update(l, 65))
+	assert.True(t, b.Update(l, 66))
+	assert.True(t, b.Update(l, 67))
 	// 68 packets tracked, 32 seen, 36 missed
 	assert.Equal(t, int64(36), b.lostCounter.Count())
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
@@ -212,10 +217,10 @@ func TestBitsLostCounter(t *testing.T) {
 func BenchmarkBits(b *testing.B) {
 	z := NewBits(10)
 	for n := 0; n < b.N; n++ {
-		for i, _ := range z.bits {
+		for i := range z.bits {
 			z.bits[i] = true
 		}
-		for i, _ := range z.bits {
+		for i := range z.bits {
 			z.bits[i] = false
 		}

--- a/cert.go
+++ b/cert.go
@@ -7,11 +7,11 @@ import (
 	"strings"
 	"time"

+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/config"
 )

-var trustedCAs *cert.NebulaCAPool
-
 type CertState struct {
 	certificate         *cert.NebulaCertificate
 	rawCertificate      []byte
@@ -46,7 +46,7 @@ func NewCertState(certificate *cert.NebulaCertificate, privateKey []byte) (*Cert
 	return cs, nil
 }

-func NewCertStateFromConfig(c *Config) (*CertState, error) {
+func NewCertStateFromConfig(c *config.C) (*CertState, error) {
 	var pemPrivateKey []byte
 	var err error

@@ -119,7 +119,7 @@ func NewCertStateFromConfig(c *Config) (*CertState, error) {
 	return NewCertState(nebulaCert, rawKey)
 }

-func loadCAFromConfig(c *Config) (*cert.NebulaCAPool, error) {
+func loadCAFromConfig(l *logrus.Logger, c *config.C) (*cert.NebulaCAPool, error) {
 	var rawCA []byte
 	var err error

@@ -149,10 +149,16 @@ func loadCAFromConfig(c *Config) (*cert.NebulaCAPool, error) {
 		return nil, fmt.Errorf("error while adding CA certificate to CA trust store: %s", err)
 	}

-	// pki.blacklist entered the scene at about the same time we aliased x509 to pki, not supporting backwards compat
+	for _, fp := range c.GetStringSlice("pki.blocklist", []string{}) {
+		l.WithField("fingerprint", fp).Infof("Blocklisting cert")
+		CAs.BlocklistFingerprint(fp)
+	}
+
+	// Support deprecated config for at leaast one minor release to allow for migrations
 	for _, fp := range c.GetStringSlice("pki.blacklist", []string{}) {
-		l.WithField("fingerprint", fp).Infof("Blacklisting cert")
-		CAs.BlacklistFingerprint(fp)
+		l.WithField("fingerprint", fp).Infof("Blocklisting cert")
+		l.Warn("pki.blacklist is deprecated and will not be supported in a future release. Please migrate your config to use pki.blocklist")
+		CAs.BlocklistFingerprint(fp)
 	}

 	return CAs, nil
--- a/cert/Makefile
+++ b/cert/Makefile
@@ -2,8 +2,8 @@ GO111MODULE = on
 export GO111MODULE

 cert.pb.go: cert.proto .FORCE
-	go build github.com/golang/protobuf/protoc-gen-go
-	PATH="$(PWD):$(PATH)" protoc --go_out=. $<
+	go build google.golang.org/protobuf/cmd/protoc-gen-go
+	PATH="$(CURDIR):$(PATH)" protoc --go_out=. --go_opt=paths=source_relative $<
 	rm protoc-gen-go

 .FORCE:
--- a/cert/ca.go
+++ b/cert/ca.go
@@ -8,14 +8,14 @@ import (

 type NebulaCAPool struct {
 	CAs           map[string]*NebulaCertificate
-	certBlacklist map[string]struct{}
+	certBlocklist map[string]struct{}
 }

 // NewCAPool creates a CAPool
 func NewCAPool() *NebulaCAPool {
 	ca := NebulaCAPool{
 		CAs:           make(map[string]*NebulaCertificate),
-		certBlacklist: make(map[string]struct{}),
+		certBlocklist: make(map[string]struct{}),
 	}

 	return &ca
@@ -67,24 +67,24 @@ func (ncp *NebulaCAPool) AddCACertificate(pemBytes []byte) ([]byte, error) {
 	return pemBytes, nil
 }

-// BlacklistFingerprint adds a cert fingerprint to the blacklist
-func (ncp *NebulaCAPool) BlacklistFingerprint(f string) {
-	ncp.certBlacklist[f] = struct{}{}
+// BlocklistFingerprint adds a cert fingerprint to the blocklist
+func (ncp *NebulaCAPool) BlocklistFingerprint(f string) {
+	ncp.certBlocklist[f] = struct{}{}
 }

-// ResetCertBlacklist removes all previously blacklisted cert fingerprints
-func (ncp *NebulaCAPool) ResetCertBlacklist() {
-	ncp.certBlacklist = make(map[string]struct{})
+// ResetCertBlocklist removes all previously blocklisted cert fingerprints
+func (ncp *NebulaCAPool) ResetCertBlocklist() {
+	ncp.certBlocklist = make(map[string]struct{})
 }

-// IsBlacklisted returns true if the fingerprint fails to generate or has been explicitly blacklisted
-func (ncp *NebulaCAPool) IsBlacklisted(c *NebulaCertificate) bool {
+// IsBlocklisted returns true if the fingerprint fails to generate or has been explicitly blocklisted
+func (ncp *NebulaCAPool) IsBlocklisted(c *NebulaCertificate) bool {
 	h, err := c.Sha256Sum()
 	if err != nil {
 		return true
 	}

-	if _, ok := ncp.certBlacklist[h]; ok {
+	if _, ok := ncp.certBlocklist[h]; ok {
 		return true
 	}

--- a/cert/cert.go
+++ b/cert/cert.go
@@ -1,18 +1,18 @@
 package cert

 import (
+	"bytes"
 	"crypto"
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/binary"
 	"encoding/hex"
+	"encoding/json"
 	"encoding/pem"
 	"fmt"
 	"net"
 	"time"

-	"bytes"
-	"encoding/json"
 	"github.com/golang/protobuf/proto"
 	"golang.org/x/crypto/curve25519"
 	"golang.org/x/crypto/ed25519"
@@ -61,6 +61,10 @@ func UnmarshalNebulaCertificate(b []byte) (*NebulaCertificate, error) {
 		return nil, err
 	}

+	if rc.Details == nil {
+		return nil, fmt.Errorf("encoded Details was nil")
+	}
+
 	if len(rc.Details.Ips)%2 != 0 {
 		return nil, fmt.Errorf("encoded IPs should be in pairs, an odd number was found")
 	}
@@ -123,6 +127,9 @@ func UnmarshalNebulaCertificateFromPEM(b []byte) (*NebulaCertificate, []byte, er
 	if p == nil {
 		return nil, r, fmt.Errorf("input did not contain a valid PEM encoded block")
 	}
+	if p.Type != CertBanner {
+		return nil, r, fmt.Errorf("bytes did not contain a proper nebula certificate banner")
+	}
 	nc, err := UnmarshalNebulaCertificate(p.Bytes)
 	return nc, r, err
 }
@@ -244,10 +251,10 @@ func (nc *NebulaCertificate) Expired(t time.Time) bool {
 	return nc.Details.NotBefore.After(t) || nc.Details.NotAfter.Before(t)
 }

-// Verify will ensure a certificate is good in all respects (expiry, group membership, signature, cert blacklist, etc)
+// Verify will ensure a certificate is good in all respects (expiry, group membership, signature, cert blocklist, etc)
 func (nc *NebulaCertificate) Verify(t time.Time, ncp *NebulaCAPool) (bool, error) {
-	if ncp.IsBlacklisted(nc) {
-		return false, fmt.Errorf("certificate has been blacklisted")
+	if ncp.IsBlocklisted(nc) {
+		return false, fmt.Errorf("certificate has been blocked")
 	}

 	signer, err := ncp.GetCAForCert(nc)
@@ -267,29 +274,80 @@ func (nc *NebulaCertificate) Verify(t time.Time, ncp *NebulaCAPool) (bool, error
 		return false, fmt.Errorf("certificate signature did not match")
 	}

-	// If the signer has a limited set of groups make sure the cert only contains a subset
-	if len(signer.Details.InvertedGroups) > 0 {
-		for _, g := range nc.Details.Groups {
-			if _, ok := signer.Details.InvertedGroups[g]; !ok {
-				return false, fmt.Errorf("certificate contained a group not present on the signing ca; %s", g)
-			}
-		}
+	if err := nc.CheckRootConstrains(signer); err != nil {
+		return false, err
 	}

 	return true, nil
 }

+// CheckRootConstrains returns an error if the certificate violates constraints set on the root (groups, ips, subnets)
+func (nc *NebulaCertificate) CheckRootConstrains(signer *NebulaCertificate) error {
+	// Make sure this cert wasn't valid before the root
+	if signer.Details.NotAfter.Before(nc.Details.NotAfter) {
+		return fmt.Errorf("certificate expires after signing certificate")
+	}
+
+	// Make sure this cert isn't valid after the root
+	if signer.Details.NotBefore.After(nc.Details.NotBefore) {
+		return fmt.Errorf("certificate is valid before the signing certificate")
+	}
+
+	// If the signer has a limited set of groups make sure the cert only contains a subset
+	if len(signer.Details.InvertedGroups) > 0 {
+		for _, g := range nc.Details.Groups {
+			if _, ok := signer.Details.InvertedGroups[g]; !ok {
+				return fmt.Errorf("certificate contained a group not present on the signing ca: %s", g)
+			}
+		}
+	}
+
+	// If the signer has a limited set of ip ranges to issue from make sure the cert only contains a subset
+	if len(signer.Details.Ips) > 0 {
+		for _, ip := range nc.Details.Ips {
+			if !netMatch(ip, signer.Details.Ips) {
+				return fmt.Errorf("certificate contained an ip assignment outside the limitations of the signing ca: %s", ip.String())
+			}
+		}
+	}
+
+	// If the signer has a limited set of subnet ranges to issue from make sure the cert only contains a subset
+	if len(signer.Details.Subnets) > 0 {
+		for _, subnet := range nc.Details.Subnets {
+			if !netMatch(subnet, signer.Details.Subnets) {
+				return fmt.Errorf("certificate contained a subnet assignment outside the limitations of the signing ca: %s", subnet)
+			}
+		}
+	}
+
+	return nil
+}
+
 // VerifyPrivateKey checks that the public key in the Nebula certificate and a supplied private key match
 func (nc *NebulaCertificate) VerifyPrivateKey(key []byte) error {
-	var dst, key32 [32]byte
-	copy(key32[:], key)
-	curve25519.ScalarBaseMult(&dst, &key32)
-	if !bytes.Equal(dst[:], nc.Details.PublicKey) {
+	if nc.Details.IsCA {
+		// the call to PublicKey below will panic slice bounds out of range otherwise
+		if len(key) != ed25519.PrivateKeySize {
+			return fmt.Errorf("key was not 64 bytes, is invalid ed25519 private key")
+		}
+
+		if !ed25519.PublicKey(nc.Details.PublicKey).Equal(ed25519.PrivateKey(key).Public()) {
 			return fmt.Errorf("public key in cert and private key supplied don't match")
 		}
 		return nil
 	}

+	pub, err := curve25519.X25519(key, curve25519.Basepoint)
+	if err != nil {
+		return err
+	}
+	if !bytes.Equal(pub, nc.Details.PublicKey) {
+		return fmt.Errorf("public key in cert and private key supplied don't match")
+	}
+
+	return nil
+}
+
 // String will return a pretty printed representation of a nebula cert
 func (nc *NebulaCertificate) String() string {
 	if nc == nil {
@@ -431,6 +489,112 @@ func (nc *NebulaCertificate) MarshalJSON() ([]byte, error) {
 	return json.Marshal(jc)
 }

+//func (nc *NebulaCertificate) Copy() *NebulaCertificate {
+//	r, err := nc.Marshal()
+//	if err != nil {
+//		//TODO
+//		return nil
+//	}
+//
+//	c, err := UnmarshalNebulaCertificate(r)
+//	return c
+//}
+
+func (nc *NebulaCertificate) Copy() *NebulaCertificate {
+	c := &NebulaCertificate{
+		Details: NebulaCertificateDetails{
+			Name:           nc.Details.Name,
+			Groups:         make([]string, len(nc.Details.Groups)),
+			Ips:            make([]*net.IPNet, len(nc.Details.Ips)),
+			Subnets:        make([]*net.IPNet, len(nc.Details.Subnets)),
+			NotBefore:      nc.Details.NotBefore,
+			NotAfter:       nc.Details.NotAfter,
+			PublicKey:      make([]byte, len(nc.Details.PublicKey)),
+			IsCA:           nc.Details.IsCA,
+			Issuer:         nc.Details.Issuer,
+			InvertedGroups: make(map[string]struct{}, len(nc.Details.InvertedGroups)),
+		},
+		Signature: make([]byte, len(nc.Signature)),
+	}
+
+	copy(c.Signature, nc.Signature)
+	copy(c.Details.Groups, nc.Details.Groups)
+	copy(c.Details.PublicKey, nc.Details.PublicKey)
+
+	for i, p := range nc.Details.Ips {
+		c.Details.Ips[i] = &net.IPNet{
+			IP:   make(net.IP, len(p.IP)),
+			Mask: make(net.IPMask, len(p.Mask)),
+		}
+		copy(c.Details.Ips[i].IP, p.IP)
+		copy(c.Details.Ips[i].Mask, p.Mask)
+	}
+
+	for i, p := range nc.Details.Subnets {
+		c.Details.Subnets[i] = &net.IPNet{
+			IP:   make(net.IP, len(p.IP)),
+			Mask: make(net.IPMask, len(p.Mask)),
+		}
+		copy(c.Details.Subnets[i].IP, p.IP)
+		copy(c.Details.Subnets[i].Mask, p.Mask)
+	}
+
+	for g := range nc.Details.InvertedGroups {
+		c.Details.InvertedGroups[g] = struct{}{}
+	}
+
+	return c
+}
+
+func netMatch(certIp *net.IPNet, rootIps []*net.IPNet) bool {
+	for _, net := range rootIps {
+		if net.Contains(certIp.IP) && maskContains(net.Mask, certIp.Mask) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func maskContains(caMask, certMask net.IPMask) bool {
+	caM := maskTo4(caMask)
+	cM := maskTo4(certMask)
+	// Make sure forcing to ipv4 didn't nuke us
+	if caM == nil || cM == nil {
+		return false
+	}
+
+	// Make sure the cert mask is not greater than the ca mask
+	for i := 0; i < len(caMask); i++ {
+		if caM[i] > cM[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+func maskTo4(ip net.IPMask) net.IPMask {
+	if len(ip) == net.IPv4len {
+		return ip
+	}
+
+	if len(ip) == net.IPv6len && isZeros(ip[0:10]) && ip[10] == 0xff && ip[11] == 0xff {
+		return ip[12:16]
+	}
+
+	return nil
+}
+
+func isZeros(b []byte) bool {
+	for i := 0; i < len(b); i++ {
+		if b[i] != 0 {
+			return false
+		}
+	}
+	return true
+}
+
 func ip2int(ip []byte) uint32 {
 	if len(ip) == 16 {
 		return binary.BigEndian.Uint32(ip[12:16])
--- a/cert/cert.pb.go
+++ b/cert/cert.pb.go
@@ -1,202 +1,298 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.26.0
+// 	protoc        v3.14.0
 // source: cert.proto

 package cert

 import (
-	fmt "fmt"
-	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )

-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)

 type RawNebulaCertificate struct {
-	Details              *RawNebulaCertificateDetails `protobuf:"bytes,1,opt,name=Details,json=details,proto3" json:"Details,omitempty"`
-	Signature            []byte                       `protobuf:"bytes,2,opt,name=Signature,json=signature,proto3" json:"Signature,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}                     `json:"-"`
-	XXX_unrecognized     []byte                       `json:"-"`
-	XXX_sizecache        int32                        `json:"-"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Details   *RawNebulaCertificateDetails `protobuf:"bytes,1,opt,name=Details,proto3" json:"Details,omitempty"`
+	Signature []byte                       `protobuf:"bytes,2,opt,name=Signature,proto3" json:"Signature,omitempty"`
+}
+
+func (x *RawNebulaCertificate) Reset() {
+	*x = RawNebulaCertificate{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_cert_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RawNebulaCertificate) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }

-func (m *RawNebulaCertificate) Reset()         { *m = RawNebulaCertificate{} }
-func (m *RawNebulaCertificate) String() string { return proto.CompactTextString(m) }
 func (*RawNebulaCertificate) ProtoMessage() {}
+
+func (x *RawNebulaCertificate) ProtoReflect() protoreflect.Message {
+	mi := &file_cert_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RawNebulaCertificate.ProtoReflect.Descriptor instead.
 func (*RawNebulaCertificate) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a142e29cbef9b1cf, []int{0}
+	return file_cert_proto_rawDescGZIP(), []int{0}
 }

-func (m *RawNebulaCertificate) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_RawNebulaCertificate.Unmarshal(m, b)
-}
-func (m *RawNebulaCertificate) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_RawNebulaCertificate.Marshal(b, m, deterministic)
-}
-func (m *RawNebulaCertificate) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_RawNebulaCertificate.Merge(m, src)
-}
-func (m *RawNebulaCertificate) XXX_Size() int {
-	return xxx_messageInfo_RawNebulaCertificate.Size(m)
-}
-func (m *RawNebulaCertificate) XXX_DiscardUnknown() {
-	xxx_messageInfo_RawNebulaCertificate.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_RawNebulaCertificate proto.InternalMessageInfo
-
-func (m *RawNebulaCertificate) GetDetails() *RawNebulaCertificateDetails {
-	if m != nil {
-		return m.Details
+func (x *RawNebulaCertificate) GetDetails() *RawNebulaCertificateDetails {
+	if x != nil {
+		return x.Details
 	}
 	return nil
 }

-func (m *RawNebulaCertificate) GetSignature() []byte {
-	if m != nil {
-		return m.Signature
+func (x *RawNebulaCertificate) GetSignature() []byte {
+	if x != nil {
+		return x.Signature
 	}
 	return nil
 }

 type RawNebulaCertificateDetails struct {
-	Name string `protobuf:"bytes,1,opt,name=Name,json=name,proto3" json:"Name,omitempty"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
 	// Ips and Subnets are in big endian 32 bit pairs, 1st the ip, 2nd the mask
-	Ips       []uint32 `protobuf:"varint,2,rep,packed,name=Ips,json=ips,proto3" json:"Ips,omitempty"`
-	Subnets   []uint32 `protobuf:"varint,3,rep,packed,name=Subnets,json=subnets,proto3" json:"Subnets,omitempty"`
-	Groups    []string `protobuf:"bytes,4,rep,name=Groups,json=groups,proto3" json:"Groups,omitempty"`
-	NotBefore int64    `protobuf:"varint,5,opt,name=NotBefore,json=notBefore,proto3" json:"NotBefore,omitempty"`
-	NotAfter  int64    `protobuf:"varint,6,opt,name=NotAfter,json=notAfter,proto3" json:"NotAfter,omitempty"`
-	PublicKey []byte   `protobuf:"bytes,7,opt,name=PublicKey,json=publicKey,proto3" json:"PublicKey,omitempty"`
-	IsCA      bool     `protobuf:"varint,8,opt,name=IsCA,json=isCA,proto3" json:"IsCA,omitempty"`
+	Ips       []uint32 `protobuf:"varint,2,rep,packed,name=Ips,proto3" json:"Ips,omitempty"`
+	Subnets   []uint32 `protobuf:"varint,3,rep,packed,name=Subnets,proto3" json:"Subnets,omitempty"`
+	Groups    []string `protobuf:"bytes,4,rep,name=Groups,proto3" json:"Groups,omitempty"`
+	NotBefore int64    `protobuf:"varint,5,opt,name=NotBefore,proto3" json:"NotBefore,omitempty"`
+	NotAfter  int64    `protobuf:"varint,6,opt,name=NotAfter,proto3" json:"NotAfter,omitempty"`
+	PublicKey []byte   `protobuf:"bytes,7,opt,name=PublicKey,proto3" json:"PublicKey,omitempty"`
+	IsCA      bool     `protobuf:"varint,8,opt,name=IsCA,proto3" json:"IsCA,omitempty"`
 	// sha-256 of the issuer certificate, if this field is blank the cert is self-signed
-	Issuer               []byte   `protobuf:"bytes,9,opt,name=Issuer,json=issuer,proto3" json:"Issuer,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Issuer []byte `protobuf:"bytes,9,opt,name=Issuer,proto3" json:"Issuer,omitempty"`
+}
+
+func (x *RawNebulaCertificateDetails) Reset() {
+	*x = RawNebulaCertificateDetails{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_cert_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RawNebulaCertificateDetails) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }

-func (m *RawNebulaCertificateDetails) Reset()         { *m = RawNebulaCertificateDetails{} }
-func (m *RawNebulaCertificateDetails) String() string { return proto.CompactTextString(m) }
 func (*RawNebulaCertificateDetails) ProtoMessage() {}
+
+func (x *RawNebulaCertificateDetails) ProtoReflect() protoreflect.Message {
+	mi := &file_cert_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RawNebulaCertificateDetails.ProtoReflect.Descriptor instead.
 func (*RawNebulaCertificateDetails) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a142e29cbef9b1cf, []int{1}
+	return file_cert_proto_rawDescGZIP(), []int{1}
 }

-func (m *RawNebulaCertificateDetails) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_RawNebulaCertificateDetails.Unmarshal(m, b)
-}
-func (m *RawNebulaCertificateDetails) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_RawNebulaCertificateDetails.Marshal(b, m, deterministic)
-}
-func (m *RawNebulaCertificateDetails) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_RawNebulaCertificateDetails.Merge(m, src)
-}
-func (m *RawNebulaCertificateDetails) XXX_Size() int {
-	return xxx_messageInfo_RawNebulaCertificateDetails.Size(m)
-}
-func (m *RawNebulaCertificateDetails) XXX_DiscardUnknown() {
-	xxx_messageInfo_RawNebulaCertificateDetails.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_RawNebulaCertificateDetails proto.InternalMessageInfo
-
-func (m *RawNebulaCertificateDetails) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *RawNebulaCertificateDetails) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }

-func (m *RawNebulaCertificateDetails) GetIps() []uint32 {
-	if m != nil {
-		return m.Ips
+func (x *RawNebulaCertificateDetails) GetIps() []uint32 {
+	if x != nil {
+		return x.Ips
 	}
 	return nil
 }

-func (m *RawNebulaCertificateDetails) GetSubnets() []uint32 {
-	if m != nil {
-		return m.Subnets
+func (x *RawNebulaCertificateDetails) GetSubnets() []uint32 {
+	if x != nil {
+		return x.Subnets
 	}
 	return nil
 }

-func (m *RawNebulaCertificateDetails) GetGroups() []string {
-	if m != nil {
-		return m.Groups
+func (x *RawNebulaCertificateDetails) GetGroups() []string {
+	if x != nil {
+		return x.Groups
 	}
 	return nil
 }

-func (m *RawNebulaCertificateDetails) GetNotBefore() int64 {
-	if m != nil {
-		return m.NotBefore
+func (x *RawNebulaCertificateDetails) GetNotBefore() int64 {
+	if x != nil {
+		return x.NotBefore
 	}
 	return 0
 }

-func (m *RawNebulaCertificateDetails) GetNotAfter() int64 {
-	if m != nil {
-		return m.NotAfter
+func (x *RawNebulaCertificateDetails) GetNotAfter() int64 {
+	if x != nil {
+		return x.NotAfter
 	}
 	return 0
 }

-func (m *RawNebulaCertificateDetails) GetPublicKey() []byte {
-	if m != nil {
-		return m.PublicKey
+func (x *RawNebulaCertificateDetails) GetPublicKey() []byte {
+	if x != nil {
+		return x.PublicKey
 	}
 	return nil
 }

-func (m *RawNebulaCertificateDetails) GetIsCA() bool {
-	if m != nil {
-		return m.IsCA
+func (x *RawNebulaCertificateDetails) GetIsCA() bool {
+	if x != nil {
+		return x.IsCA
 	}
 	return false
 }

-func (m *RawNebulaCertificateDetails) GetIssuer() []byte {
-	if m != nil {
-		return m.Issuer
+func (x *RawNebulaCertificateDetails) GetIssuer() []byte {
+	if x != nil {
+		return x.Issuer
 	}
 	return nil
 }

-func init() {
-	proto.RegisterType((*RawNebulaCertificate)(nil), "cert.RawNebulaCertificate")
-	proto.RegisterType((*RawNebulaCertificateDetails)(nil), "cert.RawNebulaCertificateDetails")
+var File_cert_proto protoreflect.FileDescriptor
+
+var file_cert_proto_rawDesc = []byte{
+	0x0a, 0x0a, 0x63, 0x65, 0x72, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x63, 0x65,
+	0x72, 0x74, 0x22, 0x71, 0x0a, 0x14, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x43,
+	0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x3b, 0x0a, 0x07, 0x44, 0x65,
+	0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x65,
+	0x72, 0x74, 0x2e, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x43, 0x65, 0x72, 0x74,
+	0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x07,
+	0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61,
+	0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x53, 0x69, 0x67, 0x6e,
+	0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0xf9, 0x01, 0x0a, 0x1b, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62,
+	0x75, 0x6c, 0x61, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x44, 0x65,
+	0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x49, 0x70, 0x73,
+	0x18, 0x02, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x03, 0x49, 0x70, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x53,
+	0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x53, 0x75,
+	0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18,
+	0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x1c, 0x0a,
+	0x09, 0x4e, 0x6f, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03,
+	0x52, 0x09, 0x4e, 0x6f, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x4e,
+	0x6f, 0x74, 0x41, 0x66, 0x74, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x4e,
+	0x6f, 0x74, 0x41, 0x66, 0x74, 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x75, 0x62, 0x6c, 0x69,
+	0x63, 0x4b, 0x65, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x50, 0x75, 0x62, 0x6c,
+	0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x73, 0x43, 0x41, 0x18, 0x08, 0x20,
+	0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x73, 0x43, 0x41, 0x12, 0x16, 0x0a, 0x06, 0x49, 0x73, 0x73,
+	0x75, 0x65, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x49, 0x73, 0x73, 0x75, 0x65,
+	0x72, 0x42, 0x20, 0x5a, 0x1e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
+	0x73, 0x6c, 0x61, 0x63, 0x6b, 0x68, 0x71, 0x2f, 0x6e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x2f, 0x63,
+	0x65, 0x72, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }

-func init() { proto.RegisterFile("cert.proto", fileDescriptor_a142e29cbef9b1cf) }
+var (
+	file_cert_proto_rawDescOnce sync.Once
+	file_cert_proto_rawDescData = file_cert_proto_rawDesc
+)

-var fileDescriptor_a142e29cbef9b1cf = []byte{
-	// 279 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x90, 0xcf, 0x4a, 0xf4, 0x30,
-	0x14, 0xc5, 0xc9, 0xa4, 0x5f, 0xdb, 0xe4, 0x53, 0x90, 0x20, 0x12, 0xd4, 0x45, 0x9c, 0x55, 0x56,
-	0xb3, 0xd0, 0xa5, 0xab, 0x71, 0x04, 0x29, 0x42, 0x91, 0xcc, 0x13, 0xa4, 0xf5, 0x76, 0x08, 0x74,
-	0x9a, 0x9a, 0x3f, 0x88, 0x8f, 0xee, 0x4e, 0x9a, 0x4e, 0x77, 0xe2, 0xee, 0x9e, 0x5f, 0xce, 0x49,
-	0x4e, 0x2e, 0xa5, 0x2d, 0xb8, 0xb0, 0x19, 0x9d, 0x0d, 0x96, 0x65, 0xd3, 0xbc, 0xfe, 0xa0, 0x97,
-	0x4a, 0x7f, 0xd6, 0xd0, 0xc4, 0x5e, 0xef, 0xc0, 0x05, 0xd3, 0x99, 0x56, 0x07, 0x60, 0x8f, 0xb4,
-	0x78, 0x86, 0xa0, 0x4d, 0xef, 0x39, 0x12, 0x48, 0xfe, 0xbf, 0xbf, 0xdb, 0xa4, 0xec, 0x6f, 0xe6,
-	0x93, 0x51, 0x15, 0xef, 0xf3, 0xc0, 0x6e, 0x29, 0xd9, 0x9b, 0xc3, 0xa0, 0x43, 0x74, 0xc0, 0x57,
-	0x02, 0xc9, 0x33, 0x45, 0xfc, 0x02, 0xd6, 0xdf, 0x88, 0xde, 0xfc, 0x71, 0x0d, 0x63, 0x34, 0xab,
-	0xf5, 0x11, 0xd2, 0xbb, 0x44, 0x65, 0x83, 0x3e, 0x02, 0xbb, 0xa0, 0xb8, 0x1a, 0x3d, 0x5f, 0x09,
-	0x2c, 0xcf, 0x15, 0x36, 0xa3, 0x67, 0x9c, 0x16, 0xfb, 0xd8, 0x0c, 0x10, 0x3c, 0xc7, 0x89, 0x16,
-	0x7e, 0x96, 0xec, 0x8a, 0xe6, 0x2f, 0xce, 0xc6, 0xd1, 0xf3, 0x4c, 0x60, 0x49, 0x54, 0x7e, 0x48,
-	0x6a, 0x6a, 0x55, 0xdb, 0xf0, 0x04, 0x9d, 0x75, 0xc0, 0xff, 0x09, 0x24, 0xb1, 0x22, 0xc3, 0x02,
-	0xd8, 0x35, 0x2d, 0x6b, 0x1b, 0xb6, 0x5d, 0x00, 0xc7, 0xf3, 0x74, 0x58, 0x0e, 0x27, 0x3d, 0x25,
-	0xdf, 0x62, 0xd3, 0x9b, 0xf6, 0x15, 0xbe, 0x78, 0x31, 0xff, 0x67, 0x5c, 0xc0, 0xd4, 0xb7, 0xf2,
-	0xbb, 0x2d, 0x2f, 0x05, 0x92, 0xa5, 0xca, 0x8c, 0xdf, 0x6d, 0xa7, 0x0e, 0x95, 0xf7, 0x11, 0x1c,
-	0x27, 0xc9, 0x9e, 0x9b, 0xa4, 0x9a, 0x3c, 0xed, 0xfe, 0xe1, 0x27, 0x00, 0x00, 0xff, 0xff, 0x2c,
-	0xe3, 0x08, 0x37, 0x89, 0x01, 0x00, 0x00,
+func file_cert_proto_rawDescGZIP() []byte {
+	file_cert_proto_rawDescOnce.Do(func() {
+		file_cert_proto_rawDescData = protoimpl.X.CompressGZIP(file_cert_proto_rawDescData)
+	})
+	return file_cert_proto_rawDescData
+}
+
+var file_cert_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_cert_proto_goTypes = []interface{}{
+	(*RawNebulaCertificate)(nil),        // 0: cert.RawNebulaCertificate
+	(*RawNebulaCertificateDetails)(nil), // 1: cert.RawNebulaCertificateDetails
+}
+var file_cert_proto_depIdxs = []int32{
+	1, // 0: cert.RawNebulaCertificate.Details:type_name -> cert.RawNebulaCertificateDetails
+	1, // [1:1] is the sub-list for method output_type
+	1, // [1:1] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
+}
+
+func init() { file_cert_proto_init() }
+func file_cert_proto_init() {
+	if File_cert_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_cert_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RawNebulaCertificate); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_cert_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RawNebulaCertificateDetails); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_cert_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_cert_proto_goTypes,
+		DependencyIndexes: file_cert_proto_depIdxs,
+		MessageInfos:      file_cert_proto_msgTypes,
+	}.Build()
+	File_cert_proto = out.File
+	file_cert_proto_rawDesc = nil
+	file_cert_proto_goTypes = nil
+	file_cert_proto_depIdxs = nil
 }
--- a/cert/cert.proto
+++ b/cert/cert.proto
@@ -1,6 +1,8 @@
 syntax = "proto3";
 package cert;

+option go_package = "github.com/slackhq/nebula/cert";
+
 //import "google/protobuf/timestamp.proto";

 message RawNebulaCertificate {
--- a/cert/cert_test.go
+++ b/cert/cert_test.go
@@ -9,6 +9,7 @@ import (
 	"time"

 	"github.com/golang/protobuf/proto"
+	"github.com/slackhq/nebula/util"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/crypto/curve25519"
 	"golang.org/x/crypto/ed25519"
@@ -44,7 +45,7 @@ func TestMarshalingNebulaCertificate(t *testing.T) {

 	b, err := nc.Marshal()
 	assert.Nil(t, err)
-	t.Log("Cert size:", len(b))
+	//t.Log("Cert size:", len(b))

 	nc2, err := UnmarshalNebulaCertificate(b)
 	assert.Nil(t, err)
@@ -103,9 +104,9 @@ func TestNebulaCertificate_Sign(t *testing.T) {
 	assert.Nil(t, nc.Sign(priv))
 	assert.True(t, nc.CheckSignature(pub))

-	b, err := nc.Marshal()
+	_, err = nc.Marshal()
 	assert.Nil(t, err)
-	t.Log("Cert size:", len(b))
+	//t.Log("Cert size:", len(b))
 }

 func TestNebulaCertificate_Expired(t *testing.T) {
@@ -158,10 +159,10 @@ func TestNebulaCertificate_MarshalJSON(t *testing.T) {
 }

 func TestNebulaCertificate_Verify(t *testing.T) {
-	ca, _, caKey, err := newTestCaCert()
+	ca, _, caKey, err := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
 	assert.Nil(t, err)

-	c, _, _, err := newTestCert(ca, caKey)
+	c, _, _, err := newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
 	assert.Nil(t, err)

 	h, err := ca.Sha256Sum()
@@ -172,13 +173,13 @@ func TestNebulaCertificate_Verify(t *testing.T) {

 	f, err := c.Sha256Sum()
 	assert.Nil(t, err)
-	caPool.BlacklistFingerprint(f)
+	caPool.BlocklistFingerprint(f)

 	v, err := c.Verify(time.Now(), caPool)
 	assert.False(t, v)
-	assert.EqualError(t, err, "certificate has been blacklisted")
+	assert.EqualError(t, err, "certificate has been blocked")

-	caPool.ResetCertBlacklist()
+	caPool.ResetCertBlocklist()
 	v, err = c.Verify(time.Now(), caPool)
 	assert.True(t, v)
 	assert.Nil(t, err)
@@ -186,13 +187,206 @@ func TestNebulaCertificate_Verify(t *testing.T) {
 	v, err = c.Verify(time.Now().Add(time.Hour*1000), caPool)
 	assert.False(t, v)
 	assert.EqualError(t, err, "root certificate is expired")
-}

-func TestNebulaVerifyPrivateKey(t *testing.T) {
-	ca, _, caKey, err := newTestCaCert()
+	c, _, _, err = newTestCert(ca, caKey, time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now().Add(time.Minute*6), caPool)
+	assert.False(t, v)
+	assert.EqualError(t, err, "certificate is expired")
+
+	// Test group assertion
+	ca, _, caKey, err = newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{"test1", "test2"})
 	assert.Nil(t, err)

-	c, _, priv, err := newTestCert(ca, caKey)
+	caPem, err := ca.MarshalToPEM()
+	assert.Nil(t, err)
+
+	caPool = NewCAPool()
+	caPool.AddCACertificate(caPem)
+
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{"test1", "bad"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.False(t, v)
+	assert.EqualError(t, err, "certificate contained a group not present on the signing ca: bad")
+
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{"test1"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.True(t, v)
+	assert.Nil(t, err)
+}
+
+func TestNebulaCertificate_Verify_IPs(t *testing.T) {
+	_, caIp1, _ := net.ParseCIDR("10.0.0.0/16")
+	_, caIp2, _ := net.ParseCIDR("192.168.0.0/24")
+	ca, _, caKey, err := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{caIp1, caIp2}, []*net.IPNet{}, []string{"test"})
+	assert.Nil(t, err)
+
+	caPem, err := ca.MarshalToPEM()
+	assert.Nil(t, err)
+
+	caPool := NewCAPool()
+	caPool.AddCACertificate(caPem)
+
+	// ip is outside the network
+	cIp1 := &net.IPNet{IP: net.ParseIP("10.1.0.0"), Mask: []byte{255, 255, 255, 0}}
+	cIp2 := &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 0, 0}}
+	c, _, _, err := newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{cIp1, cIp2}, []*net.IPNet{}, []string{"test"})
+	assert.Nil(t, err)
+	v, err := c.Verify(time.Now(), caPool)
+	assert.False(t, v)
+	assert.EqualError(t, err, "certificate contained an ip assignment outside the limitations of the signing ca: 10.1.0.0/24")
+
+	// ip is outside the network reversed order of above
+	cIp1 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}}
+	cIp2 = &net.IPNet{IP: net.ParseIP("10.1.0.0"), Mask: []byte{255, 255, 255, 0}}
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{cIp1, cIp2}, []*net.IPNet{}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.False(t, v)
+	assert.EqualError(t, err, "certificate contained an ip assignment outside the limitations of the signing ca: 10.1.0.0/24")
+
+	// ip is within the network but mask is outside
+	cIp1 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 254, 0, 0}}
+	cIp2 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}}
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{cIp1, cIp2}, []*net.IPNet{}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.False(t, v)
+	assert.EqualError(t, err, "certificate contained an ip assignment outside the limitations of the signing ca: 10.0.1.0/15")
+
+	// ip is within the network but mask is outside reversed order of above
+	cIp1 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}}
+	cIp2 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 254, 0, 0}}
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{cIp1, cIp2}, []*net.IPNet{}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.False(t, v)
+	assert.EqualError(t, err, "certificate contained an ip assignment outside the limitations of the signing ca: 10.0.1.0/15")
+
+	// ip and mask are within the network
+	cIp1 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 255, 0, 0}}
+	cIp2 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 128}}
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{cIp1, cIp2}, []*net.IPNet{}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.True(t, v)
+	assert.Nil(t, err)
+
+	// Exact matches
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{caIp1, caIp2}, []*net.IPNet{}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.True(t, v)
+	assert.Nil(t, err)
+
+	// Exact matches reversed
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{caIp2, caIp1}, []*net.IPNet{}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.True(t, v)
+	assert.Nil(t, err)
+
+	// Exact matches reversed with just 1
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{caIp1}, []*net.IPNet{}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.True(t, v)
+	assert.Nil(t, err)
+}
+
+func TestNebulaCertificate_Verify_Subnets(t *testing.T) {
+	_, caIp1, _ := net.ParseCIDR("10.0.0.0/16")
+	_, caIp2, _ := net.ParseCIDR("192.168.0.0/24")
+	ca, _, caKey, err := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{caIp1, caIp2}, []string{"test"})
+	assert.Nil(t, err)
+
+	caPem, err := ca.MarshalToPEM()
+	assert.Nil(t, err)
+
+	caPool := NewCAPool()
+	caPool.AddCACertificate(caPem)
+
+	// ip is outside the network
+	cIp1 := &net.IPNet{IP: net.ParseIP("10.1.0.0"), Mask: []byte{255, 255, 255, 0}}
+	cIp2 := &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 0, 0}}
+	c, _, _, err := newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{cIp1, cIp2}, []string{"test"})
+	assert.Nil(t, err)
+	v, err := c.Verify(time.Now(), caPool)
+	assert.False(t, v)
+	assert.EqualError(t, err, "certificate contained a subnet assignment outside the limitations of the signing ca: 10.1.0.0/24")
+
+	// ip is outside the network reversed order of above
+	cIp1 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}}
+	cIp2 = &net.IPNet{IP: net.ParseIP("10.1.0.0"), Mask: []byte{255, 255, 255, 0}}
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{cIp1, cIp2}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.False(t, v)
+	assert.EqualError(t, err, "certificate contained a subnet assignment outside the limitations of the signing ca: 10.1.0.0/24")
+
+	// ip is within the network but mask is outside
+	cIp1 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 254, 0, 0}}
+	cIp2 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}}
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{cIp1, cIp2}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.False(t, v)
+	assert.EqualError(t, err, "certificate contained a subnet assignment outside the limitations of the signing ca: 10.0.1.0/15")
+
+	// ip is within the network but mask is outside reversed order of above
+	cIp1 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}}
+	cIp2 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 254, 0, 0}}
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{cIp1, cIp2}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.False(t, v)
+	assert.EqualError(t, err, "certificate contained a subnet assignment outside the limitations of the signing ca: 10.0.1.0/15")
+
+	// ip and mask are within the network
+	cIp1 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 255, 0, 0}}
+	cIp2 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 128}}
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{cIp1, cIp2}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.True(t, v)
+	assert.Nil(t, err)
+
+	// Exact matches
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{caIp1, caIp2}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.True(t, v)
+	assert.Nil(t, err)
+
+	// Exact matches reversed
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{caIp2, caIp1}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.True(t, v)
+	assert.Nil(t, err)
+
+	// Exact matches reversed with just 1
+	c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{caIp1}, []string{"test"})
+	assert.Nil(t, err)
+	v, err = c.Verify(time.Now(), caPool)
+	assert.True(t, v)
+	assert.Nil(t, err)
+}
+
+func TestNebulaCertificate_VerifyPrivateKey(t *testing.T) {
+	ca, _, caKey, err := newTestCaCert(time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{})
+	assert.Nil(t, err)
+	err = ca.VerifyPrivateKey(caKey)
+	assert.Nil(t, err)
+
+	_, _, caKey2, err := newTestCaCert(time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{})
+	assert.Nil(t, err)
+	err = ca.VerifyPrivateKey(caKey2)
+	assert.NotNil(t, err)
+
+	c, _, priv, err := newTestCert(ca, caKey, time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{})
 	err = c.VerifyPrivateKey(priv)
 	assert.Nil(t, err)

@@ -260,6 +454,255 @@ BVG+oJpAoqokUBbI4U0N8CSfpUABEkB/Pm5A2xyH/nc8mg/wvGUWG3pZ7nHzaDMf
 	assert.Equal(t, pp.CAs[string("5c9c3f23e7ee7fe97637cbd3a0a5b854154d1d9aaaf7b566a51f4a88f76b64cd")].Details.Name, rootCA01.Details.Name)
 }

+func appendByteSlices(b ...[]byte) []byte {
+	retSlice := []byte{}
+	for _, v := range b {
+		retSlice = append(retSlice, v...)
+	}
+	return retSlice
+}
+
+func TestUnmrshalCertPEM(t *testing.T) {
+	goodCert := []byte(`
+# A good cert
+-----BEGIN NEBULA CERTIFICATE-----
+CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL
+vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv
+bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB
+-----END NEBULA CERTIFICATE-----
+`)
+	badBanner := []byte(`# A bad banner
+-----BEGIN NOT A NEBULA CERTIFICATE-----
+CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL
+vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv
+bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB
+-----END NOT A NEBULA CERTIFICATE-----
+`)
+	invalidPem := []byte(`# Not a valid PEM format
+-BEGIN NEBULA CERTIFICATE-----
+CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL
+vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv
+bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB
+-END NEBULA CERTIFICATE----`)
+
+	certBundle := appendByteSlices(goodCert, badBanner, invalidPem)
+
+	// Success test case
+	cert, rest, err := UnmarshalNebulaCertificateFromPEM(certBundle)
+	assert.NotNil(t, cert)
+	assert.Equal(t, rest, append(badBanner, invalidPem...))
+	assert.Nil(t, err)
+
+	// Fail due to invalid banner.
+	cert, rest, err = UnmarshalNebulaCertificateFromPEM(rest)
+	assert.Nil(t, cert)
+	assert.Equal(t, rest, invalidPem)
+	assert.EqualError(t, err, "bytes did not contain a proper nebula certificate banner")
+
+	// Fail due to ivalid PEM format, because
+	// it's missing the requisite pre-encapsulation boundary.
+	cert, rest, err = UnmarshalNebulaCertificateFromPEM(rest)
+	assert.Nil(t, cert)
+	assert.Equal(t, rest, invalidPem)
+	assert.EqualError(t, err, "input did not contain a valid PEM encoded block")
+}
+
+func TestUnmarshalEd25519PrivateKey(t *testing.T) {
+	privKey := []byte(`# A good key
+-----BEGIN NEBULA ED25519 PRIVATE KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
+-----END NEBULA ED25519 PRIVATE KEY-----
+`)
+	shortKey := []byte(`# A short key
+-----BEGIN NEBULA ED25519 PRIVATE KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+-----END NEBULA ED25519 PRIVATE KEY-----
+`)
+	invalidBanner := []byte(`# Invalid banner
+-----BEGIN NOT A NEBULA PRIVATE KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
+-----END NOT A NEBULA PRIVATE KEY-----
+`)
+	invalidPem := []byte(`# Not a valid PEM format
+-BEGIN NEBULA ED25519 PRIVATE KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
+-END NEBULA ED25519 PRIVATE KEY-----`)
+
+	keyBundle := appendByteSlices(privKey, shortKey, invalidBanner, invalidPem)
+
+	// Success test case
+	k, rest, err := UnmarshalEd25519PrivateKey(keyBundle)
+	assert.Len(t, k, 64)
+	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
+	assert.Nil(t, err)
+
+	// Fail due to short key
+	k, rest, err = UnmarshalEd25519PrivateKey(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem))
+	assert.EqualError(t, err, "key was not 64 bytes, is invalid ed25519 private key")
+
+	// Fail due to invalid banner
+	k, rest, err = UnmarshalEd25519PrivateKey(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, rest, invalidPem)
+	assert.EqualError(t, err, "bytes did not contain a proper nebula Ed25519 private key banner")
+
+	// Fail due to ivalid PEM format, because
+	// it's missing the requisite pre-encapsulation boundary.
+	k, rest, err = UnmarshalEd25519PrivateKey(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, rest, invalidPem)
+	assert.EqualError(t, err, "input did not contain a valid PEM encoded block")
+}
+
+func TestUnmarshalX25519PrivateKey(t *testing.T) {
+	privKey := []byte(`# A good key
+-----BEGIN NEBULA X25519 PRIVATE KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-----END NEBULA X25519 PRIVATE KEY-----
+`)
+	shortKey := []byte(`# A short key
+-----BEGIN NEBULA X25519 PRIVATE KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
+-----END NEBULA X25519 PRIVATE KEY-----
+`)
+	invalidBanner := []byte(`# Invalid banner
+-----BEGIN NOT A NEBULA PRIVATE KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-----END NOT A NEBULA PRIVATE KEY-----
+`)
+	invalidPem := []byte(`# Not a valid PEM format
+-BEGIN NEBULA X25519 PRIVATE KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-END NEBULA X25519 PRIVATE KEY-----`)
+
+	keyBundle := appendByteSlices(privKey, shortKey, invalidBanner, invalidPem)
+
+	// Success test case
+	k, rest, err := UnmarshalX25519PrivateKey(keyBundle)
+	assert.Len(t, k, 32)
+	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
+	assert.Nil(t, err)
+
+	// Fail due to short key
+	k, rest, err = UnmarshalX25519PrivateKey(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem))
+	assert.EqualError(t, err, "key was not 32 bytes, is invalid X25519 private key")
+
+	// Fail due to invalid banner
+	k, rest, err = UnmarshalX25519PrivateKey(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, rest, invalidPem)
+	assert.EqualError(t, err, "bytes did not contain a proper nebula X25519 private key banner")
+
+	// Fail due to ivalid PEM format, because
+	// it's missing the requisite pre-encapsulation boundary.
+	k, rest, err = UnmarshalX25519PrivateKey(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, rest, invalidPem)
+	assert.EqualError(t, err, "input did not contain a valid PEM encoded block")
+}
+
+func TestUnmarshalEd25519PublicKey(t *testing.T) {
+	pubKey := []byte(`# A good key
+-----BEGIN NEBULA ED25519 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-----END NEBULA ED25519 PUBLIC KEY-----
+`)
+	shortKey := []byte(`# A short key
+-----BEGIN NEBULA ED25519 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
+-----END NEBULA ED25519 PUBLIC KEY-----
+`)
+	invalidBanner := []byte(`# Invalid banner
+-----BEGIN NOT A NEBULA PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-----END NOT A NEBULA PUBLIC KEY-----
+`)
+	invalidPem := []byte(`# Not a valid PEM format
+-BEGIN NEBULA ED25519 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-END NEBULA ED25519 PUBLIC KEY-----`)
+
+	keyBundle := appendByteSlices(pubKey, shortKey, invalidBanner, invalidPem)
+
+	// Success test case
+	k, rest, err := UnmarshalEd25519PublicKey(keyBundle)
+	assert.Equal(t, len(k), 32)
+	assert.Nil(t, err)
+	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
+
+	// Fail due to short key
+	k, rest, err = UnmarshalEd25519PublicKey(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem))
+	assert.EqualError(t, err, "key was not 32 bytes, is invalid ed25519 public key")
+
+	// Fail due to invalid banner
+	k, rest, err = UnmarshalEd25519PublicKey(rest)
+	assert.Nil(t, k)
+	assert.EqualError(t, err, "bytes did not contain a proper nebula Ed25519 public key banner")
+	assert.Equal(t, rest, invalidPem)
+
+	// Fail due to ivalid PEM format, because
+	// it's missing the requisite pre-encapsulation boundary.
+	k, rest, err = UnmarshalEd25519PublicKey(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, rest, invalidPem)
+	assert.EqualError(t, err, "input did not contain a valid PEM encoded block")
+}
+
+func TestUnmarshalX25519PublicKey(t *testing.T) {
+	pubKey := []byte(`# A good key
+-----BEGIN NEBULA X25519 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-----END NEBULA X25519 PUBLIC KEY-----
+`)
+	shortKey := []byte(`# A short key
+-----BEGIN NEBULA X25519 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
+-----END NEBULA X25519 PUBLIC KEY-----
+`)
+	invalidBanner := []byte(`# Invalid banner
+-----BEGIN NOT A NEBULA PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-----END NOT A NEBULA PUBLIC KEY-----
+`)
+	invalidPem := []byte(`# Not a valid PEM format
+-BEGIN NEBULA X25519 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-END NEBULA X25519 PUBLIC KEY-----`)
+
+	keyBundle := appendByteSlices(pubKey, shortKey, invalidBanner, invalidPem)
+
+	// Success test case
+	k, rest, err := UnmarshalX25519PublicKey(keyBundle)
+	assert.Equal(t, len(k), 32)
+	assert.Nil(t, err)
+	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
+
+	// Fail due to short key
+	k, rest, err = UnmarshalX25519PublicKey(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem))
+	assert.EqualError(t, err, "key was not 32 bytes, is invalid X25519 public key")
+
+	// Fail due to invalid banner
+	k, rest, err = UnmarshalX25519PublicKey(rest)
+	assert.Nil(t, k)
+	assert.EqualError(t, err, "bytes did not contain a proper nebula X25519 public key banner")
+	assert.Equal(t, rest, invalidPem)
+
+	// Fail due to ivalid PEM format, because
+	// it's missing the requisite pre-encapsulation boundary.
+	k, rest, err = UnmarshalX25519PublicKey(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, rest, invalidPem)
+	assert.EqualError(t, err, "input did not contain a valid PEM encoded block")
+}
+
 // Ensure that upgrading the protobuf library does not change how certificates
 // are marshalled, since this would break signature verification
 func TestMarshalingNebulaCertificateConsistency(t *testing.T) {
@@ -292,30 +735,65 @@ func TestMarshalingNebulaCertificateConsistency(t *testing.T) {

 	b, err := nc.Marshal()
 	assert.Nil(t, err)
-	t.Log("Cert size:", len(b))
+	//t.Log("Cert size:", len(b))
 	assert.Equal(t, "0aa2010a0774657374696e67121b8182845080feffff0f828284508080fcff0f8382845080fe83f80f1a1b8182844880fe83f80f8282844880feffff0f838284488080fcff0f220b746573742d67726f757031220b746573742d67726f757032220b746573742d67726f75703328f0e0e7d70430a08681c4053a20313233343536373839306162636564666768696a3132333435363738393061624a081234567890abcedf1220313233343536373839306162636564666768696a313233343536373839306162", fmt.Sprintf("%x", b))

 	b, err = proto.Marshal(nc.getRawDetails())
 	assert.Nil(t, err)
-	t.Log("Raw cert size:", len(b))
+	//t.Log("Raw cert size:", len(b))
 	assert.Equal(t, "0a0774657374696e67121b8182845080feffff0f828284508080fcff0f8382845080fe83f80f1a1b8182844880fe83f80f8282844880feffff0f838284488080fcff0f220b746573742d67726f757031220b746573742d67726f757032220b746573742d67726f75703328f0e0e7d70430a08681c4053a20313233343536373839306162636564666768696a3132333435363738393061624a081234567890abcedf", fmt.Sprintf("%x", b))
 }

-func newTestCaCert() (*NebulaCertificate, []byte, []byte, error) {
+func TestNebulaCertificate_Copy(t *testing.T) {
+	ca, _, caKey, err := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
+	assert.Nil(t, err)
+
+	c, _, _, err := newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
+	assert.Nil(t, err)
+	cc := c.Copy()
+
+	util.AssertDeepCopyEqual(t, c, cc)
+}
+
+func TestUnmarshalNebulaCertificate(t *testing.T) {
+	// Test that we don't panic with an invalid certificate (#332)
+	data := []byte("\x98\x00\x00")
+	_, err := UnmarshalNebulaCertificate(data)
+	assert.EqualError(t, err, "encoded Details was nil")
+}
+
+func newTestCaCert(before, after time.Time, ips, subnets []*net.IPNet, groups []string) (*NebulaCertificate, []byte, []byte, error) {
 	pub, priv, err := ed25519.GenerateKey(rand.Reader)
-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
+	if before.IsZero() {
+		before = time.Now().Add(time.Second * -60).Round(time.Second)
+	}
+	if after.IsZero() {
+		after = time.Now().Add(time.Second * 60).Round(time.Second)
+	}

 	nc := &NebulaCertificate{
 		Details: NebulaCertificateDetails{
 			Name:           "test ca",
-			NotBefore: before,
-			NotAfter:  after,
+			NotBefore:      time.Unix(before.Unix(), 0),
+			NotAfter:       time.Unix(after.Unix(), 0),
 			PublicKey:      pub,
 			IsCA:           true,
+			InvertedGroups: make(map[string]struct{}),
 		},
 	}

+	if len(ips) > 0 {
+		nc.Details.Ips = ips
+	}
+
+	if len(subnets) > 0 {
+		nc.Details.Subnets = subnets
+	}
+
+	if len(groups) > 0 {
+		nc.Details.Groups = groups
+	}
+
 	err = nc.Sign(priv)
 	if err != nil {
 		return nil, nil, nil, err
@@ -323,35 +801,53 @@ func newTestCaCert() (*NebulaCertificate, []byte, []byte, error) {
 	return nc, pub, priv, nil
 }

-func newTestCert(ca *NebulaCertificate, key []byte) (*NebulaCertificate, []byte, []byte, error) {
+func newTestCert(ca *NebulaCertificate, key []byte, before, after time.Time, ips, subnets []*net.IPNet, groups []string) (*NebulaCertificate, []byte, []byte, error) {
 	issuer, err := ca.Sha256Sum()
 	if err != nil {
 		return nil, nil, nil, err
 	}

-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
+	if before.IsZero() {
+		before = time.Now().Add(time.Second * -60).Round(time.Second)
+	}
+	if after.IsZero() {
+		after = time.Now().Add(time.Second * 60).Round(time.Second)
+	}
+
+	if len(groups) == 0 {
+		groups = []string{"test-group1", "test-group2", "test-group3"}
+	}
+
+	if len(ips) == 0 {
+		ips = []*net.IPNet{
+			{IP: net.ParseIP("10.1.1.1").To4(), Mask: net.IPMask(net.ParseIP("255.255.255.0").To4())},
+			{IP: net.ParseIP("10.1.1.2").To4(), Mask: net.IPMask(net.ParseIP("255.255.0.0").To4())},
+			{IP: net.ParseIP("10.1.1.3").To4(), Mask: net.IPMask(net.ParseIP("255.0.255.0").To4())},
+		}
+	}
+
+	if len(subnets) == 0 {
+		subnets = []*net.IPNet{
+			{IP: net.ParseIP("9.1.1.1").To4(), Mask: net.IPMask(net.ParseIP("255.0.255.0").To4())},
+			{IP: net.ParseIP("9.1.1.2").To4(), Mask: net.IPMask(net.ParseIP("255.255.255.0").To4())},
+			{IP: net.ParseIP("9.1.1.3").To4(), Mask: net.IPMask(net.ParseIP("255.255.0.0").To4())},
+		}
+	}
+
 	pub, rawPriv := x25519Keypair()

 	nc := &NebulaCertificate{
 		Details: NebulaCertificateDetails{
 			Name:           "testing",
-			Ips: []*net.IPNet{
-				{IP: net.ParseIP("10.1.1.1"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))},
-				{IP: net.ParseIP("10.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))},
-				{IP: net.ParseIP("10.1.1.3"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))},
-			},
-			Subnets: []*net.IPNet{
-				{IP: net.ParseIP("9.1.1.1"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))},
-				{IP: net.ParseIP("9.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))},
-				{IP: net.ParseIP("9.1.1.3"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))},
-			},
-			Groups:    []string{"test-group1", "test-group2", "test-group3"},
-			NotBefore: before,
-			NotAfter:  after,
+			Ips:            ips,
+			Subnets:        subnets,
+			Groups:         groups,
+			NotBefore:      time.Unix(before.Unix(), 0),
+			NotAfter:       time.Unix(after.Unix(), 0),
 			PublicKey:      pub,
 			IsCA:           false,
 			Issuer:         issuer,
+			InvertedGroups: make(map[string]struct{}),
 		},
 	}

@@ -364,10 +860,15 @@ func newTestCert(ca *NebulaCertificate, key []byte) (*NebulaCertificate, []byte,
 }

 func x25519Keypair() ([]byte, []byte) {
-	var pubkey, privkey [32]byte
-	if _, err := io.ReadFull(rand.Reader, privkey[:]); err != nil {
+	privkey := make([]byte, 32)
+	if _, err := io.ReadFull(rand.Reader, privkey); err != nil {
 		panic(err)
 	}
-	curve25519.ScalarBaseMult(&pubkey, &privkey)
-	return pubkey[:], privkey[:]
+
+	pubkey, err := curve25519.X25519(privkey, curve25519.Basepoint)
+	if err != nil {
+		panic(err)
+	}
+
+	return pubkey, privkey
 }
--- a/cidr/parse.go
+++ b/cidr/parse.go
@@ -0,0 +1,10 @@
+package cidr
+
+import "net"
+
+// Parse is a convenience function that returns only the IPNet
+// This function ignores errors since it is primarily a test helper, the result could be nil
+func Parse(s string) *net.IPNet {
+	_, c, _ := net.ParseCIDR(s)
+	return c
+}
--- a/cidr/tree4.go
+++ b/cidr/tree4.go
@@ -1,39 +1,39 @@
-package nebula
+package cidr

 import (
-	"encoding/binary"
-	"fmt"
 	"net"
+
+	"github.com/slackhq/nebula/iputil"
 )

-type CIDRNode struct {
-	left   *CIDRNode
-	right  *CIDRNode
-	parent *CIDRNode
+type Node struct {
+	left   *Node
+	right  *Node
+	parent *Node
 	value  interface{}
 }

-type CIDRTree struct {
-	root *CIDRNode
+type Tree4 struct {
+	root *Node
 }

 const (
-	startbit = uint32(0x80000000)
+	startbit = iputil.VpnIp(0x80000000)
 )

-func NewCIDRTree() *CIDRTree {
-	tree := new(CIDRTree)
-	tree.root = &CIDRNode{}
+func NewTree4() *Tree4 {
+	tree := new(Tree4)
+	tree.root = &Node{}
 	return tree
 }

-func (tree *CIDRTree) AddCIDR(cidr *net.IPNet, val interface{}) {
+func (tree *Tree4) AddCIDR(cidr *net.IPNet, val interface{}) {
 	bit := startbit
 	node := tree.root
 	next := tree.root

-	ip := ip2int(cidr.IP)
-	mask := ip2int(cidr.Mask)
+	ip := iputil.Ip2VpnIp(cidr.IP)
+	mask := iputil.Ip2VpnIp(cidr.Mask)

 	// Find our last ancestor in the tree
 	for bit&mask != 0 {
@@ -59,7 +59,7 @@ func (tree *CIDRTree) AddCIDR(cidr *net.IPNet, val interface{}) {

 	// Build up the rest of the tree we don't already have
 	for bit&mask != 0 {
-		next = &CIDRNode{}
+		next = &Node{}
 		next.parent = node

 		if ip&bit != 0 {
@@ -76,8 +76,8 @@ func (tree *CIDRTree) AddCIDR(cidr *net.IPNet, val interface{}) {
 	node.value = val
 }

-// Finds the first match, which way be the least specific
-func (tree *CIDRTree) Contains(ip uint32) (value interface{}) {
+// Finds the first match, which may be the least specific
+func (tree *Tree4) Contains(ip iputil.VpnIp) (value interface{}) {
 	bit := startbit
 	node := tree.root

@@ -100,7 +100,29 @@ func (tree *CIDRTree) Contains(ip uint32) (value interface{}) {
 }

 // Finds the most specific match
-func (tree *CIDRTree) Match(ip uint32) (value interface{}) {
+func (tree *Tree4) MostSpecificContains(ip iputil.VpnIp) (value interface{}) {
+	bit := startbit
+	node := tree.root
+
+	for node != nil {
+		if node.value != nil {
+			value = node.value
+		}
+
+		if ip&bit != 0 {
+			node = node.right
+		} else {
+			node = node.left
+		}
+
+		bit >>= 1
+	}
+
+	return value
+}
+
+// Finds the most specific match
+func (tree *Tree4) Match(ip iputil.VpnIp) (value interface{}) {
 	bit := startbit
 	node := tree.root
 	lastNode := node
@@ -121,27 +143,3 @@ func (tree *CIDRTree) Match(ip uint32) (value interface{}) {
 	}
 	return value
 }
-
-// A helper type to avoid converting to IP when logging
-type IntIp uint32
-
-func (ip IntIp) String() string {
-	return fmt.Sprintf("%v", int2ip(uint32(ip)))
-}
-
-func (ip IntIp) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf("\"%s\"", int2ip(uint32(ip)).String())), nil
-}
-
-func ip2int(ip []byte) uint32 {
-	if len(ip) == 16 {
-		return binary.BigEndian.Uint32(ip[12:16])
-	}
-	return binary.BigEndian.Uint32(ip)
-}
-
-func int2ip(nn uint32) net.IP {
-	ip := make(net.IP, 4)
-	binary.BigEndian.PutUint32(ip, nn)
-	return ip
-}
--- a/cidr/tree4_test.go
+++ b/cidr/tree4_test.go
@@ -0,0 +1,153 @@
+package cidr
+
+import (
+	"net"
+	"testing"
+
+	"github.com/slackhq/nebula/iputil"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCIDRTree_Contains(t *testing.T) {
+	tree := NewTree4()
+	tree.AddCIDR(Parse("1.0.0.0/8"), "1")
+	tree.AddCIDR(Parse("2.1.0.0/16"), "2")
+	tree.AddCIDR(Parse("3.1.1.0/24"), "3")
+	tree.AddCIDR(Parse("4.1.1.0/24"), "4a")
+	tree.AddCIDR(Parse("4.1.1.1/32"), "4b")
+	tree.AddCIDR(Parse("4.1.2.1/32"), "4c")
+	tree.AddCIDR(Parse("254.0.0.0/4"), "5")
+
+	tests := []struct {
+		Result interface{}
+		IP     string
+	}{
+		{"1", "1.0.0.0"},
+		{"1", "1.255.255.255"},
+		{"2", "2.1.0.0"},
+		{"2", "2.1.255.255"},
+		{"3", "3.1.1.0"},
+		{"3", "3.1.1.255"},
+		{"4a", "4.1.1.255"},
+		{"4a", "4.1.1.1"},
+		{"5", "240.0.0.0"},
+		{"5", "255.255.255.255"},
+		{nil, "239.0.0.0"},
+		{nil, "4.1.2.2"},
+	}
+
+	for _, tt := range tests {
+		assert.Equal(t, tt.Result, tree.Contains(iputil.Ip2VpnIp(net.ParseIP(tt.IP))))
+	}
+
+	tree = NewTree4()
+	tree.AddCIDR(Parse("1.1.1.1/0"), "cool")
+	assert.Equal(t, "cool", tree.Contains(iputil.Ip2VpnIp(net.ParseIP("0.0.0.0"))))
+	assert.Equal(t, "cool", tree.Contains(iputil.Ip2VpnIp(net.ParseIP("255.255.255.255"))))
+}
+
+func TestCIDRTree_MostSpecificContains(t *testing.T) {
+	tree := NewTree4()
+	tree.AddCIDR(Parse("1.0.0.0/8"), "1")
+	tree.AddCIDR(Parse("2.1.0.0/16"), "2")
+	tree.AddCIDR(Parse("3.1.1.0/24"), "3")
+	tree.AddCIDR(Parse("4.1.1.0/24"), "4a")
+	tree.AddCIDR(Parse("4.1.1.0/30"), "4b")
+	tree.AddCIDR(Parse("4.1.1.1/32"), "4c")
+	tree.AddCIDR(Parse("254.0.0.0/4"), "5")
+
+	tests := []struct {
+		Result interface{}
+		IP     string
+	}{
+		{"1", "1.0.0.0"},
+		{"1", "1.255.255.255"},
+		{"2", "2.1.0.0"},
+		{"2", "2.1.255.255"},
+		{"3", "3.1.1.0"},
+		{"3", "3.1.1.255"},
+		{"4a", "4.1.1.255"},
+		{"4b", "4.1.1.2"},
+		{"4c", "4.1.1.1"},
+		{"5", "240.0.0.0"},
+		{"5", "255.255.255.255"},
+		{nil, "239.0.0.0"},
+		{nil, "4.1.2.2"},
+	}
+
+	for _, tt := range tests {
+		assert.Equal(t, tt.Result, tree.MostSpecificContains(iputil.Ip2VpnIp(net.ParseIP(tt.IP))))
+	}
+
+	tree = NewTree4()
+	tree.AddCIDR(Parse("1.1.1.1/0"), "cool")
+	assert.Equal(t, "cool", tree.MostSpecificContains(iputil.Ip2VpnIp(net.ParseIP("0.0.0.0"))))
+	assert.Equal(t, "cool", tree.MostSpecificContains(iputil.Ip2VpnIp(net.ParseIP("255.255.255.255"))))
+}
+
+func TestCIDRTree_Match(t *testing.T) {
+	tree := NewTree4()
+	tree.AddCIDR(Parse("4.1.1.0/32"), "1a")
+	tree.AddCIDR(Parse("4.1.1.1/32"), "1b")
+
+	tests := []struct {
+		Result interface{}
+		IP     string
+	}{
+		{"1a", "4.1.1.0"},
+		{"1b", "4.1.1.1"},
+	}
+
+	for _, tt := range tests {
+		assert.Equal(t, tt.Result, tree.Match(iputil.Ip2VpnIp(net.ParseIP(tt.IP))))
+	}
+
+	tree = NewTree4()
+	tree.AddCIDR(Parse("1.1.1.1/0"), "cool")
+	assert.Equal(t, "cool", tree.Contains(iputil.Ip2VpnIp(net.ParseIP("0.0.0.0"))))
+	assert.Equal(t, "cool", tree.Contains(iputil.Ip2VpnIp(net.ParseIP("255.255.255.255"))))
+}
+
+func BenchmarkCIDRTree_Contains(b *testing.B) {
+	tree := NewTree4()
+	tree.AddCIDR(Parse("1.1.0.0/16"), "1")
+	tree.AddCIDR(Parse("1.2.1.1/32"), "1")
+	tree.AddCIDR(Parse("192.2.1.1/32"), "1")
+	tree.AddCIDR(Parse("172.2.1.1/32"), "1")
+
+	ip := iputil.Ip2VpnIp(net.ParseIP("1.2.1.1"))
+	b.Run("found", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			tree.Contains(ip)
+		}
+	})
+
+	ip = iputil.Ip2VpnIp(net.ParseIP("1.2.1.255"))
+	b.Run("not found", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			tree.Contains(ip)
+		}
+	})
+}
+
+func BenchmarkCIDRTree_Match(b *testing.B) {
+	tree := NewTree4()
+	tree.AddCIDR(Parse("1.1.0.0/16"), "1")
+	tree.AddCIDR(Parse("1.2.1.1/32"), "1")
+	tree.AddCIDR(Parse("192.2.1.1/32"), "1")
+	tree.AddCIDR(Parse("172.2.1.1/32"), "1")
+
+	ip := iputil.Ip2VpnIp(net.ParseIP("1.2.1.1"))
+	b.Run("found", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			tree.Match(ip)
+		}
+	})
+
+	ip = iputil.Ip2VpnIp(net.ParseIP("1.2.1.255"))
+	b.Run("not found", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			tree.Match(ip)
+		}
+	})
+}
--- a/cidr/tree6.go
+++ b/cidr/tree6.go
@@ -0,0 +1,185 @@
+package cidr
+
+import (
+	"net"
+
+	"github.com/slackhq/nebula/iputil"
+)
+
+const startbit6 = uint64(1 << 63)
+
+type Tree6 struct {
+	root4 *Node
+	root6 *Node
+}
+
+func NewTree6() *Tree6 {
+	tree := new(Tree6)
+	tree.root4 = &Node{}
+	tree.root6 = &Node{}
+	return tree
+}
+
+func (tree *Tree6) AddCIDR(cidr *net.IPNet, val interface{}) {
+	var node, next *Node
+
+	cidrIP, ipv4 := isIPV4(cidr.IP)
+	if ipv4 {
+		node = tree.root4
+		next = tree.root4
+
+	} else {
+		node = tree.root6
+		next = tree.root6
+	}
+
+	for i := 0; i < len(cidrIP); i += 4 {
+		ip := iputil.Ip2VpnIp(cidrIP[i : i+4])
+		mask := iputil.Ip2VpnIp(cidr.Mask[i : i+4])
+		bit := startbit
+
+		// Find our last ancestor in the tree
+		for bit&mask != 0 {
+			if ip&bit != 0 {
+				next = node.right
+			} else {
+				next = node.left
+			}
+
+			if next == nil {
+				break
+			}
+
+			bit = bit >> 1
+			node = next
+		}
+
+		// Build up the rest of the tree we don't already have
+		for bit&mask != 0 {
+			next = &Node{}
+			next.parent = node
+
+			if ip&bit != 0 {
+				node.right = next
+			} else {
+				node.left = next
+			}
+
+			bit >>= 1
+			node = next
+		}
+	}
+
+	// Final node marks our cidr, set the value
+	node.value = val
+}
+
+// Finds the most specific match
+func (tree *Tree6) MostSpecificContains(ip net.IP) (value interface{}) {
+	var node *Node
+
+	wholeIP, ipv4 := isIPV4(ip)
+	if ipv4 {
+		node = tree.root4
+	} else {
+		node = tree.root6
+	}
+
+	for i := 0; i < len(wholeIP); i += 4 {
+		ip := iputil.Ip2VpnIp(wholeIP[i : i+4])
+		bit := startbit
+
+		for node != nil {
+			if node.value != nil {
+				value = node.value
+			}
+
+			if bit == 0 {
+				break
+			}
+
+			if ip&bit != 0 {
+				node = node.right
+			} else {
+				node = node.left
+			}
+
+			bit >>= 1
+		}
+	}
+
+	return value
+}
+
+func (tree *Tree6) MostSpecificContainsIpV4(ip iputil.VpnIp) (value interface{}) {
+	bit := startbit
+	node := tree.root4
+
+	for node != nil {
+		if node.value != nil {
+			value = node.value
+		}
+
+		if ip&bit != 0 {
+			node = node.right
+		} else {
+			node = node.left
+		}
+
+		bit >>= 1
+	}
+
+	return value
+}
+
+func (tree *Tree6) MostSpecificContainsIpV6(hi, lo uint64) (value interface{}) {
+	ip := hi
+	node := tree.root6
+
+	for i := 0; i < 2; i++ {
+		bit := startbit6
+
+		for node != nil {
+			if node.value != nil {
+				value = node.value
+			}
+
+			if bit == 0 {
+				break
+			}
+
+			if ip&bit != 0 {
+				node = node.right
+			} else {
+				node = node.left
+			}
+
+			bit >>= 1
+		}
+
+		ip = lo
+	}
+
+	return value
+}
+
+func isIPV4(ip net.IP) (net.IP, bool) {
+	if len(ip) == net.IPv4len {
+		return ip, true
+	}
+
+	if len(ip) == net.IPv6len && isZeros(ip[0:10]) && ip[10] == 0xff && ip[11] == 0xff {
+		return ip[12:16], true
+	}
+
+	return ip, false
+}
+
+func isZeros(p net.IP) bool {
+	for i := 0; i < len(p); i++ {
+		if p[i] != 0 {
+			return false
+		}
+	}
+	return true
+}
--- a/cidr/tree6_test.go
+++ b/cidr/tree6_test.go
@@ -0,0 +1,81 @@
+package cidr
+
+import (
+	"encoding/binary"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCIDR6Tree_MostSpecificContains(t *testing.T) {
+	tree := NewTree6()
+	tree.AddCIDR(Parse("1.0.0.0/8"), "1")
+	tree.AddCIDR(Parse("2.1.0.0/16"), "2")
+	tree.AddCIDR(Parse("3.1.1.0/24"), "3")
+	tree.AddCIDR(Parse("4.1.1.1/24"), "4a")
+	tree.AddCIDR(Parse("4.1.1.1/30"), "4b")
+	tree.AddCIDR(Parse("4.1.1.1/32"), "4c")
+	tree.AddCIDR(Parse("254.0.0.0/4"), "5")
+	tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/64"), "6a")
+	tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/80"), "6b")
+	tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/96"), "6c")
+
+	tests := []struct {
+		Result interface{}
+		IP     string
+	}{
+		{"1", "1.0.0.0"},
+		{"1", "1.255.255.255"},
+		{"2", "2.1.0.0"},
+		{"2", "2.1.255.255"},
+		{"3", "3.1.1.0"},
+		{"3", "3.1.1.255"},
+		{"4a", "4.1.1.255"},
+		{"4b", "4.1.1.2"},
+		{"4c", "4.1.1.1"},
+		{"5", "240.0.0.0"},
+		{"5", "255.255.255.255"},
+		{"6a", "1:2:0:4:1:1:1:1"},
+		{"6b", "1:2:0:4:5:1:1:1"},
+		{"6c", "1:2:0:4:5:0:0:0"},
+		{nil, "239.0.0.0"},
+		{nil, "4.1.2.2"},
+	}
+
+	for _, tt := range tests {
+		assert.Equal(t, tt.Result, tree.MostSpecificContains(net.ParseIP(tt.IP)))
+	}
+
+	tree = NewTree6()
+	tree.AddCIDR(Parse("1.1.1.1/0"), "cool")
+	tree.AddCIDR(Parse("::/0"), "cool6")
+	assert.Equal(t, "cool", tree.MostSpecificContains(net.ParseIP("0.0.0.0")))
+	assert.Equal(t, "cool", tree.MostSpecificContains(net.ParseIP("255.255.255.255")))
+	assert.Equal(t, "cool6", tree.MostSpecificContains(net.ParseIP("::")))
+	assert.Equal(t, "cool6", tree.MostSpecificContains(net.ParseIP("1:2:3:4:5:6:7:8")))
+}
+
+func TestCIDR6Tree_MostSpecificContainsIpV6(t *testing.T) {
+	tree := NewTree6()
+	tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/64"), "6a")
+	tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/80"), "6b")
+	tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/96"), "6c")
+
+	tests := []struct {
+		Result interface{}
+		IP     string
+	}{
+		{"6a", "1:2:0:4:1:1:1:1"},
+		{"6b", "1:2:0:4:5:1:1:1"},
+		{"6c", "1:2:0:4:5:0:0:0"},
+	}
+
+	for _, tt := range tests {
+		ip := net.ParseIP(tt.IP)
+		hi := binary.BigEndian.Uint64(ip[:8])
+		lo := binary.BigEndian.Uint64(ip[8:])
+
+		assert.Equal(t, tt.Result, tree.MostSpecificContainsIpV6(hi, lo))
+	}
+}
--- a/cidr_radix_test.go
+++ b/cidr_radix_test.go
@@ -1,118 +0,0 @@
-package nebula
-
-import (
-	"net"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestCIDRTree_Contains(t *testing.T) {
-	tree := NewCIDRTree()
-	tree.AddCIDR(getCIDR("1.0.0.0/8"), "1")
-	tree.AddCIDR(getCIDR("2.1.0.0/16"), "2")
-	tree.AddCIDR(getCIDR("3.1.1.0/24"), "3")
-	tree.AddCIDR(getCIDR("4.1.1.0/24"), "4a")
-	tree.AddCIDR(getCIDR("4.1.1.1/32"), "4b")
-	tree.AddCIDR(getCIDR("4.1.2.1/32"), "4c")
-	tree.AddCIDR(getCIDR("254.0.0.0/4"), "5")
-
-	tests := []struct {
-		Result interface{}
-		IP     string
-	}{
-		{"1", "1.0.0.0"},
-		{"1", "1.255.255.255"},
-		{"2", "2.1.0.0"},
-		{"2", "2.1.255.255"},
-		{"3", "3.1.1.0"},
-		{"3", "3.1.1.255"},
-		{"4a", "4.1.1.255"},
-		{"4a", "4.1.1.1"},
-		{"5", "240.0.0.0"},
-		{"5", "255.255.255.255"},
-		{nil, "239.0.0.0"},
-		{nil, "4.1.2.2"},
-	}
-
-	for _, tt := range tests {
-		assert.Equal(t, tt.Result, tree.Contains(ip2int(net.ParseIP(tt.IP))))
-	}
-
-	tree = NewCIDRTree()
-	tree.AddCIDR(getCIDR("1.1.1.1/0"), "cool")
-	assert.Equal(t, "cool", tree.Contains(ip2int(net.ParseIP("0.0.0.0"))))
-	assert.Equal(t, "cool", tree.Contains(ip2int(net.ParseIP("255.255.255.255"))))
-}
-
-func TestCIDRTree_Match(t *testing.T) {
-	tree := NewCIDRTree()
-	tree.AddCIDR(getCIDR("4.1.1.0/32"), "1a")
-	tree.AddCIDR(getCIDR("4.1.1.1/32"), "1b")
-
-	tests := []struct {
-		Result interface{}
-		IP     string
-	}{
-		{"1a", "4.1.1.0"},
-		{"1b", "4.1.1.1"},
-	}
-
-	for _, tt := range tests {
-		assert.Equal(t, tt.Result, tree.Match(ip2int(net.ParseIP(tt.IP))))
-	}
-
-	tree = NewCIDRTree()
-	tree.AddCIDR(getCIDR("1.1.1.1/0"), "cool")
-	assert.Equal(t, "cool", tree.Contains(ip2int(net.ParseIP("0.0.0.0"))))
-	assert.Equal(t, "cool", tree.Contains(ip2int(net.ParseIP("255.255.255.255"))))
-}
-
-func BenchmarkCIDRTree_Contains(b *testing.B) {
-	tree := NewCIDRTree()
-	tree.AddCIDR(getCIDR("1.1.0.0/16"), "1")
-	tree.AddCIDR(getCIDR("1.2.1.1/32"), "1")
-	tree.AddCIDR(getCIDR("192.2.1.1/32"), "1")
-	tree.AddCIDR(getCIDR("172.2.1.1/32"), "1")
-
-	ip := ip2int(net.ParseIP("1.2.1.1"))
-	b.Run("found", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			tree.Contains(ip)
-		}
-	})
-
-	ip = ip2int(net.ParseIP("1.2.1.255"))
-	b.Run("not found", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			tree.Contains(ip)
-		}
-	})
-}
-
-func BenchmarkCIDRTree_Match(b *testing.B) {
-	tree := NewCIDRTree()
-	tree.AddCIDR(getCIDR("1.1.0.0/16"), "1")
-	tree.AddCIDR(getCIDR("1.2.1.1/32"), "1")
-	tree.AddCIDR(getCIDR("192.2.1.1/32"), "1")
-	tree.AddCIDR(getCIDR("172.2.1.1/32"), "1")
-
-	ip := ip2int(net.ParseIP("1.2.1.1"))
-	b.Run("found", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			tree.Match(ip)
-		}
-	})
-
-	ip = ip2int(net.ParseIP("1.2.1.255"))
-	b.Run("not found", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			tree.Match(ip)
-		}
-	})
-}
-
-func getCIDR(s string) *net.IPNet {
-	_, c, _ := net.ParseCIDR(s)
-	return c
-}
--- a/cmd/nebula-cert/ca.go
+++ b/cmd/nebula-cert/ca.go
@@ -6,12 +6,14 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"strings"
 	"time"

-	"golang.org/x/crypto/ed25519"
+	"github.com/skip2/go-qrcode"
 	"github.com/slackhq/nebula/cert"
+	"golang.org/x/crypto/ed25519"
 )

 type caFlags struct {
@@ -20,7 +22,10 @@ type caFlags struct {
 	duration    *time.Duration
 	outKeyPath  *string
 	outCertPath *string
+	outQRPath   *string
 	groups      *string
+	ips         *string
+	subnets     *string
 }

 func newCaFlags() *caFlags {
@@ -30,7 +35,10 @@ func newCaFlags() *caFlags {
 	cf.duration = cf.set.Duration("duration", time.Duration(time.Hour*8760), "Optional: amount of time the certificate should be valid for. Valid time units are seconds: \"s\", minutes: \"m\", hours: \"h\"")
 	cf.outKeyPath = cf.set.String("out-key", "ca.key", "Optional: path to write the private key to")
 	cf.outCertPath = cf.set.String("out-crt", "ca.crt", "Optional: path to write the certificate to")
+	cf.outQRPath = cf.set.String("out-qr", "", "Optional: output a qr code image (png) of the certificate")
 	cf.groups = cf.set.String("groups", "", "Optional: comma separated list of groups. This will limit which groups subordinate certs can use")
+	cf.ips = cf.set.String("ips", "", "Optional: comma separated list of ip and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use")
+	cf.subnets = cf.set.String("subnets", "", "Optional: comma separated list of ip and network in CIDR notation. This will limit which subnet addresses and networks subordinate certs can use")
 	return &cf
 }

@@ -55,7 +63,7 @@ func ca(args []string, out io.Writer, errOut io.Writer) error {
 		return &helpError{"-duration must be greater than 0"}
 	}

-	groups := []string{}
+	var groups []string
 	if *cf.groups != "" {
 		for _, rg := range strings.Split(*cf.groups, ",") {
 			g := strings.TrimSpace(rg)
@@ -65,6 +73,36 @@ func ca(args []string, out io.Writer, errOut io.Writer) error {
 		}
 	}

+	var ips []*net.IPNet
+	if *cf.ips != "" {
+		for _, rs := range strings.Split(*cf.ips, ",") {
+			rs := strings.Trim(rs, " ")
+			if rs != "" {
+				ip, ipNet, err := net.ParseCIDR(rs)
+				if err != nil {
+					return newHelpErrorf("invalid ip definition: %s", err)
+				}
+
+				ipNet.IP = ip
+				ips = append(ips, ipNet)
+			}
+		}
+	}
+
+	var subnets []*net.IPNet
+	if *cf.subnets != "" {
+		for _, rs := range strings.Split(*cf.subnets, ",") {
+			rs := strings.Trim(rs, " ")
+			if rs != "" {
+				_, s, err := net.ParseCIDR(rs)
+				if err != nil {
+					return newHelpErrorf("invalid subnet definition: %s", err)
+				}
+				subnets = append(subnets, s)
+			}
+		}
+	}
+
 	pub, rawPriv, err := ed25519.GenerateKey(rand.Reader)
 	if err != nil {
 		return fmt.Errorf("error while generating ed25519 keys: %s", err)
@@ -74,6 +112,8 @@ func ca(args []string, out io.Writer, errOut io.Writer) error {
 		Details: cert.NebulaCertificateDetails{
 			Name:      *cf.name,
 			Groups:    groups,
+			Ips:       ips,
+			Subnets:   subnets,
 			NotBefore: time.Now(),
 			NotAfter:  time.Now().Add(*cf.duration),
 			PublicKey: pub,
@@ -109,6 +149,18 @@ func ca(args []string, out io.Writer, errOut io.Writer) error {
 		return fmt.Errorf("error while writing out-crt: %s", err)
 	}

+	if *cf.outQRPath != "" {
+		b, err = qrcode.Encode(string(b), qrcode.Medium, -5)
+		if err != nil {
+			return fmt.Errorf("error while generating qr code: %s", err)
+		}
+
+		err = ioutil.WriteFile(*cf.outQRPath, b, 0600)
+		if err != nil {
+			return fmt.Errorf("error while writing out-qr: %s", err)
+		}
+	}
+
 	return nil
 }

--- a/cmd/nebula-cert/ca_test.go
+++ b/cmd/nebula-cert/ca_test.go
@@ -1,3 +1,6 @@
+//go:build !windows
+// +build !windows
+
 package main

 import (
@@ -7,8 +10,8 @@ import (
 	"testing"
 	"time"

-	"github.com/stretchr/testify/assert"
 	"github.com/slackhq/nebula/cert"
+	"github.com/stretchr/testify/assert"
 )

 //TODO: test file permissions
@@ -27,12 +30,18 @@ func Test_caHelp(t *testing.T) {
 			"    \tOptional: amount of time the certificate should be valid for. Valid time units are seconds: \"s\", minutes: \"m\", hours: \"h\" (default 8760h0m0s)\n"+
 			"  -groups string\n"+
 			"    \tOptional: comma separated list of groups. This will limit which groups subordinate certs can use\n"+
+			"  -ips string\n"+
+			"    \tOptional: comma separated list of ip and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use\n"+
 			"  -name string\n"+
 			"    \tRequired: name of the certificate authority\n"+
 			"  -out-crt string\n"+
 			"    \tOptional: path to write the certificate to (default \"ca.crt\")\n"+
 			"  -out-key string\n"+
-			"    \tOptional: path to write the private key to (default \"ca.key\")\n",
+			"    \tOptional: path to write the private key to (default \"ca.key\")\n"+
+			"  -out-qr string\n"+
+			"    \tOptional: output a qr code image (png) of the certificate\n"+
+			"  -subnets string\n"+
+			"    \tOptional: comma separated list of ip and network in CIDR notation. This will limit which subnet addresses and networks subordinate certs can use\n",
 		ob.String(),
 	)
 }
--- a/cmd/nebula-cert/keygen_test.go
+++ b/cmd/nebula-cert/keygen_test.go
@@ -6,8 +6,8 @@ import (
 	"os"
 	"testing"

-	"github.com/stretchr/testify/assert"
 	"github.com/slackhq/nebula/cert"
+	"github.com/stretchr/testify/assert"
 )

 //TODO: test file permissions
--- a/cmd/nebula-cert/main_test.go
+++ b/cmd/nebula-cert/main_test.go
@@ -3,10 +3,11 @@ package main
 import (
 	"bytes"
 	"errors"
-	"github.com/stretchr/testify/assert"
 	"io"
 	"os"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )

 //TODO: all flag parsing continueOnError will print to stderr on its own currently
--- a/cmd/nebula-cert/print.go
+++ b/cmd/nebula-cert/print.go
@@ -7,13 +7,16 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"github.com/slackhq/nebula/cert"
 	"strings"
+
+	"github.com/skip2/go-qrcode"
+	"github.com/slackhq/nebula/cert"
 )

 type printFlags struct {
 	set       *flag.FlagSet
 	json      *bool
+	outQRPath *string
 	path      *string
 }

@@ -21,6 +24,7 @@ func newPrintFlags() *printFlags {
 	pf := printFlags{set: flag.NewFlagSet("print", flag.ContinueOnError)}
 	pf.set.Usage = func() {}
 	pf.json = pf.set.Bool("json", false, "Optional: outputs certificates in json format")
+	pf.outQRPath = pf.set.String("out-qr", "", "Optional: output a qr code image (png) of the certificate")
 	pf.path = pf.set.String("path", "", "Required: path to the certificate")

 	return &pf
@@ -43,6 +47,8 @@ func printCert(args []string, out io.Writer, errOut io.Writer) error {
 	}

 	var c *cert.NebulaCertificate
+	var qrBytes []byte
+	part := 0

 	for {
 		c, rawCert, err = cert.UnmarshalNebulaCertificateFromPEM(rawCert)
@@ -60,9 +66,31 @@ func printCert(args []string, out io.Writer, errOut io.Writer) error {
 			out.Write([]byte("\n"))
 		}

+		if *pf.outQRPath != "" {
+			b, err := c.MarshalToPEM()
+			if err != nil {
+				return fmt.Errorf("error while marshalling cert to PEM: %s", err)
+			}
+			qrBytes = append(qrBytes, b...)
+		}
+
 		if rawCert == nil || len(rawCert) == 0 || strings.TrimSpace(string(rawCert)) == "" {
 			break
 		}
+
+		part++
+	}
+
+	if *pf.outQRPath != "" {
+		b, err := qrcode.Encode(string(qrBytes), qrcode.Medium, -5)
+		if err != nil {
+			return fmt.Errorf("error while generating qr code: %s", err)
+		}
+
+		err = ioutil.WriteFile(*pf.outQRPath, b, 0600)
+		if err != nil {
+			return fmt.Errorf("error while writing out-qr: %s", err)
+		}
 	}

 	return nil
--- a/cmd/nebula-cert/print_test.go
+++ b/cmd/nebula-cert/print_test.go
@@ -2,12 +2,13 @@ package main

 import (
 	"bytes"
-	"github.com/stretchr/testify/assert"
 	"io/ioutil"
 	"os"
-	"github.com/slackhq/nebula/cert"
 	"testing"
 	"time"
+
+	"github.com/slackhq/nebula/cert"
+	"github.com/stretchr/testify/assert"
 )

 func Test_printSummary(t *testing.T) {
@@ -22,6 +23,8 @@ func Test_printHelp(t *testing.T) {
 		"Usage of "+os.Args[0]+" print <flags>: prints details about a certificate\n"+
 			"  -json\n"+
 			"    \tOptional: outputs certificates in json format\n"+
+			"  -out-qr string\n"+
+			"    \tOptional: output a qr code image (png) of the certificate\n"+
 			"  -path string\n"+
 			"    \tRequired: path to the certificate\n",
 		ob.String(),
--- a/cmd/nebula-cert/sign.go
+++ b/cmd/nebula-cert/sign.go
@@ -11,8 +11,9 @@ import (
 	"strings"
 	"time"

-	"golang.org/x/crypto/curve25519"
+	"github.com/skip2/go-qrcode"
 	"github.com/slackhq/nebula/cert"
+	"golang.org/x/crypto/curve25519"
 )

 type signFlags struct {
@@ -25,6 +26,7 @@ type signFlags struct {
 	inPubPath   *string
 	outKeyPath  *string
 	outCertPath *string
+	outQRPath   *string
 	groups      *string
 	subnets     *string
 }
@@ -36,12 +38,13 @@ func newSignFlags() *signFlags {
 	sf.caCertPath = sf.set.String("ca-crt", "ca.crt", "Optional: path to the signing CA cert")
 	sf.name = sf.set.String("name", "", "Required: name of the cert, usually a hostname")
 	sf.ip = sf.set.String("ip", "", "Required: ip and network in CIDR notation to assign the cert")
-	sf.duration = sf.set.Duration("duration", 0, "Required: how long the cert should be valid for. Valid time units are seconds: \"s\", minutes: \"m\", hours: \"h\"")
+	sf.duration = sf.set.Duration("duration", 0, "Optional: how long the cert should be valid for. The default is 1 second before the signing cert expires. Valid time units are seconds: \"s\", minutes: \"m\", hours: \"h\"")
 	sf.inPubPath = sf.set.String("in-pub", "", "Optional (if out-key not set): path to read a previously generated public key")
 	sf.outKeyPath = sf.set.String("out-key", "", "Optional (if in-pub not set): path to write the private key to")
 	sf.outCertPath = sf.set.String("out-crt", "", "Optional: path to write the certificate to")
+	sf.outQRPath = sf.set.String("out-qr", "", "Optional: output a qr code image (png) of the certificate")
 	sf.groups = sf.set.String("groups", "", "Optional: comma separated list of groups")
-	sf.subnets = sf.set.String("subnets", "", "Optional: comma seperated list of subnet this cert can serve for")
+	sf.subnets = sf.set.String("subnets", "", "Optional: comma separated list of subnet this cert can serve for")
 	return &sf

 }
@@ -89,6 +92,10 @@ func signCert(args []string, out io.Writer, errOut io.Writer) error {
 		return fmt.Errorf("error while parsing ca-crt: %s", err)
 	}

+	if err := caCert.VerifyPrivateKey(caKey); err != nil {
+		return fmt.Errorf("refusing to sign, root certificate does not match private key")
+	}
+
 	issuer, err := caCert.Sha256Sum()
 	if err != nil {
 		return fmt.Errorf("error while getting -ca-crt fingerprint: %s", err)
@@ -103,10 +110,6 @@ func signCert(args []string, out io.Writer, errOut io.Writer) error {
 		*sf.duration = time.Until(caCert.Details.NotAfter) - time.Second*1
 	}

-	if caCert.Details.NotAfter.Before(time.Now().Add(*sf.duration)) {
-		return fmt.Errorf("refusing to generate certificate with duration beyond root expiration: %s", caCert.Details.NotAfter)
-	}
-
 	ip, ipNet, err := net.ParseCIDR(*sf.ip)
 	if err != nil {
 		return newHelpErrorf("invalid ip definition: %s", err)
@@ -165,6 +168,10 @@ func signCert(args []string, out io.Writer, errOut io.Writer) error {
 		},
 	}

+	if err := nc.CheckRootConstrains(caCert); err != nil {
+		return fmt.Errorf("refusing to sign, root certificate constraints violated: %s", err)
+	}
+
 	if *sf.outKeyPath == "" {
 		*sf.outKeyPath = *sf.name + ".key"
 	}
@@ -173,10 +180,6 @@ func signCert(args []string, out io.Writer, errOut io.Writer) error {
 		*sf.outCertPath = *sf.name + ".crt"
 	}

-	if _, err := os.Stat(*sf.outKeyPath); err == nil {
-		return fmt.Errorf("refusing to overwrite existing key: %s", *sf.outKeyPath)
-	}
-
 	if _, err := os.Stat(*sf.outCertPath); err == nil {
 		return fmt.Errorf("refusing to overwrite existing cert: %s", *sf.outCertPath)
 	}
@@ -187,6 +190,10 @@ func signCert(args []string, out io.Writer, errOut io.Writer) error {
 	}

 	if *sf.inPubPath == "" {
+		if _, err := os.Stat(*sf.outKeyPath); err == nil {
+			return fmt.Errorf("refusing to overwrite existing key: %s", *sf.outKeyPath)
+		}
+
 		err = ioutil.WriteFile(*sf.outKeyPath, cert.MarshalX25519PrivateKey(rawPriv), 0600)
 		if err != nil {
 			return fmt.Errorf("error while writing out-key: %s", err)
@@ -203,16 +210,33 @@ func signCert(args []string, out io.Writer, errOut io.Writer) error {
 		return fmt.Errorf("error while writing out-crt: %s", err)
 	}

+	if *sf.outQRPath != "" {
+		b, err = qrcode.Encode(string(b), qrcode.Medium, -5)
+		if err != nil {
+			return fmt.Errorf("error while generating qr code: %s", err)
+		}
+
+		err = ioutil.WriteFile(*sf.outQRPath, b, 0600)
+		if err != nil {
+			return fmt.Errorf("error while writing out-qr: %s", err)
+		}
+	}
+
 	return nil
 }

 func x25519Keypair() ([]byte, []byte) {
-	var pubkey, privkey [32]byte
-	if _, err := io.ReadFull(rand.Reader, privkey[:]); err != nil {
+	privkey := make([]byte, 32)
+	if _, err := io.ReadFull(rand.Reader, privkey); err != nil {
 		panic(err)
 	}
-	curve25519.ScalarBaseMult(&pubkey, &privkey)
-	return pubkey[:], privkey[:]
+
+	pubkey, err := curve25519.X25519(privkey, curve25519.Basepoint)
+	if err != nil {
+		panic(err)
+	}
+
+	return pubkey, privkey
 }

 func signSummary() string {
--- a/cmd/nebula-cert/sign_test.go
+++ b/cmd/nebula-cert/sign_test.go
@@ -1,3 +1,6 @@
+//go:build !windows
+// +build !windows
+
 package main

 import (
@@ -8,9 +11,9 @@ import (
 	"testing"
 	"time"

+	"github.com/slackhq/nebula/cert"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/crypto/ed25519"
-	"github.com/slackhq/nebula/cert"
 )

 //TODO: test file permissions
@@ -30,7 +33,7 @@ func Test_signHelp(t *testing.T) {
 			"  -ca-key string\n"+
 			"    \tOptional: path to the signing CA key (default \"ca.key\")\n"+
 			"  -duration duration\n"+
-			"    \tRequired: how long the cert should be valid for. Valid time units are seconds: \"s\", minutes: \"m\", hours: \"h\"\n"+
+			"    \tOptional: how long the cert should be valid for. The default is 1 second before the signing cert expires. Valid time units are seconds: \"s\", minutes: \"m\", hours: \"h\"\n"+
 			"  -groups string\n"+
 			"    \tOptional: comma separated list of groups\n"+
 			"  -in-pub string\n"+
@@ -43,8 +46,10 @@ func Test_signHelp(t *testing.T) {
 			"    \tOptional: path to write the certificate to\n"+
 			"  -out-key string\n"+
 			"    \tOptional (if in-pub not set): path to write the private key to\n"+
+			"  -out-qr string\n"+
+			"    \tOptional: output a qr code image (png) of the certificate\n"+
 			"  -subnets string\n"+
-			"    \tOptional: comma seperated list of subnet this cert can serve for\n",
+			"    \tOptional: comma separated list of subnet this cert can serve for\n",
 		ob.String(),
 	)
 }
@@ -163,6 +168,20 @@ func Test_signCert(t *testing.T) {
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())

+	// mismatched ca key
+	_, caPriv2, _ := ed25519.GenerateKey(rand.Reader)
+	caKeyF2, err := ioutil.TempFile("", "sign-cert-2.key")
+	assert.Nil(t, err)
+	defer os.Remove(caKeyF2.Name())
+	caKeyF2.Write(cert.MarshalEd25519PrivateKey(caPriv2))
+
+	ob.Reset()
+	eb.Reset()
+	args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF2.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m", "-subnets", "a"}
+	assert.EqualError(t, signCert(args, ob, eb), "refusing to sign, root certificate does not match private key")
+	assert.Empty(t, ob.String())
+	assert.Empty(t, eb.String())
+
 	// failed key write
 	ob.Reset()
 	eb.Reset()
@@ -251,7 +270,7 @@ func Test_signCert(t *testing.T) {
 	ob.Reset()
 	eb.Reset()
 	args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "1000m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
-	assert.EqualError(t, signCert(args, ob, eb), "refusing to generate certificate with duration beyond root expiration: "+ca.Details.NotAfter.Format("2006-01-02 15:04:05 +0000 UTC"))
+	assert.EqualError(t, signCert(args, ob, eb), "refusing to sign, root certificate constraints violated: certificate expires after signing certificate")
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())

@@ -262,6 +281,7 @@ func Test_signCert(t *testing.T) {
 	assert.Nil(t, signCert(args, ob, eb))

 	// test that we won't overwrite existing key file
+	os.Remove(crtF.Name())
 	ob.Reset()
 	eb.Reset()
 	args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
@@ -269,6 +289,12 @@ func Test_signCert(t *testing.T) {
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())

+	// create valid cert/key for overwrite tests
+	os.Remove(keyF.Name())
+	os.Remove(crtF.Name())
+	args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
+	assert.Nil(t, signCert(args, ob, eb))
+
 	// test that we won't overwrite existing certificate file
 	os.Remove(keyF.Name())
 	ob.Reset()
@@ -277,5 +303,4 @@ func Test_signCert(t *testing.T) {
 	assert.EqualError(t, signCert(args, ob, eb), "refusing to overwrite existing cert: "+crtF.Name())
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
-
 }
--- a/cmd/nebula-cert/test_darwin.go
+++ b/cmd/nebula-cert/test_darwin.go
@@ -0,0 +1,4 @@
+package main
+
+const NoSuchFileError = "no such file or directory"
+const NoSuchDirError = "no such file or directory"
--- a/cmd/nebula-cert/verify.go
+++ b/cmd/nebula-cert/verify.go
@@ -6,9 +6,10 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"github.com/slackhq/nebula/cert"
 	"strings"
 	"time"
+
+	"github.com/slackhq/nebula/cert"
 )

 type verifyFlags struct {
--- a/cmd/nebula-cert/verify_test.go
+++ b/cmd/nebula-cert/verify_test.go
@@ -3,13 +3,14 @@ package main
 import (
 	"bytes"
 	"crypto/rand"
-	"github.com/stretchr/testify/assert"
-	"golang.org/x/crypto/ed25519"
 	"io/ioutil"
 	"os"
-	"github.com/slackhq/nebula/cert"
 	"testing"
 	"time"
+
+	"github.com/slackhq/nebula/cert"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/crypto/ed25519"
 )

 func Test_verifySummary(t *testing.T) {
--- a/cmd/nebula-service/logs_generic.go
+++ b/cmd/nebula-service/logs_generic.go
@@ -0,0 +1,10 @@
+//go:build !windows
+// +build !windows
+
+package main
+
+import "github.com/sirupsen/logrus"
+
+func HookLogger(l *logrus.Logger) {
+	// Do nothing, let the logs flow to stdout/stderr
+}
--- a/cmd/nebula-service/logs_windows.go
+++ b/cmd/nebula-service/logs_windows.go
@@ -0,0 +1,54 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/kardianos/service"
+	"github.com/sirupsen/logrus"
+)
+
+// HookLogger routes the logrus logs through the service logger so that they end up in the Windows Event Viewer
+// logrus output will be discarded
+func HookLogger(l *logrus.Logger) {
+	l.AddHook(newLogHook(logger))
+	l.SetOutput(ioutil.Discard)
+}
+
+type logHook struct {
+	sl service.Logger
+}
+
+func newLogHook(sl service.Logger) *logHook {
+	return &logHook{sl: sl}
+}
+
+func (h *logHook) Fire(entry *logrus.Entry) error {
+	line, err := entry.String()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err)
+		return err
+	}
+
+	switch entry.Level {
+	case logrus.PanicLevel:
+		return h.sl.Error(line)
+	case logrus.FatalLevel:
+		return h.sl.Error(line)
+	case logrus.ErrorLevel:
+		return h.sl.Error(line)
+	case logrus.WarnLevel:
+		return h.sl.Warning(line)
+	case logrus.InfoLevel:
+		return h.sl.Info(line)
+	case logrus.DebugLevel:
+		return h.sl.Info(line)
+	default:
+		return nil
+	}
+}
+
+func (h *logHook) Levels() []logrus.Level {
+	return logrus.AllLevels
+}
--- a/cmd/nebula-service/main.go
+++ b/cmd/nebula-service/main.go
@@ -0,0 +1,77 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/config"
+)
+
+// A version string that can be set with
+//
+//     -ldflags "-X main.Build=SOMEVERSION"
+//
+// at compile-time.
+var Build string
+
+func main() {
+	serviceFlag := flag.String("service", "", "Control the system service.")
+	configPath := flag.String("config", "", "Path to either a file or directory to load configuration from")
+	configTest := flag.Bool("test", false, "Test the config and print the end result. Non zero exit indicates a faulty config")
+	printVersion := flag.Bool("version", false, "Print version")
+	printUsage := flag.Bool("help", false, "Print command line usage")
+
+	flag.Parse()
+
+	if *printVersion {
+		fmt.Printf("Version: %s\n", Build)
+		os.Exit(0)
+	}
+
+	if *printUsage {
+		flag.Usage()
+		os.Exit(0)
+	}
+
+	if *serviceFlag != "" {
+		doService(configPath, configTest, Build, serviceFlag)
+		os.Exit(1)
+	}
+
+	if *configPath == "" {
+		fmt.Println("-config flag must be set")
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	l := logrus.New()
+	l.Out = os.Stdout
+
+	c := config.NewC(l)
+	err := c.Load(*configPath)
+	if err != nil {
+		fmt.Printf("failed to load config: %s", err)
+		os.Exit(1)
+	}
+
+	ctrl, err := nebula.Main(c, *configTest, Build, l, nil)
+
+	switch v := err.(type) {
+	case nebula.ContextualError:
+		v.Log(l)
+		os.Exit(1)
+	case error:
+		l.WithError(err).Error("Failed to start")
+		os.Exit(1)
+	}
+
+	if !*configTest {
+		ctrl.Start()
+		ctrl.ShutdownBlock()
+	}
+
+	os.Exit(0)
+}
--- a/cmd/nebula-service/service.go
+++ b/cmd/nebula-service/service.go
@@ -0,0 +1,115 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/kardianos/service"
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/config"
+)
+
+var logger service.Logger
+
+type program struct {
+	configPath *string
+	configTest *bool
+	build      string
+	control    *nebula.Control
+}
+
+func (p *program) Start(s service.Service) error {
+	// Start should not block.
+	logger.Info("Nebula service starting.")
+
+	l := logrus.New()
+	HookLogger(l)
+
+	c := config.NewC(l)
+	err := c.Load(*p.configPath)
+	if err != nil {
+		return fmt.Errorf("failed to load config: %s", err)
+	}
+
+	p.control, err = nebula.Main(c, *p.configTest, Build, l, nil)
+	if err != nil {
+		return err
+	}
+
+	p.control.Start()
+	return nil
+}
+
+func (p *program) Stop(s service.Service) error {
+	logger.Info("Nebula service stopping.")
+	p.control.Stop()
+	return nil
+}
+
+func doService(configPath *string, configTest *bool, build string, serviceFlag *string) {
+	if *configPath == "" {
+		ex, err := os.Executable()
+		if err != nil {
+			panic(err)
+		}
+		*configPath = filepath.Dir(ex) + "/config.yaml"
+	}
+
+	svcConfig := &service.Config{
+		Name:        "Nebula",
+		DisplayName: "Nebula Network Service",
+		Description: "Nebula network connectivity daemon for encrypted communications",
+		Arguments:   []string{"-service", "run", "-config", *configPath},
+	}
+
+	prg := &program{
+		configPath: configPath,
+		configTest: configTest,
+		build:      build,
+	}
+
+	// Here are what the different loggers are doing:
+	// - `log` is the standard go log utility, meant to be used while the process is still attached to stdout/stderr
+	// - `logger` is the service log utility that may be attached to a special place depending on OS (Windows will have it attached to the event log)
+	// - above, in `Run` we create a `logrus.Logger` which is what nebula expects to use
+	s, err := service.New(prg, svcConfig)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	errs := make(chan error, 5)
+	logger, err = s.Logger(errs)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	go func() {
+		for {
+			err := <-errs
+			if err != nil {
+				// Route any errors from the system logger to stdout as a best effort to notice issues there
+				log.Print(err)
+			}
+		}
+	}()
+
+	switch *serviceFlag {
+	case "run":
+		err = s.Run()
+		if err != nil {
+			// Route any errors to the system logger
+			logger.Error(err)
+		}
+	default:
+		err := service.Control(s, *serviceFlag)
+		if err != nil {
+			log.Printf("Valid actions: %q\n", service.ControlAction)
+			log.Fatal(err)
+		}
+		return
+	}
+
+}
--- a/cmd/nebula/main.go
+++ b/cmd/nebula/main.go
@@ -5,7 +5,9 @@ import (
 	"fmt"
 	"os"

+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/config"
 )

 // A version string that can be set with
@@ -39,5 +41,31 @@ func main() {
 		os.Exit(1)
 	}

-	nebula.Main(*configPath, *configTest, Build)
+	l := logrus.New()
+	l.Out = os.Stdout
+
+	c := config.NewC(l)
+	err := c.Load(*configPath)
+	if err != nil {
+		fmt.Printf("failed to load config: %s", err)
+		os.Exit(1)
+	}
+
+	ctrl, err := nebula.Main(c, *configTest, Build, l, nil)
+
+	switch v := err.(type) {
+	case nebula.ContextualError:
+		v.Log(l)
+		os.Exit(1)
+	case error:
+		l.WithError(err).Error("Failed to start")
+		os.Exit(1)
+	}
+
+	if !*configTest {
+		ctrl.Start()
+		ctrl.ShutdownBlock()
+	}
+
+	os.Exit(0)
 }
--- a/config/config.go
+++ b/config/config.go
@@ -1,10 +1,9 @@
-package nebula
+package config

 import (
+	"context"
+	"errors"
 	"fmt"
-	"github.com/imdario/mergo"
-	"github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"os"
 	"os/signal"
@@ -14,32 +13,42 @@ import (
 	"strings"
 	"syscall"
 	"time"
+
+	"github.com/imdario/mergo"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 )

-type Config struct {
+type C struct {
 	path        string
 	files       []string
 	Settings    map[interface{}]interface{}
 	oldSettings map[interface{}]interface{}
-	callbacks   []func(*Config)
+	callbacks   []func(*C)
+	l           *logrus.Logger
 }

-func NewConfig() *Config {
-	return &Config{
+func NewC(l *logrus.Logger) *C {
+	return &C{
 		Settings: make(map[interface{}]interface{}),
+		l:        l,
 	}
 }

 // Load will find all yaml files within path and load them in lexical order
-func (c *Config) Load(path string) error {
+func (c *C) Load(path string) error {
 	c.path = path
 	c.files = make([]string, 0)

-	err := c.resolve(path)
+	err := c.resolve(path, true)
 	if err != nil {
 		return err
 	}

+	if len(c.files) == 0 {
+		return fmt.Errorf("no config files found at %s", path)
+	}
+
 	sort.Strings(c.files)

 	err = c.parse()
@@ -50,11 +59,18 @@ func (c *Config) Load(path string) error {
 	return nil
 }

+func (c *C) LoadString(raw string) error {
+	if raw == "" {
+		return errors.New("Empty configuration")
+	}
+	return c.parseRaw([]byte(raw))
+}
+
 // RegisterReloadCallback stores a function to be called when a config reload is triggered. The functions registered
 // here should decide if they need to make a change to the current process before making the change. HasChanged can be
 // used to help decide if a change is necessary.
 // These functions should return quickly or spawn their own go routine if they will take a while
-func (c *Config) RegisterReloadCallback(f func(*Config)) {
+func (c *C) RegisterReloadCallback(f func(*C)) {
 	c.callbacks = append(c.callbacks, f)
 }

@@ -63,7 +79,7 @@ func (c *Config) RegisterReloadCallback(f func(*Config)) {
 // If k is an empty string the entire config is tested.
 // It's important to note that this is very rudimentary and susceptible to configuration ordering issues indicating
 // there is change when there actually wasn't any.
-func (c *Config) HasChanged(k string) bool {
+func (c *C) HasChanged(k string) bool {
 	if c.oldSettings == nil {
 		return false
 	}
@@ -84,12 +100,12 @@ func (c *Config) HasChanged(k string) bool {

 	newVals, err := yaml.Marshal(nv)
 	if err != nil {
-		l.WithField("config_path", k).WithError(err).Error("Error while marshaling new config")
+		c.l.WithField("config_path", k).WithError(err).Error("Error while marshaling new config")
 	}

 	oldVals, err := yaml.Marshal(ov)
 	if err != nil {
-		l.WithField("config_path", k).WithError(err).Error("Error while marshaling old config")
+		c.l.WithField("config_path", k).WithError(err).Error("Error while marshaling old config")
 	}

 	return string(newVals) != string(oldVals)
@@ -97,19 +113,26 @@ func (c *Config) HasChanged(k string) bool {

 // CatchHUP will listen for the HUP signal in a go routine and reload all configs found in the
 // original path provided to Load. The old settings are shallow copied for change detection after the reload.
-func (c *Config) CatchHUP() {
+func (c *C) CatchHUP(ctx context.Context) {
 	ch := make(chan os.Signal, 1)
 	signal.Notify(ch, syscall.SIGHUP)

 	go func() {
-		for range ch {
-			l.Info("Caught HUP, reloading config")
+		for {
+			select {
+			case <-ctx.Done():
+				signal.Stop(ch)
+				close(ch)
+				return
+			case <-ch:
+				c.l.Info("Caught HUP, reloading config")
 				c.ReloadConfig()
 			}
+		}
 	}()
 }

-func (c *Config) ReloadConfig() {
+func (c *C) ReloadConfig() {
 	c.oldSettings = make(map[interface{}]interface{})
 	for k, v := range c.Settings {
 		c.oldSettings[k] = v
@@ -117,7 +140,7 @@ func (c *Config) ReloadConfig() {

 	err := c.Load(c.path)
 	if err != nil {
-		l.WithField("config_path", c.path).WithError(err).Error("Error occurred while reloading config")
+		c.l.WithField("config_path", c.path).WithError(err).Error("Error occurred while reloading config")
 		return
 	}

@@ -127,7 +150,7 @@ func (c *Config) ReloadConfig() {
 }

 // GetString will get the string for k or return the default d if not found or invalid
-func (c *Config) GetString(k, d string) string {
+func (c *C) GetString(k, d string) string {
 	r := c.Get(k)
 	if r == nil {
 		return d
@@ -137,7 +160,7 @@ func (c *Config) GetString(k, d string) string {
 }

 // GetStringSlice will get the slice of strings for k or return the default d if not found or invalid
-func (c *Config) GetStringSlice(k string, d []string) []string {
+func (c *C) GetStringSlice(k string, d []string) []string {
 	r := c.Get(k)
 	if r == nil {
 		return d
@@ -157,7 +180,7 @@ func (c *Config) GetStringSlice(k string, d []string) []string {
 }

 // GetMap will get the map for k or return the default d if not found or invalid
-func (c *Config) GetMap(k string, d map[interface{}]interface{}) map[interface{}]interface{} {
+func (c *C) GetMap(k string, d map[interface{}]interface{}) map[interface{}]interface{} {
 	r := c.Get(k)
 	if r == nil {
 		return d
@@ -172,7 +195,7 @@ func (c *Config) GetMap(k string, d map[interface{}]interface{}) map[interface{}
 }

 // GetInt will get the int for k or return the default d if not found or invalid
-func (c *Config) GetInt(k string, d int) int {
+func (c *C) GetInt(k string, d int) int {
 	r := c.GetString(k, strconv.Itoa(d))
 	v, err := strconv.Atoi(r)
 	if err != nil {
@@ -183,7 +206,7 @@ func (c *Config) GetInt(k string, d int) int {
 }

 // GetBool will get the bool for k or return the default d if not found or invalid
-func (c *Config) GetBool(k string, d bool) bool {
+func (c *C) GetBool(k string, d bool) bool {
 	r := strings.ToLower(c.GetString(k, fmt.Sprintf("%v", d)))
 	v, err := strconv.ParseBool(r)
 	if err != nil {
@@ -200,7 +223,7 @@ func (c *Config) GetBool(k string, d bool) bool {
 }

 // GetDuration will get the duration for k or return the default d if not found or invalid
-func (c *Config) GetDuration(k string, d time.Duration) time.Duration {
+func (c *C) GetDuration(k string, d time.Duration) time.Duration {
 	r := c.GetString(k, "")
 	v, err := time.ParseDuration(r)
 	if err != nil {
@@ -209,11 +232,15 @@ func (c *Config) GetDuration(k string, d time.Duration) time.Duration {
 	return v
 }

-func (c *Config) Get(k string) interface{} {
+func (c *C) Get(k string) interface{} {
 	return c.get(k, c.Settings)
 }

-func (c *Config) get(k string, v interface{}) interface{} {
+func (c *C) IsSet(k string) bool {
+	return c.get(k, c.Settings) != nil
+}
+
+func (c *C) get(k string, v interface{}) interface{} {
 	parts := strings.Split(k, ".")
 	for _, p := range parts {
 		m, ok := v.(map[interface{}]interface{})
@@ -230,14 +257,16 @@ func (c *Config) get(k string, v interface{}) interface{} {
 	return v
 }

-func (c *Config) resolve(path string) error {
+// direct signifies if this is the config path directly specified by the user,
+// versus a file/dir found by recursing into that path
+func (c *C) resolve(path string, direct bool) error {
 	i, err := os.Stat(path)
 	if err != nil {
 		return nil
 	}

 	if !i.IsDir() {
-		c.addFile(path)
+		c.addFile(path, direct)
 		return nil
 	}

@@ -247,7 +276,7 @@ func (c *Config) resolve(path string) error {
 	}

 	for _, p := range paths {
-		err := c.resolve(filepath.Join(path, p))
+		err := c.resolve(filepath.Join(path, p), false)
 		if err != nil {
 			return err
 		}
@@ -256,10 +285,10 @@ func (c *Config) resolve(path string) error {
 	return nil
 }

-func (c *Config) addFile(path string) error {
+func (c *C) addFile(path string, direct bool) error {
 	ext := filepath.Ext(path)

-	if ext != ".yaml" && ext != ".yml" {
+	if !direct && ext != ".yaml" && ext != ".yml" {
 		return nil
 	}

@@ -272,7 +301,19 @@ func (c *Config) addFile(path string) error {
 	return nil
 }

-func (c *Config) parse() error {
+func (c *C) parseRaw(b []byte) error {
+	var m map[interface{}]interface{}
+
+	err := yaml.Unmarshal(b, &m)
+	if err != nil {
+		return err
+	}
+
+	c.Settings = m
+	return nil
+}
+
+func (c *C) parse() error {
 	var m map[interface{}]interface{}

 	for _, path := range c.files {
@@ -315,24 +356,3 @@ func readDirNames(path string) ([]string, error) {
 	sort.Strings(paths)
 	return paths, nil
 }
-
-func configLogger(c *Config) error {
-	// set up our logging level
-	logLevel, err := logrus.ParseLevel(strings.ToLower(c.GetString("logging.level", "info")))
-	if err != nil {
-		return fmt.Errorf("%s; possible levels: %s", err, logrus.AllLevels)
-	}
-	l.SetLevel(logLevel)
-
-	logFormat := strings.ToLower(c.GetString("logging.format", "text"))
-	switch logFormat {
-	case "text":
-		l.Formatter = &logrus.TextFormatter{}
-	case "json":
-		l.Formatter = &logrus.JSONFormatter{}
-	default:
-		return fmt.Errorf("unknown log format `%s`. possible formats: %s", logFormat, []string{"text", "json"})
-	}
-
-	return nil
-}
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -1,23 +1,26 @@
-package nebula
+package config

 import (
-	"github.com/stretchr/testify/assert"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
+
+	"github.com/slackhq/nebula/util"
+	"github.com/stretchr/testify/assert"
 )

 func TestConfig_Load(t *testing.T) {
+	l := util.NewTestLogger()
 	dir, err := ioutil.TempDir("", "config-test")
 	// invalid yaml
-	c := NewConfig()
+	c := NewC(l)
 	ioutil.WriteFile(filepath.Join(dir, "01.yaml"), []byte(" invalid yaml"), 0644)
 	assert.EqualError(t, c.Load(dir), "yaml: unmarshal errors:\n  line 1: cannot unmarshal !!str `invalid...` into map[interface {}]interface {}")

 	// simple multi config merge
-	c = NewConfig()
+	c = NewC(l)
 	os.RemoveAll(dir)
 	os.Mkdir(dir, 0755)

@@ -39,8 +42,9 @@ func TestConfig_Load(t *testing.T) {
 }

 func TestConfig_Get(t *testing.T) {
+	l := util.NewTestLogger()
 	// test simple type
-	c := NewConfig()
+	c := NewC(l)
 	c.Settings["firewall"] = map[interface{}]interface{}{"outbound": "hi"}
 	assert.Equal(t, "hi", c.Get("firewall.outbound"))

@@ -54,13 +58,15 @@ func TestConfig_Get(t *testing.T) {
 }

 func TestConfig_GetStringSlice(t *testing.T) {
-	c := NewConfig()
+	l := util.NewTestLogger()
+	c := NewC(l)
 	c.Settings["slice"] = []interface{}{"one", "two"}
 	assert.Equal(t, []string{"one", "two"}, c.GetStringSlice("slice", []string{}))
 }

 func TestConfig_GetBool(t *testing.T) {
-	c := NewConfig()
+	l := util.NewTestLogger()
+	c := NewC(l)
 	c.Settings["bool"] = true
 	assert.Equal(t, true, c.GetBool("bool", false))

@@ -87,20 +93,21 @@ func TestConfig_GetBool(t *testing.T) {
 }

 func TestConfig_HasChanged(t *testing.T) {
+	l := util.NewTestLogger()
 	// No reload has occurred, return false
-	c := NewConfig()
+	c := NewC(l)
 	c.Settings["test"] = "hi"
 	assert.False(t, c.HasChanged(""))

 	// Test key change
-	c = NewConfig()
+	c = NewC(l)
 	c.Settings["test"] = "hi"
 	c.oldSettings = map[interface{}]interface{}{"test": "no"}
 	assert.True(t, c.HasChanged("test"))
 	assert.True(t, c.HasChanged(""))

 	// No key change
-	c = NewConfig()
+	c = NewC(l)
 	c.Settings["test"] = "hi"
 	c.oldSettings = map[interface{}]interface{}{"test": "hi"}
 	assert.False(t, c.HasChanged("test"))
@@ -108,12 +115,13 @@ func TestConfig_HasChanged(t *testing.T) {
 }

 func TestConfig_ReloadConfig(t *testing.T) {
+	l := util.NewTestLogger()
 	done := make(chan bool, 1)
 	dir, err := ioutil.TempDir("", "config-test")
 	assert.Nil(t, err)
 	ioutil.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n  inner: hi"), 0644)

-	c := NewConfig()
+	c := NewC(l)
 	assert.Nil(t, c.Load(dir))

 	assert.False(t, c.HasChanged("outer.inner"))
@@ -122,7 +130,7 @@ func TestConfig_ReloadConfig(t *testing.T) {

 	ioutil.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n  inner: ho"), 0644)

-	c.RegisterReloadCallback(func(c *Config) {
+	c.RegisterReloadCallback(func(c *C) {
 		done <- true
 	})

--- a/connection_manager.go
+++ b/connection_manager.go
@@ -1,9 +1,13 @@
 package nebula

 import (
-	"github.com/sirupsen/logrus"
+	"context"
 	"sync"
 	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/iputil"
 )

 // TODO: incount and outcount are intended as a shortcut to locking the mutexes for every single packet
@@ -11,47 +15,49 @@ import (

 type connectionManager struct {
 	hostMap      *HostMap
-	in           map[uint32]struct{}
+	in           map[iputil.VpnIp]struct{}
 	inLock       *sync.RWMutex
 	inCount      int
-	out          map[uint32]struct{}
+	out          map[iputil.VpnIp]struct{}
 	outLock      *sync.RWMutex
 	outCount     int
 	TrafficTimer *SystemTimerWheel
 	intf         *Interface

-	pendingDeletion      map[uint32]int
+	pendingDeletion      map[iputil.VpnIp]int
 	pendingDeletionLock  *sync.RWMutex
 	pendingDeletionTimer *SystemTimerWheel

 	checkInterval           int
 	pendingDeletionInterval int

+	l *logrus.Logger
 	// I wanted to call one matLock
 }

-func newConnectionManager(intf *Interface, checkInterval, pendingDeletionInterval int) *connectionManager {
+func newConnectionManager(ctx context.Context, l *logrus.Logger, intf *Interface, checkInterval, pendingDeletionInterval int) *connectionManager {
 	nc := &connectionManager{
 		hostMap:                 intf.hostMap,
-		in:                      make(map[uint32]struct{}),
+		in:                      make(map[iputil.VpnIp]struct{}),
 		inLock:                  &sync.RWMutex{},
 		inCount:                 0,
-		out:                     make(map[uint32]struct{}),
+		out:                     make(map[iputil.VpnIp]struct{}),
 		outLock:                 &sync.RWMutex{},
 		outCount:                0,
 		TrafficTimer:            NewSystemTimerWheel(time.Millisecond*500, time.Second*60),
 		intf:                    intf,
-		pendingDeletion:         make(map[uint32]int),
+		pendingDeletion:         make(map[iputil.VpnIp]int),
 		pendingDeletionLock:     &sync.RWMutex{},
 		pendingDeletionTimer:    NewSystemTimerWheel(time.Millisecond*500, time.Second*60),
 		checkInterval:           checkInterval,
 		pendingDeletionInterval: pendingDeletionInterval,
+		l:                       l,
 	}
-	nc.Start()
+	nc.Start(ctx)
 	return nc
 }

-func (n *connectionManager) In(ip uint32) {
+func (n *connectionManager) In(ip iputil.VpnIp) {
 	n.inLock.RLock()
 	// If this already exists, return
 	if _, ok := n.in[ip]; ok {
@@ -64,7 +70,7 @@ func (n *connectionManager) In(ip uint32) {
 	n.inLock.Unlock()
 }

-func (n *connectionManager) Out(ip uint32) {
+func (n *connectionManager) Out(ip iputil.VpnIp) {
 	n.outLock.RLock()
 	// If this already exists, return
 	if _, ok := n.out[ip]; ok {
@@ -83,9 +89,9 @@ func (n *connectionManager) Out(ip uint32) {
 	n.outLock.Unlock()
 }

-func (n *connectionManager) CheckIn(vpnIP uint32) bool {
+func (n *connectionManager) CheckIn(vpnIp iputil.VpnIp) bool {
 	n.inLock.RLock()
-	if _, ok := n.in[vpnIP]; ok {
+	if _, ok := n.in[vpnIp]; ok {
 		n.inLock.RUnlock()
 		return true
 	}
@@ -93,7 +99,7 @@ func (n *connectionManager) CheckIn(vpnIP uint32) bool {
 	return false
 }

-func (n *connectionManager) ClearIP(ip uint32) {
+func (n *connectionManager) ClearIP(ip iputil.VpnIp) {
 	n.inLock.Lock()
 	n.outLock.Lock()
 	delete(n.in, ip)
@@ -102,13 +108,13 @@ func (n *connectionManager) ClearIP(ip uint32) {
 	n.outLock.Unlock()
 }

-func (n *connectionManager) ClearPendingDeletion(ip uint32) {
+func (n *connectionManager) ClearPendingDeletion(ip iputil.VpnIp) {
 	n.pendingDeletionLock.Lock()
 	delete(n.pendingDeletion, ip)
 	n.pendingDeletionLock.Unlock()
 }

-func (n *connectionManager) AddPendingDeletion(ip uint32) {
+func (n *connectionManager) AddPendingDeletion(ip iputil.VpnIp) {
 	n.pendingDeletionLock.Lock()
 	if _, ok := n.pendingDeletion[ip]; ok {
 		n.pendingDeletion[ip] += 1
@@ -119,7 +125,7 @@ func (n *connectionManager) AddPendingDeletion(ip uint32) {
 	n.pendingDeletionLock.Unlock()
 }

-func (n *connectionManager) checkPendingDeletion(ip uint32) bool {
+func (n *connectionManager) checkPendingDeletion(ip iputil.VpnIp) bool {
 	n.pendingDeletionLock.RLock()
 	if _, ok := n.pendingDeletion[ip]; ok {

@@ -130,24 +136,34 @@ func (n *connectionManager) checkPendingDeletion(ip uint32) bool {
 	return false
 }

-func (n *connectionManager) AddTrafficWatch(vpnIP uint32, seconds int) {
-	n.TrafficTimer.Add(vpnIP, time.Second*time.Duration(seconds))
+func (n *connectionManager) AddTrafficWatch(vpnIp iputil.VpnIp, seconds int) {
+	n.TrafficTimer.Add(vpnIp, time.Second*time.Duration(seconds))
 }

-func (n *connectionManager) Start() {
-	go n.Run()
+func (n *connectionManager) Start(ctx context.Context) {
+	go n.Run(ctx)
 }

-func (n *connectionManager) Run() {
-	clockSource := time.Tick(500 * time.Millisecond)
+func (n *connectionManager) Run(ctx context.Context) {
+	clockSource := time.NewTicker(500 * time.Millisecond)
+	defer clockSource.Stop()

-	for now := range clockSource {
-		n.HandleMonitorTick(now)
+	p := []byte("")
+	nb := make([]byte, 12, 12)
+	out := make([]byte, mtu)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case now := <-clockSource.C:
+			n.HandleMonitorTick(now, p, nb, out)
 			n.HandleDeletionTick(now)
 		}
 	}
+}

-func (n *connectionManager) HandleMonitorTick(now time.Time) {
+func (n *connectionManager) HandleMonitorTick(now time.Time, p, nb, out []byte) {
 	n.TrafficTimer.advance(now)
 	for {
 		ep := n.TrafficTimer.Purge()
@@ -155,44 +171,51 @@ func (n *connectionManager) HandleMonitorTick(now time.Time) {
 			break
 		}

-		vpnIP := ep.(uint32)
+		vpnIp := ep.(iputil.VpnIp)

 		// Check for traffic coming back in from this host.
-		traf := n.CheckIn(vpnIP)
+		traf := n.CheckIn(vpnIp)

-		// If we saw incoming packets from this ip, just return
+		hostinfo, err := n.hostMap.QueryVpnIp(vpnIp)
+		if err != nil {
+			n.l.Debugf("Not found in hostmap: %s", vpnIp)
+
+			if !n.intf.disconnectInvalid {
+				n.ClearIP(vpnIp)
+				n.ClearPendingDeletion(vpnIp)
+				continue
+			}
+		}
+
+		if n.handleInvalidCertificate(now, vpnIp, hostinfo) {
+			continue
+		}
+
+		// If we saw an incoming packets from this ip and peer's certificate is not
+		// expired, just ignore.
 		if traf {
-			if l.Level >= logrus.DebugLevel {
-				l.WithField("vpnIp", IntIp(vpnIP)).
+			if n.l.Level >= logrus.DebugLevel {
+				n.l.WithField("vpnIp", vpnIp).
 					WithField("tunnelCheck", m{"state": "alive", "method": "passive"}).
 					Debug("Tunnel status")
 			}
-			n.ClearIP(vpnIP)
-			n.ClearPendingDeletion(vpnIP)
+			n.ClearIP(vpnIp)
+			n.ClearPendingDeletion(vpnIp)
 			continue
 		}

-		// If we didn't we may need to probe or destroy the conn
-		hostinfo, err := n.hostMap.QueryVpnIP(vpnIP)
-		if err != nil {
-			l.Debugf("Not found in hostmap: %s", IntIp(vpnIP))
-			n.ClearIP(vpnIP)
-			n.ClearPendingDeletion(vpnIP)
-			continue
-		}
-
-		l.WithField("vpnIp", IntIp(vpnIP)).
+		hostinfo.logger(n.l).
 			WithField("tunnelCheck", m{"state": "testing", "method": "active"}).
 			Debug("Tunnel status")

 		if hostinfo != nil && hostinfo.ConnectionState != nil {
 			// Send a test packet to trigger an authenticated tunnel test, this should suss out any lingering tunnel issues
-			n.intf.SendMessageToVpnIp(test, testRequest, vpnIP, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+			n.intf.SendMessageToVpnIp(header.Test, header.TestRequest, vpnIp, p, nb, out)

 		} else {
-			l.Debugf("Hostinfo sadness: %s", IntIp(vpnIP))
+			hostinfo.logger(n.l).Debugf("Hostinfo sadness: %s", vpnIp)
 		}
-		n.AddPendingDeletion(vpnIP)
+		n.AddPendingDeletion(vpnIp)
 	}

 }
@@ -205,49 +228,88 @@ func (n *connectionManager) HandleDeletionTick(now time.Time) {
 			break
 		}

-		vpnIP := ep.(uint32)
+		vpnIp := ep.(iputil.VpnIp)

-		// If we saw incoming packets from this ip, just return
-		traf := n.CheckIn(vpnIP)
-		if traf {
-			l.WithField("vpnIp", IntIp(vpnIP)).
-				WithField("tunnelCheck", m{"state": "alive", "method": "active"}).
-				Debug("Tunnel status")
-			n.ClearIP(vpnIP)
-			n.ClearPendingDeletion(vpnIP)
+		hostinfo, err := n.hostMap.QueryVpnIp(vpnIp)
+		if err != nil {
+			n.l.Debugf("Not found in hostmap: %s", vpnIp)
+
+			if !n.intf.disconnectInvalid {
+				n.ClearIP(vpnIp)
+				n.ClearPendingDeletion(vpnIp)
+				continue
+			}
+		}
+
+		if n.handleInvalidCertificate(now, vpnIp, hostinfo) {
 			continue
 		}

-		hostinfo, err := n.hostMap.QueryVpnIP(vpnIP)
-		if err != nil {
-			n.ClearIP(vpnIP)
-			n.ClearPendingDeletion(vpnIP)
-			l.Debugf("Not found in hostmap: %s", IntIp(vpnIP))
+		// If we saw an incoming packets from this ip and peer's certificate is not
+		// expired, just ignore.
+		traf := n.CheckIn(vpnIp)
+		if traf {
+			n.l.WithField("vpnIp", vpnIp).
+				WithField("tunnelCheck", m{"state": "alive", "method": "active"}).
+				Debug("Tunnel status")
+
+			n.ClearIP(vpnIp)
+			n.ClearPendingDeletion(vpnIp)
 			continue
 		}

 		// If it comes around on deletion wheel and hasn't resolved itself, delete
-		if n.checkPendingDeletion(vpnIP) {
+		if n.checkPendingDeletion(vpnIp) {
 			cn := ""
 			if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.peerCert != nil {
 				cn = hostinfo.ConnectionState.peerCert.Details.Name
 			}
-			l.WithField("vpnIp", IntIp(vpnIP)).
+			hostinfo.logger(n.l).
 				WithField("tunnelCheck", m{"state": "dead", "method": "active"}).
 				WithField("certName", cn).
 				Info("Tunnel status")

-			n.ClearIP(vpnIP)
-			n.ClearPendingDeletion(vpnIP)
+			n.ClearIP(vpnIp)
+			n.ClearPendingDeletion(vpnIp)
 			// TODO: This is only here to let tests work. Should do proper mocking
 			if n.intf.lightHouse != nil {
-				n.intf.lightHouse.DeleteVpnIP(vpnIP)
+				n.intf.lightHouse.DeleteVpnIp(vpnIp)
 			}
-			n.hostMap.DeleteVpnIP(vpnIP)
-			n.hostMap.DeleteIndex(hostinfo.localIndexId)
+			n.hostMap.DeleteHostInfo(hostinfo)
 		} else {
-			n.ClearIP(vpnIP)
-			n.ClearPendingDeletion(vpnIP)
+			n.ClearIP(vpnIp)
+			n.ClearPendingDeletion(vpnIp)
 		}
 	}
 }
+
+// handleInvalidCertificates will destroy a tunnel if pki.disconnect_invalid is true and the certificate is no longer valid
+func (n *connectionManager) handleInvalidCertificate(now time.Time, vpnIp iputil.VpnIp, hostinfo *HostInfo) bool {
+	if !n.intf.disconnectInvalid {
+		return false
+	}
+
+	remoteCert := hostinfo.GetCert()
+	if remoteCert == nil {
+		return false
+	}
+
+	valid, err := remoteCert.Verify(now, n.intf.caPool)
+	if valid {
+		return false
+	}
+
+	fingerprint, _ := remoteCert.Sha256Sum()
+	n.l.WithField("vpnIp", vpnIp).WithError(err).
+		WithField("certName", remoteCert.Details.Name).
+		WithField("fingerprint", fingerprint).
+		Info("Remote certificate is no longer valid, tearing down the tunnel")
+
+	// Inform the remote and close the tunnel locally
+	n.intf.sendCloseTunnel(hostinfo)
+	n.intf.closeTunnel(hostinfo, false)
+
+	n.ClearIP(vpnIp)
+	n.ClearPendingDeletion(vpnIp)
+	return true
+}
--- a/connection_manager_test.go
+++ b/connection_manager_test.go
@@ -1,25 +1,33 @@
 package nebula

 import (
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
 	"net"
 	"testing"
 	"time"

 	"github.com/flynn/noise"
-	"github.com/stretchr/testify/assert"
 	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
+	"github.com/slackhq/nebula/util"
+	"github.com/stretchr/testify/assert"
 )

-var vpnIP uint32 = uint32(12341234)
+var vpnIp iputil.VpnIp

 func Test_NewConnectionManagerTest(t *testing.T) {
+	l := util.NewTestLogger()
 	//_, tuncidr, _ := net.ParseCIDR("1.1.1.1/24")
 	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
+	vpnIp = iputil.Ip2VpnIp(net.ParseIP("172.1.1.2"))
 	preferredRanges := []*net.IPNet{localrange}

 	// Very incomplete mock objects
-	hostMap := NewHostMap("test", vpncidr, preferredRanges)
+	hostMap := NewHostMap(l, "test", vpncidr, preferredRanges)
 	cs := &CertState{
 		rawCertificate:      []byte{},
 		privateKey:          []byte{},
@@ -27,62 +35,68 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 		rawCertificateNoKey: []byte{},
 	}

-	lh := NewLightHouse(false, 0, []string{}, 1000, 0, &udpConn{}, false)
+	lh := NewLightHouse(l, false, &net.IPNet{IP: net.IP{0, 0, 0, 0}, Mask: net.IPMask{0, 0, 0, 0}}, []iputil.VpnIp{}, 1000, 0, &udp.Conn{}, false, 1, false)
 	ifce := &Interface{
 		hostMap:          hostMap,
 		inside:           &Tun{},
-		outside:          &udpConn{},
+		outside:          &udp.Conn{},
 		certState:        cs,
 		firewall:         &Firewall{},
 		lightHouse:       lh,
-		handshakeManager: NewHandshakeManager(vpncidr, preferredRanges, hostMap, lh, &udpConn{}),
+		handshakeManager: NewHandshakeManager(l, vpncidr, preferredRanges, hostMap, lh, &udp.Conn{}, defaultHandshakeConfig),
+		l:                l,
 	}
 	now := time.Now()

 	// Create manager
-	nc := newConnectionManager(ifce, 5, 10)
-	nc.HandleMonitorTick(now)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	nc := newConnectionManager(ctx, l, ifce, 5, 10)
+	p := []byte("")
+	nb := make([]byte, 12, 12)
+	out := make([]byte, mtu)
+	nc.HandleMonitorTick(now, p, nb, out)
 	// Add an ip we have established a connection w/ to hostmap
-	hostinfo := nc.hostMap.AddVpnIP(vpnIP)
+	hostinfo := nc.hostMap.AddVpnIp(vpnIp)
 	hostinfo.ConnectionState = &ConnectionState{
 		certState: cs,
 		H:         &noise.HandshakeState{},
-		messageCounter: new(uint64),
 	}

-	// We saw traffic out to vpnIP
-	nc.Out(vpnIP)
-	assert.NotContains(t, nc.pendingDeletion, vpnIP)
-	assert.Contains(t, nc.hostMap.Hosts, vpnIP)
+	// We saw traffic out to vpnIp
+	nc.Out(vpnIp)
+	assert.NotContains(t, nc.pendingDeletion, vpnIp)
+	assert.Contains(t, nc.hostMap.Hosts, vpnIp)
 	// Move ahead 5s. Nothing should happen
 	next_tick := now.Add(5 * time.Second)
-	nc.HandleMonitorTick(next_tick)
+	nc.HandleMonitorTick(next_tick, p, nb, out)
 	nc.HandleDeletionTick(next_tick)
 	// Move ahead 6s. We haven't heard back
 	next_tick = now.Add(6 * time.Second)
-	nc.HandleMonitorTick(next_tick)
+	nc.HandleMonitorTick(next_tick, p, nb, out)
 	nc.HandleDeletionTick(next_tick)
 	// This host should now be up for deletion
-	assert.Contains(t, nc.pendingDeletion, vpnIP)
-	assert.Contains(t, nc.hostMap.Hosts, vpnIP)
+	assert.Contains(t, nc.pendingDeletion, vpnIp)
+	assert.Contains(t, nc.hostMap.Hosts, vpnIp)
 	// Move ahead some more
 	next_tick = now.Add(45 * time.Second)
-	nc.HandleMonitorTick(next_tick)
+	nc.HandleMonitorTick(next_tick, p, nb, out)
 	nc.HandleDeletionTick(next_tick)
 	// The host should be evicted
-	assert.NotContains(t, nc.pendingDeletion, vpnIP)
-	assert.NotContains(t, nc.hostMap.Hosts, vpnIP)
+	assert.NotContains(t, nc.pendingDeletion, vpnIp)
+	assert.NotContains(t, nc.hostMap.Hosts, vpnIp)

 }

 func Test_NewConnectionManagerTest2(t *testing.T) {
+	l := util.NewTestLogger()
 	//_, tuncidr, _ := net.ParseCIDR("1.1.1.1/24")
 	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
 	preferredRanges := []*net.IPNet{localrange}

 	// Very incomplete mock objects
-	hostMap := NewHostMap("test", vpncidr, preferredRanges)
+	hostMap := NewHostMap(l, "test", vpncidr, preferredRanges)
 	cs := &CertState{
 		rawCertificate:      []byte{},
 		privateKey:          []byte{},
@@ -90,52 +104,152 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 		rawCertificateNoKey: []byte{},
 	}

-	lh := NewLightHouse(false, 0, []string{}, 1000, 0, &udpConn{}, false)
+	lh := NewLightHouse(l, false, &net.IPNet{IP: net.IP{0, 0, 0, 0}, Mask: net.IPMask{0, 0, 0, 0}}, []iputil.VpnIp{}, 1000, 0, &udp.Conn{}, false, 1, false)
 	ifce := &Interface{
 		hostMap:          hostMap,
 		inside:           &Tun{},
-		outside:          &udpConn{},
+		outside:          &udp.Conn{},
 		certState:        cs,
 		firewall:         &Firewall{},
 		lightHouse:       lh,
-		handshakeManager: NewHandshakeManager(vpncidr, preferredRanges, hostMap, lh, &udpConn{}),
+		handshakeManager: NewHandshakeManager(l, vpncidr, preferredRanges, hostMap, lh, &udp.Conn{}, defaultHandshakeConfig),
+		l:                l,
 	}
 	now := time.Now()

 	// Create manager
-	nc := newConnectionManager(ifce, 5, 10)
-	nc.HandleMonitorTick(now)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	nc := newConnectionManager(ctx, l, ifce, 5, 10)
+	p := []byte("")
+	nb := make([]byte, 12, 12)
+	out := make([]byte, mtu)
+	nc.HandleMonitorTick(now, p, nb, out)
 	// Add an ip we have established a connection w/ to hostmap
-	hostinfo := nc.hostMap.AddVpnIP(vpnIP)
+	hostinfo := nc.hostMap.AddVpnIp(vpnIp)
 	hostinfo.ConnectionState = &ConnectionState{
 		certState: cs,
 		H:         &noise.HandshakeState{},
-		messageCounter: new(uint64),
 	}

-	// We saw traffic out to vpnIP
-	nc.Out(vpnIP)
-	assert.NotContains(t, nc.pendingDeletion, vpnIP)
-	assert.Contains(t, nc.hostMap.Hosts, vpnIP)
+	// We saw traffic out to vpnIp
+	nc.Out(vpnIp)
+	assert.NotContains(t, nc.pendingDeletion, vpnIp)
+	assert.Contains(t, nc.hostMap.Hosts, vpnIp)
 	// Move ahead 5s. Nothing should happen
 	next_tick := now.Add(5 * time.Second)
-	nc.HandleMonitorTick(next_tick)
+	nc.HandleMonitorTick(next_tick, p, nb, out)
 	nc.HandleDeletionTick(next_tick)
 	// Move ahead 6s. We haven't heard back
 	next_tick = now.Add(6 * time.Second)
-	nc.HandleMonitorTick(next_tick)
+	nc.HandleMonitorTick(next_tick, p, nb, out)
 	nc.HandleDeletionTick(next_tick)
 	// This host should now be up for deletion
-	assert.Contains(t, nc.pendingDeletion, vpnIP)
-	assert.Contains(t, nc.hostMap.Hosts, vpnIP)
+	assert.Contains(t, nc.pendingDeletion, vpnIp)
+	assert.Contains(t, nc.hostMap.Hosts, vpnIp)
 	// We heard back this time
-	nc.In(vpnIP)
+	nc.In(vpnIp)
 	// Move ahead some more
 	next_tick = now.Add(45 * time.Second)
-	nc.HandleMonitorTick(next_tick)
+	nc.HandleMonitorTick(next_tick, p, nb, out)
 	nc.HandleDeletionTick(next_tick)
 	// The host should be evicted
-	assert.NotContains(t, nc.pendingDeletion, vpnIP)
-	assert.Contains(t, nc.hostMap.Hosts, vpnIP)
+	assert.NotContains(t, nc.pendingDeletion, vpnIp)
+	assert.Contains(t, nc.hostMap.Hosts, vpnIp)

 }
+
+// Check if we can disconnect the peer.
+// Validate if the peer's certificate is invalid (expired, etc.)
+// Disconnect only if disconnectInvalid: true is set.
+func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
+	now := time.Now()
+	l := util.NewTestLogger()
+	ipNet := net.IPNet{
+		IP:   net.IPv4(172, 1, 1, 2),
+		Mask: net.IPMask{255, 255, 255, 0},
+	}
+	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
+	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
+	preferredRanges := []*net.IPNet{localrange}
+	hostMap := NewHostMap(l, "test", vpncidr, preferredRanges)
+
+	// Generate keys for CA and peer's cert.
+	pubCA, privCA, _ := ed25519.GenerateKey(rand.Reader)
+	caCert := cert.NebulaCertificate{
+		Details: cert.NebulaCertificateDetails{
+			Name:      "ca",
+			NotBefore: now,
+			NotAfter:  now.Add(1 * time.Hour),
+			IsCA:      true,
+			PublicKey: pubCA,
+		},
+	}
+	caCert.Sign(privCA)
+	ncp := &cert.NebulaCAPool{
+		CAs: cert.NewCAPool().CAs,
+	}
+	ncp.CAs["ca"] = &caCert
+
+	pubCrt, _, _ := ed25519.GenerateKey(rand.Reader)
+	peerCert := cert.NebulaCertificate{
+		Details: cert.NebulaCertificateDetails{
+			Name:      "host",
+			Ips:       []*net.IPNet{&ipNet},
+			Subnets:   []*net.IPNet{},
+			NotBefore: now,
+			NotAfter:  now.Add(60 * time.Second),
+			PublicKey: pubCrt,
+			IsCA:      false,
+			Issuer:    "ca",
+		},
+	}
+	peerCert.Sign(privCA)
+
+	cs := &CertState{
+		rawCertificate:      []byte{},
+		privateKey:          []byte{},
+		certificate:         &cert.NebulaCertificate{},
+		rawCertificateNoKey: []byte{},
+	}
+
+	lh := NewLightHouse(l, false, &net.IPNet{IP: net.IP{0, 0, 0, 0}, Mask: net.IPMask{0, 0, 0, 0}}, []iputil.VpnIp{}, 1000, 0, &udp.Conn{}, false, 1, false)
+	ifce := &Interface{
+		hostMap:           hostMap,
+		inside:            &Tun{},
+		outside:           &udp.Conn{},
+		certState:         cs,
+		firewall:          &Firewall{},
+		lightHouse:        lh,
+		handshakeManager:  NewHandshakeManager(l, vpncidr, preferredRanges, hostMap, lh, &udp.Conn{}, defaultHandshakeConfig),
+		l:                 l,
+		disconnectInvalid: true,
+		caPool:            ncp,
+	}
+
+	// Create manager
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	nc := newConnectionManager(ctx, l, ifce, 5, 10)
+	ifce.connectionManager = nc
+	hostinfo := nc.hostMap.AddVpnIp(vpnIp)
+	hostinfo.ConnectionState = &ConnectionState{
+		certState: cs,
+		peerCert:  &peerCert,
+		H:         &noise.HandshakeState{},
+	}
+
+	// Move ahead 45s.
+	// Check if to disconnect with invalid certificate.
+	// Should be alive.
+	nextTick := now.Add(45 * time.Second)
+	destroyed := nc.handleInvalidCertificate(nextTick, vpnIp, hostinfo)
+	assert.False(t, destroyed)
+
+	// Move ahead 61s.
+	// Check if to disconnect with invalid certificate.
+	// Should be disconnected.
+	nextTick = now.Add(61 * time.Second)
+	destroyed = nc.handleInvalidCertificate(nextTick, vpnIp, hostinfo)
+	assert.True(t, destroyed)
+}
--- a/connection_state.go
+++ b/connection_state.go
@@ -4,8 +4,10 @@ import (
 	"crypto/rand"
 	"encoding/json"
 	"sync"
+	"sync/atomic"

 	"github.com/flynn/noise"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 )

@@ -18,14 +20,14 @@ type ConnectionState struct {
 	certState            *CertState
 	peerCert             *cert.NebulaCertificate
 	initiator            bool
-	messageCounter *uint64
+	atomicMessageCounter uint64
 	window               *Bits
 	queueLock            sync.Mutex
 	writeLock            sync.Mutex
 	ready                bool
 }

-func (f *Interface) newConnectionState(initiator bool, pattern noise.HandshakePattern, psk []byte, pskStage int) *ConnectionState {
+func (f *Interface) newConnectionState(l *logrus.Logger, initiator bool, p []byte) (*ConnectionState, error) {
 	cs := noise.NewCipherSuite(noise.DH25519, noise.CipherAESGCM, noise.HashSHA256)
 	if f.cipher == "chachapoly" {
 		cs = noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
@@ -36,19 +38,20 @@ func (f *Interface) newConnectionState(initiator bool, pattern noise.HandshakePa

 	b := NewBits(ReplayWindow)
 	// Clear out bit 0, we never transmit it and we don't want it showing as packet loss
-	b.Update(0)
+	b.Update(l, 0)

 	hs, err := noise.NewHandshakeState(noise.Config{
 		CipherSuite:           cs,
 		Random:                rand.Reader,
-		Pattern:               pattern,
+		Pattern:               noise.HandshakeIX,
 		Initiator:             initiator,
 		StaticKeypair:         static,
-		PresharedKey:          psk,
-		PresharedKeyPlacement: pskStage,
+		PresharedKey:          p,
+		PresharedKeyPlacement: 0,
 	})
+
 	if err != nil {
-		return nil
+		return nil, err
 	}

 	// The queue and ready params prevent a counter race that would happen when
@@ -59,17 +62,16 @@ func (f *Interface) newConnectionState(initiator bool, pattern noise.HandshakePa
 		window:    b,
 		ready:     false,
 		certState: curCertState,
-		messageCounter: new(uint64),
 	}

-	return ci
+	return ci, nil
 }

 func (cs *ConnectionState) MarshalJSON() ([]byte, error) {
 	return json.Marshal(m{
 		"certificate":     cs.peerCert,
 		"initiator":       cs.initiator,
-		"message_counter": cs.messageCounter,
+		"message_counter": atomic.LoadUint64(&cs.atomicMessageCounter),
 		"ready":           cs.ready,
 	})
 }
--- a/control.go
+++ b/control.go
@@ -0,0 +1,215 @@
+package nebula
+
+import (
+	"context"
+	"net"
+	"os"
+	"os/signal"
+	"sync/atomic"
+	"syscall"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
+)
+
+// Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching
+// core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc
+
+type Control struct {
+	f          *Interface
+	l          *logrus.Logger
+	cancel     context.CancelFunc
+	sshStart   func()
+	statsStart func()
+	dnsStart   func()
+}
+
+type ControlHostInfo struct {
+	VpnIp          net.IP                  `json:"vpnIp"`
+	LocalIndex     uint32                  `json:"localIndex"`
+	RemoteIndex    uint32                  `json:"remoteIndex"`
+	RemoteAddrs    []*udp.Addr             `json:"remoteAddrs"`
+	CachedPackets  int                     `json:"cachedPackets"`
+	Cert           *cert.NebulaCertificate `json:"cert"`
+	MessageCounter uint64                  `json:"messageCounter"`
+	CurrentRemote  *udp.Addr               `json:"currentRemote"`
+}
+
+// Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock()
+func (c *Control) Start() {
+	// Activate the interface
+	c.f.activate()
+
+	// Call all the delayed funcs that waited patiently for the interface to be created.
+	if c.sshStart != nil {
+		go c.sshStart()
+	}
+	if c.statsStart != nil {
+		go c.statsStart()
+	}
+	if c.dnsStart != nil {
+		go c.dnsStart()
+	}
+
+	// Start reading packets.
+	c.f.run()
+}
+
+// Stop signals nebula to shutdown, returns after the shutdown is complete
+func (c *Control) Stop() {
+	//TODO: stop tun and udp routines, the lock on hostMap effectively does that though
+	c.CloseAllTunnels(false)
+	c.cancel()
+	c.l.Info("Goodbye")
+}
+
+// ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled
+func (c *Control) ShutdownBlock() {
+	sigChan := make(chan os.Signal)
+	signal.Notify(sigChan, syscall.SIGTERM)
+	signal.Notify(sigChan, syscall.SIGINT)
+
+	rawSig := <-sigChan
+	sig := rawSig.String()
+	c.l.WithField("signal", sig).Info("Caught signal, shutting down")
+	c.Stop()
+}
+
+// RebindUDPServer asks the UDP listener to rebind it's listener. Mainly used on mobile clients when interfaces change
+func (c *Control) RebindUDPServer() {
+	_ = c.f.outside.Rebind()
+
+	// Trigger a lighthouse update, useful for mobile clients that should have an update interval of 0
+	c.f.lightHouse.SendUpdate(c.f)
+
+	// Let the main interface know that we rebound so that underlying tunnels know to trigger punches from their remotes
+	c.f.rebindCount++
+}
+
+// ListHostmap returns details about the actual or pending (handshaking) hostmap
+func (c *Control) ListHostmap(pendingMap bool) []ControlHostInfo {
+	if pendingMap {
+		return listHostMap(c.f.handshakeManager.pendingHostMap)
+	} else {
+		return listHostMap(c.f.hostMap)
+	}
+}
+
+// GetHostInfoByVpnIp returns a single tunnels hostInfo, or nil if not found
+func (c *Control) GetHostInfoByVpnIp(vpnIp iputil.VpnIp, pending bool) *ControlHostInfo {
+	var hm *HostMap
+	if pending {
+		hm = c.f.handshakeManager.pendingHostMap
+	} else {
+		hm = c.f.hostMap
+	}
+
+	h, err := hm.QueryVpnIp(vpnIp)
+	if err != nil {
+		return nil
+	}
+
+	ch := copyHostInfo(h, c.f.hostMap.preferredRanges)
+	return &ch
+}
+
+// SetRemoteForTunnel forces a tunnel to use a specific remote
+func (c *Control) SetRemoteForTunnel(vpnIp iputil.VpnIp, addr udp.Addr) *ControlHostInfo {
+	hostInfo, err := c.f.hostMap.QueryVpnIp(vpnIp)
+	if err != nil {
+		return nil
+	}
+
+	hostInfo.SetRemote(addr.Copy())
+	ch := copyHostInfo(hostInfo, c.f.hostMap.preferredRanges)
+	return &ch
+}
+
+// CloseTunnel closes a fully established tunnel. If localOnly is false it will notify the remote end as well.
+func (c *Control) CloseTunnel(vpnIp iputil.VpnIp, localOnly bool) bool {
+	hostInfo, err := c.f.hostMap.QueryVpnIp(vpnIp)
+	if err != nil {
+		return false
+	}
+
+	if !localOnly {
+		c.f.send(
+			header.CloseTunnel,
+			0,
+			hostInfo.ConnectionState,
+			hostInfo,
+			hostInfo.remote,
+			[]byte{},
+			make([]byte, 12, 12),
+			make([]byte, mtu),
+		)
+	}
+
+	c.f.closeTunnel(hostInfo, false)
+	return true
+}
+
+// CloseAllTunnels is just like CloseTunnel except it goes through and shuts them all down, optionally you can avoid shutting down lighthouse tunnels
+// the int returned is a count of tunnels closed
+func (c *Control) CloseAllTunnels(excludeLighthouses bool) (closed int) {
+	//TODO: this is probably better as a function in ConnectionManager or HostMap directly
+	c.f.hostMap.Lock()
+	for _, h := range c.f.hostMap.Hosts {
+		if excludeLighthouses {
+			if _, ok := c.f.lightHouse.lighthouses[h.vpnIp]; ok {
+				continue
+			}
+		}
+
+		if h.ConnectionState.ready {
+			c.f.send(header.CloseTunnel, 0, h.ConnectionState, h, h.remote, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
+			c.f.closeTunnel(h, true)
+
+			c.l.WithField("vpnIp", h.vpnIp).WithField("udpAddr", h.remote).
+				Debug("Sending close tunnel message")
+			closed++
+		}
+	}
+	c.f.hostMap.Unlock()
+	return
+}
+
+func copyHostInfo(h *HostInfo, preferredRanges []*net.IPNet) ControlHostInfo {
+	chi := ControlHostInfo{
+		VpnIp:         h.vpnIp.ToIP(),
+		LocalIndex:    h.localIndexId,
+		RemoteIndex:   h.remoteIndexId,
+		RemoteAddrs:   h.remotes.CopyAddrs(preferredRanges),
+		CachedPackets: len(h.packetStore),
+	}
+
+	if h.ConnectionState != nil {
+		chi.MessageCounter = atomic.LoadUint64(&h.ConnectionState.atomicMessageCounter)
+	}
+
+	if c := h.GetCert(); c != nil {
+		chi.Cert = c.Copy()
+	}
+
+	if h.remote != nil {
+		chi.CurrentRemote = h.remote.Copy()
+	}
+
+	return chi
+}
+
+func listHostMap(hm *HostMap) []ControlHostInfo {
+	hm.RLock()
+	hosts := make([]ControlHostInfo, len(hm.Hosts))
+	i := 0
+	for _, v := range hm.Hosts {
+		hosts[i] = copyHostInfo(v, hm.preferredRanges)
+		i++
+	}
+	hm.RUnlock()
+
+	return hosts
+}
--- a/control_test.go
+++ b/control_test.go
@@ -0,0 +1,113 @@
+package nebula
+
+import (
+	"net"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
+	"github.com/slackhq/nebula/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestControl_GetHostInfoByVpnIp(t *testing.T) {
+	l := util.NewTestLogger()
+	// Special care must be taken to re-use all objects provided to the hostmap and certificate in the expectedInfo object
+	// To properly ensure we are not exposing core memory to the caller
+	hm := NewHostMap(l, "test", &net.IPNet{}, make([]*net.IPNet, 0))
+	remote1 := udp.NewAddr(net.ParseIP("0.0.0.100"), 4444)
+	remote2 := udp.NewAddr(net.ParseIP("1:2:3:4:5:6:7:8"), 4444)
+	ipNet := net.IPNet{
+		IP:   net.IPv4(1, 2, 3, 4),
+		Mask: net.IPMask{255, 255, 255, 0},
+	}
+
+	ipNet2 := net.IPNet{
+		IP:   net.ParseIP("1:2:3:4:5:6:7:8"),
+		Mask: net.IPMask{255, 255, 255, 0},
+	}
+
+	crt := &cert.NebulaCertificate{
+		Details: cert.NebulaCertificateDetails{
+			Name:           "test",
+			Ips:            []*net.IPNet{&ipNet},
+			Subnets:        []*net.IPNet{},
+			Groups:         []string{"default-group"},
+			NotBefore:      time.Unix(1, 0),
+			NotAfter:       time.Unix(2, 0),
+			PublicKey:      []byte{5, 6, 7, 8},
+			IsCA:           false,
+			Issuer:         "the-issuer",
+			InvertedGroups: map[string]struct{}{"default-group": {}},
+		},
+		Signature: []byte{1, 2, 1, 2, 1, 3},
+	}
+
+	remotes := NewRemoteList()
+	remotes.unlockedPrependV4(0, NewIp4AndPort(remote1.IP, uint32(remote1.Port)))
+	remotes.unlockedPrependV6(0, NewIp6AndPort(remote2.IP, uint32(remote2.Port)))
+	hm.Add(iputil.Ip2VpnIp(ipNet.IP), &HostInfo{
+		remote:  remote1,
+		remotes: remotes,
+		ConnectionState: &ConnectionState{
+			peerCert: crt,
+		},
+		remoteIndexId: 200,
+		localIndexId:  201,
+		vpnIp:         iputil.Ip2VpnIp(ipNet.IP),
+	})
+
+	hm.Add(iputil.Ip2VpnIp(ipNet2.IP), &HostInfo{
+		remote:  remote1,
+		remotes: remotes,
+		ConnectionState: &ConnectionState{
+			peerCert: nil,
+		},
+		remoteIndexId: 200,
+		localIndexId:  201,
+		vpnIp:         iputil.Ip2VpnIp(ipNet2.IP),
+	})
+
+	c := Control{
+		f: &Interface{
+			hostMap: hm,
+		},
+		l: logrus.New(),
+	}
+
+	thi := c.GetHostInfoByVpnIp(iputil.Ip2VpnIp(ipNet.IP), false)
+
+	expectedInfo := ControlHostInfo{
+		VpnIp:          net.IPv4(1, 2, 3, 4).To4(),
+		LocalIndex:     201,
+		RemoteIndex:    200,
+		RemoteAddrs:    []*udp.Addr{remote2, remote1},
+		CachedPackets:  0,
+		Cert:           crt.Copy(),
+		MessageCounter: 0,
+		CurrentRemote:  udp.NewAddr(net.ParseIP("0.0.0.100"), 4444),
+	}
+
+	// Make sure we don't have any unexpected fields
+	assertFields(t, []string{"VpnIp", "LocalIndex", "RemoteIndex", "RemoteAddrs", "CachedPackets", "Cert", "MessageCounter", "CurrentRemote"}, thi)
+	util.AssertDeepCopyEqual(t, &expectedInfo, thi)
+
+	// Make sure we don't panic if the host info doesn't have a cert yet
+	assert.NotPanics(t, func() {
+		thi = c.GetHostInfoByVpnIp(iputil.Ip2VpnIp(ipNet2.IP), false)
+	})
+}
+
+func assertFields(t *testing.T, expected []string, actualStruct interface{}) {
+	val := reflect.ValueOf(actualStruct).Elem()
+	fields := make([]string, val.NumField())
+	for i := 0; i < val.NumField(); i++ {
+		fields[i] = val.Type().Field(i).Name
+	}
+
+	assert.Equal(t, expected, fields)
+}
--- a/control_tester.go
+++ b/control_tester.go
@@ -0,0 +1,132 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package nebula
+
+import (
+	"net"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
+)
+
+// WaitForTypeByIndex will pipe all messages from this control device into the pipeTo control device
+// returning after a message matching the criteria has been piped
+func (c *Control) WaitForType(msgType header.MessageType, subType header.MessageSubType, pipeTo *Control) {
+	h := &header.H{}
+	for {
+		p := c.f.outside.Get(true)
+		if err := h.Parse(p.Data); err != nil {
+			panic(err)
+		}
+		pipeTo.InjectUDPPacket(p)
+		if h.Type == msgType && h.Subtype == subType {
+			return
+		}
+	}
+}
+
+// WaitForTypeByIndex is similar to WaitForType except it adds an index check
+// Useful if you have many nodes communicating and want to wait to find a specific nodes packet
+func (c *Control) WaitForTypeByIndex(toIndex uint32, msgType header.MessageType, subType header.MessageSubType, pipeTo *Control) {
+	h := &header.H{}
+	for {
+		p := c.f.outside.Get(true)
+		if err := h.Parse(p.Data); err != nil {
+			panic(err)
+		}
+		pipeTo.InjectUDPPacket(p)
+		if h.RemoteIndex == toIndex && h.Type == msgType && h.Subtype == subType {
+			return
+		}
+	}
+}
+
+// InjectLightHouseAddr will push toAddr into the local lighthouse cache for the vpnIp
+// This is necessary if you did not configure static hosts or are not running a lighthouse
+func (c *Control) InjectLightHouseAddr(vpnIp net.IP, toAddr *net.UDPAddr) {
+	c.f.lightHouse.Lock()
+	remoteList := c.f.lightHouse.unlockedGetRemoteList(iputil.Ip2VpnIp(vpnIp))
+	remoteList.Lock()
+	defer remoteList.Unlock()
+	c.f.lightHouse.Unlock()
+
+	iVpnIp := iputil.Ip2VpnIp(vpnIp)
+	if v4 := toAddr.IP.To4(); v4 != nil {
+		remoteList.unlockedPrependV4(iVpnIp, NewIp4AndPort(v4, uint32(toAddr.Port)))
+	} else {
+		remoteList.unlockedPrependV6(iVpnIp, NewIp6AndPort(toAddr.IP, uint32(toAddr.Port)))
+	}
+}
+
+// GetFromTun will pull a packet off the tun side of nebula
+func (c *Control) GetFromTun(block bool) []byte {
+	return c.f.inside.(*Tun).Get(block)
+}
+
+// GetFromUDP will pull a udp packet off the udp side of nebula
+func (c *Control) GetFromUDP(block bool) *udp.Packet {
+	return c.f.outside.Get(block)
+}
+
+func (c *Control) GetUDPTxChan() <-chan *udp.Packet {
+	return c.f.outside.TxPackets
+}
+
+func (c *Control) GetTunTxChan() <-chan []byte {
+	return c.f.inside.(*Tun).txPackets
+}
+
+// InjectUDPPacket will inject a packet into the udp side of nebula
+func (c *Control) InjectUDPPacket(p *udp.Packet) {
+	c.f.outside.Send(p)
+}
+
+// InjectTunUDPPacket puts a udp packet on the tun interface. Using UDP here because it's a simpler protocol
+func (c *Control) InjectTunUDPPacket(toIp net.IP, toPort uint16, fromPort uint16, data []byte) {
+	ip := layers.IPv4{
+		Version:  4,
+		TTL:      64,
+		Protocol: layers.IPProtocolUDP,
+		SrcIP:    c.f.inside.CidrNet().IP,
+		DstIP:    toIp,
+	}
+
+	udp := layers.UDP{
+		SrcPort: layers.UDPPort(fromPort),
+		DstPort: layers.UDPPort(toPort),
+	}
+	err := udp.SetNetworkLayerForChecksum(&ip)
+	if err != nil {
+		panic(err)
+	}
+
+	buffer := gopacket.NewSerializeBuffer()
+	opt := gopacket.SerializeOptions{
+		ComputeChecksums: true,
+		FixLengths:       true,
+	}
+	err = gopacket.SerializeLayers(buffer, opt, &ip, &udp, gopacket.Payload(data))
+	if err != nil {
+		panic(err)
+	}
+
+	c.f.inside.(*Tun).Send(buffer.Bytes())
+}
+
+func (c *Control) GetUDPAddr() string {
+	return c.f.outside.Addr.String()
+}
+
+func (c *Control) KillPendingTunnel(vpnIp net.IP) bool {
+	hostinfo, ok := c.f.handshakeManager.pendingHostMap.Hosts[iputil.Ip2VpnIp(vpnIp)]
+	if !ok {
+		return false
+	}
+
+	c.f.handshakeManager.pendingHostMap.DeleteHostInfo(hostinfo)
+	return true
+}
--- a/dist/arch/nebula.service
+++ b/dist/arch/nebula.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=nebula
+Wants=basic.target network-online.target
+After=basic.target network.target network-online.target
+
+[Service]
+SyslogIdentifier=nebula
+ExecReload=/bin/kill -HUP $MAINPID
+ExecStart=/usr/bin/nebula -config /etc/nebula/config.yml
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
--- a/dist/fedora/nebula.service
+++ b/dist/fedora/nebula.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Nebula overlay networking tool
+
+After=basic.target network.target network-online.target
+Before=sshd.service
+Wants=basic.target network-online.target
+
+[Service]
+ExecReload=/bin/kill -HUP $MAINPID
+ExecStart=/usr/bin/nebula -config /etc/nebula/config.yml
+Restart=always
+SyslogIdentifier=nebula
+
+[Install]
+WantedBy=multi-user.target
--- a/dist/wireshark/nebula.lua
+++ b/dist/wireshark/nebula.lua
@@ -0,0 +1,113 @@
+local nebula = Proto("nebula", "nebula")
+
+local default_settings = {
+    port      = 4242,
+    all_ports = false,
+}
+
+nebula.prefs.port  = Pref.uint("Port number", default_settings.port, "The UDP port number for Nebula")
+nebula.prefs.all_ports  = Pref.bool("All ports", default_settings.all_ports, "Assume nebula packets on any port, useful when dealing with hole punching")
+
+local pf_version  = ProtoField.new("version", "nebula.version", ftypes.UINT8, nil, base.DEC, 0xF0)
+local pf_type     = ProtoField.new("type", "nebula.type", ftypes.UINT8, {
+    [0] = "handshake",
+    [1] = "message",
+    [2] = "recvError",
+    [3] = "lightHouse",
+    [4] = "test",
+    [5] = "closeTunnel",
+}, base.DEC, 0x0F)
+
+local pf_subtype = ProtoField.new("subtype", "nebula.subtype", ftypes.UINT8, nil, base.DEC)
+local pf_subtype_test = ProtoField.new("subtype", "nebula.subtype", ftypes.UINT8, {
+    [0] = "request",
+    [1] = "reply",
+}, base.DEC)
+
+local pf_subtype_handshake = ProtoField.new("subtype", "nebula.subtype", ftypes.UINT8, {
+    [0] = "ix_psk0",
+}, base.DEC)
+
+local pf_reserved        = ProtoField.new("reserved",     "nebula.reserved",     ftypes.UINT16, nil, base.HEX)
+local pf_remote_index    = ProtoField.new("remote index", "nebula.remote_index", ftypes.UINT32, nil, base.DEC)
+local pf_message_counter = ProtoField.new("counter",      "nebula.counter",      ftypes.UINT64, nil, base.DEC)
+local pf_payload         = ProtoField.new("payload",      "nebula.payload",      ftypes.BYTES,  nil, base.NONE)
+
+nebula.fields = { pf_version, pf_type, pf_subtype, pf_subtype_handshake, pf_subtype_test, pf_reserved, pf_remote_index, pf_message_counter, pf_payload }
+
+local ef_holepunch = ProtoExpert.new("nebula.holepunch.expert", "Nebula hole punch packet", expert.group.PROTOCOL, expert.severity.NOTE)
+local ef_punchy = ProtoExpert.new("nebula.punchy.expert", "Nebula punchy keepalive packet", expert.group.PROTOCOL, expert.severity.NOTE)
+
+nebula.experts = { ef_holepunch, ef_punchy }
+local type_field = Field.new("nebula.type")
+local subtype_field = Field.new("nebula.subtype")
+
+function nebula.dissector(tvbuf, pktinfo, root)
+    -- set the protocol column to show our protocol name
+    pktinfo.cols.protocol:set("NEBULA")
+
+    local pktlen = tvbuf:reported_length_remaining()
+    local tree = root:add(nebula, tvbuf:range(0,pktlen))
+
+    if pktlen == 0 then
+        tree:add_proto_expert_info(ef_holepunch)
+        pktinfo.cols.info:append(" (holepunch)")
+        return
+    elseif pktlen == 1 then
+        tree:add_proto_expert_info(ef_punchy)
+        pktinfo.cols.info:append(" (punchy)")
+        return
+    end
+
+    tree:add(pf_version, tvbuf:range(0,1))
+    local type = tree:add(pf_type,    tvbuf:range(0,1))
+
+    local nebula_type = bit32.band(tvbuf:range(0,1):uint(), 0x0F)
+    if nebula_type == 0 then
+        local stage = tvbuf(8,8):uint64()
+        tree:add(pf_subtype_handshake, tvbuf:range(1,1))
+        type:append_text(" stage " .. stage)
+        pktinfo.cols.info:append(" (" .. type_field().display .. ", stage " .. stage .. ", " .. subtype_field().display .. ")")
+    elseif nebula_type == 4 then
+        tree:add(pf_subtype_test, tvbuf:range(1,1))
+        pktinfo.cols.info:append(" (" .. type_field().display .. ", " .. subtype_field().display .. ")")
+    else
+        tree:add(pf_subtype, tvbuf:range(1,1))
+        pktinfo.cols.info:append(" (" .. type_field().display .. ")")
+    end
+
+    tree:add(pf_reserved,        tvbuf:range(2,2))
+    tree:add(pf_remote_index,    tvbuf:range(4,4))
+    tree:add(pf_message_counter, tvbuf:range(8,8))
+    tree:add(pf_payload,         tvbuf:range(16,tvbuf:len() - 16))
+end
+
+function nebula.prefs_changed()
+    if default_settings.all_ports == nebula.prefs.all_ports and default_settings.port == nebula.prefs.port then
+        -- Nothing changed, bail
+        return
+    end
+
+    -- Remove our old dissector
+    DissectorTable.get("udp.port"):remove_all(nebula)
+
+    if nebula.prefs.all_ports and default_settings.all_ports ~= nebula.prefs.all_ports then
+        default_settings.all_port = nebula.prefs.all_ports
+
+        for i=0, 65535 do
+            DissectorTable.get("udp.port"):add(i, nebula)
+        end
+
+        -- no need to establish again on specific ports
+        return
+    end
+
+
+    if default_settings.all_ports ~= nebula.prefs.all_ports then
+        -- Add our new port dissector
+        default_settings.port = nebula.prefs.port
+        DissectorTable.get("udp.port"):add(default_settings.port, nebula)
+    end
+end
+
+DissectorTable.get("udp.port"):add(default_settings.port, nebula)
--- a/dns_server.go
+++ b/dns_server.go
@@ -7,11 +7,16 @@ import (
 	"sync"

 	"github.com/miekg/dns"
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/iputil"
 )

 // This whole thing should be rewritten to use context

 var dnsR *dnsRecords
+var dnsServer *dns.Server
+var dnsAddr string

 type dnsRecords struct {
 	sync.RWMutex
@@ -41,8 +46,8 @@ func (d *dnsRecords) QueryCert(data string) string {
 	if ip == nil {
 		return ""
 	}
-	iip := ip2int(ip)
-	hostinfo, err := d.hostMap.QueryVpnIP(iip)
+	iip := iputil.Ip2VpnIp(ip)
+	hostinfo, err := d.hostMap.QueryVpnIp(iip)
 	if err != nil {
 		return ""
 	}
@@ -61,7 +66,7 @@ func (d *dnsRecords) Add(host, data string) {
 	d.Unlock()
 }

-func parseQuery(m *dns.Msg, w dns.ResponseWriter) {
+func parseQuery(l *logrus.Logger, m *dns.Msg, w dns.ResponseWriter) {
 	for _, q := range m.Question {
 		switch q.Qtype {
 		case dns.TypeA:
@@ -93,33 +98,58 @@ func parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 	}
 }

-func handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
+func handleDnsRequest(l *logrus.Logger, w dns.ResponseWriter, r *dns.Msg) {
 	m := new(dns.Msg)
 	m.SetReply(r)
 	m.Compress = false

 	switch r.Opcode {
 	case dns.OpcodeQuery:
-		parseQuery(m, w)
+		parseQuery(l, m, w)
 	}

 	w.WriteMsg(m)
 }

-func dnsMain(hostMap *HostMap) {
-
+func dnsMain(l *logrus.Logger, hostMap *HostMap, c *config.C) func() {
 	dnsR = newDnsRecords(hostMap)

 	// attach request handler func
-	dns.HandleFunc(".", handleDnsRequest)
+	dns.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) {
+		handleDnsRequest(l, w, r)
+	})

-	// start server
-	port := 53
-	server := &dns.Server{Addr: ":" + strconv.Itoa(port), Net: "udp"}
-	l.Debugf("Starting DNS responder at %d\n", port)
-	err := server.ListenAndServe()
-	defer server.Shutdown()
+	c.RegisterReloadCallback(func(c *config.C) {
+		reloadDns(l, c)
+	})
+
+	return func() {
+		startDns(l, c)
+	}
+}
+
+func getDnsServerAddr(c *config.C) string {
+	return c.GetString("lighthouse.dns.host", "") + ":" + strconv.Itoa(c.GetInt("lighthouse.dns.port", 53))
+}
+
+func startDns(l *logrus.Logger, c *config.C) {
+	dnsAddr = getDnsServerAddr(c)
+	dnsServer = &dns.Server{Addr: dnsAddr, Net: "udp"}
+	l.WithField("dnsListener", dnsAddr).Infof("Starting DNS responder")
+	err := dnsServer.ListenAndServe()
+	defer dnsServer.Shutdown()
 	if err != nil {
 		l.Errorf("Failed to start server: %s\n ", err.Error())
 	}
 }
+
+func reloadDns(l *logrus.Logger, c *config.C) {
+	if dnsAddr == getDnsServerAddr(c) {
+		l.Debug("No DNS server config change detected")
+		return
+	}
+
+	l.Debug("Restarting DNS server")
+	dnsServer.Shutdown()
+	go startDns(l, c)
+}
--- a/e2e/doc.go
+++ b/e2e/doc.go
@@ -0,0 +1,3 @@
+package e2e
+
+// This file exists to allow `go fmt` to traverse here on its own. The build tags were keeping it out before
--- a/e2e/handshakes_test.go
+++ b/e2e/handshakes_test.go
@@ -0,0 +1,333 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package e2e
+
+import (
+	"net"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGoodHandshake(t *testing.T) {
+	ca, _, caKey, _ := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
+	myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 1}, nil)
+	theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, nil)
+
+	// Put their info in our lighthouse
+	myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
+
+	// Start the servers
+	myControl.Start()
+	theirControl.Start()
+
+	t.Log("Send a udp packet through to begin standing up the tunnel, this should come out the other side")
+	myControl.InjectTunUDPPacket(theirVpnIp, 80, 80, []byte("Hi from me"))
+
+	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
+	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
+
+	t.Log("Get their stage 1 packet so that we can play with it")
+	stage1Packet := theirControl.GetFromUDP(true)
+
+	t.Log("I consume a garbage packet with a proper nebula header for our tunnel")
+	// this should log a statement and get ignored, allowing the real handshake packet to complete the tunnel
+	badPacket := stage1Packet.Copy()
+	badPacket.Data = badPacket.Data[:len(badPacket.Data)-header.Len]
+	myControl.InjectUDPPacket(badPacket)
+
+	t.Log("Have me consume their real stage 1 packet. I have a tunnel now")
+	myControl.InjectUDPPacket(stage1Packet)
+
+	t.Log("Wait until we see my cached packet come through")
+	myControl.WaitForType(1, 0, theirControl)
+
+	t.Log("Make sure our host infos are correct")
+	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIp, theirVpnIp, myControl, theirControl)
+
+	t.Log("Get that cached packet and make sure it looks right")
+	myCachedPacket := theirControl.GetFromTun(true)
+	assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIp, theirVpnIp, 80, 80)
+
+	t.Log("Do a bidirectional tunnel test")
+	assertTunnel(t, myVpnIp, theirVpnIp, myControl, theirControl, router.NewR(myControl, theirControl))
+
+	myControl.Stop()
+	theirControl.Stop()
+	//TODO: assert hostmaps
+}
+
+func TestWrongResponderHandshake(t *testing.T) {
+	ca, _, caKey, _ := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
+
+	// The IPs here are chosen on purpose:
+	// The current remote handling will sort by preference, public, and then lexically.
+	// So we need them to have a higher address than evil (we could apply a preference though)
+	myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 100}, nil)
+	theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 99}, nil)
+	evilControl, evilVpnIp, evilUdpAddr := newSimpleServer(ca, caKey, "evil", net.IP{10, 0, 0, 2}, nil)
+
+	// Add their real udp addr, which should be tried after evil.
+	myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
+
+	// Put the evil udp addr in for their vpn Ip, this is a case of being lied to by the lighthouse.
+	myControl.InjectLightHouseAddr(theirVpnIp, evilUdpAddr)
+
+	// Build a router so we don't have to reason who gets which packet
+	r := router.NewR(myControl, theirControl, evilControl)
+
+	// Start the servers
+	myControl.Start()
+	theirControl.Start()
+	evilControl.Start()
+
+	t.Log("Start the handshake process, we will route until we see our cached packet get sent to them")
+	myControl.InjectTunUDPPacket(theirVpnIp, 80, 80, []byte("Hi from me"))
+	r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
+		h := &header.H{}
+		err := h.Parse(p.Data)
+		if err != nil {
+			panic(err)
+		}
+
+		if p.ToIp.Equal(theirUdpAddr.IP) && p.ToPort == uint16(theirUdpAddr.Port) && h.Type == 1 {
+			return router.RouteAndExit
+		}
+
+		return router.KeepRouting
+	})
+
+	//TODO: Assert pending hostmap - I should have a correct hostinfo for them now
+
+	t.Log("My cached packet should be received by them")
+	myCachedPacket := theirControl.GetFromTun(true)
+	assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIp, theirVpnIp, 80, 80)
+
+	t.Log("Test the tunnel with them")
+	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIp, theirVpnIp, myControl, theirControl)
+	assertTunnel(t, myVpnIp, theirVpnIp, myControl, theirControl, r)
+
+	t.Log("Flush all packets from all controllers")
+	r.FlushAll()
+
+	t.Log("Ensure ensure I don't have any hostinfo artifacts from evil")
+	assert.Nil(t, myControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(evilVpnIp), true), "My pending hostmap should not contain evil")
+	assert.Nil(t, myControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(evilVpnIp), false), "My main hostmap should not contain evil")
+	//NOTE: if evil lost the handshake race it may still have a tunnel since me would reject the handshake since the tunnel is complete
+
+	//TODO: assert hostmaps for everyone
+	t.Log("Success!")
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func Test_Case1_Stage1Race(t *testing.T) {
+	ca, _, caKey, _ := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
+	myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me  ", net.IP{10, 0, 0, 1}, nil)
+	theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, nil)
+
+	// Put their info in our lighthouse and vice versa
+	myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIp, myUdpAddr)
+
+	// Build a router so we don't have to reason who gets which packet
+	r := router.NewR(myControl, theirControl)
+
+	// Start the servers
+	myControl.Start()
+	theirControl.Start()
+
+	t.Log("Trigger a handshake to start on both me and them")
+	myControl.InjectTunUDPPacket(theirVpnIp, 80, 80, []byte("Hi from me"))
+	theirControl.InjectTunUDPPacket(myVpnIp, 80, 80, []byte("Hi from them"))
+
+	t.Log("Get both stage 1 handshake packets")
+	myHsForThem := myControl.GetFromUDP(true)
+	theirHsForMe := theirControl.GetFromUDP(true)
+
+	t.Log("Now inject both stage 1 handshake packets")
+	myControl.InjectUDPPacket(theirHsForMe)
+	theirControl.InjectUDPPacket(myHsForThem)
+	//TODO: they should win, grab their index for me and make sure I use it in the end.
+
+	t.Log("They should not have a stage 2 (won the race) but I should send one")
+	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
+
+	t.Log("Route for me until I send a message packet to them")
+	myControl.WaitForType(1, 0, theirControl)
+
+	t.Log("My cached packet should be received by them")
+	myCachedPacket := theirControl.GetFromTun(true)
+	assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIp, theirVpnIp, 80, 80)
+
+	t.Log("Route for them until I send a message packet to me")
+	theirControl.WaitForType(1, 0, myControl)
+
+	t.Log("Their cached packet should be received by me")
+	theirCachedPacket := myControl.GetFromTun(true)
+	assertUdpPacket(t, []byte("Hi from them"), theirCachedPacket, theirVpnIp, myVpnIp, 80, 80)
+
+	t.Log("Do a bidirectional tunnel test")
+	assertTunnel(t, myVpnIp, theirVpnIp, myControl, theirControl, r)
+
+	myControl.Stop()
+	theirControl.Stop()
+	//TODO: assert hostmaps
+}
+
+//TODO: add a test with many lies
+
+func TestPSK(t *testing.T) {
+	tests := []struct {
+		name         string
+		myPskMode    nebula.PskMode
+		theirPskMode nebula.PskMode
+	}{
+		// None and transitional-accepting both ways
+		{
+			name:         "none to transitional-accepting",
+			myPskMode:    nebula.PskNone,
+			theirPskMode: nebula.PskTransitionalAccepting,
+		},
+		{
+			name:         "transitional-accepting to none",
+			myPskMode:    nebula.PskTransitionalAccepting,
+			theirPskMode: nebula.PskNone,
+		},
+
+		// All transitional-accepting
+		{
+			name:         "both transitional-accepting",
+			myPskMode:    nebula.PskTransitionalAccepting,
+			theirPskMode: nebula.PskTransitionalAccepting,
+		},
+
+		// transitional-accepting and transitional-sending both ways
+		{
+			name:         "transitional-accepting to transitional-sending",
+			myPskMode:    nebula.PskTransitionalAccepting,
+			theirPskMode: nebula.PskTransitionalSending,
+		},
+		{
+			name:         "transitional-sending to transitional-accepting",
+			myPskMode:    nebula.PskTransitionalSending,
+			theirPskMode: nebula.PskTransitionalAccepting,
+		},
+
+		// All transitional-sending
+		{
+			name:         "transitional-sending to transitional-sending",
+			myPskMode:    nebula.PskTransitionalSending,
+			theirPskMode: nebula.PskTransitionalSending,
+		},
+
+		// enforced and transitional-sending both ways
+		{
+			name:         "enforced to transitional-sending",
+			myPskMode:    nebula.PskEnforced,
+			theirPskMode: nebula.PskTransitionalSending,
+		},
+		{
+			name:         "transitional-sending to enforced",
+			myPskMode:    nebula.PskTransitionalSending,
+			theirPskMode: nebula.PskEnforced,
+		},
+
+		// All enforced
+		{
+			name:         "both enforced",
+			myPskMode:    nebula.PskEnforced,
+			theirPskMode: nebula.PskEnforced,
+		},
+
+		// Enforced can technically handshake with a traditional-accepting but it is bad to be in this state
+		{
+			name:         "enforced to traditional-accepting",
+			myPskMode:    nebula.PskEnforced,
+			theirPskMode: nebula.PskTransitionalAccepting,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var myPskSettings, theirPskSettings *m
+
+			switch test.myPskMode {
+			case nebula.PskNone:
+				myPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "none"}}}
+			case nebula.PskTransitionalAccepting:
+				myPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "transitional-accepting", "keys": []string{"this is a key"}}}}
+			case nebula.PskTransitionalSending:
+				myPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "transitional-sending", "keys": []string{"this is a key"}}}}
+			case nebula.PskEnforced:
+				myPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "enforced", "keys": []string{"this is a key"}}}}
+			}
+
+			switch test.theirPskMode {
+			case nebula.PskNone:
+				theirPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "none"}}}
+			case nebula.PskTransitionalAccepting:
+				theirPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "transitional-accepting", "keys": []string{"this is a key"}}}}
+			case nebula.PskTransitionalSending:
+				theirPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "transitional-sending", "keys": []string{"this is a key"}}}}
+			case nebula.PskEnforced:
+				theirPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "enforced", "keys": []string{"this is a key"}}}}
+			}
+
+			ca, _, caKey, _ := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
+			myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 1}, myPskSettings)
+			theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, theirPskSettings)
+
+			myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
+			r := router.NewR(myControl, theirControl)
+
+			// Start the servers
+			myControl.Start()
+			theirControl.Start()
+
+			t.Log("Route until we see our cached packet flow")
+			myControl.InjectTunUDPPacket(theirVpnIp, 80, 80, []byte("Hi from me"))
+			r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
+				h := &header.H{}
+				err := h.Parse(p.Data)
+				if err != nil {
+					panic(err)
+				}
+
+				// If this is the stage 1 handshake packet and I am configured to send with a psk, my cert name should
+				// not appear. It would likely be more obvious to unmarshal the payload and check but this works fine for now
+				if test.myPskMode == nebula.PskEnforced || test.myPskMode == nebula.PskTransitionalSending {
+					if h.Type == 0 && h.MessageCounter == 1 {
+						assert.NotContains(t, string(p.Data), "test me")
+					}
+				}
+
+				if p.ToIp.Equal(theirUdpAddr.IP) && p.ToPort == uint16(theirUdpAddr.Port) && h.Type == 1 {
+					return router.RouteAndExit
+				}
+
+				return router.KeepRouting
+			})
+
+			t.Log("My cached packet should be received by them")
+			myCachedPacket := theirControl.GetFromTun(true)
+			assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIp, theirVpnIp, 80, 80)
+
+			t.Log("Test the tunnel with them")
+			assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIp, theirVpnIp, myControl, theirControl)
+			assertTunnel(t, myVpnIp, theirVpnIp, myControl, theirControl, r)
+
+			myControl.Stop()
+			theirControl.Stop()
+			//TODO: assert hostmaps
+		})
+	}
+
+}
--- a/e2e/helpers_test.go
+++ b/e2e/helpers_test.go
@@ -0,0 +1,330 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package e2e
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/imdario/mergo"
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/ed25519"
+	"gopkg.in/yaml.v2"
+)
+
+type m map[string]interface{}
+
+// newSimpleServer creates a nebula instance with many assumptions
+func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, udpIp net.IP, customConfig *m) (*nebula.Control, net.IP, *net.UDPAddr) {
+	l := NewTestLogger()
+
+	vpnIpNet := &net.IPNet{IP: make([]byte, len(udpIp)), Mask: net.IPMask{255, 255, 255, 0}}
+	copy(vpnIpNet.IP, udpIp)
+	vpnIpNet.IP[1] += 128
+	udpAddr := net.UDPAddr{
+		IP:   udpIp,
+		Port: 4242,
+	}
+	_, _, myPrivKey, myPEM := newTestCert(caCrt, caKey, "test "+name, time.Now(), time.Now().Add(5*time.Minute), vpnIpNet, nil, []string{})
+
+	caB, err := caCrt.MarshalToPEM()
+	if err != nil {
+		panic(err)
+	}
+
+	mc := m{
+		"pki": m{
+			"ca":   string(caB),
+			"cert": string(myPEM),
+			"key":  string(myPrivKey),
+		},
+		//"tun": m{"disabled": true},
+		"firewall": m{
+			"outbound": []m{{
+				"proto": "any",
+				"port":  "any",
+				"host":  "any",
+			}},
+			"inbound": []m{{
+				"proto": "any",
+				"port":  "any",
+				"host":  "any",
+			}},
+		},
+		//"handshakes": m{
+		//	"try_interval": "1s",
+		//},
+		"listen": m{
+			"host": udpAddr.IP.String(),
+			"port": udpAddr.Port,
+		},
+		"logging": m{
+			"timestamp_format": fmt.Sprintf("%v 15:04:05.000000", name),
+			"level":            l.Level.String(),
+		},
+	}
+	cb, err := yaml.Marshal(mc)
+	if err != nil {
+		panic(err)
+	}
+
+	c := config.NewC(l)
+	c.LoadString(string(cb))
+
+	if customConfig != nil {
+		ccb, err := yaml.Marshal(customConfig)
+		if err != nil {
+			panic(err)
+		}
+
+		ccm := map[interface{}]interface{}{}
+		err = yaml.Unmarshal(ccb, &ccm)
+		if err != nil {
+			panic(err)
+		}
+
+		err = mergo.Merge(&c.Settings, ccm, mergo.WithAppendSlice)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	control, err := nebula.Main(c, false, "e2e-test", l, nil)
+
+	if err != nil {
+		panic(err)
+	}
+
+	return control, vpnIpNet.IP, &udpAddr
+}
+
+// newTestCaCert will generate a CA cert
+func newTestCaCert(before, after time.Time, ips, subnets []*net.IPNet, groups []string) (*cert.NebulaCertificate, []byte, []byte, []byte) {
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	if before.IsZero() {
+		before = time.Now().Add(time.Second * -60).Round(time.Second)
+	}
+	if after.IsZero() {
+		after = time.Now().Add(time.Second * 60).Round(time.Second)
+	}
+
+	nc := &cert.NebulaCertificate{
+		Details: cert.NebulaCertificateDetails{
+			Name:           "test ca",
+			NotBefore:      time.Unix(before.Unix(), 0),
+			NotAfter:       time.Unix(after.Unix(), 0),
+			PublicKey:      pub,
+			IsCA:           true,
+			InvertedGroups: make(map[string]struct{}),
+		},
+	}
+
+	if len(ips) > 0 {
+		nc.Details.Ips = ips
+	}
+
+	if len(subnets) > 0 {
+		nc.Details.Subnets = subnets
+	}
+
+	if len(groups) > 0 {
+		nc.Details.Groups = groups
+	}
+
+	err = nc.Sign(priv)
+	if err != nil {
+		panic(err)
+	}
+
+	pem, err := nc.MarshalToPEM()
+	if err != nil {
+		panic(err)
+	}
+
+	return nc, pub, priv, pem
+}
+
+// newTestCert will generate a signed certificate with the provided details.
+// Expiry times are defaulted if you do not pass them in
+func newTestCert(ca *cert.NebulaCertificate, key []byte, name string, before, after time.Time, ip *net.IPNet, subnets []*net.IPNet, groups []string) (*cert.NebulaCertificate, []byte, []byte, []byte) {
+	issuer, err := ca.Sha256Sum()
+	if err != nil {
+		panic(err)
+	}
+
+	if before.IsZero() {
+		before = time.Now().Add(time.Second * -60).Round(time.Second)
+	}
+
+	if after.IsZero() {
+		after = time.Now().Add(time.Second * 60).Round(time.Second)
+	}
+
+	pub, rawPriv := x25519Keypair()
+
+	nc := &cert.NebulaCertificate{
+		Details: cert.NebulaCertificateDetails{
+			Name:           name,
+			Ips:            []*net.IPNet{ip},
+			Subnets:        subnets,
+			Groups:         groups,
+			NotBefore:      time.Unix(before.Unix(), 0),
+			NotAfter:       time.Unix(after.Unix(), 0),
+			PublicKey:      pub,
+			IsCA:           false,
+			Issuer:         issuer,
+			InvertedGroups: make(map[string]struct{}),
+		},
+	}
+
+	err = nc.Sign(key)
+	if err != nil {
+		panic(err)
+	}
+
+	pem, err := nc.MarshalToPEM()
+	if err != nil {
+		panic(err)
+	}
+
+	return nc, pub, cert.MarshalX25519PrivateKey(rawPriv), pem
+}
+
+func x25519Keypair() ([]byte, []byte) {
+	privkey := make([]byte, 32)
+	if _, err := io.ReadFull(rand.Reader, privkey); err != nil {
+		panic(err)
+	}
+
+	pubkey, err := curve25519.X25519(privkey, curve25519.Basepoint)
+	if err != nil {
+		panic(err)
+	}
+
+	return pubkey, privkey
+}
+
+type doneCb func()
+
+func deadline(t *testing.T, seconds time.Duration) doneCb {
+	timeout := time.After(seconds * time.Second)
+	done := make(chan bool)
+	go func() {
+		select {
+		case <-timeout:
+			t.Fatal("Test did not finish in time")
+		case <-done:
+		}
+	}()
+
+	return func() {
+		done <- true
+	}
+}
+
+func assertTunnel(t *testing.T, vpnIpA, vpnIpB net.IP, controlA, controlB *nebula.Control, r *router.R) {
+	// Send a packet from them to me
+	controlB.InjectTunUDPPacket(vpnIpA, 80, 90, []byte("Hi from B"))
+	bPacket := r.RouteUntilTxTun(controlB, controlA)
+	assertUdpPacket(t, []byte("Hi from B"), bPacket, vpnIpB, vpnIpA, 90, 80)
+
+	// And once more from me to them
+	controlA.InjectTunUDPPacket(vpnIpB, 80, 90, []byte("Hello from A"))
+	aPacket := r.RouteUntilTxTun(controlA, controlB)
+	assertUdpPacket(t, []byte("Hello from A"), aPacket, vpnIpA, vpnIpB, 90, 80)
+}
+
+func assertHostInfoPair(t *testing.T, addrA, addrB *net.UDPAddr, vpnIpA, vpnIpB net.IP, controlA, controlB *nebula.Control) {
+	// Get both host infos
+	hBinA := controlA.GetHostInfoByVpnIp(iputil.Ip2VpnIp(vpnIpB), false)
+	assert.NotNil(t, hBinA, "Host B was not found by vpnIp in controlA")
+
+	hAinB := controlB.GetHostInfoByVpnIp(iputil.Ip2VpnIp(vpnIpA), false)
+	assert.NotNil(t, hAinB, "Host A was not found by vpnIp in controlB")
+
+	// Check that both vpn and real addr are correct
+	assert.Equal(t, vpnIpB, hBinA.VpnIp, "Host B VpnIp is wrong in control A")
+	assert.Equal(t, vpnIpA, hAinB.VpnIp, "Host A VpnIp is wrong in control B")
+
+	assert.Equal(t, addrB.IP.To16(), hBinA.CurrentRemote.IP.To16(), "Host B remote ip is wrong in control A")
+	assert.Equal(t, addrA.IP.To16(), hAinB.CurrentRemote.IP.To16(), "Host A remote ip is wrong in control B")
+
+	assert.Equal(t, addrB.Port, int(hBinA.CurrentRemote.Port), "Host B remote port is wrong in control A")
+	assert.Equal(t, addrA.Port, int(hAinB.CurrentRemote.Port), "Host A remote port is wrong in control B")
+
+	// Check that our indexes match
+	assert.Equal(t, hBinA.LocalIndex, hAinB.RemoteIndex, "Host B local index does not match host A remote index")
+	assert.Equal(t, hBinA.RemoteIndex, hAinB.LocalIndex, "Host B remote index does not match host A local index")
+
+	//TODO: Would be nice to assert this memory
+	//checkIndexes := func(name string, hm *HostMap, hi *HostInfo) {
+	//	hBbyIndex := hmA.Indexes[hBinA.localIndexId]
+	//	assert.NotNil(t, hBbyIndex, "Could not host info by local index in %s", name)
+	//	assert.Equal(t, &hBbyIndex, &hBinA, "%s Indexes map did not point to the right host info", name)
+	//
+	//	//TODO: remote indexes are susceptible to collision
+	//	hBbyRemoteIndex := hmA.RemoteIndexes[hBinA.remoteIndexId]
+	//	assert.NotNil(t, hBbyIndex, "Could not host info by remote index in %s", name)
+	//	assert.Equal(t, &hBbyRemoteIndex, &hBinA, "%s RemoteIndexes did not point to the right host info", name)
+	//}
+	//
+	//// Check hostmap indexes too
+	//checkIndexes("hmA", hmA, hBinA)
+	//checkIndexes("hmB", hmB, hAinB)
+}
+
+func assertUdpPacket(t *testing.T, expected, b []byte, fromIp, toIp net.IP, fromPort, toPort uint16) {
+	packet := gopacket.NewPacket(b, layers.LayerTypeIPv4, gopacket.Lazy)
+	v4 := packet.Layer(layers.LayerTypeIPv4).(*layers.IPv4)
+	assert.NotNil(t, v4, "No ipv4 data found")
+
+	assert.Equal(t, fromIp, v4.SrcIP, "Source ip was incorrect")
+	assert.Equal(t, toIp, v4.DstIP, "Dest ip was incorrect")
+
+	udp := packet.Layer(layers.LayerTypeUDP).(*layers.UDP)
+	assert.NotNil(t, udp, "No udp data found")
+
+	assert.Equal(t, fromPort, uint16(udp.SrcPort), "Source port was incorrect")
+	assert.Equal(t, toPort, uint16(udp.DstPort), "Dest port was incorrect")
+
+	data := packet.ApplicationLayer()
+	assert.NotNil(t, data)
+	assert.Equal(t, expected, data.Payload(), "Data was incorrect")
+}
+
+func NewTestLogger() *logrus.Logger {
+	l := logrus.New()
+
+	v := os.Getenv("TEST_LOGS")
+	if v == "" {
+		l.SetOutput(ioutil.Discard)
+		return l
+	}
+
+	switch v {
+	case "2":
+		l.SetLevel(logrus.DebugLevel)
+	case "3":
+		l.SetLevel(logrus.TraceLevel)
+	default:
+		l.SetLevel(logrus.InfoLevel)
+	}
+
+	return l
+}
--- a/e2e/router/doc.go
+++ b/e2e/router/doc.go
@@ -0,0 +1,3 @@
+package router
+
+// This file exists to allow `go fmt` to traverse here on its own. The build tags were keeping it out before
--- a/e2e/router/router.go
+++ b/e2e/router/router.go
@@ -0,0 +1,323 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package router
+
+import (
+	"fmt"
+	"net"
+	"reflect"
+	"strconv"
+	"sync"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/udp"
+)
+
+type R struct {
+	// Simple map of the ip:port registered on a control to the control
+	// Basically a router, right?
+	controls map[string]*nebula.Control
+
+	// A map for inbound packets for a control that doesn't know about this address
+	inNat map[string]*nebula.Control
+
+	// A last used map, if an inbound packet hit the inNat map then
+	// all return packets should use the same last used inbound address for the outbound sender
+	// map[from address + ":" + to address] => ip:port to rewrite in the udp packet to receiver
+	outNat map[string]net.UDPAddr
+
+	// All interactions are locked to help serialize behavior
+	sync.Mutex
+}
+
+type ExitType int
+
+const (
+	// Keeps routing, the function will get called again on the next packet
+	KeepRouting ExitType = 0
+	// Does not route this packet and exits immediately
+	ExitNow ExitType = 1
+	// Routes this packet and exits immediately afterwards
+	RouteAndExit ExitType = 2
+)
+
+type ExitFunc func(packet *udp.Packet, receiver *nebula.Control) ExitType
+
+func NewR(controls ...*nebula.Control) *R {
+	r := &R{
+		controls: make(map[string]*nebula.Control),
+		inNat:    make(map[string]*nebula.Control),
+		outNat:   make(map[string]net.UDPAddr),
+	}
+
+	for _, c := range controls {
+		addr := c.GetUDPAddr()
+		if _, ok := r.controls[addr]; ok {
+			panic("Duplicate listen address: " + addr)
+		}
+		r.controls[addr] = c
+	}
+
+	return r
+}
+
+// AddRoute will place the nebula controller at the ip and port specified.
+// It does not look at the addr attached to the instance.
+// If a route is used, this will behave like a NAT for the return path.
+// Rewriting the source ip:port to what was last sent to from the origin
+func (r *R) AddRoute(ip net.IP, port uint16, c *nebula.Control) {
+	r.Lock()
+	defer r.Unlock()
+
+	inAddr := net.JoinHostPort(ip.String(), fmt.Sprintf("%v", port))
+	if _, ok := r.inNat[inAddr]; ok {
+		panic("Duplicate listen address inNat: " + inAddr)
+	}
+	r.inNat[inAddr] = c
+}
+
+// OnceFrom will route a single packet from sender then return
+// If the router doesn't have the nebula controller for that address, we panic
+func (r *R) OnceFrom(sender *nebula.Control) {
+	r.RouteExitFunc(sender, func(*udp.Packet, *nebula.Control) ExitType {
+		return RouteAndExit
+	})
+}
+
+// RouteUntilTxTun will route for sender and return when a packet is seen on receivers tun
+// If the router doesn't have the nebula controller for that address, we panic
+func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []byte {
+	tunTx := receiver.GetTunTxChan()
+	udpTx := sender.GetUDPTxChan()
+
+	for {
+		select {
+		// Maybe we already have something on the tun for us
+		case b := <-tunTx:
+			return b
+
+		// Nope, lets push the sender along
+		case p := <-udpTx:
+			outAddr := sender.GetUDPAddr()
+			r.Lock()
+			inAddr := net.JoinHostPort(p.ToIp.String(), fmt.Sprintf("%v", p.ToPort))
+			c := r.getControl(outAddr, inAddr, p)
+			if c == nil {
+				r.Unlock()
+				panic("No control for udp tx")
+			}
+
+			c.InjectUDPPacket(p)
+			r.Unlock()
+		}
+	}
+}
+
+// RouteExitFunc will call the whatDo func with each udp packet from sender.
+// whatDo can return:
+//   - exitNow: the packet will not be routed and this call will return immediately
+//   - routeAndExit: this call will return immediately after routing the last packet from sender
+//   - keepRouting: the packet will be routed and whatDo will be called again on the next packet from sender
+func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
+	h := &header.H{}
+	for {
+		p := sender.GetFromUDP(true)
+		r.Lock()
+		if err := h.Parse(p.Data); err != nil {
+			panic(err)
+		}
+
+		outAddr := sender.GetUDPAddr()
+		inAddr := net.JoinHostPort(p.ToIp.String(), fmt.Sprintf("%v", p.ToPort))
+		receiver := r.getControl(outAddr, inAddr, p)
+		if receiver == nil {
+			r.Unlock()
+			panic("Can't route for host: " + inAddr)
+		}
+
+		e := whatDo(p, receiver)
+		switch e {
+		case ExitNow:
+			r.Unlock()
+			return
+
+		case RouteAndExit:
+			receiver.InjectUDPPacket(p)
+			r.Unlock()
+			return
+
+		case KeepRouting:
+			receiver.InjectUDPPacket(p)
+
+		default:
+			panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
+		}
+
+		r.Unlock()
+	}
+}
+
+// RouteUntilAfterMsgType will route for sender until a message type is seen and sent from sender
+// If the router doesn't have the nebula controller for that address, we panic
+func (r *R) RouteUntilAfterMsgType(sender *nebula.Control, msgType header.MessageType, subType header.MessageSubType) {
+	h := &header.H{}
+	r.RouteExitFunc(sender, func(p *udp.Packet, r *nebula.Control) ExitType {
+		if err := h.Parse(p.Data); err != nil {
+			panic(err)
+		}
+		if h.Type == msgType && h.Subtype == subType {
+			return RouteAndExit
+		}
+
+		return KeepRouting
+	})
+}
+
+// RouteForUntilAfterToAddr will route for sender and return only after it sees and sends a packet destined for toAddr
+// finish can be any of the exitType values except `keepRouting`, the default value is `routeAndExit`
+// If the router doesn't have the nebula controller for that address, we panic
+func (r *R) RouteForUntilAfterToAddr(sender *nebula.Control, toAddr *net.UDPAddr, finish ExitType) {
+	if finish == KeepRouting {
+		finish = RouteAndExit
+	}
+
+	r.RouteExitFunc(sender, func(p *udp.Packet, r *nebula.Control) ExitType {
+		if p.ToIp.Equal(toAddr.IP) && p.ToPort == uint16(toAddr.Port) {
+			return finish
+		}
+
+		return KeepRouting
+	})
+}
+
+// RouteForAllExitFunc will route for every registered controller and calls the whatDo func with each udp packet from
+// whatDo can return:
+//   - exitNow: the packet will not be routed and this call will return immediately
+//   - routeAndExit: this call will return immediately after routing the last packet from sender
+//   - keepRouting: the packet will be routed and whatDo will be called again on the next packet from sender
+func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
+	sc := make([]reflect.SelectCase, len(r.controls))
+	cm := make([]*nebula.Control, len(r.controls))
+
+	i := 0
+	for _, c := range r.controls {
+		sc[i] = reflect.SelectCase{
+			Dir:  reflect.SelectRecv,
+			Chan: reflect.ValueOf(c.GetUDPTxChan()),
+			Send: reflect.Value{},
+		}
+
+		cm[i] = c
+		i++
+	}
+
+	for {
+		x, rx, _ := reflect.Select(sc)
+		r.Lock()
+
+		p := rx.Interface().(*udp.Packet)
+
+		outAddr := cm[x].GetUDPAddr()
+		inAddr := net.JoinHostPort(p.ToIp.String(), fmt.Sprintf("%v", p.ToPort))
+		receiver := r.getControl(outAddr, inAddr, p)
+		if receiver == nil {
+			r.Unlock()
+			panic("Can't route for host: " + inAddr)
+		}
+
+		e := whatDo(p, receiver)
+		switch e {
+		case ExitNow:
+			r.Unlock()
+			return
+
+		case RouteAndExit:
+			receiver.InjectUDPPacket(p)
+			r.Unlock()
+			return
+
+		case KeepRouting:
+			receiver.InjectUDPPacket(p)
+
+		default:
+			panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
+		}
+		r.Unlock()
+	}
+}
+
+// FlushAll will route for every registered controller, exiting once there are no packets left to route
+func (r *R) FlushAll() {
+	sc := make([]reflect.SelectCase, len(r.controls))
+	cm := make([]*nebula.Control, len(r.controls))
+
+	i := 0
+	for _, c := range r.controls {
+		sc[i] = reflect.SelectCase{
+			Dir:  reflect.SelectRecv,
+			Chan: reflect.ValueOf(c.GetUDPTxChan()),
+			Send: reflect.Value{},
+		}
+
+		cm[i] = c
+		i++
+	}
+
+	// Add a default case to exit when nothing is left to send
+	sc = append(sc, reflect.SelectCase{
+		Dir:  reflect.SelectDefault,
+		Chan: reflect.Value{},
+		Send: reflect.Value{},
+	})
+
+	for {
+		x, rx, ok := reflect.Select(sc)
+		if !ok {
+			return
+		}
+		r.Lock()
+
+		p := rx.Interface().(*udp.Packet)
+
+		outAddr := cm[x].GetUDPAddr()
+		inAddr := net.JoinHostPort(p.ToIp.String(), fmt.Sprintf("%v", p.ToPort))
+		receiver := r.getControl(outAddr, inAddr, p)
+		if receiver == nil {
+			r.Unlock()
+			panic("Can't route for host: " + inAddr)
+		}
+		r.Unlock()
+	}
+}
+
+// getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change
+// This is an internal router function, the caller must hold the lock
+func (r *R) getControl(fromAddr, toAddr string, p *udp.Packet) *nebula.Control {
+	if newAddr, ok := r.outNat[fromAddr+":"+toAddr]; ok {
+		p.FromIp = newAddr.IP
+		p.FromPort = uint16(newAddr.Port)
+	}
+
+	c, ok := r.inNat[toAddr]
+	if ok {
+		sHost, sPort, err := net.SplitHostPort(toAddr)
+		if err != nil {
+			panic(err)
+		}
+
+		port, err := strconv.Atoi(sPort)
+		if err != nil {
+			panic(err)
+		}
+
+		r.outNat[c.GetUDPAddr()+":"+fromAddr] = net.UDPAddr{
+			IP:   net.ParseIP(sHost),
+			Port: port,
+		}
+		return c
+	}
+
+	return r.controls[toAddr]
+}
--- a/examples/config.yaml
+++ b/examples/config.yaml
@@ -1,160 +0,0 @@
-# This is the nebula example configuration file. You must edit, at a minimum, the static_host_map, lighthouse, and firewall sections
-
-
-# PKI defines the location of credentials for this node. Each of these can also be inlined by using the yaml ": |" syntax.
-pki:
-  ca: /etc/nebula/ca.crt
-  cert: /etc/nebula/host.crt
-  key: /etc/nebula/host.key
-  #blacklist is a list of certificate fingerprints that we will refuse to talk to
-  #blacklist:
-  #  - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72
-
-# The static host map defines a set of hosts with with fixed IP addresses on the internet (or any network).
-# A host can have multiple fixed IP addresses defined here, and nebula will try each when establishing a tunnel.
-# The syntax is:
-#   "{nebula ip}": ["{routable ip/dns name}:{routable port}"]
-# Example, if your lighthouse has the nebula IP of 192.168.100.1 and has the real ip address of 100.64.22.11 and runs on port 4242:
-static_host_map:
-  "192.168.100.1": ["100.64.22.11:4242"]
-
-
-lighthouse:
-  # am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
-  # you have configured to be lighthouses in your network
-  am_lighthouse: false
-  # serve_dns optionally starts a dns listener that responds to various queries and can even be
-  # delegated to for resolution
-  #serve_dns: false
-  # interval is the number of seconds between updates from this node to a lighthouse.
-  # during updates, a node sends information about its current IP addresses to each node.
-  interval: 60
-  # hosts is a list of lighthouse hosts this node should report to and query from
-  # IMPORTANT: THIS SHOULD BE EMPTY ON LIGHTHOUSE NODES
-  hosts:
-    - "192.168.100.1"
-
-# Port Nebula will be listening on. The default here is 4242. For a lighthouse node, the port should be defined,
-# however using port 0 will dynamically assign a port and is recommended for roaming nodes.
-listen:
-  host: 0.0.0.0
-  port: 4242
-  # Sets the max number of packets to pull from the kernel for each syscall (under systems that support recvmmsg)
-  # default is 64, does not support reload
-  #batch: 64
-  # Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
-  # Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
-  # Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide
-  # max, net.core.rmem_max and net.core.wmem_max
-  #read_buffer: 10485760
-  #write_buffer: 10485760
-
-
-# Local range is used to define a hint about the local network range, which speeds up discovering the fastest
-# path to a network adjacent nebula node.
-#local_range: "172.16.0.0/24"
-
-# Handshake mac is an optional network-wide handshake authentication step that is used to prevent nebula from
-# responding to handshakes from nodes not in possession of the shared secret. This is primarily used to prevent
-# detection of nebula nodes when someone is scanning a network.
-#handshake_mac:
-  #key: "DONOTUSETHISKEY"
-  # You can define multiple accepted keys
-  #accepted_keys:
-    #- "DONOTUSETHISKEY"
-    #- "dontusethiseither"
-
-# sshd can expose informational and administrative functions via ssh this is a
-#sshd:
-  # Toggles the feature
-  #enabled: true
-  # Host and port to listen on, port 22 is not allowed for your safety
-  #listen: 127.0.0.1:2222
-  # A file containing the ssh host private key to use
-  # A decent way to generate one: ssh-keygen -t ed25519 -f ssh_host_ed25519_key -N "" < /dev/null
-  #host_key: ./ssh_host_ed25519_key
-  # A file containing a list of authorized public keys
-  #authorized_users:
-    #- user: steeeeve
-      # keys can be an array of strings or single string
-      #keys:
-        #- "ssh public key string"
-
-# Configure the private interface. Note: addr is baked into the nebula certificate
-tun:
-  # Name of the device
-  dev: nebula1
-  # Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert
-  drop_local_broadcast: false
-  # Toggles forwarding of multicast packets
-  drop_multicast: false
-  # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500
-  tx_queue: 500
-  # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
-  mtu: 1300
-  # Route based MTU overrides, you have known vpn ip paths that can support larger MTUs you can increase/decrease them here
-  routes:
-    #- mtu: 8800
-    #  route: 10.0.0.0/16
-
-# TODO
-# Configure logging level
-logging:
-  # panic, fatal, error, warning, info, or debug. Default is info
-  level: info
-  # json or text formats currently available. Default is text
-  format: text
-
-#stats:
-  #type: graphite
-  #prefix: nebula
-  #protocol: tcp
-  #host: 127.0.0.1:9999
-  #interval: 10s
-
-  #type: prometheus
-  #listen: 127.0.0.1:8080
-  #path: /metrics
-  #namespace: prometheusns
-  #subsystem: nebula
-  #interval: 10s
-
-# Nebula security group configuration
-firewall:
-  conntrack:
-    tcp_timeout: 120h
-    udp_timeout: 3m
-    default_timeout: 10m
-    max_connections: 100000
-
-  # The firewall is default deny. There is no way to write a deny rule.
-  # Rules are comprised of a protocol, port, and one or more of host, group, or CIDR
-  # Logical evaluation is roughly: port AND proto AND ca_sha AND ca_name AND (host OR group OR groups OR cidr)
-  # - port: Takes `0` or `any` as any, a single number `80`, a range `200-901`, or `fragment` to match second and further fragments of fragmented packets (since there is no port available).
-  #   code: same as port but makes more sense when talking about ICMP, TODO: this is not currently implemented in a way that works, use `any`
-  #   proto: `any`, `tcp`, `udp`, or `icmp`
-  #   host: `any` or a literal hostname, ie `test-host`
-  #   group: `any` or a literal group name, ie `default-group`
-  #   groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass
-  #   cidr: a CIDR, `0.0.0.0/0` is any.
-  #   ca_name: An issuing CA name
-  #   ca_sha: An issuing CA shasum
-
-  outbound:
-    # Allow all outbound traffic from this node
-    - port: any
-      proto: any
-      host: any
-
-  inbound:
-    # Allow icmp between any nebula hosts
-    - port: any
-      proto: icmp
-      host: any
-
-    # Allow tcp/443 from any host with BOTH laptop and home group
-    - port: 443
-      proto: tcp
-      groups:
-        - laptop
-        - home
--- a/examples/config.yml
+++ b/examples/config.yml
@@ -0,0 +1,296 @@
+# This is the nebula example configuration file. You must edit, at a minimum, the static_host_map, lighthouse, and firewall sections
+# Some options in this file are HUPable, including the pki section. (A HUP will reload credentials from disk without affecting existing tunnels)
+
+# PKI defines the location of credentials for this node. Each of these can also be inlined by using the yaml ": |" syntax.
+pki:
+  # The CAs that are accepted by this node. Must contain one or more certificates created by 'nebula-cert ca'
+  ca: /etc/nebula/ca.crt
+  cert: /etc/nebula/host.crt
+  key: /etc/nebula/host.key
+  # blocklist is a list of certificate fingerprints that we will refuse to talk to
+  #blocklist:
+  #  - c99d4e650533b92061b09918e838a5a0a6aaee21eed1d12fd937682865936c72
+  # disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid.
+  #disconnect_invalid: false
+
+# The static host map defines a set of hosts with fixed IP addresses on the internet (or any network).
+# A host can have multiple fixed IP addresses defined here, and nebula will try each when establishing a tunnel.
+# The syntax is:
+#   "{nebula ip}": ["{routable ip/dns name}:{routable port}"]
+# Example, if your lighthouse has the nebula IP of 192.168.100.1 and has the real ip address of 100.64.22.11 and runs on port 4242:
+static_host_map:
+  "192.168.100.1": ["100.64.22.11:4242"]
+
+
+lighthouse:
+  # am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
+  # you have configured to be lighthouses in your network
+  am_lighthouse: false
+  # serve_dns optionally starts a dns listener that responds to various queries and can even be
+  # delegated to for resolution
+  #serve_dns: false
+  #dns:
+    # The DNS host defines the IP to bind the dns listener to. This also allows binding to the nebula node IP.
+    #host: 0.0.0.0
+    #port: 53
+  # interval is the number of seconds between updates from this node to a lighthouse.
+  # during updates, a node sends information about its current IP addresses to each node.
+  interval: 60
+  # hosts is a list of lighthouse hosts this node should report to and query from
+  # IMPORTANT: THIS SHOULD BE EMPTY ON LIGHTHOUSE NODES
+  # IMPORTANT2: THIS SHOULD BE LIGHTHOUSES' NEBULA IPs, NOT LIGHTHOUSES' REAL ROUTABLE IPs
+  hosts:
+    - "192.168.100.1"
+
+  # remote_allow_list allows you to control ip ranges that this node will
+  # consider when handshaking to another node. By default, any remote IPs are
+  # allowed. You can provide CIDRs here with `true` to allow and `false` to
+  # deny. The most specific CIDR rule applies to each remote. If all rules are
+  # "allow", the default will be "deny", and vice-versa. If both "allow" and
+  # "deny" rules are present, then you MUST set a rule for "0.0.0.0/0" as the
+  # default.
+  #remote_allow_list:
+    # Example to block IPs from this subnet from being used for remote IPs.
+    #"172.16.0.0/12": false
+
+    # A more complicated example, allow public IPs but only private IPs from a specific subnet
+    #"0.0.0.0/0": true
+    #"10.0.0.0/8": false
+    #"10.42.42.0/24": true
+
+  # EXPERIMENTAL: This option my change or disappear in the future.
+  # Optionally allows the definition of remote_allow_list blocks
+  # specific to an inside VPN IP CIDR.
+  #remote_allow_ranges:
+    # This rule would only allow only private IPs for this VPN range
+    #"10.42.42.0/24":
+      #"192.168.0.0/16": true
+
+  # local_allow_list allows you to filter which local IP addresses we advertise
+  # to the lighthouses. This uses the same logic as `remote_allow_list`, but
+  # additionally, you can specify an `interfaces` map of regular expressions
+  # to match against interface names. The regexp must match the entire name.
+  # All interface rules must be either true or false (and the default will be
+  # the inverse). CIDR rules are matched after interface name rules.
+  # Default is all local IP addresses.
+  #local_allow_list:
+    # Example to block tun0 and all docker interfaces.
+    #interfaces:
+      #tun0: false
+      #'docker.*': false
+    # Example to only advertise this subnet to the lighthouse.
+    #"10.0.0.0/8": true
+
+# Port Nebula will be listening on. The default here is 4242. For a lighthouse node, the port should be defined,
+# however using port 0 will dynamically assign a port and is recommended for roaming nodes.
+listen:
+  # To listen on both any ipv4 and ipv6 use "[::]"
+  host: 0.0.0.0
+  port: 4242
+  # Sets the max number of packets to pull from the kernel for each syscall (under systems that support recvmmsg)
+  # default is 64, does not support reload
+  #batch: 64
+  # Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
+  # Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
+  # Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide
+  # max, net.core.rmem_max and net.core.wmem_max
+  #read_buffer: 10485760
+  #write_buffer: 10485760
+
+# EXPERIMENTAL: This option is currently only supported on linux and may
+# change in future minor releases.
+#
+# Routines is the number of thread pairs to run that consume from the tun and UDP queues.
+# Currently, this defaults to 1 which means we have 1 tun queue reader and 1
+# UDP queue reader. Setting this above one will set IFF_MULTI_QUEUE on the tun
+# device and SO_REUSEPORT on the UDP socket to allow multiple queues.
+#routines: 1
+
+punchy:
+  # Continues to punch inbound/outbound at a regular interval to avoid expiration of firewall nat mappings
+  punch: true
+
+  # respond means that a node you are trying to reach will connect back out to you if your hole punching fails
+  # this is extremely useful if one node is behind a difficult nat, such as a symmetric NAT
+  # Default is false
+  #respond: true
+
+  # delays a punch response for misbehaving NATs, default is 1 second, respond must be true to take effect
+  #delay: 1s
+
+# Cipher allows you to choose between the available ciphers for your network. Options are chachapoly or aes
+# IMPORTANT: this value must be identical on ALL NODES/LIGHTHOUSES. We do not/will not support use of different ciphers simultaneously!
+#cipher: chachapoly
+
+# Preferred ranges is used to define a hint about the local network ranges, which speeds up discovering the fastest
+# path to a network adjacent nebula node.
+# NOTE: the previous option "local_range" only allowed definition of a single range
+# and has been deprecated for "preferred_ranges"
+#preferred_ranges: ["172.16.0.0/24"]
+
+# sshd can expose informational and administrative functions via ssh this is a
+#sshd:
+  # Toggles the feature
+  #enabled: true
+  # Host and port to listen on, port 22 is not allowed for your safety
+  #listen: 127.0.0.1:2222
+  # A file containing the ssh host private key to use
+  # A decent way to generate one: ssh-keygen -t ed25519 -f ssh_host_ed25519_key -N "" < /dev/null
+  #host_key: ./ssh_host_ed25519_key
+  # A file containing a list of authorized public keys
+  #authorized_users:
+    #- user: steeeeve
+      # keys can be an array of strings or single string
+      #keys:
+        #- "ssh public key string"
+
+# Configure the private interface. Note: addr is baked into the nebula certificate
+tun:
+  # When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root)
+  disabled: false
+  # Name of the device
+  dev: nebula1
+  # Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert
+  drop_local_broadcast: false
+  # Toggles forwarding of multicast packets
+  drop_multicast: false
+  # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500
+  tx_queue: 500
+  # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
+  mtu: 1300
+  # Route based MTU overrides, you have known vpn ip paths that can support larger MTUs you can increase/decrease them here
+  routes:
+    #- mtu: 8800
+    #  route: 10.0.0.0/16
+  # Unsafe routes allows you to route traffic over nebula to non-nebula nodes
+  # Unsafe routes should be avoided unless you have hosts/services that cannot run nebula
+  # NOTE: The nebula certificate of the "via" node *MUST* have the "route" defined as a subnet in its certificate
+  # `mtu` will default to tun mtu if this option is not specified
+  # `metric` will default to 0 if this option is not specified
+  unsafe_routes:
+    #- route: 172.16.1.0/24
+    #  via: 192.168.100.99
+    #  mtu: 1300
+    #  metric: 100
+
+
+# TODO
+# Configure logging level
+logging:
+  # panic, fatal, error, warning, info, or debug. Default is info
+  level: info
+  # json or text formats currently available. Default is text
+  format: text
+  # Disable timestamp logging. useful when output is redirected to logging system that already adds timestamps. Default is false
+  #disable_timestamp: true
+  # timestamp format is specified in Go time format, see:
+  #     https://golang.org/pkg/time/#pkg-constants
+  # default when `format: json`: "2006-01-02T15:04:05Z07:00" (RFC3339)
+  # default when `format: text`:
+  #     when TTY attached: seconds since beginning of execution
+  #     otherwise: "2006-01-02T15:04:05Z07:00" (RFC3339)
+  # As an example, to log as RFC3339 with millisecond precision, set to:
+  #timestamp_format: "2006-01-02T15:04:05.000Z07:00"
+
+#stats:
+  #type: graphite
+  #prefix: nebula
+  #protocol: tcp
+  #host: 127.0.0.1:9999
+  #interval: 10s
+
+  #type: prometheus
+  #listen: 127.0.0.1:8080
+  #path: /metrics
+  #namespace: prometheusns
+  #subsystem: nebula
+  #interval: 10s
+
+  # enables counter metrics for meta packets
+  #   e.g.: `messages.tx.handshake`
+  # NOTE: `message.{tx,rx}.recv_error` is always emitted
+  #message_metrics: false
+
+  # enables detailed counter metrics for lighthouse packets
+  #   e.g.: `lighthouse.rx.HostQuery`
+  #lighthouse_metrics: false
+
+# Handshake Manger Settings
+handshakes:
+  # Handshakes are sent to all known addresses at each interval with a linear backoff,
+  # Wait try_interval after the 1st attempt, 2 * try_interval after the 2nd, etc, until the handshake is older than timeout
+  # A 100ms interval with the default 10 retries will give a handshake 5.5 seconds to resolve before timing out
+  #try_interval: 100ms
+  #retries: 20
+
+  # trigger_buffer is the size of the buffer channel for quickly sending handshakes
+  # after receiving the response for lighthouse queries
+  #trigger_buffer: 64
+
+  # psk can be used to mask the contents of handshakes and makes handshaking with unintended recipients more difficult
+  # all settings respond to a reload
+  psk:
+    # mode defines how the pre shared keys can be used in a handshake
+    # `none` (the default) does not send or receive using a psk. Ideally `enforced` is used
+    # `transitional-accepting` will send handshakes without using a psk and can receive handshakes using a psk we know about
+    # `transitional-sending` will send handshakes using a psk but will still accept handshakes without them
+    # `enforced` enforces the use of a psk for all tunnels. Any node not also using `enforced` or `transitional-sending` can not handshake with us
+    #
+    # When moving from `none` to `enforced` you will want to change every node in the mesh to `transitional-accepting` and reload
+    # then move every node to `transitional-sending` then reload, and finally `enforced` then reload. This allows you to
+    # avoid stopping the world to use psk. You must ensure at `transitional-accepting` that all nodes have the same psks.
+    #mode: none
+
+    # In `transitional-accepting`, `transitional-sending` and `enforced` modes, the keys provided here are sent through
+    # hkdf with the intended recipients ip used in the info section. This helps guard against handshaking with the wrong
+    # host if your static_host_map or lighthouse(s) has incorrect information.
+    #
+    # Setting keys if mode is `none` has no effect.
+    #
+    # Only the first key is used for outbound handshakes but all keys provided will be tried in the order specified, on
+    # incoming handshakes. This is to allow for psk rotation.
+    #keys:
+      # - shared secret string, this one is used in all outbound handshakes
+      # - this is a fallback key, received handshakes can use this
+      # - another fallback, received handshakes can use this one too
+      # - "\x68\x65\x6c\x6c\x6f\x20\x66\x72\x69\x65\x6e\x64\x73" # for raw bytes if you desire
+
+# Nebula security group configuration
+firewall:
+  conntrack:
+    tcp_timeout: 12m
+    udp_timeout: 3m
+    default_timeout: 10m
+    max_connections: 100000
+
+  # The firewall is default deny. There is no way to write a deny rule.
+  # Rules are comprised of a protocol, port, and one or more of host, group, or CIDR
+  # Logical evaluation is roughly: port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr)
+  # - port: Takes `0` or `any` as any, a single number `80`, a range `200-901`, or `fragment` to match second and further fragments of fragmented packets (since there is no port available).
+  #   code: same as port but makes more sense when talking about ICMP, TODO: this is not currently implemented in a way that works, use `any`
+  #   proto: `any`, `tcp`, `udp`, or `icmp`
+  #   host: `any` or a literal hostname, ie `test-host`
+  #   group: `any` or a literal group name, ie `default-group`
+  #   groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass
+  #   cidr: a CIDR, `0.0.0.0/0` is any.
+  #   ca_name: An issuing CA name
+  #   ca_sha: An issuing CA shasum
+
+  outbound:
+    # Allow all outbound traffic from this node
+    - port: any
+      proto: any
+      host: any
+
+  inbound:
+    # Allow icmp between any nebula hosts
+    - port: any
+      proto: icmp
+      host: any
+
+    # Allow tcp/443 from any host with BOTH laptop and home group
+    - port: 443
+      proto: tcp
+      groups:
+        - laptop
+        - home
--- a/examples/quickstart-vagrant/README.md
+++ b/examples/quickstart-vagrant/README.md
@@ -2,22 +2,7 @@

 This guide is intended to bring up a vagrant environment with 1 lighthouse and 2 generic hosts running nebula.

-## Pre-requisites
-
-There are two pre-requisites prior to bringing up the vagrant environment
-
-  - build the binaries locally for the vagrant deploy
-  - create a virtualenv for ansible
-
-### Building the binaries
-
-Build the `nebula` and `nebula-cert` binaries for vagrant by doing the following
-
-`make bin-vagrant` (under the src directory with Makefile)
-
-For convenience, ansible will run this for you in every deploy (see `ansible/playbook.yml`)
-
-### Creating the virtualenv
+## Creating the virtualenv for ansible

 Within the `quickstart/` directory, do the following

--- a/examples/quickstart-vagrant/ansible/filter_plugins/to_nebula_ip.py
+++ b/examples/quickstart-vagrant/ansible/filter_plugins/to_nebula_ip.py
@@ -9,7 +9,7 @@ class FilterModule(object):
        }

    def to_nebula_ip(self, ip_str):
-        ip_list = map(int, ip_str.split("."))
+        ip_list = list(map(int, ip_str.split(".")))
        ip_list[0] = 10
        ip_list[1] = 168
        ip = '.'.join(map(str, ip_list))
--- a/examples/quickstart-vagrant/ansible/playbook.yml
+++ b/examples/quickstart-vagrant/ansible/playbook.yml
@@ -8,7 +8,10 @@
  connection: local
  hosts: localhost
  tasks:
-    - command: chdir=../../../ make bin-vagrant
+    - command: chdir=../../../ make build/linux-amd64/"{{ item }}"
+      with_items:
+        - nebula
+        - nebula-cert
  tags:
    - build-nebula
   
--- a/examples/quickstart-vagrant/ansible/roles/nebula/files/systemd.nebula.service
+++ b/examples/quickstart-vagrant/ansible/roles/nebula/files/systemd.nebula.service
@@ -5,8 +5,6 @@ After=basic.target network.target

 [Service]
 SyslogIdentifier=nebula
-StandardOutput=syslog
-StandardError=syslog
 ExecReload=/bin/kill -HUP $MAINPID
 ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml
 Restart=always
--- a/examples/quickstart-vagrant/ansible/roles/nebula/tasks/main.yml
+++ b/examples/quickstart-vagrant/ansible/roles/nebula/tasks/main.yml
@@ -8,7 +8,7 @@
    - nebula-conf
  
 - name: install built nebula binary
-  copy: src=../../../../../{{ item }} dest=/usr/local/bin mode=0755
+  copy: src="../../../../../build/linux-amd64/{{ item }}" dest="/usr/local/bin" mode=0755
  with_items:
    - nebula
    - nebula-cert
@@ -22,6 +22,12 @@
    - vagrant-test-ca.key
    - vagrant-test-ca.crt

+- name: remove previously signed host certificate
+  file: dest=/etc/nebula/{{ item }} state=absent
+  with_items:
+    - host.crt
+    - host.key
+
 - name: sign using the root key
  command: nebula-cert sign -ca-crt /opt/vagrant-test-ca.crt -ca-key /opt/vagrant-test-ca.key -duration 4320h -groups vagrant -ip {{ hostvars[inventory_hostname][vagrant_ifce]['ipv4']['address'] | to_nebula_ip }}/9 -name {{ ansible_hostname }}.nebula -out-crt /etc/nebula/host.crt -out-key /etc/nebula/host.key

--- a/examples/quickstart-vagrant/ansible/roles/nebula/templates/config.yml.j2
+++ b/examples/quickstart-vagrant/ansible/roles/nebula/templates/config.yml.j2
@@ -42,8 +42,10 @@ lighthouse:
  am_lighthouse: false
 {% endif %}
  interval: 60
+{% if 'generic' in group_names %}
  hosts:
    - {{ hostvars[groups['lighthouse'][0]][vagrant_ifce]['ipv4']['address'] | to_nebula_ip }}
+{% endif %}
  
 # Configure the private interface
 tun:
--- a/examples/service_scripts/nebula.service
+++ b/examples/service_scripts/nebula.service
@@ -2,11 +2,10 @@
 Description=nebula
 Wants=basic.target
 After=basic.target network.target
+Before=sshd.service

 [Service]
 SyslogIdentifier=nebula
-StandardOutput=syslog
-StandardError=syslog
 ExecReload=/bin/kill -HUP $MAINPID
 ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml
 Restart=always
--- a/firewall.go
+++ b/firewall.go
@@ -1,32 +1,24 @@
 package nebula

 import (
-	"encoding/binary"
-	"encoding/json"
-	"fmt"
-	"net"
-	"sync"
-	"time"
-
 	"crypto/sha256"
+	"encoding/binary"
 	"encoding/hex"
 	"errors"
+	"fmt"
+	"net"
 	"reflect"
 	"strconv"
 	"strings"
+	"sync"
+	"time"

 	"github.com/rcrowley/go-metrics"
+	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
-)
-
-const (
-	fwProtoAny  = 0 // When we want to handle HOPOPT (0) we can change this, if ever
-	fwProtoTCP  = 6
-	fwProtoUDP  = 17
-	fwProtoICMP = 1
-
-	fwPortAny      = 0  // Special value for matching `port: any`
-	fwPortFragment = -1 // Special value for matching `port: fragment`
+	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/firewall"
 )

 const tcpACK = 0x10
@@ -38,13 +30,19 @@ type FirewallInterface interface {

 type conn struct {
 	Expires time.Time // Time when this conntrack entry will expire
-	Seq     uint32    // If tcp rtt tracking is enabled this will be the seq we are looking for an ack
 	Sent    time.Time // If tcp rtt tracking is enabled this will be when Seq was last set
+	Seq     uint32    // If tcp rtt tracking is enabled this will be the seq we are looking for an ack
+
+	// record why the original connection passed the firewall, so we can re-validate
+	// after ruleset changes. Note, rulesVersion is a uint16 so that these two
+	// fields pack for free after the uint32 above
+	incoming     bool
+	rulesVersion uint16
 }

 // TODO: need conntrack max tracked connections handling
 type Firewall struct {
-	Conns map[FirewallPacket]*conn
+	Conntrack *FirewallConntrack

 	InRules  *FirewallTable
 	OutRules *FirewallTable
@@ -55,16 +53,31 @@ type Firewall struct {
 	UDPTimeout     time.Duration //linux: 180s max
 	DefaultTimeout time.Duration //linux: 600s

-	TimerWheel *TimerWheel
-
 	// Used to ensure we don't emit local packets for ips we don't own
-	localIps *CIDRTree
+	localIps *cidr.Tree4

-	connMutex sync.Mutex
 	rules        string
+	rulesVersion uint16

 	trackTCPRTT     bool
 	metricTCPRTT    metrics.Histogram
+	incomingMetrics firewallMetrics
+	outgoingMetrics firewallMetrics
+
+	l *logrus.Logger
+}
+
+type firewallMetrics struct {
+	droppedLocalIP  metrics.Counter
+	droppedRemoteIP metrics.Counter
+	droppedNoRule   metrics.Counter
+}
+
+type FirewallConntrack struct {
+	sync.Mutex
+
+	Conns      map[firewall.Packet]*conn
+	TimerWheel *TimerWheel
 }

 type FirewallTable struct {
@@ -83,64 +96,26 @@ func newFirewallTable() *FirewallTable {
 	}
 }

+type FirewallCA struct {
+	Any     *FirewallRule
+	CANames map[string]*FirewallRule
+	CAShas  map[string]*FirewallRule
+}
+
 type FirewallRule struct {
-	// Any makes Hosts, Groups, and CIDR irrelevant. CAName and CASha still need to be checked
+	// Any makes Hosts, Groups, and CIDR irrelevant
 	Any    bool
 	Hosts  map[string]struct{}
 	Groups [][]string
-	CIDR    *CIDRTree
-	CANames map[string]struct{}
-	CAShas  map[string]struct{}
+	CIDR   *cidr.Tree4
 }

 // Even though ports are uint16, int32 maps are faster for lookup
 // Plus we can use `-1` for fragment rules
-type firewallPort map[int32]*FirewallRule
-
-type FirewallPacket struct {
-	LocalIP    uint32
-	RemoteIP   uint32
-	LocalPort  uint16
-	RemotePort uint16
-	Protocol   uint8
-	Fragment   bool
-}
-
-func (fp *FirewallPacket) Copy() *FirewallPacket {
-	return &FirewallPacket{
-		LocalIP:    fp.LocalIP,
-		RemoteIP:   fp.RemoteIP,
-		LocalPort:  fp.LocalPort,
-		RemotePort: fp.RemotePort,
-		Protocol:   fp.Protocol,
-		Fragment:   fp.Fragment,
-	}
-}
-
-func (fp FirewallPacket) MarshalJSON() ([]byte, error) {
-	var proto string
-	switch fp.Protocol {
-	case fwProtoTCP:
-		proto = "tcp"
-	case fwProtoICMP:
-		proto = "icmp"
-	case fwProtoUDP:
-		proto = "udp"
-	default:
-		proto = fmt.Sprintf("unknown %v", fp.Protocol)
-	}
-	return json.Marshal(m{
-		"LocalIP":    int2ip(fp.LocalIP).String(),
-		"RemoteIP":   int2ip(fp.RemoteIP).String(),
-		"LocalPort":  fp.LocalPort,
-		"RemotePort": fp.RemotePort,
-		"Protocol":   proto,
-		"Fragment":   fp.Fragment,
-	})
-}
+type firewallPort map[int32]*FirewallCA

 // NewFirewall creates a new Firewall object. A TimerWheel is created for you from the provided timeouts.
-func NewFirewall(tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c *cert.NebulaCertificate) *Firewall {
+func NewFirewall(l *logrus.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c *cert.NebulaCertificate) *Firewall {
 	//TODO: error on 0 duration
 	var min, max time.Duration

@@ -158,7 +133,7 @@ func NewFirewall(tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c *cert.N
 		max = defaultTimeout
 	}

-	localIps := NewCIDRTree()
+	localIps := cidr.NewTree4()
 	for _, ip := range c.Details.Ips {
 		localIps.AddCIDR(&net.IPNet{IP: ip.IP, Mask: net.IPMask{255, 255, 255, 255}}, struct{}{})
 	}
@@ -168,33 +143,48 @@ func NewFirewall(tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c *cert.N
 	}

 	return &Firewall{
-		Conns:          make(map[FirewallPacket]*conn),
+		Conntrack: &FirewallConntrack{
+			Conns:      make(map[firewall.Packet]*conn),
+			TimerWheel: NewTimerWheel(min, max),
+		},
 		InRules:        newFirewallTable(),
 		OutRules:       newFirewallTable(),
-		TimerWheel:     NewTimerWheel(min, max),
 		TCPTimeout:     tcpTimeout,
 		UDPTimeout:     UDPTimeout,
 		DefaultTimeout: defaultTimeout,
 		localIps:       localIps,
+		l:              l,
+
 		metricTCPRTT: metrics.GetOrRegisterHistogram("network.tcp.rtt", nil, metrics.NewExpDecaySample(1028, 0.015)),
+		incomingMetrics: firewallMetrics{
+			droppedLocalIP:  metrics.GetOrRegisterCounter("firewall.incoming.dropped.local_ip", nil),
+			droppedRemoteIP: metrics.GetOrRegisterCounter("firewall.incoming.dropped.remote_ip", nil),
+			droppedNoRule:   metrics.GetOrRegisterCounter("firewall.incoming.dropped.no_rule", nil),
+		},
+		outgoingMetrics: firewallMetrics{
+			droppedLocalIP:  metrics.GetOrRegisterCounter("firewall.outgoing.dropped.local_ip", nil),
+			droppedRemoteIP: metrics.GetOrRegisterCounter("firewall.outgoing.dropped.remote_ip", nil),
+			droppedNoRule:   metrics.GetOrRegisterCounter("firewall.outgoing.dropped.no_rule", nil),
+		},
 	}
 }

-func NewFirewallFromConfig(nc *cert.NebulaCertificate, c *Config) (*Firewall, error) {
+func NewFirewallFromConfig(l *logrus.Logger, nc *cert.NebulaCertificate, c *config.C) (*Firewall, error) {
 	fw := NewFirewall(
-		c.GetDuration("firewall.conntrack.tcp_timeout", time.Duration(time.Minute*12)),
-		c.GetDuration("firewall.conntrack.udp_timeout", time.Duration(time.Minute*3)),
-		c.GetDuration("firewall.conntrack.default_timeout", time.Duration(time.Minute*10)),
+		l,
+		c.GetDuration("firewall.conntrack.tcp_timeout", time.Minute*12),
+		c.GetDuration("firewall.conntrack.udp_timeout", time.Minute*3),
+		c.GetDuration("firewall.conntrack.default_timeout", time.Minute*10),
 		nc,
 		//TODO: max_connections
 	)

-	err := AddFirewallRulesFromConfig(false, c, fw)
+	err := AddFirewallRulesFromConfig(l, false, c, fw)
 	if err != nil {
 		return nil, err
 	}

-	err = AddFirewallRulesFromConfig(true, c, fw)
+	err = AddFirewallRulesFromConfig(l, true, c, fw)
 	if err != nil {
 		return nil, err
 	}
@@ -204,11 +194,17 @@ func NewFirewallFromConfig(nc *cert.NebulaCertificate, c *Config) (*Firewall, er

 // AddRule properly creates the in memory rule structure for a firewall table.
 func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip *net.IPNet, caName string, caSha string) error {
+	// Under gomobile, stringing a nil pointer with fmt causes an abort in debug mode for iOS
+	// https://github.com/golang/go/issues/14131
+	sIp := ""
+	if ip != nil {
+		sIp = ip.String()
+	}

 	// We need this rule string because we generate a hash. Removing this will break firewall reload.
 	ruleString := fmt.Sprintf(
 		"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, caName: %v, caSha: %s",
-		incoming, proto, startPort, endPort, groups, host, ip, caName, caSha,
+		incoming, proto, startPort, endPort, groups, host, sIp, caName, caSha,
 	)
 	f.rules += ruleString + "\n"

@@ -216,7 +212,7 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 	if !incoming {
 		direction = "outgoing"
 	}
-	l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "ip": ip, "caName": caName, "caSha": caSha}).
+	f.l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "ip": sIp, "caName": caName, "caSha": caSha}).
 		Info("Firewall rule added")

 	var (
@@ -231,13 +227,13 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 	}

 	switch proto {
-	case fwProtoTCP:
+	case firewall.ProtoTCP:
 		fp = ft.TCP
-	case fwProtoUDP:
+	case firewall.ProtoUDP:
 		fp = ft.UDP
-	case fwProtoICMP:
+	case firewall.ProtoICMP:
 		fp = ft.ICMP
-	case fwProtoAny:
+	case firewall.ProtoAny:
 		fp = ft.AnyProto
 	default:
 		return fmt.Errorf("unknown protocol %v", proto)
@@ -252,7 +248,7 @@ func (f *Firewall) GetRuleHash() string {
 	return hex.EncodeToString(sum[:])
 }

-func AddFirewallRulesFromConfig(inbound bool, config *Config, fw FirewallInterface) error {
+func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw FirewallInterface) error {
 	var table string
 	if inbound {
 		table = "firewall.inbound"
@@ -260,7 +256,7 @@ func AddFirewallRulesFromConfig(inbound bool, config *Config, fw FirewallInterfa
 		table = "firewall.outbound"
 	}

-	r := config.Get(table)
+	r := c.Get(table)
 	if r == nil {
 		return nil
 	}
@@ -272,7 +268,7 @@ func AddFirewallRulesFromConfig(inbound bool, config *Config, fw FirewallInterfa

 	for i, t := range rs {
 		var groups []string
-		r, err := convertRule(t)
+		r, err := convertRule(l, t, table, i)
 		if err != nil {
 			return fmt.Errorf("%s rule #%v; %s", table, i, err)
 		}
@@ -315,13 +311,13 @@ func AddFirewallRulesFromConfig(inbound bool, config *Config, fw FirewallInterfa
 		var proto uint8
 		switch r.Proto {
 		case "any":
-			proto = fwProtoAny
+			proto = firewall.ProtoAny
 		case "tcp":
-			proto = fwProtoTCP
+			proto = firewall.ProtoTCP
 		case "udp":
-			proto = fwProtoUDP
+			proto = firewall.ProtoUDP
 		case "icmp":
-			proto = fwProtoICMP
+			proto = firewall.ProtoICMP
 		default:
 			return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
 		}
@@ -343,15 +339,36 @@ func AddFirewallRulesFromConfig(inbound bool, config *Config, fw FirewallInterfa
 	return nil
 }

-func (f *Firewall) Drop(packet []byte, fp FirewallPacket, incoming bool, c *cert.NebulaCertificate, caPool *cert.NebulaCAPool) bool {
+var ErrInvalidRemoteIP = errors.New("remote IP is not in remote certificate subnets")
+var ErrInvalidLocalIP = errors.New("local IP is not in list of handled local IPs")
+var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
+
+// Drop returns an error if the packet should be dropped, explaining why. It
+// returns nil if the packet should not be dropped.
+func (f *Firewall) Drop(packet []byte, fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) error {
 	// Check if we spoke to this tuple, if we did then allow this packet
-	if f.inConns(packet, fp, incoming) {
-		return false
+	if f.inConns(packet, fp, incoming, h, caPool, localCache) {
+		return nil
+	}
+
+	// Make sure remote address matches nebula certificate
+	if remoteCidr := h.remoteCidr; remoteCidr != nil {
+		if remoteCidr.Contains(fp.RemoteIP) == nil {
+			f.metrics(incoming).droppedRemoteIP.Inc(1)
+			return ErrInvalidRemoteIP
+		}
+	} else {
+		// Simple case: Certificate has one IP and no subnets
+		if fp.RemoteIP != h.vpnIp {
+			f.metrics(incoming).droppedRemoteIP.Inc(1)
+			return ErrInvalidRemoteIP
+		}
 	}

 	// Make sure we are supposed to be handling this local ip address
 	if f.localIps.Contains(fp.LocalIP) == nil {
-		return true
+		f.metrics(incoming).droppedLocalIP.Inc(1)
+		return ErrInvalidLocalIP
 	}

 	table := f.OutRules
@@ -360,14 +377,23 @@ func (f *Firewall) Drop(packet []byte, fp FirewallPacket, incoming bool, c *cert
 	}

 	// We now know which firewall table to check against
-	if !table.match(fp, incoming, c, caPool) {
-		return true
+	if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
+		f.metrics(incoming).droppedNoRule.Inc(1)
+		return ErrNoMatchingRule
 	}

 	// We always want to conntrack since it is a faster operation
 	f.addConn(packet, fp, incoming)

-	return false
+	return nil
+}
+
+func (f *Firewall) metrics(incoming bool) firewallMetrics {
+	if incoming {
+		return f.incomingMetrics
+	} else {
+		return f.outgoingMetrics
+	}
 }

 // Destroy cleans up any known cyclical references so the object can be free'd my GC. This should be called if a new
@@ -377,77 +403,132 @@ func (f *Firewall) Destroy() {
 }

 func (f *Firewall) EmitStats() {
-	conntrackCount := len(f.Conns)
+	conntrack := f.Conntrack
+	conntrack.Lock()
+	conntrackCount := len(conntrack.Conns)
+	conntrack.Unlock()
 	metrics.GetOrRegisterGauge("firewall.conntrack.count", nil).Update(int64(conntrackCount))
+	metrics.GetOrRegisterGauge("firewall.rules.version", nil).Update(int64(f.rulesVersion))
 }

-func (f *Firewall) inConns(packet []byte, fp FirewallPacket, incoming bool) bool {
-	f.connMutex.Lock()
+func (f *Firewall) inConns(packet []byte, fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) bool {
+	if localCache != nil {
+		if _, ok := localCache[fp]; ok {
+			return true
+		}
+	}
+	conntrack := f.Conntrack
+	conntrack.Lock()

 	// Purge every time we test
-	ep, has := f.TimerWheel.Purge()
+	ep, has := conntrack.TimerWheel.Purge()
 	if has {
 		f.evict(ep)
 	}

-	c, ok := f.Conns[fp]
+	c, ok := conntrack.Conns[fp]

 	if !ok {
-		f.connMutex.Unlock()
+		conntrack.Unlock()
 		return false
 	}

+	if c.rulesVersion != f.rulesVersion {
+		// This conntrack entry was for an older rule set, validate
+		// it still passes with the current rule set
+		table := f.OutRules
+		if c.incoming {
+			table = f.InRules
+		}
+
+		// We now know which firewall table to check against
+		if !table.match(fp, c.incoming, h.ConnectionState.peerCert, caPool) {
+			if f.l.Level >= logrus.DebugLevel {
+				h.logger(f.l).
+					WithField("fwPacket", fp).
+					WithField("incoming", c.incoming).
+					WithField("rulesVersion", f.rulesVersion).
+					WithField("oldRulesVersion", c.rulesVersion).
+					Debugln("dropping old conntrack entry, does not match new ruleset")
+			}
+			delete(conntrack.Conns, fp)
+			conntrack.Unlock()
+			return false
+		}
+
+		if f.l.Level >= logrus.DebugLevel {
+			h.logger(f.l).
+				WithField("fwPacket", fp).
+				WithField("incoming", c.incoming).
+				WithField("rulesVersion", f.rulesVersion).
+				WithField("oldRulesVersion", c.rulesVersion).
+				Debugln("keeping old conntrack entry, does match new ruleset")
+		}
+
+		c.rulesVersion = f.rulesVersion
+	}
+
 	switch fp.Protocol {
-	case fwProtoTCP:
+	case firewall.ProtoTCP:
 		c.Expires = time.Now().Add(f.TCPTimeout)
 		if incoming {
 			f.checkTCPRTT(c, packet)
 		} else {
 			setTCPRTTTracking(c, packet)
 		}
-	case fwProtoUDP:
+	case firewall.ProtoUDP:
 		c.Expires = time.Now().Add(f.UDPTimeout)
 	default:
 		c.Expires = time.Now().Add(f.DefaultTimeout)
 	}

-	f.connMutex.Unlock()
+	conntrack.Unlock()
+
+	if localCache != nil {
+		localCache[fp] = struct{}{}
+	}

 	return true
 }

-func (f *Firewall) addConn(packet []byte, fp FirewallPacket, incoming bool) {
+func (f *Firewall) addConn(packet []byte, fp firewall.Packet, incoming bool) {
 	var timeout time.Duration
 	c := &conn{}

 	switch fp.Protocol {
-	case fwProtoTCP:
+	case firewall.ProtoTCP:
 		timeout = f.TCPTimeout
 		if !incoming {
 			setTCPRTTTracking(c, packet)
 		}
-	case fwProtoUDP:
+	case firewall.ProtoUDP:
 		timeout = f.UDPTimeout
 	default:
 		timeout = f.DefaultTimeout
 	}

-	f.connMutex.Lock()
-	if _, ok := f.Conns[fp]; !ok {
-		f.TimerWheel.Add(fp, timeout)
+	conntrack := f.Conntrack
+	conntrack.Lock()
+	if _, ok := conntrack.Conns[fp]; !ok {
+		conntrack.TimerWheel.Add(fp, timeout)
 	}

+	// Record which rulesVersion allowed this connection, so we can retest after
+	// firewall reload
+	c.incoming = incoming
+	c.rulesVersion = f.rulesVersion
 	c.Expires = time.Now().Add(timeout)
-	f.Conns[fp] = c
-	f.connMutex.Unlock()
+	conntrack.Conns[fp] = c
+	conntrack.Unlock()
 }

 // Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel
 // Caller must own the connMutex lock!
-func (f *Firewall) evict(p FirewallPacket) {
+func (f *Firewall) evict(p firewall.Packet) {
 	//TODO: report a stat if the tcp rtt tracking was never resolved?
 	// Are we still tracking this conn?
-	t, ok := f.Conns[p]
+	conntrack := f.Conntrack
+	t, ok := conntrack.Conns[p]
 	if !ok {
 		return
 	}
@@ -456,29 +537,29 @@ func (f *Firewall) evict(p FirewallPacket) {

 	// Timeout is in the future, re-add the timer
 	if newT > 0 {
-		f.TimerWheel.Add(p, newT)
+		conntrack.TimerWheel.Add(p, newT)
 		return
 	}

 	// This conn is done
-	delete(f.Conns, p)
+	delete(conntrack.Conns, p)
 }

-func (ft *FirewallTable) match(p FirewallPacket, incoming bool, c *cert.NebulaCertificate, caPool *cert.NebulaCAPool) bool {
+func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.NebulaCertificate, caPool *cert.NebulaCAPool) bool {
 	if ft.AnyProto.match(p, incoming, c, caPool) {
 		return true
 	}

 	switch p.Protocol {
-	case fwProtoTCP:
+	case firewall.ProtoTCP:
 		if ft.TCP.match(p, incoming, c, caPool) {
 			return true
 		}
-	case fwProtoUDP:
+	case firewall.ProtoUDP:
 		if ft.UDP.match(p, incoming, c, caPool) {
 			return true
 		}
-	case fwProtoICMP:
+	case firewall.ProtoICMP:
 		if ft.ICMP.match(p, incoming, c, caPool) {
 			return true
 		}
@@ -494,12 +575,9 @@ func (fp firewallPort) addRule(startPort int32, endPort int32, groups []string,

 	for i := startPort; i <= endPort; i++ {
 		if _, ok := fp[i]; !ok {
-			fp[i] = &FirewallRule{
-				Groups:  make([][]string, 0),
-				Hosts:   make(map[string]struct{}),
-				CIDR:    NewCIDRTree(),
-				CANames: make(map[string]struct{}),
-				CAShas:  make(map[string]struct{}),
+			fp[i] = &FirewallCA{
+				CANames: make(map[string]*FirewallRule),
+				CAShas:  make(map[string]*FirewallRule),
 			}
 		}

@@ -511,7 +589,7 @@ func (fp firewallPort) addRule(startPort int32, endPort int32, groups []string,
 	return nil
 }

-func (fp firewallPort) match(p FirewallPacket, incoming bool, c *cert.NebulaCertificate, caPool *cert.NebulaCAPool) bool {
+func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.NebulaCertificate, caPool *cert.NebulaCAPool) bool {
 	// We don't have any allowed ports, bail
 	if fp == nil {
 		return false
@@ -520,7 +598,7 @@ func (fp firewallPort) match(p FirewallPacket, incoming bool, c *cert.NebulaCert
 	var port int32

 	if p.Fragment {
-		port = fwPortFragment
+		port = firewall.PortFragment
 	} else if incoming {
 		port = int32(p.LocalPort)
 	} else {
@@ -531,18 +609,73 @@ func (fp firewallPort) match(p FirewallPacket, incoming bool, c *cert.NebulaCert
 		return true
 	}

-	return fp[fwPortAny].match(p, c, caPool)
+	return fp[firewall.PortAny].match(p, c, caPool)
 }

-func (fr *FirewallRule) addRule(groups []string, host string, ip *net.IPNet, caName string, caSha string) error {
-	if caName != "" {
-		fr.CANames[caName] = struct{}{}
+func (fc *FirewallCA) addRule(groups []string, host string, ip *net.IPNet, caName, caSha string) error {
+	fr := func() *FirewallRule {
+		return &FirewallRule{
+			Hosts:  make(map[string]struct{}),
+			Groups: make([][]string, 0),
+			CIDR:   cidr.NewTree4(),
+		}
+	}
+
+	if caSha == "" && caName == "" {
+		if fc.Any == nil {
+			fc.Any = fr()
+		}
+
+		return fc.Any.addRule(groups, host, ip)
 	}

 	if caSha != "" {
-		fr.CAShas[caSha] = struct{}{}
+		if _, ok := fc.CAShas[caSha]; !ok {
+			fc.CAShas[caSha] = fr()
+		}
+		err := fc.CAShas[caSha].addRule(groups, host, ip)
+		if err != nil {
+			return err
+		}
 	}

+	if caName != "" {
+		if _, ok := fc.CANames[caName]; !ok {
+			fc.CANames[caName] = fr()
+		}
+		err := fc.CANames[caName].addRule(groups, host, ip)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (fc *FirewallCA) match(p firewall.Packet, c *cert.NebulaCertificate, caPool *cert.NebulaCAPool) bool {
+	if fc == nil {
+		return false
+	}
+
+	if fc.Any.match(p, c) {
+		return true
+	}
+
+	if t, ok := fc.CAShas[c.Details.Issuer]; ok {
+		if t.match(p, c) {
+			return true
+		}
+	}
+
+	s, err := caPool.GetCAForCert(c)
+	if err != nil {
+		return false
+	}
+
+	return fc.CANames[s.Details.Name].match(p, c)
+}
+
+func (fr *FirewallRule) addRule(groups []string, host string, ip *net.IPNet) error {
 	if fr.Any {
 		return nil
 	}
@@ -552,7 +685,7 @@ func (fr *FirewallRule) addRule(groups []string, host string, ip *net.IPNet, caN
 		// If it's any we need to wipe out any pre-existing rules to save on memory
 		fr.Groups = make([][]string, 0)
 		fr.Hosts = make(map[string]struct{})
-		fr.CIDR = NewCIDRTree()
+		fr.CIDR = cidr.NewTree4()
 	} else {
 		if len(groups) > 0 {
 			fr.Groups = append(fr.Groups, groups)
@@ -571,6 +704,10 @@ func (fr *FirewallRule) addRule(groups []string, host string, ip *net.IPNet, caN
 }

 func (fr *FirewallRule) isAny(groups []string, host string, ip *net.IPNet) bool {
+	if len(groups) == 0 && host == "" && ip == nil {
+		return true
+	}
+
 	for _, group := range groups {
 		if group == "any" {
 			return true
@@ -588,28 +725,11 @@ func (fr *FirewallRule) isAny(groups []string, host string, ip *net.IPNet) bool
 	return false
 }

-func (fr *FirewallRule) match(p FirewallPacket, c *cert.NebulaCertificate, caPool *cert.NebulaCAPool) bool {
+func (fr *FirewallRule) match(p firewall.Packet, c *cert.NebulaCertificate) bool {
 	if fr == nil {
 		return false
 	}

-	// CASha and CAName always need to be checked
-	if len(fr.CAShas) > 0 {
-		if _, ok := fr.CAShas[c.Details.Issuer]; !ok {
-			return false
-		}
-	}
-
-	if len(fr.CANames) > 0 {
-		s, err := caPool.GetCAForCert(c)
-		if err != nil {
-			return false
-		}
-		if _, ok := fr.CANames[s.Details.Name]; !ok {
-			return false
-		}
-	}
-
 	// Shortcut path for if groups, hosts, or cidr contained an `any`
 	if fr.Any {
 		return true
@@ -659,7 +779,7 @@ type rule struct {
 	CASha  string
 }

-func convertRule(p interface{}) (rule, error) {
+func convertRule(l *logrus.Logger, p interface{}, table string, i int) (rule, error) {
 	r := rule{}

 	m, ok := p.(map[interface{}]interface{})
@@ -679,11 +799,21 @@ func convertRule(p interface{}) (rule, error) {
 	r.Code = toString("code", m)
 	r.Proto = toString("proto", m)
 	r.Host = toString("host", m)
-	r.Group = toString("group", m)
 	r.Cidr = toString("cidr", m)
 	r.CAName = toString("ca_name", m)
 	r.CASha = toString("ca_sha", m)

+	// Make sure group isn't an array
+	if v, ok := m["group"].([]interface{}); ok {
+		if len(v) > 1 {
+			return r, errors.New("group should contain a single value, an array with more than one entry was provided")
+		}
+
+		l.Warnf("%s rule #%v; group was an array with a single value, converting to simple value", table, i)
+		m["group"] = v[0]
+	}
+	r.Group = toString("group", m)
+
 	if rg, ok := m["groups"]; ok {
 		switch reflect.TypeOf(rg).Kind() {
 		case reflect.Slice:
@@ -704,12 +834,12 @@ func convertRule(p interface{}) (rule, error) {

 func parsePort(s string) (startPort, endPort int32, err error) {
 	if s == "any" {
-		startPort = fwPortAny
-		endPort = fwPortAny
+		startPort = firewall.PortAny
+		endPort = firewall.PortAny

 	} else if s == "fragment" {
-		startPort = fwPortFragment
-		endPort = fwPortFragment
+		startPort = firewall.PortFragment
+		endPort = firewall.PortFragment

 	} else if strings.Contains(s, `-`) {
 		sPorts := strings.SplitN(s, `-`, 2)
@@ -733,8 +863,8 @@ func parsePort(s string) (startPort, endPort int32, err error) {
 		startPort = int32(rStartPort)
 		endPort = int32(rEndPort)

-		if startPort == fwPortAny {
-			endPort = fwPortAny
+		if startPort == firewall.PortAny {
+			endPort = firewall.PortAny
 		}

 	} else {
@@ -758,7 +888,7 @@ func setTCPRTTTracking(c *conn, p []byte) {
 	ihl := int(p[0]&0x0f) << 2

 	// Don't track FIN packets
-	if uint8(p[ihl+13])&tcpFIN != 0 {
+	if p[ihl+13]&tcpFIN != 0 {
 		return
 	}

@@ -772,7 +902,7 @@ func (f *Firewall) checkTCPRTT(c *conn, p []byte) bool {
 	}

 	ihl := int(p[0]&0x0f) << 2
-	if uint8(p[ihl+13])&tcpACK == 0 {
+	if p[ihl+13]&tcpACK == 0 {
 		return false
 	}

--- a/firewall/cache.go
+++ b/firewall/cache.go
@@ -0,0 +1,59 @@
+package firewall
+
+import (
+	"sync/atomic"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// ConntrackCache is used as a local routine cache to know if a given flow
+// has been seen in the conntrack table.
+type ConntrackCache map[Packet]struct{}
+
+type ConntrackCacheTicker struct {
+	cacheV    uint64
+	cacheTick uint64
+
+	cache ConntrackCache
+}
+
+func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
+	if d == 0 {
+		return nil
+	}
+
+	c := &ConntrackCacheTicker{
+		cache: ConntrackCache{},
+	}
+
+	go c.tick(d)
+
+	return c
+}
+
+func (c *ConntrackCacheTicker) tick(d time.Duration) {
+	for {
+		time.Sleep(d)
+		atomic.AddUint64(&c.cacheTick, 1)
+	}
+}
+
+// Get checks if the cache ticker has moved to the next version before returning
+// the map. If it has moved, we reset the map.
+func (c *ConntrackCacheTicker) Get(l *logrus.Logger) ConntrackCache {
+	if c == nil {
+		return nil
+	}
+	if tick := atomic.LoadUint64(&c.cacheTick); tick != c.cacheV {
+		c.cacheV = tick
+		if ll := len(c.cache); ll > 0 {
+			if l.Level == logrus.DebugLevel {
+				l.WithField("len", ll).Debug("resetting conntrack cache")
+			}
+			c.cache = make(ConntrackCache, ll)
+		}
+	}
+
+	return c.cache
+}
--- a/firewall/packet.go
+++ b/firewall/packet.go
@@ -0,0 +1,62 @@
+package firewall
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/slackhq/nebula/iputil"
+)
+
+type m map[string]interface{}
+
+const (
+	ProtoAny  = 0 // When we want to handle HOPOPT (0) we can change this, if ever
+	ProtoTCP  = 6
+	ProtoUDP  = 17
+	ProtoICMP = 1
+
+	PortAny      = 0  // Special value for matching `port: any`
+	PortFragment = -1 // Special value for matching `port: fragment`
+)
+
+type Packet struct {
+	LocalIP    iputil.VpnIp
+	RemoteIP   iputil.VpnIp
+	LocalPort  uint16
+	RemotePort uint16
+	Protocol   uint8
+	Fragment   bool
+}
+
+func (fp *Packet) Copy() *Packet {
+	return &Packet{
+		LocalIP:    fp.LocalIP,
+		RemoteIP:   fp.RemoteIP,
+		LocalPort:  fp.LocalPort,
+		RemotePort: fp.RemotePort,
+		Protocol:   fp.Protocol,
+		Fragment:   fp.Fragment,
+	}
+}
+
+func (fp Packet) MarshalJSON() ([]byte, error) {
+	var proto string
+	switch fp.Protocol {
+	case ProtoTCP:
+		proto = "tcp"
+	case ProtoICMP:
+		proto = "icmp"
+	case ProtoUDP:
+		proto = "udp"
+	default:
+		proto = fmt.Sprintf("unknown %v", fp.Protocol)
+	}
+	return json.Marshal(m{
+		"LocalIP":    fp.LocalIP.String(),
+		"RemoteIP":   fp.RemoteIP.String(),
+		"LocalPort":  fp.LocalPort,
+		"RemotePort": fp.RemotePort,
+		"Protocol":   proto,
+		"Fragment":   fp.Fragment,
+	})
+}
--- a/firewall_test.go
+++ b/firewall_test.go
@@ -1,143 +1,148 @@
 package nebula

 import (
+	"bytes"
 	"encoding/binary"
 	"errors"
-	"github.com/rcrowley/go-metrics"
-	"github.com/stretchr/testify/assert"
 	"math"
 	"net"
-	"github.com/slackhq/nebula/cert"
 	"testing"
 	"time"
+
+	"github.com/rcrowley/go-metrics"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/firewall"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/util"
+	"github.com/stretchr/testify/assert"
 )

 func TestNewFirewall(t *testing.T) {
+	l := util.NewTestLogger()
 	c := &cert.NebulaCertificate{}
-	fw := NewFirewall(time.Second, time.Minute, time.Hour, c)
-	assert.NotNil(t, fw.Conns)
+	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c)
+	conntrack := fw.Conntrack
+	assert.NotNil(t, conntrack)
+	assert.NotNil(t, conntrack.Conns)
+	assert.NotNil(t, conntrack.TimerWheel)
 	assert.NotNil(t, fw.InRules)
 	assert.NotNil(t, fw.OutRules)
-	assert.NotNil(t, fw.TimerWheel)
 	assert.Equal(t, time.Second, fw.TCPTimeout)
 	assert.Equal(t, time.Minute, fw.UDPTimeout)
 	assert.Equal(t, time.Hour, fw.DefaultTimeout)

-	assert.Equal(t, time.Hour, fw.TimerWheel.wheelDuration)
-	assert.Equal(t, time.Hour, fw.TimerWheel.wheelDuration)
-	assert.Equal(t, 3601, fw.TimerWheel.wheelLen)
+	assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
+	assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
+	assert.Equal(t, 3601, conntrack.TimerWheel.wheelLen)

-	fw = NewFirewall(time.Second, time.Hour, time.Minute, c)
-	assert.Equal(t, time.Hour, fw.TimerWheel.wheelDuration)
-	assert.Equal(t, 3601, fw.TimerWheel.wheelLen)
+	fw = NewFirewall(l, time.Second, time.Hour, time.Minute, c)
+	assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
+	assert.Equal(t, 3601, conntrack.TimerWheel.wheelLen)

-	fw = NewFirewall(time.Hour, time.Second, time.Minute, c)
-	assert.Equal(t, time.Hour, fw.TimerWheel.wheelDuration)
-	assert.Equal(t, 3601, fw.TimerWheel.wheelLen)
+	fw = NewFirewall(l, time.Hour, time.Second, time.Minute, c)
+	assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
+	assert.Equal(t, 3601, conntrack.TimerWheel.wheelLen)

-	fw = NewFirewall(time.Hour, time.Minute, time.Second, c)
-	assert.Equal(t, time.Hour, fw.TimerWheel.wheelDuration)
-	assert.Equal(t, 3601, fw.TimerWheel.wheelLen)
+	fw = NewFirewall(l, time.Hour, time.Minute, time.Second, c)
+	assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
+	assert.Equal(t, 3601, conntrack.TimerWheel.wheelLen)

-	fw = NewFirewall(time.Minute, time.Hour, time.Second, c)
-	assert.Equal(t, time.Hour, fw.TimerWheel.wheelDuration)
-	assert.Equal(t, 3601, fw.TimerWheel.wheelLen)
+	fw = NewFirewall(l, time.Minute, time.Hour, time.Second, c)
+	assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
+	assert.Equal(t, 3601, conntrack.TimerWheel.wheelLen)

-	fw = NewFirewall(time.Minute, time.Second, time.Hour, c)
-	assert.Equal(t, time.Hour, fw.TimerWheel.wheelDuration)
-	assert.Equal(t, 3601, fw.TimerWheel.wheelLen)
+	fw = NewFirewall(l, time.Minute, time.Second, time.Hour, c)
+	assert.Equal(t, time.Hour, conntrack.TimerWheel.wheelDuration)
+	assert.Equal(t, 3601, conntrack.TimerWheel.wheelLen)
 }

 func TestFirewall_AddRule(t *testing.T) {
+	l := util.NewTestLogger()
+	ob := &bytes.Buffer{}
+	l.SetOutput(ob)
+
 	c := &cert.NebulaCertificate{}
-	fw := NewFirewall(time.Second, time.Minute, time.Hour, c)
+	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	assert.NotNil(t, fw.InRules)
 	assert.NotNil(t, fw.OutRules)

 	_, ti, _ := net.ParseCIDR("1.2.3.4/32")

-	assert.Nil(t, fw.AddRule(true, fwProtoTCP, 1, 1, []string{}, "", nil, "", ""))
-	// Make sure an empty rule creates structure but doesn't allow anything to flow
-	//TODO: ideally an empty rule would return an error
-	assert.False(t, fw.InRules.TCP[1].Any)
-	assert.Empty(t, fw.InRules.TCP[1].Groups)
-	assert.Empty(t, fw.InRules.TCP[1].Hosts)
-	assert.Nil(t, fw.InRules.TCP[1].CIDR.root.left)
-	assert.Nil(t, fw.InRules.TCP[1].CIDR.root.right)
-	assert.Nil(t, fw.InRules.TCP[1].CIDR.root.value)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoTCP, 1, 1, []string{}, "", nil, "", ""))
+	// An empty rule is any
+	assert.True(t, fw.InRules.TCP[1].Any.Any)
+	assert.Empty(t, fw.InRules.TCP[1].Any.Groups)
+	assert.Empty(t, fw.InRules.TCP[1].Any.Hosts)

-	fw = NewFirewall(time.Second, time.Minute, time.Hour, c)
-	assert.Nil(t, fw.AddRule(true, fwProtoUDP, 1, 1, []string{"g1"}, "", nil, "", ""))
-	assert.False(t, fw.InRules.UDP[1].Any)
-	assert.Contains(t, fw.InRules.UDP[1].Groups[0], "g1")
-	assert.Empty(t, fw.InRules.UDP[1].Hosts)
-	assert.Nil(t, fw.InRules.UDP[1].CIDR.root.left)
-	assert.Nil(t, fw.InRules.UDP[1].CIDR.root.right)
-	assert.Nil(t, fw.InRules.UDP[1].CIDR.root.value)
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", nil, "", ""))
+	assert.False(t, fw.InRules.UDP[1].Any.Any)
+	assert.Contains(t, fw.InRules.UDP[1].Any.Groups[0], "g1")
+	assert.Empty(t, fw.InRules.UDP[1].Any.Hosts)

-	fw = NewFirewall(time.Second, time.Minute, time.Hour, c)
-	assert.Nil(t, fw.AddRule(true, fwProtoICMP, 1, 1, []string{}, "h1", nil, "", ""))
-	assert.False(t, fw.InRules.ICMP[1].Any)
-	assert.Empty(t, fw.InRules.ICMP[1].Groups)
-	assert.Contains(t, fw.InRules.ICMP[1].Hosts, "h1")
-	assert.Nil(t, fw.InRules.ICMP[1].CIDR.root.left)
-	assert.Nil(t, fw.InRules.ICMP[1].CIDR.root.right)
-	assert.Nil(t, fw.InRules.ICMP[1].CIDR.root.value)
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoICMP, 1, 1, []string{}, "h1", nil, "", ""))
+	assert.False(t, fw.InRules.ICMP[1].Any.Any)
+	assert.Empty(t, fw.InRules.ICMP[1].Any.Groups)
+	assert.Contains(t, fw.InRules.ICMP[1].Any.Hosts, "h1")

-	fw = NewFirewall(time.Second, time.Minute, time.Hour, c)
-	assert.Nil(t, fw.AddRule(false, fwProtoAny, 1, 1, []string{}, "", ti, "", ""))
-	assert.False(t, fw.OutRules.AnyProto[1].Any)
-	assert.Empty(t, fw.OutRules.AnyProto[1].Groups)
-	assert.Empty(t, fw.OutRules.AnyProto[1].Hosts)
-	assert.NotNil(t, fw.OutRules.AnyProto[1].CIDR.Match(ip2int(ti.IP)))
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
+	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", ti, "", ""))
+	assert.False(t, fw.OutRules.AnyProto[1].Any.Any)
+	assert.Empty(t, fw.OutRules.AnyProto[1].Any.Groups)
+	assert.Empty(t, fw.OutRules.AnyProto[1].Any.Hosts)
+	assert.NotNil(t, fw.OutRules.AnyProto[1].Any.CIDR.Match(iputil.Ip2VpnIp(ti.IP)))

-	fw = NewFirewall(time.Second, time.Minute, time.Hour, c)
-	assert.Nil(t, fw.AddRule(true, fwProtoUDP, 1, 1, []string{"g1"}, "", nil, "ca-name", ""))
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", nil, "ca-name", ""))
 	assert.Contains(t, fw.InRules.UDP[1].CANames, "ca-name")

-	fw = NewFirewall(time.Second, time.Minute, time.Hour, c)
-	assert.Nil(t, fw.AddRule(true, fwProtoUDP, 1, 1, []string{"g1"}, "", nil, "", "ca-sha"))
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", nil, "", "ca-sha"))
 	assert.Contains(t, fw.InRules.UDP[1].CAShas, "ca-sha")

 	// Set any and clear fields
-	fw = NewFirewall(time.Second, time.Minute, time.Hour, c)
-	assert.Nil(t, fw.AddRule(false, fwProtoAny, 0, 0, []string{"g1", "g2"}, "h1", ti, "", ""))
-	assert.Equal(t, []string{"g1", "g2"}, fw.OutRules.AnyProto[0].Groups[0])
-	assert.Contains(t, fw.OutRules.AnyProto[0].Hosts, "h1")
-	assert.NotNil(t, fw.OutRules.AnyProto[0].CIDR.Match(ip2int(ti.IP)))
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
+	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"g1", "g2"}, "h1", ti, "", ""))
+	assert.Equal(t, []string{"g1", "g2"}, fw.OutRules.AnyProto[0].Any.Groups[0])
+	assert.Contains(t, fw.OutRules.AnyProto[0].Any.Hosts, "h1")
+	assert.NotNil(t, fw.OutRules.AnyProto[0].Any.CIDR.Match(iputil.Ip2VpnIp(ti.IP)))

 	// run twice just to make sure
-	assert.Nil(t, fw.AddRule(false, fwProtoAny, 0, 0, []string{"any"}, "", nil, "", ""))
-	assert.Nil(t, fw.AddRule(false, fwProtoAny, 0, 0, []string{}, "any", nil, "", ""))
-	assert.True(t, fw.OutRules.AnyProto[0].Any)
-	assert.Empty(t, fw.OutRules.AnyProto[0].Groups)
-	assert.Empty(t, fw.OutRules.AnyProto[0].Hosts)
-	assert.Nil(t, fw.OutRules.AnyProto[0].CIDR.root.left)
-	assert.Nil(t, fw.OutRules.AnyProto[0].CIDR.root.right)
-	assert.Nil(t, fw.OutRules.AnyProto[0].CIDR.root.value)
+	//TODO: these ANY rules should clear the CA firewall portion
+	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", nil, "", ""))
+	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "any", nil, "", ""))
+	assert.True(t, fw.OutRules.AnyProto[0].Any.Any)
+	assert.Empty(t, fw.OutRules.AnyProto[0].Any.Groups)
+	assert.Empty(t, fw.OutRules.AnyProto[0].Any.Hosts)

-	fw = NewFirewall(time.Second, time.Minute, time.Hour, c)
-	assert.Nil(t, fw.AddRule(false, fwProtoAny, 0, 0, []string{}, "any", nil, "", ""))
-	assert.True(t, fw.OutRules.AnyProto[0].Any)
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
+	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "any", nil, "", ""))
+	assert.True(t, fw.OutRules.AnyProto[0].Any.Any)

-	fw = NewFirewall(time.Second, time.Minute, time.Hour, c)
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	_, anyIp, _ := net.ParseCIDR("0.0.0.0/0")
-	assert.Nil(t, fw.AddRule(false, fwProtoAny, 0, 0, []string{}, "", anyIp, "", ""))
-	assert.True(t, fw.OutRules.AnyProto[0].Any)
+	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", anyIp, "", ""))
+	assert.True(t, fw.OutRules.AnyProto[0].Any.Any)

 	// Test error conditions
-	fw = NewFirewall(time.Second, time.Minute, time.Hour, c)
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	assert.Error(t, fw.AddRule(true, math.MaxUint8, 0, 0, []string{}, "", nil, "", ""))
-	assert.Error(t, fw.AddRule(true, fwProtoAny, 10, 0, []string{}, "", nil, "", ""))
+	assert.Error(t, fw.AddRule(true, firewall.ProtoAny, 10, 0, []string{}, "", nil, "", ""))
 }

 func TestFirewall_Drop(t *testing.T) {
-	p := FirewallPacket{
-		ip2int(net.IPv4(1, 2, 3, 4)),
-		101,
+	l := util.NewTestLogger()
+	ob := &bytes.Buffer{}
+	l.SetOutput(ob)
+
+	p := firewall.Packet{
+		iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)),
+		iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)),
 		10,
 		90,
-		fwProtoUDP,
+		firewall.ProtoUDP,
 		false,
 	}

@@ -151,42 +156,61 @@ func TestFirewall_Drop(t *testing.T) {
 			Name:           "host1",
 			Ips:            []*net.IPNet{&ipNet},
 			Groups:         []string{"default-group"},
+			InvertedGroups: map[string]struct{}{"default-group": {}},
 			Issuer:         "signer-shasum",
 		},
 	}
+	h := HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &c,
+		},
+		vpnIp: iputil.Ip2VpnIp(ipNet.IP),
+	}
+	h.CreateRemoteCIDR(&c)

-	fw := NewFirewall(time.Second, time.Minute, time.Hour, &c)
-	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"any"}, "", nil, "", ""))
+	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", nil, "", ""))
 	cp := cert.NewCAPool()

 	// Drop outbound
-	assert.True(t, fw.Drop([]byte{}, p, false, &c, cp))
+	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule)
 	// Allow inbound
-	assert.False(t, fw.Drop([]byte{}, p, true, &c, cp))
+	resetConntrack(fw)
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	assert.False(t, fw.Drop([]byte{}, p, false, &c, cp))
+	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil))

-	// test caSha assertions true
-	fw = NewFirewall(time.Second, time.Minute, time.Hour, &c)
-	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"any"}, "", nil, "", "signer-shasum"))
-	assert.False(t, fw.Drop([]byte{}, p, true, &c, cp))
+	// test remote mismatch
+	oldRemote := p.RemoteIP
+	p.RemoteIP = iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 10))
+	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrInvalidRemoteIP)
+	p.RemoteIP = oldRemote

-	// test caSha assertions false
-	fw = NewFirewall(time.Second, time.Minute, time.Hour, &c)
-	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"any"}, "", nil, "", "signer-shasum-nope"))
-	assert.True(t, fw.Drop([]byte{}, p, true, &c, cp))
+	// ensure signer doesn't get in the way of group checks
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, "", "signer-shasum"))
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, "", "signer-shasum-bad"))
+	assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp, nil), ErrNoMatchingRule)

-	// test caName true
+	// test caSha doesn't drop on match
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, "", "signer-shasum-bad"))
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, "", "signer-shasum"))
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+
+	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.NebulaCertificate{Details: cert.NebulaCertificateDetails{Name: "ca-good"}}
-	fw = NewFirewall(time.Second, time.Minute, time.Hour, &c)
-	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"any"}, "", nil, "ca-good", ""))
-	assert.False(t, fw.Drop([]byte{}, p, true, &c, cp))
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, "ca-good", ""))
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, "ca-good-bad", ""))
+	assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp, nil), ErrNoMatchingRule)

-	// test caName false
+	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.NebulaCertificate{Details: cert.NebulaCertificateDetails{Name: "ca-good"}}
-	fw = NewFirewall(time.Second, time.Minute, time.Hour, &c)
-	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"any"}, "", nil, "ca-bad", ""))
-	assert.True(t, fw.Drop([]byte{}, p, true, &c, cp))
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, "ca-good-bad", ""))
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, "ca-good", ""))
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
 }

 func BenchmarkFirewallTable_match(b *testing.B) {
@@ -195,24 +219,24 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 	}

 	_, n, _ := net.ParseCIDR("172.1.1.1/32")
-	ft.TCP.addRule(10, 10, []string{"good-group"}, "good-host", n, "", "")
-	ft.TCP.addRule(10, 10, []string{"good-group2"}, "good-host", n, "", "")
-	ft.TCP.addRule(10, 10, []string{"good-group3"}, "good-host", n, "", "")
-	ft.TCP.addRule(10, 10, []string{"good-group4"}, "good-host", n, "", "")
-	ft.TCP.addRule(10, 10, []string{"good-group, good-group1"}, "good-host", n, "", "")
+	_ = ft.TCP.addRule(10, 10, []string{"good-group"}, "good-host", n, "", "")
+	_ = ft.TCP.addRule(10, 10, []string{"good-group2"}, "good-host", n, "", "")
+	_ = ft.TCP.addRule(10, 10, []string{"good-group3"}, "good-host", n, "", "")
+	_ = ft.TCP.addRule(10, 10, []string{"good-group4"}, "good-host", n, "", "")
+	_ = ft.TCP.addRule(10, 10, []string{"good-group, good-group1"}, "good-host", n, "", "")
 	cp := cert.NewCAPool()

 	b.Run("fail on proto", func(b *testing.B) {
 		c := &cert.NebulaCertificate{}
 		for n := 0; n < b.N; n++ {
-			ft.match(FirewallPacket{Protocol: fwProtoUDP}, true, c, cp)
+			ft.match(firewall.Packet{Protocol: firewall.ProtoUDP}, true, c, cp)
 		}
 	})

 	b.Run("fail on port", func(b *testing.B) {
 		c := &cert.NebulaCertificate{}
 		for n := 0; n < b.N; n++ {
-			ft.match(FirewallPacket{Protocol: fwProtoTCP, LocalPort: 1}, true, c, cp)
+			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 1}, true, c, cp)
 		}
 	})

@@ -226,7 +250,7 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 			},
 		}
 		for n := 0; n < b.N; n++ {
-			ft.match(FirewallPacket{Protocol: fwProtoTCP, LocalPort: 10}, true, c, cp)
+			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp)
 		}
 	})

@@ -238,7 +262,7 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 			},
 		}
 		for n := 0; n < b.N; n++ {
-			ft.match(FirewallPacket{Protocol: fwProtoTCP, LocalPort: 10}, true, c, cp)
+			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp)
 		}
 	})

@@ -250,12 +274,12 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 			},
 		}
 		for n := 0; n < b.N; n++ {
-			ft.match(FirewallPacket{Protocol: fwProtoTCP, LocalPort: 10}, true, c, cp)
+			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp)
 		}
 	})

 	b.Run("pass on ip", func(b *testing.B) {
-		ip := ip2int(net.IPv4(172, 1, 1, 1))
+		ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1))
 		c := &cert.NebulaCertificate{
 			Details: cert.NebulaCertificateDetails{
 				InvertedGroups: map[string]struct{}{"nope": {}},
@@ -263,14 +287,14 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 			},
 		}
 		for n := 0; n < b.N; n++ {
-			ft.match(FirewallPacket{Protocol: fwProtoTCP, LocalPort: 10, RemoteIP: ip}, true, c, cp)
+			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10, RemoteIP: ip}, true, c, cp)
 		}
 	})

-	ft.TCP.addRule(0, 0, []string{"good-group"}, "good-host", n, "", "")
+	_ = ft.TCP.addRule(0, 0, []string{"good-group"}, "good-host", n, "", "")

 	b.Run("pass on ip with any port", func(b *testing.B) {
-		ip := ip2int(net.IPv4(172, 1, 1, 1))
+		ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1))
 		c := &cert.NebulaCertificate{
 			Details: cert.NebulaCertificateDetails{
 				InvertedGroups: map[string]struct{}{"nope": {}},
@@ -278,18 +302,22 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 			},
 		}
 		for n := 0; n < b.N; n++ {
-			ft.match(FirewallPacket{Protocol: fwProtoTCP, LocalPort: 100, RemoteIP: ip}, true, c, cp)
+			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, RemoteIP: ip}, true, c, cp)
 		}
 	})
 }

 func TestFirewall_Drop2(t *testing.T) {
-	p := FirewallPacket{
-		ip2int(net.IPv4(1, 2, 3, 4)),
-		101,
+	l := util.NewTestLogger()
+	ob := &bytes.Buffer{}
+	l.SetOutput(ob)
+
+	p := firewall.Packet{
+		iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)),
+		iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)),
 		10,
 		90,
-		fwProtoUDP,
+		firewall.ProtoUDP,
 		false,
 	}

@@ -305,6 +333,13 @@ func TestFirewall_Drop2(t *testing.T) {
 			InvertedGroups: map[string]struct{}{"default-group": {}, "test-group": {}},
 		},
 	}
+	h := HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &c,
+		},
+		vpnIp: iputil.Ip2VpnIp(ipNet.IP),
+	}
+	h.CreateRemoteCIDR(&c)

 	c1 := cert.NebulaCertificate{
 		Details: cert.NebulaCertificateDetails{
@@ -313,15 +348,175 @@ func TestFirewall_Drop2(t *testing.T) {
 			InvertedGroups: map[string]struct{}{"default-group": {}, "test-group-not": {}},
 		},
 	}
+	h1 := HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &c1,
+		},
+	}
+	h1.CreateRemoteCIDR(&c1)

-	fw := NewFirewall(time.Second, time.Minute, time.Hour, &c)
-	assert.Nil(t, fw.AddRule(true, fwProtoAny, 0, 0, []string{"default-group", "test-group"}, "", nil, "", ""))
+	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group", "test-group"}, "", nil, "", ""))
 	cp := cert.NewCAPool()

-	// c1 lacks the proper groups
-	assert.True(t, fw.Drop([]byte{}, p, true, &c1, cp))
+	// h1/c1 lacks the proper groups
+	assert.Error(t, fw.Drop([]byte{}, p, true, &h1, cp, nil), ErrNoMatchingRule)
 	// c has the proper groups
-	assert.False(t, fw.Drop([]byte{}, p, true, &c, cp))
+	resetConntrack(fw)
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+}
+
+func TestFirewall_Drop3(t *testing.T) {
+	l := util.NewTestLogger()
+	ob := &bytes.Buffer{}
+	l.SetOutput(ob)
+
+	p := firewall.Packet{
+		iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)),
+		iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)),
+		1,
+		1,
+		firewall.ProtoUDP,
+		false,
+	}
+
+	ipNet := net.IPNet{
+		IP:   net.IPv4(1, 2, 3, 4),
+		Mask: net.IPMask{255, 255, 255, 0},
+	}
+
+	c := cert.NebulaCertificate{
+		Details: cert.NebulaCertificateDetails{
+			Name: "host-owner",
+			Ips:  []*net.IPNet{&ipNet},
+		},
+	}
+
+	c1 := cert.NebulaCertificate{
+		Details: cert.NebulaCertificateDetails{
+			Name:   "host1",
+			Ips:    []*net.IPNet{&ipNet},
+			Issuer: "signer-sha-bad",
+		},
+	}
+	h1 := HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &c1,
+		},
+		vpnIp: iputil.Ip2VpnIp(ipNet.IP),
+	}
+	h1.CreateRemoteCIDR(&c1)
+
+	c2 := cert.NebulaCertificate{
+		Details: cert.NebulaCertificateDetails{
+			Name:   "host2",
+			Ips:    []*net.IPNet{&ipNet},
+			Issuer: "signer-sha",
+		},
+	}
+	h2 := HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &c2,
+		},
+		vpnIp: iputil.Ip2VpnIp(ipNet.IP),
+	}
+	h2.CreateRemoteCIDR(&c2)
+
+	c3 := cert.NebulaCertificate{
+		Details: cert.NebulaCertificateDetails{
+			Name:   "host3",
+			Ips:    []*net.IPNet{&ipNet},
+			Issuer: "signer-sha-bad",
+		},
+	}
+	h3 := HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &c3,
+		},
+		vpnIp: iputil.Ip2VpnIp(ipNet.IP),
+	}
+	h3.CreateRemoteCIDR(&c3)
+
+	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "host1", nil, "", ""))
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", nil, "", "signer-sha"))
+	cp := cert.NewCAPool()
+
+	// c1 should pass because host match
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h1, cp, nil))
+	// c2 should pass because ca sha match
+	resetConntrack(fw)
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h2, cp, nil))
+	// c3 should fail because no match
+	resetConntrack(fw)
+	assert.Equal(t, fw.Drop([]byte{}, p, true, &h3, cp, nil), ErrNoMatchingRule)
+}
+
+func TestFirewall_DropConntrackReload(t *testing.T) {
+	l := util.NewTestLogger()
+	ob := &bytes.Buffer{}
+	l.SetOutput(ob)
+
+	p := firewall.Packet{
+		iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)),
+		iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)),
+		10,
+		90,
+		firewall.ProtoUDP,
+		false,
+	}
+
+	ipNet := net.IPNet{
+		IP:   net.IPv4(1, 2, 3, 4),
+		Mask: net.IPMask{255, 255, 255, 0},
+	}
+
+	c := cert.NebulaCertificate{
+		Details: cert.NebulaCertificateDetails{
+			Name:           "host1",
+			Ips:            []*net.IPNet{&ipNet},
+			Groups:         []string{"default-group"},
+			InvertedGroups: map[string]struct{}{"default-group": {}},
+			Issuer:         "signer-shasum",
+		},
+	}
+	h := HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &c,
+		},
+		vpnIp: iputil.Ip2VpnIp(ipNet.IP),
+	}
+	h.CreateRemoteCIDR(&c)
+
+	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", nil, "", ""))
+	cp := cert.NewCAPool()
+
+	// Drop outbound
+	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule)
+	// Allow inbound
+	resetConntrack(fw)
+	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+	// Allow outbound because conntrack
+	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil))
+
+	oldFw := fw
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 10, 10, []string{"any"}, "", nil, "", ""))
+	fw.Conntrack = oldFw.Conntrack
+	fw.rulesVersion = oldFw.rulesVersion + 1
+
+	// Allow outbound because conntrack and new rules allow port 10
+	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil))
+
+	oldFw = fw
+	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
+	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 11, 11, []string{"any"}, "", nil, "", ""))
+	fw.Conntrack = oldFw.Conntrack
+	fw.rulesVersion = oldFw.rulesVersion + 1
+
+	// Drop outbound because conntrack doesn't match new ruleset
+	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule)
 }

 func BenchmarkLookup(b *testing.B) {
@@ -440,124 +635,126 @@ func Test_parsePort(t *testing.T) {
 }

 func TestNewFirewallFromConfig(t *testing.T) {
+	l := util.NewTestLogger()
 	// Test a bad rule definition
 	c := &cert.NebulaCertificate{}
-	conf := NewConfig()
+	conf := config.NewC(l)
 	conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": "asdf"}
-	_, err := NewFirewallFromConfig(c, conf)
+	_, err := NewFirewallFromConfig(l, c, conf)
 	assert.EqualError(t, err, "firewall.outbound failed to parse, should be an array of rules")

 	// Test both port and code
-	conf = NewConfig()
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"port": "1", "code": "2"}}}
-	_, err = NewFirewallFromConfig(c, conf)
+	_, err = NewFirewallFromConfig(l, c, conf)
 	assert.EqualError(t, err, "firewall.outbound rule #0; only one of port or code should be provided")

 	// Test missing host, group, cidr, ca_name and ca_sha
-	conf = NewConfig()
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{}}}
-	_, err = NewFirewallFromConfig(c, conf)
+	_, err = NewFirewallFromConfig(l, c, conf)
 	assert.EqualError(t, err, "firewall.outbound rule #0; at least one of host, group, cidr, ca_name, or ca_sha must be provided")

 	// Test code/port error
-	conf = NewConfig()
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"code": "a", "host": "testh"}}}
-	_, err = NewFirewallFromConfig(c, conf)
+	_, err = NewFirewallFromConfig(l, c, conf)
 	assert.EqualError(t, err, "firewall.outbound rule #0; code was not a number; `a`")

 	conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"port": "a", "host": "testh"}}}
-	_, err = NewFirewallFromConfig(c, conf)
+	_, err = NewFirewallFromConfig(l, c, conf)
 	assert.EqualError(t, err, "firewall.outbound rule #0; port was not a number; `a`")

 	// Test proto error
-	conf = NewConfig()
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"code": "1", "host": "testh"}}}
-	_, err = NewFirewallFromConfig(c, conf)
+	_, err = NewFirewallFromConfig(l, c, conf)
 	assert.EqualError(t, err, "firewall.outbound rule #0; proto was not understood; ``")

 	// Test cidr parse error
-	conf = NewConfig()
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"code": "1", "cidr": "testh", "proto": "any"}}}
-	_, err = NewFirewallFromConfig(c, conf)
+	_, err = NewFirewallFromConfig(l, c, conf)
 	assert.EqualError(t, err, "firewall.outbound rule #0; cidr did not parse; invalid CIDR address: testh")

 	// Test both group and groups
-	conf = NewConfig()
+	conf = config.NewC(l)
 	conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "group": "a", "groups": []string{"b", "c"}}}}
-	_, err = NewFirewallFromConfig(c, conf)
+	_, err = NewFirewallFromConfig(l, c, conf)
 	assert.EqualError(t, err, "firewall.inbound rule #0; only one of group or groups should be defined, both provided")
 }

 func TestAddFirewallRulesFromConfig(t *testing.T) {
+	l := util.NewTestLogger()
 	// Test adding tcp rule
-	conf := NewConfig()
+	conf := config.NewC(l)
 	mf := &mockFirewall{}
 	conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "tcp", "host": "a"}}}
-	assert.Nil(t, AddFirewallRulesFromConfig(false, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: false, proto: fwProtoTCP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil}, mf.lastCall)
+	assert.Nil(t, AddFirewallRulesFromConfig(l, false, conf, mf))
+	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoTCP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil}, mf.lastCall)

 	// Test adding udp rule
-	conf = NewConfig()
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "udp", "host": "a"}}}
-	assert.Nil(t, AddFirewallRulesFromConfig(false, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: false, proto: fwProtoUDP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil}, mf.lastCall)
+	assert.Nil(t, AddFirewallRulesFromConfig(l, false, conf, mf))
+	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoUDP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil}, mf.lastCall)

 	// Test adding icmp rule
-	conf = NewConfig()
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "icmp", "host": "a"}}}
-	assert.Nil(t, AddFirewallRulesFromConfig(false, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: false, proto: fwProtoICMP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil}, mf.lastCall)
+	assert.Nil(t, AddFirewallRulesFromConfig(l, false, conf, mf))
+	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoICMP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil}, mf.lastCall)

 	// Test adding any rule
-	conf = NewConfig()
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "host": "a"}}}
-	assert.Nil(t, AddFirewallRulesFromConfig(true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: fwProtoAny, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil}, mf.lastCall)
+	assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf))
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil}, mf.lastCall)

 	// Test adding rule with ca_sha
-	conf = NewConfig()
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "ca_sha": "12312313123"}}}
-	assert.Nil(t, AddFirewallRulesFromConfig(true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: fwProtoAny, startPort: 1, endPort: 1, groups: nil, ip: nil, caSha: "12312313123"}, mf.lastCall)
+	assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf))
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: nil, caSha: "12312313123"}, mf.lastCall)

 	// Test adding rule with ca_name
-	conf = NewConfig()
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "ca_name": "root01"}}}
-	assert.Nil(t, AddFirewallRulesFromConfig(true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: fwProtoAny, startPort: 1, endPort: 1, groups: nil, ip: nil, caName: "root01"}, mf.lastCall)
+	assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf))
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: nil, caName: "root01"}, mf.lastCall)

 	// Test single group
-	conf = NewConfig()
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "group": "a"}}}
-	assert.Nil(t, AddFirewallRulesFromConfig(true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: fwProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: nil}, mf.lastCall)
+	assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf))
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: nil}, mf.lastCall)

 	// Test single groups
-	conf = NewConfig()
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "groups": "a"}}}
-	assert.Nil(t, AddFirewallRulesFromConfig(true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: fwProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: nil}, mf.lastCall)
+	assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf))
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: nil}, mf.lastCall)

 	// Test multiple AND groups
-	conf = NewConfig()
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "groups": []string{"a", "b"}}}}
-	assert.Nil(t, AddFirewallRulesFromConfig(true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: fwProtoAny, startPort: 1, endPort: 1, groups: []string{"a", "b"}, ip: nil}, mf.lastCall)
+	assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf))
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a", "b"}, ip: nil}, mf.lastCall)

 	// Test Add error
-	conf = NewConfig()
+	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	mf.nextCallReturn = errors.New("test error")
 	conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "host": "a"}}}
-	assert.EqualError(t, AddFirewallRulesFromConfig(true, conf, mf), "firewall.inbound rule #0; `test error`")
+	assert.EqualError(t, AddFirewallRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; `test error`")
 }

 func TestTCPRTTTracking(t *testing.T) {
@@ -651,6 +848,42 @@ func TestTCPRTTTracking(t *testing.T) {
 	assert.Equal(t, uint32(0), c.Seq)
 }

+func TestFirewall_convertRule(t *testing.T) {
+	l := util.NewTestLogger()
+	ob := &bytes.Buffer{}
+	l.SetOutput(ob)
+
+	// Ensure group array of 1 is converted and a warning is printed
+	c := map[interface{}]interface{}{
+		"group": []interface{}{"group1"},
+	}
+
+	r, err := convertRule(l, c, "test", 1)
+	assert.Contains(t, ob.String(), "test rule #1; group was an array with a single value, converting to simple value")
+	assert.Nil(t, err)
+	assert.Equal(t, "group1", r.Group)
+
+	// Ensure group array of > 1 is errord
+	ob.Reset()
+	c = map[interface{}]interface{}{
+		"group": []interface{}{"group1", "group2"},
+	}
+
+	r, err = convertRule(l, c, "test", 1)
+	assert.Equal(t, "", ob.String())
+	assert.Error(t, err, "group should contain a single value, an array with more than one entry was provided")
+
+	// Make sure a well formed group is alright
+	ob.Reset()
+	c = map[interface{}]interface{}{
+		"group": "group1",
+	}
+
+	r, err = convertRule(l, c, "test", 1)
+	assert.Nil(t, err)
+	assert.Equal(t, "group1", r.Group)
+}
+
 type addRuleCall struct {
 	incoming  bool
 	proto     uint8
@@ -685,3 +918,9 @@ func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, end
 	mf.nextCallReturn = nil
 	return err
 }
+
+func resetConntrack(fw *Firewall) {
+	fw.Conntrack.Lock()
+	fw.Conntrack.Conns = map[firewall.Packet]*conn{}
+	fw.Conntrack.Unlock()
+}
--- a/go.mod
+++ b/go.mod
@@ -1,32 +1,43 @@
 module github.com/slackhq/nebula

-go 1.12
+go 1.17

 require (
-	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239
+	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
 	github.com/armon/go-radix v1.0.0
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
-	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
-	github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6
-	github.com/golang/protobuf v1.3.1
-	github.com/imdario/mergo v0.3.7
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/miekg/dns v1.1.12
-	github.com/nbrownus/go-metrics-prometheus v0.0.0-20180622211546-6e6d5173d99c
-	github.com/prometheus/client_golang v0.9.3
-	github.com/prometheus/common v0.4.1 // indirect
-	github.com/prometheus/procfs v0.0.0-20190523193104-a7aeb8df3389 // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a
-	github.com/sirupsen/logrus v1.4.2
-	github.com/songgao/water v0.0.0-20190402020555-6ad6edefb15c
-	github.com/stretchr/testify v1.3.0
-	github.com/vishvananda/netlink v1.0.1-0.20190522153524-00009fb8606a
-	github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect
-	golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f
-	golang.org/x/net v0.0.0-20190522155817-f3200d17e092
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
-	golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/yaml.v2 v2.2.2
+	github.com/flynn/noise v1.0.0
+	github.com/gogo/protobuf v1.3.2
+	github.com/golang/protobuf v1.5.2
+	github.com/google/gopacket v1.1.19
+	github.com/imdario/mergo v0.3.8
+	github.com/kardianos/service v1.2.0
+	github.com/miekg/dns v1.1.43
+	github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
+	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
+	github.com/sirupsen/logrus v1.8.1
+	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
+	github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
+	github.com/stretchr/testify v1.7.0
+	github.com/vishvananda/netlink v1.1.0
+	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
+	golang.org/x/net v0.0.0-20211101193420-4a448f8816b3
+	golang.org/x/sys v0.0.0-20211102192858-4dd72447c267
+	google.golang.org/protobuf v1.27.1
+	gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -1,112 +1,529 @@
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 h1:M5QgkYacWj0Xs8MhpIK/5uwU02icXpEoSo9sM2aRCps=
 github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432/go.mod h1:xwIwAxMvYnVrGJPe2FKx5prTrnAjGOD8zvDOnxnrrkM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
-github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ=
+github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
-github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
+github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kardianos/service v1.2.0 h1:bGuZ/epo3vrt8IPC7mnKQolqFeYJb7Cs8Rk4PSOBB/g=
+github.com/kardianos/service v1.2.0/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA=
-github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
+github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nbrownus/go-metrics-prometheus v0.0.0-20180622211546-6e6d5173d99c h1:G/mfx/MWYuaaGlHkZQBBXFAJiYnRt/GaOVxnRHjlxg4=
-github.com/nbrownus/go-metrics-prometheus v0.0.0-20180622211546-6e6d5173d99c/go.mod h1:1yMri853KAI2pPAUnESjaqZj9JeImOUM+6A4GuuPmTs=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f h1:8dM0ilqKL0Uzl42GABzzC4Oqlc3kGRILz0vgoff7nwg=
+github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f/go.mod h1:nwPd6pDNId/Xi16qtKrFHrauSwMNuvk+zcjk89wrnlA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
+github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.0-20190523193104-a7aeb8df3389 h1:F/k2nob1S9M6v5Xkq7KjSTQirOYaYQord0jR4TwyVmY=
-github.com/prometheus/procfs v0.0.0-20190523193104-a7aeb8df3389/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
-github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/songgao/water v0.0.0-20190402020555-6ad6edefb15c h1:VZsL/fAl8XnHj5Zn+cRvLcFbMHmCj7tdPrkKZSRziJ0=
-github.com/songgao/water v0.0.0-20190402020555-6ad6edefb15c/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
+github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
+github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8=
+github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/vishvananda/netlink v1.0.1-0.20190522153524-00009fb8606a h1:Bt1IVPhiCDMqwGrc2nnbIN4QKvJGx6SK2NzWBmW00ao=
-github.com/vishvananda/netlink v1.0.1-0.20190522153524-00009fb8606a/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
-github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4=
-github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
+github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=
-golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211101193420-4a448f8816b3 h1:VrJZAjbekhoRn7n5FBujY31gboH+iB3pdLxn3gE9FjU=
+golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4 h1:VSJ45BzqrVgR4clSx415y1rHH7QAGhGt71J0ZmhLYrc=
-golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211102192858-4dd72447c267 h1:7zYaz3tjChtpayGDzu6H0hDAUM5zIGA2XW7kRNgQ0jc=
+golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
--- a/handshake.go
+++ b/handshake.go
@@ -1,82 +1,29 @@
 package nebula

 import (
-	"crypto/hmac"
-	"crypto/sha256"
-	"errors"
-	"github.com/golang/protobuf/proto"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/udp"
 )

-const (
-	handshakeIXPSK0 = 0
-	handshakeXXPSK0 = 1
-)
+func HandleIncomingHandshake(f *Interface, addr *udp.Addr, packet []byte, h *header.H, hostinfo *HostInfo) {
+	// First remote allow list check before we know the vpnIp
+	if !f.lightHouse.remoteAllowList.AllowUnknownVpnIp(addr.IP) {
+		f.l.WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
+		return
+	}

-func HandleIncomingHandshake(f *Interface, addr *udpAddr, packet []byte, h *Header, hostinfo *HostInfo) {
-	newHostinfo, _ := f.handshakeManager.QueryIndex(h.RemoteIndex)
-	//TODO: For stage 1 we won't have hostinfo yet but stage 2 and above would require it, this check may be helpful in those cases
-	//if err != nil {
-	//	l.WithError(err).WithField("udpAddr", addr).Error("Error while finding host info for handshake message")
-	//	return
-	//}
-
-	tearDown := false
 	switch h.Subtype {
-	case handshakeIXPSK0:
+	case header.HandshakeIXPSK0:
 		switch h.MessageCounter {
 		case 1:
-			tearDown = ixHandshakeStage1(f, addr, newHostinfo, packet, h)
+			ixHandshakeStage1(f, addr, packet, h)
 		case 2:
-			tearDown = ixHandshakeStage2(f, addr, newHostinfo, packet, h)
-		}
-	}
-
+			newHostinfo, _ := f.handshakeManager.QueryIndex(h.RemoteIndex)
+			tearDown := ixHandshakeStage2(f, addr, newHostinfo, packet, h)
 			if tearDown && newHostinfo != nil {
-		f.handshakeManager.DeleteIndex(newHostinfo.localIndexId)
-		f.handshakeManager.DeleteVpnIP(newHostinfo.hostId)
+				f.handshakeManager.DeleteHostInfo(newHostinfo)
+			}
 		}
 	}

-func HandshakeBytesWithMAC(details *NebulaHandshakeDetails, key []byte) ([]byte, error) {
-	mac := hmac.New(sha256.New, key)
-
-	b, err := proto.Marshal(details)
-	if err != nil {
-		return nil, errors.New("Unable to marshal nebula handshake")
-	}
-	mac.Write(b)
-	sum := mac.Sum(nil)
-
-	hs := &NebulaHandshake{
-		Details: details,
-		Hmac:    sum,
-	}
-
-	hsBytes, err := proto.Marshal(hs)
-	if err != nil {
-		l.Debugln("failed to generate NebulaHandshake protobuf", err)
-	}
-
-	return hsBytes, nil
-}
-
-func (hs *NebulaHandshake) CheckHandshakeMAC(keys [][]byte) bool {
-
-	b, err := proto.Marshal(hs.Details)
-	if err != nil {
-		return false
-	}
-
-	for _, k := range keys {
-		mac := hmac.New(sha256.New, k)
-		mac.Write(b)
-		expectedMAC := mac.Sum(nil)
-		if hmac.Equal(hs.Hmac, expectedMAC) {
-			return true
-		}
-	}
-
-	//l.Debugln(hs.Hmac, expectedMAC)
-
-	return false
 }
--- a/handshake_ix.go
+++ b/handshake_ix.go
@@ -4,269 +4,370 @@ import (
 	"sync/atomic"
 	"time"

-	"bytes"
-
-	"github.com/flynn/noise"
 	"github.com/golang/protobuf/proto"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
 )

 // NOISE IX Handshakes

 // This function constructs a handshake packet, but does not actually send it
 // Sending is done by the handshake manager
-func ixHandshakeStage0(f *Interface, vpnIp uint32, hostinfo *HostInfo) {
+func ixHandshakeStage0(f *Interface, vpnIp iputil.VpnIp, hostinfo *HostInfo) {
 	// This queries the lighthouse if we don't know a remote for the host
+	// We do it here to provoke the lighthouse to preempt our timer wheel and trigger the stage 1 packet to send
+	// more quickly, effect is a quicker handshake.
 	if hostinfo.remote == nil {
-		ips, err := f.lightHouse.Query(vpnIp, f)
-		if err != nil {
-			//l.Debugln(err)
-		}
-		for _, ip := range ips {
-			hostinfo.AddRemote(ip)
-		}
+		f.lightHouse.QueryServer(vpnIp, f)
 	}

-	myIndex, err := generateIndex()
+	err := f.handshakeManager.AddIndexHostInfo(hostinfo)
 	if err != nil {
-		l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).
+		f.l.WithError(err).WithField("vpnIp", vpnIp).
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to generate index")
 		return
 	}
+
 	ci := hostinfo.ConnectionState
-	f.handshakeManager.AddIndexHostInfo(myIndex, hostinfo)

 	hsProto := &NebulaHandshakeDetails{
-		InitiatorIndex: myIndex,
-		Time:           uint64(time.Now().Unix()),
+		InitiatorIndex: hostinfo.localIndexId,
+		Time:           uint64(time.Now().UnixNano()),
 		Cert:           ci.certState.rawCertificateNoKey,
 	}

+	hsBytes := []byte{}
+
 	hs := &NebulaHandshake{
 		Details: hsProto,
-		Hmac:    nil,
 	}
+	hsBytes, err = proto.Marshal(hs)

-	hsBytes, err := proto.Marshal(hs)
-	//hsBytes, err := HandshakeBytesWithMAC(hsProto, f.handshakeMACKey)
 	if err != nil {
-		l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).
+		f.l.WithError(err).WithField("vpnIp", vpnIp).
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to marshal handshake message")
 		return
 	}

-	header := HeaderEncode(make([]byte, HeaderLen), Version, uint8(handshake), handshakeIXPSK0, 0, 1)
-	atomic.AddUint64(ci.messageCounter, 1)
+	h := header.Encode(make([]byte, header.Len), header.Version, header.Handshake, header.HandshakeIXPSK0, 0, 1)
+	atomic.AddUint64(&ci.atomicMessageCounter, 1)

-	msg, _, _, err := ci.H.WriteMessage(header, hsBytes)
+	msg, _, _, err := ci.H.WriteMessage(h, hsBytes)
 	if err != nil {
-		l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).
+		f.l.WithError(err).WithField("vpnIp", vpnIp).
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage")
 		return
 	}

+	// We are sending handshake packet 1, so we don't expect to receive
+	// handshake packet 1 from the responder
+	ci.window.Update(f.l, 1)
+
 	hostinfo.HandshakePacket[0] = msg
 	hostinfo.HandshakeReady = true
 	hostinfo.handshakeStart = time.Now()
-	/*
-		l.Debugln("ZZZZZZZZZZREMOTE: ", hostinfo.remote)
-	*/
-
 }

-func ixHandshakeStage1(f *Interface, addr *udpAddr, hostinfo *HostInfo, packet []byte, h *Header) bool {
-	var ip uint32
-	if h.RemoteIndex == 0 {
-		ci := f.newConnectionState(false, noise.HandshakeIX, []byte{}, 0)
-		// Mark packet 1 as seen so it doesn't show up as missed
-		ci.window.Update(1)
-
-		msg, _, _, err := ci.H.ReadMessage(nil, packet[HeaderLen:])
-		if err != nil {
-			l.WithError(err).WithField("udpAddr", addr).
-				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.ReadMessage")
-			return true
-		}
+func ixHandshakeStage1(f *Interface, addr *udp.Addr, packet []byte, h *header.H) {
+	var (
+		err error
+		ci  *ConnectionState
+		msg []byte
+	)

 	hs := &NebulaHandshake{}
+
+	// Handle multiple possible psk options, ensure the protobuf comes out clean too
+	for _, p := range f.psk.Cache {
+		//TODO: benchmark generation time of makePsk
+		ci, err = f.newConnectionState(f.l, false, p)
+		if err != nil {
+			f.l.WithError(err).WithField("udpAddr", addr).Error("Failed to get a new connection state")
+			continue
+		}
+
+		msg, _, _, err = ci.H.ReadMessage(nil, packet[header.Len:])
+		if err != nil {
+			// Calls to ReadMessage with an incorrect psk should fail, try the next one if we have one
+			continue
+		}
+
+		// Sometimes ReadMessage returns fine with a nil psk even if the handshake is using a psk, ensure our protobuf
+		// comes out clean as well
 		err = proto.Unmarshal(msg, hs)
-		/*
-			l.Debugln("GOT INDEX: ", hs.Details.InitiatorIndex)
-		*/
-		if err != nil || hs.Details == nil {
-			l.WithError(err).WithField("udpAddr", addr).
-				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed unmarshal handshake message")
-			return true
+		if err == nil {
+			// There was no error, we can continue with this handshake
+			break
 		}

-		hostinfo, _ := f.handshakeManager.pendingHostMap.QueryReverseIndex(hs.Details.InitiatorIndex)
-		if hostinfo != nil && bytes.Equal(hostinfo.HandshakePacket[0], packet[HeaderLen:]) {
-			if msg, ok := hostinfo.HandshakePacket[2]; ok {
-				err := f.outside.WriteTo(msg, addr)
+		// The unmarshal failed, try the next psk if we have one
+	}
+
+	// We finished with an error, log it and get out
 	if err != nil {
-					l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
-						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
-						WithError(err).Error("Failed to send handshake message")
-				} else {
-					l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
-						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
-						Info("Handshake message sent")
-				}
-				return false
+		// We aren't logging the error here because we can't be sure of the failure when using psk
+		f.l.WithField("udpAddr", addr).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			Error("Was unable to decrypt the handshake")
+		return
 	}

-			l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
-				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cached", true).
-				WithField("packets", hostinfo.HandshakePacket).
-				Error("Seen this handshake packet already but don't have a cached packet to return")
-		}
+	// Mark packet 1 as seen so it doesn't show up as missed
+	ci.window.Update(f.l, 1)

-		remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert)
+	remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.caPool)
 	if err != nil {
-			l.WithError(err).WithField("udpAddr", addr).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cert", remoteCert).
 			Info("Invalid certificate from host")
-			return true
+		return
 	}
-		vpnIP := ip2int(remoteCert.Details.Ips[0].IP)
+	vpnIp := iputil.Ip2VpnIp(remoteCert.Details.Ips[0].IP)
+	certName := remoteCert.Details.Name
+	fingerprint, _ := remoteCert.Sha256Sum()
+	issuer := remoteCert.Details.Issuer

-		myIndex, err := generateIndex()
+	if vpnIp == f.myVpnIp {
+		f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Refusing to handshake with myself")
+		return
+	}
+
+	if !f.lightHouse.remoteAllowList.Allow(vpnIp, addr.IP) {
+		f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
+		return
+	}
+
+	myIndex, err := generateIndex(f.l)
 	if err != nil {
-			l.WithError(err).WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
+		f.l.WithError(err).WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to generate index")
-			return true
+		return
 	}

-		hostinfo, err = f.handshakeManager.AddIndex(myIndex, ci)
-		if err != nil {
-			l.WithError(err).WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
-				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Error adding index to connection manager")
-
-			return true
+	hostinfo := &HostInfo{
+		ConnectionState:   ci,
+		localIndexId:      myIndex,
+		remoteIndexId:     hs.Details.InitiatorIndex,
+		vpnIp:             vpnIp,
+		HandshakePacket:   make(map[uint8][]byte, 0),
+		lastHandshakeTime: hs.Details.Time,
 	}
-		l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
+
+	hostinfo.Lock()
+	defer hostinfo.Unlock()
+
+	f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
+		WithField("certName", certName).
+		WithField("fingerprint", fingerprint).
+		WithField("issuer", issuer).
 		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
 		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 		Info("Handshake message received")

-		hostinfo.remoteIndexId = hs.Details.InitiatorIndex
 	hs.Details.ResponderIndex = myIndex
 	hs.Details.Cert = ci.certState.rawCertificateNoKey
-		hs.Hmac = nil
+	// Update the time in case their clock is way off from ours
+	hs.Details.Time = uint64(time.Now().UnixNano())

 	hsBytes, err := proto.Marshal(hs)
 	if err != nil {
-			l.WithError(err).WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+		f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to marshal handshake message")
-			return true
+		return
 	}

-		header := HeaderEncode(make([]byte, HeaderLen), Version, uint8(handshake), handshakeIXPSK0, hs.Details.InitiatorIndex, 2)
-		msg, dKey, eKey, err := ci.H.WriteMessage(header, hsBytes)
+	nh := header.Encode(make([]byte, header.Len), header.Version, header.Handshake, header.HandshakeIXPSK0, hs.Details.InitiatorIndex, 2)
+	msg, dKey, eKey, err := ci.H.WriteMessage(nh, hsBytes)
 	if err != nil {
-			l.WithError(err).WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+		f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage")
-			return true
+		return
+	} else if dKey == nil || eKey == nil {
+		f.l.WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Noise did not arrive at a key")
+		return
 	}

-		if f.hostMap.CheckHandshakeCompleteIP(vpnIP) && vpnIP < ip2int(f.certState.certificate.Details.Ips[0].IP) {
-			l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
-				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-				Info("Prevented a handshake race")
-
-			// Send a test packet to trigger an authenticated tunnel test, this should suss out any lingering tunnel issues
-			f.SendMessageToVpnIp(test, testRequest, vpnIP, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
-			return true
-		}
-
-		hostinfo.HandshakePacket[0] = make([]byte, len(packet[HeaderLen:]))
-		copy(hostinfo.HandshakePacket[0], packet[HeaderLen:])
+	hostinfo.HandshakePacket[0] = make([]byte, len(packet[header.Len:]))
+	copy(hostinfo.HandshakePacket[0], packet[header.Len:])

 	// Regardless of whether you are the sender or receiver, you should arrive here
 	// and complete standing up the connection.
-		if dKey != nil && eKey != nil {
 	hostinfo.HandshakePacket[2] = make([]byte, len(msg))
 	copy(hostinfo.HandshakePacket[2], msg)

+	// We are sending handshake packet 2, so we don't expect to receive
+	// handshake packet 2 from the initiator.
+	ci.window.Update(f.l, 2)
+
+	ci.peerCert = remoteCert
+	ci.dKey = NewNebulaCipherState(dKey)
+	ci.eKey = NewNebulaCipherState(eKey)
+
+	hostinfo.remotes = f.lightHouse.QueryCache(vpnIp)
+	hostinfo.SetRemote(addr)
+	hostinfo.CreateRemoteCIDR(remoteCert)
+
+	// Only overwrite existing record if we should win the handshake race
+	overwrite := vpnIp > f.myVpnIp
+	existing, err := f.handshakeManager.CheckAndComplete(hostinfo, 0, overwrite, f)
+	if err != nil {
+		switch err {
+		case ErrAlreadySeen:
+			// Update remote if preferred (Note we have to switch to locking
+			// the existing hostinfo, and then switch back so the defer Unlock
+			// higher in this function still works)
+			hostinfo.Unlock()
+			existing.Lock()
+			// Update remote if preferred
+			if existing.SetRemoteIfPreferred(f.hostMap, addr) {
+				// Send a test packet to ensure the other side has also switched to
+				// the preferred remote
+				f.SendMessageToVpnIp(header.Test, header.TestRequest, vpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+			}
+			existing.Unlock()
+			hostinfo.Lock()
+
+			msg = existing.HandshakePacket[2]
+			f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
 			err := f.outside.WriteTo(msg, addr)
 			if err != nil {
-				l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
+				f.l.WithField("vpnIp", existing.vpnIp).WithField("udpAddr", addr).
+					WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
+					WithError(err).Error("Failed to send handshake message")
+			} else {
+				f.l.WithField("vpnIp", existing.vpnIp).WithField("udpAddr", addr).
+					WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
+					Info("Handshake message sent")
+			}
+			return
+		case ErrExistingHostInfo:
+			// This means there was an existing tunnel and this handshake was older than the one we are currently based on
+			f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("oldHandshakeTime", existing.lastHandshakeTime).
+				WithField("newHandshakeTime", hostinfo.lastHandshakeTime).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+				Info("Handshake too old")
+
+			// Send a test packet to trigger an authenticated tunnel test, this should suss out any lingering tunnel issues
+			f.SendMessageToVpnIp(header.Test, header.TestRequest, vpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+			return
+		case ErrLocalIndexCollision:
+			// This means we failed to insert because of collision on localIndexId. Just let the next handshake packet retry
+			f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+				WithField("localIndex", hostinfo.localIndexId).WithField("collision", existing.vpnIp).
+				Error("Failed to add HostInfo due to localIndex collision")
+			return
+		case ErrExistingHandshake:
+			// We have a race where both parties think they are an initiator and this tunnel lost, let the other one finish
+			f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+				Error("Prevented a pending handshake race")
+			return
+		default:
+			// Shouldn't happen, but just in case someone adds a new error type to CheckAndComplete
+			// And we forget to update it here
+			f.l.WithError(err).WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+				Error("Failed to add HostInfo to HostMap")
+			return
+		}
+	}
+
+	// Do the send
+	f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
+	err = f.outside.WriteTo(msg, addr)
+	if err != nil {
+		f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
 			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
 			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			WithError(err).Error("Failed to send handshake")
 	} else {
-				l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
+		f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
 			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
 			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			WithField("sentCachedPackets", len(hostinfo.packetStore)).
 			Info("Handshake message sent")
 	}

-			ip = ip2int(remoteCert.Details.Ips[0].IP)
-			ci.peerCert = remoteCert
-			ci.dKey = NewNebulaCipherState(dKey)
-			ci.eKey = NewNebulaCipherState(eKey)
-			//l.Debugln("got symmetric pairs")
+	hostinfo.handshakeComplete(f.l, f.cachedPacketMetrics)

-			//hostinfo.ClearRemotes()
-			hostinfo.AddRemote(*addr)
-			f.lightHouse.AddRemoteAndReset(ip, addr)
-			if f.serveDns {
-				dnsR.Add(remoteCert.Details.Name+".", remoteCert.Details.Ips[0].IP.String())
+	return
 }

-			ho, err := f.hostMap.QueryVpnIP(vpnIP)
-			if err == nil && ho.localIndexId != 0 {
-				l.WithField("vpnIp", vpnIP).
-					WithField("action", "removing stale index").
-					WithField("index", ho.localIndexId).
-					Debug("Handshake processing")
-				f.hostMap.DeleteIndex(ho.localIndexId)
-			}
-
-			f.hostMap.AddIndexHostInfo(hostinfo.localIndexId, hostinfo)
-			f.hostMap.AddVpnIPHostInfo(vpnIP, hostinfo)
-
-			hostinfo.handshakeComplete()
-		} else {
-			l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
-				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-				Error("Noise did not arrive at a key")
-			return true
-		}
-
-	}
-
-	f.hostMap.AddRemote(ip, addr)
-	/*
-		l.Debugln("111 ZZZZZZZZZZADDR: ", addr)
-		l.Debugln("111 ZZZZZZZZZZREMOTE: ", hostinfo.remote)
-		l.Debugln("111 ZZZZZZZZZZREMOTEs: ", hostinfo.Remotes[0].addr)
-	*/
-	return false
-}
-
-func ixHandshakeStage2(f *Interface, addr *udpAddr, hostinfo *HostInfo, packet []byte, h *Header) bool {
+func ixHandshakeStage2(f *Interface, addr *udp.Addr, hostinfo *HostInfo, packet []byte, h *header.H) bool {
 	if hostinfo == nil {
+		// Nothing here to tear down, got a bogus stage 2 packet
 		return true
 	}

-	if bytes.Equal(hostinfo.HandshakePacket[2], packet[HeaderLen:]) {
-		l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
-			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("header", h).
-			Error("Already seen this handshake packet")
+	hostinfo.Lock()
+	defer hostinfo.Unlock()
+
+	if !f.lightHouse.remoteAllowList.Allow(hostinfo.vpnIp, addr.IP) {
+		f.l.WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
 		return false
 	}

 	ci := hostinfo.ConnectionState
-	// Mark packet 2 as seen so it doesn't show up as missed
-	ci.window.Update(2)
+	if ci.ready {
+		f.l.WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("header", h).
+			Info("Handshake is already complete")

-	hostinfo.HandshakePacket[2] = make([]byte, len(packet[HeaderLen:]))
-	copy(hostinfo.HandshakePacket[2], packet[HeaderLen:])
+		// Update remote if preferred
+		if hostinfo.SetRemoteIfPreferred(f.hostMap, addr) {
+			// Send a test packet to ensure the other side has also switched to
+			// the preferred remote
+			f.SendMessageToVpnIp(header.Test, header.TestRequest, hostinfo.vpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+		}

-	msg, eKey, dKey, err := ci.H.ReadMessage(nil, packet[HeaderLen:])
+		// We already have a complete tunnel, there is nothing that can be done by processing further stage 1 packets
+		return false
+	}
+
+	msg, eKey, dKey, err := ci.H.ReadMessage(nil, packet[header.Len:])
 	if err != nil {
-		l.WithError(err).WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+		f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("header", h).
 			Error("Failed to call noise.ReadMessage")

@@ -274,83 +375,114 @@ func ixHandshakeStage2(f *Interface, addr *udpAddr, hostinfo *HostInfo, packet [
 		// to DOS us. Every other error condition after should to allow a possible good handshake to complete in the
 		// near future
 		return false
+	} else if dKey == nil || eKey == nil {
+		f.l.WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			Error("Noise did not arrive at a key")
+
+		// This should be impossible in IX but just in case, if we get here then there is no chance to recover
+		// the handshake state machine. Tear it down
+		return true
 	}

 	hs := &NebulaHandshake{}
 	err = proto.Unmarshal(msg, hs)
 	if err != nil || hs.Details == nil {
-		l.WithError(err).WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+		f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).Error("Failed unmarshal handshake message")
+
+		// The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again
 		return true
 	}

-	remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert)
+	remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.caPool)
 	if err != nil {
-		l.WithError(err).WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
+		f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
 			WithField("cert", remoteCert).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			Error("Invalid certificate from host")
+
+		// The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again
 		return true
 	}
-	vpnIP := ip2int(remoteCert.Details.Ips[0].IP)
+
+	vpnIp := iputil.Ip2VpnIp(remoteCert.Details.Ips[0].IP)
+	certName := remoteCert.Details.Name
+	fingerprint, _ := remoteCert.Sha256Sum()
+	issuer := remoteCert.Details.Issuer
+
+	// Ensure the right host responded
+	if vpnIp != hostinfo.vpnIp {
+		f.l.WithField("intendedVpnIp", hostinfo.vpnIp).WithField("haveVpnIp", vpnIp).
+			WithField("udpAddr", addr).WithField("certName", certName).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			Info("Incorrect host responded to handshake")
+
+		// Release our old handshake from pending, it should not continue
+		f.handshakeManager.pendingHostMap.DeleteHostInfo(hostinfo)
+
+		// Create a new hostinfo/handshake for the intended vpn ip
+		//TODO: this adds it to the timer wheel in a way that aggressively retries
+		newHostInfo := f.getOrHandshake(hostinfo.vpnIp)
+		newHostInfo.Lock()
+
+		// Block the current used address
+		newHostInfo.remotes = hostinfo.remotes
+		newHostInfo.remotes.BlockRemote(addr)
+
+		// Get the correct remote list for the host we did handshake with
+		hostinfo.remotes = f.lightHouse.QueryCache(vpnIp)
+
+		f.l.WithField("blockedUdpAddrs", newHostInfo.remotes.CopyBlockedRemotes()).WithField("vpnIp", vpnIp).
+			WithField("remotes", newHostInfo.remotes.CopyAddrs(f.hostMap.preferredRanges)).
+			Info("Blocked addresses for handshakes")
+
+		// Swap the packet store to benefit the original intended recipient
+		hostinfo.ConnectionState.queueLock.Lock()
+		newHostInfo.packetStore = hostinfo.packetStore
+		hostinfo.packetStore = []*cachedPacket{}
+		hostinfo.ConnectionState.queueLock.Unlock()
+
+		// Finally, put the correct vpn ip in the host info, tell them to close the tunnel, and return true to tear down
+		hostinfo.vpnIp = vpnIp
+		f.sendCloseTunnel(hostinfo)
+		newHostInfo.Unlock()
+
+		return true
+	}
+
+	// Mark packet 2 as seen so it doesn't show up as missed
+	ci.window.Update(f.l, 2)

 	duration := time.Since(hostinfo.handshakeStart).Nanoseconds()
-	l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", addr).
+	f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
+		WithField("certName", certName).
+		WithField("fingerprint", fingerprint).
+		WithField("issuer", issuer).
 		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
 		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 		WithField("durationNs", duration).
+		WithField("sentCachedPackets", len(hostinfo.packetStore)).
 		Info("Handshake message received")

-	//ci.remoteIndex = hs.ResponderIndex
 	hostinfo.remoteIndexId = hs.Details.ResponderIndex
-	hs.Details.Cert = ci.certState.rawCertificateNoKey
+	hostinfo.lastHandshakeTime = hs.Details.Time

-	/*
-		hsBytes, err := proto.Marshal(hs)
-		if err != nil {
-			l.Debugln("Failed to marshal handshake: ", err)
-			return
-		}
-	*/
-
-	// Regardless of whether you are the sender or receiver, you should arrive here
-	// and complete standing up the connection.
-	if dKey != nil && eKey != nil {
-		ip := ip2int(remoteCert.Details.Ips[0].IP)
+	// Store their cert and our symmetric keys
 	ci.peerCert = remoteCert
 	ci.dKey = NewNebulaCipherState(dKey)
 	ci.eKey = NewNebulaCipherState(eKey)
-		//l.Debugln("got symmetric pairs")

-		//hostinfo.ClearRemotes()
-		f.hostMap.AddRemote(ip, addr)
-		f.lightHouse.AddRemoteAndReset(ip, addr)
-		if f.serveDns {
-			dnsR.Add(remoteCert.Details.Name+".", remoteCert.Details.Ips[0].IP.String())
-		}
+	// Make sure the current udpAddr being used is set for responding
+	hostinfo.SetRemote(addr)

-		ho, err := f.hostMap.QueryVpnIP(vpnIP)
-		if err == nil && ho.localIndexId != 0 {
-			l.WithField("vpnIp", vpnIP).
-				WithField("action", "removing stale index").
-				WithField("index", ho.localIndexId).
-				Debug("Handshake processing")
-			f.hostMap.DeleteIndex(ho.localIndexId)
-		}
+	// Build up the radix for the firewall if we have subnets in the cert
+	hostinfo.CreateRemoteCIDR(remoteCert)

-		f.hostMap.AddVpnIPHostInfo(vpnIP, hostinfo)
-		f.hostMap.AddIndexHostInfo(hostinfo.localIndexId, hostinfo)
-
-		hostinfo.handshakeComplete()
+	// Complete our handshake and update metrics, this will replace any existing tunnels for this vpnIp
+	//TODO: Complete here does not do a race avoidance, it will just take the new tunnel. Is this ok?
+	f.handshakeManager.Complete(hostinfo, f)
+	hostinfo.handshakeComplete(f.l, f.cachedPacketMetrics)
 	f.metricHandshakes.Update(duration)
-	} else {
-		l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("udpAddr", addr).
-			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
-			Error("Noise did not arrive at a key")
-		return true
-	}

 	return false
-	/*
-		l.Debugln("222 ZZZZZZZZZZREMOTE: ", hostinfo.remote)
-	*/
 }
--- a/handshake_manager.go
+++ b/handshake_manager.go
@@ -1,175 +1,360 @@
 package nebula

 import (
+	"bytes"
+	"context"
 	"crypto/rand"
 	"encoding/binary"
-	"fmt"
+	"errors"
 	"net"
 	"time"

+	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
 )

 const (
-	// Total time to try a handshake = sequence of HandshakeTryInterval * HandshakeRetries
-	// With 100ms interval and 20 retries is 23.5 seconds
-	HandshakeTryInterval = time.Millisecond * 100
-	HandshakeRetries     = 20
-	// HandshakeWaitRotation is the number of handshake attempts to do before starting to use other ips addresses
-	HandshakeWaitRotation = 5
+	DefaultHandshakeTryInterval   = time.Millisecond * 100
+	DefaultHandshakeRetries       = 10
+	DefaultHandshakeTriggerBuffer = 64
 )

+var (
+	defaultHandshakeConfig = HandshakeConfig{
+		tryInterval:   DefaultHandshakeTryInterval,
+		retries:       DefaultHandshakeRetries,
+		triggerBuffer: DefaultHandshakeTriggerBuffer,
+	}
+)
+
+type HandshakeConfig struct {
+	tryInterval   time.Duration
+	retries       int
+	triggerBuffer int
+
+	messageMetrics *MessageMetrics
+}
+
 type HandshakeManager struct {
 	pendingHostMap         *HostMap
 	mainHostMap            *HostMap
 	lightHouse             *LightHouse
-	outside        *udpConn
-
+	outside                *udp.Conn
+	config                 HandshakeConfig
 	OutboundHandshakeTimer *SystemTimerWheel
-	InboundHandshakeTimer  *SystemTimerWheel
+	messageMetrics         *MessageMetrics
+	metricInitiated        metrics.Counter
+	metricTimedOut         metrics.Counter
+	l                      *logrus.Logger
+
+	// can be used to trigger outbound handshake for the given vpnIp
+	trigger chan iputil.VpnIp
 }

-func NewHandshakeManager(tunCidr *net.IPNet, preferredRanges []*net.IPNet, mainHostMap *HostMap, lightHouse *LightHouse, outside *udpConn) *HandshakeManager {
+func NewHandshakeManager(l *logrus.Logger, tunCidr *net.IPNet, preferredRanges []*net.IPNet, mainHostMap *HostMap, lightHouse *LightHouse, outside *udp.Conn, config HandshakeConfig) *HandshakeManager {
 	return &HandshakeManager{
-		pendingHostMap: NewHostMap("pending", tunCidr, preferredRanges),
+		pendingHostMap:         NewHostMap(l, "pending", tunCidr, preferredRanges),
 		mainHostMap:            mainHostMap,
 		lightHouse:             lightHouse,
 		outside:                outside,
-
-		OutboundHandshakeTimer: NewSystemTimerWheel(HandshakeTryInterval, HandshakeTryInterval*HandshakeRetries),
-		InboundHandshakeTimer:  NewSystemTimerWheel(HandshakeTryInterval, HandshakeTryInterval*HandshakeRetries),
+		config:                 config,
+		trigger:                make(chan iputil.VpnIp, config.triggerBuffer),
+		OutboundHandshakeTimer: NewSystemTimerWheel(config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
+		messageMetrics:         config.messageMetrics,
+		metricInitiated:        metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
+		metricTimedOut:         metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
+		l:                      l,
 	}
 }

-func (c *HandshakeManager) Run(f EncWriter) {
-	clockSource := time.Tick(HandshakeTryInterval)
-	for now := range clockSource {
+func (c *HandshakeManager) Run(ctx context.Context, f udp.EncWriter) {
+	clockSource := time.NewTicker(c.config.tryInterval)
+	defer clockSource.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case vpnIP := <-c.trigger:
+			c.l.WithField("vpnIp", vpnIP).Debug("HandshakeManager: triggered")
+			c.handleOutbound(vpnIP, f, true)
+		case now := <-clockSource.C:
 			c.NextOutboundHandshakeTimerTick(now, f)
-		c.NextInboundHandshakeTimerTick(now)
+		}
 	}
 }

-func (c *HandshakeManager) NextOutboundHandshakeTimerTick(now time.Time, f EncWriter) {
+func (c *HandshakeManager) NextOutboundHandshakeTimerTick(now time.Time, f udp.EncWriter) {
 	c.OutboundHandshakeTimer.advance(now)
 	for {
 		ep := c.OutboundHandshakeTimer.Purge()
 		if ep == nil {
 			break
 		}
-		vpnIP := ep.(uint32)
+		vpnIp := ep.(iputil.VpnIp)
+		c.handleOutbound(vpnIp, f, false)
+	}
+}

-		index, err := c.pendingHostMap.GetIndexByVpnIP(vpnIP)
+func (c *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, f udp.EncWriter, lighthouseTriggered bool) {
+	hostinfo, err := c.pendingHostMap.QueryVpnIp(vpnIp)
 	if err != nil {
-			continue
+		return
+	}
+	hostinfo.Lock()
+	defer hostinfo.Unlock()
+
+	// We may have raced to completion but now that we have a lock we should ensure we have not yet completed.
+	if hostinfo.HandshakeComplete {
+		// Ensure we don't exist in the pending hostmap anymore since we have completed
+		c.pendingHostMap.DeleteHostInfo(hostinfo)
+		return
 	}

-		hostinfo, err := c.pendingHostMap.QueryVpnIP(vpnIP)
-		if err != nil {
-			continue
+	// Check if we have a handshake packet to transmit yet
+	if !hostinfo.HandshakeReady {
+		// There is currently a slight race in getOrHandshake due to ConnectionState not being part of the HostInfo directly
+		// Our hostinfo here was added to the pending map and the wheel may have ticked to us before we created ConnectionState
+		c.OutboundHandshakeTimer.Add(vpnIp, c.config.tryInterval*time.Duration(hostinfo.HandshakeCounter))
+		return
 	}

-		// If we haven't finished the handshake and we haven't hit max retries, query
-		// lighthouse and then send the handshake packet again.
-		if hostinfo.HandshakeCounter < HandshakeRetries && !hostinfo.HandshakeComplete {
-			if hostinfo.remote == nil {
-				// We continue to query the lighthouse because hosts may
-				// come online during handshake retries. If the query
-				// succeeds (no error), add the lighthouse info to hostinfo
-				ips, err := c.lightHouse.Query(vpnIP, f)
-				if err == nil {
-					for _, ip := range ips {
-						hostinfo.AddRemote(ip)
-					}
-					hostinfo.ForcePromoteBest(c.mainHostMap.preferredRanges)
-				}
-			}
-
-			hostinfo.HandshakeCounter++
-
-			// We want to use the "best" calculated ip for the first 5 attempts, after that we just blindly rotate through
-			// all the others until we can stand up a connection.
-			if hostinfo.HandshakeCounter > HandshakeWaitRotation {
-				hostinfo.rotateRemote()
-			}
-
-			// Ensure the handshake is ready to avoid a race in timer tick and stage 0 handshake generation
-			if hostinfo.HandshakeReady && hostinfo.remote != nil {
-				err := c.outside.WriteTo(hostinfo.HandshakePacket[0], hostinfo.remote)
-				if err != nil {
-					l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", hostinfo.remote).
+	// If we are out of time, clean up
+	if hostinfo.HandshakeCounter >= c.config.retries {
+		hostinfo.logger(c.l).WithField("udpAddrs", hostinfo.remotes.CopyAddrs(c.pendingHostMap.preferredRanges)).
 			WithField("initiatorIndex", hostinfo.localIndexId).
 			WithField("remoteIndex", hostinfo.remoteIndexId).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-						WithError(err).Error("Failed to send handshake message")
-				} else {
-					//TODO: this log line is assuming a lot of stuff around the cached stage 0 handshake packet, we should
-					// keep the real packet struct around for logging purposes
-					l.WithField("vpnIp", IntIp(vpnIP)).WithField("udpAddr", hostinfo.remote).
+			WithField("durationNs", time.Since(hostinfo.handshakeStart).Nanoseconds()).
+			Info("Handshake timed out")
+		c.metricTimedOut.Inc(1)
+		c.pendingHostMap.DeleteHostInfo(hostinfo)
+		return
+	}
+
+	// We only care about a lighthouse trigger before the first handshake transmit attempt. This is a very specific
+	// optimization for a fast lighthouse reply
+	//TODO: it would feel better to do this once, anytime, as our delay increases over time
+	if lighthouseTriggered && hostinfo.HandshakeCounter > 0 {
+		// If we didn't return here a lighthouse could cause us to aggressively send handshakes
+		return
+	}
+
+	// Get a remotes object if we don't already have one.
+	// This is mainly to protect us as this should never be the case
+	if hostinfo.remotes == nil {
+		hostinfo.remotes = c.lightHouse.QueryCache(vpnIp)
+	}
+
+	//TODO: this will generate a load of queries for hosts with only 1 ip (i'm not using a lighthouse, static mapped)
+	if hostinfo.remotes.Len(c.pendingHostMap.preferredRanges) <= 1 {
+		// If we only have 1 remote it is highly likely our query raced with the other host registered within the lighthouse
+		// Our vpnIp here has a tunnel with a lighthouse but has yet to send a host update packet there so we only know about
+		// the learned public ip for them. Query again to short circuit the promotion counter
+		c.lightHouse.QueryServer(vpnIp, f)
+	}
+
+	// Send a the handshake to all known ips, stage 2 takes care of assigning the hostinfo.remote based on the first to reply
+	var sentTo []*udp.Addr
+	hostinfo.remotes.ForEach(c.pendingHostMap.preferredRanges, func(addr *udp.Addr, _ bool) {
+		c.messageMetrics.Tx(header.Handshake, header.MessageSubType(hostinfo.HandshakePacket[0][1]), 1)
+		err = c.outside.WriteTo(hostinfo.HandshakePacket[0], addr)
+		if err != nil {
+			hostinfo.logger(c.l).WithField("udpAddr", addr).
+				WithField("initiatorIndex", hostinfo.localIndexId).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+				WithError(err).Error("Failed to send handshake message")
+
+		} else {
+			sentTo = append(sentTo, addr)
+		}
+	})
+
+	// Don't be too noisy or confusing if we fail to send a handshake - if we don't get through we'll eventually log a timeout
+	if len(sentTo) > 0 {
+		hostinfo.logger(c.l).WithField("udpAddrs", sentTo).
 			WithField("initiatorIndex", hostinfo.localIndexId).
-						WithField("remoteIndex", hostinfo.remoteIndexId).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Info("Handshake message sent")
 	}
-			}

-			// Readd to the timer wheel so we continue trying wait HandshakeTryInterval * counter longer for next try
-			//l.Infoln("Interval: ", HandshakeTryInterval*time.Duration(hostinfo.HandshakeCounter))
-			c.OutboundHandshakeTimer.Add(vpnIP, HandshakeTryInterval*time.Duration(hostinfo.HandshakeCounter))
-		} else {
-			c.pendingHostMap.DeleteVpnIP(vpnIP)
-			c.pendingHostMap.DeleteIndex(index)
-		}
+	// Increment the counter to increase our delay, linear backoff
+	hostinfo.HandshakeCounter++
+
+	// If a lighthouse triggered this attempt then we are still in the timer wheel and do not need to re-add
+	if !lighthouseTriggered {
+		//TODO: feel like we dupe handshake real fast in a tight loop, why?
+		c.OutboundHandshakeTimer.Add(vpnIp, c.config.tryInterval*time.Duration(hostinfo.HandshakeCounter))
 	}
 }

-func (c *HandshakeManager) NextInboundHandshakeTimerTick(now time.Time) {
-	c.InboundHandshakeTimer.advance(now)
-	for {
-		ep := c.InboundHandshakeTimer.Purge()
-		if ep == nil {
-			break
-		}
-		index := ep.(uint32)
-
-		vpnIP, err := c.pendingHostMap.GetVpnIPByIndex(index)
-		if err != nil {
-			continue
-		}
-		c.pendingHostMap.DeleteIndex(index)
-		c.pendingHostMap.DeleteVpnIP(vpnIP)
-	}
-}
-
-func (c *HandshakeManager) AddVpnIP(vpnIP uint32) *HostInfo {
-	hostinfo := c.pendingHostMap.AddVpnIP(vpnIP)
+func (c *HandshakeManager) AddVpnIp(vpnIp iputil.VpnIp) *HostInfo {
+	hostinfo := c.pendingHostMap.AddVpnIp(vpnIp)
 	// We lock here and use an array to insert items to prevent locking the
 	// main receive thread for very long by waiting to add items to the pending map
-	c.OutboundHandshakeTimer.Add(vpnIP, HandshakeTryInterval)
+	//TODO: what lock?
+	c.OutboundHandshakeTimer.Add(vpnIp, c.config.tryInterval)
+	c.metricInitiated.Inc(1)
+
 	return hostinfo
 }

-func (c *HandshakeManager) DeleteVpnIP(vpnIP uint32) {
-	//l.Debugln("Deleting pending vpn ip :", IntIp(vpnIP))
-	c.pendingHostMap.DeleteVpnIP(vpnIP)
+var (
+	ErrExistingHostInfo    = errors.New("existing hostinfo")
+	ErrAlreadySeen         = errors.New("already seen")
+	ErrLocalIndexCollision = errors.New("local index collision")
+	ErrExistingHandshake   = errors.New("existing handshake")
+)
+
+// CheckAndComplete checks for any conflicts in the main and pending hostmap
+// before adding hostinfo to main. If err is nil, it was added. Otherwise err will be:
+//
+// ErrAlreadySeen if we already have an entry in the hostmap that has seen the
+// exact same handshake packet
+//
+// ErrExistingHostInfo if we already have an entry in the hostmap for this
+// VpnIp and the new handshake was older than the one we currently have
+//
+// ErrLocalIndexCollision if we already have an entry in the main or pending
+// hostmap for the hostinfo.localIndexId.
+func (c *HandshakeManager) CheckAndComplete(hostinfo *HostInfo, handshakePacket uint8, overwrite bool, f *Interface) (*HostInfo, error) {
+	c.pendingHostMap.Lock()
+	defer c.pendingHostMap.Unlock()
+	c.mainHostMap.Lock()
+	defer c.mainHostMap.Unlock()
+
+	// Check if we already have a tunnel with this vpn ip
+	existingHostInfo, found := c.mainHostMap.Hosts[hostinfo.vpnIp]
+	if found && existingHostInfo != nil {
+		// Is it just a delayed handshake packet?
+		if bytes.Equal(hostinfo.HandshakePacket[handshakePacket], existingHostInfo.HandshakePacket[handshakePacket]) {
+			return existingHostInfo, ErrAlreadySeen
 		}

-func (c *HandshakeManager) AddIndex(index uint32, ci *ConnectionState) (*HostInfo, error) {
-	hostinfo, err := c.pendingHostMap.AddIndex(index, ci)
+		// Is this a newer handshake?
+		if existingHostInfo.lastHandshakeTime >= hostinfo.lastHandshakeTime {
+			return existingHostInfo, ErrExistingHostInfo
+		}
+
+		existingHostInfo.logger(c.l).Info("Taking new handshake")
+	}
+
+	existingIndex, found := c.mainHostMap.Indexes[hostinfo.localIndexId]
+	if found {
+		// We have a collision, but for a different hostinfo
+		return existingIndex, ErrLocalIndexCollision
+	}
+
+	existingIndex, found = c.pendingHostMap.Indexes[hostinfo.localIndexId]
+	if found && existingIndex != hostinfo {
+		// We have a collision, but for a different hostinfo
+		return existingIndex, ErrLocalIndexCollision
+	}
+
+	existingRemoteIndex, found := c.mainHostMap.RemoteIndexes[hostinfo.remoteIndexId]
+	if found && existingRemoteIndex != nil && existingRemoteIndex.vpnIp != hostinfo.vpnIp {
+		// We have a collision, but this can happen since we can't control
+		// the remote ID. Just log about the situation as a note.
+		hostinfo.logger(c.l).
+			WithField("remoteIndex", hostinfo.remoteIndexId).WithField("collision", existingRemoteIndex.vpnIp).
+			Info("New host shadows existing host remoteIndex")
+	}
+
+	// Check if we are also handshaking with this vpn ip
+	pendingHostInfo, found := c.pendingHostMap.Hosts[hostinfo.vpnIp]
+	if found && pendingHostInfo != nil {
+		if !overwrite {
+			// We won, let our pending handshake win
+			return pendingHostInfo, ErrExistingHandshake
+		}
+
+		// We lost, take this handshake and move any cached packets over so they get sent
+		pendingHostInfo.ConnectionState.queueLock.Lock()
+		hostinfo.packetStore = append(hostinfo.packetStore, pendingHostInfo.packetStore...)
+		c.pendingHostMap.unlockedDeleteHostInfo(pendingHostInfo)
+		pendingHostInfo.ConnectionState.queueLock.Unlock()
+		pendingHostInfo.logger(c.l).Info("Handshake race lost, replacing pending handshake with completed tunnel")
+	}
+
+	if existingHostInfo != nil {
+		// We are going to overwrite this entry, so remove the old references
+		delete(c.mainHostMap.Hosts, existingHostInfo.vpnIp)
+		delete(c.mainHostMap.Indexes, existingHostInfo.localIndexId)
+		delete(c.mainHostMap.RemoteIndexes, existingHostInfo.remoteIndexId)
+	}
+
+	c.mainHostMap.addHostInfo(hostinfo, f)
+	return existingHostInfo, nil
+}
+
+// Complete is a simpler version of CheckAndComplete when we already know we
+// won't have a localIndexId collision because we already have an entry in the
+// pendingHostMap
+func (c *HandshakeManager) Complete(hostinfo *HostInfo, f *Interface) {
+	c.pendingHostMap.Lock()
+	defer c.pendingHostMap.Unlock()
+	c.mainHostMap.Lock()
+	defer c.mainHostMap.Unlock()
+
+	existingHostInfo, found := c.mainHostMap.Hosts[hostinfo.vpnIp]
+	if found && existingHostInfo != nil {
+		// We are going to overwrite this entry, so remove the old references
+		delete(c.mainHostMap.Hosts, existingHostInfo.vpnIp)
+		delete(c.mainHostMap.Indexes, existingHostInfo.localIndexId)
+		delete(c.mainHostMap.RemoteIndexes, existingHostInfo.remoteIndexId)
+	}
+
+	existingRemoteIndex, found := c.mainHostMap.RemoteIndexes[hostinfo.remoteIndexId]
+	if found && existingRemoteIndex != nil {
+		// We have a collision, but this can happen since we can't control
+		// the remote ID. Just log about the situation as a note.
+		hostinfo.logger(c.l).
+			WithField("remoteIndex", hostinfo.remoteIndexId).WithField("collision", existingRemoteIndex.vpnIp).
+			Info("New host shadows existing host remoteIndex")
+	}
+
+	c.mainHostMap.addHostInfo(hostinfo, f)
+	c.pendingHostMap.unlockedDeleteHostInfo(hostinfo)
+}
+
+// AddIndexHostInfo generates a unique localIndexId for this HostInfo
+// and adds it to the pendingHostMap. Will error if we are unable to generate
+// a unique localIndexId
+func (c *HandshakeManager) AddIndexHostInfo(h *HostInfo) error {
+	c.pendingHostMap.Lock()
+	defer c.pendingHostMap.Unlock()
+	c.mainHostMap.RLock()
+	defer c.mainHostMap.RUnlock()
+
+	for i := 0; i < 32; i++ {
+		index, err := generateIndex(c.l)
 		if err != nil {
-		return nil, fmt.Errorf("Issue adding index: %d", index)
-	}
-	//c.mainHostMap.AddIndexHostInfo(index, hostinfo)
-	c.InboundHandshakeTimer.Add(index, time.Second*10)
-	return hostinfo, nil
+			return err
 		}

-func (c *HandshakeManager) AddIndexHostInfo(index uint32, h *HostInfo) {
-	c.pendingHostMap.AddIndexHostInfo(index, h)
+		_, inPending := c.pendingHostMap.Indexes[index]
+		_, inMain := c.mainHostMap.Indexes[index]
+
+		if !inMain && !inPending {
+			h.localIndexId = index
+			c.pendingHostMap.Indexes[index] = h
+			return nil
+		}
 	}

-func (c *HandshakeManager) DeleteIndex(index uint32) {
-	//l.Debugln("Deleting pending index :", index)
-	c.pendingHostMap.DeleteIndex(index)
+	return errors.New("failed to generate unique localIndexId")
+}
+
+func (c *HandshakeManager) addRemoteIndexHostInfo(index uint32, h *HostInfo) {
+	c.pendingHostMap.addRemoteIndexHostInfo(index, h)
+}
+
+func (c *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) {
+	//l.Debugln("Deleting pending hostinfo :", hostinfo)
+	c.pendingHostMap.DeleteHostInfo(hostinfo)
 }

 func (c *HandshakeManager) QueryIndex(index uint32) (*HostInfo, error) {
@@ -183,18 +368,28 @@ func (c *HandshakeManager) EmitStats() {

 // Utility functions below

-func generateIndex() (uint32, error) {
+func generateIndex(l *logrus.Logger) (uint32, error) {
 	b := make([]byte, 4)
+
+	// Let zero mean we don't know the ID, so don't generate zero
+	var index uint32
+	for index == 0 {
 		_, err := rand.Read(b)
 		if err != nil {
 			l.Errorln(err)
 			return 0, err
 		}

-	index := binary.BigEndian.Uint32(b)
+		index = binary.BigEndian.Uint32(b)
+	}
+
 	if l.Level >= logrus.DebugLevel {
 		l.WithField("index", index).
 			Debug("Generated index")
 	}
 	return index, nil
 }
+
+func hsTimeout(tries int, interval time.Duration) time.Duration {
+	return time.Duration(tries / 2 * ((2 * int(interval)) + (tries-1)*int(interval)))
+}
--- a/handshake_manager_test.go
+++ b/handshake_manager_test.go
@@ -5,187 +5,108 @@ import (
 	"testing"
 	"time"

+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
+	"github.com/slackhq/nebula/util"
 	"github.com/stretchr/testify/assert"
 )

-var indexes []uint32 = []uint32{1000, 2000, 3000, 4000}
-
-//var ips []uint32 = []uint32{9000, 9999999, 3, 292394923}
-var ips []uint32 = []uint32{9000}
-
-func Test_NewHandshakeManagerIndex(t *testing.T) {
-	_, tuncidr, _ := net.ParseCIDR("1.1.1.1/24")
-	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
-	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
-	preferredRanges := []*net.IPNet{localrange}
-	mainHM := NewHostMap("test", vpncidr, preferredRanges)
-
-	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{})
-
-	now := time.Now()
-	blah.NextInboundHandshakeTimerTick(now)
-
-	// Add four indexes
-	for _, v := range indexes {
-		blah.AddIndex(v, &ConnectionState{})
-	}
-	// Confirm they are in the pending index list
-	for _, v := range indexes {
-		assert.Contains(t, blah.pendingHostMap.Indexes, uint32(v))
-	}
-	// Adding something to pending should not affect the main hostmap
-	assert.Len(t, mainHM.Indexes, 0)
-	// Jump ahead 8 seconds
-	for i := 1; i <= HandshakeRetries; i++ {
-		next_tick := now.Add(HandshakeTryInterval * time.Duration(i))
-		blah.NextInboundHandshakeTimerTick(next_tick)
-	}
-	// Confirm they are still in the pending index list
-	for _, v := range indexes {
-		assert.Contains(t, blah.pendingHostMap.Indexes, uint32(v))
-	}
-	// Jump ahead 4 more seconds
-	next_tick := now.Add(12 * time.Second)
-	blah.NextInboundHandshakeTimerTick(next_tick)
-	// Confirm they have been removed
-	for _, v := range indexes {
-		assert.NotContains(t, blah.pendingHostMap.Indexes, uint32(v))
-	}
-}
-
-func Test_NewHandshakeManagerVpnIP(t *testing.T) {
-	_, tuncidr, _ := net.ParseCIDR("1.1.1.1/24")
+func Test_NewHandshakeManagerVpnIp(t *testing.T) {
+	l := util.NewTestLogger()
+	_, tuncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
+	ip := iputil.Ip2VpnIp(net.ParseIP("172.1.1.2"))
 	preferredRanges := []*net.IPNet{localrange}
 	mw := &mockEncWriter{}
-	mainHM := NewHostMap("test", vpncidr, preferredRanges)
+	mainHM := NewHostMap(l, "test", vpncidr, preferredRanges)

-	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{})
+	blah := NewHandshakeManager(l, tuncidr, preferredRanges, mainHM, &LightHouse{}, &udp.Conn{}, defaultHandshakeConfig)

 	now := time.Now()
 	blah.NextOutboundHandshakeTimerTick(now, mw)

-	// Add four "IPs" - which are just uint32s
-	for _, v := range ips {
-		blah.AddVpnIP(v)
-	}
+	i := blah.AddVpnIp(ip)
+	i.remotes = NewRemoteList()
+	i.HandshakeReady = true
+
 	// Adding something to pending should not affect the main hostmap
 	assert.Len(t, mainHM.Hosts, 0)
-	// Confirm they are in the pending index list
-	for _, v := range ips {
-		assert.Contains(t, blah.pendingHostMap.Hosts, uint32(v))
-	}

-	// Jump ahead `HandshakeRetries` ticks
-	cumulative := time.Duration(0)
-	for i := 0; i <= HandshakeRetries+1; i++ {
-		cumulative += time.Duration(i)*HandshakeTryInterval + 1
-		next_tick := now.Add(cumulative)
-		//l.Infoln(next_tick)
-		blah.NextOutboundHandshakeTimerTick(next_tick, mw)
+	// Confirm they are in the pending index list
+	assert.Contains(t, blah.pendingHostMap.Hosts, ip)
+
+	// Jump ahead `HandshakeRetries` ticks, offset by one to get the sleep logic right
+	for i := 1; i <= DefaultHandshakeRetries+1; i++ {
+		now = now.Add(time.Duration(i) * DefaultHandshakeTryInterval)
+		blah.NextOutboundHandshakeTimerTick(now, mw)
 	}

 	// Confirm they are still in the pending index list
-	for _, v := range ips {
-		assert.Contains(t, blah.pendingHostMap.Hosts, uint32(v))
-	}
-	// Jump ahead 1 more second
-	cumulative += time.Duration(HandshakeRetries+1) * HandshakeTryInterval
-	next_tick := now.Add(cumulative)
-	//l.Infoln(next_tick)
-	blah.NextOutboundHandshakeTimerTick(next_tick, mw)
+	assert.Contains(t, blah.pendingHostMap.Hosts, ip)
+
+	// Tick 1 more time, a minute will certainly flush it out
+	blah.NextOutboundHandshakeTimerTick(now.Add(time.Minute), mw)
+
 	// Confirm they have been removed
-	for _, v := range ips {
-		assert.NotContains(t, blah.pendingHostMap.Hosts, uint32(v))
-	}
+	assert.NotContains(t, blah.pendingHostMap.Hosts, ip)
 }

-func Test_NewHandshakeManagerVpnIPcleanup(t *testing.T) {
-	_, tuncidr, _ := net.ParseCIDR("1.1.1.1/24")
+func Test_NewHandshakeManagerTrigger(t *testing.T) {
+	l := util.NewTestLogger()
+	_, tuncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
+	ip := iputil.Ip2VpnIp(net.ParseIP("172.1.1.2"))
 	preferredRanges := []*net.IPNet{localrange}
 	mw := &mockEncWriter{}
-	mainHM := NewHostMap("test", vpncidr, preferredRanges)
+	mainHM := NewHostMap(l, "test", vpncidr, preferredRanges)
+	lh := &LightHouse{addrMap: make(map[iputil.VpnIp]*RemoteList), l: l}

-	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{})
+	blah := NewHandshakeManager(l, tuncidr, preferredRanges, mainHM, lh, &udp.Conn{}, defaultHandshakeConfig)

 	now := time.Now()
 	blah.NextOutboundHandshakeTimerTick(now, mw)

-	hostinfo := blah.AddVpnIP(101010)
-	// Pretned we have an index too
-	blah.AddIndexHostInfo(12341234, hostinfo)
-	assert.Contains(t, blah.pendingHostMap.Indexes, uint32(12341234))
+	assert.Equal(t, 0, testCountTimerWheelEntries(blah.OutboundHandshakeTimer))

-	// Jump ahead `HandshakeRetries` ticks. Eviction should happen in pending
-	// but not main hostmap
-	cumulative := time.Duration(0)
-	for i := 1; i <= HandshakeRetries+2; i++ {
-		cumulative += HandshakeTryInterval * time.Duration(i)
-		next_tick := now.Add(cumulative)
-		blah.NextOutboundHandshakeTimerTick(next_tick, mw)
-	}
-	/*
-		for i := 0; i <= HandshakeRetries+1; i++ {
-			next_tick := now.Add(cumulative)
-			//l.Infoln(next_tick)
-			blah.NextOutboundHandshakeTimerTick(next_tick)
-		}
-	*/
-	/*
-		for i := 0; i <= HandshakeRetries+1; i++ {
-			next_tick := now.Add(time.Duration(i) * time.Second)
-			blah.NextOutboundHandshakeTimerTick(next_tick)
-		}
-	*/
+	hi := blah.AddVpnIp(ip)
+	hi.HandshakeReady = true
+	assert.Equal(t, 1, testCountTimerWheelEntries(blah.OutboundHandshakeTimer))
+	assert.Equal(t, 0, hi.HandshakeCounter, "Should not have attempted a handshake yet")

-	/*
-		cumulative += HandshakeTryInterval*time.Duration(HandshakeRetries) + 3
-		next_tick := now.Add(cumulative)
-		l.Infoln(cumulative, next_tick)
-		blah.NextOutboundHandshakeTimerTick(next_tick)
-	*/
-	assert.NotContains(t, blah.pendingHostMap.Hosts, uint32(101010))
-	assert.NotContains(t, blah.pendingHostMap.Indexes, uint32(12341234))
+	// Trigger the same method the channel will but, this should set our remotes pointer
+	blah.handleOutbound(ip, mw, true)
+	assert.Equal(t, 1, hi.HandshakeCounter, "Trigger should have done a handshake attempt")
+	assert.NotNil(t, hi.remotes, "Manager should have set my remotes pointer")
+
+	// Make sure the trigger doesn't double schedule the timer entry
+	assert.Equal(t, 1, testCountTimerWheelEntries(blah.OutboundHandshakeTimer))
+
+	uaddr := udp.NewAddrFromString("10.1.1.1:4242")
+	hi.remotes.unlockedPrependV4(ip, NewIp4AndPort(uaddr.IP, uint32(uaddr.Port)))
+
+	// We now have remotes but only the first trigger should have pushed things forward
+	blah.handleOutbound(ip, mw, true)
+	assert.Equal(t, 1, hi.HandshakeCounter, "Trigger should have not done a handshake attempt")
+	assert.Equal(t, 1, testCountTimerWheelEntries(blah.OutboundHandshakeTimer))
 }

-func Test_NewHandshakeManagerIndexcleanup(t *testing.T) {
-	_, tuncidr, _ := net.ParseCIDR("1.1.1.1/24")
-	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
-	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
-	preferredRanges := []*net.IPNet{localrange}
-	mainHM := NewHostMap("test", vpncidr, preferredRanges)
-
-	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{})
-
-	now := time.Now()
-	blah.NextInboundHandshakeTimerTick(now)
-
-	hostinfo, _ := blah.AddIndex(12341234, &ConnectionState{})
-	// Pretned we have an index too
-	blah.pendingHostMap.AddVpnIPHostInfo(101010, hostinfo)
-	assert.Contains(t, blah.pendingHostMap.Hosts, uint32(101010))
-
-	for i := 1; i <= HandshakeRetries+2; i++ {
-		next_tick := now.Add(HandshakeTryInterval * time.Duration(i))
-		blah.NextInboundHandshakeTimerTick(next_tick)
+func testCountTimerWheelEntries(tw *SystemTimerWheel) (c int) {
+	for _, i := range tw.wheel {
+		n := i.Head
+		for n != nil {
+			c++
+			n = n.Next
 		}
-
-	next_tick := now.Add(HandshakeTryInterval*HandshakeRetries + 3)
-	blah.NextInboundHandshakeTimerTick(next_tick)
-	assert.NotContains(t, blah.pendingHostMap.Hosts, uint32(101010))
-	assert.NotContains(t, blah.pendingHostMap.Indexes, uint32(12341234))
+	}
+	return c
 }

 type mockEncWriter struct {
 }

-func (mw *mockEncWriter) SendMessageToVpnIp(t NebulaMessageType, st NebulaMessageSubType, vpnIp uint32, p, nb, out []byte) {
-	return
-}
-
-func (mw *mockEncWriter) SendMessageToAll(t NebulaMessageType, st NebulaMessageSubType, vpnIp uint32, p, nb, out []byte) {
+func (mw *mockEncWriter) SendMessageToVpnIp(t header.MessageType, st header.MessageSubType, vpnIp iputil.VpnIp, p, nb, out []byte) {
 	return
 }
--- a/header/header.go
+++ b/header/header.go
@@ -1,4 +1,4 @@
-package nebula
+package header

 import (
 	"encoding/binary"
@@ -19,82 +19,78 @@ import (
 // |-----------------------------------------------------------------------|
 // |                               payload...                              |

+type m map[string]interface{}
+
 const (
 	Version uint8 = 1
-	HeaderLen       = 16
+	Len           = 16
 )

-type NebulaMessageType uint8
-type NebulaMessageSubType uint8
+type MessageType uint8
+type MessageSubType uint8

 const (
-	handshake   NebulaMessageType = 0
-	message     NebulaMessageType = 1
-	recvError   NebulaMessageType = 2
-	lightHouse  NebulaMessageType = 3
-	test        NebulaMessageType = 4
-	closeTunnel NebulaMessageType = 5
-
-	//TODO These are deprecated as of 06/12/2018 - NB
-	testRemote      NebulaMessageType = 6
-	testRemoteReply NebulaMessageType = 7
+	Handshake   MessageType = 0
+	Message     MessageType = 1
+	RecvError   MessageType = 2
+	LightHouse  MessageType = 3
+	Test        MessageType = 4
+	CloseTunnel MessageType = 5
 )

-var typeMap = map[NebulaMessageType]string{
-	handshake:   "handshake",
-	message:     "message",
-	recvError:   "recvError",
-	lightHouse:  "lightHouse",
-	test:        "test",
-	closeTunnel: "closeTunnel",
-
-	//TODO These are deprecated as of 06/12/2018 - NB
-	testRemote:      "testRemote",
-	testRemoteReply: "testRemoteReply",
+var typeMap = map[MessageType]string{
+	Handshake:   "handshake",
+	Message:     "message",
+	RecvError:   "recvError",
+	LightHouse:  "lightHouse",
+	Test:        "test",
+	CloseTunnel: "closeTunnel",
 }

 const (
-	testRequest NebulaMessageSubType = 0
-	testReply   NebulaMessageSubType = 1
+	TestRequest MessageSubType = 0
+	TestReply   MessageSubType = 1
 )

-var eHeaderTooShort = errors.New("header is too short")
+const (
+	HandshakeIXPSK0 MessageSubType = 0
+	HandshakeXXPSK0 MessageSubType = 1
+)

-var subTypeTestMap = map[NebulaMessageSubType]string{
-	testRequest: "testRequest",
-	testReply:   "testReply",
+var ErrHeaderTooShort = errors.New("header is too short")
+
+var subTypeTestMap = map[MessageSubType]string{
+	TestRequest: "testRequest",
+	TestReply:   "testReply",
 }

-var subTypeNoneMap = map[NebulaMessageSubType]string{0: "none"}
+var subTypeNoneMap = map[MessageSubType]string{0: "none"}

-var subTypeMap = map[NebulaMessageType]*map[NebulaMessageSubType]string{
-	message:     &subTypeNoneMap,
-	recvError:   &subTypeNoneMap,
-	lightHouse:  &subTypeNoneMap,
-	test:        &subTypeTestMap,
-	closeTunnel: &subTypeNoneMap,
-	handshake: {
-		handshakeIXPSK0: "ix_psk0",
+var subTypeMap = map[MessageType]*map[MessageSubType]string{
+	Message:     &subTypeNoneMap,
+	RecvError:   &subTypeNoneMap,
+	LightHouse:  &subTypeNoneMap,
+	Test:        &subTypeTestMap,
+	CloseTunnel: &subTypeNoneMap,
+	Handshake: {
+		HandshakeIXPSK0: "ix_psk0",
 	},
-	//TODO: these are deprecated
-	testRemote:      &subTypeNoneMap,
-	testRemoteReply: &subTypeNoneMap,
 }

-type Header struct {
+type H struct {
 	Version        uint8
-	Type           NebulaMessageType
-	Subtype        NebulaMessageSubType
+	Type           MessageType
+	Subtype        MessageSubType
 	Reserved       uint16
 	RemoteIndex    uint32
 	MessageCounter uint64
 }

-// HeaderEncode uses the provided byte array to encode the provided header values into.
+// Encode uses the provided byte array to encode the provided header values into.
 // Byte array must be capped higher than HeaderLen or this will panic
-func HeaderEncode(b []byte, v uint8, t uint8, st uint8, ri uint32, c uint64) []byte {
-	b = b[:HeaderLen]
-	b[0] = byte(v<<4 | (t & 0x0f))
+func Encode(b []byte, v uint8, t MessageType, st MessageSubType, ri uint32, c uint64) []byte {
+	b = b[:Len]
+	b[0] = v<<4 | byte(t&0x0f)
 	b[1] = byte(st)
 	binary.BigEndian.PutUint16(b[2:4], 0)
 	binary.BigEndian.PutUint32(b[4:8], ri)
@@ -103,7 +99,7 @@ func HeaderEncode(b []byte, v uint8, t uint8, st uint8, ri uint32, c uint64) []b
 }

 // String creates a readable string representation of a header
-func (h *Header) String() string {
+func (h *H) String() string {
 	if h == nil {
 		return "<nil>"
 	}
@@ -112,7 +108,7 @@ func (h *Header) String() string {
 }

 // MarshalJSON creates a json string representation of a header
-func (h *Header) MarshalJSON() ([]byte, error) {
+func (h *H) MarshalJSON() ([]byte, error) {
 	return json.Marshal(m{
 		"version":        h.Version,
 		"type":           h.TypeName(),
@@ -124,24 +120,24 @@ func (h *Header) MarshalJSON() ([]byte, error) {
 }

 // Encode turns header into bytes
-func (h *Header) Encode(b []byte) ([]byte, error) {
+func (h *H) Encode(b []byte) ([]byte, error) {
 	if h == nil {
 		return nil, errors.New("nil header")
 	}

-	return HeaderEncode(b, h.Version, uint8(h.Type), uint8(h.Subtype), h.RemoteIndex, h.MessageCounter), nil
+	return Encode(b, h.Version, h.Type, h.Subtype, h.RemoteIndex, h.MessageCounter), nil
 }

 // Parse is a helper function to parses given bytes into new Header struct
-func (h *Header) Parse(b []byte) error {
-	if len(b) < HeaderLen {
-		return eHeaderTooShort
+func (h *H) Parse(b []byte) error {
+	if len(b) < Len {
+		return ErrHeaderTooShort
 	}
 	// get upper 4 bytes
 	h.Version = uint8((b[0] >> 4) & 0x0f)
 	// get lower 4 bytes
-	h.Type = NebulaMessageType(b[0] & 0x0f)
-	h.Subtype = NebulaMessageSubType(b[1])
+	h.Type = MessageType(b[0] & 0x0f)
+	h.Subtype = MessageSubType(b[1])
 	h.Reserved = binary.BigEndian.Uint16(b[2:4])
 	h.RemoteIndex = binary.BigEndian.Uint32(b[4:8])
 	h.MessageCounter = binary.BigEndian.Uint64(b[8:16])
@@ -149,12 +145,12 @@ func (h *Header) Parse(b []byte) error {
 }

 // TypeName will transform the headers message type into a human string
-func (h *Header) TypeName() string {
+func (h *H) TypeName() string {
 	return TypeName(h.Type)
 }

 // TypeName will transform a nebula message type into a human string
-func TypeName(t NebulaMessageType) string {
+func TypeName(t MessageType) string {
 	if n, ok := typeMap[t]; ok {
 		return n
 	}
@@ -163,12 +159,12 @@ func TypeName(t NebulaMessageType) string {
 }

 // SubTypeName will transform the headers message sub type into a human string
-func (h *Header) SubTypeName() string {
+func (h *H) SubTypeName() string {
 	return SubTypeName(h.Type, h.Subtype)
 }

 // SubTypeName will transform a nebula message sub type into a human string
-func SubTypeName(t NebulaMessageType, s NebulaMessageSubType) string {
+func SubTypeName(t MessageType, s MessageSubType) string {
 	if n, ok := subTypeMap[t]; ok {
 		if x, ok := (*n)[s]; ok {
 			return x
@@ -179,8 +175,8 @@ func SubTypeName(t NebulaMessageType, s NebulaMessageSubType) string {
 }

 // NewHeader turns bytes into a header
-func NewHeader(b []byte) (*Header, error) {
-	h := new(Header)
+func NewHeader(b []byte) (*H, error) {
+	h := new(H)
 	if err := h.Parse(b); err != nil {
 		return nil, err
 	}
--- a/header/header_test.go
+++ b/header/header_test.go
@@ -0,0 +1,115 @@
+package header
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type headerTest struct {
+	expectedBytes []byte
+	*H
+}
+
+// 0001 0010 00010010
+var headerBigEndianTests = []headerTest{{
+	expectedBytes: []byte{0x54, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x9},
+	// 1010 0000
+	H: &H{
+		// 1111 1+2+4+8 = 15
+		Version:        5,
+		Type:           4,
+		Subtype:        0,
+		Reserved:       0,
+		RemoteIndex:    10,
+		MessageCounter: 9,
+	},
+},
+}
+
+func TestEncode(t *testing.T) {
+	for _, tt := range headerBigEndianTests {
+		b, err := tt.Encode(make([]byte, Len))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		assert.Equal(t, tt.expectedBytes, b)
+	}
+}
+
+func TestParse(t *testing.T) {
+	for _, tt := range headerBigEndianTests {
+		b := tt.expectedBytes
+		parsedHeader := &H{}
+		parsedHeader.Parse(b)
+
+		if !reflect.DeepEqual(tt.H, parsedHeader) {
+			t.Fatalf("got %#v; want %#v", parsedHeader, tt.H)
+		}
+	}
+}
+
+func TestTypeName(t *testing.T) {
+	assert.Equal(t, "test", TypeName(Test))
+	assert.Equal(t, "test", (&H{Type: Test}).TypeName())
+
+	assert.Equal(t, "unknown", TypeName(99))
+	assert.Equal(t, "unknown", (&H{Type: 99}).TypeName())
+}
+
+func TestSubTypeName(t *testing.T) {
+	assert.Equal(t, "testRequest", SubTypeName(Test, TestRequest))
+	assert.Equal(t, "testRequest", (&H{Type: Test, Subtype: TestRequest}).SubTypeName())
+
+	assert.Equal(t, "unknown", SubTypeName(99, TestRequest))
+	assert.Equal(t, "unknown", (&H{Type: 99, Subtype: TestRequest}).SubTypeName())
+
+	assert.Equal(t, "unknown", SubTypeName(Test, 99))
+	assert.Equal(t, "unknown", (&H{Type: Test, Subtype: 99}).SubTypeName())
+
+	assert.Equal(t, "none", SubTypeName(Message, 0))
+	assert.Equal(t, "none", (&H{Type: Message, Subtype: 0}).SubTypeName())
+}
+
+func TestTypeMap(t *testing.T) {
+	// Force people to document this stuff
+	assert.Equal(t, map[MessageType]string{
+		Handshake:   "handshake",
+		Message:     "message",
+		RecvError:   "recvError",
+		LightHouse:  "lightHouse",
+		Test:        "test",
+		CloseTunnel: "closeTunnel",
+	}, typeMap)
+
+	assert.Equal(t, map[MessageType]*map[MessageSubType]string{
+		Message:     &subTypeNoneMap,
+		RecvError:   &subTypeNoneMap,
+		LightHouse:  &subTypeNoneMap,
+		Test:        &subTypeTestMap,
+		CloseTunnel: &subTypeNoneMap,
+		Handshake: {
+			HandshakeIXPSK0: "ix_psk0",
+		},
+	}, subTypeMap)
+}
+
+func TestHeader_String(t *testing.T) {
+	assert.Equal(
+		t,
+		"ver=100 type=test subtype=testRequest reserved=0x63 remoteindex=98 messagecounter=97",
+		(&H{100, Test, TestRequest, 99, 98, 97}).String(),
+	)
+}
+
+func TestHeader_MarshalJSON(t *testing.T) {
+	b, err := (&H{100, Test, TestRequest, 99, 98, 97}).MarshalJSON()
+	assert.Nil(t, err)
+	assert.Equal(
+		t,
+		"{\"messageCounter\":97,\"remoteIndex\":98,\"reserved\":99,\"subType\":\"testRequest\",\"type\":\"test\",\"version\":100}",
+		string(b),
+	)
+}
--- a/header_test.go
+++ b/header_test.go
@@ -1,118 +0,0 @@
-package nebula
-
-import (
-	"github.com/stretchr/testify/assert"
-	"reflect"
-	"testing"
-)
-
-type headerTest struct {
-	expectedBytes []byte
-	*Header
-}
-
-// 0001 0010 00010010
-var headerBigEndianTests = []headerTest{{
-	expectedBytes: []byte{0x54, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x9},
-	// 1010 0000
-	Header: &Header{
-		// 1111 1+2+4+8 = 15
-		Version:        5,
-		Type:           4,
-		Subtype:        0,
-		Reserved:       0,
-		RemoteIndex:    10,
-		MessageCounter: 9,
-	},
-},
-}
-
-func TestEncode(t *testing.T) {
-	for _, tt := range headerBigEndianTests {
-		b, err := tt.Encode(make([]byte, HeaderLen))
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		assert.Equal(t, tt.expectedBytes, b)
-	}
-}
-
-func TestParse(t *testing.T) {
-	for _, tt := range headerBigEndianTests {
-		b := tt.expectedBytes
-		parsedHeader := &Header{}
-		parsedHeader.Parse(b)
-
-		if !reflect.DeepEqual(tt.Header, parsedHeader) {
-			t.Fatalf("got %#v; want %#v", parsedHeader, tt.Header)
-		}
-	}
-}
-
-func TestTypeName(t *testing.T) {
-	assert.Equal(t, "test", TypeName(test))
-	assert.Equal(t, "test", (&Header{Type: test}).TypeName())
-
-	assert.Equal(t, "unknown", TypeName(99))
-	assert.Equal(t, "unknown", (&Header{Type: 99}).TypeName())
-}
-
-func TestSubTypeName(t *testing.T) {
-	assert.Equal(t, "testRequest", SubTypeName(test, testRequest))
-	assert.Equal(t, "testRequest", (&Header{Type: test, Subtype: testRequest}).SubTypeName())
-
-	assert.Equal(t, "unknown", SubTypeName(99, testRequest))
-	assert.Equal(t, "unknown", (&Header{Type: 99, Subtype: testRequest}).SubTypeName())
-
-	assert.Equal(t, "unknown", SubTypeName(test, 99))
-	assert.Equal(t, "unknown", (&Header{Type: test, Subtype: 99}).SubTypeName())
-
-	assert.Equal(t, "none", SubTypeName(message, 0))
-	assert.Equal(t, "none", (&Header{Type: message, Subtype: 0}).SubTypeName())
-}
-
-func TestTypeMap(t *testing.T) {
-	// Force people to document this stuff
-	assert.Equal(t, map[NebulaMessageType]string{
-		handshake:       "handshake",
-		message:         "message",
-		recvError:       "recvError",
-		lightHouse:      "lightHouse",
-		test:            "test",
-		closeTunnel:     "closeTunnel",
-		testRemote:      "testRemote",
-		testRemoteReply: "testRemoteReply",
-	}, typeMap)
-
-	assert.Equal(t, map[NebulaMessageType]*map[NebulaMessageSubType]string{
-		message:     &subTypeNoneMap,
-		recvError:   &subTypeNoneMap,
-		lightHouse:  &subTypeNoneMap,
-		test:        &subTypeTestMap,
-		closeTunnel: &subTypeNoneMap,
-		handshake: {
-			handshakeIXPSK0: "ix_psk0",
-		},
-		testRemote:      &subTypeNoneMap,
-		testRemoteReply: &subTypeNoneMap,
-	}, subTypeMap)
-}
-
-func TestHeader_String(t *testing.T) {
-	assert.Equal(
-		t,
-		"ver=100 type=test subtype=testRequest reserved=0x63 remoteindex=98 messagecounter=97",
-		(&Header{100, test, testRequest, 99, 98, 97}).String(),
-	)
-}
-
-func TestHeader_MarshalJSON(t *testing.T) {
-	b, err := (&Header{100, test, testRequest, 99, 98, 97}).MarshalJSON()
-	assert.Nil(t, err)
-	assert.Equal(
-		t,
-		"{\"messageCounter\":97,\"remoteIndex\":98,\"reserved\":99,\"subType\":\"testRequest\",\"type\":\"test\",\"version\":100}",
-		string(b),
-	)
-}
--- a/hostmap.go
+++ b/hostmap.go
--- a/hostmap_test.go
+++ b/hostmap_test.go
@@ -1,166 +1 @@
 package nebula
-
-import (
-	"net"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-/*
-func TestHostInfoDestProbe(t *testing.T) {
-	a, _ := net.ResolveUDPAddr("udp", "1.0.0.1:22222")
-	d := NewHostInfoDest(a)
-
-	// 999 probes that all return should give a 100% success rate
-	for i := 0; i < 999; i++ {
-		meh := d.Probe()
-		d.ProbeReceived(meh)
-	}
-	assert.Equal(t, d.Grade(), float64(1))
-
-	// 999 probes of which only half return should give a 50% success rate
-	for i := 0; i < 999; i++ {
-		meh := d.Probe()
-		if i%2 == 0 {
-			d.ProbeReceived(meh)
-		}
-	}
-	assert.Equal(t, d.Grade(), float64(.5))
-
-	// 999 probes of which none return should give a 0% success rate
-	for i := 0; i < 999; i++ {
-		d.Probe()
-	}
-	assert.Equal(t, d.Grade(), float64(0))
-
-	// 999 probes of which only 1/4 return should give a 25% success rate
-	for i := 0; i < 999; i++ {
-		meh := d.Probe()
-		if i%4 == 0 {
-			d.ProbeReceived(meh)
-		}
-	}
-	assert.Equal(t, d.Grade(), float64(.25))
-
-	// 999 probes of which only half return and are duplicates should give a 50% success rate
-	for i := 0; i < 999; i++ {
-		meh := d.Probe()
-		if i%2 == 0 {
-			d.ProbeReceived(meh)
-			d.ProbeReceived(meh)
-		}
-	}
-	assert.Equal(t, d.Grade(), float64(.5))
-
-	// 999 probes of which only way old replies return should give a 0% success rate
-	for i := 0; i < 999; i++ {
-		meh := d.Probe()
-		d.ProbeReceived(meh - 101)
-	}
-	assert.Equal(t, d.Grade(), float64(0))
-
-}
-*/
-
-func TestHostmap(t *testing.T) {
-	_, myNet, _ := net.ParseCIDR("10.128.0.0/16")
-	_, localToMe, _ := net.ParseCIDR("192.168.1.0/24")
-	myNets := []*net.IPNet{myNet}
-	preferredRanges := []*net.IPNet{localToMe}
-
-	m := NewHostMap("test", myNet, preferredRanges)
-
-	a := NewUDPAddrFromString("10.127.0.3:11111")
-	b := NewUDPAddrFromString("1.0.0.1:22222")
-	y := NewUDPAddrFromString("10.128.0.3:11111")
-	m.AddRemote(ip2int(net.ParseIP("127.0.0.1")), a)
-	m.AddRemote(ip2int(net.ParseIP("127.0.0.1")), b)
-	m.AddRemote(ip2int(net.ParseIP("127.0.0.1")), y)
-
-	info, _ := m.QueryVpnIP(ip2int(net.ParseIP("127.0.0.1")))
-
-	// There should be three remotes in the host map
-	assert.Equal(t, 3, len(info.Remotes))
-
-	// Adding an identical remote should not change the count
-	m.AddRemote(ip2int(net.ParseIP("127.0.0.1")), y)
-	assert.Equal(t, 3, len(info.Remotes))
-
-	// Adding a fresh remote should add one
-	y = NewUDPAddrFromString("10.18.0.3:11111")
-	m.AddRemote(ip2int(net.ParseIP("127.0.0.1")), y)
-	assert.Equal(t, 4, len(info.Remotes))
-
-	// Query and reference remote should get the first one (and not nil)
-	info, _ = m.QueryVpnIP(ip2int(net.ParseIP("127.0.0.1")))
-	assert.NotNil(t, info.remote)
-
-	// Promotion should ensure that the best remote is chosen (y)
-	info.ForcePromoteBest(myNets)
-	assert.True(t, myNet.Contains(udp2ip(info.remote)))
-
-}
-
-func TestHostmapdebug(t *testing.T) {
-	_, myNet, _ := net.ParseCIDR("10.128.0.0/16")
-	_, localToMe, _ := net.ParseCIDR("192.168.1.0/24")
-	preferredRanges := []*net.IPNet{localToMe}
-	m := NewHostMap("test", myNet, preferredRanges)
-
-	a := NewUDPAddrFromString("10.127.0.3:11111")
-	b := NewUDPAddrFromString("1.0.0.1:22222")
-	y := NewUDPAddrFromString("10.128.0.3:11111")
-	m.AddRemote(ip2int(net.ParseIP("127.0.0.1")), a)
-	m.AddRemote(ip2int(net.ParseIP("127.0.0.1")), b)
-	m.AddRemote(ip2int(net.ParseIP("127.0.0.1")), y)
-
-	//t.Errorf("%s", m.DebugRemotes(1))
-}
-
-func TestHostMap_rotateRemote(t *testing.T) {
-	h := HostInfo{}
-	// 0 remotes, no panic
-	h.rotateRemote()
-	assert.Nil(t, h.remote)
-
-	// 1 remote, no panic
-	h.AddRemote(*NewUDPAddr(ip2int(net.IP{1, 1, 1, 1}), 0))
-	h.rotateRemote()
-	assert.Equal(t, udp2ipInt(h.remote), ip2int(net.IP{1, 1, 1, 1}))
-
-	h.AddRemote(*NewUDPAddr(ip2int(net.IP{1, 1, 1, 2}), 0))
-	h.AddRemote(*NewUDPAddr(ip2int(net.IP{1, 1, 1, 3}), 0))
-	h.AddRemote(*NewUDPAddr(ip2int(net.IP{1, 1, 1, 4}), 0))
-
-	// Rotate through those 3
-	h.rotateRemote()
-	assert.Equal(t, udp2ipInt(h.remote), ip2int(net.IP{1, 1, 1, 2}))
-
-	h.rotateRemote()
-	assert.Equal(t, udp2ipInt(h.remote), ip2int(net.IP{1, 1, 1, 3}))
-
-	h.rotateRemote()
-	assert.Equal(t, udp2ipInt(h.remote), ip2int(net.IP{1, 1, 1, 4}))
-
-	// Finally, we should start over
-	h.rotateRemote()
-	assert.Equal(t, udp2ipInt(h.remote), ip2int(net.IP{1, 1, 1, 1}))
-}
-
-func BenchmarkHostmappromote2(b *testing.B) {
-	for n := 0; n < b.N; n++ {
-		_, myNet, _ := net.ParseCIDR("10.128.0.0/16")
-		_, localToMe, _ := net.ParseCIDR("192.168.1.0/24")
-		preferredRanges := []*net.IPNet{localToMe}
-		m := NewHostMap("test", myNet, preferredRanges)
-		y := NewUDPAddrFromString("10.128.0.3:11111")
-		a := NewUDPAddrFromString("10.127.0.3:11111")
-		g := NewUDPAddrFromString("1.0.0.1:22222")
-		m.AddRemote(ip2int(net.ParseIP("127.0.0.1")), a)
-		m.AddRemote(ip2int(net.ParseIP("127.0.0.1")), g)
-		m.AddRemote(ip2int(net.ParseIP("127.0.0.1")), y)
-	}
-	b.Errorf("hi")
-
-}
--- a/inside.go
+++ b/inside.go
@@ -3,14 +3,17 @@ package nebula
 import (
 	"sync/atomic"

-	"github.com/flynn/noise"
 	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/firewall"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
 )

-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *FirewallPacket, nb, out []byte) {
+func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
 	err := newPacket(packet, false, fwPacket)
 	if err != nil {
-		l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
+		f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
 		return
 	}

@@ -19,12 +22,25 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *FirewallPacket,
 		return
 	}

+	// Ignore packets from self to self
+	if fwPacket.RemoteIP == f.myVpnIp {
+		return
+	}
+
 	// Ignore broadcast packets
 	if f.dropMulticast && isMulticast(fwPacket.RemoteIP) {
 		return
 	}

 	hostinfo := f.getOrHandshake(fwPacket.RemoteIP)
+	if hostinfo == nil {
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("vpnIp", fwPacket.RemoteIP).
+				WithField("fwPacket", fwPacket).
+				Debugln("dropping outbound packet, vpnIp not in our CIDR or in unsafe routes")
+		}
+		return
+	}
 	ci := hostinfo.ConnectionState

 	if ci.ready == false {
@@ -32,92 +48,132 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *FirewallPacket,
 		// the packet queue.
 		ci.queueLock.Lock()
 		if !ci.ready {
-			hostinfo.cachePacket(message, 0, packet, f.sendMessageNow)
+			hostinfo.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
 			ci.queueLock.Unlock()
 			return
 		}
 		ci.queueLock.Unlock()
 	}

-	if !f.firewall.Drop(packet, *fwPacket, false, ci.peerCert, trustedCAs) {
-		f.send(message, 0, ci, hostinfo, hostinfo.remote, packet, nb, out)
-		if f.lightHouse != nil && *ci.messageCounter%5000 == 0 {
-			f.lightHouse.Query(fwPacket.RemoteIP, f)
-		}
+	dropReason := f.firewall.Drop(packet, *fwPacket, false, hostinfo, f.caPool, localCache)
+	if dropReason == nil {
+		f.sendNoMetrics(header.Message, 0, ci, hostinfo, hostinfo.remote, packet, nb, out, q)

-	} else if l.Level >= logrus.DebugLevel {
-		l.WithField("vpnIp", IntIp(hostinfo.hostId)).WithField("fwPacket", fwPacket).
+	} else if f.l.Level >= logrus.DebugLevel {
+		hostinfo.logger(f.l).
+			WithField("fwPacket", fwPacket).
+			WithField("reason", dropReason).
 			Debugln("dropping outbound packet")
 	}
 }

-func (f *Interface) getOrHandshake(vpnIp uint32) *HostInfo {
-	hostinfo, err := f.hostMap.PromoteBestQueryVpnIP(vpnIp, f)
-
-	//if err != nil || hostinfo.ConnectionState == nil {
-	if err != nil {
-		hostinfo, err = f.handshakeManager.pendingHostMap.QueryVpnIP(vpnIp)
-		if err != nil {
-			hostinfo = f.handshakeManager.AddVpnIP(vpnIp)
+// getOrHandshake returns nil if the vpnIp is not routable
+func (f *Interface) getOrHandshake(vpnIp iputil.VpnIp) *HostInfo {
+	//TODO: we can find contains without converting back to bytes
+	if f.hostMap.vpnCIDR.Contains(vpnIp.ToIP()) == false {
+		vpnIp = f.hostMap.queryUnsafeRoute(vpnIp)
+		if vpnIp == 0 {
+			return nil
 		}
 	}
+	hostinfo, err := f.hostMap.PromoteBestQueryVpnIp(vpnIp, f)

+	if err != nil {
+		hostinfo, err = f.handshakeManager.pendingHostMap.QueryVpnIp(vpnIp)
+		if err != nil {
+			hostinfo = f.handshakeManager.AddVpnIp(vpnIp)
+		}
+	}
 	ci := hostinfo.ConnectionState

 	if ci != nil && ci.eKey != nil && ci.ready {
 		return hostinfo
 	}

+	// Handshake is not ready, we need to grab the lock now before we start the handshake process
+	hostinfo.Lock()
+	defer hostinfo.Unlock()
+
+	// Double check, now that we have the lock
+	ci = hostinfo.ConnectionState
+	if ci != nil && ci.eKey != nil && ci.ready {
+		return hostinfo
+	}
+
+	// Create a connection state if we don't have one yet
 	if ci == nil {
-		// if we don't have a connection state, then send a handshake initiation
-		ci = f.newConnectionState(true, noise.HandshakeIX, []byte{}, 0)
-		// FIXME: Maybe make XX selectable, but probably not since psk makes it nearly pointless for us.
-		//ci = f.newConnectionState(true, noise.HandshakeXX, []byte{}, 0)
+		// Generate a PSK based on our config, this may be nil
+		p, err := f.psk.MakeFor(vpnIp)
+		if err != nil {
+			//TODO: This isn't fatal specifically but it's pretty bad
+			f.l.WithError(err).WithField("vpnIp", vpnIp).Error("Failed to get a PSK KDF")
+			return hostinfo
+		}
+
+		ci, err = f.newConnectionState(f.l, true, p)
+		if err != nil {
+			f.l.WithError(err).WithField("vpnIp", vpnIp).Error("Failed to get a connection state")
+			return hostinfo
+		}
 		hostinfo.ConnectionState = ci
-	} else if ci.eKey == nil {
-		// if we don't have any state at all, create it
 	}

 	// If we have already created the handshake packet, we don't want to call the function at all.
 	if !hostinfo.HandshakeReady {
 		ixHandshakeStage0(f, vpnIp, hostinfo)
-		// FIXME: Maybe make XX selectable, but probably not since psk makes it nearly pointless for us.
-		//xx_handshakeStage0(f, ip, hostinfo)
+
+		// If this is a static host, we don't need to wait for the HostQueryReply
+		// We can trigger the handshake right now
+		if _, ok := f.lightHouse.staticList[vpnIp]; ok {
+			select {
+			case f.handshakeManager.trigger <- vpnIp:
+			default:
+			}
+		}
 	}

 	return hostinfo
 }

-func (f *Interface) sendMessageNow(t NebulaMessageType, st NebulaMessageSubType, hostInfo *HostInfo, p, nb, out []byte) {
-	fp := &FirewallPacket{}
+func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostInfo *HostInfo, p, nb, out []byte) {
+	fp := &firewall.Packet{}
 	err := newPacket(p, false, fp)
 	if err != nil {
-		l.Warnf("error while parsing outgoing packet for firewall check; %v", err)
+		f.l.Warnf("error while parsing outgoing packet for firewall check; %v", err)
 		return
 	}

 	// check if packet is in outbound fw rules
-	if f.firewall.Drop(p, *fp, false, hostInfo.ConnectionState.peerCert, trustedCAs) {
-		l.WithField("fwPacket", fp).Debugln("dropping cached packet")
+	dropReason := f.firewall.Drop(p, *fp, false, hostInfo, f.caPool, nil)
+	if dropReason != nil {
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("fwPacket", fp).
+				WithField("reason", dropReason).
+				Debugln("dropping cached packet")
+		}
 		return
 	}

-	f.send(message, st, hostInfo.ConnectionState, hostInfo, hostInfo.remote, p, nb, out)
-	if f.lightHouse != nil && *hostInfo.ConnectionState.messageCounter%5000 == 0 {
-		f.lightHouse.Query(fp.RemoteIP, f)
-	}
+	f.sendNoMetrics(header.Message, st, hostInfo.ConnectionState, hostInfo, hostInfo.remote, p, nb, out, 0)
 }

 // SendMessageToVpnIp handles real ip:port lookup and sends to the current best known address for vpnIp
-func (f *Interface) SendMessageToVpnIp(t NebulaMessageType, st NebulaMessageSubType, vpnIp uint32, p, nb, out []byte) {
+func (f *Interface) SendMessageToVpnIp(t header.MessageType, st header.MessageSubType, vpnIp iputil.VpnIp, p, nb, out []byte) {
 	hostInfo := f.getOrHandshake(vpnIp)
+	if hostInfo == nil {
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("vpnIp", vpnIp).
+				Debugln("dropping SendMessageToVpnIp, vpnIp not in our CIDR or in unsafe routes")
+		}
+		return
+	}

 	if !hostInfo.ConnectionState.ready {
 		// Because we might be sending stored packets, lock here to stop new things going to
 		// the packet queue.
 		hostInfo.ConnectionState.queueLock.Lock()
 		if !hostInfo.ConnectionState.ready {
-			hostInfo.cachePacket(t, st, p, f.sendMessageToVpnIp)
+			hostInfo.cachePacket(f.l, t, st, p, f.sendMessageToVpnIp, f.cachedPacketMetrics)
 			hostInfo.ConnectionState.queueLock.Unlock()
 			return
 		}
@@ -128,37 +184,16 @@ func (f *Interface) SendMessageToVpnIp(t NebulaMessageType, st NebulaMessageSubT
 	return
 }

-func (f *Interface) sendMessageToVpnIp(t NebulaMessageType, st NebulaMessageSubType, hostInfo *HostInfo, p, nb, out []byte) {
+func (f *Interface) sendMessageToVpnIp(t header.MessageType, st header.MessageSubType, hostInfo *HostInfo, p, nb, out []byte) {
 	f.send(t, st, hostInfo.ConnectionState, hostInfo, hostInfo.remote, p, nb, out)
 }

-// SendMessageToAll handles real ip:port lookup and sends to all known addresses for vpnIp
-func (f *Interface) SendMessageToAll(t NebulaMessageType, st NebulaMessageSubType, vpnIp uint32, p, nb, out []byte) {
-	hostInfo := f.getOrHandshake(vpnIp)
-
-	if hostInfo.ConnectionState.ready == false {
-		// Because we might be sending stored packets, lock here to stop new things going to
-		// the packet queue.
-		hostInfo.ConnectionState.queueLock.Lock()
-		if !hostInfo.ConnectionState.ready {
-			hostInfo.cachePacket(t, st, p, f.sendMessageToAll)
-			hostInfo.ConnectionState.queueLock.Unlock()
-			return
-		}
-		hostInfo.ConnectionState.queueLock.Unlock()
+func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote *udp.Addr, p, nb, out []byte) {
+	f.messageMetrics.Tx(t, st, 1)
+	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0)
 }

-	f.sendMessageToAll(t, st, hostInfo, p, nb, out)
-	return
-}
-
-func (f *Interface) sendMessageToAll(t NebulaMessageType, st NebulaMessageSubType, hostInfo *HostInfo, p, nb, b []byte) {
-	for _, r := range hostInfo.RemoteUDPAddrs() {
-		f.send(t, st, hostInfo.ConnectionState, hostInfo, r, p, nb, b)
-	}
-}
-
-func (f *Interface) send(t NebulaMessageType, st NebulaMessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote *udpAddr, p, nb, out []byte) {
+func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote *udp.Addr, p, nb, out []byte, q int) {
 	if ci.eKey == nil {
 		//TODO: log warning
 		return
@@ -167,31 +202,44 @@ func (f *Interface) send(t NebulaMessageType, st NebulaMessageSubType, ci *Conne
 	var err error
 	//TODO: enable if we do more than 1 tun queue
 	//ci.writeLock.Lock()
-	c := atomic.AddUint64(ci.messageCounter, 1)
+	c := atomic.AddUint64(&ci.atomicMessageCounter, 1)

 	//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
-	out = HeaderEncode(out, Version, uint8(t), uint8(st), hostinfo.remoteIndexId, c)
-	f.connectionManager.Out(hostinfo.hostId)
+	out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
+	f.connectionManager.Out(hostinfo.vpnIp)
+
+	// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against
+	// all our IPs and enable a faster roaming.
+	if t != header.CloseTunnel && hostinfo.lastRebindCount != f.rebindCount {
+		//NOTE: there is an update hole if a tunnel isn't used and exactly 256 rebinds occur before the tunnel is
+		// finally used again. This tunnel would eventually be torn down and recreated if this action didn't help.
+		f.lightHouse.QueryServer(hostinfo.vpnIp, f)
+		hostinfo.lastRebindCount = f.rebindCount
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("vpnIp", hostinfo.vpnIp).Debug("Lighthouse update triggered for punch due to rebind counter")
+		}
+	}

 	out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
 	//TODO: see above note on lock
 	//ci.writeLock.Unlock()
 	if err != nil {
-		l.WithError(err).WithField("vpnIp", IntIp(hostinfo.hostId)).
+		hostinfo.logger(f.l).WithError(err).
 			WithField("udpAddr", remote).WithField("counter", c).
-			WithField("attemptedCounter", ci.messageCounter).
+			WithField("attemptedCounter", c).
 			Error("Failed to encrypt outgoing packet")
 		return
 	}

-	err = f.outside.WriteTo(out, remote)
+	err = f.writers[q].WriteTo(out, remote)
 	if err != nil {
-		l.WithError(err).WithField("vpnIp", IntIp(hostinfo.hostId)).
+		hostinfo.logger(f.l).WithError(err).
 			WithField("udpAddr", remote).Error("Failed to write outgoing packet")
 	}
+	return
 }

-func isMulticast(ip uint32) bool {
+func isMulticast(ip iputil.VpnIp) bool {
 	// Class D multicast
 	if (((ip >> 24) & 0xff) & 0xf0) == 0xe0 {
 		return true
--- a/interface.go
+++ b/interface.go
@@ -1,23 +1,40 @@
 package nebula

 import (
-	"crypto/sha256"
+	"context"
 	"errors"
-	"fmt"
 	"io"
+	"net"
 	"os"
+	"runtime"
+	"sync/atomic"
 	"time"
+	"unsafe"

 	"github.com/rcrowley/go-metrics"
-	"golang.org/x/crypto/hkdf"
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/firewall"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
 )

 const mtu = 9001

+type Inside interface {
+	io.ReadWriteCloser
+	Activate() error
+	CidrNet() *net.IPNet
+	DeviceName() string
+	WriteRaw([]byte) error
+	NewMultiQueueReader() (io.ReadWriteCloser, error)
+}
+
 type InterfaceConfig struct {
 	HostMap                 *HostMap
-	Outside                  *udpConn
-	Inside                   *Tun
+	Outside                 *udp.Conn
+	Inside                  Inside
 	certState               *CertState
 	Cipher                  string
 	Firewall                *Firewall
@@ -26,17 +43,23 @@ type InterfaceConfig struct {
 	lightHouse              *LightHouse
 	checkInterval           int
 	pendingDeletionInterval int
-	handshakeMACKey          string
-	handshakeAcceptedMACKeys []string
 	DropLocalBroadcast      bool
 	DropMulticast           bool
-	UDPBatchSize             int
+	routines                int
+	MessageMetrics          *MessageMetrics
+	version                 string
+	caPool                  *cert.NebulaCAPool
+	disconnectInvalid       bool
+	psk                     *Psk
+
+	ConntrackCacheTimeout time.Duration
+	l                     *logrus.Logger
 }

 type Interface struct {
 	hostMap            *HostMap
-	outside                  *udpConn
-	inside                   *Tun
+	outside            *udp.Conn
+	inside             Inside
 	certState          *CertState
 	cipher             string
 	firewall           *Firewall
@@ -45,20 +68,31 @@ type Interface struct {
 	serveDns           bool
 	createTime         time.Time
 	lightHouse         *LightHouse
-	handshakeMACKey          []byte
-	handshakeAcceptedMACKeys [][]byte
-	localBroadcast           uint32
+	localBroadcast     iputil.VpnIp
+	myVpnIp            iputil.VpnIp
 	dropLocalBroadcast bool
 	dropMulticast      bool
-	udpBatchSize             int
+	routines           int
+	caPool             *cert.NebulaCAPool
+	disconnectInvalid  bool
+
+	// rebindCount is used to decide if an active tunnel should trigger a punch notification through a lighthouse
+	rebindCount int8
 	version     string

-	metricRxRecvError metrics.Counter
-	metricTxRecvError metrics.Counter
+	conntrackCacheTimeout time.Duration
+	psk                   *Psk
+	writers               []*udp.Conn
+	readers               []io.ReadWriteCloser
+
 	metricHandshakes    metrics.Histogram
+	messageMetrics      *MessageMetrics
+	cachedPacketMetrics *cachedPacketMetrics
+
+	l *logrus.Logger
 }

-func NewInterface(c *InterfaceConfig) (*Interface, error) {
+func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 	if c.Outside == nil {
 		return nil, errors.New("no outside connection")
 	}
@@ -72,36 +106,7 @@ func NewInterface(c *InterfaceConfig) (*Interface, error) {
 		return nil, errors.New("no firewall rules")
 	}

-	// Use KDF to make this useful
-	hmacKey, err := sha256KdfFromString(c.handshakeMACKey)
-	if err != nil {
-		l.Debugln(err)
-	}
-
-	allowedMacs := make([][]byte, 0)
-	//allowedMacs = append(allowedMacs, mac)
-	if len(c.handshakeAcceptedMACKeys) > 0 {
-		for _, k := range c.handshakeAcceptedMACKeys {
-			// Use KDF to make these useful too
-			hmacKey, err := sha256KdfFromString(k)
-			if err != nil {
-				l.Debugln(err)
-			}
-			allowedMacs = append(allowedMacs, hmacKey)
-		}
-	} else {
-		if len(c.handshakeMACKey) > 0 {
-			l.Warnln("You have set an outgoing MAC but do not accept any incoming. This is probably not what you want.")
-		} else {
-			// This else is a fallback if we have not set any mac keys at all
-			hmacKey, err := sha256KdfFromString("")
-			if err != nil {
-				l.Debugln(err)
-			}
-			allowedMacs = append(allowedMacs, hmacKey)
-
-		}
-	}
+	myVpnIp := iputil.Ip2VpnIp(c.certState.certificate.Details.Ips[0].IP)

 	ifce := &Interface{
 		hostMap:            c.HostMap,
@@ -114,109 +119,147 @@ func NewInterface(c *InterfaceConfig) (*Interface, error) {
 		handshakeManager:   c.HandshakeManager,
 		createTime:         time.Now(),
 		lightHouse:         c.lightHouse,
-		handshakeMACKey:          hmacKey,
-		handshakeAcceptedMACKeys: allowedMacs,
-		localBroadcast:           ip2int(c.certState.certificate.Details.Ips[0].IP) | ^ip2int(c.certState.certificate.Details.Ips[0].Mask),
+		localBroadcast:     myVpnIp | ^iputil.Ip2VpnIp(c.certState.certificate.Details.Ips[0].Mask),
 		dropLocalBroadcast: c.DropLocalBroadcast,
 		dropMulticast:      c.DropMulticast,
-		udpBatchSize:             c.UDPBatchSize,
+		routines:           c.routines,
+		version:            c.version,
+		writers:            make([]*udp.Conn, c.routines),
+		readers:            make([]io.ReadWriteCloser, c.routines),
+		caPool:             c.caPool,
+		disconnectInvalid:  c.disconnectInvalid,
+		psk:                c.psk,
+		myVpnIp:            myVpnIp,
+
+		conntrackCacheTimeout: c.ConntrackCacheTimeout,

-		metricRxRecvError: metrics.GetOrRegisterCounter("messages.rx.recv_error", nil),
-		metricTxRecvError: metrics.GetOrRegisterCounter("messages.tx.recv_error", nil),
 		metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
+		messageMetrics:   c.MessageMetrics,
+		cachedPacketMetrics: &cachedPacketMetrics{
+			sent:    metrics.GetOrRegisterCounter("hostinfo.cached_packets.sent", nil),
+			dropped: metrics.GetOrRegisterCounter("hostinfo.cached_packets.dropped", nil),
+		},
+
+		l: c.l,
 	}

-	ifce.connectionManager = newConnectionManager(ifce, c.checkInterval, c.pendingDeletionInterval)
+	ifce.connectionManager = newConnectionManager(ctx, c.l, ifce, c.checkInterval, c.pendingDeletionInterval)

 	return ifce, nil
 }

-func (f *Interface) Run(tunRoutines, udpRoutines int, buildVersion string) {
+// activate creates the interface on the host. After the interface is created, any
+// other services that want to bind listeners to its IP may do so successfully. However,
+// the interface isn't going to process anything until run() is called.
+func (f *Interface) activate() {
 	// actually turn on tun dev
-	if err := f.inside.Activate(); err != nil {
-		l.Fatal(err)
+
+	addr, err := f.outside.LocalAddr()
+	if err != nil {
+		f.l.WithError(err).Error("Failed to get udp listen address")
 	}

-	f.version = buildVersion
-	l.WithField("interface", f.inside.Device).WithField("network", f.inside.Cidr.String()).
-		WithField("build", buildVersion).
+	f.l.WithField("interface", f.inside.DeviceName()).WithField("network", f.inside.CidrNet().String()).
+		WithField("build", f.version).WithField("udpAddr", addr).
 		Info("Nebula interface is active")

+	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
+
+	// Prepare n tun queues
+	var reader io.ReadWriteCloser = f.inside
+	for i := 0; i < f.routines; i++ {
+		if i > 0 {
+			reader, err = f.inside.NewMultiQueueReader()
+			if err != nil {
+				f.l.Fatal(err)
+			}
+		}
+		f.readers[i] = reader
+	}
+
+	if err := f.inside.Activate(); err != nil {
+		f.l.Fatal(err)
+	}
+}
+
+func (f *Interface) run() {
 	// Launch n queues to read packets from udp
-	for i := 0; i < udpRoutines; i++ {
+	for i := 0; i < f.routines; i++ {
 		go f.listenOut(i)
 	}

 	// Launch n queues to read packets from tun dev
-	for i := 0; i < tunRoutines; i++ {
-		go f.listenIn(i)
+	for i := 0; i < f.routines; i++ {
+		go f.listenIn(f.readers[i], i)
 	}
 }

 func (f *Interface) listenOut(i int) {
-	//TODO: handle error
-	addr, err := f.outside.LocalAddr()
-	if err != nil {
-		l.WithError(err).Error("failed to discover udp listening address")
-	}
+	runtime.LockOSThread()

-	var li *udpConn
+	var li *udp.Conn
+	// TODO clean this up with a coherent interface for each outside connection
 	if i > 0 {
-		//TODO: handle error
-		li, err = NewListener(udp2ip(addr).String(), int(addr.Port), i > 0)
-		if err != nil {
-			l.WithError(err).Error("failed to make a new udp listener")
-		}
+		li = f.writers[i]
 	} else {
 		li = f.outside
 	}

-	li.ListenOut(f)
+	lhh := f.lightHouse.NewRequestHandler()
+	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	li.ListenOut(f.readOutsidePackets, lhh.HandleRequest, conntrackCache, i)
 }

-func (f *Interface) listenIn(i int) {
+func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
+	runtime.LockOSThread()
+
 	packet := make([]byte, mtu)
 	out := make([]byte, mtu)
-	fwPacket := &FirewallPacket{}
+	fwPacket := &firewall.Packet{}
 	nb := make([]byte, 12, 12)

+	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+
 	for {
-		n, err := f.inside.Read(packet)
+		n, err := reader.Read(packet)
 		if err != nil {
-			l.WithError(err).Error("Error while reading outbound packet")
+			f.l.WithError(err).Error("Error while reading outbound packet")
 			// This only seems to happen when something fatal happens to the fd, so exit.
 			os.Exit(2)
 		}

-		f.consumeInsidePacket(packet[:n], fwPacket, nb, out)
+		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get(f.l))
 	}
 }

-func (f *Interface) RegisterConfigChangeCallbacks(c *Config) {
+func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
 	c.RegisterReloadCallback(f.reloadCA)
 	c.RegisterReloadCallback(f.reloadCertKey)
 	c.RegisterReloadCallback(f.reloadFirewall)
-	c.RegisterReloadCallback(f.outside.reloadConfig)
+	for _, udpConn := range f.writers {
+		c.RegisterReloadCallback(udpConn.ReloadConfig)
+	}
+	c.RegisterReloadCallback(f.reloadPSKs)
 }

-func (f *Interface) reloadCA(c *Config) {
+func (f *Interface) reloadCA(c *config.C) {
 	// reload and check regardless
 	// todo: need mutex?
-	newCAs, err := loadCAFromConfig(c)
+	newCAs, err := loadCAFromConfig(f.l, c)
 	if err != nil {
-		l.WithError(err).Error("Could not refresh trusted CA certificates")
+		f.l.WithError(err).Error("Could not refresh trusted CA certificates")
 		return
 	}

-	trustedCAs = newCAs
-	l.WithField("fingerprints", trustedCAs.GetFingerprints()).Info("Trusted CA certificates refreshed")
+	f.caPool = newCAs
+	f.l.WithField("fingerprints", f.caPool.GetFingerprints()).Info("Trusted CA certificates refreshed")
 }

-func (f *Interface) reloadCertKey(c *Config) {
+func (f *Interface) reloadCertKey(c *config.C) {
 	// reload and check in all cases
 	cs, err := NewCertStateFromConfig(c)
 	if err != nil {
-		l.WithError(err).Error("Could not refresh client cert")
+		f.l.WithError(err).Error("Could not refresh client cert")
 		return
 	}

@@ -224,54 +267,80 @@ func (f *Interface) reloadCertKey(c *Config) {
 	oldIPs := f.certState.certificate.Details.Ips
 	newIPs := cs.certificate.Details.Ips
 	if len(oldIPs) > 0 && len(newIPs) > 0 && oldIPs[0].String() != newIPs[0].String() {
-		l.WithField("new_ip", newIPs[0]).WithField("old_ip", oldIPs[0]).Error("IP in new cert was different from old")
+		f.l.WithField("new_ip", newIPs[0]).WithField("old_ip", oldIPs[0]).Error("IP in new cert was different from old")
 		return
 	}

 	f.certState = cs
-	l.WithField("cert", cs.certificate).Info("Client cert refreshed from disk")
+	f.l.WithField("cert", cs.certificate).Info("Client cert refreshed from disk")
 }

-func (f *Interface) reloadFirewall(c *Config) {
+func (f *Interface) reloadFirewall(c *config.C) {
 	//TODO: need to trigger/detect if the certificate changed too
 	if c.HasChanged("firewall") == false {
-		l.Debug("No firewall config change detected")
+		f.l.Debug("No firewall config change detected")
 		return
 	}

-	fw, err := NewFirewallFromConfig(f.certState.certificate, c)
+	fw, err := NewFirewallFromConfig(f.l, f.certState.certificate, c)
 	if err != nil {
-		l.WithError(err).Error("Error while creating firewall during reload")
+		f.l.WithError(err).Error("Error while creating firewall during reload")
 		return
 	}

 	oldFw := f.firewall
+	conntrack := oldFw.Conntrack
+	conntrack.Lock()
+	defer conntrack.Unlock()
+
+	fw.rulesVersion = oldFw.rulesVersion + 1
+	// If rulesVersion is back to zero, we have wrapped all the way around. Be
+	// safe and just reset conntrack in this case.
+	if fw.rulesVersion == 0 {
+		f.l.WithField("firewallHash", fw.GetRuleHash()).
+			WithField("oldFirewallHash", oldFw.GetRuleHash()).
+			WithField("rulesVersion", fw.rulesVersion).
+			Warn("firewall rulesVersion has overflowed, resetting conntrack")
+	} else {
+		fw.Conntrack = conntrack
+	}
+
 	f.firewall = fw

 	oldFw.Destroy()
-	l.WithField("firewallHash", fw.GetRuleHash()).
+	f.l.WithField("firewallHash", fw.GetRuleHash()).
 		WithField("oldFirewallHash", oldFw.GetRuleHash()).
+		WithField("rulesVersion", fw.rulesVersion).
 		Info("New firewall has been installed")
 }

-func (f *Interface) emitStats(i time.Duration) {
-	ticker := time.NewTicker(i)
-	for range ticker.C {
-		f.firewall.EmitStats()
-		f.handshakeManager.EmitStats()
-	}
+func (f *Interface) reloadPSKs(c *config.C) {
+	psk, err := NewPskFromConfig(c, f.myVpnIp)
+	if err != nil {
+		f.l.WithError(err).Error("Error while reloading PSKs")
+		return
 	}

-func sha256KdfFromString(secret string) ([]byte, error) {
-	// Use KDF to make this useful
-	mac := []byte(secret)
-	hmacKey := make([]byte, sha256.BlockSize)
-	hash := sha256.New
-	hkdfer := hkdf.New(hash, []byte(mac), nil, nil)
-	n, err := io.ReadFull(hkdfer, hmacKey)
-	if n != len(hmacKey) || err != nil {
-		l.Errorln("KDF Failed!")
-		return nil, fmt.Errorf("%s", err)
+	atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&f.psk)), unsafe.Pointer(psk))
+
+	f.l.WithField("pskMode", psk.mode).WithField("keysLen", len(psk.Cache)).
+		Info("New psks are in use")
+}
+
+func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
+	ticker := time.NewTicker(i)
+	defer ticker.Stop()
+
+	udpStats := udp.NewUDPStatsEmitter(f.writers)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			f.firewall.EmitStats()
+			f.handshakeManager.EmitStats()
+			udpStats()
+		}
 	}
-	return hmacKey, nil
 }
--- a/iputil/util.go
+++ b/iputil/util.go
@@ -0,0 +1,66 @@
+package iputil
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net"
+)
+
+type VpnIp uint32
+
+const maxIPv4StringLen = len("255.255.255.255")
+
+func (ip VpnIp) String() string {
+	b := make([]byte, maxIPv4StringLen)
+
+	n := ubtoa(b, 0, byte(ip>>24))
+	b[n] = '.'
+	n++
+
+	n += ubtoa(b, n, byte(ip>>16&255))
+	b[n] = '.'
+	n++
+
+	n += ubtoa(b, n, byte(ip>>8&255))
+	b[n] = '.'
+	n++
+
+	n += ubtoa(b, n, byte(ip&255))
+	return string(b[:n])
+}
+
+func (ip VpnIp) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf("\"%s\"", ip.String())), nil
+}
+
+func (ip VpnIp) ToIP() net.IP {
+	nip := make(net.IP, 4)
+	binary.BigEndian.PutUint32(nip, uint32(ip))
+	return nip
+}
+
+func Ip2VpnIp(ip []byte) VpnIp {
+	if len(ip) == 16 {
+		return VpnIp(binary.BigEndian.Uint32(ip[12:16]))
+	}
+	return VpnIp(binary.BigEndian.Uint32(ip))
+}
+
+// ubtoa encodes the string form of the integer v to dst[start:] and
+// returns the number of bytes written to dst. The caller must ensure
+// that dst has sufficient length.
+func ubtoa(dst []byte, start int, v byte) int {
+	if v < 10 {
+		dst[start] = v + '0'
+		return 1
+	} else if v < 100 {
+		dst[start+1] = v%10 + '0'
+		dst[start] = v/10 + '0'
+		return 2
+	}
+
+	dst[start+2] = v%10 + '0'
+	dst[start+1] = (v/10)%10 + '0'
+	dst[start] = v/100 + '0'
+	return 3
+}
--- a/iputil/util_test.go
+++ b/iputil/util_test.go
@@ -0,0 +1,17 @@
+package iputil
+
+import (
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestVpnIp_String(t *testing.T) {
+	assert.Equal(t, "255.255.255.255", Ip2VpnIp(net.ParseIP("255.255.255.255")).String())
+	assert.Equal(t, "1.255.255.255", Ip2VpnIp(net.ParseIP("1.255.255.255")).String())
+	assert.Equal(t, "1.1.255.255", Ip2VpnIp(net.ParseIP("1.1.255.255")).String())
+	assert.Equal(t, "1.1.1.255", Ip2VpnIp(net.ParseIP("1.1.1.255")).String())
+	assert.Equal(t, "1.1.1.1", Ip2VpnIp(net.ParseIP("1.1.1.1")).String())
+	assert.Equal(t, "0.0.0.0", Ip2VpnIp(net.ParseIP("0.0.0.0")).String())
+}
--- a/lighthouse.go
+++ b/lighthouse.go
@@ -1,91 +1,125 @@
 package nebula

 import (
+	"context"
+	"encoding/binary"
+	"errors"
 	"fmt"
 	"net"
 	"sync"
 	"time"

 	"github.com/golang/protobuf/proto"
-	"github.com/slackhq/nebula/cert"
+	"github.com/rcrowley/go-metrics"
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
 )

+//TODO: if a lighthouse doesn't have an answer, clients AGGRESSIVELY REQUERY.. why? handshake manager and/or getOrHandshake?
+//TODO: nodes are roaming lighthouses, this is bad. How are they learning?
+
+var ErrHostNotKnown = errors.New("host not known")
+
 type LightHouse struct {
+	//TODO: We need a timer wheel to kick out vpnIps that haven't reported in a long time
 	sync.RWMutex //Because we concurrently read and write to our maps
 	amLighthouse bool
-	myIp         uint32
-	punchConn    *udpConn
+	myVpnIp      iputil.VpnIp
+	myVpnZeros   iputil.VpnIp
+	punchConn    *udp.Conn

 	// Local cache of answers from light houses
-	addrMap map[uint32][]udpAddr
+	// map of vpn Ip to answers
+	addrMap map[iputil.VpnIp]*RemoteList
+
+	// filters remote addresses allowed for each host
+	// - When we are a lighthouse, this filters what addresses we store and
+	// respond with.
+	// - When we are not a lighthouse, this filters which addresses we accept
+	// from lighthouses.
+	remoteAllowList *RemoteAllowList
+
+	// filters local addresses that we advertise to lighthouses
+	localAllowList *LocalAllowList
+
+	// used to trigger the HandshakeManager when we receive HostQueryReply
+	handshakeTrigger chan<- iputil.VpnIp

 	// staticList exists to avoid having a bool in each addrMap entry
 	// since static should be rare
-	staticList  map[uint32]struct{}
-	lighthouses map[uint32]struct{}
+	staticList  map[iputil.VpnIp]struct{}
+	lighthouses map[iputil.VpnIp]struct{}
 	interval    int
-	nebulaPort  int
+	nebulaPort  uint32 // 32 bits because protobuf does not have a uint16
 	punchBack   bool
+	punchDelay  time.Duration
+
+	metrics           *MessageMetrics
+	metricHolepunchTx metrics.Counter
+	l                 *logrus.Logger
 }

-type EncWriter interface {
-	SendMessageToVpnIp(t NebulaMessageType, st NebulaMessageSubType, vpnIp uint32, p, nb, out []byte)
-	SendMessageToAll(t NebulaMessageType, st NebulaMessageSubType, vpnIp uint32, p, nb, out []byte)
-}
-
-func NewLightHouse(amLighthouse bool, myIp uint32, ips []string, interval int, nebulaPort int, pc *udpConn, punchBack bool) *LightHouse {
+func NewLightHouse(l *logrus.Logger, amLighthouse bool, myVpnIpNet *net.IPNet, ips []iputil.VpnIp, interval int, nebulaPort uint32, pc *udp.Conn, punchBack bool, punchDelay time.Duration, metricsEnabled bool) *LightHouse {
+	ones, _ := myVpnIpNet.Mask.Size()
 	h := LightHouse{
 		amLighthouse: amLighthouse,
-		myIp:         myIp,
-		addrMap:      make(map[uint32][]udpAddr),
+		myVpnIp:      iputil.Ip2VpnIp(myVpnIpNet.IP),
+		myVpnZeros:   iputil.VpnIp(32 - ones),
+		addrMap:      make(map[iputil.VpnIp]*RemoteList),
 		nebulaPort:   nebulaPort,
-		lighthouses:  make(map[uint32]struct{}),
-		staticList:   make(map[uint32]struct{}),
+		lighthouses:  make(map[iputil.VpnIp]struct{}),
+		staticList:   make(map[iputil.VpnIp]struct{}),
 		interval:     interval,
 		punchConn:    pc,
 		punchBack:    punchBack,
+		punchDelay:   punchDelay,
+		l:            l,
 	}

-	for _, rIp := range ips {
-		h.lighthouses[ip2int(net.ParseIP(rIp))] = struct{}{}
+	if metricsEnabled {
+		h.metrics = newLighthouseMetrics()
+
+		h.metricHolepunchTx = metrics.GetOrRegisterCounter("messages.tx.holepunch", nil)
+	} else {
+		h.metricHolepunchTx = metrics.NilCounter{}
+	}
+
+	for _, ip := range ips {
+		h.lighthouses[ip] = struct{}{}
 	}

 	return &h
 }

-func (lh *LightHouse) Query(ip uint32, f EncWriter) ([]udpAddr, error) {
+func (lh *LightHouse) SetRemoteAllowList(allowList *RemoteAllowList) {
+	lh.Lock()
+	defer lh.Unlock()
+
+	lh.remoteAllowList = allowList
+}
+
+func (lh *LightHouse) SetLocalAllowList(allowList *LocalAllowList) {
+	lh.Lock()
+	defer lh.Unlock()
+
+	lh.localAllowList = allowList
+}
+
+func (lh *LightHouse) ValidateLHStaticEntries() error {
+	for lhIP, _ := range lh.lighthouses {
+		if _, ok := lh.staticList[lhIP]; !ok {
+			return fmt.Errorf("Lighthouse %s does not have a static_host_map entry", lhIP)
+		}
+	}
+	return nil
+}
+
+func (lh *LightHouse) Query(ip iputil.VpnIp, f udp.EncWriter) *RemoteList {
 	if !lh.IsLighthouseIP(ip) {
 		lh.QueryServer(ip, f)
 	}
-	lh.RLock()
-	if v, ok := lh.addrMap[ip]; ok {
-		lh.RUnlock()
-		return v, nil
-	}
-	lh.RUnlock()
-	return nil, fmt.Errorf("host %s not known, queries sent to lighthouses", IntIp(ip))
-}
-
-// This is asynchronous so no reply should be expected
-func (lh *LightHouse) QueryServer(ip uint32, f EncWriter) {
-	if !lh.amLighthouse {
-		// Send a query to the lighthouses and hope for the best next time
-		query, err := proto.Marshal(NewLhQueryByInt(ip))
-		if err != nil {
-			l.WithError(err).WithField("vpnIp", IntIp(ip)).Error("Failed to marshal lighthouse query payload")
-			return
-		}
-
-		nb := make([]byte, 12, 12)
-		out := make([]byte, mtu)
-		for n := range lh.lighthouses {
-			f.SendMessageToVpnIp(lightHouse, 0, n, query, nb, out)
-		}
-	}
-}
-
-// Query our local lighthouse cached results
-func (lh *LightHouse) QueryCache(ip uint32) []udpAddr {
 	lh.RLock()
 	if v, ok := lh.addrMap[ip]; ok {
 		lh.RUnlock()
@@ -95,274 +129,525 @@ func (lh *LightHouse) QueryCache(ip uint32) []udpAddr {
 	return nil
 }

-func (lh *LightHouse) DeleteVpnIP(vpnIP uint32) {
+// This is asynchronous so no reply should be expected
+func (lh *LightHouse) QueryServer(ip iputil.VpnIp, f udp.EncWriter) {
+	if lh.amLighthouse {
+		return
+	}
+
+	if lh.IsLighthouseIP(ip) {
+		return
+	}
+
+	// Send a query to the lighthouses and hope for the best next time
+	query, err := proto.Marshal(NewLhQueryByInt(ip))
+	if err != nil {
+		lh.l.WithError(err).WithField("vpnIp", ip).Error("Failed to marshal lighthouse query payload")
+		return
+	}
+
+	lh.metricTx(NebulaMeta_HostQuery, int64(len(lh.lighthouses)))
+	nb := make([]byte, 12, 12)
+	out := make([]byte, mtu)
+	for n := range lh.lighthouses {
+		f.SendMessageToVpnIp(header.LightHouse, 0, n, query, nb, out)
+	}
+}
+
+func (lh *LightHouse) QueryCache(ip iputil.VpnIp) *RemoteList {
+	lh.RLock()
+	if v, ok := lh.addrMap[ip]; ok {
+		lh.RUnlock()
+		return v
+	}
+	lh.RUnlock()
+
+	lh.Lock()
+	defer lh.Unlock()
+	// Add an entry if we don't already have one
+	return lh.unlockedGetRemoteList(ip)
+}
+
+// queryAndPrepMessage is a lock helper on RemoteList, assisting the caller to build a lighthouse message containing
+// details from the remote list. It looks for a hit in the addrMap and a hit in the RemoteList under the owner vpnIp
+// If one is found then f() is called with proper locking, f() must return result of n.MarshalTo()
+func (lh *LightHouse) queryAndPrepMessage(vpnIp iputil.VpnIp, f func(*cache) (int, error)) (bool, int, error) {
+	lh.RLock()
+	// Do we have an entry in the main cache?
+	if v, ok := lh.addrMap[vpnIp]; ok {
+		// Swap lh lock for remote list lock
+		v.RLock()
+		defer v.RUnlock()
+
+		lh.RUnlock()
+
+		// vpnIp should also be the owner here since we are a lighthouse.
+		c := v.cache[vpnIp]
+		// Make sure we have
+		if c != nil {
+			n, err := f(c)
+			return true, n, err
+		}
+		return false, 0, nil
+	}
+	lh.RUnlock()
+	return false, 0, nil
+}
+
+func (lh *LightHouse) DeleteVpnIp(vpnIp iputil.VpnIp) {
 	// First we check the static mapping
 	// and do nothing if it is there
-	if _, ok := lh.staticList[vpnIP]; ok {
+	if _, ok := lh.staticList[vpnIp]; ok {
 		return
 	}
 	lh.Lock()
 	//l.Debugln(lh.addrMap)
-	delete(lh.addrMap, vpnIP)
-	l.Debugf("deleting %s from lighthouse.", IntIp(vpnIP))
+	delete(lh.addrMap, vpnIp)
+
+	if lh.l.Level >= logrus.DebugLevel {
+		lh.l.Debugf("deleting %s from lighthouse.", vpnIp)
+	}
+
 	lh.Unlock()
 }

-func (lh *LightHouse) AddRemote(vpnIP uint32, toIp *udpAddr, static bool) {
-	// First we check if the sender thinks this is a static entry
-	// and do nothing if it is not, but should be considered static
-	if static == false {
-		if _, ok := lh.staticList[vpnIP]; ok {
-			return
-		}
-	}
-
+// AddStaticRemote adds a static host entry for vpnIp as ourselves as the owner
+// We are the owner because we don't want a lighthouse server to advertise for static hosts it was configured with
+// And we don't want a lighthouse query reply to interfere with our learned cache if we are a client
+func (lh *LightHouse) AddStaticRemote(vpnIp iputil.VpnIp, toAddr *udp.Addr) {
 	lh.Lock()
-	for _, v := range lh.addrMap[vpnIP] {
-		if v.Equals(toIp) {
+	am := lh.unlockedGetRemoteList(vpnIp)
+	am.Lock()
+	defer am.Unlock()
 	lh.Unlock()
+
+	if ipv4 := toAddr.IP.To4(); ipv4 != nil {
+		to := NewIp4AndPort(ipv4, uint32(toAddr.Port))
+		if !lh.unlockedShouldAddV4(vpnIp, to) {
 			return
 		}
+		am.unlockedPrependV4(lh.myVpnIp, to)
+
+	} else {
+		to := NewIp6AndPort(toAddr.IP, uint32(toAddr.Port))
+		if !lh.unlockedShouldAddV6(vpnIp, to) {
+			return
 		}
-	//l.Debugf("Adding reply of %s as %s\n", IntIp(vpnIP), toIp)
-	if static {
-		lh.staticList[vpnIP] = struct{}{}
-	}
-	lh.addrMap[vpnIP] = append(lh.addrMap[vpnIP], *toIp)
-	lh.Unlock()
+		am.unlockedPrependV6(lh.myVpnIp, to)
 	}

-func (lh *LightHouse) AddRemoteAndReset(vpnIP uint32, toIp *udpAddr) {
-	if lh.amLighthouse {
-		lh.DeleteVpnIP(vpnIP)
-		lh.AddRemote(vpnIP, toIp, false)
+	// Mark it as static
+	lh.staticList[vpnIp] = struct{}{}
 }

+// unlockedGetRemoteList assumes you have the lh lock
+func (lh *LightHouse) unlockedGetRemoteList(vpnIp iputil.VpnIp) *RemoteList {
+	am, ok := lh.addrMap[vpnIp]
+	if !ok {
+		am = NewRemoteList()
+		lh.addrMap[vpnIp] = am
+	}
+	return am
 }

-func (lh *LightHouse) IsLighthouseIP(vpnIP uint32) bool {
-	if _, ok := lh.lighthouses[vpnIP]; ok {
+// unlockedShouldAddV4 checks if to is allowed by our allow list
+func (lh *LightHouse) unlockedShouldAddV4(vpnIp iputil.VpnIp, to *Ip4AndPort) bool {
+	allow := lh.remoteAllowList.AllowIpV4(vpnIp, iputil.VpnIp(to.Ip))
+	if lh.l.Level >= logrus.TraceLevel {
+		lh.l.WithField("remoteIp", vpnIp).WithField("allow", allow).Trace("remoteAllowList.Allow")
+	}
+
+	if !allow || ipMaskContains(lh.myVpnIp, lh.myVpnZeros, iputil.VpnIp(to.Ip)) {
+		return false
+	}
+
+	return true
+}
+
+// unlockedShouldAddV6 checks if to is allowed by our allow list
+func (lh *LightHouse) unlockedShouldAddV6(vpnIp iputil.VpnIp, to *Ip6AndPort) bool {
+	allow := lh.remoteAllowList.AllowIpV6(vpnIp, to.Hi, to.Lo)
+	if lh.l.Level >= logrus.TraceLevel {
+		lh.l.WithField("remoteIp", lhIp6ToIp(to)).WithField("allow", allow).Trace("remoteAllowList.Allow")
+	}
+
+	// We don't check our vpn network here because nebula does not support ipv6 on the inside
+	if !allow {
+		return false
+	}
+
+	return true
+}
+
+func lhIp6ToIp(v *Ip6AndPort) net.IP {
+	ip := make(net.IP, 16)
+	binary.BigEndian.PutUint64(ip[:8], v.Hi)
+	binary.BigEndian.PutUint64(ip[8:], v.Lo)
+	return ip
+}
+
+func (lh *LightHouse) IsLighthouseIP(vpnIp iputil.VpnIp) bool {
+	if _, ok := lh.lighthouses[vpnIp]; ok {
 		return true
 	}
 	return false
 }

-// Quick generators for protobuf
-
-func NewLhQueryByIpString(VpnIp string) *NebulaMeta {
-	return NewLhQueryByInt(ip2int(net.ParseIP(VpnIp)))
-}
-
-func NewLhQueryByInt(VpnIp uint32) *NebulaMeta {
+func NewLhQueryByInt(VpnIp iputil.VpnIp) *NebulaMeta {
 	return &NebulaMeta{
 		Type: NebulaMeta_HostQuery,
 		Details: &NebulaMetaDetails{
-			VpnIp: VpnIp,
+			VpnIp: uint32(VpnIp),
 		},
 	}
 }

-func NewLhWhoami() *NebulaMeta {
-	return &NebulaMeta{
-		Type:    NebulaMeta_HostWhoami,
-		Details: &NebulaMetaDetails{},
+func NewIp4AndPort(ip net.IP, port uint32) *Ip4AndPort {
+	ipp := Ip4AndPort{Port: port}
+	ipp.Ip = uint32(iputil.Ip2VpnIp(ip))
+	return &ipp
+}
+
+func NewIp6AndPort(ip net.IP, port uint32) *Ip6AndPort {
+	return &Ip6AndPort{
+		Hi:   binary.BigEndian.Uint64(ip[:8]),
+		Lo:   binary.BigEndian.Uint64(ip[8:]),
+		Port: port,
 	}
 }

-// End Quick generators for protobuf
-
-func NewIpAndPortFromUDPAddr(addr udpAddr) *IpAndPort {
-	return &IpAndPort{Ip: udp2ipInt(&addr), Port: uint32(addr.Port)}
+func NewUDPAddrFromLH4(ipp *Ip4AndPort) *udp.Addr {
+	ip := ipp.Ip
+	return udp.NewAddr(
+		net.IPv4(byte(ip&0xff000000>>24), byte(ip&0x00ff0000>>16), byte(ip&0x0000ff00>>8), byte(ip&0x000000ff)),
+		uint16(ipp.Port),
+	)
 }

-func NewIpAndPortsFromNetIps(ips []udpAddr) *[]*IpAndPort {
-	var iap []*IpAndPort
-	for _, e := range ips {
-		// Only add IPs that aren't my VPN/tun IP
-		iap = append(iap, NewIpAndPortFromUDPAddr(e))
-	}
-	return &iap
+func NewUDPAddrFromLH6(ipp *Ip6AndPort) *udp.Addr {
+	return udp.NewAddr(lhIp6ToIp(ipp), uint16(ipp.Port))
 }

-func (lh *LightHouse) LhUpdateWorker(f EncWriter) {
-	if lh.amLighthouse {
+func (lh *LightHouse) LhUpdateWorker(ctx context.Context, f udp.EncWriter) {
+	if lh.amLighthouse || lh.interval == 0 {
 		return
 	}

-	for {
-		ipp := []*IpAndPort{}
+	clockSource := time.NewTicker(time.Second * time.Duration(lh.interval))
+	defer clockSource.Stop()
+
+	for {
+		lh.SendUpdate(f)
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-clockSource.C:
+			continue
+		}
+	}
+}
+
+func (lh *LightHouse) SendUpdate(f udp.EncWriter) {
+	var v4 []*Ip4AndPort
+	var v6 []*Ip6AndPort
+
+	for _, e := range *localIps(lh.l, lh.localAllowList) {
+		if ip4 := e.To4(); ip4 != nil && ipMaskContains(lh.myVpnIp, lh.myVpnZeros, iputil.Ip2VpnIp(ip4)) {
+			continue
+		}

-		for _, e := range *localIps() {
 		// Only add IPs that aren't my VPN/tun IP
-			if ip2int(e) != lh.myIp {
-				ipp = append(ipp, &IpAndPort{Ip: ip2int(e), Port: uint32(lh.nebulaPort)})
-				//fmt.Println(e)
+		if ip := e.To4(); ip != nil {
+			v4 = append(v4, NewIp4AndPort(e, lh.nebulaPort))
+		} else {
+			v6 = append(v6, NewIp6AndPort(e, lh.nebulaPort))
 		}
 	}
 	m := &NebulaMeta{
 		Type: NebulaMeta_HostUpdateNotification,
 		Details: &NebulaMetaDetails{
-				VpnIp:      lh.myIp,
-				IpAndPorts: ipp,
+			VpnIp:       uint32(lh.myVpnIp),
+			Ip4AndPorts: v4,
+			Ip6AndPorts: v6,
 		},
 	}

+	lh.metricTx(NebulaMeta_HostUpdateNotification, int64(len(lh.lighthouses)))
 	nb := make([]byte, 12, 12)
 	out := make([]byte, mtu)
-		for vpnIp := range lh.lighthouses {
+
 	mm, err := proto.Marshal(m)
 	if err != nil {
-				l.Debugf("Invalid marshal to update")
+		lh.l.WithError(err).Error("Error while marshaling for lighthouse update")
+		return
 	}
-			//l.Error("LIGHTHOUSE PACKET SEND", mm)
-			f.SendMessageToVpnIp(lightHouse, 0, vpnIp, mm, nb, out)

-		}
-		time.Sleep(time.Second * time.Duration(lh.interval))
+	for vpnIp := range lh.lighthouses {
+		f.SendMessageToVpnIp(header.LightHouse, 0, vpnIp, mm, nb, out)
 	}
 }

-func (lh *LightHouse) HandleRequest(rAddr *udpAddr, vpnIp uint32, p []byte, c *cert.NebulaCertificate, f EncWriter) {
-	n := &NebulaMeta{}
-	err := proto.Unmarshal(p, n)
+type LightHouseHandler struct {
+	lh   *LightHouse
+	nb   []byte
+	out  []byte
+	pb   []byte
+	meta *NebulaMeta
+	l    *logrus.Logger
+}
+
+func (lh *LightHouse) NewRequestHandler() *LightHouseHandler {
+	lhh := &LightHouseHandler{
+		lh:  lh,
+		nb:  make([]byte, 12, 12),
+		out: make([]byte, mtu),
+		l:   lh.l,
+		pb:  make([]byte, mtu),
+
+		meta: &NebulaMeta{
+			Details: &NebulaMetaDetails{},
+		},
+	}
+
+	return lhh
+}
+
+func (lh *LightHouse) metricRx(t NebulaMeta_MessageType, i int64) {
+	lh.metrics.Rx(header.MessageType(t), 0, i)
+}
+
+func (lh *LightHouse) metricTx(t NebulaMeta_MessageType, i int64) {
+	lh.metrics.Tx(header.MessageType(t), 0, i)
+}
+
+// This method is similar to Reset(), but it re-uses the pointer structs
+// so that we don't have to re-allocate them
+func (lhh *LightHouseHandler) resetMeta() *NebulaMeta {
+	details := lhh.meta.Details
+	lhh.meta.Reset()
+
+	// Keep the array memory around
+	details.Ip4AndPorts = details.Ip4AndPorts[:0]
+	details.Ip6AndPorts = details.Ip6AndPorts[:0]
+	lhh.meta.Details = details
+
+	return lhh.meta
+}
+
+func (lhh *LightHouseHandler) HandleRequest(rAddr *udp.Addr, vpnIp iputil.VpnIp, p []byte, w udp.EncWriter) {
+	n := lhh.resetMeta()
+	err := n.Unmarshal(p)
 	if err != nil {
-		l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).WithField("udpAddr", rAddr).
+		lhh.l.WithError(err).WithField("vpnIp", vpnIp).WithField("udpAddr", rAddr).
 			Error("Failed to unmarshal lighthouse packet")
 		//TODO: send recv_error?
 		return
 	}

 	if n.Details == nil {
-		l.WithField("vpnIp", IntIp(vpnIp)).WithField("udpAddr", rAddr).
+		lhh.l.WithField("vpnIp", vpnIp).WithField("udpAddr", rAddr).
 			Error("Invalid lighthouse update")
 		//TODO: send recv_error?
 		return
 	}

+	lhh.lh.metricRx(n.Type, 1)
+
 	switch n.Type {
 	case NebulaMeta_HostQuery:
-		// Exit if we don't answer queries
-		if !lh.amLighthouse {
-			l.Debugln("I don't answer queries, but received from: ", rAddr)
-			return
-		}
-
-		//l.Debugln("Got Query")
-		ips, err := lh.Query(n.Details.VpnIp, f)
-		if err != nil {
-			//l.Debugf("Can't answer query %s from %s because error: %s", IntIp(n.Details.VpnIp), rAddr, err)
-			return
-		} else {
-			iap := NewIpAndPortsFromNetIps(ips)
-			answer := &NebulaMeta{
-				Type: NebulaMeta_HostQueryReply,
-				Details: &NebulaMetaDetails{
-					VpnIp:      n.Details.VpnIp,
-					IpAndPorts: *iap,
-				},
-			}
-			reply, err := proto.Marshal(answer)
-			if err != nil {
-				l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).Error("Failed to marshal lighthouse host query reply")
-				return
-			}
-			f.SendMessageToVpnIp(lightHouse, 0, vpnIp, reply, make([]byte, 12, 12), make([]byte, mtu))
-
-			// This signals the other side to punch some zero byte udp packets
-			ips, err = lh.Query(vpnIp, f)
-			if err != nil {
-				l.WithField("vpnIp", IntIp(vpnIp)).Debugln("Can't notify host to punch")
-				return
-			} else {
-				//l.Debugln("Notify host to punch", iap)
-				iap = NewIpAndPortsFromNetIps(ips)
-				answer = &NebulaMeta{
-					Type: NebulaMeta_HostPunchNotification,
-					Details: &NebulaMetaDetails{
-						VpnIp:      vpnIp,
-						IpAndPorts: *iap,
-					},
-				}
-				reply, _ := proto.Marshal(answer)
-				f.SendMessageToVpnIp(lightHouse, 0, n.Details.VpnIp, reply, make([]byte, 12, 12), make([]byte, mtu))
-			}
-			//fmt.Println(reply, remoteaddr)
-		}
+		lhh.handleHostQuery(n, vpnIp, rAddr, w)

 	case NebulaMeta_HostQueryReply:
-		if !lh.IsLighthouseIP(vpnIp) {
-			return
-		}
-		for _, a := range n.Details.IpAndPorts {
-			//first := n.Details.IpAndPorts[0]
-			ans := NewUDPAddr(a.Ip, uint16(a.Port))
-			lh.AddRemote(n.Details.VpnIp, ans, false)
-		}
+		lhh.handleHostQueryReply(n, vpnIp)

 	case NebulaMeta_HostUpdateNotification:
-		//Simple check that the host sent this not someone else
-		if n.Details.VpnIp != vpnIp {
-			l.WithField("vpnIp", IntIp(vpnIp)).WithField("answer", IntIp(n.Details.VpnIp)).Debugln("Host sent invalid update")
-			return
-		}
-		for _, a := range n.Details.IpAndPorts {
-			ans := NewUDPAddr(a.Ip, uint16(a.Port))
-			lh.AddRemote(n.Details.VpnIp, ans, false)
-		}
+		lhh.handleHostUpdateNotification(n, vpnIp)
+
 	case NebulaMeta_HostMovedNotification:
 	case NebulaMeta_HostPunchNotification:
-		if !lh.IsLighthouseIP(vpnIp) {
+		lhh.handleHostPunchNotification(n, vpnIp, w)
+	}
+}
+
+func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, vpnIp iputil.VpnIp, addr *udp.Addr, w udp.EncWriter) {
+	// Exit if we don't answer queries
+	if !lhh.lh.amLighthouse {
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.Debugln("I don't answer queries, but received from: ", addr)
+		}
+		return
+	}
+
+	//TODO: we can DRY this further
+	reqVpnIp := n.Details.VpnIp
+	//TODO: Maybe instead of marshalling into n we marshal into a new `r` to not nuke our current request data
+	found, ln, err := lhh.lh.queryAndPrepMessage(iputil.VpnIp(n.Details.VpnIp), func(c *cache) (int, error) {
+		n = lhh.resetMeta()
+		n.Type = NebulaMeta_HostQueryReply
+		n.Details.VpnIp = reqVpnIp
+
+		lhh.coalesceAnswers(c, n)
+
+		return n.MarshalTo(lhh.pb)
+	})
+
+	if !found {
+		return
+	}
+
+	if err != nil {
+		lhh.l.WithError(err).WithField("vpnIp", vpnIp).Error("Failed to marshal lighthouse host query reply")
+		return
+	}
+
+	lhh.lh.metricTx(NebulaMeta_HostQueryReply, 1)
+	w.SendMessageToVpnIp(header.LightHouse, 0, vpnIp, lhh.pb[:ln], lhh.nb, lhh.out[:0])
+
+	// This signals the other side to punch some zero byte udp packets
+	found, ln, err = lhh.lh.queryAndPrepMessage(vpnIp, func(c *cache) (int, error) {
+		n = lhh.resetMeta()
+		n.Type = NebulaMeta_HostPunchNotification
+		n.Details.VpnIp = uint32(vpnIp)
+
+		lhh.coalesceAnswers(c, n)
+
+		return n.MarshalTo(lhh.pb)
+	})
+
+	if !found {
+		return
+	}
+
+	if err != nil {
+		lhh.l.WithError(err).WithField("vpnIp", vpnIp).Error("Failed to marshal lighthouse host was queried for")
+		return
+	}
+
+	lhh.lh.metricTx(NebulaMeta_HostPunchNotification, 1)
+	w.SendMessageToVpnIp(header.LightHouse, 0, iputil.VpnIp(reqVpnIp), lhh.pb[:ln], lhh.nb, lhh.out[:0])
+}
+
+func (lhh *LightHouseHandler) coalesceAnswers(c *cache, n *NebulaMeta) {
+	if c.v4 != nil {
+		if c.v4.learned != nil {
+			n.Details.Ip4AndPorts = append(n.Details.Ip4AndPorts, c.v4.learned)
+		}
+		if c.v4.reported != nil && len(c.v4.reported) > 0 {
+			n.Details.Ip4AndPorts = append(n.Details.Ip4AndPorts, c.v4.reported...)
+		}
+	}
+
+	if c.v6 != nil {
+		if c.v6.learned != nil {
+			n.Details.Ip6AndPorts = append(n.Details.Ip6AndPorts, c.v6.learned)
+		}
+		if c.v6.reported != nil && len(c.v6.reported) > 0 {
+			n.Details.Ip6AndPorts = append(n.Details.Ip6AndPorts, c.v6.reported...)
+		}
+	}
+}
+
+func (lhh *LightHouseHandler) handleHostQueryReply(n *NebulaMeta, vpnIp iputil.VpnIp) {
+	if !lhh.lh.IsLighthouseIP(vpnIp) {
+		return
+	}
+
+	lhh.lh.Lock()
+	am := lhh.lh.unlockedGetRemoteList(iputil.VpnIp(n.Details.VpnIp))
+	am.Lock()
+	lhh.lh.Unlock()
+
+	certVpnIp := iputil.VpnIp(n.Details.VpnIp)
+	am.unlockedSetV4(vpnIp, certVpnIp, n.Details.Ip4AndPorts, lhh.lh.unlockedShouldAddV4)
+	am.unlockedSetV6(vpnIp, certVpnIp, n.Details.Ip6AndPorts, lhh.lh.unlockedShouldAddV6)
+	am.Unlock()
+
+	// Non-blocking attempt to trigger, skip if it would block
+	select {
+	case lhh.lh.handshakeTrigger <- iputil.VpnIp(n.Details.VpnIp):
+	default:
+	}
+}
+
+func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, vpnIp iputil.VpnIp) {
+	if !lhh.lh.amLighthouse {
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.Debugln("I am not a lighthouse, do not take host updates: ", vpnIp)
+		}
+		return
+	}
+
+	//Simple check that the host sent this not someone else
+	if n.Details.VpnIp != uint32(vpnIp) {
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.WithField("vpnIp", vpnIp).WithField("answer", iputil.VpnIp(n.Details.VpnIp)).Debugln("Host sent invalid update")
+		}
+		return
+	}
+
+	lhh.lh.Lock()
+	am := lhh.lh.unlockedGetRemoteList(vpnIp)
+	am.Lock()
+	lhh.lh.Unlock()
+
+	certVpnIp := iputil.VpnIp(n.Details.VpnIp)
+	am.unlockedSetV4(vpnIp, certVpnIp, n.Details.Ip4AndPorts, lhh.lh.unlockedShouldAddV4)
+	am.unlockedSetV6(vpnIp, certVpnIp, n.Details.Ip6AndPorts, lhh.lh.unlockedShouldAddV6)
+	am.Unlock()
+}
+
+func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, vpnIp iputil.VpnIp, w udp.EncWriter) {
+	if !lhh.lh.IsLighthouseIP(vpnIp) {
 		return
 	}

 	empty := []byte{0}
-		for _, a := range n.Details.IpAndPorts {
-			vpnPeer := NewUDPAddr(a.Ip, uint16(a.Port))
-			go func() {
-				for i := 0; i < 5; i++ {
-					lh.punchConn.WriteTo(empty, vpnPeer)
-					time.Sleep(time.Second * 1)
+	punch := func(vpnPeer *udp.Addr) {
+		if vpnPeer == nil {
+			return
 		}

+		go func() {
+			time.Sleep(lhh.lh.punchDelay)
+			lhh.lh.metricHolepunchTx.Inc(1)
+			lhh.lh.punchConn.WriteTo(empty, vpnPeer)
 		}()
-			l.Debugf("Punching %s on %d for %s", IntIp(a.Ip), a.Port, IntIp(n.Details.VpnIp))
+
+		if lhh.l.Level >= logrus.DebugLevel {
+			//TODO: lacking the ip we are actually punching on, old: l.Debugf("Punching %s on %d for %s", IntIp(a.Ip), a.Port, IntIp(n.Details.VpnIp))
+			lhh.l.Debugf("Punching on %d for %s", vpnPeer.Port, iputil.VpnIp(n.Details.VpnIp))
 		}
+	}
+
+	for _, a := range n.Details.Ip4AndPorts {
+		punch(NewUDPAddrFromLH4(a))
+	}
+
+	for _, a := range n.Details.Ip6AndPorts {
+		punch(NewUDPAddrFromLH6(a))
+	}
+
 	// This sends a nebula test packet to the host trying to contact us. In the case
 	// of a double nat or other difficult scenario, this may help establish
 	// a tunnel.
-		if lh.punchBack {
+	if lhh.lh.punchBack {
+		queryVpnIp := iputil.VpnIp(n.Details.VpnIp)
 		go func() {
 			time.Sleep(time.Second * 5)
-				l.Debugf("Sending a nebula test packet to vpn ip %s", IntIp(n.Details.VpnIp))
-				f.SendMessageToVpnIp(test, testRequest, n.Details.VpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+			if lhh.l.Level >= logrus.DebugLevel {
+				lhh.l.Debugf("Sending a nebula test packet to vpn ip %s", queryVpnIp)
+			}
+			//NOTE: we have to allocate a new output buffer here since we are spawning a new goroutine
+			// for each punchBack packet. We should move this into a timerwheel or a single goroutine
+			// managed by a channel.
+			w.SendMessageToVpnIp(header.Test, header.TestRequest, queryVpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
 		}()
 	}
 }
-}

-/*
-func (f *Interface) sendPathCheck(ci *ConnectionState, endpoint *net.UDPAddr, counter int) {
-	c := ci.messageCounter
-    b := HeaderEncode(nil, Version, uint8(path_check), 0, ci.remoteIndex, c)
-	ci.messageCounter++
-
-	if ci.eKey != nil {
-		msg := ci.eKey.EncryptDanger(b, nil, []byte(strconv.Itoa(counter)), c)
-		//msg := ci.eKey.EncryptDanger(b, nil, []byte(fmt.Sprintf("%d", counter)), c)
-		f.outside.WriteTo(msg, endpoint)
-		l.Debugf("path_check sent, remote index: %d, pathCounter %d", ci.remoteIndex, counter)
+// ipMaskContains checks if testIp is contained by ip after applying a cidr
+// zeros is 32 - bits from net.IPMask.Size()
+func ipMaskContains(ip iputil.VpnIp, zeros iputil.VpnIp, testIp iputil.VpnIp) bool {
+	return (testIp^ip)>>zeros == 0
 }
-}
-
-func (f *Interface) sendPathCheckReply(ci *ConnectionState, endpoint *net.UDPAddr, counter []byte) {
-	c := ci.messageCounter
-    b := HeaderEncode(nil, Version, uint8(path_check_reply), 0, ci.remoteIndex, c)
-	ci.messageCounter++
-
-	if ci.eKey != nil {
-		msg := ci.eKey.EncryptDanger(b, nil, counter, c)
-		f.outside.WriteTo(msg, endpoint)
-		l.Debugln("path_check sent, remote index: ", ci.remoteIndex)
-	}
-}
-*/
--- a/lighthouse_test.go
+++ b/lighthouse_test.go
@@ -1,16 +1,32 @@
 package nebula

 import (
+	"fmt"
 	"net"
 	"testing"

-	proto "github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/proto"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/udp"
+	"github.com/slackhq/nebula/util"
 	"github.com/stretchr/testify/assert"
 )

+//TODO: Add a test to ensure udpAddr is copied and not reused
+
+func TestOldIPv4Only(t *testing.T) {
+	// This test ensures our new ipv6 enabled LH protobuf IpAndPorts works with the old style to enable backwards compatibility
+	b := []byte{8, 129, 130, 132, 80, 16, 10}
+	var m Ip4AndPort
+	err := proto.Unmarshal(b, &m)
+	assert.NoError(t, err)
+	assert.Equal(t, "10.1.1.1", iputil.VpnIp(m.GetIp()).String())
+}
+
 func TestNewLhQuery(t *testing.T) {
 	myIp := net.ParseIP("192.1.1.1")
-	myIpint := ip2int(myIp)
+	myIpint := iputil.Ip2VpnIp(myIp)

 	// Generating a new lh query should work
 	a := NewLhQueryByInt(myIpint)
@@ -29,48 +45,346 @@ func TestNewLhQuery(t *testing.T) {

 }

-func TestNewipandportfromudpaddr(t *testing.T) {
-	blah := NewUDPAddrFromString("1.2.2.3:12345")
-	meh := NewIpAndPortFromUDPAddr(*blah)
-	assert.Equal(t, uint32(16908803), meh.Ip)
-	assert.Equal(t, uint32(12345), meh.Port)
+func Test_lhStaticMapping(t *testing.T) {
+	l := util.NewTestLogger()
+	lh1 := "10.128.0.2"
+	lh1IP := net.ParseIP(lh1)
+
+	udpServer, _ := udp.NewListener(l, "0.0.0.0", 0, true, 2)
+
+	meh := NewLightHouse(l, true, &net.IPNet{IP: net.IP{0, 0, 0, 1}, Mask: net.IPMask{255, 255, 255, 255}}, []iputil.VpnIp{iputil.Ip2VpnIp(lh1IP)}, 10, 10003, udpServer, false, 1, false)
+	meh.AddStaticRemote(iputil.Ip2VpnIp(lh1IP), udp.NewAddr(lh1IP, uint16(4242)))
+	err := meh.ValidateLHStaticEntries()
+	assert.Nil(t, err)
+
+	lh2 := "10.128.0.3"
+	lh2IP := net.ParseIP(lh2)
+
+	meh = NewLightHouse(l, true, &net.IPNet{IP: net.IP{0, 0, 0, 1}, Mask: net.IPMask{255, 255, 255, 255}}, []iputil.VpnIp{iputil.Ip2VpnIp(lh1IP), iputil.Ip2VpnIp(lh2IP)}, 10, 10003, udpServer, false, 1, false)
+	meh.AddStaticRemote(iputil.Ip2VpnIp(lh1IP), udp.NewAddr(lh1IP, uint16(4242)))
+	err = meh.ValidateLHStaticEntries()
+	assert.EqualError(t, err, "Lighthouse 10.128.0.3 does not have a static_host_map entry")
 }

-func TestNewipandportsfromudpaddrs(t *testing.T) {
-	blah := NewUDPAddrFromString("1.2.2.3:12345")
-	blah2 := NewUDPAddrFromString("9.9.9.9:47828")
-	group := []udpAddr{*blah, *blah2}
-	hah := NewIpAndPortsFromNetIps(group)
-	assert.IsType(t, &[]*IpAndPort{}, hah)
-	//t.Error(reflect.TypeOf(hah))
+func BenchmarkLighthouseHandleRequest(b *testing.B) {
+	l := util.NewTestLogger()
+	lh1 := "10.128.0.2"
+	lh1IP := net.ParseIP(lh1)

+	udpServer, _ := udp.NewListener(l, "0.0.0.0", 0, true, 2)
+
+	lh := NewLightHouse(l, true, &net.IPNet{IP: net.IP{0, 0, 0, 1}, Mask: net.IPMask{0, 0, 0, 0}}, []iputil.VpnIp{iputil.Ip2VpnIp(lh1IP)}, 10, 10003, udpServer, false, 1, false)
+
+	hAddr := udp.NewAddrFromString("4.5.6.7:12345")
+	hAddr2 := udp.NewAddrFromString("4.5.6.7:12346")
+	lh.addrMap[3] = NewRemoteList()
+	lh.addrMap[3].unlockedSetV4(
+		3,
+		3,
+		[]*Ip4AndPort{
+			NewIp4AndPort(hAddr.IP, uint32(hAddr.Port)),
+			NewIp4AndPort(hAddr2.IP, uint32(hAddr2.Port)),
+		},
+		func(iputil.VpnIp, *Ip4AndPort) bool { return true },
+	)
+
+	rAddr := udp.NewAddrFromString("1.2.2.3:12345")
+	rAddr2 := udp.NewAddrFromString("1.2.2.3:12346")
+	lh.addrMap[2] = NewRemoteList()
+	lh.addrMap[2].unlockedSetV4(
+		3,
+		3,
+		[]*Ip4AndPort{
+			NewIp4AndPort(rAddr.IP, uint32(rAddr.Port)),
+			NewIp4AndPort(rAddr2.IP, uint32(rAddr2.Port)),
+		},
+		func(iputil.VpnIp, *Ip4AndPort) bool { return true },
+	)
+
+	mw := &mockEncWriter{}
+
+	b.Run("notfound", func(b *testing.B) {
+		lhh := lh.NewRequestHandler()
+		req := &NebulaMeta{
+			Type: NebulaMeta_HostQuery,
+			Details: &NebulaMetaDetails{
+				VpnIp:       4,
+				Ip4AndPorts: nil,
+			},
+		}
+		p, err := proto.Marshal(req)
+		assert.NoError(b, err)
+		for n := 0; n < b.N; n++ {
+			lhh.HandleRequest(rAddr, 2, p, mw)
+		}
+	})
+	b.Run("found", func(b *testing.B) {
+		lhh := lh.NewRequestHandler()
+		req := &NebulaMeta{
+			Type: NebulaMeta_HostQuery,
+			Details: &NebulaMetaDetails{
+				VpnIp:       3,
+				Ip4AndPorts: nil,
+			},
+		}
+		p, err := proto.Marshal(req)
+		assert.NoError(b, err)
+
+		for n := 0; n < b.N; n++ {
+			lhh.HandleRequest(rAddr, 2, p, mw)
+		}
+	})
 }

-/*
-func TestLHQuery(t *testing.T) {
-	//n := NewLhQueryByIpString("10.128.0.3")
-	_, myNet, _ := net.ParseCIDR("10.128.0.0/16")
-	m := NewHostMap(myNet)
-	y, _ := net.ResolveUDPAddr("udp", "10.128.0.3:11111")
-	m.Add(ip2int(net.ParseIP("127.0.0.1")), y)
-	//t.Errorf("%s", m)
-	_ = m
+func TestLighthouse_Memory(t *testing.T) {
+	l := util.NewTestLogger()

-	_, n, _ := net.ParseCIDR("127.0.0.1/8")
+	myUdpAddr0 := &udp.Addr{IP: net.ParseIP("10.0.0.2"), Port: 4242}
+	myUdpAddr1 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4242}
+	myUdpAddr2 := &udp.Addr{IP: net.ParseIP("172.16.0.2"), Port: 4242}
+	myUdpAddr3 := &udp.Addr{IP: net.ParseIP("100.152.0.2"), Port: 4242}
+	myUdpAddr4 := &udp.Addr{IP: net.ParseIP("24.15.0.2"), Port: 4242}
+	myUdpAddr5 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4243}
+	myUdpAddr6 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4244}
+	myUdpAddr7 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4245}
+	myUdpAddr8 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4246}
+	myUdpAddr9 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4247}
+	myUdpAddr10 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4248}
+	myUdpAddr11 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4249}
+	myVpnIp := iputil.Ip2VpnIp(net.ParseIP("10.128.0.2"))

-	/*udpServer, err := net.ListenUDP("udp", &net.UDPAddr{Port: 10009})
+	theirUdpAddr0 := &udp.Addr{IP: net.ParseIP("10.0.0.3"), Port: 4242}
+	theirUdpAddr1 := &udp.Addr{IP: net.ParseIP("192.168.0.3"), Port: 4242}
+	theirUdpAddr2 := &udp.Addr{IP: net.ParseIP("172.16.0.3"), Port: 4242}
+	theirUdpAddr3 := &udp.Addr{IP: net.ParseIP("100.152.0.3"), Port: 4242}
+	theirUdpAddr4 := &udp.Addr{IP: net.ParseIP("24.15.0.3"), Port: 4242}
+	theirVpnIp := iputil.Ip2VpnIp(net.ParseIP("10.128.0.3"))
+
+	udpServer, _ := udp.NewListener(l, "0.0.0.0", 0, true, 2)
+	lh := NewLightHouse(l, true, &net.IPNet{IP: net.IP{10, 128, 0, 1}, Mask: net.IPMask{255, 255, 255, 0}}, []iputil.VpnIp{}, 10, 10003, udpServer, false, 1, false)
+	lhh := lh.NewRequestHandler()
+
+	// Test that my first update responds with just that
+	newLHHostUpdate(myUdpAddr0, myVpnIp, []*udp.Addr{myUdpAddr1, myUdpAddr2}, lhh)
+	r := newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh)
+	assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr1, myUdpAddr2)
+
+	// Ensure we don't accumulate addresses
+	newLHHostUpdate(myUdpAddr0, myVpnIp, []*udp.Addr{myUdpAddr3}, lhh)
+	r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh)
+	assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr3)
+
+	// Grow it back to 2
+	newLHHostUpdate(myUdpAddr0, myVpnIp, []*udp.Addr{myUdpAddr1, myUdpAddr4}, lhh)
+	r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh)
+	assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr1, myUdpAddr4)
+
+	// Update a different host
+	newLHHostUpdate(theirUdpAddr0, theirVpnIp, []*udp.Addr{theirUdpAddr1, theirUdpAddr2, theirUdpAddr3, theirUdpAddr4}, lhh)
+	r = newLHHostRequest(theirUdpAddr0, theirVpnIp, myVpnIp, lhh)
+	assertIp4InArray(t, r.msg.Details.Ip4AndPorts, theirUdpAddr1, theirUdpAddr2, theirUdpAddr3, theirUdpAddr4)
+
+	// Make sure we didn't get changed
+	r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh)
+	assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr1, myUdpAddr4)
+
+	// Ensure proper ordering and limiting
+	// Send 12 addrs, get 10 back, the last 2 removed, allowing the duplicate to remain (clients dedupe)
+	newLHHostUpdate(
+		myUdpAddr0,
+		myVpnIp,
+		[]*udp.Addr{
+			myUdpAddr1,
+			myUdpAddr2,
+			myUdpAddr3,
+			myUdpAddr4,
+			myUdpAddr5,
+			myUdpAddr5, //Duplicated on purpose
+			myUdpAddr6,
+			myUdpAddr7,
+			myUdpAddr8,
+			myUdpAddr9,
+			myUdpAddr10,
+			myUdpAddr11, // This should get cut
+		}, lhh)
+
+	r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh)
+	assertIp4InArray(
+		t,
+		r.msg.Details.Ip4AndPorts,
+		myUdpAddr1, myUdpAddr2, myUdpAddr3, myUdpAddr4, myUdpAddr5, myUdpAddr5, myUdpAddr6, myUdpAddr7, myUdpAddr8, myUdpAddr9,
+	)
+
+	// Make sure we won't add ips in our vpn network
+	bad1 := &udp.Addr{IP: net.ParseIP("10.128.0.99"), Port: 4242}
+	bad2 := &udp.Addr{IP: net.ParseIP("10.128.0.100"), Port: 4242}
+	good := &udp.Addr{IP: net.ParseIP("1.128.0.99"), Port: 4242}
+	newLHHostUpdate(myUdpAddr0, myVpnIp, []*udp.Addr{bad1, bad2, good}, lhh)
+	r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh)
+	assertIp4InArray(t, r.msg.Details.Ip4AndPorts, good)
+}
+
+func newLHHostRequest(fromAddr *udp.Addr, myVpnIp, queryVpnIp iputil.VpnIp, lhh *LightHouseHandler) testLhReply {
+	req := &NebulaMeta{
+		Type: NebulaMeta_HostQuery,
+		Details: &NebulaMetaDetails{
+			VpnIp: uint32(queryVpnIp),
+		},
+	}
+
+	b, err := req.Marshal()
 	if err != nil {
-		t.Errorf("%s", err)
+		panic(err)
 	}

-	meh := NewLightHouse(n, m, []string{"10.128.0.2"}, false, 10, 10003, 10004)
-	//t.Error(m.Hosts)
-	meh2, err := meh.Query(ip2int(net.ParseIP("10.128.0.3")))
-	t.Error(err)
+	w := &testEncWriter{}
+	lhh.HandleRequest(fromAddr, myVpnIp, b, w)
+	return w.lastReply
+}
+
+func newLHHostUpdate(fromAddr *udp.Addr, vpnIp iputil.VpnIp, addrs []*udp.Addr, lhh *LightHouseHandler) {
+	req := &NebulaMeta{
+		Type: NebulaMeta_HostUpdateNotification,
+		Details: &NebulaMetaDetails{
+			VpnIp:       uint32(vpnIp),
+			Ip4AndPorts: make([]*Ip4AndPort, len(addrs)),
+		},
+	}
+
+	for k, v := range addrs {
+		req.Details.Ip4AndPorts[k] = &Ip4AndPort{Ip: uint32(iputil.Ip2VpnIp(v.IP)), Port: uint32(v.Port)}
+	}
+
+	b, err := req.Marshal()
 	if err != nil {
-		return
+		panic(err)
 	}
-	t.Errorf("%s", meh2)
-	t.Errorf("%s", n)
+
+	w := &testEncWriter{}
+	lhh.HandleRequest(fromAddr, vpnIp, b, w)
+}
+
+//TODO: this is a RemoteList test
+//func Test_lhRemoteAllowList(t *testing.T) {
+//	l := NewTestLogger()
+//	c := NewConfig(l)
+//	c.Settings["remoteallowlist"] = map[interface{}]interface{}{
+//		"10.20.0.0/12": false,
+//	}
+//	allowList, err := c.GetAllowList("remoteallowlist", false)
+//	assert.Nil(t, err)
+//
+//	lh1 := "10.128.0.2"
+//	lh1IP := net.ParseIP(lh1)
+//
+//	udpServer, _ := NewListener(l, "0.0.0.0", 0, true)
+//
+//	lh := NewLightHouse(l, true, &net.IPNet{IP: net.IP{0, 0, 0, 1}, Mask: net.IPMask{255, 255, 255, 0}}, []uint32{ip2int(lh1IP)}, 10, 10003, udpServer, false, 1, false)
+//	lh.SetRemoteAllowList(allowList)
+//
+//	// A disallowed ip should not enter the cache but we should end up with an empty entry in the addrMap
+//	remote1IP := net.ParseIP("10.20.0.3")
+//	remotes := lh.unlockedGetRemoteList(ip2int(remote1IP))
+//	remotes.unlockedPrependV4(ip2int(remote1IP), NewIp4AndPort(remote1IP, 4242))
+//	assert.NotNil(t, lh.addrMap[ip2int(remote1IP)])
+//	assert.Empty(t, lh.addrMap[ip2int(remote1IP)].CopyAddrs([]*net.IPNet{}))
+//
+//	// Make sure a good ip enters the cache and addrMap
+//	remote2IP := net.ParseIP("10.128.0.3")
+//	remote2UDPAddr := NewUDPAddr(remote2IP, uint16(4242))
+//	lh.addRemoteV4(ip2int(remote2IP), ip2int(remote2IP), NewIp4AndPort(remote2UDPAddr.IP, uint32(remote2UDPAddr.Port)), false, false)
+//	assertUdpAddrInArray(t, lh.addrMap[ip2int(remote2IP)].CopyAddrs([]*net.IPNet{}), remote2UDPAddr)
+//
+//	// Another good ip gets into the cache, ordering is inverted
+//	remote3IP := net.ParseIP("10.128.0.4")
+//	remote3UDPAddr := NewUDPAddr(remote3IP, uint16(4243))
+//	lh.addRemoteV4(ip2int(remote2IP), ip2int(remote2IP), NewIp4AndPort(remote3UDPAddr.IP, uint32(remote3UDPAddr.Port)), false, false)
+//	assertUdpAddrInArray(t, lh.addrMap[ip2int(remote2IP)].CopyAddrs([]*net.IPNet{}), remote2UDPAddr, remote3UDPAddr)
+//
+//	// If we exceed the length limit we should only have the most recent addresses
+//	addedAddrs := []*udpAddr{}
+//	for i := 0; i < 11; i++ {
+//		remoteUDPAddr := NewUDPAddr(net.IP{10, 128, 0, 4}, uint16(4243+i))
+//		lh.addRemoteV4(ip2int(remote2IP), ip2int(remote2IP), NewIp4AndPort(remoteUDPAddr.IP, uint32(remoteUDPAddr.Port)), false, false)
+//		// The first entry here is a duplicate, don't add it to the assert list
+//		if i != 0 {
+//			addedAddrs = append(addedAddrs, remoteUDPAddr)
+//		}
+//	}
+//
+//	// We should only have the last 10 of what we tried to add
+//	assert.True(t, len(addedAddrs) >= 10, "We should have tried to add at least 10 addresses")
+//	assertUdpAddrInArray(
+//		t,
+//		lh.addrMap[ip2int(remote2IP)].CopyAddrs([]*net.IPNet{}),
+//		addedAddrs[0],
+//		addedAddrs[1],
+//		addedAddrs[2],
+//		addedAddrs[3],
+//		addedAddrs[4],
+//		addedAddrs[5],
+//		addedAddrs[6],
+//		addedAddrs[7],
+//		addedAddrs[8],
+//		addedAddrs[9],
+//	)
+//}
+
+func Test_ipMaskContains(t *testing.T) {
+	assert.True(t, ipMaskContains(iputil.Ip2VpnIp(net.ParseIP("10.0.0.1")), 32-24, iputil.Ip2VpnIp(net.ParseIP("10.0.0.255"))))
+	assert.False(t, ipMaskContains(iputil.Ip2VpnIp(net.ParseIP("10.0.0.1")), 32-24, iputil.Ip2VpnIp(net.ParseIP("10.0.1.1"))))
+	assert.True(t, ipMaskContains(iputil.Ip2VpnIp(net.ParseIP("10.0.0.1")), 32, iputil.Ip2VpnIp(net.ParseIP("10.0.1.1"))))
+}
+
+type testLhReply struct {
+	nebType    header.MessageType
+	nebSubType header.MessageSubType
+	vpnIp      iputil.VpnIp
+	msg        *NebulaMeta
+}
+
+type testEncWriter struct {
+	lastReply testLhReply
+}
+
+func (tw *testEncWriter) SendMessageToVpnIp(t header.MessageType, st header.MessageSubType, vpnIp iputil.VpnIp, p, _, _ []byte) {
+	tw.lastReply = testLhReply{
+		nebType:    t,
+		nebSubType: st,
+		vpnIp:      vpnIp,
+		msg:        &NebulaMeta{},
+	}
+
+	err := proto.Unmarshal(p, tw.lastReply.msg)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// assertIp4InArray asserts every address in want is at the same position in have and that the lengths match
+func assertIp4InArray(t *testing.T, have []*Ip4AndPort, want ...*udp.Addr) {
+	assert.Len(t, have, len(want))
+	for k, w := range want {
+		if !(have[k].Ip == uint32(iputil.Ip2VpnIp(w.IP)) && have[k].Port == uint32(w.Port)) {
+			assert.Fail(t, fmt.Sprintf("Response did not contain: %v:%v at %v; %v", w.IP, w.Port, k, translateV4toUdpAddr(have)))
+		}
+	}
+}
+
+// assertUdpAddrInArray asserts every address in want is at the same position in have and that the lengths match
+func assertUdpAddrInArray(t *testing.T, have []*udp.Addr, want ...*udp.Addr) {
+	assert.Len(t, have, len(want))
+	for k, w := range want {
+		if !(have[k].IP.Equal(w.IP) && have[k].Port == w.Port) {
+			assert.Fail(t, fmt.Sprintf("Response did not contain: %v at %v; %v", w, k, have))
+		}
+	}
+}
+
+func translateV4toUdpAddr(ips []*Ip4AndPort) []*udp.Addr {
+	addrs := make([]*udp.Addr, len(ips))
+	for k, v := range ips {
+		addrs[k] = NewUDPAddrFromLH4(v)
+	}
+	return addrs
 }
-*/
--- a/logger.go
+++ b/logger.go
@@ -0,0 +1,78 @@
+package nebula
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/config"
+)
+
+type ContextualError struct {
+	RealError error
+	Fields    map[string]interface{}
+	Context   string
+}
+
+func NewContextualError(msg string, fields map[string]interface{}, realError error) ContextualError {
+	return ContextualError{Context: msg, Fields: fields, RealError: realError}
+}
+
+func (ce ContextualError) Error() string {
+	if ce.RealError == nil {
+		return ce.Context
+	}
+	return ce.RealError.Error()
+}
+
+func (ce ContextualError) Unwrap() error {
+	if ce.RealError == nil {
+		return errors.New(ce.Context)
+	}
+	return ce.RealError
+}
+
+func (ce *ContextualError) Log(lr *logrus.Logger) {
+	if ce.RealError != nil {
+		lr.WithFields(ce.Fields).WithError(ce.RealError).Error(ce.Context)
+	} else {
+		lr.WithFields(ce.Fields).Error(ce.Context)
+	}
+}
+
+func configLogger(l *logrus.Logger, c *config.C) error {
+	// set up our logging level
+	logLevel, err := logrus.ParseLevel(strings.ToLower(c.GetString("logging.level", "info")))
+	if err != nil {
+		return fmt.Errorf("%s; possible levels: %s", err, logrus.AllLevels)
+	}
+	l.SetLevel(logLevel)
+
+	disableTimestamp := c.GetBool("logging.disable_timestamp", false)
+	timestampFormat := c.GetString("logging.timestamp_format", "")
+	fullTimestamp := (timestampFormat != "")
+	if timestampFormat == "" {
+		timestampFormat = time.RFC3339
+	}
+
+	logFormat := strings.ToLower(c.GetString("logging.format", "text"))
+	switch logFormat {
+	case "text":
+		l.Formatter = &logrus.TextFormatter{
+			TimestampFormat:  timestampFormat,
+			FullTimestamp:    fullTimestamp,
+			DisableTimestamp: disableTimestamp,
+		}
+	case "json":
+		l.Formatter = &logrus.JSONFormatter{
+			TimestampFormat:  timestampFormat,
+			DisableTimestamp: disableTimestamp,
+		}
+	default:
+		return fmt.Errorf("unknown log format `%s`. possible formats: %s", logFormat, []string{"text", "json"})
+	}
+
+	return nil
+}
--- a/logger_test.go
+++ b/logger_test.go
@@ -0,0 +1,67 @@
+package nebula
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+type TestLogWriter struct {
+	Logs []string
+}
+
+func NewTestLogWriter() *TestLogWriter {
+	return &TestLogWriter{Logs: make([]string, 0)}
+}
+
+func (tl *TestLogWriter) Write(p []byte) (n int, err error) {
+	tl.Logs = append(tl.Logs, string(p))
+	return len(p), nil
+}
+
+func (tl *TestLogWriter) Reset() {
+	tl.Logs = tl.Logs[:0]
+}
+
+func TestContextualError_Log(t *testing.T) {
+	l := logrus.New()
+	l.Formatter = &logrus.TextFormatter{
+		DisableTimestamp: true,
+		DisableColors:    true,
+	}
+
+	tl := NewTestLogWriter()
+	l.Out = tl
+
+	// Test a full context line
+	tl.Reset()
+	e := NewContextualError("test message", m{"field": "1"}, errors.New("error"))
+	e.Log(l)
+	assert.Equal(t, []string{"level=error msg=\"test message\" error=error field=1\n"}, tl.Logs)
+
+	// Test a line with an error and msg but no fields
+	tl.Reset()
+	e = NewContextualError("test message", nil, errors.New("error"))
+	e.Log(l)
+	assert.Equal(t, []string{"level=error msg=\"test message\" error=error\n"}, tl.Logs)
+
+	// Test just a context and fields
+	tl.Reset()
+	e = NewContextualError("test message", m{"field": "1"}, nil)
+	e.Log(l)
+	assert.Equal(t, []string{"level=error msg=\"test message\" field=1\n"}, tl.Logs)
+
+	// Test just a context
+	tl.Reset()
+	e = NewContextualError("test message", nil, nil)
+	e.Log(l)
+	assert.Equal(t, []string{"level=error msg=\"test message\"\n"}, tl.Logs)
+
+	// Test just an error
+	tl.Reset()
+	e = NewContextualError("", nil, errors.New("error"))
+	e.Log(l)
+	assert.Equal(t, []string{"level=error error=error\n"}, tl.Logs)
+}
--- a/main.go
+++ b/main.go
@@ -1,137 +1,220 @@
 package nebula

 import (
+	"context"
 	"encoding/binary"
 	"fmt"
 	"net"
-	"os"
-	"os/signal"
-	"strconv"
-	"strings"
-	"syscall"
 	"time"

 	"github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/iputil"
 	"github.com/slackhq/nebula/sshd"
+	"github.com/slackhq/nebula/udp"
+	"gopkg.in/yaml.v2"
 )

-var l = logrus.New()
-
 type m map[string]interface{}

-func Main(configPath string, configTest bool, buildVersion string) {
-	l.Out = os.Stdout
+func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logger, tunFd *int) (retcon *Control, reterr error) {
+	ctx, cancel := context.WithCancel(context.Background())
+	// Automatically cancel the context if Main returns an error, to signal all created goroutines to quit.
+	defer func() {
+		if reterr != nil {
+			cancel()
+		}
+	}()
+
+	l := logger
 	l.Formatter = &logrus.TextFormatter{
 		FullTimestamp: true,
 	}

-	config := NewConfig()
-	err := config.Load(configPath)
-	if err != nil {
-		l.WithError(err).Error("Failed to load config")
-		os.Exit(1)
-	}
-
 	// Print the config if in test, the exit comes later
 	if configTest {
-		b, err := yaml.Marshal(config.Settings)
+		b, err := yaml.Marshal(c.Settings)
 		if err != nil {
-			l.Println(err)
-			os.Exit(1)
+			return nil, err
 		}
+
+		// Print the final config
 		l.Println(string(b))
 	}

-	err = configLogger(config)
+	err := configLogger(l, c)
 	if err != nil {
-		l.WithError(err).Error("Failed to configure the logger")
+		return nil, NewContextualError("Failed to configure the logger", nil, err)
 	}

-	config.RegisterReloadCallback(func(c *Config) {
-		err := configLogger(c)
+	c.RegisterReloadCallback(func(c *config.C) {
+		err := configLogger(l, c)
 		if err != nil {
 			l.WithError(err).Error("Failed to configure the logger")
 		}
 	})

-	// trustedCAs is currently a global, so loadCA operates on that global directly
-	trustedCAs, err = loadCAFromConfig(config)
+	caPool, err := loadCAFromConfig(l, c)
 	if err != nil {
 		//The errors coming out of loadCA are already nicely formatted
-		l.Fatal(err)
+		return nil, NewContextualError("Failed to load ca from config", nil, err)
 	}
-	l.WithField("fingerprints", trustedCAs.GetFingerprints()).Debug("Trusted CA fingerprints")
+	l.WithField("fingerprints", caPool.GetFingerprints()).Debug("Trusted CA fingerprints")

-	cs, err := NewCertStateFromConfig(config)
+	cs, err := NewCertStateFromConfig(c)
 	if err != nil {
 		//The errors coming out of NewCertStateFromConfig are already nicely formatted
-		l.Fatal(err)
+		return nil, NewContextualError("Failed to load certificate from config", nil, err)
 	}
 	l.WithField("cert", cs.certificate).Debug("Client nebula certificate")

-	fw, err := NewFirewallFromConfig(cs.certificate, config)
+	fw, err := NewFirewallFromConfig(l, cs.certificate, c)
 	if err != nil {
-		l.Fatal("Error while loading firewall rules: ", err)
+		return nil, NewContextualError("Error while loading firewall rules", nil, err)
 	}
 	l.WithField("firewallHash", fw.GetRuleHash()).Info("Firewall started")

 	// TODO: make sure mask is 4 bytes
 	tunCidr := cs.certificate.Details.Ips[0]
-	routes, err := parseRoutes(config, tunCidr)
+	routes, err := parseRoutes(c, tunCidr)
 	if err != nil {
-		l.WithError(err).Fatal("Could not parse tun.routes")
+		return nil, NewContextualError("Could not parse tun.routes", nil, err)
+	}
+	unsafeRoutes, err := parseUnsafeRoutes(c, tunCidr)
+	if err != nil {
+		return nil, NewContextualError("Could not parse tun.unsafe_routes", nil, err)
 	}

 	ssh, err := sshd.NewSSHServer(l.WithField("subsystem", "sshd"))
-	wireSSHReload(ssh, config)
-	if config.GetBool("sshd.enabled", false) {
-		err = configSSH(ssh, config)
+	wireSSHReload(l, ssh, c)
+	var sshStart func()
+	if c.GetBool("sshd.enabled", false) {
+		sshStart, err = configSSH(l, ssh, c)
 		if err != nil {
-			l.WithError(err).Fatal("Error while configuring the sshd")
+			return nil, NewContextualError("Error while configuring the sshd", nil, err)
 		}
 	}

+	psk, err := NewPskFromConfig(c, iputil.Ip2VpnIp(tunCidr.IP))
+	if err != nil {
+		return nil, NewContextualError("Failed to create psk", nil, err)
+	}
+
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 	// All non system modifying configuration consumption should live above this line
 	// tun config, listeners, anything modifying the computer should be below
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

-	if configTest {
-		os.Exit(0)
+	var routines int
+
+	// If `routines` is set, use that and ignore the specific values
+	if routines = c.GetInt("routines", 0); routines != 0 {
+		if routines < 1 {
+			routines = 1
+		}
+		if routines > 1 {
+			l.WithField("routines", routines).Info("Using multiple routines")
+		}
+	} else {
+		// deprecated and undocumented
+		tunQueues := c.GetInt("tun.routines", 1)
+		udpQueues := c.GetInt("listen.routines", 1)
+		if tunQueues > udpQueues {
+			routines = tunQueues
+		} else {
+			routines = udpQueues
+		}
+		if routines != 1 {
+			l.WithField("routines", routines).Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead")
+		}
 	}

-	config.CatchHUP()
+	// EXPERIMENTAL
+	// Intentionally not documented yet while we do more testing and determine
+	// a good default value.
+	conntrackCacheTimeout := c.GetDuration("firewall.conntrack.routine_cache_timeout", 0)
+	if routines > 1 && !c.IsSet("firewall.conntrack.routine_cache_timeout") {
+		// Use a different default if we are running with multiple routines
+		conntrackCacheTimeout = 1 * time.Second
+	}
+	if conntrackCacheTimeout > 0 {
+		l.WithField("duration", conntrackCacheTimeout).Info("Using routine-local conntrack cache")
+	}

-	// set up our tun dev
-	tun, err := newTun(
-		config.GetString("tun.dev", ""),
+	var tun Inside
+	if !configTest {
+		c.CatchHUP(ctx)
+
+		switch {
+		case c.GetBool("tun.disabled", false):
+			tun = newDisabledTun(tunCidr, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
+		case tunFd != nil:
+			tun, err = newTunFromFd(
+				l,
+				*tunFd,
 				tunCidr,
-		config.GetInt("tun.mtu", 1300),
+				c.GetInt("tun.mtu", DEFAULT_MTU),
 				routes,
-		config.GetInt("tun.tx_queue", 500),
+				unsafeRoutes,
+				c.GetInt("tun.tx_queue", 500),
+			)
+		default:
+			tun, err = newTun(
+				l,
+				c.GetString("tun.dev", ""),
+				tunCidr,
+				c.GetInt("tun.mtu", DEFAULT_MTU),
+				routes,
+				unsafeRoutes,
+				c.GetInt("tun.tx_queue", 500),
+				routines > 1,
 			)
-	if err != nil {
-		l.Fatal(err)
 		}

+		if err != nil {
+			return nil, NewContextualError("Failed to get a tun/tap device", nil, err)
+		}
+	}
+
+	defer func() {
+		if reterr != nil {
+			tun.Close()
+		}
+	}()
+
 	// set up our UDP listener
-	udpQueues := config.GetInt("listen.routines", 1)
-	udpServer, err := NewListener(config.GetString("listen.host", "0.0.0.0"), config.GetInt("listen.port", 0), udpQueues > 1)
+	udpConns := make([]*udp.Conn, routines)
+	port := c.GetInt("listen.port", 0)
+
+	if !configTest {
+		for i := 0; i < routines; i++ {
+			udpServer, err := udp.NewListener(l, c.GetString("listen.host", "0.0.0.0"), port, routines > 1, c.GetInt("listen.batch", 64))
 			if err != nil {
-		l.Fatal(err)
+				return nil, NewContextualError("Failed to open udp listener", m{"queue": i}, err)
+			}
+			udpServer.ReloadConfig(c)
+			udpConns[i] = udpServer
+
+			// If port is dynamic, discover it
+			if port == 0 {
+				uPort, err := udpServer.LocalAddr()
+				if err != nil {
+					return nil, NewContextualError("Failed to get listening port", nil, err)
+				}
+				port = int(uPort.Port)
+			}
+		}
 	}
-	udpServer.reloadConfig(config)

 	// Set up my internal host map
 	var preferredRanges []*net.IPNet
-	rawPreferredRanges := config.GetStringSlice("preferred_ranges", []string{})
+	rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{})
 	// First, check if 'preferred_ranges' is set and fallback to 'local_range'
 	if len(rawPreferredRanges) > 0 {
 		for _, rawPreferredRange := range rawPreferredRanges {
 			_, preferredRange, err := net.ParseCIDR(rawPreferredRange)
 			if err != nil {
-				l.Fatal(err)
+				return nil, NewContextualError("Failed to parse preferred ranges", nil, err)
 			}
 			preferredRanges = append(preferredRanges, preferredRange)
 		}
@@ -140,11 +223,11 @@ func Main(configPath string, configTest bool, buildVersion string) {
 	// local_range was superseded by preferred_ranges. If it is still present,
 	// merge the local_range setting into preferred_ranges. We will probably
 	// deprecate local_range and remove in the future.
-	rawLocalRange := config.GetString("local_range", "")
+	rawLocalRange := c.GetString("local_range", "")
 	if rawLocalRange != "" {
 		_, localRange, err := net.ParseCIDR(rawLocalRange)
 		if err != nil {
-			l.Fatal(err)
+			return nil, NewContextualError("Failed to parse local_range", nil, err)
 		}

 		// Check if the entry for local_range was already specified in
@@ -161,8 +244,11 @@ func Main(configPath string, configTest bool, buildVersion string) {
 		}
 	}

-	hostMap := NewHostMap("main", tunCidr, preferredRanges)
-	hostMap.SetDefaultRoute(ip2int(net.ParseIP(config.GetString("default_route", "0.0.0.0"))))
+	hostMap := NewHostMap(l, "main", tunCidr, preferredRanges)
+
+	hostMap.addUnsafeRoutes(&unsafeRoutes)
+	hostMap.metricsEnabled = c.GetBool("stats.message_metrics", false)
+
 	l.WithField("network", hostMap.vpnCIDR).WithField("preferredRanges", hostMap.preferredRanges).Info("Main HostMap created")

 	/*
@@ -170,152 +256,196 @@ func Main(configPath string, configTest bool, buildVersion string) {
 		go hostMap.Promoter(config.GetInt("promoter.interval"))
 	*/

-	punchy := config.GetBool("punchy", false)
-	if punchy == true {
+	punchy := NewPunchyFromConfig(c)
+	if punchy.Punch && !configTest {
 		l.Info("UDP hole punching enabled")
-		go hostMap.Punchy(udpServer)
+		go hostMap.Punchy(ctx, udpConns[0])
 	}

-	port := config.GetInt("listen.port", 0)
-	// If port is dynamic, discover it
-	if port == 0 {
-		uPort, err := udpServer.LocalAddr()
-		if err != nil {
-			l.WithError(err).Fatal("Failed to get listening port")
-		}
-		port = int(uPort.Port)
+	amLighthouse := c.GetBool("lighthouse.am_lighthouse", false)
+
+	// fatal if am_lighthouse is enabled but we are using an ephemeral port
+	if amLighthouse && (c.GetInt("listen.port", 0) == 0) {
+		return nil, NewContextualError("lighthouse.am_lighthouse enabled on node but no port number is set in config", nil, nil)
+	}
+
+	// warn if am_lighthouse is enabled but upstream lighthouses exists
+	rawLighthouseHosts := c.GetStringSlice("lighthouse.hosts", []string{})
+	if amLighthouse && len(rawLighthouseHosts) != 0 {
+		l.Warn("lighthouse.am_lighthouse enabled on node but upstream lighthouses exist in config")
+	}
+
+	lighthouseHosts := make([]iputil.VpnIp, len(rawLighthouseHosts))
+	for i, host := range rawLighthouseHosts {
+		ip := net.ParseIP(host)
+		if ip == nil {
+			return nil, NewContextualError("Unable to parse lighthouse host entry", m{"host": host, "entry": i + 1}, nil)
+		}
+		if !tunCidr.Contains(ip) {
+			return nil, NewContextualError("lighthouse host is not in our subnet, invalid", m{"vpnIp": ip, "network": tunCidr.String()}, nil)
+		}
+		lighthouseHosts[i] = iputil.Ip2VpnIp(ip)
 	}

-	punchBack := config.GetBool("punch_back", false)
-	amLighthouse := config.GetBool("lighthouse.am_lighthouse", false)
-	serveDns := config.GetBool("lighthouse.serve_dns", false)
 	lightHouse := NewLightHouse(
+		l,
 		amLighthouse,
-		ip2int(tunCidr.IP),
-		config.GetStringSlice("lighthouse.hosts", []string{}),
+		tunCidr,
+		lighthouseHosts,
 		//TODO: change to a duration
-		config.GetInt("lighthouse.interval", 10),
-		port,
-		udpServer,
-		punchBack,
+		c.GetInt("lighthouse.interval", 10),
+		uint32(port),
+		udpConns[0],
+		punchy.Respond,
+		punchy.Delay,
+		c.GetBool("stats.lighthouse_metrics", false),
 	)

-	if amLighthouse && serveDns {
-		l.Debugln("Starting dns server")
-		go dnsMain(hostMap)
+	remoteAllowList, err := NewRemoteAllowListFromConfig(c, "lighthouse.remote_allow_list", "lighthouse.remote_allow_ranges")
+	if err != nil {
+		return nil, NewContextualError("Invalid lighthouse.remote_allow_list", nil, err)
 	}
+	lightHouse.SetRemoteAllowList(remoteAllowList)

-	for k, v := range config.GetMap("static_host_map", map[interface{}]interface{}{}) {
-		vpnIp := net.ParseIP(fmt.Sprintf("%v", k))
+	localAllowList, err := NewLocalAllowListFromConfig(c, "lighthouse.local_allow_list")
+	if err != nil {
+		return nil, NewContextualError("Invalid lighthouse.local_allow_list", nil, err)
+	}
+	lightHouse.SetLocalAllowList(localAllowList)
+
+	//TODO: Move all of this inside functions in lighthouse.go
+	for k, v := range c.GetMap("static_host_map", map[interface{}]interface{}{}) {
+		ip := net.ParseIP(fmt.Sprintf("%v", k))
+		vpnIp := iputil.Ip2VpnIp(ip)
+		if !tunCidr.Contains(ip) {
+			return nil, NewContextualError("static_host_map key is not in our subnet, invalid", m{"vpnIp": vpnIp, "network": tunCidr.String()}, nil)
+		}
 		vals, ok := v.([]interface{})
 		if ok {
 			for _, v := range vals {
-				parts := strings.Split(fmt.Sprintf("%v", v), ":")
-				addr, err := net.ResolveIPAddr("ip", parts[0])
-				if err == nil {
-					ip := addr.IP
-					port, err := strconv.Atoi(parts[1])
+				ip, port, err := udp.ParseIPAndPort(fmt.Sprintf("%v", v))
 				if err != nil {
-						l.Fatalf("Static host address for %s could not be parsed: %s", vpnIp, v)
-					}
-					lightHouse.AddRemote(ip2int(vpnIp), NewUDPAddr(ip2int(ip), uint16(port)), true)
+					return nil, NewContextualError("Static host address could not be parsed", m{"vpnIp": vpnIp}, err)
 				}
+				lightHouse.AddStaticRemote(vpnIp, udp.NewAddr(ip, port))
 			}
 		} else {
-			//TODO: make this all a helper
-			parts := strings.Split(fmt.Sprintf("%v", v), ":")
-			addr, err := net.ResolveIPAddr("ip", parts[0])
-			if err == nil {
-				ip := addr.IP
-				port, err := strconv.Atoi(parts[1])
+			ip, port, err := udp.ParseIPAndPort(fmt.Sprintf("%v", v))
 			if err != nil {
-					l.Fatalf("Static host address for %s could not be parsed: %s", vpnIp, v)
-				}
-				lightHouse.AddRemote(ip2int(vpnIp), NewUDPAddr(ip2int(ip), uint16(port)), true)
+				return nil, NewContextualError("Static host address could not be parsed", m{"vpnIp": vpnIp}, err)
 			}
+			lightHouse.AddStaticRemote(vpnIp, udp.NewAddr(ip, port))
 		}
 	}

-	handshakeManager := NewHandshakeManager(tunCidr, preferredRanges, hostMap, lightHouse, udpServer)
+	err = lightHouse.ValidateLHStaticEntries()
+	if err != nil {
+		l.WithError(err).Error("Lighthouse unreachable")
+	}

-	handshakeMACKey := config.GetString("handshake_mac.key", "")
-	handshakeAcceptedMACKeys := config.GetStringSlice("handshake_mac.accepted_keys", []string{})
+	var messageMetrics *MessageMetrics
+	if c.GetBool("stats.message_metrics", false) {
+		messageMetrics = newMessageMetrics()
+	} else {
+		messageMetrics = newMessageMetricsOnlyRecvError()
+	}

-	checkInterval := config.GetInt("timers.connection_alive_interval", 5)
-	pendingDeletionInterval := config.GetInt("timers.pending_deletion_interval", 10)
+	handshakeConfig := HandshakeConfig{
+		tryInterval:   c.GetDuration("handshakes.try_interval", DefaultHandshakeTryInterval),
+		retries:       c.GetInt("handshakes.retries", DefaultHandshakeRetries),
+		triggerBuffer: c.GetInt("handshakes.trigger_buffer", DefaultHandshakeTriggerBuffer),
+
+		messageMetrics: messageMetrics,
+	}
+
+	handshakeManager := NewHandshakeManager(l, tunCidr, preferredRanges, hostMap, lightHouse, udpConns[0], handshakeConfig)
+	lightHouse.handshakeTrigger = handshakeManager.trigger
+
+	serveDns := false
+	if c.GetBool("lighthouse.serve_dns", false) {
+		if c.GetBool("lighthouse.am_lighthouse", false) {
+			serveDns = true
+		} else {
+			l.Warn("DNS server refusing to run because this host is not a lighthouse.")
+		}
+	}
+
+	checkInterval := c.GetInt("timers.connection_alive_interval", 5)
+	pendingDeletionInterval := c.GetInt("timers.pending_deletion_interval", 10)
 	ifConfig := &InterfaceConfig{
 		HostMap:                 hostMap,
 		Inside:                  tun,
-		Outside:                  udpServer,
+		Outside:                 udpConns[0],
 		certState:               cs,
-		Cipher:                   config.GetString("cipher", "aes"),
+		Cipher:                  c.GetString("cipher", "aes"),
 		Firewall:                fw,
 		ServeDns:                serveDns,
 		HandshakeManager:        handshakeManager,
 		lightHouse:              lightHouse,
 		checkInterval:           checkInterval,
 		pendingDeletionInterval: pendingDeletionInterval,
-		handshakeMACKey:          handshakeMACKey,
-		handshakeAcceptedMACKeys: handshakeAcceptedMACKeys,
-		DropLocalBroadcast:       config.GetBool("tun.drop_local_broadcast", false),
-		DropMulticast:            config.GetBool("tun.drop_multicast", false),
-		UDPBatchSize:             config.GetInt("listen.batch", 64),
+		DropLocalBroadcast:      c.GetBool("tun.drop_local_broadcast", false),
+		DropMulticast:           c.GetBool("tun.drop_multicast", false),
+		routines:                routines,
+		MessageMetrics:          messageMetrics,
+		version:                 buildVersion,
+		caPool:                  caPool,
+		disconnectInvalid:       c.GetBool("pki.disconnect_invalid", false),
+		psk:                     psk,
+
+		ConntrackCacheTimeout: conntrackCacheTimeout,
+		l:                     l,
 	}

 	switch ifConfig.Cipher {
 	case "aes":
-		noiseEndiannes = binary.BigEndian
+		noiseEndianness = binary.BigEndian
 	case "chachapoly":
-		noiseEndiannes = binary.LittleEndian
+		noiseEndianness = binary.LittleEndian
 	default:
-		l.Fatalf("Unknown cipher: %v", ifConfig.Cipher)
+		return nil, fmt.Errorf("unknown cipher: %v", ifConfig.Cipher)
 	}

-	ifce, err := NewInterface(ifConfig)
+	var ifce *Interface
+	if !configTest {
+		ifce, err = NewInterface(ctx, ifConfig)
 		if err != nil {
-		l.Fatal(err)
+			return nil, fmt.Errorf("failed to initialize interface: %s", err)
 		}

-	ifce.RegisterConfigChangeCallbacks(config)
+		// TODO: Better way to attach these, probably want a new interface in InterfaceConfig
+		// I don't want to make this initial commit too far-reaching though
+		ifce.writers = udpConns

-	go handshakeManager.Run(ifce)
-	go lightHouse.LhUpdateWorker(ifce)
+		ifce.RegisterConfigChangeCallbacks(c)
+
+		go handshakeManager.Run(ctx, ifce)
+		go lightHouse.LhUpdateWorker(ctx, ifce)
+	}
+
+	// TODO - stats third-party modules start uncancellable goroutines. Update those libs to accept
+	// a context so that they can exit when the context is Done.
+	statsStart, err := startStats(l, c, buildVersion, configTest)

-	err = startStats(config)
 	if err != nil {
-		l.Fatal(err)
+		return nil, NewContextualError("Failed to start stats emitter", nil, err)
+	}
+
+	if configTest {
+		return nil, nil
 	}

 	//TODO: check if we _should_ be emitting stats
-	go ifce.emitStats(config.GetDuration("stats.interval", time.Second*10))
+	go ifce.emitStats(ctx, c.GetDuration("stats.interval", time.Second*10))

-	attachCommands(ssh, hostMap, handshakeManager.pendingHostMap, lightHouse, ifce)
-	ifce.Run(config.GetInt("tun.routines", 1), udpQueues, buildVersion)
+	attachCommands(l, ssh, hostMap, handshakeManager.pendingHostMap, lightHouse, ifce)

-	// Just sit here and be friendly, main thread.
-	shutdownBlock(ifce)
+	// Start DNS server last to allow using the nebula IP as lighthouse.dns.host
+	var dnsStart func()
+	if amLighthouse && serveDns {
+		l.Debugln("Starting dns server")
+		dnsStart = dnsMain(l, hostMap, c)
 	}

-func shutdownBlock(ifce *Interface) {
-	var sigChan = make(chan os.Signal)
-	signal.Notify(sigChan, syscall.SIGTERM)
-	signal.Notify(sigChan, syscall.SIGINT)
-
-	sig := <-sigChan
-	l.WithField("signal", sig).Info("Caught signal, shutting down")
-
-	//TODO: stop tun and udp routines, the lock on hostMap does effectively does that though
-	//TODO: this is probably better as a function in ConnectionManager or HostMap directly
-	ifce.hostMap.Lock()
-	for _, h := range ifce.hostMap.Hosts {
-		if h.ConnectionState.ready {
-			ifce.send(closeTunnel, 0, h.ConnectionState, h, h.remote, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
-			l.WithField("vpnIp", IntIp(h.hostId)).WithField("udpAddr", h.remote).
-				Debug("Sending close tunnel message")
-		}
-	}
-	ifce.hostMap.Unlock()
-
-	l.WithField("signal", sig).Info("Goodbye")
-	os.Exit(0)
+	return &Control{ifce, l, cancel, sshStart, statsStart, dnsStart}, nil
 }
--- a/main_test.go
+++ b/main_test.go
@@ -1 +0,0 @@
-package nebula
--- a/message_metrics.go
+++ b/message_metrics.go
@@ -0,0 +1,100 @@
+package nebula
+
+import (
+	"fmt"
+
+	"github.com/rcrowley/go-metrics"
+	"github.com/slackhq/nebula/header"
+)
+
+//TODO: this can probably move into the header package
+
+type MessageMetrics struct {
+	rx [][]metrics.Counter
+	tx [][]metrics.Counter
+
+	rxUnknown metrics.Counter
+	txUnknown metrics.Counter
+}
+
+func (m *MessageMetrics) Rx(t header.MessageType, s header.MessageSubType, i int64) {
+	if m != nil {
+		if t >= 0 && int(t) < len(m.rx) && s >= 0 && int(s) < len(m.rx[t]) {
+			m.rx[t][s].Inc(i)
+		} else if m.rxUnknown != nil {
+			m.rxUnknown.Inc(i)
+		}
+	}
+}
+func (m *MessageMetrics) Tx(t header.MessageType, s header.MessageSubType, i int64) {
+	if m != nil {
+		if t >= 0 && int(t) < len(m.tx) && s >= 0 && int(s) < len(m.tx[t]) {
+			m.tx[t][s].Inc(i)
+		} else if m.txUnknown != nil {
+			m.txUnknown.Inc(i)
+		}
+	}
+}
+
+func newMessageMetrics() *MessageMetrics {
+	gen := func(t string) [][]metrics.Counter {
+		return [][]metrics.Counter{
+			{
+				metrics.GetOrRegisterCounter(fmt.Sprintf("messages.%s.handshake_ixpsk0", t), nil),
+			},
+			nil,
+			{metrics.GetOrRegisterCounter(fmt.Sprintf("messages.%s.recv_error", t), nil)},
+			{metrics.GetOrRegisterCounter(fmt.Sprintf("messages.%s.lighthouse", t), nil)},
+			{
+				metrics.GetOrRegisterCounter(fmt.Sprintf("messages.%s.test_request", t), nil),
+				metrics.GetOrRegisterCounter(fmt.Sprintf("messages.%s.test_response", t), nil),
+			},
+			{metrics.GetOrRegisterCounter(fmt.Sprintf("messages.%s.close_tunnel", t), nil)},
+		}
+	}
+	return &MessageMetrics{
+		rx: gen("rx"),
+		tx: gen("tx"),
+
+		rxUnknown: metrics.GetOrRegisterCounter("messages.rx.other", nil),
+		txUnknown: metrics.GetOrRegisterCounter("messages.tx.other", nil),
+	}
+}
+
+// Historically we only recorded recv_error, so this is backwards compat
+func newMessageMetricsOnlyRecvError() *MessageMetrics {
+	gen := func(t string) [][]metrics.Counter {
+		return [][]metrics.Counter{
+			nil,
+			nil,
+			{metrics.GetOrRegisterCounter(fmt.Sprintf("messages.%s.recv_error", t), nil)},
+		}
+	}
+	return &MessageMetrics{
+		rx: gen("rx"),
+		tx: gen("tx"),
+	}
+}
+
+func newLighthouseMetrics() *MessageMetrics {
+	gen := func(t string) [][]metrics.Counter {
+		h := make([][]metrics.Counter, len(NebulaMeta_MessageType_name))
+		used := []NebulaMeta_MessageType{
+			NebulaMeta_HostQuery,
+			NebulaMeta_HostQueryReply,
+			NebulaMeta_HostUpdateNotification,
+			NebulaMeta_HostPunchNotification,
+		}
+		for _, i := range used {
+			h[i] = []metrics.Counter{metrics.GetOrRegisterCounter(fmt.Sprintf("lighthouse.%s.%s", t, i.String()), nil)}
+		}
+		return h
+	}
+	return &MessageMetrics{
+		rx: gen("rx"),
+		tx: gen("tx"),
+
+		rxUnknown: metrics.GetOrRegisterCounter("lighthouse.rx.other", nil),
+		txUnknown: metrics.GetOrRegisterCounter("lighthouse.tx.other", nil),
+	}
+}
--- a/nebula.pb.go
+++ b/nebula.pb.go
--- a/nebula.proto
+++ b/nebula.proto
@@ -1,6 +1,8 @@
 syntax = "proto3";
 package nebula;

+option go_package = "github.com/slackhq/nebula";
+
 message NebulaMeta {
  enum MessageType {
    None = 0;
@@ -20,19 +22,23 @@ message NebulaMeta {
  NebulaMetaDetails Details = 2;
 }

-
 message NebulaMetaDetails {
-
  uint32 VpnIp = 1;
-  repeated IpAndPort IpAndPorts = 2;
+  repeated Ip4AndPort Ip4AndPorts = 2;
+  repeated Ip6AndPort Ip6AndPorts = 4;
  uint32 counter = 3;
 }

-message IpAndPort {
+message Ip4AndPort {
  uint32 Ip = 1;
  uint32 Port = 2;
 }

+message Ip6AndPort {
+  uint64 Hi = 1;
+  uint64 Lo = 2;
+  uint32 Port = 3;
+}

 message NebulaPing {
  enum MessageType {
--- a/noise.go
+++ b/noise.go
@@ -8,11 +8,11 @@ import (
 	"github.com/flynn/noise"
 )

-type endiannes interface {
+type endianness interface {
 	PutUint64(b []byte, v uint64)
 }

-var noiseEndiannes endiannes = binary.BigEndian
+var noiseEndianness endianness = binary.BigEndian

 type NebulaCipherState struct {
 	c noise.Cipher
@@ -22,7 +22,6 @@ type NebulaCipherState struct {

 func NewNebulaCipherState(s *noise.CipherState) *NebulaCipherState {
 	return &NebulaCipherState{c: s.Cipher()}
-
 }

 func (s *NebulaCipherState) EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error) {
@@ -37,7 +36,7 @@ func (s *NebulaCipherState) EncryptDanger(out, ad, plaintext []byte, n uint64, n
 		nb[1] = 0
 		nb[2] = 0
 		nb[3] = 0
-		noiseEndiannes.PutUint64(nb[4:], n)
+		noiseEndianness.PutUint64(nb[4:], n)
 		out = s.c.(cipher.AEAD).Seal(out, nb, plaintext, ad)
 		//l.Debugf("Encryption: outlen: %d, nonce: %d, ad: %s, plainlen %d", len(out), n, ad, len(plaintext))
 		return out, nil
@@ -52,7 +51,7 @@ func (s *NebulaCipherState) DecryptDanger(out, ad, ciphertext []byte, n uint64,
 		nb[1] = 0
 		nb[2] = 0
 		nb[3] = 0
-		noiseEndiannes.PutUint64(nb[4:], n)
+		noiseEndianness.PutUint64(nb[4:], n)
 		return s.c.(cipher.AEAD).Open(out, nb, ciphertext, ad)
 	} else {
 		return []byte{}, nil
--- a/Show More
+++ b/Show More