v1.5.2 (#612)

Update CHANGELOG for Nebula v1.5.2

.github/workflows/release.yml

@@ -25,7 +25,7 @@ jobs:
           mv build/*.tar.gz release
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v2
         with:
           name: linux-latest
           path: release
@@ -47,16 +47,20 @@ jobs:
           echo $Env:GITHUB_REF.Substring(11)
           go build -trimpath -ldflags "-X main.Build=$($Env:GITHUB_REF.Substring(11))" -o build\nebula.exe ./cmd/nebula-service
           go build -trimpath -ldflags "-X main.Build=$($Env:GITHUB_REF.Substring(11))" -o build\nebula-cert.exe ./cmd/nebula-cert
+          mkdir build\dist\windows
+          mv dist\windows\wintun build\dist\windows\
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v2
         with:
           name: windows-latest
           path: build
 
   build-darwin:
-    name: Build Darwin amd64
-    runs-on: macOS-latest
+    name: Build Universal Darwin
+    env:
+      HAS_SIGNING_CREDS: ${{ secrets.AC_USERNAME != '' }}
+    runs-on: macos-11
     steps:
       - name: Set up Go 1.17
         uses: actions/setup-go@v2
@@ -66,43 +70,54 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
 
-      - name: Build
+      - name: Import certificates
+        if: env.HAS_SIGNING_CREDS == 'true'
+        uses: Apple-Actions/import-codesign-certs@v1
+        with:
+          p12-file-base64: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }}
+          p12-password: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }}
+
+      - name: Build, sign, and notarize
+        env:
+          AC_USERNAME: ${{ secrets.AC_USERNAME }}
+          AC_PASSWORD: ${{ secrets.AC_PASSWORD }}
         run: |
-          make BUILD_NUMBER="${GITHUB_REF#refs/tags/v}" service build/nebula-darwin-amd64.tar.gz
-          make BUILD_NUMBER="${GITHUB_REF#refs/tags/v}" service build/nebula-darwin-arm64.tar.gz
+          rm -rf release
           mkdir release
-          mv build/*.tar.gz release
+          make BUILD_NUMBER="${GITHUB_REF#refs/tags/v}" service build/darwin-amd64/nebula build/darwin-amd64/nebula-cert
+          make BUILD_NUMBER="${GITHUB_REF#refs/tags/v}" service build/darwin-arm64/nebula build/darwin-arm64/nebula-cert
+          lipo -create -output ./release/nebula ./build/darwin-amd64/nebula ./build/darwin-arm64/nebula
+          lipo -create -output ./release/nebula-cert ./build/darwin-amd64/nebula-cert ./build/darwin-arm64/nebula-cert
+
+          if [ -n "$AC_USERNAME" ]; then
+            codesign -s "10BC1FDDEB6CE753550156C0669109FAC49E4D1E" -f -v --timestamp --options=runtime -i "net.defined.nebula" ./release/nebula
+            codesign -s "10BC1FDDEB6CE753550156C0669109FAC49E4D1E" -f -v --timestamp --options=runtime -i "net.defined.nebula-cert" ./release/nebula-cert
+          fi
+
+          zip -j release/nebula-darwin.zip release/nebula-cert release/nebula
+
+          if [ -n "$AC_USERNAME" ]; then
+            xcrun notarytool submit ./release/nebula-darwin.zip --team-id "576H3XS7FP" --apple-id "$AC_USERNAME" --password "$AC_PASSWORD" --wait
+          fi
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v2
         with:
           name: darwin-latest
-          path: release
+          path: ./release/*
 
   release:
     name: Create and Upload Release
     needs: [build-linux, build-darwin, build-windows]
     runs-on: ubuntu-latest
     steps:
-      - name: Download Linux artifacts
-        uses: actions/download-artifact@v1
-        with:
-          name: linux-latest
-
-      - name: Download Darwin artifacts
-        uses: actions/download-artifact@v1
-        with:
-          name: darwin-latest
-
-      - name: Download Windows artifacts
-        uses: actions/download-artifact@v1
-        with:
-          name: windows-latest
+      - name: Download artifacts
+        uses: actions/download-artifact@v2
 
       - name: Zip Windows
         run: |
           cd windows-latest
-          zip nebula-windows-amd64.zip nebula.exe nebula-cert.exe
+          zip -r nebula-windows-amd64.zip nebula.exe nebula-cert.exe dist
 
       - name: Create sha256sum
         run: |
@@ -115,12 +130,17 @@ jobs:
                 sha256sum <nebula.exe | sed 's=-$=nebula-windows-amd64.zip/nebula.exe='
                 sha256sum <nebula-cert.exe | sed 's=-$=nebula-windows-amd64.zip/nebula-cert.exe='
                 sha256sum nebula-windows-amd64.zip
+              elif [ "$dir" = darwin-latest ]
+              then
+                sha256sum <nebula-darwin.zip | sed 's=-$=nebula-darwin.zip='
+                sha256sum <nebula | sed 's=-$=nebula-darwin.zip/nebula='
+                sha256sum <nebula-cert | sed 's=-$=nebula-darwin.zip/nebula-cert='
               else
-                  for v in *.tar.gz
-                  do
-                    sha256sum $v
-                    tar zxf $v --to-command='sh -c "sha256sum | sed s=-$='$v'/$TAR_FILENAME="'
-                  done
+                for v in *.tar.gz
+                do
+                  sha256sum $v
+                  tar zxf $v --to-command='sh -c "sha256sum | sed s=-$='$v'/$TAR_FILENAME="'
+                done
               fi
             )
           done | sort -k 2 >SHASUM256.txt
@@ -150,25 +170,15 @@ jobs:
           asset_name: SHASUM256.txt
           asset_content_type: text/plain
 
-      - name: Upload darwin-amd64
+      - name: Upload darwin zip
         uses: actions/upload-release-asset@v1.0.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./darwin-latest/nebula-darwin-amd64.tar.gz
-          asset_name: nebula-darwin-amd64.tar.gz
-          asset_content_type: application/gzip
-
-      - name: Upload darwin-arm64
-        uses: actions/upload-release-asset@v1.0.1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./darwin-latest/nebula-darwin-arm64.tar.gz
-          asset_name: nebula-darwin-arm64.tar.gz
-          asset_content_type: application/gzip
+          asset_path: ./darwin-latest/nebula-darwin.zip
+          asset_name: nebula-darwin.zip
+          asset_content_type: application/zip
 
       - name: Upload windows-amd64
         uses: actions/upload-release-asset@v1.0.1

.github/workflows/test.yml

@@ -48,7 +48,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, macOS-latest]
+        os: [windows-latest, macos-11]
     steps:
 
     - name: Set up Go 1.17

CHANGELOG.md

@@ -7,6 +7,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.5.2] - 2021-12-14
+
+### Added
+
+- Warn when a non lighthouse node does not have lighthouse hosts configured. (#587)
+
+### Changed
+
+- No longer fatals if expired CA certificates are present in `pki.ca`, as long as 1 valid CA is present. (#599)
+
+- `nebula-cert` will now enforce ipv4 addresses. (#604)
+
+- Warn on macOS if an unsafe route cannot be created due to a collision with an
+  existing route. (#610)
+
+- Warn if you set a route MTU on platforms where we don't support it. (#611)
+
+### Fixed
+
+- Rare race condition when tearing down a tunnel due to `recv_error` and sending packets on another thread. (#590)
+
+- Bug in `routes` and `unsafe_routes` handling that was introduced in 1.5.0. (#595)
+
+- `-test` mode no longer results in a crash. (#602)
+
+### Removed
+
+- `x509.ca` config alias for `pki.ca`. (#604)
+
+### Security
+
+- Upgraded `golang.org/x/crypto` to address an issue which allowed unauthenticated clients to cause a panic in SSH
+  servers. (#603)
+
+## 1.5.1 - 2021-12-13
+
+(This release was skipped due to discovering #610 and #611 after the tag was
+created.)
+
+## [1.5.0] - 2021-11-11
+
 ### Added
 
 - SSH `print-cert` has a new `-raw` flag to get the PEM representation of a certificate. (#483)
@@ -20,12 +61,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   certificates since the last handshake. (#370)
 
 - New config option `unsafe_routes.<route>.metric` will set a metric for a specific unsafe route. It's useful if you have
-  more than one identical route and want to prefer one against the other.
+  more than one identical route and want to prefer one against the other. (#353)
 
 ### Changed
 
 - Build against go 1.17. (#553)
 
+- Build with `CGO_ENABLED=0` set, to create more portable binaries. This could
+  have an effect on DNS resolution if you rely on anything non-standard. (#421)
+
+- Windows now uses the [wintun](https://www.wintun.net/) driver which does not require installation. This driver
+  is a large improvement over the TAP driver that was used in previous versions. If you had a previous version
+  of `nebula` running, you will want to disable the tap driver in Control Panel, or uninstall the `tap0901` driver
+  before running this version. (#289)
+
+- Darwin binaries are now universal (works on both amd64 and arm64), signed, and shipped in a notarized zip file.
+  `nebula-darwin.zip` will be the only darwin release artifact. (#571)
+
+- Darwin uses syscalls and AF_ROUTE to configure the routing table, instead of
+  using `/sbin/route`. Setting `tun.dev` is now allowed on Darwin as well, it
+  must be in the format `utun[0-9]+` or it will be ignored. (#163)
+
 ### Deprecated
 
 - The `preferred_ranges` option has been supported as a replacement for
@@ -45,10 +101,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   will immediately switch to a preferred remote address after the reception of
   a handshake packet (instead of waiting until 1,000 packets have been sent).
   (#532)
-  
+
 - A race condition when `punchy.respond` is enabled and ensures the correct
   vpn ip is sent a punch back response in highly queried node. (#566)
 
+- Fix a rare crash during handshake due to a race condition. (#535)
+
 ## [1.4.0] - 2021-05-11
 
 ### Added
@@ -287,7 +345,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Initial public release.
 
-[Unreleased]: https://github.com/slackhq/nebula/compare/v1.4.0...HEAD
+[Unreleased]: https://github.com/slackhq/nebula/compare/v1.5.2...HEAD
+[1.5.2]: https://github.com/slackhq/nebula/releases/tag/v1.5.2
+[1.5.0]: https://github.com/slackhq/nebula/releases/tag/v1.5.0
 [1.4.0]: https://github.com/slackhq/nebula/releases/tag/v1.4.0
 [1.3.0]: https://github.com/slackhq/nebula/releases/tag/v1.3.0
 [1.2.0]: https://github.com/slackhq/nebula/releases/tag/v1.2.0

Makefile

@@ -2,6 +2,8 @@ GOMINVERSION = 1.17
 NEBULA_CMD_PATH = "./cmd/nebula"
 GO111MODULE = on
 export GO111MODULE
+CGO_ENABLED = 0
+export CGO_ENABLED
 
 # Set up OS specific bits
 ifeq ($(OS),Windows_NT)

README.md

@@ -50,9 +50,9 @@ Nebula's user-defined groups allow for provider agnostic traffic filtering betwe
 Discovery nodes allow individual peers to find each other and optionally use UDP hole punching to establish connections from behind most firewalls or NATs.
 Users can move data between nodes in any number of cloud service providers, datacenters, and endpoints, without needing to maintain a particular addressing scheme.
 
-Nebula uses elliptic curve Diffie-Hellman key exchange, and AES-256-GCM in its default configuration.
+Nebula uses Elliptic-curve Diffie-Hellman (`ECDH`) key exchange and `AES-256-GCM` in its default configuration.
 
-Nebula was created to provide a mechanism for groups hosts to communicate securely, even across the internet, while enabling expressive firewall definitions similar in style to cloud security groups.
+Nebula was created to provide a mechanism for groups of hosts to communicate securely, even across the internet, while enabling expressive firewall definitions similar in style to cloud security groups.
 
 ## Getting started (quickly)
 

allow_list_test.go

@@ -7,12 +7,12 @@ import (
 
 	"github.com/slackhq/nebula/cidr"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/util"
+	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestNewAllowListFromConfig(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	c := config.NewC(l)
 	c.Settings["allowlist"] = map[interface{}]interface{}{
 		"192.168.0.0": true,

bits_test.go

@@ -3,12 +3,12 @@ package nebula
 import (
 	"testing"
 
-	"github.com/slackhq/nebula/util"
+	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestBits(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	b := NewBits(10)
 
 	// make sure it is the right size
@@ -76,7 +76,7 @@ func TestBits(t *testing.T) {
 }
 
 func TestBitsDupeCounter(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	b := NewBits(10)
 	b.lostCounter.Clear()
 	b.dupeCounter.Clear()
@@ -101,7 +101,7 @@ func TestBitsDupeCounter(t *testing.T) {
 }
 
 func TestBitsOutOfWindowCounter(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	b := NewBits(10)
 	b.lostCounter.Clear()
 	b.dupeCounter.Clear()
@@ -131,7 +131,7 @@ func TestBitsOutOfWindowCounter(t *testing.T) {
 }
 
 func TestBitsLostCounter(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	b := NewBits(10)
 	b.lostCounter.Clear()
 	b.dupeCounter.Clear()

cert.go

@@ -124,19 +124,13 @@ func loadCAFromConfig(l *logrus.Logger, c *config.C) (*cert.NebulaCAPool, error)
 	var err error
 
 	caPathOrPEM := c.GetString("pki.ca", "")
-	if caPathOrPEM == "" {
-		// Support backwards compat with the old x509
-		//TODO: remove after this is rolled out everywhere - NB 2018/02/23
-		caPathOrPEM = c.GetString("x509.ca", "")
-	}
-
 	if caPathOrPEM == "" {
 		return nil, errors.New("no pki.ca path or PEM data provided")
 	}
 
 	if strings.Contains(caPathOrPEM, "-----BEGIN") {
 		rawCA = []byte(caPathOrPEM)
-		caPathOrPEM = "<inline>"
+
 	} else {
 		rawCA, err = ioutil.ReadFile(caPathOrPEM)
 		if err != nil {
@@ -145,7 +139,20 @@ func loadCAFromConfig(l *logrus.Logger, c *config.C) (*cert.NebulaCAPool, error)
 	}
 
 	CAs, err := cert.NewCAPoolFromBytes(rawCA)
-	if err != nil {
+	if errors.Is(err, cert.ErrExpired) {
+		var expired int
+		for _, cert := range CAs.CAs {
+			if cert.Expired(time.Now()) {
+				expired++
+				l.WithField("cert", cert).Warn("expired certificate present in CA pool")
+			}
+		}
+
+		if expired >= len(CAs.CAs) {
+			return nil, errors.New("no valid CA certificates present")
+		}
+
+	} else if err != nil {
 		return nil, fmt.Errorf("error while adding CA certificate to CA trust store: %s", err)
 	}
 
@@ -154,7 +161,8 @@ func loadCAFromConfig(l *logrus.Logger, c *config.C) (*cert.NebulaCAPool, error)
 		CAs.BlocklistFingerprint(fp)
 	}
 
-	// Support deprecated config for at leaast one minor release to allow for migrations
+	// Support deprecated config for at least one minor release to allow for migrations
+	//TODO: remove in 2022 or later
 	for _, fp := range c.GetStringSlice("pki.blacklist", []string{}) {
 		l.WithField("fingerprint", fp).Infof("Blocklisting cert")
 		l.Warn("pki.blacklist is deprecated and will not be supported in a future release. Please migrate your config to use pki.blocklist")

cert/ca.go

@@ -1,6 +1,7 @@
 package cert
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -21,19 +22,32 @@ func NewCAPool() *NebulaCAPool {
 	return &ca
 }
 
+// NewCAPoolFromBytes will create a new CA pool from the provided
+// input bytes, which must be a PEM-encoded set of nebula certificates.
+// If the pool contains any expired certificates, an ErrExpired will be
+// returned along with the pool. The caller must handle any such errors.
 func NewCAPoolFromBytes(caPEMs []byte) (*NebulaCAPool, error) {
 	pool := NewCAPool()
 	var err error
+	var expired bool
 	for {
 		caPEMs, err = pool.AddCACertificate(caPEMs)
+		if errors.Is(err, ErrExpired) {
+			expired = true
+			err = nil
+		}
 		if err != nil {
 			return nil, err
 		}
-		if caPEMs == nil || len(caPEMs) == 0 || strings.TrimSpace(string(caPEMs)) == "" {
+		if len(caPEMs) == 0 || strings.TrimSpace(string(caPEMs)) == "" {
 			break
 		}
 	}
 
+	if expired {
+		return pool, ErrExpired
+	}
+
 	return pool, nil
 }
 
@@ -47,15 +61,11 @@ func (ncp *NebulaCAPool) AddCACertificate(pemBytes []byte) ([]byte, error) {
 	}
 
 	if !c.Details.IsCA {
-		return pemBytes, fmt.Errorf("provided certificate was not a CA; %s", c.Details.Name)
+		return pemBytes, fmt.Errorf("%s: %w", c.Details.Name, ErrNotCA)
 	}
 
 	if !c.CheckSignature(c.Details.PublicKey) {
-		return pemBytes, fmt.Errorf("provided certificate was not self signed; %s", c.Details.Name)
-	}
-
-	if c.Expired(time.Now()) {
-		return pemBytes, fmt.Errorf("provided CA certificate is expired; %s", c.Details.Name)
+		return pemBytes, fmt.Errorf("%s: %w", c.Details.Name, ErrNotSelfSigned)
 	}
 
 	sum, err := c.Sha256Sum()
@@ -64,6 +74,10 @@ func (ncp *NebulaCAPool) AddCACertificate(pemBytes []byte) ([]byte, error) {
 	}
 
 	ncp.CAs[sum] = c
+	if c.Expired(time.Now()) {
+		return pemBytes, fmt.Errorf("%s: %w", c.Details.Name, ErrExpired)
+	}
+
 	return pemBytes, nil
 }
 

cert/cert_test.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
-	"github.com/slackhq/nebula/util"
+	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/crypto/curve25519"
 	"golang.org/x/crypto/ed25519"
@@ -429,6 +429,15 @@ BVG+oJpAoqokUBbI4U0N8CSfpUABEkB/Pm5A2xyH/nc8mg/wvGUWG3pZ7nHzaDMf
 8/phAUt+FLzqTECzQKisYswKvE3pl9mbEYKbOdIHrxdIp95mo4sF
 -----END NEBULA CERTIFICATE-----
 
+`
+
+	expired := `
+# expired certificate
+-----BEGIN NEBULA CERTIFICATE-----
+CjkKB2V4cGlyZWQouPmWjQYwufmWjQY6ILCRaoCkJlqHgv5jfDN4lzLHBvDzaQm4
+vZxfu144hmgjQAESQG4qlnZi8DncvD/LDZnLgJHOaX1DWCHHEh59epVsC+BNgTie
+WH1M9n4O7cFtGlM6sJJOS+rCVVEJ3ABS7+MPdQs=
+-----END NEBULA CERTIFICATE-----
 `
 
 	rootCA := NebulaCertificate{
@@ -452,6 +461,19 @@ BVG+oJpAoqokUBbI4U0N8CSfpUABEkB/Pm5A2xyH/nc8mg/wvGUWG3pZ7nHzaDMf
 	assert.Nil(t, err)
 	assert.Equal(t, pp.CAs[string("c9bfaf7ce8e84b2eeda2e27b469f4b9617bde192efd214b68891ecda6ed49522")].Details.Name, rootCA.Details.Name)
 	assert.Equal(t, pp.CAs[string("5c9c3f23e7ee7fe97637cbd3a0a5b854154d1d9aaaf7b566a51f4a88f76b64cd")].Details.Name, rootCA01.Details.Name)
+
+	// expired cert, no valid certs
+	ppp, err := NewCAPoolFromBytes([]byte(expired))
+	assert.Equal(t, ErrExpired, err)
+	assert.Equal(t, ppp.CAs[string("152070be6bb19bc9e3bde4c2f0e7d8f4ff5448b4c9856b8eccb314fade0229b0")].Details.Name, "expired")
+
+	// expired cert, with valid certs
+	pppp, err := NewCAPoolFromBytes(append([]byte(expired), noNewLines...))
+	assert.Equal(t, ErrExpired, err)
+	assert.Equal(t, pppp.CAs[string("c9bfaf7ce8e84b2eeda2e27b469f4b9617bde192efd214b68891ecda6ed49522")].Details.Name, rootCA.Details.Name)
+	assert.Equal(t, pppp.CAs[string("5c9c3f23e7ee7fe97637cbd3a0a5b854154d1d9aaaf7b566a51f4a88f76b64cd")].Details.Name, rootCA01.Details.Name)
+	assert.Equal(t, pppp.CAs[string("152070be6bb19bc9e3bde4c2f0e7d8f4ff5448b4c9856b8eccb314fade0229b0")].Details.Name, "expired")
+	assert.Equal(t, len(pppp.CAs), 3)
 }
 
 func appendByteSlices(b ...[]byte) []byte {
@@ -752,7 +774,7 @@ func TestNebulaCertificate_Copy(t *testing.T) {
 	assert.Nil(t, err)
 	cc := c.Copy()
 
-	util.AssertDeepCopyEqual(t, c, cc)
+	test.AssertDeepCopyEqual(t, c, cc)
 }
 
 func TestUnmarshalNebulaCertificate(t *testing.T) {

cert/errors.go

@@ -0,0 +1,9 @@
+package cert
+
+import "errors"
+
+var (
+	ErrExpired       = errors.New("certificate is expired")
+	ErrNotCA         = errors.New("certificate is not a CA")
+	ErrNotSelfSigned = errors.New("certificate is not self-signed")
+)

cmd/nebula-cert/ca.go

@@ -37,8 +37,8 @@ func newCaFlags() *caFlags {
 	cf.outCertPath = cf.set.String("out-crt", "ca.crt", "Optional: path to write the certificate to")
 	cf.outQRPath = cf.set.String("out-qr", "", "Optional: output a qr code image (png) of the certificate")
 	cf.groups = cf.set.String("groups", "", "Optional: comma separated list of groups. This will limit which groups subordinate certs can use")
-	cf.ips = cf.set.String("ips", "", "Optional: comma separated list of ip and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use")
-	cf.subnets = cf.set.String("subnets", "", "Optional: comma separated list of ip and network in CIDR notation. This will limit which subnet addresses and networks subordinate certs can use")
+	cf.ips = cf.set.String("ips", "", "Optional: comma separated list of ipv4 address and network in CIDR notation. This will limit which ipv4 addresses and networks subordinate certs can use for ip addresses")
+	cf.subnets = cf.set.String("subnets", "", "Optional: comma separated list of ipv4 address and network in CIDR notation. This will limit which ipv4 addresses and networks subordinate certs can use in subnets")
 	return &cf
 }
 
@@ -82,6 +82,9 @@ func ca(args []string, out io.Writer, errOut io.Writer) error {
 				if err != nil {
 					return newHelpErrorf("invalid ip definition: %s", err)
 				}
+				if ip.To4() == nil {
+					return newHelpErrorf("invalid ip definition: can only be ipv4, have %s", rs)
+				}
 
 				ipNet.IP = ip
 				ips = append(ips, ipNet)
@@ -98,6 +101,9 @@ func ca(args []string, out io.Writer, errOut io.Writer) error {
 				if err != nil {
 					return newHelpErrorf("invalid subnet definition: %s", err)
 				}
+				if s.IP.To4() == nil {
+					return newHelpErrorf("invalid subnet definition: can only be ipv4, have %s", rs)
+				}
 				subnets = append(subnets, s)
 			}
 		}

cmd/nebula-cert/ca_test.go

@@ -31,7 +31,7 @@ func Test_caHelp(t *testing.T) {
 			"  -groups string\n"+
 			"    \tOptional: comma separated list of groups. This will limit which groups subordinate certs can use\n"+
 			"  -ips string\n"+
-			"    \tOptional: comma separated list of ip and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use\n"+
+			"    \tOptional: comma separated list of ipv4 address and network in CIDR notation. This will limit which ipv4 addresses and networks subordinate certs can use for ip addresses\n"+
 			"  -name string\n"+
 			"    \tRequired: name of the certificate authority\n"+
 			"  -out-crt string\n"+
@@ -41,7 +41,7 @@ func Test_caHelp(t *testing.T) {
 			"  -out-qr string\n"+
 			"    \tOptional: output a qr code image (png) of the certificate\n"+
 			"  -subnets string\n"+
-			"    \tOptional: comma separated list of ip and network in CIDR notation. This will limit which subnet addresses and networks subordinate certs can use\n",
+			"    \tOptional: comma separated list of ipv4 address and network in CIDR notation. This will limit which ipv4 addresses and networks subordinate certs can use in subnets\n",
 		ob.String(),
 	)
 }
@@ -55,6 +55,16 @@ func Test_ca(t *testing.T) {
 	assert.Equal(t, "", ob.String())
 	assert.Equal(t, "", eb.String())
 
+	// ipv4 only ips
+	assertHelpError(t, ca([]string{"-name", "ipv6", "-ips", "100::100/100"}, ob, eb), "invalid ip definition: can only be ipv4, have 100::100/100")
+	assert.Equal(t, "", ob.String())
+	assert.Equal(t, "", eb.String())
+
+	// ipv4 only subnets
+	assertHelpError(t, ca([]string{"-name", "ipv6", "-subnets", "100::100/100"}, ob, eb), "invalid subnet definition: can only be ipv4, have 100::100/100")
+	assert.Equal(t, "", ob.String())
+	assert.Equal(t, "", eb.String())
+
 	// failed key write
 	ob.Reset()
 	eb.Reset()

cmd/nebula-cert/sign.go

@@ -37,14 +37,14 @@ func newSignFlags() *signFlags {
 	sf.caKeyPath = sf.set.String("ca-key", "ca.key", "Optional: path to the signing CA key")
 	sf.caCertPath = sf.set.String("ca-crt", "ca.crt", "Optional: path to the signing CA cert")
 	sf.name = sf.set.String("name", "", "Required: name of the cert, usually a hostname")
-	sf.ip = sf.set.String("ip", "", "Required: ip and network in CIDR notation to assign the cert")
+	sf.ip = sf.set.String("ip", "", "Required: ipv4 address and network in CIDR notation to assign the cert")
 	sf.duration = sf.set.Duration("duration", 0, "Optional: how long the cert should be valid for. The default is 1 second before the signing cert expires. Valid time units are seconds: \"s\", minutes: \"m\", hours: \"h\"")
 	sf.inPubPath = sf.set.String("in-pub", "", "Optional (if out-key not set): path to read a previously generated public key")
 	sf.outKeyPath = sf.set.String("out-key", "", "Optional (if in-pub not set): path to write the private key to")
 	sf.outCertPath = sf.set.String("out-crt", "", "Optional: path to write the certificate to")
 	sf.outQRPath = sf.set.String("out-qr", "", "Optional: output a qr code image (png) of the certificate")
 	sf.groups = sf.set.String("groups", "", "Optional: comma separated list of groups")
-	sf.subnets = sf.set.String("subnets", "", "Optional: comma separated list of subnet this cert can serve for")
+	sf.subnets = sf.set.String("subnets", "", "Optional: comma separated list of ipv4 address and network in CIDR notation. Subnets this cert can serve for")
 	return &sf
 
 }
@@ -114,6 +114,9 @@ func signCert(args []string, out io.Writer, errOut io.Writer) error {
 	if err != nil {
 		return newHelpErrorf("invalid ip definition: %s", err)
 	}
+	if ip.To4() == nil {
+		return newHelpErrorf("invalid ip definition: can only be ipv4, have %s", *sf.ip)
+	}
 	ipNet.IP = ip
 
 	groups := []string{}
@@ -135,6 +138,9 @@ func signCert(args []string, out io.Writer, errOut io.Writer) error {
 				if err != nil {
 					return newHelpErrorf("invalid subnet definition: %s", err)
 				}
+				if s.IP.To4() == nil {
+					return newHelpErrorf("invalid subnet definition: can only be ipv4, have %s", rs)
+				}
 				subnets = append(subnets, s)
 			}
 		}

cmd/nebula-cert/sign_test.go

@@ -39,7 +39,7 @@ func Test_signHelp(t *testing.T) {
 			"  -in-pub string\n"+
 			"    \tOptional (if out-key not set): path to read a previously generated public key\n"+
 			"  -ip string\n"+
-			"    \tRequired: ip and network in CIDR notation to assign the cert\n"+
+			"    \tRequired: ipv4 address and network in CIDR notation to assign the cert\n"+
 			"  -name string\n"+
 			"    \tRequired: name of the cert, usually a hostname\n"+
 			"  -out-crt string\n"+
@@ -49,7 +49,7 @@ func Test_signHelp(t *testing.T) {
 			"  -out-qr string\n"+
 			"    \tOptional: output a qr code image (png) of the certificate\n"+
 			"  -subnets string\n"+
-			"    \tOptional: comma separated list of subnet this cert can serve for\n",
+			"    \tOptional: comma separated list of ipv4 address and network in CIDR notation. Subnets this cert can serve for\n",
 		ob.String(),
 	)
 }
@@ -59,7 +59,6 @@ func Test_signCert(t *testing.T) {
 	eb := &bytes.Buffer{}
 
 	// required args
-
 	assertHelpError(t, signCert([]string{"-ca-crt", "./nope", "-ca-key", "./nope", "-ip", "1.1.1.1/24", "-out-key", "nope", "-out-crt", "nope"}, ob, eb), "-name is required")
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
@@ -160,6 +159,13 @@ func Test_signCert(t *testing.T) {
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
 
+	ob.Reset()
+	eb.Reset()
+	args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "100::100/100", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"}
+	assertHelpError(t, signCert(args, ob, eb), "invalid ip definition: can only be ipv4, have 100::100/100")
+	assert.Empty(t, ob.String())
+	assert.Empty(t, eb.String())
+
 	// bad subnet cidr
 	ob.Reset()
 	eb.Reset()
@@ -168,6 +174,13 @@ func Test_signCert(t *testing.T) {
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
 
+	ob.Reset()
+	eb.Reset()
+	args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m", "-subnets", "100::100/100"}
+	assertHelpError(t, signCert(args, ob, eb), "invalid subnet definition: can only be ipv4, have 100::100/100")
+	assert.Empty(t, ob.String())
+	assert.Empty(t, eb.String())
+
 	// mismatched ca key
 	_, caPriv2, _ := ed25519.GenerateKey(rand.Reader)
 	caKeyF2, err := ioutil.TempFile("", "sign-cert-2.key")

cmd/nebula-cert/verify_test.go

@@ -72,7 +72,7 @@ func Test_verify(t *testing.T) {
 		Details: cert.NebulaCertificateDetails{
 			Name:      "test-ca",
 			NotBefore: time.Now().Add(time.Hour * -1),
-			NotAfter:  time.Now().Add(time.Hour),
+			NotAfter:  time.Now().Add(time.Hour * 2),
 			PublicKey: caPub,
 			IsCA:      true,
 		},

cmd/nebula-service/main.go

@@ -8,6 +8,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/util"
 )
 
 // A version string that can be set with
@@ -60,7 +61,7 @@ func main() {
 	ctrl, err := nebula.Main(c, *configTest, Build, l, nil)
 
 	switch v := err.(type) {
-	case nebula.ContextualError:
+	case util.ContextualError:
 		v.Log(l)
 		os.Exit(1)
 	case error:

cmd/nebula/main.go

@@ -8,6 +8,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/util"
 )
 
 // A version string that can be set with
@@ -54,7 +55,7 @@ func main() {
 	ctrl, err := nebula.Main(c, *configTest, Build, l, nil)
 
 	switch v := err.(type) {
-	case nebula.ContextualError:
+	case util.ContextualError:
 		v.Log(l)
 		os.Exit(1)
 	case error:

config/config_test.go

@@ -7,12 +7,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/slackhq/nebula/util"
+	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestConfig_Load(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	dir, err := ioutil.TempDir("", "config-test")
 	// invalid yaml
 	c := NewC(l)
@@ -42,7 +42,7 @@ func TestConfig_Load(t *testing.T) {
 }
 
 func TestConfig_Get(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	// test simple type
 	c := NewC(l)
 	c.Settings["firewall"] = map[interface{}]interface{}{"outbound": "hi"}
@@ -58,14 +58,14 @@ func TestConfig_Get(t *testing.T) {
 }
 
 func TestConfig_GetStringSlice(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	c := NewC(l)
 	c.Settings["slice"] = []interface{}{"one", "two"}
 	assert.Equal(t, []string{"one", "two"}, c.GetStringSlice("slice", []string{}))
 }
 
 func TestConfig_GetBool(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	c := NewC(l)
 	c.Settings["bool"] = true
 	assert.Equal(t, true, c.GetBool("bool", false))
@@ -93,7 +93,7 @@ func TestConfig_GetBool(t *testing.T) {
 }
 
 func TestConfig_HasChanged(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	// No reload has occurred, return false
 	c := NewC(l)
 	c.Settings["test"] = "hi"
@@ -115,7 +115,7 @@ func TestConfig_HasChanged(t *testing.T) {
 }
 
 func TestConfig_ReloadConfig(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	done := make(chan bool, 1)
 	dir, err := ioutil.TempDir("", "config-test")
 	assert.Nil(t, err)

connection_manager_test.go

@@ -11,15 +11,15 @@ import (
 	"github.com/flynn/noise"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/test"
 	"github.com/slackhq/nebula/udp"
-	"github.com/slackhq/nebula/util"
 	"github.com/stretchr/testify/assert"
 )
 
 var vpnIp iputil.VpnIp
 
 func Test_NewConnectionManagerTest(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	//_, tuncidr, _ := net.ParseCIDR("1.1.1.1/24")
 	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
@@ -38,7 +38,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	lh := NewLightHouse(l, false, &net.IPNet{IP: net.IP{0, 0, 0, 0}, Mask: net.IPMask{0, 0, 0, 0}}, []iputil.VpnIp{}, 1000, 0, &udp.Conn{}, false, 1, false)
 	ifce := &Interface{
 		hostMap:          hostMap,
-		inside:           &Tun{},
+		inside:           &test.NoopTun{},
 		outside:          &udp.Conn{},
 		certState:        cs,
 		firewall:         &Firewall{},
@@ -57,7 +57,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	out := make([]byte, mtu)
 	nc.HandleMonitorTick(now, p, nb, out)
 	// Add an ip we have established a connection w/ to hostmap
-	hostinfo := nc.hostMap.AddVpnIp(vpnIp)
+	hostinfo, _ := nc.hostMap.AddVpnIp(vpnIp, nil)
 	hostinfo.ConnectionState = &ConnectionState{
 		certState: cs,
 		H:         &noise.HandshakeState{},
@@ -89,7 +89,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 }
 
 func Test_NewConnectionManagerTest2(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	//_, tuncidr, _ := net.ParseCIDR("1.1.1.1/24")
 	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
@@ -107,7 +107,7 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	lh := NewLightHouse(l, false, &net.IPNet{IP: net.IP{0, 0, 0, 0}, Mask: net.IPMask{0, 0, 0, 0}}, []iputil.VpnIp{}, 1000, 0, &udp.Conn{}, false, 1, false)
 	ifce := &Interface{
 		hostMap:          hostMap,
-		inside:           &Tun{},
+		inside:           &test.NoopTun{},
 		outside:          &udp.Conn{},
 		certState:        cs,
 		firewall:         &Firewall{},
@@ -126,7 +126,7 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	out := make([]byte, mtu)
 	nc.HandleMonitorTick(now, p, nb, out)
 	// Add an ip we have established a connection w/ to hostmap
-	hostinfo := nc.hostMap.AddVpnIp(vpnIp)
+	hostinfo, _ := nc.hostMap.AddVpnIp(vpnIp, nil)
 	hostinfo.ConnectionState = &ConnectionState{
 		certState: cs,
 		H:         &noise.HandshakeState{},
@@ -164,7 +164,7 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 // Disconnect only if disconnectInvalid: true is set.
 func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 	now := time.Now()
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	ipNet := net.IPNet{
 		IP:   net.IPv4(172, 1, 1, 2),
 		Mask: net.IPMask{255, 255, 255, 0},
@@ -216,7 +216,7 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 	lh := NewLightHouse(l, false, &net.IPNet{IP: net.IP{0, 0, 0, 0}, Mask: net.IPMask{0, 0, 0, 0}}, []iputil.VpnIp{}, 1000, 0, &udp.Conn{}, false, 1, false)
 	ifce := &Interface{
 		hostMap:           hostMap,
-		inside:            &Tun{},
+		inside:            &test.NoopTun{},
 		outside:           &udp.Conn{},
 		certState:         cs,
 		firewall:          &Firewall{},
@@ -232,7 +232,7 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 	defer cancel()
 	nc := newConnectionManager(ctx, l, ifce, 5, 10)
 	ifce.connectionManager = nc
-	hostinfo := nc.hostMap.AddVpnIp(vpnIp)
+	hostinfo, _ := nc.hostMap.AddVpnIp(vpnIp, nil)
 	hostinfo.ConnectionState = &ConnectionState{
 		certState: cs,
 		peerCert:  &peerCert,

connection_state.go

@@ -27,7 +27,7 @@ type ConnectionState struct {
 	ready                bool
 }
 
-func (f *Interface) newConnectionState(l *logrus.Logger, initiator bool, p []byte) (*ConnectionState, error) {
+func (f *Interface) newConnectionState(l *logrus.Logger, initiator bool, pattern noise.HandshakePattern, psk []byte, pskStage int) *ConnectionState {
 	cs := noise.NewCipherSuite(noise.DH25519, noise.CipherAESGCM, noise.HashSHA256)
 	if f.cipher == "chachapoly" {
 		cs = noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256)
@@ -43,15 +43,14 @@ func (f *Interface) newConnectionState(l *logrus.Logger, initiator bool, p []byt
 	hs, err := noise.NewHandshakeState(noise.Config{
 		CipherSuite:           cs,
 		Random:                rand.Reader,
-		Pattern:               noise.HandshakeIX,
+		Pattern:               pattern,
 		Initiator:             initiator,
 		StaticKeypair:         static,
-		PresharedKey:          p,
-		PresharedKeyPlacement: 0,
+		PresharedKey:          psk,
+		PresharedKeyPlacement: pskStage,
 	})
-
 	if err != nil {
-		return nil, err
+		return nil
 	}
 
 	// The queue and ready params prevent a counter race that would happen when
@@ -64,7 +63,7 @@ func (f *Interface) newConnectionState(l *logrus.Logger, initiator bool, p []byt
 		certState: curCertState,
 	}
 
-	return ci, nil
+	return ci
 }
 
 func (cs *ConnectionState) MarshalJSON() ([]byte, error) {

control.go

@@ -62,6 +62,9 @@ func (c *Control) Start() {
 func (c *Control) Stop() {
 	//TODO: stop tun and udp routines, the lock on hostMap effectively does that though
 	c.CloseAllTunnels(false)
+	if err := c.f.Close(); err != nil {
+		c.l.WithError(err).Error("Close interface failed")
+	}
 	c.cancel()
 	c.l.Info("Goodbye")
 }

control_test.go

@@ -9,13 +9,13 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/test"
 	"github.com/slackhq/nebula/udp"
-	"github.com/slackhq/nebula/util"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestControl_GetHostInfoByVpnIp(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	// Special care must be taken to re-use all objects provided to the hostmap and certificate in the expectedInfo object
 	// To properly ensure we are not exposing core memory to the caller
 	hm := NewHostMap(l, "test", &net.IPNet{}, make([]*net.IPNet, 0))
@@ -94,7 +94,7 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 
 	// Make sure we don't have any unexpected fields
 	assertFields(t, []string{"VpnIp", "LocalIndex", "RemoteIndex", "RemoteAddrs", "CachedPackets", "Cert", "MessageCounter", "CurrentRemote"}, thi)
-	util.AssertDeepCopyEqual(t, &expectedInfo, thi)
+	test.AssertDeepCopyEqual(t, &expectedInfo, thi)
 
 	// Make sure we don't panic if the host info doesn't have a cert yet
 	assert.NotPanics(t, func() {

control_tester.go

@@ -10,6 +10,7 @@ import (
 	"github.com/google/gopacket/layers"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/udp"
 )
 
@@ -64,7 +65,7 @@ func (c *Control) InjectLightHouseAddr(vpnIp net.IP, toAddr *net.UDPAddr) {
 
 // GetFromTun will pull a packet off the tun side of nebula
 func (c *Control) GetFromTun(block bool) []byte {
-	return c.f.inside.(*Tun).Get(block)
+	return c.f.inside.(*overlay.TestTun).Get(block)
 }
 
 // GetFromUDP will pull a udp packet off the udp side of nebula
@@ -77,7 +78,7 @@ func (c *Control) GetUDPTxChan() <-chan *udp.Packet {
 }
 
 func (c *Control) GetTunTxChan() <-chan []byte {
-	return c.f.inside.(*Tun).txPackets
+	return c.f.inside.(*overlay.TestTun).TxPackets
 }
 
 // InjectUDPPacket will inject a packet into the udp side of nebula
@@ -91,7 +92,7 @@ func (c *Control) InjectTunUDPPacket(toIp net.IP, toPort uint16, fromPort uint16
 		Version:  4,
 		TTL:      64,
 		Protocol: layers.IPProtocolUDP,
-		SrcIP:    c.f.inside.CidrNet().IP,
+		SrcIP:    c.f.inside.Cidr().IP,
 		DstIP:    toIp,
 	}
 
@@ -114,7 +115,7 @@ func (c *Control) InjectTunUDPPacket(toIp net.IP, toPort uint16, fromPort uint16
 		panic(err)
 	}
 
-	c.f.inside.(*Tun).Send(buffer.Bytes())
+	c.f.inside.(*overlay.TestTun).Send(buffer.Bytes())
 }
 
 func (c *Control) GetUDPAddr() string {

dist/windows/wintun/LICENSE.txt

@@ -0,0 +1,84 @@
+Prebuilt Binaries License
+-------------------------
+
+1. DEFINITIONS. "Software" means the precise contents of the "wintun.dll"
+   files that are included in the .zip file that contains this document as
+   downloaded from wintun.net/builds.
+
+2. LICENSE GRANT. WireGuard LLC grants to you a non-exclusive and
+   non-transferable right to use Software for lawful purposes under certain
+   obligations and limited rights as set forth in this agreement.
+
+3. RESTRICTIONS. Software is owned and copyrighted by WireGuard LLC. It is
+   licensed, not sold. Title to Software and all associated intellectual
+   property rights are retained by WireGuard. You must not:
+   a. reverse engineer, decompile, disassemble, extract from, or otherwise
+      modify the Software;
+   b. modify or create derivative work based upon Software in whole or in
+      parts, except insofar as only the API interfaces of the "wintun.h" file
+      distributed alongside the Software (the "Permitted API") are used;
+   c. remove any proprietary notices, labels, or copyrights from the Software;
+   d. resell, redistribute, lease, rent, transfer, sublicense, or otherwise
+      transfer rights of the Software without the prior written consent of
+      WireGuard LLC, except insofar as the Software is distributed alongside
+      other software that uses the Software only via the Permitted API;
+   e. use the name of WireGuard LLC, the WireGuard project, the Wintun
+      project, or the names of its contributors to endorse or promote products
+      derived from the Software without specific prior written consent.
+
+4. LIMITED WARRANTY. THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTY OF
+   ANY KIND. WIREGUARD LLC HEREBY EXCLUDES AND DISCLAIMS ALL IMPLIED OR
+   STATUTORY WARRANTIES, INCLUDING ANY WARRANTIES OF MERCHANTABILITY, FITNESS
+   FOR A PARTICULAR PURPOSE, QUALITY, NON-INFRINGEMENT, TITLE, RESULTS,
+   EFFORTS, OR QUIET ENJOYMENT. THERE IS NO WARRANTY THAT THE PRODUCT WILL BE
+   ERROR-FREE OR WILL FUNCTION WITHOUT INTERRUPTION. YOU ASSUME THE ENTIRE
+   RISK FOR THE RESULTS OBTAINED USING THE PRODUCT. TO THE EXTENT THAT
+   WIREGUARD LLC MAY NOT DISCLAIM ANY WARRANTY AS A MATTER OF APPLICABLE LAW,
+   THE SCOPE AND DURATION OF SUCH WARRANTY WILL BE THE MINIMUM PERMITTED UNDER
+   SUCH LAW. ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND
+   WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR
+   A PARTICULAR PURPOSE OR NON-INFRINGEMENT ARE DISCLAIMED, EXCEPT TO THE
+   EXTENT THAT THESE DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
+
+5. LIMITATION OF LIABILITY. To the extent not prohibited by law, in no event
+   WireGuard LLC or any third-party-developer will be liable for any lost
+   revenue, profit or data or for special, indirect, consequential, incidental
+   or punitive damages, however caused regardless of the theory of liability,
+   arising out of or related to the use of or inability to use Software, even
+   if WireGuard LLC has been advised of the possibility of such damages.
+   Solely you are responsible for determining the appropriateness of using
+   Software and accept full responsibility for all risks associated with its
+   exercise of rights under this agreement, including but not limited to the
+   risks and costs of program errors, compliance with applicable laws, damage
+   to or loss of data, programs or equipment, and unavailability or
+   interruption of operations. The foregoing limitations will apply even if
+   the above stated warranty fails of its essential purpose. You acknowledge,
+   that it is in the nature of software that software is complex and not
+   completely free of errors. In no event shall WireGuard LLC or any
+   third-party-developer be liable to you under any theory for any damages
+   suffered by you or any user of Software or for any special, incidental,
+   indirect, consequential or similar damages (including without limitation
+   damages for loss of business profits, business interruption, loss of
+   business information or any other pecuniary loss) arising out of the use or
+   inability to use Software, even if WireGuard LLC has been advised of the
+   possibility of such damages and regardless of the legal or quitable theory
+   (contract, tort, or otherwise) upon which the claim is based.
+
+6. TERMINATION. This agreement is affected until terminated. You may
+   terminate this agreement at any time. This agreement will terminate
+   immediately without notice from WireGuard LLC if you fail to comply with
+   the terms and conditions of this agreement. Upon termination, you must
+   delete Software and all copies of Software and cease all forms of
+   distribution of Software.
+
+7. SEVERABILITY. If any provision of this agreement is held to be
+   unenforceable, this agreement will remain in effect with the provision
+   omitted, unless omission would frustrate the intent of the parties, in
+   which case this agreement will immediately terminate.
+
+8. RESERVATION OF RIGHTS. All rights not expressly granted in this agreement
+   are reserved by WireGuard LLC. For example, WireGuard LLC reserves the
+   right at any time to cease development of Software, to alter distribution
+   details, features, specifications, capabilities, functions, licensing
+   terms, release dates, APIs, ABIs, general availability, or other
+   characteristics of the Software.

dist/windows/wintun/README.md

@@ -0,0 +1,339 @@
+# [Wintun Network Adapter](https://www.wintun.net/)
+### TUN Device Driver for Windows
+
+This is a layer 3 TUN driver for Windows 7, 8, 8.1, and 10. Originally created for [WireGuard](https://www.wireguard.com/), it is intended to be useful to a wide variety of projects that require layer 3 tunneling devices with implementations primarily in userspace.
+
+## Installation
+
+Wintun is deployed as a platform-specific `wintun.dll` file. Install the `wintun.dll` file side-by-side with your application. Download the dll from [wintun.net](https://www.wintun.net/), alongside the header file for your application described below.
+
+## Usage
+
+Include the [`wintun.h` file](https://git.zx2c4.com/wintun/tree/api/wintun.h) in your project simply by copying it there and dynamically load the `wintun.dll` using [`LoadLibraryEx()`](https://docs.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-loadlibraryexa) and [`GetProcAddress()`](https://docs.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-getprocaddress) to resolve each function, using the typedefs provided in the header file. The [`InitializeWintun` function in the example.c code](https://git.zx2c4.com/wintun/tree/example/example.c) provides this in a function that you can simply copy and paste.
+
+With the library setup, Wintun can then be used by first creating an adapter, configuring it, and then setting its status to "up". Adapters have names (e.g. "OfficeNet") and types (e.g. "Wintun").
+
+```C
+WINTUN_ADAPTER_HANDLE Adapter1 = WintunCreateAdapter(L"OfficeNet", L"Wintun", &SomeFixedGUID1);
+WINTUN_ADAPTER_HANDLE Adapter2 = WintunCreateAdapter(L"HomeNet", L"Wintun", &SomeFixedGUID2);
+WINTUN_ADAPTER_HANDLE Adapter3 = WintunCreateAdapter(L"Data Center", L"Wintun", &SomeFixedGUID3);
+```
+
+After creating an adapter, we can use it by starting a session:
+
+```C
+WINTUN_SESSION_HANDLE Session = WintunStartSession(Adapter2, 0x400000);
+```
+
+Then, the `WintunAllocateSendPacket` and `WintunSendPacket` functions can be used for sending packets ([used by `SendPackets` in the example.c code](https://git.zx2c4.com/wintun/tree/example/example.c)):
+
+```C
+BYTE *OutgoingPacket = WintunAllocateSendPacket(Session, PacketDataSize);
+if (OutgoingPacket)
+{
+    memcpy(OutgoingPacket, PacketData, PacketDataSize);
+    WintunSendPacket(Session, OutgoingPacket);
+}
+else if (GetLastError() != ERROR_BUFFER_OVERFLOW) // Silently drop packets if the ring is full
+    Log(L"Packet write failed");
+```
+
+And the `WintunReceivePacket` and `WintunReleaseReceivePacket` functions can be used for receiving packets ([used by `ReceivePackets` in the example.c code](https://git.zx2c4.com/wintun/tree/example/example.c)):
+
+```C
+for (;;)
+{
+    DWORD IncomingPacketSize;
+    BYTE *IncomingPacket = WintunReceivePacket(Session, &IncomingPacketSize);
+    if (IncomingPacket)
+    {
+        DoSomethingWithPacket(IncomingPacket, IncomingPacketSize);
+        WintunReleaseReceivePacket(Session, IncomingPacket);
+    }
+    else if (GetLastError() == ERROR_NO_MORE_ITEMS)
+        WaitForSingleObject(WintunGetReadWaitEvent(Session), INFINITE);
+    else
+    {
+        Log(L"Packet read failed");
+        break;
+    }
+}
+```
+
+Some high performance use cases may want to spin on `WintunReceivePackets` for a number of cycles before falling back to waiting on the read-wait event.
+
+You are **highly encouraged** to read the [**example.c short example**](https://git.zx2c4.com/wintun/tree/example/example.c) to see how to put together a simple userspace network tunnel.
+
+The various functions and definitions are [documented in the reference below](#Reference).
+
+## Reference
+
+### Macro Definitions
+
+#### WINTUN\_MAX\_POOL
+
+`#define WINTUN_MAX_POOL   256`
+
+Maximum pool name length including zero terminator
+
+#### WINTUN\_MIN\_RING\_CAPACITY
+
+`#define WINTUN_MIN_RING_CAPACITY   0x20000 /* 128kiB */`
+
+Minimum ring capacity.
+
+#### WINTUN\_MAX\_RING\_CAPACITY
+
+`#define WINTUN_MAX_RING_CAPACITY   0x4000000 /* 64MiB */`
+
+Maximum ring capacity.
+
+#### WINTUN\_MAX\_IP\_PACKET\_SIZE
+
+`#define WINTUN_MAX_IP_PACKET_SIZE   0xFFFF`
+
+Maximum IP packet size
+
+### Typedefs
+
+#### WINTUN\_ADAPTER\_HANDLE
+
+`typedef void* WINTUN_ADAPTER_HANDLE`
+
+A handle representing Wintun adapter
+
+#### WINTUN\_ENUM\_CALLBACK
+
+`typedef BOOL(* WINTUN_ENUM_CALLBACK) (WINTUN_ADAPTER_HANDLE Adapter, LPARAM Param)`
+
+Called by WintunEnumAdapters for each adapter in the pool.
+
+**Parameters**
+
+- *Adapter*: Adapter handle, which will be freed when this function returns.
+- *Param*: An application-defined value passed to the WintunEnumAdapters.
+
+**Returns**
+
+Non-zero to continue iterating adapters; zero to stop.
+
+#### WINTUN\_LOGGER\_CALLBACK
+
+`typedef void(* WINTUN_LOGGER_CALLBACK) (WINTUN_LOGGER_LEVEL Level, DWORD64 Timestamp, const WCHAR *Message)`
+
+Called by internal logger to report diagnostic messages
+
+**Parameters**
+
+- *Level*: Message level.
+- *Timestamp*: Message timestamp in in 100ns intervals since 1601-01-01 UTC.
+- *Message*: Message text.
+
+#### WINTUN\_SESSION\_HANDLE
+
+`typedef void* WINTUN_SESSION_HANDLE`
+
+A handle representing Wintun session
+
+### Enumeration Types
+
+#### WINTUN\_LOGGER\_LEVEL
+
+`enum WINTUN_LOGGER_LEVEL`
+
+Determines the level of logging, passed to WINTUN\_LOGGER\_CALLBACK.
+
+- *WINTUN\_LOG\_INFO*: Informational
+- *WINTUN\_LOG\_WARN*: Warning
+- *WINTUN\_LOG\_ERR*: Error
+
+Enumerator
+
+### Functions
+
+#### WintunCreateAdapter()
+
+`WINTUN_ADAPTER_HANDLE WintunCreateAdapter (const WCHAR * Name, const WCHAR * TunnelType, const GUID * RequestedGUID)`
+
+Creates a new Wintun adapter.
+
+**Parameters**
+
+- *Name*: The requested name of the adapter. Zero-terminated string of up to MAX\_ADAPTER\_NAME-1 characters.
+- *Name*: Name of the adapter tunnel type. Zero-terminated string of up to MAX\_ADAPTER\_NAME-1 characters.
+- *RequestedGUID*: The GUID of the created network adapter, which then influences NLA generation deterministically. If it is set to NULL, the GUID is chosen by the system at random, and hence a new NLA entry is created for each new adapter. It is called "requested" GUID because the API it uses is completely undocumented, and so there could be minor interesting complications with its usage.
+
+**Returns**
+
+If the function succeeds, the return value is the adapter handle. Must be released with WintunCloseAdapter. If the function fails, the return value is NULL. To get extended error information, call GetLastError.
+
+#### WintunOpenAdapter()
+
+`WINTUN_ADAPTER_HANDLE WintunOpenAdapter (const WCHAR * Name)`
+
+Opens an existing Wintun adapter.
+
+**Parameters**
+
+- *Name*: The requested name of the adapter. Zero-terminated string of up to MAX\_ADAPTER\_NAME-1 characters.
+
+**Returns**
+
+If the function succeeds, the return value is adapter handle. Must be released with WintunCloseAdapter. If the function fails, the return value is NULL. To get extended error information, call GetLastError.
+
+#### WintunCloseAdapter()
+
+`void WintunCloseAdapter (WINTUN_ADAPTER_HANDLE Adapter)`
+
+Releases Wintun adapter resources and, if adapter was created with WintunCreateAdapter, removes adapter.
+
+**Parameters**
+
+- *Adapter*: Adapter handle obtained with WintunCreateAdapter or WintunOpenAdapter.
+
+#### WintunDeleteDriver()
+
+`BOOL WintunDeleteDriver ()`
+
+Deletes the Wintun driver if there are no more adapters in use.
+
+**Returns**
+
+If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To get extended error information, call GetLastError.
+
+#### WintunGetAdapterLuid()
+
+`void WintunGetAdapterLuid (WINTUN_ADAPTER_HANDLE Adapter, NET_LUID * Luid)`
+
+Returns the LUID of the adapter.
+
+**Parameters**
+
+- *Adapter*: Adapter handle obtained with WintunOpenAdapter or WintunCreateAdapter
+- *Luid*: Pointer to LUID to receive adapter LUID.
+
+#### WintunGetRunningDriverVersion()
+
+`DWORD WintunGetRunningDriverVersion (void )`
+
+Determines the version of the Wintun driver currently loaded.
+
+**Returns**
+
+If the function succeeds, the return value is the version number. If the function fails, the return value is zero. To get extended error information, call GetLastError. Possible errors include the following: ERROR\_FILE\_NOT\_FOUND Wintun not loaded
+
+#### WintunSetLogger()
+
+`void WintunSetLogger (WINTUN_LOGGER_CALLBACK NewLogger)`
+
+Sets logger callback function.
+
+**Parameters**
+
+- *NewLogger*: Pointer to callback function to use as a new global logger. NewLogger may be called from various threads concurrently. Should the logging require serialization, you must handle serialization in NewLogger. Set to NULL to disable.
+
+#### WintunStartSession()
+
+`WINTUN_SESSION_HANDLE WintunStartSession (WINTUN_ADAPTER_HANDLE Adapter, DWORD Capacity)`
+
+Starts Wintun session.
+
+**Parameters**
+
+- *Adapter*: Adapter handle obtained with WintunOpenAdapter or WintunCreateAdapter
+- *Capacity*: Rings capacity. Must be between WINTUN\_MIN\_RING\_CAPACITY and WINTUN\_MAX\_RING\_CAPACITY (incl.) Must be a power of two.
+
+**Returns**
+
+Wintun session handle. Must be released with WintunEndSession. If the function fails, the return value is NULL. To get extended error information, call GetLastError.
+
+#### WintunEndSession()
+
+`void WintunEndSession (WINTUN_SESSION_HANDLE Session)`
+
+Ends Wintun session.
+
+**Parameters**
+
+- *Session*: Wintun session handle obtained with WintunStartSession
+
+#### WintunGetReadWaitEvent()
+
+`HANDLE WintunGetReadWaitEvent (WINTUN_SESSION_HANDLE Session)`
+
+Gets Wintun session's read-wait event handle.
+
+**Parameters**
+
+- *Session*: Wintun session handle obtained with WintunStartSession
+
+**Returns**
+
+Pointer to receive event handle to wait for available data when reading. Should WintunReceivePackets return ERROR\_NO\_MORE\_ITEMS (after spinning on it for a while under heavy load), wait for this event to become signaled before retrying WintunReceivePackets. Do not call CloseHandle on this event - it is managed by the session.
+
+#### WintunReceivePacket()
+
+`BYTE* WintunReceivePacket (WINTUN_SESSION_HANDLE Session, DWORD * PacketSize)`
+
+Retrieves one or packet. After the packet content is consumed, call WintunReleaseReceivePacket with Packet returned from this function to release internal buffer. This function is thread-safe.
+
+**Parameters**
+
+- *Session*: Wintun session handle obtained with WintunStartSession
+- *PacketSize*: Pointer to receive packet size.
+
+**Returns**
+
+Pointer to layer 3 IPv4 or IPv6 packet. Client may modify its content at will. If the function fails, the return value is NULL. To get extended error information, call GetLastError. Possible errors include the following: ERROR\_HANDLE\_EOF Wintun adapter is terminating; ERROR\_NO\_MORE\_ITEMS Wintun buffer is exhausted; ERROR\_INVALID\_DATA Wintun buffer is corrupt
+
+#### WintunReleaseReceivePacket()
+
+`void WintunReleaseReceivePacket (WINTUN_SESSION_HANDLE Session, const BYTE * Packet)`
+
+Releases internal buffer after the received packet has been processed by the client. This function is thread-safe.
+
+**Parameters**
+
+- *Session*: Wintun session handle obtained with WintunStartSession
+- *Packet*: Packet obtained with WintunReceivePacket
+
+#### WintunAllocateSendPacket()
+
+`BYTE* WintunAllocateSendPacket (WINTUN_SESSION_HANDLE Session, DWORD PacketSize)`
+
+Allocates memory for a packet to send. After the memory is filled with packet data, call WintunSendPacket to send and release internal buffer. WintunAllocateSendPacket is thread-safe and the WintunAllocateSendPacket order of calls define the packet sending order.
+
+**Parameters**
+
+- *Session*: Wintun session handle obtained with WintunStartSession
+- *PacketSize*: Exact packet size. Must be less or equal to WINTUN\_MAX\_IP\_PACKET\_SIZE.
+
+**Returns**
+
+Returns pointer to memory where to prepare layer 3 IPv4 or IPv6 packet for sending. If the function fails, the return value is NULL. To get extended error information, call GetLastError. Possible errors include the following: ERROR\_HANDLE\_EOF Wintun adapter is terminating; ERROR\_BUFFER\_OVERFLOW Wintun buffer is full;
+
+#### WintunSendPacket()
+
+`void WintunSendPacket (WINTUN_SESSION_HANDLE Session, const BYTE * Packet)`
+
+Sends the packet and releases internal buffer. WintunSendPacket is thread-safe, but the WintunAllocateSendPacket order of calls define the packet sending order. This means the packet is not guaranteed to be sent in the WintunSendPacket yet.
+
+**Parameters**
+
+- *Session*: Wintun session handle obtained with WintunStartSession
+- *Packet*: Packet obtained with WintunAllocateSendPacket
+
+## Building
+
+**Do not distribute drivers or files named "Wintun", as they will most certainly clash with official deployments. Instead distribute [`wintun.dll` as downloaded from wintun.net](https://www.wintun.net).**
+
+General requirements:
+
+- [Visual Studio 2019](https://visualstudio.microsoft.com/downloads/) with Windows SDK
+- [Windows Driver Kit](https://docs.microsoft.com/en-us/windows-hardware/drivers/download-the-wdk)
+
+`wintun.sln` may be opened in Visual Studio for development and building. Be sure to run `bcdedit /set testsigning on` and then reboot before to enable unsigned driver loading. The default run sequence (F5) in Visual Studio will build the example project and its dependencies.
+
+## License
+
+The entire contents of [the repository](https://git.zx2c4.com/wintun/), including all documentation and example code, is "Copyright © 2018-2021 WireGuard LLC. All Rights Reserved." Source code is licensed under the [GPLv2](COPYING). Prebuilt binaries from [wintun.net](https://www.wintun.net/) are released under a more permissive license suitable for more forms of software contained inside of the .zip files distributed there.

dist/windows/wintun/include/wintun.h

@@ -0,0 +1,270 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT
+ *
+ * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+#pragma once
+
+#include <winsock2.h>
+#include <windows.h>
+#include <ipexport.h>
+#include <ifdef.h>
+#include <ws2ipdef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef ALIGNED
+#    if defined(_MSC_VER)
+#        define ALIGNED(n) __declspec(align(n))
+#    elif defined(__GNUC__)
+#        define ALIGNED(n) __attribute__((aligned(n)))
+#    else
+#        error "Unable to define ALIGNED"
+#    endif
+#endif
+
+/* MinGW is missing this one, unfortunately. */
+#ifndef _Post_maybenull_
+#    define _Post_maybenull_
+#endif
+
+#pragma warning(push)
+#pragma warning(disable : 4324) /* structure was padded due to alignment specifier */
+
+/**
+ * A handle representing Wintun adapter
+ */
+typedef struct _WINTUN_ADAPTER *WINTUN_ADAPTER_HANDLE;
+
+/**
+ * Creates a new Wintun adapter.
+ *
+ * @param Name          The requested name of the adapter. Zero-terminated string of up to MAX_ADAPTER_NAME-1
+ *                      characters.
+ *
+ * @param TunnelType    Name of the adapter tunnel type. Zero-terminated string of up to MAX_ADAPTER_NAME-1
+ *                      characters.
+ *
+ * @param RequestedGUID The GUID of the created network adapter, which then influences NLA generation deterministically.
+ *                      If it is set to NULL, the GUID is chosen by the system at random, and hence a new NLA entry is
+ *                      created for each new adapter. It is called "requested" GUID because the API it uses is
+ *                      completely undocumented, and so there could be minor interesting complications with its usage.
+ *
+ * @return If the function succeeds, the return value is the adapter handle. Must be released with
+ * WintunCloseAdapter. If the function fails, the return value is NULL. To get extended error information, call
+ * GetLastError.
+ */
+typedef _Must_inspect_result_
+_Return_type_success_(return != NULL)
+_Post_maybenull_
+WINTUN_ADAPTER_HANDLE(WINAPI WINTUN_CREATE_ADAPTER_FUNC)
+(_In_z_ LPCWSTR Name, _In_z_ LPCWSTR TunnelType, _In_opt_ const GUID *RequestedGUID);
+
+/**
+ * Opens an existing Wintun adapter.
+ *
+ * @param Name          The requested name of the adapter. Zero-terminated string of up to MAX_ADAPTER_NAME-1
+ *                      characters.
+ *
+ * @return If the function succeeds, the return value is the adapter handle. Must be released with
+ * WintunCloseAdapter. If the function fails, the return value is NULL. To get extended error information, call
+ * GetLastError.
+ */
+typedef _Must_inspect_result_
+_Return_type_success_(return != NULL)
+_Post_maybenull_
+WINTUN_ADAPTER_HANDLE(WINAPI WINTUN_OPEN_ADAPTER_FUNC)(_In_z_ LPCWSTR Name);
+
+/**
+ * Releases Wintun adapter resources and, if adapter was created with WintunCreateAdapter, removes adapter.
+ *
+ * @param Adapter       Adapter handle obtained with WintunCreateAdapter or WintunOpenAdapter.
+ */
+typedef VOID(WINAPI WINTUN_CLOSE_ADAPTER_FUNC)(_In_opt_ WINTUN_ADAPTER_HANDLE Adapter);
+
+/**
+ * Deletes the Wintun driver if there are no more adapters in use.
+ *
+ * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To
+ *         get extended error information, call GetLastError.
+ */
+typedef _Return_type_success_(return != FALSE)
+BOOL(WINAPI WINTUN_DELETE_DRIVER_FUNC)(VOID);
+
+/**
+ * Returns the LUID of the adapter.
+ *
+ * @param Adapter       Adapter handle obtained with WintunCreateAdapter or WintunOpenAdapter
+ *
+ * @param Luid          Pointer to LUID to receive adapter LUID.
+ */
+typedef VOID(WINAPI WINTUN_GET_ADAPTER_LUID_FUNC)(_In_ WINTUN_ADAPTER_HANDLE Adapter, _Out_ NET_LUID *Luid);
+
+/**
+ * Determines the version of the Wintun driver currently loaded.
+ *
+ * @return If the function succeeds, the return value is the version number. If the function fails, the return value is
+ *         zero. To get extended error information, call GetLastError. Possible errors include the following:
+ *         ERROR_FILE_NOT_FOUND  Wintun not loaded
+ */
+typedef _Return_type_success_(return != 0)
+DWORD(WINAPI WINTUN_GET_RUNNING_DRIVER_VERSION_FUNC)(VOID);
+
+/**
+ * Determines the level of logging, passed to WINTUN_LOGGER_CALLBACK.
+ */
+typedef enum
+{
+    WINTUN_LOG_INFO, /**< Informational */
+    WINTUN_LOG_WARN, /**< Warning */
+    WINTUN_LOG_ERR   /**< Error */
+} WINTUN_LOGGER_LEVEL;
+
+/**
+ * Called by internal logger to report diagnostic messages
+ *
+ * @param Level         Message level.
+ *
+ * @param Timestamp     Message timestamp in in 100ns intervals since 1601-01-01 UTC.
+ *
+ * @param Message       Message text.
+ */
+typedef VOID(CALLBACK *WINTUN_LOGGER_CALLBACK)(
+    _In_ WINTUN_LOGGER_LEVEL Level,
+    _In_ DWORD64 Timestamp,
+    _In_z_ LPCWSTR Message);
+
+/**
+ * Sets logger callback function.
+ *
+ * @param NewLogger     Pointer to callback function to use as a new global logger. NewLogger may be called from various
+ *                      threads concurrently. Should the logging require serialization, you must handle serialization in
+ *                      NewLogger. Set to NULL to disable.
+ */
+typedef VOID(WINAPI WINTUN_SET_LOGGER_FUNC)(_In_ WINTUN_LOGGER_CALLBACK NewLogger);
+
+/**
+ * Minimum ring capacity.
+ */
+#define WINTUN_MIN_RING_CAPACITY 0x20000 /* 128kiB */
+
+/**
+ * Maximum ring capacity.
+ */
+#define WINTUN_MAX_RING_CAPACITY 0x4000000 /* 64MiB */
+
+/**
+ * A handle representing Wintun session
+ */
+typedef struct _TUN_SESSION *WINTUN_SESSION_HANDLE;
+
+/**
+ * Starts Wintun session.
+ *
+ * @param Adapter       Adapter handle obtained with WintunOpenAdapter or WintunCreateAdapter
+ *
+ * @param Capacity      Rings capacity. Must be between WINTUN_MIN_RING_CAPACITY and WINTUN_MAX_RING_CAPACITY (incl.)
+ *                      Must be a power of two.
+ *
+ * @return Wintun session handle. Must be released with WintunEndSession. If the function fails, the return value is
+ *         NULL. To get extended error information, call GetLastError.
+ */
+typedef _Must_inspect_result_
+_Return_type_success_(return != NULL)
+_Post_maybenull_
+WINTUN_SESSION_HANDLE(WINAPI WINTUN_START_SESSION_FUNC)(_In_ WINTUN_ADAPTER_HANDLE Adapter, _In_ DWORD Capacity);
+
+/**
+ * Ends Wintun session.
+ *
+ * @param Session       Wintun session handle obtained with WintunStartSession
+ */
+typedef VOID(WINAPI WINTUN_END_SESSION_FUNC)(_In_ WINTUN_SESSION_HANDLE Session);
+
+/**
+ * Gets Wintun session's read-wait event handle.
+ *
+ * @param Session       Wintun session handle obtained with WintunStartSession
+ *
+ * @return Pointer to receive event handle to wait for available data when reading. Should
+ *         WintunReceivePackets return ERROR_NO_MORE_ITEMS (after spinning on it for a while under heavy
+ *         load), wait for this event to become signaled before retrying WintunReceivePackets. Do not call
+ *         CloseHandle on this event - it is managed by the session.
+ */
+typedef HANDLE(WINAPI WINTUN_GET_READ_WAIT_EVENT_FUNC)(_In_ WINTUN_SESSION_HANDLE Session);
+
+/**
+ * Maximum IP packet size
+ */
+#define WINTUN_MAX_IP_PACKET_SIZE 0xFFFF
+
+/**
+ * Retrieves one or packet. After the packet content is consumed, call WintunReleaseReceivePacket with Packet returned
+ * from this function to release internal buffer. This function is thread-safe.
+ *
+ * @param Session       Wintun session handle obtained with WintunStartSession
+ *
+ * @param PacketSize    Pointer to receive packet size.
+ *
+ * @return Pointer to layer 3 IPv4 or IPv6 packet. Client may modify its content at will. If the function fails, the
+ *         return value is NULL. To get extended error information, call GetLastError. Possible errors include the
+ *         following:
+ *         ERROR_HANDLE_EOF     Wintun adapter is terminating;
+ *         ERROR_NO_MORE_ITEMS  Wintun buffer is exhausted;
+ *         ERROR_INVALID_DATA   Wintun buffer is corrupt
+ */
+typedef _Must_inspect_result_
+_Return_type_success_(return != NULL)
+_Post_maybenull_
+_Post_writable_byte_size_(*PacketSize)
+BYTE *(WINAPI WINTUN_RECEIVE_PACKET_FUNC)(_In_ WINTUN_SESSION_HANDLE Session, _Out_ DWORD *PacketSize);
+
+/**
+ * Releases internal buffer after the received packet has been processed by the client. This function is thread-safe.
+ *
+ * @param Session       Wintun session handle obtained with WintunStartSession
+ *
+ * @param Packet        Packet obtained with WintunReceivePacket
+ */
+typedef VOID(
+    WINAPI WINTUN_RELEASE_RECEIVE_PACKET_FUNC)(_In_ WINTUN_SESSION_HANDLE Session, _In_ const BYTE *Packet);
+
+/**
+ * Allocates memory for a packet to send. After the memory is filled with packet data, call WintunSendPacket to send
+ * and release internal buffer. WintunAllocateSendPacket is thread-safe and the WintunAllocateSendPacket order of
+ * calls define the packet sending order.
+ *
+ * @param Session       Wintun session handle obtained with WintunStartSession
+ *
+ * @param PacketSize    Exact packet size. Must be less or equal to WINTUN_MAX_IP_PACKET_SIZE.
+ *
+ * @return Returns pointer to memory where to prepare layer 3 IPv4 or IPv6 packet for sending. If the function fails,
+ *         the return value is NULL. To get extended error information, call GetLastError. Possible errors include the
+ *         following:
+ *         ERROR_HANDLE_EOF       Wintun adapter is terminating;
+ *         ERROR_BUFFER_OVERFLOW  Wintun buffer is full;
+ */
+typedef _Must_inspect_result_
+_Return_type_success_(return != NULL)
+_Post_maybenull_
+_Post_writable_byte_size_(PacketSize)
+BYTE *(WINAPI WINTUN_ALLOCATE_SEND_PACKET_FUNC)(_In_ WINTUN_SESSION_HANDLE Session, _In_ DWORD PacketSize);
+
+/**
+ * Sends the packet and releases internal buffer. WintunSendPacket is thread-safe, but the WintunAllocateSendPacket
+ * order of calls define the packet sending order. This means the packet is not guaranteed to be sent in the
+ * WintunSendPacket yet.
+ *
+ * @param Session       Wintun session handle obtained with WintunStartSession
+ *
+ * @param Packet        Packet obtained with WintunAllocateSendPacket
+ */
+typedef VOID(WINAPI WINTUN_SEND_PACKET_FUNC)(_In_ WINTUN_SESSION_HANDLE Session, _In_ const BYTE *Packet);
+
+#pragma warning(pop)
+
+#ifdef __cplusplus
+}
+#endif

e2e/handshakes_test.go

@@ -18,8 +18,8 @@ import (
 
 func TestGoodHandshake(t *testing.T) {
 	ca, _, caKey, _ := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
-	myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 1}, nil)
-	theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, nil)
+	myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 1})
+	theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2})
 
 	// Put their info in our lighthouse
 	myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
@@ -70,9 +70,9 @@ func TestWrongResponderHandshake(t *testing.T) {
 	// The IPs here are chosen on purpose:
 	// The current remote handling will sort by preference, public, and then lexically.
 	// So we need them to have a higher address than evil (we could apply a preference though)
-	myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 100}, nil)
-	theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 99}, nil)
-	evilControl, evilVpnIp, evilUdpAddr := newSimpleServer(ca, caKey, "evil", net.IP{10, 0, 0, 2}, nil)
+	myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 100})
+	theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 99})
+	evilControl, evilVpnIp, evilUdpAddr := newSimpleServer(ca, caKey, "evil", net.IP{10, 0, 0, 2})
 
 	// Add their real udp addr, which should be tried after evil.
 	myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
@@ -130,8 +130,8 @@ func TestWrongResponderHandshake(t *testing.T) {
 
 func Test_Case1_Stage1Race(t *testing.T) {
 	ca, _, caKey, _ := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
-	myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me  ", net.IP{10, 0, 0, 1}, nil)
-	theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, nil)
+	myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me  ", net.IP{10, 0, 0, 1})
+	theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2})
 
 	// Put their info in our lighthouse and vice versa
 	myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
@@ -183,151 +183,3 @@ func Test_Case1_Stage1Race(t *testing.T) {
 }
 
 //TODO: add a test with many lies
-
-func TestPSK(t *testing.T) {
-	tests := []struct {
-		name         string
-		myPskMode    nebula.PskMode
-		theirPskMode nebula.PskMode
-	}{
-		// None and transitional-accepting both ways
-		{
-			name:         "none to transitional-accepting",
-			myPskMode:    nebula.PskNone,
-			theirPskMode: nebula.PskTransitionalAccepting,
-		},
-		{
-			name:         "transitional-accepting to none",
-			myPskMode:    nebula.PskTransitionalAccepting,
-			theirPskMode: nebula.PskNone,
-		},
-
-		// All transitional-accepting
-		{
-			name:         "both transitional-accepting",
-			myPskMode:    nebula.PskTransitionalAccepting,
-			theirPskMode: nebula.PskTransitionalAccepting,
-		},
-
-		// transitional-accepting and transitional-sending both ways
-		{
-			name:         "transitional-accepting to transitional-sending",
-			myPskMode:    nebula.PskTransitionalAccepting,
-			theirPskMode: nebula.PskTransitionalSending,
-		},
-		{
-			name:         "transitional-sending to transitional-accepting",
-			myPskMode:    nebula.PskTransitionalSending,
-			theirPskMode: nebula.PskTransitionalAccepting,
-		},
-
-		// All transitional-sending
-		{
-			name:         "transitional-sending to transitional-sending",
-			myPskMode:    nebula.PskTransitionalSending,
-			theirPskMode: nebula.PskTransitionalSending,
-		},
-
-		// enforced and transitional-sending both ways
-		{
-			name:         "enforced to transitional-sending",
-			myPskMode:    nebula.PskEnforced,
-			theirPskMode: nebula.PskTransitionalSending,
-		},
-		{
-			name:         "transitional-sending to enforced",
-			myPskMode:    nebula.PskTransitionalSending,
-			theirPskMode: nebula.PskEnforced,
-		},
-
-		// All enforced
-		{
-			name:         "both enforced",
-			myPskMode:    nebula.PskEnforced,
-			theirPskMode: nebula.PskEnforced,
-		},
-
-		// Enforced can technically handshake with a traditional-accepting but it is bad to be in this state
-		{
-			name:         "enforced to traditional-accepting",
-			myPskMode:    nebula.PskEnforced,
-			theirPskMode: nebula.PskTransitionalAccepting,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			var myPskSettings, theirPskSettings *m
-
-			switch test.myPskMode {
-			case nebula.PskNone:
-				myPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "none"}}}
-			case nebula.PskTransitionalAccepting:
-				myPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "transitional-accepting", "keys": []string{"this is a key"}}}}
-			case nebula.PskTransitionalSending:
-				myPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "transitional-sending", "keys": []string{"this is a key"}}}}
-			case nebula.PskEnforced:
-				myPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "enforced", "keys": []string{"this is a key"}}}}
-			}
-
-			switch test.theirPskMode {
-			case nebula.PskNone:
-				theirPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "none"}}}
-			case nebula.PskTransitionalAccepting:
-				theirPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "transitional-accepting", "keys": []string{"this is a key"}}}}
-			case nebula.PskTransitionalSending:
-				theirPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "transitional-sending", "keys": []string{"this is a key"}}}}
-			case nebula.PskEnforced:
-				theirPskSettings = &m{"handshakes": &m{"psk": &m{"mode": "enforced", "keys": []string{"this is a key"}}}}
-			}
-
-			ca, _, caKey, _ := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
-			myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 1}, myPskSettings)
-			theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, theirPskSettings)
-
-			myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
-			r := router.NewR(myControl, theirControl)
-
-			// Start the servers
-			myControl.Start()
-			theirControl.Start()
-
-			t.Log("Route until we see our cached packet flow")
-			myControl.InjectTunUDPPacket(theirVpnIp, 80, 80, []byte("Hi from me"))
-			r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
-				h := &header.H{}
-				err := h.Parse(p.Data)
-				if err != nil {
-					panic(err)
-				}
-
-				// If this is the stage 1 handshake packet and I am configured to send with a psk, my cert name should
-				// not appear. It would likely be more obvious to unmarshal the payload and check but this works fine for now
-				if test.myPskMode == nebula.PskEnforced || test.myPskMode == nebula.PskTransitionalSending {
-					if h.Type == 0 && h.MessageCounter == 1 {
-						assert.NotContains(t, string(p.Data), "test me")
-					}
-				}
-
-				if p.ToIp.Equal(theirUdpAddr.IP) && p.ToPort == uint16(theirUdpAddr.Port) && h.Type == 1 {
-					return router.RouteAndExit
-				}
-
-				return router.KeepRouting
-			})
-
-			t.Log("My cached packet should be received by them")
-			myCachedPacket := theirControl.GetFromTun(true)
-			assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIp, theirVpnIp, 80, 80)
-
-			t.Log("Test the tunnel with them")
-			assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIp, theirVpnIp, myControl, theirControl)
-			assertTunnel(t, myVpnIp, theirVpnIp, myControl, theirControl, r)
-
-			myControl.Stop()
-			theirControl.Stop()
-			//TODO: assert hostmaps
-		})
-	}
-
-}

e2e/helpers_test.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
-	"github.com/imdario/mergo"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/cert"
@@ -31,7 +30,7 @@ import (
 type m map[string]interface{}
 
 // newSimpleServer creates a nebula instance with many assumptions
-func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, udpIp net.IP, customConfig *m) (*nebula.Control, net.IP, *net.UDPAddr) {
+func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, udpIp net.IP) (*nebula.Control, net.IP, *net.UDPAddr) {
 	l := NewTestLogger()
 
 	vpnIpNet := &net.IPNet{IP: make([]byte, len(udpIp)), Mask: net.IPMask{255, 255, 255, 0}}
@@ -41,7 +40,7 @@ func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, u
 		IP:   udpIp,
 		Port: 4242,
 	}
-	_, _, myPrivKey, myPEM := newTestCert(caCrt, caKey, "test "+name, time.Now(), time.Now().Add(5*time.Minute), vpnIpNet, nil, []string{})
+	_, _, myPrivKey, myPEM := newTestCert(caCrt, caKey, name, time.Now(), time.Now().Add(5*time.Minute), vpnIpNet, nil, []string{})
 
 	caB, err := caCrt.MarshalToPEM()
 	if err != nil {
@@ -87,24 +86,6 @@ func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, u
 	c := config.NewC(l)
 	c.LoadString(string(cb))
 
-	if customConfig != nil {
-		ccb, err := yaml.Marshal(customConfig)
-		if err != nil {
-			panic(err)
-		}
-
-		ccm := map[interface{}]interface{}{}
-		err = yaml.Unmarshal(ccb, &ccm)
-		if err != nil {
-			panic(err)
-		}
-
-		err = mergo.Merge(&c.Settings, ccm, mergo.WithAppendSlice)
-		if err != nil {
-			panic(err)
-		}
-	}
-
 	control, err := nebula.Main(c, false, "e2e-test", l, nil)
 
 	if err != nil {

examples/config.yml

@@ -148,7 +148,9 @@ punchy:
 tun:
   # When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root)
   disabled: false
-  # Name of the device
+  # Name of the device. If not set, a default will be chosen by the OS.
+  # For macOS: if set, must be in the form `utun[0-9]+`.
+  # For FreeBSD: Required to be set, must be in the form `tun[0-9]+`.
   dev: nebula1
   # Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert
   drop_local_broadcast: false
@@ -215,45 +217,17 @@ logging:
   #   e.g.: `lighthouse.rx.HostQuery`
   #lighthouse_metrics: false
 
-# Handshake Manger Settings
-handshakes:
+# Handshake Manager Settings
+#handshakes:
   # Handshakes are sent to all known addresses at each interval with a linear backoff,
   # Wait try_interval after the 1st attempt, 2 * try_interval after the 2nd, etc, until the handshake is older than timeout
   # A 100ms interval with the default 10 retries will give a handshake 5.5 seconds to resolve before timing out
   #try_interval: 100ms
   #retries: 20
-
   # trigger_buffer is the size of the buffer channel for quickly sending handshakes
   # after receiving the response for lighthouse queries
   #trigger_buffer: 64
 
-  # psk can be used to mask the contents of handshakes and makes handshaking with unintended recipients more difficult
-  # all settings respond to a reload
-  psk:
-    # mode defines how the pre shared keys can be used in a handshake
-    # `none` (the default) does not send or receive using a psk. Ideally `enforced` is used
-    # `transitional-accepting` will send handshakes without using a psk and can receive handshakes using a psk we know about
-    # `transitional-sending` will send handshakes using a psk but will still accept handshakes without them
-    # `enforced` enforces the use of a psk for all tunnels. Any node not also using `enforced` or `transitional-sending` can not handshake with us
-    #
-    # When moving from `none` to `enforced` you will want to change every node in the mesh to `transitional-accepting` and reload
-    # then move every node to `transitional-sending` then reload, and finally `enforced` then reload. This allows you to
-    # avoid stopping the world to use psk. You must ensure at `transitional-accepting` that all nodes have the same psks.
-    #mode: none
-
-    # In `transitional-accepting`, `transitional-sending` and `enforced` modes, the keys provided here are sent through
-    # hkdf with the intended recipients ip used in the info section. This helps guard against handshaking with the wrong
-    # host if your static_host_map or lighthouse(s) has incorrect information.
-    #
-    # Setting keys if mode is `none` has no effect.
-    #
-    # Only the first key is used for outbound handshakes but all keys provided will be tried in the order specified, on
-    # incoming handshakes. This is to allow for psk rotation.
-    #keys:
-      # - shared secret string, this one is used in all outbound handshakes
-      # - this is a fallback key, received handshakes can use this
-      # - another fallback, received handshakes can use this one too
-      # - "\x68\x65\x6c\x6c\x6f\x20\x66\x72\x69\x65\x6e\x64\x73" # for raw bytes if you desire
 
 # Nebula security group configuration
 firewall:

firewall_test.go

@@ -14,12 +14,12 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/iputil"
-	"github.com/slackhq/nebula/util"
+	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestNewFirewall(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	c := &cert.NebulaCertificate{}
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	conntrack := fw.Conntrack
@@ -58,7 +58,7 @@ func TestNewFirewall(t *testing.T) {
 }
 
 func TestFirewall_AddRule(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
 
@@ -133,7 +133,7 @@ func TestFirewall_AddRule(t *testing.T) {
 }
 
 func TestFirewall_Drop(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
 
@@ -308,7 +308,7 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 }
 
 func TestFirewall_Drop2(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
 
@@ -367,7 +367,7 @@ func TestFirewall_Drop2(t *testing.T) {
 }
 
 func TestFirewall_Drop3(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
 
@@ -453,7 +453,7 @@ func TestFirewall_Drop3(t *testing.T) {
 }
 
 func TestFirewall_DropConntrackReload(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
 
@@ -635,7 +635,7 @@ func Test_parsePort(t *testing.T) {
 }
 
 func TestNewFirewallFromConfig(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	// Test a bad rule definition
 	c := &cert.NebulaCertificate{}
 	conf := config.NewC(l)
@@ -685,7 +685,7 @@ func TestNewFirewallFromConfig(t *testing.T) {
 }
 
 func TestAddFirewallRulesFromConfig(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	// Test adding tcp rule
 	conf := config.NewC(l)
 	mf := &mockFirewall{}
@@ -849,7 +849,7 @@ func TestTCPRTTTracking(t *testing.T) {
 }
 
 func TestFirewall_convertRule(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
 

go.mod

@@ -25,9 +25,11 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/vishvananda/netlink v1.1.0
 	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/net v0.0.0-20211101193420-4a448f8816b3
-	golang.org/x/sys v0.0.0-20211102192858-4dd72447c267
+	golang.org/x/crypto v0.0.0-20211202192323-5770296d904e
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
+	golang.org/x/sys v0.0.0-20211103235746-7861aae1554b
+	golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224
+	golang.zx2c4.com/wireguard/windows v0.5.1
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -160,6 +160,8 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
+github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
@@ -226,6 +228,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -240,6 +243,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8=
+golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -270,6 +275,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -301,8 +307,12 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211101193420-4a448f8816b3 h1:VrJZAjbekhoRn7n5FBujY31gboH+iB3pdLxn3gE9FjU=
 golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -357,14 +367,17 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211102192858-4dd72447c267 h1:7zYaz3tjChtpayGDzu6H0hDAUM5zIGA2XW7kRNgQ0jc=
-golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211103235746-7861aae1554b h1:1VkfZQv42XQlA/jchYumAnv1UPo6RgF9rJFkTgZIxO4=
+golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -374,6 +387,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -419,11 +433,16 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 h1:Ug9qvr1myri/zFN6xL17LSCBGFDnphBBhzmILHsM5TY=
+golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
+golang.zx2c4.com/wireguard/windows v0.5.1 h1:OnYw96PF+CsIMrqWo5QP3Q59q5hY1rFErk/yN3cS+JQ=
+golang.zx2c4.com/wireguard/windows v0.5.1/go.mod h1:EApyTk/ZNrkbZjurHL1nleDYnsPpJYBO7LZEBCyDAHk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=

handshake_ix.go

@@ -4,6 +4,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/flynn/noise"
 	"github.com/golang/protobuf/proto"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
@@ -70,50 +71,27 @@ func ixHandshakeStage0(f *Interface, vpnIp iputil.VpnIp, hostinfo *HostInfo) {
 }
 
 func ixHandshakeStage1(f *Interface, addr *udp.Addr, packet []byte, h *header.H) {
-	var (
-		err error
-		ci  *ConnectionState
-		msg []byte
-	)
+	ci := f.newConnectionState(f.l, false, noise.HandshakeIX, []byte{}, 0)
+	// Mark packet 1 as seen so it doesn't show up as missed
+	ci.window.Update(f.l, 1)
 
-	hs := &NebulaHandshake{}
-
-	// Handle multiple possible psk options, ensure the protobuf comes out clean too
-	for _, p := range f.psk.Cache {
-		//TODO: benchmark generation time of makePsk
-		ci, err = f.newConnectionState(f.l, false, p)
-		if err != nil {
-			f.l.WithError(err).WithField("udpAddr", addr).Error("Failed to get a new connection state")
-			continue
-		}
-
-		msg, _, _, err = ci.H.ReadMessage(nil, packet[header.Len:])
-		if err != nil {
-			// Calls to ReadMessage with an incorrect psk should fail, try the next one if we have one
-			continue
-		}
-
-		// Sometimes ReadMessage returns fine with a nil psk even if the handshake is using a psk, ensure our protobuf
-		// comes out clean as well
-		err = proto.Unmarshal(msg, hs)
-		if err == nil {
-			// There was no error, we can continue with this handshake
-			break
-		}
-
-		// The unmarshal failed, try the next psk if we have one
-	}
-
-	// We finished with an error, log it and get out
+	msg, _, _, err := ci.H.ReadMessage(nil, packet[header.Len:])
 	if err != nil {
-		// We aren't logging the error here because we can't be sure of the failure when using psk
-		f.l.WithField("udpAddr", addr).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-			Error("Was unable to decrypt the handshake")
+		f.l.WithError(err).WithField("udpAddr", addr).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.ReadMessage")
 		return
 	}
 
-	// Mark packet 1 as seen so it doesn't show up as missed
-	ci.window.Update(f.l, 1)
+	hs := &NebulaHandshake{}
+	err = proto.Unmarshal(msg, hs)
+	/*
+		l.Debugln("GOT INDEX: ", hs.Details.InitiatorIndex)
+	*/
+	if err != nil || hs.Details == nil {
+		f.l.WithError(err).WithField("udpAddr", addr).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed unmarshal handshake message")
+		return
+	}
 
 	remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.caPool)
 	if err != nil {

handshake_manager.go

@@ -191,13 +191,13 @@ func (c *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, f udp.EncWriter, l
 	}
 }
 
-func (c *HandshakeManager) AddVpnIp(vpnIp iputil.VpnIp) *HostInfo {
-	hostinfo := c.pendingHostMap.AddVpnIp(vpnIp)
-	// We lock here and use an array to insert items to prevent locking the
-	// main receive thread for very long by waiting to add items to the pending map
-	//TODO: what lock?
-	c.OutboundHandshakeTimer.Add(vpnIp, c.config.tryInterval)
-	c.metricInitiated.Inc(1)
+func (c *HandshakeManager) AddVpnIp(vpnIp iputil.VpnIp, init func(*HostInfo)) *HostInfo {
+	hostinfo, created := c.pendingHostMap.AddVpnIp(vpnIp, init)
+
+	if created {
+		c.OutboundHandshakeTimer.Add(vpnIp, c.config.tryInterval)
+		c.metricInitiated.Inc(1)
+	}
 
 	return hostinfo
 }

handshake_manager_test.go

@@ -7,13 +7,13 @@ import (
 
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/test"
 	"github.com/slackhq/nebula/udp"
-	"github.com/slackhq/nebula/util"
 	"github.com/stretchr/testify/assert"
 )
 
 func Test_NewHandshakeManagerVpnIp(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	_, tuncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
@@ -27,7 +27,19 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
 	now := time.Now()
 	blah.NextOutboundHandshakeTimerTick(now, mw)
 
-	i := blah.AddVpnIp(ip)
+	var initCalled bool
+	initFunc := func(*HostInfo) {
+		initCalled = true
+	}
+
+	i := blah.AddVpnIp(ip, initFunc)
+	assert.True(t, initCalled)
+
+	initCalled = false
+	i2 := blah.AddVpnIp(ip, initFunc)
+	assert.False(t, initCalled)
+	assert.Same(t, i, i2)
+
 	i.remotes = NewRemoteList()
 	i.HandshakeReady = true
 
@@ -54,7 +66,7 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
 }
 
 func Test_NewHandshakeManagerTrigger(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	_, tuncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
@@ -71,7 +83,7 @@ func Test_NewHandshakeManagerTrigger(t *testing.T) {
 
 	assert.Equal(t, 0, testCountTimerWheelEntries(blah.OutboundHandshakeTimer))
 
-	hi := blah.AddVpnIp(ip)
+	hi := blah.AddVpnIp(ip, nil)
 	hi.HandshakeReady = true
 	assert.Equal(t, 1, testCountTimerWheelEntries(blah.OutboundHandshakeTimer))
 	assert.Equal(t, 0, hi.HandshakeCounter, "Should not have attempted a handshake yet")

hostmap.go

@@ -35,7 +35,6 @@ type HostMap struct {
 	Hosts           map[iputil.VpnIp]*HostInfo
 	preferredRanges []*net.IPNet
 	vpnCIDR         *net.IPNet
-	unsafeRoutes    *cidr.Tree4
 	metricsEnabled  bool
 	l               *logrus.Logger
 }
@@ -98,7 +97,6 @@ func NewHostMap(l *logrus.Logger, name string, vpnCIDR *net.IPNet, preferredRang
 		Hosts:           h,
 		preferredRanges: preferredRanges,
 		vpnCIDR:         vpnCIDR,
-		unsafeRoutes:    cidr.NewTree4(),
 		l:               l,
 	}
 	return &m
@@ -134,24 +132,25 @@ func (hm *HostMap) Add(ip iputil.VpnIp, hostinfo *HostInfo) {
 	hm.Unlock()
 }
 
-func (hm *HostMap) AddVpnIp(vpnIp iputil.VpnIp) *HostInfo {
-	h := &HostInfo{}
+func (hm *HostMap) AddVpnIp(vpnIp iputil.VpnIp, init func(hostinfo *HostInfo)) (hostinfo *HostInfo, created bool) {
 	hm.RLock()
-	if _, ok := hm.Hosts[vpnIp]; !ok {
+	if h, ok := hm.Hosts[vpnIp]; !ok {
 		hm.RUnlock()
 		h = &HostInfo{
 			promoteCounter:  0,
 			vpnIp:           vpnIp,
 			HandshakePacket: make(map[uint8][]byte, 0),
 		}
+		if init != nil {
+			init(h)
+		}
 		hm.Lock()
 		hm.Hosts[vpnIp] = h
 		hm.Unlock()
-		return h
+		return h, true
 	} else {
-		h = hm.Hosts[vpnIp]
 		hm.RUnlock()
-		return h
+		return h, false
 	}
 }
 
@@ -331,15 +330,6 @@ func (hm *HostMap) queryVpnIp(vpnIp iputil.VpnIp, promoteIfce *Interface) (*Host
 	return nil, errors.New("unable to find host")
 }
 
-func (hm *HostMap) queryUnsafeRoute(ip iputil.VpnIp) iputil.VpnIp {
-	r := hm.unsafeRoutes.MostSpecificContains(ip)
-	if r != nil {
-		return r.(iputil.VpnIp)
-	} else {
-		return 0
-	}
-}
-
 // We already have the hm Lock when this is called, so make sure to not call
 // any other methods that might try to grab it again
 func (hm *HostMap) addHostInfo(hostinfo *HostInfo, f *Interface) {
@@ -407,17 +397,6 @@ func (hm *HostMap) Punchy(ctx context.Context, conn *udp.Conn) {
 	}
 }
 
-func (hm *HostMap) addUnsafeRoutes(routes *[]route) {
-	for _, r := range *routes {
-		hm.l.WithField("route", r.route).WithField("via", r.via).Warn("Adding UNSAFE Route")
-		hm.unsafeRoutes.AddCIDR(r.route, iputil.Ip2VpnIp(*r.via))
-	}
-}
-
-func (i *HostInfo) BindConnectionState(cs *ConnectionState) {
-	i.ConnectionState = cs
-}
-
 // TryPromoteBest handles re-querying lighthouses and probing for better paths
 // NOTE: It is an error to call this if you are a lighthouse since they should not roam clients!
 func (i *HostInfo) TryPromoteBest(preferredRanges []*net.IPNet, ifce *Interface) {
@@ -558,10 +537,6 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, newRemote *udp.Addr) bool {
 	return false
 }
 
-func (i *HostInfo) ClearConnectionState() {
-	i.ConnectionState = nil
-}
-
 func (i *HostInfo) RecvErrorExceeded() bool {
 	if i.recvError < 3 {
 		i.recvError += 1

inside.go

@@ -3,6 +3,7 @@ package nebula
 import (
 	"sync/atomic"
 
+	"github.com/flynn/noise"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
@@ -71,17 +72,18 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 func (f *Interface) getOrHandshake(vpnIp iputil.VpnIp) *HostInfo {
 	//TODO: we can find contains without converting back to bytes
 	if f.hostMap.vpnCIDR.Contains(vpnIp.ToIP()) == false {
-		vpnIp = f.hostMap.queryUnsafeRoute(vpnIp)
+		vpnIp = f.inside.RouteFor(vpnIp)
 		if vpnIp == 0 {
 			return nil
 		}
 	}
 	hostinfo, err := f.hostMap.PromoteBestQueryVpnIp(vpnIp, f)
 
+	//if err != nil || hostinfo.ConnectionState == nil {
 	if err != nil {
 		hostinfo, err = f.handshakeManager.pendingHostMap.QueryVpnIp(vpnIp)
 		if err != nil {
-			hostinfo = f.handshakeManager.AddVpnIp(vpnIp)
+			hostinfo = f.handshakeManager.AddVpnIp(vpnIp, f.initHostInfo)
 		}
 	}
 	ci := hostinfo.ConnectionState
@@ -100,27 +102,11 @@ func (f *Interface) getOrHandshake(vpnIp iputil.VpnIp) *HostInfo {
 		return hostinfo
 	}
 
-	// Create a connection state if we don't have one yet
-	if ci == nil {
-		// Generate a PSK based on our config, this may be nil
-		p, err := f.psk.MakeFor(vpnIp)
-		if err != nil {
-			//TODO: This isn't fatal specifically but it's pretty bad
-			f.l.WithError(err).WithField("vpnIp", vpnIp).Error("Failed to get a PSK KDF")
-			return hostinfo
-		}
-
-		ci, err = f.newConnectionState(f.l, true, p)
-		if err != nil {
-			f.l.WithError(err).WithField("vpnIp", vpnIp).Error("Failed to get a connection state")
-			return hostinfo
-		}
-		hostinfo.ConnectionState = ci
-	}
-
 	// If we have already created the handshake packet, we don't want to call the function at all.
 	if !hostinfo.HandshakeReady {
 		ixHandshakeStage0(f, vpnIp, hostinfo)
+		// FIXME: Maybe make XX selectable, but probably not since psk makes it nearly pointless for us.
+		//xx_handshakeStage0(f, ip, hostinfo)
 
 		// If this is a static host, we don't need to wait for the HostQueryReply
 		// We can trigger the handshake right now
@@ -135,6 +121,12 @@ func (f *Interface) getOrHandshake(vpnIp iputil.VpnIp) *HostInfo {
 	return hostinfo
 }
 
+// initHostInfo is the init function to pass to (*HandshakeManager).AddVpnIP that
+// will create the initial Noise ConnectionState
+func (f *Interface) initHostInfo(hostinfo *HostInfo) {
+	hostinfo.ConnectionState = f.newConnectionState(f.l, true, noise.HandshakeIX, []byte{}, 0)
+}
+
 func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostInfo *HostInfo, p, nb, out []byte) {
 	fp := &firewall.Packet{}
 	err := newPacket(p, false, fp)

interface.go

@@ -4,12 +4,10 @@ import (
 	"context"
 	"errors"
 	"io"
-	"net"
 	"os"
 	"runtime"
 	"sync/atomic"
 	"time"
-	"unsafe"
 
 	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
@@ -17,24 +15,16 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/udp"
 )
 
 const mtu = 9001
 
-type Inside interface {
-	io.ReadWriteCloser
-	Activate() error
-	CidrNet() *net.IPNet
-	DeviceName() string
-	WriteRaw([]byte) error
-	NewMultiQueueReader() (io.ReadWriteCloser, error)
-}
-
 type InterfaceConfig struct {
 	HostMap                 *HostMap
 	Outside                 *udp.Conn
-	Inside                  Inside
+	Inside                  overlay.Device
 	certState               *CertState
 	Cipher                  string
 	Firewall                *Firewall
@@ -50,7 +40,6 @@ type InterfaceConfig struct {
 	version                 string
 	caPool                  *cert.NebulaCAPool
 	disconnectInvalid       bool
-	psk                     *Psk
 
 	ConntrackCacheTimeout time.Duration
 	l                     *logrus.Logger
@@ -59,7 +48,7 @@ type InterfaceConfig struct {
 type Interface struct {
 	hostMap            *HostMap
 	outside            *udp.Conn
-	inside             Inside
+	inside             overlay.Device
 	certState          *CertState
 	cipher             string
 	firewall           *Firewall
@@ -75,15 +64,16 @@ type Interface struct {
 	routines           int
 	caPool             *cert.NebulaCAPool
 	disconnectInvalid  bool
+	closed             int32
 
 	// rebindCount is used to decide if an active tunnel should trigger a punch notification through a lighthouse
 	rebindCount int8
 	version     string
 
 	conntrackCacheTimeout time.Duration
-	psk                   *Psk
-	writers               []*udp.Conn
-	readers               []io.ReadWriteCloser
+
+	writers []*udp.Conn
+	readers []io.ReadWriteCloser
 
 	metricHandshakes    metrics.Histogram
 	messageMetrics      *MessageMetrics
@@ -107,7 +97,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 	}
 
 	myVpnIp := iputil.Ip2VpnIp(c.certState.certificate.Details.Ips[0].IP)
-
 	ifce := &Interface{
 		hostMap:            c.HostMap,
 		outside:            c.Outside,
@@ -128,7 +117,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		readers:            make([]io.ReadWriteCloser, c.routines),
 		caPool:             c.caPool,
 		disconnectInvalid:  c.disconnectInvalid,
-		psk:                c.psk,
 		myVpnIp:            myVpnIp,
 
 		conntrackCacheTimeout: c.ConntrackCacheTimeout,
@@ -159,7 +147,7 @@ func (f *Interface) activate() {
 		f.l.WithError(err).Error("Failed to get udp listen address")
 	}
 
-	f.l.WithField("interface", f.inside.DeviceName()).WithField("network", f.inside.CidrNet().String()).
+	f.l.WithField("interface", f.inside.Name()).WithField("network", f.inside.Cidr().String()).
 		WithField("build", f.version).WithField("udpAddr", addr).
 		Info("Nebula interface is active")
 
@@ -178,6 +166,7 @@ func (f *Interface) activate() {
 	}
 
 	if err := f.inside.Activate(); err != nil {
+		f.inside.Close()
 		f.l.Fatal(err)
 	}
 }
@@ -223,6 +212,10 @@ func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
 	for {
 		n, err := reader.Read(packet)
 		if err != nil {
+			if errors.Is(err, os.ErrClosed) && atomic.LoadInt32(&f.closed) != 0 {
+				return
+			}
+
 			f.l.WithError(err).Error("Error while reading outbound packet")
 			// This only seems to happen when something fatal happens to the fd, so exit.
 			os.Exit(2)
@@ -239,7 +232,6 @@ func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
 	for _, udpConn := range f.writers {
 		c.RegisterReloadCallback(udpConn.ReloadConfig)
 	}
-	c.RegisterReloadCallback(f.reloadPSKs)
 }
 
 func (f *Interface) reloadCA(c *config.C) {
@@ -314,19 +306,6 @@ func (f *Interface) reloadFirewall(c *config.C) {
 		Info("New firewall has been installed")
 }
 
-func (f *Interface) reloadPSKs(c *config.C) {
-	psk, err := NewPskFromConfig(c, f.myVpnIp)
-	if err != nil {
-		f.l.WithError(err).Error("Error while reloading PSKs")
-		return
-	}
-
-	atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&f.psk)), unsafe.Pointer(psk))
-
-	f.l.WithField("pskMode", psk.mode).WithField("keysLen", len(psk.Cache)).
-		Info("New psks are in use")
-}
-
 func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
 	ticker := time.NewTicker(i)
 	defer ticker.Stop()
@@ -344,3 +323,10 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
 		}
 	}
 }
+
+func (f *Interface) Close() error {
+	atomic.StoreInt32(&f.closed, 1)
+
+	// Release the tun device
+	return f.inside.Close()
+}

lighthouse_test.go

@@ -8,8 +8,8 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/test"
 	"github.com/slackhq/nebula/udp"
-	"github.com/slackhq/nebula/util"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -46,7 +46,7 @@ func TestNewLhQuery(t *testing.T) {
 }
 
 func Test_lhStaticMapping(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	lh1 := "10.128.0.2"
 	lh1IP := net.ParseIP(lh1)
 
@@ -67,7 +67,7 @@ func Test_lhStaticMapping(t *testing.T) {
 }
 
 func BenchmarkLighthouseHandleRequest(b *testing.B) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	lh1 := "10.128.0.2"
 	lh1IP := net.ParseIP(lh1)
 
@@ -137,7 +137,7 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) {
 }
 
 func TestLighthouse_Memory(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 
 	myUdpAddr0 := &udp.Addr{IP: net.ParseIP("10.0.0.2"), Port: 4242}
 	myUdpAddr1 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4242}
@@ -266,7 +266,7 @@ func newLHHostUpdate(fromAddr *udp.Addr, vpnIp iputil.VpnIp, addrs []*udp.Addr,
 
 //TODO: this is a RemoteList test
 //func Test_lhRemoteAllowList(t *testing.T) {
-//	l := NewTestLogger()
+//	l := NewLogger()
 //	c := NewConfig(l)
 //	c.Settings["remoteallowlist"] = map[interface{}]interface{}{
 //		"10.20.0.0/12": false,

logger.go

@@ -1,7 +1,6 @@
 package nebula
 
 import (
-	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -10,38 +9,6 @@ import (
 	"github.com/slackhq/nebula/config"
 )
 
-type ContextualError struct {
-	RealError error
-	Fields    map[string]interface{}
-	Context   string
-}
-
-func NewContextualError(msg string, fields map[string]interface{}, realError error) ContextualError {
-	return ContextualError{Context: msg, Fields: fields, RealError: realError}
-}
-
-func (ce ContextualError) Error() string {
-	if ce.RealError == nil {
-		return ce.Context
-	}
-	return ce.RealError.Error()
-}
-
-func (ce ContextualError) Unwrap() error {
-	if ce.RealError == nil {
-		return errors.New(ce.Context)
-	}
-	return ce.RealError
-}
-
-func (ce *ContextualError) Log(lr *logrus.Logger) {
-	if ce.RealError != nil {
-		lr.WithFields(ce.Fields).WithError(ce.RealError).Error(ce.Context)
-	} else {
-		lr.WithFields(ce.Fields).Error(ce.Context)
-	}
-}
-
 func configLogger(l *logrus.Logger, c *config.C) error {
 	// set up our logging level
 	logLevel, err := logrus.ParseLevel(strings.ToLower(c.GetString("logging.level", "info")))

main.go

@@ -10,8 +10,10 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/sshd"
 	"github.com/slackhq/nebula/udp"
+	"github.com/slackhq/nebula/util"
 	"gopkg.in/yaml.v2"
 )
 
@@ -44,7 +46,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 
 	err := configLogger(l, c)
 	if err != nil {
-		return nil, NewContextualError("Failed to configure the logger", nil, err)
+		return nil, util.NewContextualError("Failed to configure the logger", nil, err)
 	}
 
 	c.RegisterReloadCallback(func(c *config.C) {
@@ -57,33 +59,25 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	caPool, err := loadCAFromConfig(l, c)
 	if err != nil {
 		//The errors coming out of loadCA are already nicely formatted
-		return nil, NewContextualError("Failed to load ca from config", nil, err)
+		return nil, util.NewContextualError("Failed to load ca from config", nil, err)
 	}
 	l.WithField("fingerprints", caPool.GetFingerprints()).Debug("Trusted CA fingerprints")
 
 	cs, err := NewCertStateFromConfig(c)
 	if err != nil {
 		//The errors coming out of NewCertStateFromConfig are already nicely formatted
-		return nil, NewContextualError("Failed to load certificate from config", nil, err)
+		return nil, util.NewContextualError("Failed to load certificate from config", nil, err)
 	}
 	l.WithField("cert", cs.certificate).Debug("Client nebula certificate")
 
 	fw, err := NewFirewallFromConfig(l, cs.certificate, c)
 	if err != nil {
-		return nil, NewContextualError("Error while loading firewall rules", nil, err)
+		return nil, util.NewContextualError("Error while loading firewall rules", nil, err)
 	}
 	l.WithField("firewallHash", fw.GetRuleHash()).Info("Firewall started")
 
 	// TODO: make sure mask is 4 bytes
 	tunCidr := cs.certificate.Details.Ips[0]
-	routes, err := parseRoutes(c, tunCidr)
-	if err != nil {
-		return nil, NewContextualError("Could not parse tun.routes", nil, err)
-	}
-	unsafeRoutes, err := parseUnsafeRoutes(c, tunCidr)
-	if err != nil {
-		return nil, NewContextualError("Could not parse tun.unsafe_routes", nil, err)
-	}
 
 	ssh, err := sshd.NewSSHServer(l.WithField("subsystem", "sshd"))
 	wireSSHReload(l, ssh, c)
@@ -91,15 +85,10 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	if c.GetBool("sshd.enabled", false) {
 		sshStart, err = configSSH(l, ssh, c)
 		if err != nil {
-			return nil, NewContextualError("Error while configuring the sshd", nil, err)
+			return nil, util.NewContextualError("Error while configuring the sshd", nil, err)
 		}
 	}
 
-	psk, err := NewPskFromConfig(c, iputil.Ip2VpnIp(tunCidr.IP))
-	if err != nil {
-		return nil, NewContextualError("Failed to create psk", nil, err)
-	}
-
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 	// All non system modifying configuration consumption should live above this line
 	// tun config, listeners, anything modifying the computer should be below
@@ -141,46 +130,21 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		l.WithField("duration", conntrackCacheTimeout).Info("Using routine-local conntrack cache")
 	}
 
-	var tun Inside
+	var tun overlay.Device
 	if !configTest {
 		c.CatchHUP(ctx)
 
-		switch {
-		case c.GetBool("tun.disabled", false):
-			tun = newDisabledTun(tunCidr, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
-		case tunFd != nil:
-			tun, err = newTunFromFd(
-				l,
-				*tunFd,
-				tunCidr,
-				c.GetInt("tun.mtu", DEFAULT_MTU),
-				routes,
-				unsafeRoutes,
-				c.GetInt("tun.tx_queue", 500),
-			)
-		default:
-			tun, err = newTun(
-				l,
-				c.GetString("tun.dev", ""),
-				tunCidr,
-				c.GetInt("tun.mtu", DEFAULT_MTU),
-				routes,
-				unsafeRoutes,
-				c.GetInt("tun.tx_queue", 500),
-				routines > 1,
-			)
-		}
-
+		tun, err = overlay.NewDeviceFromConfig(c, l, tunCidr, tunFd, routines)
 		if err != nil {
-			return nil, NewContextualError("Failed to get a tun/tap device", nil, err)
+			return nil, util.NewContextualError("Failed to get a tun/tap device", nil, err)
 		}
-	}
 
-	defer func() {
-		if reterr != nil {
-			tun.Close()
-		}
-	}()
+		defer func() {
+			if reterr != nil {
+				tun.Close()
+			}
+		}()
+	}
 
 	// set up our UDP listener
 	udpConns := make([]*udp.Conn, routines)
@@ -190,7 +154,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		for i := 0; i < routines; i++ {
 			udpServer, err := udp.NewListener(l, c.GetString("listen.host", "0.0.0.0"), port, routines > 1, c.GetInt("listen.batch", 64))
 			if err != nil {
-				return nil, NewContextualError("Failed to open udp listener", m{"queue": i}, err)
+				return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err)
 			}
 			udpServer.ReloadConfig(c)
 			udpConns[i] = udpServer
@@ -199,7 +163,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 			if port == 0 {
 				uPort, err := udpServer.LocalAddr()
 				if err != nil {
-					return nil, NewContextualError("Failed to get listening port", nil, err)
+					return nil, util.NewContextualError("Failed to get listening port", nil, err)
 				}
 				port = int(uPort.Port)
 			}
@@ -214,7 +178,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		for _, rawPreferredRange := range rawPreferredRanges {
 			_, preferredRange, err := net.ParseCIDR(rawPreferredRange)
 			if err != nil {
-				return nil, NewContextualError("Failed to parse preferred ranges", nil, err)
+				return nil, util.NewContextualError("Failed to parse preferred ranges", nil, err)
 			}
 			preferredRanges = append(preferredRanges, preferredRange)
 		}
@@ -227,7 +191,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	if rawLocalRange != "" {
 		_, localRange, err := net.ParseCIDR(rawLocalRange)
 		if err != nil {
-			return nil, NewContextualError("Failed to parse local_range", nil, err)
+			return nil, util.NewContextualError("Failed to parse local_range", nil, err)
 		}
 
 		// Check if the entry for local_range was already specified in
@@ -245,8 +209,6 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	}
 
 	hostMap := NewHostMap(l, "main", tunCidr, preferredRanges)
-
-	hostMap.addUnsafeRoutes(&unsafeRoutes)
 	hostMap.metricsEnabled = c.GetBool("stats.message_metrics", false)
 
 	l.WithField("network", hostMap.vpnCIDR).WithField("preferredRanges", hostMap.preferredRanges).Info("Main HostMap created")
@@ -266,7 +228,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 
 	// fatal if am_lighthouse is enabled but we are using an ephemeral port
 	if amLighthouse && (c.GetInt("listen.port", 0) == 0) {
-		return nil, NewContextualError("lighthouse.am_lighthouse enabled on node but no port number is set in config", nil, nil)
+		return nil, util.NewContextualError("lighthouse.am_lighthouse enabled on node but no port number is set in config", nil, nil)
 	}
 
 	// warn if am_lighthouse is enabled but upstream lighthouses exists
@@ -279,14 +241,18 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	for i, host := range rawLighthouseHosts {
 		ip := net.ParseIP(host)
 		if ip == nil {
-			return nil, NewContextualError("Unable to parse lighthouse host entry", m{"host": host, "entry": i + 1}, nil)
+			return nil, util.NewContextualError("Unable to parse lighthouse host entry", m{"host": host, "entry": i + 1}, nil)
 		}
 		if !tunCidr.Contains(ip) {
-			return nil, NewContextualError("lighthouse host is not in our subnet, invalid", m{"vpnIp": ip, "network": tunCidr.String()}, nil)
+			return nil, util.NewContextualError("lighthouse host is not in our subnet, invalid", m{"vpnIp": ip, "network": tunCidr.String()}, nil)
 		}
 		lighthouseHosts[i] = iputil.Ip2VpnIp(ip)
 	}
 
+	if !amLighthouse && len(lighthouseHosts) == 0 {
+		l.Warn("No lighthouses.hosts configured, this host will only be able to initiate tunnels with static_host_map entries")
+	}
+
 	lightHouse := NewLightHouse(
 		l,
 		amLighthouse,
@@ -303,13 +269,13 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 
 	remoteAllowList, err := NewRemoteAllowListFromConfig(c, "lighthouse.remote_allow_list", "lighthouse.remote_allow_ranges")
 	if err != nil {
-		return nil, NewContextualError("Invalid lighthouse.remote_allow_list", nil, err)
+		return nil, util.NewContextualError("Invalid lighthouse.remote_allow_list", nil, err)
 	}
 	lightHouse.SetRemoteAllowList(remoteAllowList)
 
 	localAllowList, err := NewLocalAllowListFromConfig(c, "lighthouse.local_allow_list")
 	if err != nil {
-		return nil, NewContextualError("Invalid lighthouse.local_allow_list", nil, err)
+		return nil, util.NewContextualError("Invalid lighthouse.local_allow_list", nil, err)
 	}
 	lightHouse.SetLocalAllowList(localAllowList)
 
@@ -318,21 +284,21 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		ip := net.ParseIP(fmt.Sprintf("%v", k))
 		vpnIp := iputil.Ip2VpnIp(ip)
 		if !tunCidr.Contains(ip) {
-			return nil, NewContextualError("static_host_map key is not in our subnet, invalid", m{"vpnIp": vpnIp, "network": tunCidr.String()}, nil)
+			return nil, util.NewContextualError("static_host_map key is not in our subnet, invalid", m{"vpnIp": vpnIp, "network": tunCidr.String()}, nil)
 		}
 		vals, ok := v.([]interface{})
 		if ok {
 			for _, v := range vals {
 				ip, port, err := udp.ParseIPAndPort(fmt.Sprintf("%v", v))
 				if err != nil {
-					return nil, NewContextualError("Static host address could not be parsed", m{"vpnIp": vpnIp}, err)
+					return nil, util.NewContextualError("Static host address could not be parsed", m{"vpnIp": vpnIp}, err)
 				}
 				lightHouse.AddStaticRemote(vpnIp, udp.NewAddr(ip, port))
 			}
 		} else {
 			ip, port, err := udp.ParseIPAndPort(fmt.Sprintf("%v", v))
 			if err != nil {
-				return nil, NewContextualError("Static host address could not be parsed", m{"vpnIp": vpnIp}, err)
+				return nil, util.NewContextualError("Static host address could not be parsed", m{"vpnIp": vpnIp}, err)
 			}
 			lightHouse.AddStaticRemote(vpnIp, udp.NewAddr(ip, port))
 		}
@@ -361,6 +327,10 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	handshakeManager := NewHandshakeManager(l, tunCidr, preferredRanges, hostMap, lightHouse, udpConns[0], handshakeConfig)
 	lightHouse.handshakeTrigger = handshakeManager.trigger
 
+	//TODO: These will be reused for psk
+	//handshakeMACKey := config.GetString("handshake_mac.key", "")
+	//handshakeAcceptedMACKeys := config.GetStringSlice("handshake_mac.accepted_keys", []string{})
+
 	serveDns := false
 	if c.GetBool("lighthouse.serve_dns", false) {
 		if c.GetBool("lighthouse.am_lighthouse", false) {
@@ -391,7 +361,6 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		version:                 buildVersion,
 		caPool:                  caPool,
 		disconnectInvalid:       c.GetBool("pki.disconnect_invalid", false),
-		psk:                     psk,
 
 		ConntrackCacheTimeout: conntrackCacheTimeout,
 		l:                     l,
@@ -428,7 +397,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	statsStart, err := startStats(l, c, buildVersion, configTest)
 
 	if err != nil {
-		return nil, NewContextualError("Failed to start stats emitter", nil, err)
+		return nil, util.NewContextualError("Failed to start stats emitter", nil, err)
 	}
 
 	if configTest {

noise.go

@@ -22,6 +22,7 @@ type NebulaCipherState struct {
 
 func NewNebulaCipherState(s *noise.CipherState) *NebulaCipherState {
 	return &NebulaCipherState{c: s.Cipher()}
+
 }
 
 func (s *NebulaCipherState) EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error) {

outside.go

@@ -349,12 +349,9 @@ func (f *Interface) handleRecvError(addr *udp.Addr, h *header.H) {
 		return
 	}
 
-	// We delete this host from the main hostmap
-	f.hostMap.DeleteHostInfo(hostinfo)
-	// We also delete it from pending to allow for
-	// fast reconnect. We must null the connectionstate
-	// or a counter reuse may happen
-	hostinfo.ConnectionState = nil
+	f.closeTunnel(hostinfo, false)
+	// We also delete it from pending hostmap to allow for
+	// fast reconnect.
 	f.handshakeManager.DeleteHostInfo(hostinfo)
 }
 

overlay/device.go

@@ -0,0 +1,17 @@
+package overlay
+
+import (
+	"io"
+	"net"
+
+	"github.com/slackhq/nebula/iputil"
+)
+
+type Device interface {
+	io.ReadWriteCloser
+	Activate() error
+	Cidr() *net.IPNet
+	Name() string
+	RouteFor(iputil.VpnIp) iputil.VpnIp
+	NewMultiQueueReader() (io.ReadWriteCloser, error)
+}

overlay/route.go

@@ -1,29 +1,45 @@
-package nebula
+package overlay
 
 import (
 	"fmt"
 	"math"
 	"net"
+	"runtime"
 	"strconv"
 
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/cidr"
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/iputil"
 )
 
-const DEFAULT_MTU = 1300
-
-type route struct {
-	mtu    int
-	metric int
-	route  *net.IPNet
-	via    *net.IP
+type Route struct {
+	MTU    int
+	Metric int
+	Cidr   *net.IPNet
+	Via    *iputil.VpnIp
 }
 
-func parseRoutes(c *config.C, network *net.IPNet) ([]route, error) {
+func makeRouteTree(l *logrus.Logger, routes []Route, allowMTU bool) (*cidr.Tree4, error) {
+	routeTree := cidr.NewTree4()
+	for _, r := range routes {
+		if !allowMTU && r.MTU > 0 {
+			l.WithField("route", r).Warnf("route MTU is not supported in %s", runtime.GOOS)
+		}
+
+		if r.Via != nil {
+			routeTree.AddCIDR(r.Cidr, *r.Via)
+		}
+	}
+	return routeTree, nil
+}
+
+func parseRoutes(c *config.C, network *net.IPNet) ([]Route, error) {
 	var err error
 
 	r := c.Get("tun.routes")
 	if r == nil {
-		return []route{}, nil
+		return []Route{}, nil
 	}
 
 	rawRoutes, ok := r.([]interface{})
@@ -32,10 +48,10 @@ func parseRoutes(c *config.C, network *net.IPNet) ([]route, error) {
 	}
 
 	if len(rawRoutes) < 1 {
-		return []route{}, nil
+		return []Route{}, nil
 	}
 
-	routes := make([]route, len(rawRoutes))
+	routes := make([]Route, len(rawRoutes))
 	for i, r := range rawRoutes {
 		m, ok := r.(map[interface{}]interface{})
 		if !ok {
@@ -64,20 +80,20 @@ func parseRoutes(c *config.C, network *net.IPNet) ([]route, error) {
 			return nil, fmt.Errorf("entry %v.route in tun.routes is not present", i+1)
 		}
 
-		r := route{
-			mtu: mtu,
+		r := Route{
+			MTU: mtu,
 		}
 
-		_, r.route, err = net.ParseCIDR(fmt.Sprintf("%v", rRoute))
+		_, r.Cidr, err = net.ParseCIDR(fmt.Sprintf("%v", rRoute))
 		if err != nil {
 			return nil, fmt.Errorf("entry %v.route in tun.routes failed to parse: %v", i+1, err)
 		}
 
-		if !ipWithin(network, r.route) {
+		if !ipWithin(network, r.Cidr) {
 			return nil, fmt.Errorf(
 				"entry %v.route in tun.routes is not contained within the network attached to the certificate; route: %v, network: %v",
 				i+1,
-				r.route.String(),
+				r.Cidr.String(),
 				network.String(),
 			)
 		}
@@ -88,12 +104,12 @@ func parseRoutes(c *config.C, network *net.IPNet) ([]route, error) {
 	return routes, nil
 }
 
-func parseUnsafeRoutes(c *config.C, network *net.IPNet) ([]route, error) {
+func parseUnsafeRoutes(c *config.C, network *net.IPNet) ([]Route, error) {
 	var err error
 
 	r := c.Get("tun.unsafe_routes")
 	if r == nil {
-		return []route{}, nil
+		return []Route{}, nil
 	}
 
 	rawRoutes, ok := r.([]interface{})
@@ -102,31 +118,29 @@ func parseUnsafeRoutes(c *config.C, network *net.IPNet) ([]route, error) {
 	}
 
 	if len(rawRoutes) < 1 {
-		return []route{}, nil
+		return []Route{}, nil
 	}
 
-	routes := make([]route, len(rawRoutes))
+	routes := make([]Route, len(rawRoutes))
 	for i, r := range rawRoutes {
 		m, ok := r.(map[interface{}]interface{})
 		if !ok {
 			return nil, fmt.Errorf("entry %v in tun.unsafe_routes is invalid", i+1)
 		}
 
-		rMtu, ok := m["mtu"]
-		if !ok {
-			rMtu = c.GetInt("tun.mtu", DEFAULT_MTU)
-		}
-
-		mtu, ok := rMtu.(int)
-		if !ok {
-			mtu, err = strconv.Atoi(rMtu.(string))
-			if err != nil {
-				return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err)
+		var mtu int
+		if rMtu, ok := m["mtu"]; ok {
+			mtu, ok = rMtu.(int)
+			if !ok {
+				mtu, err = strconv.Atoi(rMtu.(string))
+				if err != nil {
+					return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err)
+				}
 			}
-		}
 
-		if mtu < 500 {
-			return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is below 500: %v", i+1, mtu)
+			if mtu != 0 && mtu < 500 {
+				return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is below 500: %v", i+1, mtu)
+			}
 		}
 
 		rMetric, ok := m["metric"]
@@ -166,22 +180,24 @@ func parseUnsafeRoutes(c *config.C, network *net.IPNet) ([]route, error) {
 			return nil, fmt.Errorf("entry %v.route in tun.unsafe_routes is not present", i+1)
 		}
 
-		r := route{
-			via:    &nVia,
-			mtu:    mtu,
-			metric: metric,
+		viaVpnIp := iputil.Ip2VpnIp(nVia)
+
+		r := Route{
+			Via:    &viaVpnIp,
+			MTU:    mtu,
+			Metric: metric,
 		}
 
-		_, r.route, err = net.ParseCIDR(fmt.Sprintf("%v", rRoute))
+		_, r.Cidr, err = net.ParseCIDR(fmt.Sprintf("%v", rRoute))
 		if err != nil {
 			return nil, fmt.Errorf("entry %v.route in tun.unsafe_routes failed to parse: %v", i+1, err)
 		}
 
-		if ipWithin(network, r.route) {
+		if ipWithin(network, r.Cidr) {
 			return nil, fmt.Errorf(
 				"entry %v.route in tun.unsafe_routes is contained within the network attached to the certificate; route: %v, network: %v",
 				i+1,
-				r.route.String(),
+				r.Cidr.String(),
 				network.String(),
 			)
 		}

overlay/route_test.go

@@ -1,4 +1,4 @@
-package nebula
+package overlay
 
 import (
 	"fmt"
@@ -6,12 +6,13 @@ import (
 	"testing"
 
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/util"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 )
 
 func Test_parseRoutes(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	c := config.NewC(l)
 	_, n, _ := net.ParseCIDR("10.0.0.0/24")
 
@@ -91,12 +92,12 @@ func Test_parseRoutes(t *testing.T) {
 
 	tested := 0
 	for _, r := range routes {
-		if r.mtu == 8000 {
-			assert.Equal(t, "10.0.0.1/32", r.route.String())
+		if r.MTU == 8000 {
+			assert.Equal(t, "10.0.0.1/32", r.Cidr.String())
 			tested++
 		} else {
-			assert.Equal(t, 9000, r.mtu)
-			assert.Equal(t, "10.0.0.0/29", r.route.String())
+			assert.Equal(t, 9000, r.MTU)
+			assert.Equal(t, "10.0.0.0/29", r.Cidr.String())
 			tested++
 		}
 	}
@@ -107,7 +108,7 @@ func Test_parseRoutes(t *testing.T) {
 }
 
 func Test_parseUnsafeRoutes(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	c := config.NewC(l)
 	_, n, _ := net.ParseCIDR("10.0.0.0/24")
 
@@ -190,7 +191,7 @@ func Test_parseUnsafeRoutes(t *testing.T) {
 	c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "route": "1.0.0.0/8"}}}
 	routes, err = parseUnsafeRoutes(c, n)
 	assert.Len(t, routes, 1)
-	assert.Equal(t, DEFAULT_MTU, routes[0].mtu)
+	assert.Equal(t, 0, routes[0].MTU)
 
 	// bad mtu
 	c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "nope"}}}
@@ -216,17 +217,17 @@ func Test_parseUnsafeRoutes(t *testing.T) {
 
 	tested := 0
 	for _, r := range routes {
-		if r.mtu == 8000 {
-			assert.Equal(t, "1.0.0.1/32", r.route.String())
+		if r.MTU == 8000 {
+			assert.Equal(t, "1.0.0.1/32", r.Cidr.String())
 			tested++
-		} else if r.mtu == 9000 {
-			assert.Equal(t, 9000, r.mtu)
-			assert.Equal(t, "1.0.0.0/29", r.route.String())
+		} else if r.MTU == 9000 {
+			assert.Equal(t, 9000, r.MTU)
+			assert.Equal(t, "1.0.0.0/29", r.Cidr.String())
 			tested++
 		} else {
-			assert.Equal(t, 1500, r.mtu)
-			assert.Equal(t, 1234, r.metric)
-			assert.Equal(t, "1.0.0.2/32", r.route.String())
+			assert.Equal(t, 1500, r.MTU)
+			assert.Equal(t, 1234, r.Metric)
+			assert.Equal(t, "1.0.0.2/32", r.Cidr.String())
 			tested++
 		}
 	}
@@ -235,3 +236,35 @@ func Test_parseUnsafeRoutes(t *testing.T) {
 		t.Fatal("Did not see both unsafe_routes")
 	}
 }
+
+func Test_makeRouteTree(t *testing.T) {
+	l := test.NewLogger()
+	c := config.NewC(l)
+	_, n, _ := net.ParseCIDR("10.0.0.0/24")
+
+	c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{
+		map[interface{}]interface{}{"via": "192.168.0.1", "route": "1.0.0.0/28"},
+		map[interface{}]interface{}{"via": "192.168.0.2", "route": "1.0.0.1/32"},
+	}}
+	routes, err := parseUnsafeRoutes(c, n)
+	assert.NoError(t, err)
+	assert.Len(t, routes, 2)
+	routeTree, err := makeRouteTree(l, routes, true)
+	assert.NoError(t, err)
+
+	ip := iputil.Ip2VpnIp(net.ParseIP("1.0.0.2"))
+	r := routeTree.MostSpecificContains(ip)
+	assert.NotNil(t, r)
+	assert.IsType(t, iputil.VpnIp(0), r)
+	assert.EqualValues(t, iputil.Ip2VpnIp(net.ParseIP("192.168.0.1")), r)
+
+	ip = iputil.Ip2VpnIp(net.ParseIP("1.0.0.1"))
+	r = routeTree.MostSpecificContains(ip)
+	assert.NotNil(t, r)
+	assert.IsType(t, iputil.VpnIp(0), r)
+	assert.EqualValues(t, iputil.Ip2VpnIp(net.ParseIP("192.168.0.2")), r)
+
+	ip = iputil.Ip2VpnIp(net.ParseIP("1.1.0.1"))
+	r = routeTree.MostSpecificContains(ip)
+	assert.Nil(t, r)
+}

overlay/tun.go

@@ -0,0 +1,51 @@
+package overlay
+
+import (
+	"net"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/util"
+)
+
+const DefaultMTU = 1300
+
+func NewDeviceFromConfig(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, fd *int, routines int) (Device, error) {
+	routes, err := parseRoutes(c, tunCidr)
+	if err != nil {
+		return nil, util.NewContextualError("Could not parse tun.routes", nil, err)
+	}
+
+	unsafeRoutes, err := parseUnsafeRoutes(c, tunCidr)
+	if err != nil {
+		return nil, util.NewContextualError("Could not parse tun.unsafe_routes", nil, err)
+	}
+	routes = append(routes, unsafeRoutes...)
+
+	switch {
+	case c.GetBool("tun.disabled", false):
+		tun := newDisabledTun(tunCidr, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
+		return tun, nil
+
+	case fd != nil:
+		return newTunFromFd(
+			l,
+			*fd,
+			tunCidr,
+			c.GetInt("tun.mtu", DefaultMTU),
+			routes,
+			c.GetInt("tun.tx_queue", 500),
+		)
+
+	default:
+		return newTun(
+			l,
+			c.GetString("tun.dev", ""),
+			tunCidr,
+			c.GetInt("tun.mtu", DefaultMTU),
+			routes,
+			c.GetInt("tun.tx_queue", 500),
+			routines > 1,
+		)
+	}
+}

overlay/tun_android.go

@@ -0,0 +1,61 @@
+//go:build !e2e_testing
+// +build !e2e_testing
+
+package overlay
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"runtime"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/iputil"
+)
+
+type tun struct {
+	io.ReadWriteCloser
+	fd   int
+	cidr *net.IPNet
+	l    *logrus.Logger
+}
+
+func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, _ int, routes []Route, _ int) (*tun, error) {
+	if len(routes) > 0 {
+		return nil, fmt.Errorf("routes are not supported in %s", runtime.GOOS)
+	}
+
+	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
+
+	return &tun{
+		ReadWriteCloser: file,
+		fd:              int(file.Fd()),
+		cidr:            cidr,
+		l:               l,
+	}, nil
+}
+
+func newTun(_ *logrus.Logger, _ string, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*tun, error) {
+	return nil, fmt.Errorf("newTun not supported in Android")
+}
+
+func (t *tun) RouteFor(iputil.VpnIp) iputil.VpnIp {
+	return 0
+}
+
+func (t tun) Activate() error {
+	return nil
+}
+
+func (t *tun) Cidr() *net.IPNet {
+	return t.cidr
+}
+
+func (t *tun) Name() string {
+	return "android"
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
+}

overlay/tun_darwin.go

@@ -0,0 +1,425 @@
+//go:build !ios && !e2e_testing
+// +build !ios,!e2e_testing
+
+package overlay
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"syscall"
+	"unsafe"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/iputil"
+	netroute "golang.org/x/net/route"
+	"golang.org/x/sys/unix"
+)
+
+type tun struct {
+	io.ReadWriteCloser
+	Device     string
+	cidr       *net.IPNet
+	DefaultMTU int
+	Routes     []Route
+	routeTree  *cidr.Tree4
+	l          *logrus.Logger
+
+	// cache out buffer since we need to prepend 4 bytes for tun metadata
+	out []byte
+}
+
+type sockaddrCtl struct {
+	scLen      uint8
+	scFamily   uint8
+	ssSysaddr  uint16
+	scID       uint32
+	scUnit     uint32
+	scReserved [5]uint32
+}
+
+type ifReq struct {
+	Name  [16]byte
+	Flags uint16
+	pad   [8]byte
+}
+
+func ioctl(a1, a2, a3 uintptr) error {
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, a1, a2, a3)
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}
+
+var sockaddrCtlSize uintptr = 32
+
+const (
+	_SYSPROTO_CONTROL = 2              //define SYSPROTO_CONTROL 2 /* kernel control protocol */
+	_AF_SYS_CONTROL   = 2              //#define AF_SYS_CONTROL 2 /* corresponding sub address type */
+	_PF_SYSTEM        = unix.AF_SYSTEM //#define PF_SYSTEM AF_SYSTEM
+	_CTLIOCGINFO      = 3227799043     //#define CTLIOCGINFO     _IOWR('N', 3, struct ctl_info)
+	utunControlName   = "com.apple.net.utun_control"
+)
+
+type ifreqAddr struct {
+	Name [16]byte
+	Addr unix.RawSockaddrInet4
+	pad  [8]byte
+}
+
+type ifreqMTU struct {
+	Name [16]byte
+	MTU  int32
+	pad  [8]byte
+}
+
+func newTun(l *logrus.Logger, name string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool) (*tun, error) {
+	routeTree, err := makeRouteTree(l, routes, false)
+	if err != nil {
+		return nil, err
+	}
+
+	ifIndex := -1
+	if name != "" && name != "utun" {
+		_, err := fmt.Sscanf(name, "utun%d", &ifIndex)
+		if err != nil || ifIndex < 0 {
+			// NOTE: we don't make this error so we don't break existing
+			// configs that set a name before it was used.
+			l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring")
+			ifIndex = -1
+		}
+	}
+
+	fd, err := unix.Socket(_PF_SYSTEM, unix.SOCK_DGRAM, _SYSPROTO_CONTROL)
+	if err != nil {
+		return nil, fmt.Errorf("system socket: %v", err)
+	}
+
+	var ctlInfo = &struct {
+		ctlID   uint32
+		ctlName [96]byte
+	}{}
+
+	copy(ctlInfo.ctlName[:], utunControlName)
+
+	err = ioctl(uintptr(fd), uintptr(_CTLIOCGINFO), uintptr(unsafe.Pointer(ctlInfo)))
+	if err != nil {
+		return nil, fmt.Errorf("CTLIOCGINFO: %v", err)
+	}
+
+	sc := sockaddrCtl{
+		scLen:     uint8(sockaddrCtlSize),
+		scFamily:  unix.AF_SYSTEM,
+		ssSysaddr: _AF_SYS_CONTROL,
+		scID:      ctlInfo.ctlID,
+		scUnit:    uint32(ifIndex) + 1,
+	}
+
+	_, _, errno := unix.RawSyscall(
+		unix.SYS_CONNECT,
+		uintptr(fd),
+		uintptr(unsafe.Pointer(&sc)),
+		sockaddrCtlSize,
+	)
+	if errno != 0 {
+		return nil, fmt.Errorf("SYS_CONNECT: %v", errno)
+	}
+
+	var ifName struct {
+		name [16]byte
+	}
+	ifNameSize := uintptr(len(ifName.name))
+	_, _, errno = syscall.Syscall6(syscall.SYS_GETSOCKOPT, uintptr(fd),
+		2, // SYSPROTO_CONTROL
+		2, // UTUN_OPT_IFNAME
+		uintptr(unsafe.Pointer(&ifName)),
+		uintptr(unsafe.Pointer(&ifNameSize)), 0)
+	if errno != 0 {
+		return nil, fmt.Errorf("SYS_GETSOCKOPT: %v", errno)
+	}
+	name = string(ifName.name[:ifNameSize-1])
+
+	err = syscall.SetNonblock(fd, true)
+	if err != nil {
+		return nil, fmt.Errorf("SetNonblock: %v", err)
+	}
+
+	file := os.NewFile(uintptr(fd), "")
+
+	tun := &tun{
+		ReadWriteCloser: file,
+		Device:          name,
+		cidr:            cidr,
+		DefaultMTU:      defaultMTU,
+		Routes:          routes,
+		routeTree:       routeTree,
+		l:               l,
+	}
+
+	return tun, nil
+}
+
+func (t *tun) deviceBytes() (o [16]byte) {
+	for i, c := range t.Device {
+		o[i] = byte(c)
+	}
+	return
+}
+
+func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int) (*tun, error) {
+	return nil, fmt.Errorf("newTunFromFd not supported in Darwin")
+}
+
+func (t *tun) Close() error {
+	if t.ReadWriteCloser != nil {
+		return t.ReadWriteCloser.Close()
+	}
+	return nil
+}
+
+func (t *tun) Activate() error {
+	devName := t.deviceBytes()
+
+	var addr, mask [4]byte
+
+	copy(addr[:], t.cidr.IP.To4())
+	copy(mask[:], t.cidr.Mask)
+
+	s, err := unix.Socket(
+		unix.AF_INET,
+		unix.SOCK_DGRAM,
+		unix.IPPROTO_IP,
+	)
+
+	if err != nil {
+		return err
+	}
+
+	fd := uintptr(s)
+
+	ifra := ifreqAddr{
+		Name: devName,
+		Addr: unix.RawSockaddrInet4{
+			Family: unix.AF_INET,
+			Addr:   addr,
+		},
+	}
+
+	// Set the device ip address
+	if err = ioctl(fd, unix.SIOCSIFADDR, uintptr(unsafe.Pointer(&ifra))); err != nil {
+		return fmt.Errorf("failed to set tun address: %s", err)
+	}
+
+	// Set the device network
+	ifra.Addr.Addr = mask
+	if err = ioctl(fd, unix.SIOCSIFNETMASK, uintptr(unsafe.Pointer(&ifra))); err != nil {
+		return fmt.Errorf("failed to set tun netmask: %s", err)
+	}
+
+	// Set the device name
+	ifrf := ifReq{Name: devName}
+	if err = ioctl(fd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
+		return fmt.Errorf("failed to set tun device name: %s", err)
+	}
+
+	// Set the MTU on the device
+	ifm := ifreqMTU{Name: devName, MTU: int32(t.DefaultMTU)}
+	if err = ioctl(fd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
+		return fmt.Errorf("failed to set tun mtu: %v", err)
+	}
+
+	/*
+		// Set the transmit queue length
+		ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)}
+		if err = ioctl(fd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil {
+			// If we can't set the queue length nebula will still work but it may lead to packet loss
+			l.WithError(err).Error("Failed to set tun tx queue length")
+		}
+	*/
+
+	// Bring up the interface
+	ifrf.Flags = ifrf.Flags | unix.IFF_UP
+	if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
+		return fmt.Errorf("failed to bring the tun device up: %s", err)
+	}
+
+	routeSock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
+	}
+	defer func() {
+		unix.Shutdown(routeSock, unix.SHUT_RDWR)
+		err := unix.Close(routeSock)
+		if err != nil {
+			t.l.WithError(err).Error("failed to close AF_ROUTE socket")
+		}
+	}()
+
+	routeAddr := &netroute.Inet4Addr{}
+	maskAddr := &netroute.Inet4Addr{}
+	linkAddr, err := getLinkAddr(t.Device)
+	if err != nil {
+		return err
+	}
+	if linkAddr == nil {
+		return fmt.Errorf("unable to discover link_addr for tun interface")
+	}
+
+	copy(routeAddr.IP[:], addr[:])
+	copy(maskAddr.IP[:], mask[:])
+	err = addRoute(routeSock, routeAddr, maskAddr, linkAddr)
+	if err != nil {
+		if errors.Is(err, unix.EEXIST) {
+			err = fmt.Errorf("unable to add tun route, identical route already exists: %s", t.cidr)
+		}
+		return err
+	}
+
+	// Run the interface
+	ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
+	if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
+		return fmt.Errorf("failed to run tun device: %s", err)
+	}
+
+	// Unsafe path routes
+	for _, r := range t.Routes {
+		if r.Via == nil {
+			// We don't allow route MTUs so only install routes with a via
+			continue
+		}
+
+		copy(routeAddr.IP[:], r.Cidr.IP.To4())
+		copy(maskAddr.IP[:], net.IP(r.Cidr.Mask).To4())
+
+		err = addRoute(routeSock, routeAddr, maskAddr, linkAddr)
+		if err != nil {
+			if errors.Is(err, unix.EEXIST) {
+				t.l.WithField("route", r.Cidr).
+					Warnf("unable to add unsafe_route, identical route already exists")
+			} else {
+				return err
+			}
+		}
+
+		// TODO how to set metric
+	}
+
+	return nil
+}
+
+func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
+	r := t.routeTree.MostSpecificContains(ip)
+	if r != nil {
+		return r.(iputil.VpnIp)
+	}
+
+	return 0
+}
+
+// Get the LinkAddr for the interface of the given name
+// TODO: Is there an easier way to fetch this when we create the interface?
+// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
+func getLinkAddr(name string) (*netroute.LinkAddr, error) {
+	rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0)
+	if err != nil {
+		return nil, err
+	}
+	msgs, err := netroute.ParseRIB(unix.NET_RT_IFLIST, rib)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, m := range msgs {
+		switch m := m.(type) {
+		case *netroute.InterfaceMessage:
+			if m.Name == name {
+				sa, ok := m.Addrs[unix.RTAX_IFP].(*netroute.LinkAddr)
+				if ok {
+					return sa, nil
+				}
+			}
+		}
+	}
+
+	return nil, nil
+}
+
+func addRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) error {
+	r := netroute.RouteMessage{
+		Version: unix.RTM_VERSION,
+		Type:    unix.RTM_ADD,
+		Flags:   unix.RTF_UP,
+		Seq:     1,
+		Addrs: []netroute.Addr{
+			unix.RTAX_DST:     addr,
+			unix.RTAX_GATEWAY: link,
+			unix.RTAX_NETMASK: mask,
+		},
+	}
+
+	data, err := r.Marshal()
+	if err != nil {
+		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
+	}
+	_, err = unix.Write(sock, data[:])
+	if err != nil {
+		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
+	}
+
+	return nil
+}
+
+func (t *tun) Read(to []byte) (int, error) {
+
+	buf := make([]byte, len(to)+4)
+
+	n, err := t.ReadWriteCloser.Read(buf)
+
+	copy(to, buf[4:])
+	return n - 4, err
+}
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	buf := t.out
+	if cap(buf) < len(from)+4 {
+		buf = make([]byte, len(from)+4)
+		t.out = buf
+	}
+	buf = buf[:len(from)+4]
+
+	if len(from) == 0 {
+		return 0, syscall.EIO
+	}
+
+	// Determine the IP Family for the NULL L2 Header
+	ipVer := from[0] >> 4
+	if ipVer == 4 {
+		buf[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		buf[3] = syscall.AF_INET6
+	} else {
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+
+	copy(buf[4:], from)
+
+	n, err := t.ReadWriteCloser.Write(buf)
+	return n - 4, err
+}
+
+func (t *tun) Cidr() *net.IPNet {
+	return t.cidr
+}
+
+func (t *tun) Name() string {
+	return t.Device
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
+}

overlay/tun_disabled.go

@@ -1,4 +1,4 @@
-package nebula
+package overlay
 
 import (
 	"encoding/binary"
@@ -9,6 +9,7 @@ import (
 
 	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/iputil"
 )
 
 type disabledTun struct {
@@ -43,11 +44,15 @@ func (*disabledTun) Activate() error {
 	return nil
 }
 
-func (t *disabledTun) CidrNet() *net.IPNet {
+func (*disabledTun) RouteFor(iputil.VpnIp) iputil.VpnIp {
+	return 0
+}
+
+func (t *disabledTun) Cidr() *net.IPNet {
 	return t.cidr
 }
 
-func (*disabledTun) DeviceName() string {
+func (*disabledTun) Name() string {
 	return "disabled"
 }
 
@@ -71,7 +76,8 @@ func (t *disabledTun) Read(b []byte) (int, error) {
 
 func (t *disabledTun) handleICMPEchoRequest(b []byte) bool {
 	// Return early if this is not a simple ICMP Echo Request
-	if !(len(b) >= 28 && len(b) <= mtu && b[0] == 0x45 && b[9] == 0x01 && b[20] == 0x08) {
+	//TODO: make constants out of these
+	if !(len(b) >= 28 && len(b) <= 9001 && b[0] == 0x45 && b[9] == 0x01 && b[20] == 0x08) {
 		return false
 	}
 
@@ -122,11 +128,6 @@ func (t *disabledTun) Write(b []byte) (int, error) {
 	return len(b), nil
 }
 
-func (t *disabledTun) WriteRaw(b []byte) error {
-	_, err := t.Write(b)
-	return err
-}
-
 func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return t, nil
 }

overlay/tun_freebsd.go

@@ -0,0 +1,122 @@
+//go:build !e2e_testing
+// +build !e2e_testing
+
+package overlay
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/iputil"
+)
+
+var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
+
+type tun struct {
+	Device    string
+	cidr      *net.IPNet
+	MTU       int
+	Routes    []Route
+	routeTree *cidr.Tree4
+	l         *logrus.Logger
+
+	io.ReadWriteCloser
+}
+
+func (t *tun) Close() error {
+	if t.ReadWriteCloser != nil {
+		return t.ReadWriteCloser.Close()
+	}
+	return nil
+}
+
+func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int) (*tun, error) {
+	return nil, fmt.Errorf("newTunFromFd not supported in FreeBSD")
+}
+
+func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool) (*tun, error) {
+	routeTree, err := makeRouteTree(l, routes, false)
+	if err != nil {
+		return nil, err
+	}
+
+	if strings.HasPrefix(deviceName, "/dev/") {
+		deviceName = strings.TrimPrefix(deviceName, "/dev/")
+	}
+	if !deviceNameRE.MatchString(deviceName) {
+		return nil, fmt.Errorf("tun.dev must match `tun[0-9]+`")
+	}
+	return &tun{
+		Device:    deviceName,
+		cidr:      cidr,
+		MTU:       defaultMTU,
+		Routes:    routes,
+		routeTree: routeTree,
+		l:         l,
+	}, nil
+}
+
+func (t *tun) Activate() error {
+	var err error
+	t.ReadWriteCloser, err = os.OpenFile("/dev/"+t.Device, os.O_RDWR, 0)
+	if err != nil {
+		return fmt.Errorf("activate failed: %v", err)
+	}
+
+	// TODO use syscalls instead of exec.Command
+	t.l.Debug("command: ifconfig", t.Device, t.cidr.String(), t.cidr.IP.String())
+	if err = exec.Command("/sbin/ifconfig", t.Device, t.cidr.String(), t.cidr.IP.String()).Run(); err != nil {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
+	}
+	t.l.Debug("command: route", "-n", "add", "-net", t.cidr.String(), "-interface", t.Device)
+	if err = exec.Command("/sbin/route", "-n", "add", "-net", t.cidr.String(), "-interface", t.Device).Run(); err != nil {
+		return fmt.Errorf("failed to run 'route add': %s", err)
+	}
+	t.l.Debug("command: ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU))
+	if err = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU)).Run(); err != nil {
+		return fmt.Errorf("failed to run 'ifconfig': %s", err)
+	}
+	// Unsafe path routes
+	for _, r := range t.Routes {
+		if r.Via == nil {
+			// We don't allow route MTUs so only install routes with a via
+			continue
+		}
+
+		t.l.Debug("command: route", "-n", "add", "-net", r.Cidr.String(), "-interface", t.Device)
+		if err = exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), "-interface", t.Device).Run(); err != nil {
+			return fmt.Errorf("failed to run 'route add' for unsafe_route %s: %s", r.Cidr.String(), err)
+		}
+	}
+
+	return nil
+}
+
+func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
+	r := t.routeTree.MostSpecificContains(ip)
+	if r != nil {
+		return r.(iputil.VpnIp)
+	}
+
+	return 0
+}
+
+func (t *tun) Cidr() *net.IPNet {
+	return t.cidr
+}
+
+func (t *tun) Name() string {
+	return t.Device
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
+}

overlay/tun_ios.go

@@ -0,0 +1,117 @@
+//go:build ios && !e2e_testing
+// +build ios,!e2e_testing
+
+package overlay
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"runtime"
+	"sync"
+	"syscall"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/iputil"
+)
+
+type tun struct {
+	io.ReadWriteCloser
+	cidr *net.IPNet
+}
+
+func newTun(_ *logrus.Logger, _ string, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*tun, error) {
+	return nil, fmt.Errorf("newTun not supported in iOS")
+}
+
+func newTunFromFd(_ *logrus.Logger, deviceFd int, cidr *net.IPNet, _ int, routes []Route, _ int) (*tun, error) {
+	if len(routes) > 0 {
+		return nil, fmt.Errorf("routes are not supported in %s", runtime.GOOS)
+	}
+
+	file := os.NewFile(uintptr(deviceFd), "/dev/tun")
+	return &tun{
+		cidr:            cidr,
+		ReadWriteCloser: &tunReadCloser{f: file},
+	}, nil
+}
+
+func (t *tun) Activate() error {
+	return nil
+}
+
+func (t *tun) RouteFor(iputil.VpnIp) iputil.VpnIp {
+	return 0
+}
+
+// The following is hoisted up from water, we do this so we can inject our own fd on iOS
+type tunReadCloser struct {
+	f io.ReadWriteCloser
+
+	rMu  sync.Mutex
+	rBuf []byte
+
+	wMu  sync.Mutex
+	wBuf []byte
+}
+
+func (tr *tunReadCloser) Read(to []byte) (int, error) {
+	tr.rMu.Lock()
+	defer tr.rMu.Unlock()
+
+	if cap(tr.rBuf) < len(to)+4 {
+		tr.rBuf = make([]byte, len(to)+4)
+	}
+	tr.rBuf = tr.rBuf[:len(to)+4]
+
+	n, err := tr.f.Read(tr.rBuf)
+	copy(to, tr.rBuf[4:])
+	return n - 4, err
+}
+
+func (tr *tunReadCloser) Write(from []byte) (int, error) {
+	if len(from) == 0 {
+		return 0, syscall.EIO
+	}
+
+	tr.wMu.Lock()
+	defer tr.wMu.Unlock()
+
+	if cap(tr.wBuf) < len(from)+4 {
+		tr.wBuf = make([]byte, len(from)+4)
+	}
+	tr.wBuf = tr.wBuf[:len(from)+4]
+
+	// Determine the IP Family for the NULL L2 Header
+	ipVer := from[0] >> 4
+	if ipVer == 4 {
+		tr.wBuf[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		tr.wBuf[3] = syscall.AF_INET6
+	} else {
+		return 0, errors.New("unable to determine IP version from packet")
+	}
+
+	copy(tr.wBuf[4:], from)
+
+	n, err := tr.f.Write(tr.wBuf)
+	return n - 4, err
+}
+
+func (tr *tunReadCloser) Close() error {
+	return tr.f.Close()
+}
+
+func (t *tun) Cidr() *net.IPNet {
+	return t.cidr
+}
+
+func (t *tun) Name() string {
+	return "iOS"
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
+}

overlay/tun_linux.go

@@ -1,7 +1,7 @@
 //go:build !android && !e2e_testing
 // +build !android,!e2e_testing
 
-package nebula
+package overlay
 
 import (
 	"fmt"
@@ -12,21 +12,23 @@ import (
 	"unsafe"
 
 	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/iputil"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
 )
 
-type Tun struct {
+type tun struct {
 	io.ReadWriteCloser
-	fd           int
-	Device       string
-	Cidr         *net.IPNet
-	MaxMTU       int
-	DefaultMTU   int
-	TXQueueLen   int
-	Routes       []route
-	UnsafeRoutes []route
-	l            *logrus.Logger
+	fd         int
+	Device     string
+	cidr       *net.IPNet
+	MaxMTU     int
+	DefaultMTU int
+	TXQueueLen int
+	Routes     []Route
+	routeTree  *cidr.Tree4
+	l          *logrus.Logger
 }
 
 type ifReq struct {
@@ -43,26 +45,6 @@ func ioctl(a1, a2, a3 uintptr) error {
 	return nil
 }
 
-/*
-func ipv4(addr string) (o [4]byte, err error) {
-	ip := net.ParseIP(addr).To4()
-	if ip == nil {
-		err = fmt.Errorf("failed to parse addr %s", addr)
-		return
-	}
-	for i, b := range ip {
-		o[i] = b
-	}
-	return
-}
-*/
-
-const (
-	cIFF_TUN         = 0x0001
-	cIFF_NO_PI       = 0x1000
-	cIFF_MULTI_QUEUE = 0x0100
-)
-
 type ifreqAddr struct {
 	Name [16]byte
 	Addr unix.RawSockaddrInet4
@@ -81,34 +63,37 @@ type ifreqQLEN struct {
 	pad   [8]byte
 }
 
-func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
+func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []Route, txQueueLen int) (*tun, error) {
+	routeTree, err := makeRouteTree(l, routes, true)
+	if err != nil {
+		return nil, err
+	}
 
 	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
 
-	ifce = &Tun{
+	return &tun{
 		ReadWriteCloser: file,
 		fd:              int(file.Fd()),
 		Device:          "tun0",
-		Cidr:            cidr,
+		cidr:            cidr,
 		DefaultMTU:      defaultMTU,
 		TXQueueLen:      txQueueLen,
 		Routes:          routes,
-		UnsafeRoutes:    unsafeRoutes,
+		routeTree:       routeTree,
 		l:               l,
-	}
-	return
+	}, nil
 }
 
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
+func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, txQueueLen int, multiqueue bool) (*tun, error) {
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
 	}
 
 	var req ifReq
-	req.Flags = uint16(cIFF_TUN | cIFF_NO_PI)
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
 	if multiqueue {
-		req.Flags |= cIFF_MULTI_QUEUE
+		req.Flags |= unix.IFF_MULTI_QUEUE
 	}
 	copy(req.Name[:], deviceName)
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
@@ -120,35 +105,43 @@ func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int
 
 	maxMTU := defaultMTU
 	for _, r := range routes {
-		if r.mtu > maxMTU {
-			maxMTU = r.mtu
+		if r.MTU == 0 {
+			r.MTU = defaultMTU
+		}
+
+		if r.MTU > maxMTU {
+			maxMTU = r.MTU
 		}
 	}
 
-	ifce = &Tun{
+	routeTree, err := makeRouteTree(l, routes, true)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tun{
 		ReadWriteCloser: file,
 		fd:              int(file.Fd()),
 		Device:          name,
-		Cidr:            cidr,
+		cidr:            cidr,
 		MaxMTU:          maxMTU,
 		DefaultMTU:      defaultMTU,
 		TXQueueLen:      txQueueLen,
 		Routes:          routes,
-		UnsafeRoutes:    unsafeRoutes,
+		routeTree:       routeTree,
 		l:               l,
-	}
-	return
+	}, nil
 }
 
-func (c *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
 	}
 
 	var req ifReq
-	req.Flags = uint16(cIFF_TUN | cIFF_NO_PI | cIFF_MULTI_QUEUE)
-	copy(req.Name[:], c.Device)
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
+	copy(req.Name[:], t.Device)
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
 		return nil, err
 	}
@@ -158,46 +151,52 @@ func (c *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return file, nil
 }
 
-func (c *Tun) WriteRaw(b []byte) error {
+func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
+	r := t.routeTree.MostSpecificContains(ip)
+	if r != nil {
+		return r.(iputil.VpnIp)
+	}
+
+	return 0
+}
+
+func (t *tun) Write(b []byte) (int, error) {
 	var nn int
+	max := len(b)
+
 	for {
-		max := len(b)
-		n, err := unix.Write(c.fd, b[nn:max])
+		n, err := unix.Write(t.fd, b[nn:max])
 		if n > 0 {
 			nn += n
 		}
 		if nn == len(b) {
-			return err
+			return nn, err
 		}
 
 		if err != nil {
-			return err
+			return nn, err
 		}
 
 		if n == 0 {
-			return io.ErrUnexpectedEOF
+			return nn, io.ErrUnexpectedEOF
 		}
 	}
 }
 
-func (c *Tun) Write(b []byte) (int, error) {
-	return len(b), c.WriteRaw(b)
-}
-
-func (c Tun) deviceBytes() (o [16]byte) {
-	for i, c := range c.Device {
+func (t tun) deviceBytes() (o [16]byte) {
+	for i, c := range t.Device {
 		o[i] = byte(c)
 	}
 	return
 }
 
-func (c Tun) Activate() error {
-	devName := c.deviceBytes()
+func (t tun) Activate() error {
+	devName := t.deviceBytes()
 
 	var addr, mask [4]byte
 
-	copy(addr[:], c.Cidr.IP.To4())
-	copy(mask[:], c.Cidr.Mask)
+	copy(addr[:], t.cidr.IP.To4())
+	copy(mask[:], t.cidr.Mask)
 
 	s, err := unix.Socket(
 		unix.AF_INET,
@@ -235,17 +234,17 @@ func (c Tun) Activate() error {
 	}
 
 	// Set the MTU on the device
-	ifm := ifreqMTU{Name: devName, MTU: int32(c.MaxMTU)}
+	ifm := ifreqMTU{Name: devName, MTU: int32(t.MaxMTU)}
 	if err = ioctl(fd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
 		// This is currently a non fatal condition because the route table must have the MTU set appropriately as well
-		c.l.WithError(err).Error("Failed to set tun mtu")
+		t.l.WithError(err).Error("Failed to set tun mtu")
 	}
 
 	// Set the transmit queue length
-	ifrq := ifreqQLEN{Name: devName, Value: int32(c.TXQueueLen)}
+	ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)}
 	if err = ioctl(fd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil {
 		// If we can't set the queue length nebula will still work but it may lead to packet loss
-		c.l.WithError(err).Error("Failed to set tun tx queue length")
+		t.l.WithError(err).Error("Failed to set tun tx queue length")
 	}
 
 	// Bring up the interface
@@ -255,59 +254,46 @@ func (c Tun) Activate() error {
 	}
 
 	// Set the routes
-	link, err := netlink.LinkByName(c.Device)
+	link, err := netlink.LinkByName(t.Device)
 	if err != nil {
 		return fmt.Errorf("failed to get tun device link: %s", err)
 	}
 
 	// Default route
-	dr := &net.IPNet{IP: c.Cidr.IP.Mask(c.Cidr.Mask), Mask: c.Cidr.Mask}
+	dr := &net.IPNet{IP: t.cidr.IP.Mask(t.cidr.Mask), Mask: t.cidr.Mask}
 	nr := netlink.Route{
 		LinkIndex: link.Attrs().Index,
 		Dst:       dr,
-		MTU:       c.DefaultMTU,
-		AdvMSS:    c.advMSS(route{}),
+		MTU:       t.DefaultMTU,
+		AdvMSS:    t.advMSS(Route{}),
 		Scope:     unix.RT_SCOPE_LINK,
-		Src:       c.Cidr.IP,
+		Src:       t.cidr.IP,
 		Protocol:  unix.RTPROT_KERNEL,
 		Table:     unix.RT_TABLE_MAIN,
 		Type:      unix.RTN_UNICAST,
 	}
 	err = netlink.RouteReplace(&nr)
 	if err != nil {
-		return fmt.Errorf("failed to set mtu %v on the default route %v; %v", c.DefaultMTU, dr, err)
+		return fmt.Errorf("failed to set mtu %v on the default route %v; %v", t.DefaultMTU, dr, err)
 	}
 
 	// Path routes
-	for _, r := range c.Routes {
+	for _, r := range t.Routes {
 		nr := netlink.Route{
 			LinkIndex: link.Attrs().Index,
-			Dst:       r.route,
-			MTU:       r.mtu,
-			AdvMSS:    c.advMSS(r),
+			Dst:       r.Cidr,
+			MTU:       r.MTU,
+			AdvMSS:    t.advMSS(r),
 			Scope:     unix.RT_SCOPE_LINK,
 		}
 
+		if r.Metric > 0 {
+			nr.Priority = r.Metric
+		}
+
 		err = netlink.RouteAdd(&nr)
 		if err != nil {
-			return fmt.Errorf("failed to set mtu %v on route %v; %v", r.mtu, r.route, err)
-		}
-	}
-
-	// Unsafe path routes
-	for _, r := range c.UnsafeRoutes {
-		nr := netlink.Route{
-			LinkIndex: link.Attrs().Index,
-			Dst:       r.route,
-			MTU:       r.mtu,
-			Priority:  r.metric,
-			AdvMSS:    c.advMSS(r),
-			Scope:     unix.RT_SCOPE_LINK,
-		}
-
-		err = netlink.RouteAdd(&nr)
-		if err != nil {
-			return fmt.Errorf("failed to set mtu %v on route %v; %v", r.mtu, r.route, err)
+			return fmt.Errorf("failed to set mtu %v on route %v; %v", r.MTU, r.Cidr, err)
 		}
 	}
 
@@ -320,22 +306,22 @@ func (c Tun) Activate() error {
 	return nil
 }
 
-func (c *Tun) CidrNet() *net.IPNet {
-	return c.Cidr
+func (t *tun) Cidr() *net.IPNet {
+	return t.cidr
 }
 
-func (c *Tun) DeviceName() string {
-	return c.Device
+func (t *tun) Name() string {
+	return t.Device
 }
 
-func (c Tun) advMSS(r route) int {
-	mtu := r.mtu
-	if r.mtu == 0 {
-		mtu = c.DefaultMTU
+func (t tun) advMSS(r Route) int {
+	mtu := r.MTU
+	if r.MTU == 0 {
+		mtu = t.DefaultMTU
 	}
 
 	// We only need to set advmss if the route MTU does not match the device MTU
-	if mtu != c.MaxMTU {
+	if mtu != t.MaxMTU {
 		return mtu - 40
 	}
 	return 0

overlay/tun_linux_test.go

@@ -1,25 +1,25 @@
 //go:build !e2e_testing
 // +build !e2e_testing
 
-package nebula
+package overlay
 
 import "testing"
 
 var runAdvMSSTests = []struct {
 	name     string
-	tun      Tun
-	r        route
+	tun      tun
+	r        Route
 	expected int
 }{
 	// Standard case, default MTU is the device max MTU
-	{"default", Tun{DefaultMTU: 1440, MaxMTU: 1440}, route{}, 0},
-	{"default-min", Tun{DefaultMTU: 1440, MaxMTU: 1440}, route{mtu: 1440}, 0},
-	{"default-low", Tun{DefaultMTU: 1440, MaxMTU: 1440}, route{mtu: 1200}, 1160},
+	{"default", tun{DefaultMTU: 1440, MaxMTU: 1440}, Route{}, 0},
+	{"default-min", tun{DefaultMTU: 1440, MaxMTU: 1440}, Route{MTU: 1440}, 0},
+	{"default-low", tun{DefaultMTU: 1440, MaxMTU: 1440}, Route{MTU: 1200}, 1160},
 
 	// Case where we have a route MTU set higher than the default
-	{"route", Tun{DefaultMTU: 1440, MaxMTU: 8941}, route{}, 1400},
-	{"route-min", Tun{DefaultMTU: 1440, MaxMTU: 8941}, route{mtu: 1440}, 1400},
-	{"route-high", Tun{DefaultMTU: 1440, MaxMTU: 8941}, route{mtu: 8941}, 0},
+	{"route", tun{DefaultMTU: 1440, MaxMTU: 8941}, Route{}, 1400},
+	{"route-min", tun{DefaultMTU: 1440, MaxMTU: 8941}, Route{MTU: 1440}, 1400},
+	{"route-high", tun{DefaultMTU: 1440, MaxMTU: 8941}, Route{MTU: 8941}, 0},
 }
 
 func TestTunAdvMSS(t *testing.T) {

overlay/tun_tester.go

@@ -0,0 +1,117 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package overlay
+
+import (
+	"fmt"
+	"io"
+	"net"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/iputil"
+)
+
+type TestTun struct {
+	Device    string
+	cidr      *net.IPNet
+	Routes    []Route
+	routeTree *cidr.Tree4
+	l         *logrus.Logger
+
+	rxPackets chan []byte // Packets to receive into nebula
+	TxPackets chan []byte // Packets transmitted outside by nebula
+}
+
+func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, _ int, routes []Route, _ int, _ bool) (*TestTun, error) {
+	routeTree, err := makeRouteTree(l, routes, false)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TestTun{
+		Device:    deviceName,
+		cidr:      cidr,
+		Routes:    routes,
+		routeTree: routeTree,
+		l:         l,
+		rxPackets: make(chan []byte, 1),
+		TxPackets: make(chan []byte, 1),
+	}, nil
+}
+
+func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int) (*TestTun, error) {
+	return nil, fmt.Errorf("newTunFromFd not supported")
+}
+
+// Send will place a byte array onto the receive queue for nebula to consume
+// These are unencrypted ip layer frames destined for another nebula node.
+// packets should exit the udp side, capture them with udpConn.Get
+func (t *TestTun) Send(packet []byte) {
+	t.l.WithField("dataLen", len(packet)).Info("Tun receiving injected packet")
+	t.rxPackets <- packet
+}
+
+// Get will pull an unencrypted ip layer frame from the transmit queue
+// nebula meant to send this message to some application on the local system
+// packets were ingested from the udp side, you can send them with udpConn.Send
+func (t *TestTun) Get(block bool) []byte {
+	if block {
+		return <-t.TxPackets
+	}
+
+	select {
+	case p := <-t.TxPackets:
+		return p
+	default:
+		return nil
+	}
+}
+
+//********************************************************************************************************************//
+// Below this is boilerplate implementation to make nebula actually work
+//********************************************************************************************************************//
+
+func (t *TestTun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
+	r := t.routeTree.MostSpecificContains(ip)
+	if r != nil {
+		return r.(iputil.VpnIp)
+	}
+
+	return 0
+}
+
+func (t *TestTun) Activate() error {
+	return nil
+}
+
+func (t *TestTun) Cidr() *net.IPNet {
+	return t.cidr
+}
+
+func (t *TestTun) Name() string {
+	return t.Device
+}
+
+func (t *TestTun) Write(b []byte) (n int, err error) {
+	packet := make([]byte, len(b), len(b))
+	copy(packet, b)
+	t.TxPackets <- packet
+	return len(b), nil
+}
+
+func (t *TestTun) Close() error {
+	close(t.rxPackets)
+	return nil
+}
+
+func (t *TestTun) Read(b []byte) (int, error) {
+	p := <-t.rxPackets
+	copy(b, p)
+	return len(p), nil
+}
+
+func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented")
+}

overlay/tun_water_windows.go

@@ -0,0 +1,126 @@
+package overlay
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"os/exec"
+	"strconv"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/songgao/water"
+)
+
+type waterTun struct {
+	Device    string
+	cidr      *net.IPNet
+	MTU       int
+	Routes    []Route
+	routeTree *cidr.Tree4
+
+	*water.Interface
+}
+
+func newWaterTun(l *logrus.Logger, cidr *net.IPNet, defaultMTU int, routes []Route) (*waterTun, error) {
+	routeTree, err := makeRouteTree(l, routes, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// NOTE: You cannot set the deviceName under Windows, so you must check tun.Device after calling .Activate()
+	return &waterTun{
+		cidr:      cidr,
+		MTU:       defaultMTU,
+		Routes:    routes,
+		routeTree: routeTree,
+	}, nil
+}
+
+func (t *waterTun) Activate() error {
+	var err error
+	t.Interface, err = water.New(water.Config{
+		DeviceType: water.TUN,
+		PlatformSpecificParams: water.PlatformSpecificParams{
+			ComponentID: "tap0901",
+			Network:     t.cidr.String(),
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("activate failed: %v", err)
+	}
+
+	t.Device = t.Interface.Name()
+
+	// TODO use syscalls instead of exec.Command
+	err = exec.Command(
+		`C:\Windows\System32\netsh.exe`, "interface", "ipv4", "set", "address",
+		fmt.Sprintf("name=%s", t.Device),
+		"source=static",
+		fmt.Sprintf("addr=%s", t.cidr.IP),
+		fmt.Sprintf("mask=%s", net.IP(t.cidr.Mask)),
+		"gateway=none",
+	).Run()
+	if err != nil {
+		return fmt.Errorf("failed to run 'netsh' to set address: %s", err)
+	}
+	err = exec.Command(
+		`C:\Windows\System32\netsh.exe`, "interface", "ipv4", "set", "interface",
+		t.Device,
+		fmt.Sprintf("mtu=%d", t.MTU),
+	).Run()
+	if err != nil {
+		return fmt.Errorf("failed to run 'netsh' to set MTU: %s", err)
+	}
+
+	iface, err := net.InterfaceByName(t.Device)
+	if err != nil {
+		return fmt.Errorf("failed to find interface named %s: %v", t.Device, err)
+	}
+
+	for _, r := range t.Routes {
+		if r.Via == nil {
+			// We don't allow route MTUs so only install routes with a via
+			continue
+		}
+
+		err = exec.Command(
+			"C:\\Windows\\System32\\route.exe", "add", r.Cidr.String(), r.Via.String(), "IF", strconv.Itoa(iface.Index), "METRIC", strconv.Itoa(r.Metric),
+		).Run()
+		if err != nil {
+			return fmt.Errorf("failed to add the unsafe_route %s: %v", r.Cidr.String(), err)
+		}
+	}
+
+	return nil
+}
+
+func (t *waterTun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
+	r := t.routeTree.MostSpecificContains(ip)
+	if r != nil {
+		return r.(iputil.VpnIp)
+	}
+
+	return 0
+}
+
+func (t *waterTun) Cidr() *net.IPNet {
+	return t.cidr
+}
+
+func (t *waterTun) Name() string {
+	return t.Device
+}
+
+func (t *waterTun) Close() error {
+	if t.Interface == nil {
+		return nil
+	}
+
+	return t.Interface.Close()
+}
+
+func (t *waterTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
+}

overlay/tun_windows.go

@@ -0,0 +1,62 @@
+//go:build !e2e_testing
+// +build !e2e_testing
+
+package overlay
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"runtime"
+	"syscall"
+
+	"github.com/sirupsen/logrus"
+)
+
+func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int) (Device, error) {
+	return nil, fmt.Errorf("newTunFromFd not supported in Windows")
+}
+
+func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool) (Device, error) {
+	if len(routes) > 0 {
+		return nil, fmt.Errorf("route MTU not supported in Windows")
+	}
+
+	useWintun := true
+	if err := checkWinTunExists(); err != nil {
+		l.WithError(err).Warn("Check Wintun driver failed, fallback to wintap driver")
+		useWintun = false
+	}
+
+	if useWintun {
+		device, err := newWinTun(l, deviceName, cidr, defaultMTU, routes)
+		if err != nil {
+			return nil, fmt.Errorf("create Wintun interface failed, %w", err)
+		}
+		return device, nil
+	}
+
+	device, err := newWaterTun(l, cidr, defaultMTU, routes)
+	if err != nil {
+		return nil, fmt.Errorf("create wintap driver failed, %w", err)
+	}
+	return device, nil
+}
+
+func checkWinTunExists() error {
+	myPath, err := os.Executable()
+	if err != nil {
+		return err
+	}
+
+	arch := runtime.GOARCH
+	switch arch {
+	case "386":
+		//NOTE: wintun bundles 386 as x86
+		arch = "x86"
+	}
+
+	_, err = syscall.LoadDLL(filepath.Join(filepath.Dir(myPath), "dist", "windows", "wintun", "bin", arch, "wintun.dll"))
+	return err
+}

overlay/tun_wintun_windows.go

@@ -0,0 +1,170 @@
+package overlay
+
+import (
+	"crypto"
+	"fmt"
+	"io"
+	"net"
+	"unsafe"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/wintun"
+	"golang.org/x/sys/windows"
+	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
+)
+
+const tunGUIDLabel = "Fixed Nebula Windows GUID v1"
+
+type winTun struct {
+	Device    string
+	cidr      *net.IPNet
+	MTU       int
+	Routes    []Route
+	routeTree *cidr.Tree4
+
+	tun *wintun.NativeTun
+}
+
+func generateGUIDByDeviceName(name string) (*windows.GUID, error) {
+	// GUID is 128 bit
+	hash := crypto.MD5.New()
+
+	_, err := hash.Write([]byte(tunGUIDLabel))
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = hash.Write([]byte(name))
+	if err != nil {
+		return nil, err
+	}
+
+	sum := hash.Sum(nil)
+
+	return (*windows.GUID)(unsafe.Pointer(&sum[0])), nil
+}
+
+func newWinTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route) (*winTun, error) {
+	guid, err := generateGUIDByDeviceName(deviceName)
+	if err != nil {
+		return nil, fmt.Errorf("generate GUID failed: %w", err)
+	}
+
+	tunDevice, err := wintun.CreateTUNWithRequestedGUID(deviceName, guid, defaultMTU)
+	if err != nil {
+		return nil, fmt.Errorf("create TUN device failed: %w", err)
+	}
+
+	routeTree, err := makeRouteTree(l, routes, false)
+	if err != nil {
+		return nil, err
+	}
+
+	return &winTun{
+		Device:    deviceName,
+		cidr:      cidr,
+		MTU:       defaultMTU,
+		Routes:    routes,
+		routeTree: routeTree,
+
+		tun: tunDevice.(*wintun.NativeTun),
+	}, nil
+}
+
+func (t *winTun) Activate() error {
+	luid := winipcfg.LUID(t.tun.LUID())
+
+	if err := luid.SetIPAddresses([]net.IPNet{*t.cidr}); err != nil {
+		return fmt.Errorf("failed to set address: %w", err)
+	}
+
+	foundDefault4 := false
+	routes := make([]*winipcfg.RouteData, 0, len(t.Routes)+1)
+
+	for _, r := range t.Routes {
+		if r.Via == nil {
+			// We don't allow route MTUs so only install routes with a via
+			continue
+		}
+
+		if !foundDefault4 {
+			if ones, bits := r.Cidr.Mask.Size(); ones == 0 && bits != 0 {
+				foundDefault4 = true
+			}
+		}
+
+		// Add our unsafe route
+		routes = append(routes, &winipcfg.RouteData{
+			Destination: *r.Cidr,
+			NextHop:     r.Via.ToIP(),
+			Metric:      uint32(r.Metric),
+		})
+	}
+
+	if err := luid.AddRoutes(routes); err != nil {
+		return fmt.Errorf("failed to add routes: %w", err)
+	}
+
+	ipif, err := luid.IPInterface(windows.AF_INET)
+	if err != nil {
+		return fmt.Errorf("failed to get ip interface: %w", err)
+	}
+
+	ipif.NLMTU = uint32(t.MTU)
+	if foundDefault4 {
+		ipif.UseAutomaticMetric = false
+		ipif.Metric = 0
+	}
+
+	if err := ipif.Set(); err != nil {
+		return fmt.Errorf("failed to set ip interface: %w", err)
+	}
+
+	return nil
+}
+
+func (t *winTun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
+	r := t.routeTree.MostSpecificContains(ip)
+	if r != nil {
+		return r.(iputil.VpnIp)
+	}
+
+	return 0
+}
+
+func (t *winTun) Cidr() *net.IPNet {
+	return t.cidr
+}
+
+func (t *winTun) Name() string {
+	return t.Device
+}
+
+func (t *winTun) Read(b []byte) (int, error) {
+	return t.tun.Read(b, 0)
+}
+
+func (t *winTun) Write(b []byte) (int, error) {
+	return t.tun.Write(b, 0)
+}
+
+func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
+}
+
+func (t *winTun) Close() error {
+	// It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes,
+	// so to be certain, just remove everything before destroying.
+	luid := winipcfg.LUID(t.tun.LUID())
+	_ = luid.FlushRoutes(windows.AF_INET)
+	_ = luid.FlushIPAddresses(windows.AF_INET)
+	/* We don't support IPV6 yet
+	_ = luid.FlushRoutes(windows.AF_INET6)
+	_ = luid.FlushIPAddresses(windows.AF_INET6)
+	*/
+	_ = luid.FlushDNS(windows.AF_INET)
+
+	return t.tun.Close()
+}

psk.go

@@ -1,183 +0,0 @@
-package nebula
-
-import (
-	"crypto/sha256"
-	"errors"
-	"fmt"
-	"io"
-
-	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/iputil"
-	"golang.org/x/crypto/hkdf"
-)
-
-var ErrNotAPskMode = errors.New("not a psk mode")
-var ErrKeyTooShort = errors.New("key is too short")
-var ErrNotEnoughPskKeys = errors.New("at least 1 key is required")
-
-// The minimum length that we accept for a user defined psk, the choice is arbitrary
-const MinPskLength = 8
-
-type PskMode int
-
-func (p PskMode) String() string {
-	switch p {
-	case PskNone:
-		return "none"
-	case PskTransitionalAccepting:
-		return "transitional-accepting"
-	case PskTransitionalSending:
-		return "transitional-sending"
-	case PskEnforced:
-		return "enforced"
-	}
-
-	return "unknown"
-}
-
-func NewPskMode(m string) (PskMode, error) {
-	switch m {
-	case "none":
-		return PskNone, nil
-	case "transitional-accepting":
-		return PskTransitionalAccepting, nil
-	case "transitional-sending":
-		return PskTransitionalSending, nil
-	case "enforced":
-		return PskEnforced, nil
-	}
-	return PskNone, ErrNotAPskMode
-}
-
-const (
-	PskNone                  PskMode = 0
-	PskTransitionalAccepting PskMode = 1
-	PskTransitionalSending   PskMode = 2
-	PskEnforced              PskMode = 3
-)
-
-type Psk struct {
-	// pskMode sets how psk works, ignored, allowed for incoming, or enforced for all
-	mode PskMode
-
-	// Cache holds all pre-computed psk hkdfs
-	// Handshakes iterate this directly
-	Cache [][]byte
-
-	// The key has already been extracted and is ready to be expanded for use
-	// MakeFor does the final expand step mixing in the intended recipients vpn ip
-	key []byte
-}
-
-// NewPskFromConfig is a helper for initial boot and config reloading.
-func NewPskFromConfig(c *config.C, myVpnIp iputil.VpnIp) (*Psk, error) {
-	sMode := c.GetString("handshakes.psk.mode", "none")
-	mode, err := NewPskMode(sMode)
-	if err != nil {
-		return nil, NewContextualError("Could not parse handshakes.psk.mode", m{"mode": mode}, err)
-	}
-
-	return NewPsk(
-		mode,
-		c.GetStringSlice("handshakes.psk.keys", nil),
-		myVpnIp,
-	)
-}
-
-// NewPsk creates a new Psk object and handles the caching of all accepted keys and preparation of the primary key
-func NewPsk(mode PskMode, keys []string, myVpnIp iputil.VpnIp) (*Psk, error) {
-	psk := &Psk{
-		mode: mode,
-	}
-
-	err := psk.preparePrimaryKey(keys)
-	if err != nil {
-		return nil, err
-	}
-
-	err = psk.cachePsks(myVpnIp, keys)
-	if err != nil {
-		return nil, err
-	}
-
-	return psk, nil
-}
-
-// MakeFor if we are in enforced mode, the final hkdf expand stage is done on the pre extracted primary key,
-// mixing in the intended recipients vpn ip and the result is returned.
-// If we are transitional or not using psks, an empty byte slice is returned
-func (p *Psk) MakeFor(vpnIp iputil.VpnIp) ([]byte, error) {
-	if p.mode == PskNone || p.mode == PskTransitionalAccepting {
-		return []byte{}, nil
-	}
-
-	hmacKey := make([]byte, sha256.Size)
-	_, err := io.ReadFull(hkdf.Expand(sha256.New, p.key, vpnIp.ToIP()), hmacKey)
-	if err != nil {
-		return nil, err
-	}
-
-	return hmacKey, nil
-}
-
-// cachePsks generates all psks we accept and caches them to speed up handshaking
-func (p *Psk) cachePsks(myVpnIp iputil.VpnIp, keys []string) error {
-	// If PskNone is set then we are using the nil byte array for a psk, we can return
-	if p.mode == PskNone {
-		p.Cache = [][]byte{nil}
-		return nil
-	}
-
-	if len(keys) < 1 {
-		return ErrNotEnoughPskKeys
-	}
-
-	p.Cache = [][]byte{}
-
-	if p.mode == PskTransitionalAccepting || p.mode == PskTransitionalSending {
-		// We are transitional, we accept empty psks
-		p.Cache = append(p.Cache, nil)
-	}
-
-	// We are either PskAuto or PskTransitional, build all possibilities
-	for i, rk := range keys {
-		k, err := sha256KdfFromString(rk, myVpnIp)
-		if err != nil {
-			return fmt.Errorf("failed to generate key for position %v: %w", i, err)
-		}
-
-		p.Cache = append(p.Cache, k)
-	}
-
-	return nil
-}
-
-// preparePrimaryKey if we are in enforced mode, will do an hkdf extract on the first key to benefit
-// outgoing handshake performance, MakeFor does the final expand step
-func (p *Psk) preparePrimaryKey(keys []string) error {
-	if p.mode == PskNone || p.mode == PskTransitionalAccepting {
-		// If we aren't enforcing then there is nothing to prepare
-		return nil
-	}
-
-	if len(keys) < 1 {
-		return ErrNotEnoughPskKeys
-	}
-
-	p.key = hkdf.Extract(sha256.New, []byte(keys[0]), nil)
-	return nil
-}
-
-// sha256KdfFromString generates a full hkdf
-func sha256KdfFromString(secret string, vpnIp iputil.VpnIp) ([]byte, error) {
-	if len(secret) < MinPskLength {
-		return nil, ErrKeyTooShort
-	}
-
-	hmacKey := make([]byte, sha256.Size)
-	_, err := io.ReadFull(hkdf.New(sha256.New, []byte(secret), nil, vpnIp.ToIP()), hmacKey)
-	if err != nil {
-		return nil, err
-	}
-	return hmacKey, nil
-}

psk_test.go

@@ -1,103 +0,0 @@
-package nebula
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNewPsk(t *testing.T) {
-	t.Run("mode none", func(t *testing.T) {
-		p, err := NewPsk(PskNone, nil, 1)
-		assert.NoError(t, err)
-		assert.Equal(t, PskNone, p.mode)
-		assert.Empty(t, p.key)
-
-		assert.Len(t, p.Cache, 1)
-		assert.Nil(t, p.Cache[0])
-
-		b, err := p.MakeFor(0)
-		assert.Equal(t, []byte{}, b)
-	})
-
-	t.Run("mode transitional-accepting", func(t *testing.T) {
-		p, err := NewPsk(PskTransitionalAccepting, nil, 1)
-		assert.Error(t, ErrNotEnoughPskKeys, err)
-
-		p, err = NewPsk(PskTransitionalAccepting, []string{"1234567"}, 1)
-		assert.Error(t, ErrKeyTooShort)
-
-		p, err = NewPsk(PskTransitionalAccepting, []string{"hi there friends"}, 1)
-		assert.NoError(t, err)
-		assert.Equal(t, PskTransitionalAccepting, p.mode)
-		assert.Empty(t, p.key)
-
-		assert.Len(t, p.Cache, 2)
-		assert.Nil(t, p.Cache[0])
-
-		expectedCache := []byte{146, 120, 135, 31, 158, 102, 45, 189, 128, 190, 37, 101, 58, 254, 6, 166, 91, 209, 148, 131, 27, 193, 24, 25, 170, 65, 130, 189, 7, 179, 255, 17}
-		assert.Equal(t, expectedCache, p.Cache[1])
-
-		b, err := p.MakeFor(0)
-		assert.Equal(t, []byte{}, b)
-	})
-
-	t.Run("mode transitional-sending", func(t *testing.T) {
-		p, err := NewPsk(PskTransitionalSending, nil, 1)
-		assert.Error(t, ErrNotEnoughPskKeys, err)
-
-		p, err = NewPsk(PskTransitionalSending, []string{"1234567"}, 1)
-		assert.Error(t, ErrKeyTooShort)
-
-		p, err = NewPsk(PskTransitionalSending, []string{"hi there friends"}, 1)
-		assert.NoError(t, err)
-		assert.Equal(t, PskTransitionalSending, p.mode)
-
-		expectedKey := []byte{0x9c, 0x67, 0xab, 0x58, 0x79, 0x5c, 0x8a, 0xf0, 0xaa, 0xf0, 0x4c, 0x6c, 0x9a, 0x42, 0x6b, 0xe, 0xe2, 0x94, 0xb1, 0x0, 0x28, 0x1c, 0xdc, 0x88, 0x44, 0x35, 0x3f, 0xb7, 0xd5, 0x9, 0xc0, 0xda}
-		assert.Equal(t, expectedKey, p.key)
-
-		assert.Len(t, p.Cache, 2)
-		assert.Nil(t, p.Cache[0])
-
-		expectedCache := []byte{146, 120, 135, 31, 158, 102, 45, 189, 128, 190, 37, 101, 58, 254, 6, 166, 91, 209, 148, 131, 27, 193, 24, 25, 170, 65, 130, 189, 7, 179, 255, 17}
-		assert.Equal(t, expectedCache, p.Cache[1])
-
-		expectedPsk := []byte{0xd9, 0x16, 0xa3, 0x66, 0x6a, 0x20, 0x26, 0xcf, 0x5d, 0x93, 0xad, 0xa3, 0x88, 0x2d, 0x57, 0xac, 0x9b, 0xc3, 0x5a, 0xb7, 0x8f, 0x6, 0x71, 0xc4, 0x3e, 0x5, 0x9e, 0xbc, 0x4e, 0xc8, 0x24, 0x17}
-		b, err := p.MakeFor(0)
-		assert.Equal(t, expectedPsk, b)
-	})
-
-	t.Run("mode enforced", func(t *testing.T) {
-		p, err := NewPsk(PskEnforced, nil, 1)
-		assert.Error(t, ErrNotEnoughPskKeys, err)
-
-		p, err = NewPsk(PskEnforced, []string{"hi there friends"}, 1)
-		assert.NoError(t, err)
-		assert.Equal(t, PskEnforced, p.mode)
-
-		expectedKey := []byte{156, 103, 171, 88, 121, 92, 138, 240, 170, 240, 76, 108, 154, 66, 107, 14, 226, 148, 177, 0, 40, 28, 220, 136, 68, 53, 63, 183, 213, 9, 192, 218}
-		assert.Equal(t, expectedKey, p.key)
-
-		assert.Len(t, p.Cache, 1)
-		expectedCache := []byte{146, 120, 135, 31, 158, 102, 45, 189, 128, 190, 37, 101, 58, 254, 6, 166, 91, 209, 148, 131, 27, 193, 24, 25, 170, 65, 130, 189, 7, 179, 255, 17}
-		assert.Equal(t, expectedCache, p.Cache[0])
-
-		expectedPsk := []byte{0xd9, 0x16, 0xa3, 0x66, 0x6a, 0x20, 0x26, 0xcf, 0x5d, 0x93, 0xad, 0xa3, 0x88, 0x2d, 0x57, 0xac, 0x9b, 0xc3, 0x5a, 0xb7, 0x8f, 0x6, 0x71, 0xc4, 0x3e, 0x5, 0x9e, 0xbc, 0x4e, 0xc8, 0x24, 0x17}
-		b, err := p.MakeFor(0)
-		assert.Equal(t, expectedPsk, b)
-
-		// Make sure different vpn ips generate different psks
-		expectedPsk = []byte{0x92, 0x78, 0x87, 0x1f, 0x9e, 0x66, 0x2d, 0xbd, 0x80, 0xbe, 0x25, 0x65, 0x3a, 0xfe, 0x6, 0xa6, 0x5b, 0xd1, 0x94, 0x83, 0x1b, 0xc1, 0x18, 0x19, 0xaa, 0x41, 0x82, 0xbd, 0x7, 0xb3, 0xff, 0x11}
-		b, err = p.MakeFor(1)
-		assert.Equal(t, expectedPsk, b)
-	})
-}
-
-func BenchmarkPsk_MakeFor(b *testing.B) {
-	p, err := NewPsk(PskEnforced, []string{"hi there friends"}, 1)
-	assert.NoError(b, err)
-
-	for n := 0; n < b.N; n++ {
-		p.MakeFor(99)
-	}
-}

punchy_test.go

@@ -5,12 +5,12 @@ import (
 	"time"
 
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/util"
+	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestNewPunchyFromConfig(t *testing.T) {
-	l := util.NewTestLogger()
+	l := test.NewLogger()
 	c := config.NewC(l)
 
 	// Test defaults

ssh.go

@@ -569,7 +569,7 @@ func sshCreateTunnel(ifce *Interface, fs interface{}, a []string, w sshd.StringW
 		}
 	}
 
-	hostInfo = ifce.handshakeManager.AddVpnIp(vpnIp)
+	hostInfo = ifce.handshakeManager.AddVpnIp(vpnIp, ifce.initHostInfo)
 	if addr != nil {
 		hostInfo.SetRemote(addr)
 	}

test/assert.go

@@ -1,4 +1,4 @@
-package util
+package test
 
 import (
 	"fmt"

test/logger.go

@@ -1,4 +1,4 @@
-package util
+package test
 
 import (
 	"io/ioutil"
@@ -7,7 +7,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func NewTestLogger() *logrus.Logger {
+func NewLogger() *logrus.Logger {
 	l := logrus.New()
 
 	v := os.Getenv("TEST_LOGS")

test/tun.go

@@ -0,0 +1,43 @@
+package test
+
+import (
+	"errors"
+	"io"
+	"net"
+
+	"github.com/slackhq/nebula/iputil"
+)
+
+type NoopTun struct{}
+
+func (NoopTun) RouteFor(iputil.VpnIp) iputil.VpnIp {
+	return 0
+}
+
+func (NoopTun) Activate() error {
+	return nil
+}
+
+func (NoopTun) Cidr() *net.IPNet {
+	return nil
+}
+
+func (NoopTun) Name() string {
+	return "noop"
+}
+
+func (NoopTun) Read([]byte) (int, error) {
+	return 0, nil
+}
+
+func (NoopTun) Write([]byte) (int, error) {
+	return 0, nil
+}
+
+func (NoopTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, errors.New("unsupported")
+}
+
+func (NoopTun) Close() error {
+	return nil
+}

tun_android.go

@@ -1,86 +0,0 @@
-//go:build !e2e_testing
-// +build !e2e_testing
-
-package nebula
-
-import (
-	"fmt"
-	"io"
-	"net"
-	"os"
-
-	"github.com/sirupsen/logrus"
-	"golang.org/x/sys/unix"
-)
-
-type Tun struct {
-	io.ReadWriteCloser
-	fd           int
-	Device       string
-	Cidr         *net.IPNet
-	MaxMTU       int
-	DefaultMTU   int
-	TXQueueLen   int
-	Routes       []route
-	UnsafeRoutes []route
-	l            *logrus.Logger
-}
-
-func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
-	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
-
-	ifce = &Tun{
-		ReadWriteCloser: file,
-		fd:              int(file.Fd()),
-		Device:          "android",
-		Cidr:            cidr,
-		DefaultMTU:      defaultMTU,
-		TXQueueLen:      txQueueLen,
-		Routes:          routes,
-		UnsafeRoutes:    unsafeRoutes,
-		l:               l,
-	}
-	return
-}
-
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
-	return nil, fmt.Errorf("newTun not supported in Android")
-}
-
-func (c *Tun) WriteRaw(b []byte) error {
-	var nn int
-	for {
-		max := len(b)
-		n, err := unix.Write(c.fd, b[nn:max])
-		if n > 0 {
-			nn += n
-		}
-		if nn == len(b) {
-			return err
-		}
-
-		if err != nil {
-			return err
-		}
-
-		if n == 0 {
-			return io.ErrUnexpectedEOF
-		}
-	}
-}
-
-func (c Tun) Activate() error {
-	return nil
-}
-
-func (c *Tun) CidrNet() *net.IPNet {
-	return c.Cidr
-}
-
-func (c *Tun) DeviceName() string {
-	return c.Device
-}
-
-func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
-}

tun_darwin.go

@@ -1,97 +0,0 @@
-//go:build !ios && !e2e_testing
-// +build !ios,!e2e_testing
-
-package nebula
-
-import (
-	"fmt"
-	"io"
-	"net"
-	"os/exec"
-	"strconv"
-
-	"github.com/sirupsen/logrus"
-	"github.com/songgao/water"
-)
-
-type Tun struct {
-	Device       string
-	Cidr         *net.IPNet
-	MTU          int
-	UnsafeRoutes []route
-	l            *logrus.Logger
-	*water.Interface
-}
-
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
-	if len(routes) > 0 {
-		return nil, fmt.Errorf("route MTU not supported in Darwin")
-	}
-
-	// NOTE: You cannot set the deviceName under Darwin, so you must check tun.Device after calling .Activate()
-	return &Tun{
-		Cidr:         cidr,
-		MTU:          defaultMTU,
-		UnsafeRoutes: unsafeRoutes,
-		l:            l,
-	}, nil
-}
-
-func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
-	return nil, fmt.Errorf("newTunFromFd not supported in Darwin")
-}
-
-func (c *Tun) Close() error {
-	if c.Interface != nil {
-		return c.Interface.Close()
-	}
-	return nil
-}
-
-func (c *Tun) Activate() error {
-	var err error
-	c.Interface, err = water.New(water.Config{
-		DeviceType: water.TUN,
-	})
-	if err != nil {
-		return fmt.Errorf("activate failed: %v", err)
-	}
-
-	c.Device = c.Interface.Name()
-
-	// TODO use syscalls instead of exec.Command
-	if err = exec.Command("/sbin/ifconfig", c.Device, c.Cidr.String(), c.Cidr.IP.String()).Run(); err != nil {
-		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-	}
-	if err = exec.Command("/sbin/route", "-n", "add", "-net", c.Cidr.String(), "-interface", c.Device).Run(); err != nil {
-		return fmt.Errorf("failed to run 'route add': %s", err)
-	}
-	if err = exec.Command("/sbin/ifconfig", c.Device, "mtu", strconv.Itoa(c.MTU)).Run(); err != nil {
-		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-	}
-	// Unsafe path routes
-	for _, r := range c.UnsafeRoutes {
-		if err = exec.Command("/sbin/route", "-n", "add", "-net", r.route.String(), "-interface", c.Device).Run(); err != nil {
-			return fmt.Errorf("failed to run 'route add' for unsafe_route %s: %s", r.route.String(), err)
-		}
-	}
-
-	return nil
-}
-
-func (c *Tun) CidrNet() *net.IPNet {
-	return c.Cidr
-}
-
-func (c *Tun) DeviceName() string {
-	return c.Device
-}
-
-func (c *Tun) WriteRaw(b []byte) error {
-	_, err := c.Write(b)
-	return err
-}
-
-func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
-}

tun_freebsd.go

@@ -1,107 +0,0 @@
-//go:build !e2e_testing
-// +build !e2e_testing
-
-package nebula
-
-import (
-	"fmt"
-	"io"
-	"net"
-	"os"
-	"os/exec"
-	"regexp"
-	"strconv"
-	"strings"
-
-	"github.com/sirupsen/logrus"
-)
-
-var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
-
-type Tun struct {
-	Device       string
-	Cidr         *net.IPNet
-	MTU          int
-	UnsafeRoutes []route
-	l            *logrus.Logger
-
-	io.ReadWriteCloser
-}
-
-func (c *Tun) Close() error {
-	if c.ReadWriteCloser != nil {
-		return c.ReadWriteCloser.Close()
-	}
-	return nil
-}
-
-func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
-	return nil, fmt.Errorf("newTunFromFd not supported in FreeBSD")
-}
-
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
-	if len(routes) > 0 {
-		return nil, fmt.Errorf("Route MTU not supported in FreeBSD")
-	}
-	if strings.HasPrefix(deviceName, "/dev/") {
-		deviceName = strings.TrimPrefix(deviceName, "/dev/")
-	}
-	if !deviceNameRE.MatchString(deviceName) {
-		return nil, fmt.Errorf("tun.dev must match `tun[0-9]+`")
-	}
-	return &Tun{
-		Device:       deviceName,
-		Cidr:         cidr,
-		MTU:          defaultMTU,
-		UnsafeRoutes: unsafeRoutes,
-		l:            l,
-	}, nil
-}
-
-func (c *Tun) Activate() error {
-	var err error
-	c.ReadWriteCloser, err = os.OpenFile("/dev/"+c.Device, os.O_RDWR, 0)
-	if err != nil {
-		return fmt.Errorf("Activate failed: %v", err)
-	}
-
-	// TODO use syscalls instead of exec.Command
-	c.l.Debug("command: ifconfig", c.Device, c.Cidr.String(), c.Cidr.IP.String())
-	if err = exec.Command("/sbin/ifconfig", c.Device, c.Cidr.String(), c.Cidr.IP.String()).Run(); err != nil {
-		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-	}
-	c.l.Debug("command: route", "-n", "add", "-net", c.Cidr.String(), "-interface", c.Device)
-	if err = exec.Command("/sbin/route", "-n", "add", "-net", c.Cidr.String(), "-interface", c.Device).Run(); err != nil {
-		return fmt.Errorf("failed to run 'route add': %s", err)
-	}
-	c.l.Debug("command: ifconfig", c.Device, "mtu", strconv.Itoa(c.MTU))
-	if err = exec.Command("/sbin/ifconfig", c.Device, "mtu", strconv.Itoa(c.MTU)).Run(); err != nil {
-		return fmt.Errorf("failed to run 'ifconfig': %s", err)
-	}
-	// Unsafe path routes
-	for _, r := range c.UnsafeRoutes {
-		c.l.Debug("command: route", "-n", "add", "-net", r.route.String(), "-interface", c.Device)
-		if err = exec.Command("/sbin/route", "-n", "add", "-net", r.route.String(), "-interface", c.Device).Run(); err != nil {
-			return fmt.Errorf("failed to run 'route add' for unsafe_route %s: %s", r.route.String(), err)
-		}
-	}
-
-	return nil
-}
-
-func (c *Tun) CidrNet() *net.IPNet {
-	return c.Cidr
-}
-
-func (c *Tun) DeviceName() string {
-	return c.Device
-}
-
-func (c *Tun) WriteRaw(b []byte) error {
-	_, err := c.Write(b)
-	return err
-}
-
-func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
-}

tun_ios.go

@@ -1,120 +0,0 @@
-//go:build ios && !e2e_testing
-// +build ios,!e2e_testing
-
-package nebula
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"net"
-	"os"
-	"sync"
-	"syscall"
-
-	"github.com/sirupsen/logrus"
-)
-
-type Tun struct {
-	io.ReadWriteCloser
-	Device string
-	Cidr   *net.IPNet
-}
-
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
-	return nil, fmt.Errorf("newTun not supported in iOS")
-}
-
-func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
-	if len(routes) > 0 {
-		return nil, fmt.Errorf("route MTU not supported in Darwin")
-	}
-
-	file := os.NewFile(uintptr(deviceFd), "/dev/tun")
-	ifce = &Tun{
-		Cidr:            cidr,
-		Device:          "iOS",
-		ReadWriteCloser: &tunReadCloser{f: file},
-	}
-	return
-}
-
-func (c *Tun) Activate() error {
-	return nil
-}
-
-func (c *Tun) WriteRaw(b []byte) error {
-	_, err := c.Write(b)
-	return err
-}
-
-// The following is hoisted up from water, we do this so we can inject our own fd on iOS
-type tunReadCloser struct {
-	f io.ReadWriteCloser
-
-	rMu  sync.Mutex
-	rBuf []byte
-
-	wMu  sync.Mutex
-	wBuf []byte
-}
-
-func (t *tunReadCloser) Read(to []byte) (int, error) {
-	t.rMu.Lock()
-	defer t.rMu.Unlock()
-
-	if cap(t.rBuf) < len(to)+4 {
-		t.rBuf = make([]byte, len(to)+4)
-	}
-	t.rBuf = t.rBuf[:len(to)+4]
-
-	n, err := t.f.Read(t.rBuf)
-	copy(to, t.rBuf[4:])
-	return n - 4, err
-}
-
-func (t *tunReadCloser) Write(from []byte) (int, error) {
-
-	if len(from) == 0 {
-		return 0, syscall.EIO
-	}
-
-	t.wMu.Lock()
-	defer t.wMu.Unlock()
-
-	if cap(t.wBuf) < len(from)+4 {
-		t.wBuf = make([]byte, len(from)+4)
-	}
-	t.wBuf = t.wBuf[:len(from)+4]
-
-	// Determine the IP Family for the NULL L2 Header
-	ipVer := from[0] >> 4
-	if ipVer == 4 {
-		t.wBuf[3] = syscall.AF_INET
-	} else if ipVer == 6 {
-		t.wBuf[3] = syscall.AF_INET6
-	} else {
-		return 0, errors.New("unable to determine IP version from packet")
-	}
-
-	copy(t.wBuf[4:], from)
-
-	n, err := t.f.Write(t.wBuf)
-	return n - 4, err
-}
-
-func (t *tunReadCloser) Close() error {
-	return t.f.Close()
-}
-
-func (c *Tun) CidrNet() *net.IPNet {
-	return c.Cidr
-}
-
-func (c *Tun) DeviceName() string {
-	return c.Device
-}
-
-func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
-}

tun_tester.go

@@ -1,105 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package nebula
-
-import (
-	"fmt"
-	"io"
-	"net"
-
-	"github.com/sirupsen/logrus"
-)
-
-type Tun struct {
-	Device       string
-	Cidr         *net.IPNet
-	MTU          int
-	UnsafeRoutes []route
-	l            *logrus.Logger
-
-	rxPackets chan []byte // Packets to receive into nebula
-	txPackets chan []byte // Packets transmitted outside by nebula
-}
-
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, _ []route, unsafeRoutes []route, _ int, _ bool) (ifce *Tun, err error) {
-	return &Tun{
-		Device:       deviceName,
-		Cidr:         cidr,
-		MTU:          defaultMTU,
-		UnsafeRoutes: unsafeRoutes,
-		l:            l,
-		rxPackets:    make(chan []byte, 1),
-		txPackets:    make(chan []byte, 1),
-	}, nil
-}
-
-func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []route, _ []route, _ int) (ifce *Tun, err error) {
-	return nil, fmt.Errorf("newTunFromFd not supported")
-}
-
-// Send will place a byte array onto the receive queue for nebula to consume
-// These are unencrypted ip layer frames destined for another nebula node.
-// packets should exit the udp side, capture them with udpConn.Get
-func (c *Tun) Send(packet []byte) {
-	c.l.WithField("dataLen", len(packet)).Info("Tun receiving injected packet")
-	c.rxPackets <- packet
-}
-
-// Get will pull an unencrypted ip layer frame from the transmit queue
-// nebula meant to send this message to some application on the local system
-// packets were ingested from the udp side, you can send them with udpConn.Send
-func (c *Tun) Get(block bool) []byte {
-	if block {
-		return <-c.txPackets
-	}
-
-	select {
-	case p := <-c.txPackets:
-		return p
-	default:
-		return nil
-	}
-}
-
-//********************************************************************************************************************//
-// Below this is boilerplate implementation to make nebula actually work
-//********************************************************************************************************************//
-
-func (c *Tun) Activate() error {
-	return nil
-}
-
-func (c *Tun) CidrNet() *net.IPNet {
-	return c.Cidr
-}
-
-func (c *Tun) DeviceName() string {
-	return c.Device
-}
-
-func (c *Tun) Write(b []byte) (n int, err error) {
-	return len(b), c.WriteRaw(b)
-}
-
-func (c *Tun) Close() error {
-	close(c.rxPackets)
-	return nil
-}
-
-func (c *Tun) WriteRaw(b []byte) error {
-	packet := make([]byte, len(b), len(b))
-	copy(packet, b)
-	c.txPackets <- packet
-	return nil
-}
-
-func (c *Tun) Read(b []byte) (int, error) {
-	p := <-c.rxPackets
-	copy(b, p)
-	return len(p), nil
-}
-
-func (c *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented")
-}

tun_windows.go

@@ -1,120 +0,0 @@
-//go:build !e2e_testing
-// +build !e2e_testing
-
-package nebula
-
-import (
-	"fmt"
-	"io"
-	"net"
-	"os/exec"
-	"strconv"
-
-	"github.com/sirupsen/logrus"
-	"github.com/songgao/water"
-)
-
-type Tun struct {
-	Device       string
-	Cidr         *net.IPNet
-	MTU          int
-	UnsafeRoutes []route
-	l            *logrus.Logger
-
-	*water.Interface
-}
-
-func (c *Tun) Close() error {
-	if c.Interface != nil {
-		return c.Interface.Close()
-	}
-	return nil
-}
-
-func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) {
-	return nil, fmt.Errorf("newTunFromFd not supported in Windows")
-}
-
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
-	if len(routes) > 0 {
-		return nil, fmt.Errorf("route MTU not supported in Windows")
-	}
-
-	// NOTE: You cannot set the deviceName under Windows, so you must check tun.Device after calling .Activate()
-	return &Tun{
-		Cidr:         cidr,
-		MTU:          defaultMTU,
-		UnsafeRoutes: unsafeRoutes,
-		l:            l,
-	}, nil
-}
-
-func (c *Tun) Activate() error {
-	var err error
-	c.Interface, err = water.New(water.Config{
-		DeviceType: water.TUN,
-		PlatformSpecificParams: water.PlatformSpecificParams{
-			ComponentID: "tap0901",
-			Network:     c.Cidr.String(),
-		},
-	})
-	if err != nil {
-		return fmt.Errorf("Activate failed: %v", err)
-	}
-
-	c.Device = c.Interface.Name()
-
-	// TODO use syscalls instead of exec.Command
-	err = exec.Command(
-		`C:\Windows\System32\netsh.exe`, "interface", "ipv4", "set", "address",
-		fmt.Sprintf("name=%s", c.Device),
-		"source=static",
-		fmt.Sprintf("addr=%s", c.Cidr.IP),
-		fmt.Sprintf("mask=%s", net.IP(c.Cidr.Mask)),
-		"gateway=none",
-	).Run()
-	if err != nil {
-		return fmt.Errorf("failed to run 'netsh' to set address: %s", err)
-	}
-	err = exec.Command(
-		`C:\Windows\System32\netsh.exe`, "interface", "ipv4", "set", "interface",
-		c.Device,
-		fmt.Sprintf("mtu=%d", c.MTU),
-	).Run()
-	if err != nil {
-		return fmt.Errorf("failed to run 'netsh' to set MTU: %s", err)
-	}
-
-	iface, err := net.InterfaceByName(c.Device)
-	if err != nil {
-		return fmt.Errorf("failed to find interface named %s: %v", c.Device, err)
-	}
-
-	for _, r := range c.UnsafeRoutes {
-		err = exec.Command(
-			"C:\\Windows\\System32\\route.exe", "add", r.route.String(), r.via.String(), "IF", strconv.Itoa(iface.Index), "METRIC", strconv.Itoa(r.metric),
-		).Run()
-		if err != nil {
-			return fmt.Errorf("failed to add the unsafe_route %s: %v", r.route.String(), err)
-		}
-	}
-
-	return nil
-}
-
-func (c *Tun) CidrNet() *net.IPNet {
-	return c.Cidr
-}
-
-func (c *Tun) DeviceName() string {
-	return c.Device
-}
-
-func (c *Tun) WriteRaw(b []byte) error {
-	_, err := c.Write(b)
-	return err
-}
-
-func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
-}

util/error.go

@@ -0,0 +1,39 @@
+package util
+
+import (
+	"errors"
+
+	"github.com/sirupsen/logrus"
+)
+
+type ContextualError struct {
+	RealError error
+	Fields    map[string]interface{}
+	Context   string
+}
+
+func NewContextualError(msg string, fields map[string]interface{}, realError error) ContextualError {
+	return ContextualError{Context: msg, Fields: fields, RealError: realError}
+}
+
+func (ce ContextualError) Error() string {
+	if ce.RealError == nil {
+		return ce.Context
+	}
+	return ce.RealError.Error()
+}
+
+func (ce ContextualError) Unwrap() error {
+	if ce.RealError == nil {
+		return errors.New(ce.Context)
+	}
+	return ce.RealError
+}
+
+func (ce *ContextualError) Log(lr *logrus.Logger) {
+	if ce.RealError != nil {
+		lr.WithFields(ce.Fields).WithError(ce.RealError).Error(ce.Context)
+	} else {
+		lr.WithFields(ce.Fields).Error(ce.Context)
+	}
+}

util/error_test.go

@@ -1,4 +1,4 @@
-package nebula
+package util
 
 import (
 	"errors"
@@ -8,6 +8,8 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+type m map[string]interface{}
+
 type TestLogWriter struct {
 	Logs []string
 }

wintun/device.go

@@ -0,0 +1,24 @@
+//go:build windows
+// +build windows
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+//NOTE: this file was forked from https://git.zx2c4.com/wireguard-go/tree/tun/tun.go?id=851efb1bb65555e0f765a3361c8eb5ac47435b19
+
+package wintun
+
+import (
+	"os"
+)
+
+type Device interface {
+	File() *os.File                 // returns the file descriptor of the device
+	Read([]byte, int) (int, error)  // read a packet from the device (without any additional headers)
+	Write([]byte, int) (int, error) // writes a packet to the device (without any additional headers)
+	Flush() error                   // flush all previous writes to the device
+	Name() (string, error)          // fetches and returns the current name
+	Close() error                   // stops the device and closes the event channel
+}

wintun/tun.go

@@ -0,0 +1,213 @@
+//go:build windows
+// +build windows
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+//NOTE: This file was forked from https://git.zx2c4.com/wireguard-go/tree/tun/tun_windows.go?id=851efb1bb65555e0f765a3361c8eb5ac47435b19
+// Mainly to shed functionality we won't be using and to fix names that display in the system
+
+package wintun
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"sync"
+	"sync/atomic"
+	"time"
+	_ "unsafe"
+
+	"golang.org/x/sys/windows"
+
+	"golang.zx2c4.com/wintun"
+)
+
+const (
+	rateMeasurementGranularity = uint64((time.Second / 2) / time.Nanosecond)
+	spinloopRateThreshold      = 800000000 / 8                                   // 800mbps
+	spinloopDuration           = uint64(time.Millisecond / 80 / time.Nanosecond) // ~1gbit/s
+)
+
+type rateJuggler struct {
+	current       uint64
+	nextByteCount uint64
+	nextStartTime int64
+	changing      int32
+}
+
+type NativeTun struct {
+	wt        *wintun.Adapter
+	name      string
+	handle    windows.Handle
+	rate      rateJuggler
+	session   wintun.Session
+	readWait  windows.Handle
+	running   sync.WaitGroup
+	closeOnce sync.Once
+	close     int32
+}
+
+var WintunTunnelType = "Nebula"
+var WintunStaticRequestedGUID *windows.GUID
+
+//go:linkname procyield runtime.procyield
+func procyield(cycles uint32)
+
+//go:linkname nanotime runtime.nanotime
+func nanotime() int64
+
+//
+// CreateTUN creates a Wintun interface with the given name. Should a Wintun
+// interface with the same name exist, it is reused.
+//
+func CreateTUN(ifname string, mtu int) (Device, error) {
+	return CreateTUNWithRequestedGUID(ifname, WintunStaticRequestedGUID, mtu)
+}
+
+//
+// CreateTUNWithRequestedGUID creates a Wintun interface with the given name and
+// a requested GUID. Should a Wintun interface with the same name exist, it is reused.
+//
+func CreateTUNWithRequestedGUID(ifname string, requestedGUID *windows.GUID, mtu int) (Device, error) {
+	wt, err := wintun.CreateAdapter(ifname, WintunTunnelType, requestedGUID)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating interface: %w", err)
+	}
+
+	tun := &NativeTun{
+		wt:     wt,
+		name:   ifname,
+		handle: windows.InvalidHandle,
+	}
+
+	tun.session, err = wt.StartSession(0x800000) // Ring capacity, 8 MiB
+	if err != nil {
+		tun.wt.Close()
+		return nil, fmt.Errorf("Error starting session: %w", err)
+	}
+	tun.readWait = tun.session.ReadWaitEvent()
+	return tun, nil
+}
+
+func (tun *NativeTun) Name() (string, error) {
+	return tun.name, nil
+}
+
+func (tun *NativeTun) File() *os.File {
+	return nil
+}
+
+func (tun *NativeTun) Close() error {
+	var err error
+	tun.closeOnce.Do(func() {
+		atomic.StoreInt32(&tun.close, 1)
+		windows.SetEvent(tun.readWait)
+		tun.running.Wait()
+		tun.session.End()
+		if tun.wt != nil {
+			tun.wt.Close()
+		}
+	})
+	return err
+}
+
+// Note: Read() and Write() assume the caller comes only from a single thread; there's no locking.
+
+func (tun *NativeTun) Read(buff []byte, offset int) (int, error) {
+	tun.running.Add(1)
+	defer tun.running.Done()
+retry:
+	if atomic.LoadInt32(&tun.close) == 1 {
+		return 0, os.ErrClosed
+	}
+	start := nanotime()
+	shouldSpin := atomic.LoadUint64(&tun.rate.current) >= spinloopRateThreshold && uint64(start-atomic.LoadInt64(&tun.rate.nextStartTime)) <= rateMeasurementGranularity*2
+	for {
+		if atomic.LoadInt32(&tun.close) == 1 {
+			return 0, os.ErrClosed
+		}
+		packet, err := tun.session.ReceivePacket()
+		switch err {
+		case nil:
+			packetSize := len(packet)
+			copy(buff[offset:], packet)
+			tun.session.ReleaseReceivePacket(packet)
+			tun.rate.update(uint64(packetSize))
+			return packetSize, nil
+		case windows.ERROR_NO_MORE_ITEMS:
+			if !shouldSpin || uint64(nanotime()-start) >= spinloopDuration {
+				windows.WaitForSingleObject(tun.readWait, windows.INFINITE)
+				goto retry
+			}
+			procyield(1)
+			continue
+		case windows.ERROR_HANDLE_EOF:
+			return 0, os.ErrClosed
+		case windows.ERROR_INVALID_DATA:
+			return 0, errors.New("Send ring corrupt")
+		}
+		return 0, fmt.Errorf("Read failed: %w", err)
+	}
+}
+
+func (tun *NativeTun) Flush() error {
+	return nil
+}
+
+func (tun *NativeTun) Write(buff []byte, offset int) (int, error) {
+	tun.running.Add(1)
+	defer tun.running.Done()
+	if atomic.LoadInt32(&tun.close) == 1 {
+		return 0, os.ErrClosed
+	}
+
+	packetSize := len(buff) - offset
+	tun.rate.update(uint64(packetSize))
+
+	packet, err := tun.session.AllocateSendPacket(packetSize)
+	if err == nil {
+		copy(packet, buff[offset:])
+		tun.session.SendPacket(packet)
+		return packetSize, nil
+	}
+	switch err {
+	case windows.ERROR_HANDLE_EOF:
+		return 0, os.ErrClosed
+	case windows.ERROR_BUFFER_OVERFLOW:
+		return 0, nil // Dropping when ring is full.
+	}
+	return 0, fmt.Errorf("Write failed: %w", err)
+}
+
+// LUID returns Windows interface instance ID.
+func (tun *NativeTun) LUID() uint64 {
+	tun.running.Add(1)
+	defer tun.running.Done()
+	if atomic.LoadInt32(&tun.close) == 1 {
+		return 0
+	}
+	return tun.wt.LUID()
+}
+
+// RunningVersion returns the running version of the Wintun driver.
+func (tun *NativeTun) RunningVersion() (version uint32, err error) {
+	return wintun.RunningVersion()
+}
+
+func (rate *rateJuggler) update(packetLen uint64) {
+	now := nanotime()
+	total := atomic.AddUint64(&rate.nextByteCount, packetLen)
+	period := uint64(now - atomic.LoadInt64(&rate.nextStartTime))
+	if period >= rateMeasurementGranularity {
+		if !atomic.CompareAndSwapInt32(&rate.changing, 0, 1) {
+			return
+		}
+		atomic.StoreInt64(&rate.nextStartTime, now)
+		atomic.StoreUint64(&rate.current, total*uint64(time.Second/time.Nanosecond)/period)
+		atomic.StoreUint64(&rate.nextByteCount, 0)
+		atomic.StoreInt32(&rate.changing, 0)
+	}
+}