add locking for stop crash

.github/workflows/gofmt.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:

.github/workflows/release.yml

@@ -10,7 +10,7 @@ jobs:
     name: Build Linux/BSD All
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v6
         with:
@@ -24,7 +24,7 @@ jobs:
           mv build/*.tar.gz release
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v4
         with:
           name: linux-latest
           path: release
@@ -33,7 +33,7 @@ jobs:
     name: Build Windows
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v6
         with:
@@ -55,7 +55,7 @@ jobs:
           mv dist\windows\wintun build\dist\windows\
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v4
         with:
           name: windows-latest
           path: build
@@ -66,7 +66,7 @@ jobs:
       HAS_SIGNING_CREDS: ${{ secrets.AC_USERNAME != '' }}
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v6
         with:
@@ -75,7 +75,7 @@ jobs:
 
       - name: Import certificates
         if: env.HAS_SIGNING_CREDS == 'true'
-        uses: Apple-Actions/import-codesign-certs@v6
+        uses: Apple-Actions/import-codesign-certs@v5
         with:
           p12-file-base64: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }}
           p12-password: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }}
@@ -104,7 +104,7 @@ jobs:
           fi
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v4
         with:
           name: darwin-latest
           path: ./release/*
@@ -124,11 +124,11 @@ jobs:
       # be overwritten
       - name: Checkout code
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: Download artifacts
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v4
         with:
           name: linux-latest
           path: artifacts
@@ -160,10 +160,10 @@ jobs:
     needs: [build-linux, build-darwin, build-windows]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
       - name: Download artifacts
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v4
         with:
           path: artifacts
 

.github/workflows/smoke-extra.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:

.github/workflows/smoke.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:

.github/workflows/test.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:
@@ -32,7 +32,7 @@ jobs:
       run: make vet
 
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@v9
+      uses: golangci/golangci-lint-action@v8
       with:
         version: v2.5
 
@@ -45,7 +45,7 @@ jobs:
     - name: Build test mobile
       run: make build-test-mobile
 
-    - uses: actions/upload-artifact@v5
+    - uses: actions/upload-artifact@v4
       with:
         name: e2e packet flow linux-latest
         path: e2e/mermaid/linux-latest
@@ -56,7 +56,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:
@@ -77,7 +77,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:
@@ -98,7 +98,7 @@ jobs:
         os: [windows-latest, macos-latest]
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:
@@ -115,7 +115,7 @@ jobs:
       run: make vet
 
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@v9
+      uses: golangci/golangci-lint-action@v8
       with:
         version: v2.5
 
@@ -125,7 +125,7 @@ jobs:
     - name: End 2 end
       run: make e2evv
 
-    - uses: actions/upload-artifact@v5
+    - uses: actions/upload-artifact@v4
       with:
         name: e2e packet flow ${{ matrix.os }}
         path: e2e/mermaid/${{ matrix.os }}

CHANGELOG.md

@@ -7,85 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-## [1.10.0] - 2025-12-04
-
-See the [v1.10.0](https://github.com/slackhq/nebula/milestone/16?closed=1) milestone for a complete list of changes.
-
-### Added
-
-- Support for ipv6 and multiple ipv4/6 addresses in the overlay.
-  A new v2 ASN.1 based certificate format.
-  Certificates now have a unified interface for external implementations.
-  (#1212, #1216, #1345, #1359, #1381, #1419, #1464, #1466, #1451, #1476, #1467, #1481, #1399, #1488, #1492, #1495, #1468, #1521, #1535, #1538)
-- Add the ability to mark packets on linux to better target nebula packets in iptables/nftables. (#1331)
-- Add ECMP support for `unsafe_routes`. (#1332)
-- PKCS11 support for P256 keys when built with `pkcs11` tag (#1153, #1482)
-
 ### Changed
 
-- **NOTE**: `default_local_cidr_any` now defaults to false, meaning that any firewall rule
+- `default_local_cidr_any` now defaults to false, meaning that any firewall rule
   intended to target an `unsafe_routes` entry must explicitly declare it via the
   `local_cidr` field. This is almost always the intended behavior. This flag is
-  deprecated and will be removed in a future release. (#1373)
+  deprecated and will be removed in a future release.
-- Improve logging when a relay is in use on an inbound packet. (#1533)
-- Avoid fatal errors if `rountines` is > 1 on systems that don't support more than 1 routine. (#1531)
-- Log a warning if a firewall rule contains an `any` that negates a more restrictive filter. (#1513)
-- Accept encrypted CA passphrase from an environment variable. (#1421)
-- Allow handshaking with any trusted remote. (#1509)
-- Log only the count of blocklisted certificate fingerprints instead of the entire list. (#1525)
-- Don't fatal when the ssh server is unable to be configured successfully. (#1520)
-- Update to build against go v1.25. (#1483)
-- Allow projects using `nebula` as a library with userspace networking to configure the `logger` and build version. (#1239) 
-- Upgrade to `yaml.v3`. (#1148, #1371, #1438, #1478)
-
-### Fixed
-
-- Fix a potential bug with udp ipv4 only on darwin. (#1532)
-- Improve lost packet statistics. (#1441, #1537)
-- Honor `remote_allow_list` in hole punch response. (#1186)
-- Fix a panic when `tun.use_system_route_table` is `true` and a route lacks a destination. (#1437) 
-- Fix an issue when `tun.use_system_route_table: true` could result in heavy CPU utilization when many thousands of routes
-  are present. (#1326) 
-- Fix tests for 32 bit machines. (#1394)
-- Fix a possible 32bit integer underflow in config handling. (#1353)
-- Fix moving a udp address from one vpn address to another in the `static_host_map`
-  which could cause rapid re-handshaking with an incorrect remote. (#1259)
-- Improve smoke tests in environments where the docker network is not the default. (#1347)
-
-## [1.9.7] - 2025-10-10
-
-### Security
-
-- Fix an issue where Nebula could incorrectly accept and process a packet from an erroneous source IP when the sender's
-  certificate is configured with unsafe_routes (cert v1/v2) or multiple IPs (cert v2). (#1494)
-
-### Changed
-
-- Disable sending `recv_error` messages when a packet is received outside the allowable counter window. (#1459)
-- Improve error messages and remove some unnecessary fatal conditions in the Windows and generic udp listener. (#1453)
-
-## [1.9.6] - 2025-7-15
-
-### Added
-
-- Support dropping inactive tunnels. This is disabled by default in this release but can be enabled with `tunnels.drop_inactive`. See example config for more details. (#1413)
-
-### Fixed
-
-- Fix Darwin freeze due to presence of some Network Extensions (#1426)
-- Ensure the same relay tunnel is always used when multiple relay tunnels are present (#1422)
-- Fix Windows freeze due to ICMP error handling (#1412)
-- Fix relay migration panic (#1403)
-
-## [1.9.5] - 2024-12-05
-
-### Added
-
-- Gracefully ignore v2 certificates. (#1282)
-
-### Fixed
-
-- Fix relays that refuse to re-establish after one of the remote tunnel pairs breaks. (#1277)
 
 ## [1.9.4] - 2024-09-09
 
@@ -744,11 +671,7 @@ created.)
 
 - Initial public release.
 
-[Unreleased]: https://github.com/slackhq/nebula/compare/v1.10.0...HEAD
+[Unreleased]: https://github.com/slackhq/nebula/compare/v1.9.4...HEAD
-[1.10.0]: https://github.com/slackhq/nebula/releases/tag/v1.10.0
-[1.9.7]: https://github.com/slackhq/nebula/releases/tag/v1.9.7
-[1.9.6]: https://github.com/slackhq/nebula/releases/tag/v1.9.6
-[1.9.5]: https://github.com/slackhq/nebula/releases/tag/v1.9.5
 [1.9.4]: https://github.com/slackhq/nebula/releases/tag/v1.9.4
 [1.9.3]: https://github.com/slackhq/nebula/releases/tag/v1.9.3
 [1.9.2]: https://github.com/slackhq/nebula/releases/tag/v1.9.2

batch_pipeline.go

@@ -0,0 +1,164 @@
+package nebula
+
+import (
+	"net/netip"
+
+	"github.com/slackhq/nebula/overlay"
+	"github.com/slackhq/nebula/udp"
+)
+
+// batchPipelines tracks whether the inside device can operate on packet batches
+// and, if so, holds the shared packet pool sized for the virtio headroom and
+// payload limits advertised by the device. It also owns the fan-in/fan-out
+// queues between the TUN readers, encrypt/decrypt workers, and the UDP writers.
+type batchPipelines struct {
+	enabled    bool
+	inside     overlay.BatchCapableDevice
+	headroom   int
+	payloadCap int
+	pool       *overlay.PacketPool
+	batchSize  int
+	routines   int
+	rxQueues   []chan *overlay.Packet
+	txQueues   []chan queuedDatagram
+	tunQueues  []chan *overlay.Packet
+}
+
+type queuedDatagram struct {
+	packet *overlay.Packet
+	addr   netip.AddrPort
+}
+
+func (bp *batchPipelines) init(device overlay.Device, routines int, queueDepth int, maxSegments int) {
+	if device == nil || routines <= 0 {
+		return
+	}
+	bcap, ok := device.(overlay.BatchCapableDevice)
+	if !ok {
+		return
+	}
+	headroom := bcap.BatchHeadroom()
+	payload := bcap.BatchPayloadCap()
+	if maxSegments < 1 {
+		maxSegments = 1
+	}
+	requiredPayload := udp.MTU * maxSegments
+	if payload < requiredPayload {
+		payload = requiredPayload
+	}
+	batchSize := bcap.BatchSize()
+	if headroom <= 0 || payload <= 0 || batchSize <= 0 {
+		return
+	}
+	bp.enabled = true
+	bp.inside = bcap
+	bp.headroom = headroom
+	bp.payloadCap = payload
+	bp.batchSize = batchSize
+	bp.routines = routines
+	bp.pool = overlay.NewPacketPool(headroom, payload)
+	queueCap := batchSize * defaultBatchQueueDepthFactor
+	if queueDepth > 0 {
+		queueCap = queueDepth
+	}
+	if queueCap < batchSize {
+		queueCap = batchSize
+	}
+	bp.rxQueues = make([]chan *overlay.Packet, routines)
+	bp.txQueues = make([]chan queuedDatagram, routines)
+	bp.tunQueues = make([]chan *overlay.Packet, routines)
+	for i := 0; i < routines; i++ {
+		bp.rxQueues[i] = make(chan *overlay.Packet, queueCap)
+		bp.txQueues[i] = make(chan queuedDatagram, queueCap)
+		bp.tunQueues[i] = make(chan *overlay.Packet, queueCap)
+	}
+}
+
+func (bp *batchPipelines) Pool() *overlay.PacketPool {
+	if bp == nil || !bp.enabled {
+		return nil
+	}
+	return bp.pool
+}
+
+func (bp *batchPipelines) Enabled() bool {
+	return bp != nil && bp.enabled
+}
+
+func (bp *batchPipelines) batchSizeHint() int {
+	if bp == nil || bp.batchSize <= 0 {
+		return 1
+	}
+	return bp.batchSize
+}
+
+func (bp *batchPipelines) rxQueue(i int) chan *overlay.Packet {
+	if bp == nil || !bp.enabled || i < 0 || i >= len(bp.rxQueues) {
+		return nil
+	}
+	return bp.rxQueues[i]
+}
+
+func (bp *batchPipelines) txQueue(i int) chan queuedDatagram {
+	if bp == nil || !bp.enabled || i < 0 || i >= len(bp.txQueues) {
+		return nil
+	}
+	return bp.txQueues[i]
+}
+
+func (bp *batchPipelines) tunQueue(i int) chan *overlay.Packet {
+	if bp == nil || !bp.enabled || i < 0 || i >= len(bp.tunQueues) {
+		return nil
+	}
+	return bp.tunQueues[i]
+}
+
+func (bp *batchPipelines) txQueueLen(i int) int {
+	q := bp.txQueue(i)
+	if q == nil {
+		return 0
+	}
+	return len(q)
+}
+
+func (bp *batchPipelines) tunQueueLen(i int) int {
+	q := bp.tunQueue(i)
+	if q == nil {
+		return 0
+	}
+	return len(q)
+}
+
+func (bp *batchPipelines) enqueueRx(i int, pkt *overlay.Packet) bool {
+	q := bp.rxQueue(i)
+	if q == nil {
+		return false
+	}
+	q <- pkt
+	return true
+}
+
+func (bp *batchPipelines) enqueueTx(i int, pkt *overlay.Packet, addr netip.AddrPort) bool {
+	q := bp.txQueue(i)
+	if q == nil {
+		return false
+	}
+	q <- queuedDatagram{packet: pkt, addr: addr}
+	return true
+}
+
+func (bp *batchPipelines) enqueueTun(i int, pkt *overlay.Packet) bool {
+	q := bp.tunQueue(i)
+	if q == nil {
+		return false
+	}
+	q <- pkt
+	return true
+}
+
+func (bp *batchPipelines) newPacket() *overlay.Packet {
+	if bp == nil || !bp.enabled || bp.pool == nil {
+		return nil
+	}
+	return bp.pool.Get()
+}

bits.go

@@ -9,13 +9,14 @@ type Bits struct {
 	length             uint64
 	current            uint64
 	bits               []bool
+	firstSeen          bool
 	lostCounter        metrics.Counter
 	dupeCounter        metrics.Counter
 	outOfWindowCounter metrics.Counter
 }
 
 func NewBits(bits uint64) *Bits {
-	b := &Bits{
+	return &Bits{
 		length:             bits,
 		bits:               make([]bool, bits, bits),
 		current:            0,
@@ -23,37 +24,34 @@ func NewBits(bits uint64) *Bits {
 		dupeCounter:        metrics.GetOrRegisterCounter("network.packets.duplicate", nil),
 		outOfWindowCounter: metrics.GetOrRegisterCounter("network.packets.out_of_window", nil),
 	}
-
-	// There is no counter value 0, mark it to avoid counting a lost packet later.
-	b.bits[0] = true
-	b.current = 0
-	return b
 }
 
-func (b *Bits) Check(l *logrus.Logger, i uint64) bool {
+func (b *Bits) Check(l logrus.FieldLogger, i uint64) bool {
 	// If i is the next number, return true.
-	if i > b.current {
+	if i > b.current || (i == 0 && b.firstSeen == false && b.current < b.length) {
 		return true
 	}
 
-	// If i is within the window, check if it's been set already.
+	// If i is within the window, check if it's been set already. The first window will fail this check
-	if i > b.current-b.length || i < b.length && b.current < b.length {
+	if i > b.current-b.length {
+		return !b.bits[i%b.length]
+	}
+
+	// If i is within the first window
+	if i < b.length {
 		return !b.bits[i%b.length]
 	}
 
 	// Not within the window
-	if l.Level >= logrus.DebugLevel {
+	l.Debugf("rejected a packet (top) %d %d\n", b.current, i)
-		l.Debugf("rejected a packet (top) %d %d\n", b.current, i)
-	}
 	return false
 }
 
 func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 	// If i is the next number, return true and update current.
 	if i == b.current+1 {
-		// Check if the oldest bit was lost since we are shifting the window by 1 and occupying it with this counter
+		// Report missed packets, we can only understand what was missed after the first window has been gone through
-		// The very first window can only be tracked as lost once we are on the 2nd window or greater
+		if i > b.length && b.bits[i%b.length] == false {
-		if b.bits[i%b.length] == false && i > b.length {
 			b.lostCounter.Inc(1)
 		}
 		b.bits[i%b.length] = true
@@ -61,32 +59,61 @@ func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 		return true
 	}
 
-	// If i is a jump, adjust the window, record lost, update current, and return true
+	// If i packet is greater than current but less than the maximum length of our bitmap,
-	if i > b.current {
+	// flip everything in between to false and move ahead.
-		lost := int64(0)
+	if i > b.current && i < b.current+b.length {
-		// Zero out the bits between the current and the new counter value, limited by the window size,
+		// In between current and i need to be zero'd to allow those packets to come in later
-		// since the window is shifting
+		for n := b.current + 1; n < i; n++ {
-		for n := b.current + 1; n <= min(i, b.current+b.length); n++ {
-			if b.bits[n%b.length] == false && n > b.length {
-				lost++
-			}
 			b.bits[n%b.length] = false
 		}
 
-		// Only record any skipped packets as a result of the window moving further than the window length
+		b.bits[i%b.length] = true
-		// Any loss within the new window will be accounted for in future calls
+		b.current = i
-		lost += max(0, int64(i-b.current-b.length))
+		//l.Debugf("missed %d packets between %d and %d\n", i-b.current, i, b.current)
+		return true
+	}
+
+	// If i is greater than the delta between current and the total length of our bitmap,
+	// just flip everything in the map and move ahead.
+	if i >= b.current+b.length {
+		// The current window loss will be accounted for later, only record the jump as loss up until then
+		lost := maxInt64(0, int64(i-b.current-b.length))
+		//TODO: explain this
+		if b.current == 0 {
+			lost++
+		}
+
+		for n := range b.bits {
+			// Don't want to count the first window as a loss
+			//TODO: this is likely wrong, we are wanting to track only the bit slots that we aren't going to track anymore and this is marking everything as missed
+			//if b.bits[n] == false {
+			//	lost++
+			//}
+			b.bits[n] = false
+		}
+
 		b.lostCounter.Inc(lost)
 
+		if l.Level >= logrus.DebugLevel {
+			l.WithField("receiveWindow", m{"accepted": true, "currentCounter": b.current, "incomingCounter": i, "reason": "window shifting"}).
+				Debug("Receive window")
+		}
 		b.bits[i%b.length] = true
 		b.current = i
 		return true
 	}
 
-	// If i is within the current window but below the current counter,
+	// Allow for the 0 packet to come in within the first window
-	// Check to see if it's a duplicate
+	if i == 0 && b.firstSeen == false && b.current < b.length {
-	if i > b.current-b.length || i < b.length && b.current < b.length {
+		b.firstSeen = true
-		if b.current == i || b.bits[i%b.length] == true {
+		b.bits[i%b.length] = true
+		return true
+	}
+
+	// If i is within the window of current minus length (the total pat window size),
+	// allow it and flip to true but to NOT change current. We also have to account for the first window
+	if ((b.current >= b.length && i > b.current-b.length) || (b.current < b.length && i < b.length)) && i <= b.current {
+		if b.current == i {
 			if l.Level >= logrus.DebugLevel {
 				l.WithField("receiveWindow", m{"accepted": false, "currentCounter": b.current, "incomingCounter": i, "reason": "duplicate"}).
 					Debug("Receive window")
@@ -95,8 +122,18 @@ func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 			return false
 		}
 
+		if b.bits[i%b.length] == true {
+			if l.Level >= logrus.DebugLevel {
+				l.WithField("receiveWindow", m{"accepted": false, "currentCounter": b.current, "incomingCounter": i, "reason": "old duplicate"}).
+					Debug("Receive window")
+			}
+			b.dupeCounter.Inc(1)
+			return false
+		}
+
 		b.bits[i%b.length] = true
 		return true
+
 	}
 
 	// In all other cases, fail and don't change current.
@@ -110,3 +147,11 @@ func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 	}
 	return false
 }
+
+func maxInt64(a, b int64) int64 {
+	if a > b {
+		return a
+	}
+
+	return b
+}

bits_test.go

@@ -15,41 +15,48 @@ func TestBits(t *testing.T) {
 	assert.Len(t, b.bits, 10)
 
 	// This is initialized to zero - receive one. This should work.
+
 	assert.True(t, b.Check(l, 1))
-	assert.True(t, b.Update(l, 1))
+	u := b.Update(l, 1)
+	assert.True(t, u)
 	assert.EqualValues(t, 1, b.current)
-	g := []bool{true, true, false, false, false, false, false, false, false, false}
+	g := []bool{false, true, false, false, false, false, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Receive two
 	assert.True(t, b.Check(l, 2))
-	assert.True(t, b.Update(l, 2))
+	u = b.Update(l, 2)
+	assert.True(t, u)
 	assert.EqualValues(t, 2, b.current)
-	g = []bool{true, true, true, false, false, false, false, false, false, false}
+	g = []bool{false, true, true, false, false, false, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Receive two again - it will fail
 	assert.False(t, b.Check(l, 2))
-	assert.False(t, b.Update(l, 2))
+	u = b.Update(l, 2)
+	assert.False(t, u)
 	assert.EqualValues(t, 2, b.current)
 
 	// Jump ahead to 15, which should clear everything and set the 6th element
 	assert.True(t, b.Check(l, 15))
-	assert.True(t, b.Update(l, 15))
+	u = b.Update(l, 15)
+	assert.True(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, false, true, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Mark 14, which is allowed because it is in the window
 	assert.True(t, b.Check(l, 14))
-	assert.True(t, b.Update(l, 14))
+	u = b.Update(l, 14)
+	assert.True(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, true, true, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Mark 5, which is not allowed because it is not in the window
 	assert.False(t, b.Check(l, 5))
-	assert.False(t, b.Update(l, 5))
+	u = b.Update(l, 5)
+	assert.False(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, true, true, false, false, false, false}
 	assert.Equal(t, g, b.bits)
@@ -62,29 +69,10 @@ func TestBits(t *testing.T) {
 
 	// Walk through a few windows in order
 	b = NewBits(10)
-	for i := uint64(1); i <= 100; i++ {
+	for i := uint64(0); i <= 100; i++ {
 		assert.True(t, b.Check(l, i), "Error while checking %v", i)
 		assert.True(t, b.Update(l, i), "Error while updating %v", i)
 	}
-
-	assert.False(t, b.Check(l, 1), "Out of window check")
-}
-
-func TestBitsLargeJumps(t *testing.T) {
-	l := test.NewLogger()
-	b := NewBits(10)
-	b.lostCounter.Clear()
-
-	b = NewBits(10)
-	b.lostCounter.Clear()
-	assert.True(t, b.Update(l, 55)) // We saw packet 55 and can still track 45,46,47,48,49,50,51,52,53,54
-	assert.Equal(t, int64(45), b.lostCounter.Count())
-
-	assert.True(t, b.Update(l, 100)) // We saw packet 55 and 100 and can still track 90,91,92,93,94,95,96,97,98,99
-	assert.Equal(t, int64(89), b.lostCounter.Count())
-
-	assert.True(t, b.Update(l, 200)) // We saw packet 55, 100, and 200 and can still track 190,191,192,193,194,195,196,197,198,199
-	assert.Equal(t, int64(188), b.lostCounter.Count())
 }
 
 func TestBitsDupeCounter(t *testing.T) {
@@ -136,7 +124,8 @@ func TestBitsOutOfWindowCounter(t *testing.T) {
 	assert.False(t, b.Update(l, 0))
 	assert.Equal(t, int64(1), b.outOfWindowCounter.Count())
 
-	assert.Equal(t, int64(19), b.lostCounter.Count()) // packet 0 wasn't lost
+	//tODO: make sure lostcounter doesn't increase in orderly increment
+	assert.Equal(t, int64(20), b.lostCounter.Count())
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
 	assert.Equal(t, int64(1), b.outOfWindowCounter.Count())
 }
@@ -148,6 +137,8 @@ func TestBitsLostCounter(t *testing.T) {
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()
 
+	//assert.True(t, b.Update(0))
+	assert.True(t, b.Update(l, 0))
 	assert.True(t, b.Update(l, 20))
 	assert.True(t, b.Update(l, 21))
 	assert.True(t, b.Update(l, 22))
@@ -158,7 +149,7 @@ func TestBitsLostCounter(t *testing.T) {
 	assert.True(t, b.Update(l, 27))
 	assert.True(t, b.Update(l, 28))
 	assert.True(t, b.Update(l, 29))
-	assert.Equal(t, int64(19), b.lostCounter.Count()) // packet 0 wasn't lost
+	assert.Equal(t, int64(20), b.lostCounter.Count())
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 
@@ -167,6 +158,8 @@ func TestBitsLostCounter(t *testing.T) {
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()
 
+	assert.True(t, b.Update(l, 0))
+	assert.Equal(t, int64(0), b.lostCounter.Count())
 	assert.True(t, b.Update(l, 9))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
 	// 10 will set 0 index, 0 was already set, no lost packets
@@ -221,62 +214,6 @@ func TestBitsLostCounter(t *testing.T) {
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 }
 
-func TestBitsLostCounterIssue1(t *testing.T) {
-	l := test.NewLogger()
-	b := NewBits(10)
-	b.lostCounter.Clear()
-	b.dupeCounter.Clear()
-	b.outOfWindowCounter.Clear()
-
-	assert.True(t, b.Update(l, 4))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 1))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 9))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 2))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 3))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 5))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 6))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 7))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	// assert.True(t, b.Update(l, 8))
-	assert.True(t, b.Update(l, 10))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 11))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-
-	assert.True(t, b.Update(l, 14))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	// Issue seems to be here, we reset missing packet 8 to false here and don't increment the lost counter
-	assert.True(t, b.Update(l, 19))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 12))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 13))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 15))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 16))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 17))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 18))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 20))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 21))
-
-	// We missed packet 8 above
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.Equal(t, int64(0), b.dupeCounter.Count())
-	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
-}
-
 func BenchmarkBits(b *testing.B) {
 	z := NewBits(10)
 	for n := 0; n < b.N; n++ {

cert/cert_v1_test.go

@@ -99,19 +99,13 @@ func TestCertificateV1_PublicKeyPem(t *testing.T) {
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 AAAAAAAAAAAAAAAAAAAAAAA=
 -----END NEBULA P256 PUBLIC KEY-----
-`)
-
-	pubP256KeyPemCA := []byte(`-----BEGIN NEBULA ECDSA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA ECDSA P256 PUBLIC KEY-----
 `)
 	pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem)
 	require.NoError(t, err)
 	nc.details.curve = Curve_P256
 	nc.details.publicKey = pubP256Key
 	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPemCA))
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
 	assert.True(t, nc.IsCA())
 
 	nc.details.isCA = false

cert/cert_v2_test.go

@@ -114,19 +114,12 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 AAAAAAAAAAAAAAAAAAAAAAA=
 -----END NEBULA P256 PUBLIC KEY-----
 `)
-
-	pubP256KeyPemCA := []byte(`-----BEGIN NEBULA ECDSA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA ECDSA P256 PUBLIC KEY-----
-`)
-
 	pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem)
 	require.NoError(t, err)
 	nc.curve = Curve_P256
 	nc.publicKey = pubP256Key
 	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPemCA))
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
 	assert.True(t, nc.IsCA())
 
 	nc.details.isCA = false

cert/pem.go

@@ -1,8 +1,10 @@
 package cert
 
 import (
+	"encoding/hex"
 	"encoding/pem"
 	"fmt"
+	"time"
 
 	"golang.org/x/crypto/ed25519"
 )
@@ -86,7 +88,7 @@ func MarshalSigningPublicKeyToPEM(curve Curve, b []byte) []byte {
 	case Curve_CURVE25519:
 		return pem.EncodeToMemory(&pem.Block{Type: Ed25519PublicKeyBanner, Bytes: b})
 	case Curve_P256:
-		return pem.EncodeToMemory(&pem.Block{Type: ECDSAP256PublicKeyBanner, Bytes: b})
+		return pem.EncodeToMemory(&pem.Block{Type: P256PublicKeyBanner, Bytes: b})
 	default:
 		return nil
 	}
@@ -138,6 +140,101 @@ func MarshalSigningPrivateKeyToPEM(curve Curve, b []byte) []byte {
 	}
 }
 
+// Backward compatibility functions for older API
+func MarshalX25519PublicKey(b []byte) []byte {
+	return MarshalPublicKeyToPEM(Curve_CURVE25519, b)
+}
+
+func MarshalX25519PrivateKey(b []byte) []byte {
+	return MarshalPrivateKeyToPEM(Curve_CURVE25519, b)
+}
+
+func MarshalPublicKey(curve Curve, b []byte) []byte {
+	return MarshalPublicKeyToPEM(curve, b)
+}
+
+func MarshalPrivateKey(curve Curve, b []byte) []byte {
+	return MarshalPrivateKeyToPEM(curve, b)
+}
+
+// NebulaCertificate is a compatibility wrapper for the old API
+type NebulaCertificate struct {
+	Details   NebulaCertificateDetails
+	Signature []byte
+	cert      Certificate
+}
+
+// NebulaCertificateDetails is a compatibility wrapper for certificate details
+type NebulaCertificateDetails struct {
+	Name      string
+	NotBefore time.Time
+	NotAfter  time.Time
+	PublicKey []byte
+	IsCA      bool
+	Issuer    []byte
+	Curve     Curve
+}
+
+// UnmarshalNebulaCertificateFromPEM provides backward compatibility with the old API
+func UnmarshalNebulaCertificateFromPEM(b []byte) (*NebulaCertificate, []byte, error) {
+	c, rest, err := UnmarshalCertificateFromPEM(b)
+	if err != nil {
+		return nil, rest, err
+	}
+
+	issuerBytes, err := func() ([]byte, error) {
+		issuer := c.Issuer()
+		if issuer == "" {
+			return nil, nil
+		}
+		decoded, err := hex.DecodeString(issuer)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode issuer fingerprint: %w", err)
+		}
+		return decoded, nil
+	}()
+	if err != nil {
+		return nil, rest, err
+	}
+
+	pubKey := c.PublicKey()
+	if pubKey != nil {
+		pubKey = append([]byte(nil), pubKey...)
+	}
+
+	sig := c.Signature()
+	if sig != nil {
+		sig = append([]byte(nil), sig...)
+	}
+
+	return &NebulaCertificate{
+		Details: NebulaCertificateDetails{
+			Name:      c.Name(),
+			NotBefore: c.NotBefore(),
+			NotAfter:  c.NotAfter(),
+			PublicKey: pubKey,
+			IsCA:      c.IsCA(),
+			Issuer:    issuerBytes,
+			Curve:     c.Curve(),
+		},
+		Signature: sig,
+		cert:      c,
+	}, rest, nil
+}
+
+// IssuerString returns the issuer in hex format for compatibility
+func (n *NebulaCertificate) IssuerString() string {
+	if n.Details.Issuer == nil {
+		return ""
+	}
+	return hex.EncodeToString(n.Details.Issuer)
+}
+
+// Certificate returns the underlying certificate (read-only)
+func (n *NebulaCertificate) Certificate() Certificate {
+	return n.cert
+}
+
 // UnmarshalPrivateKeyFromPEM will try to unmarshal the first pem block in a byte array, returning any non
 // consumed data or an error on failure
 func UnmarshalPrivateKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {

cmd/nebula-cert/ca.go

@@ -173,26 +173,23 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
 
 	var passphrase []byte
 	if !isP11 && *cf.encryption {
-		passphrase = []byte(os.Getenv("NEBULA_CA_PASSPHRASE"))
+		for i := 0; i < 5; i++ {
+			out.Write([]byte("Enter passphrase: "))
+			passphrase, err = pr.ReadPassword()
+
+			if err == ErrNoTerminal {
+				return fmt.Errorf("out-key must be encrypted interactively")
+			} else if err != nil {
+				return fmt.Errorf("error reading passphrase: %s", err)
+			}
+
+			if len(passphrase) > 0 {
+				break
+			}
+		}
+
 		if len(passphrase) == 0 {
-			for i := 0; i < 5; i++ {
+			return fmt.Errorf("no passphrase specified, remove -encrypt flag to write out-key in plaintext")
-				out.Write([]byte("Enter passphrase: "))
-				passphrase, err = pr.ReadPassword()
-
-				if err == ErrNoTerminal {
-					return fmt.Errorf("out-key must be encrypted interactively")
-				} else if err != nil {
-					return fmt.Errorf("error reading passphrase: %s", err)
-				}
-
-				if len(passphrase) > 0 {
-					break
-				}
-			}
-
-			if len(passphrase) == 0 {
-				return fmt.Errorf("no passphrase specified, remove -encrypt flag to write out-key in plaintext")
-			}
 		}
 	}
 

cmd/nebula-cert/ca_test.go

@@ -171,17 +171,6 @@ func Test_ca(t *testing.T) {
 	assert.Equal(t, pwPromptOb, ob.String())
 	assert.Empty(t, eb.String())
 
-	// test encrypted key with passphrase environment variable
-	os.Remove(keyF.Name())
-	os.Remove(crtF.Name())
-	ob.Reset()
-	eb.Reset()
-	args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
-	os.Setenv("NEBULA_CA_PASSPHRASE", string(passphrase))
-	require.NoError(t, ca(args, ob, eb, testpw))
-	assert.Empty(t, eb.String())
-	os.Setenv("NEBULA_CA_PASSPHRASE", "")
-
 	// read encrypted key file and verify default params
 	rb, _ = os.ReadFile(keyF.Name())
 	k, _ := pem.Decode(rb)

cmd/nebula-cert/main.go

@@ -5,28 +5,10 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"runtime/debug"
-	"strings"
 )
 
-// A version string that can be set with
-//
-//	-ldflags "-X main.Build=SOMEVERSION"
-//
-// at compile-time.
 var Build string
 
-func init() {
-	if Build == "" {
-		info, ok := debug.ReadBuildInfo()
-		if !ok {
-			return
-		}
-
-		Build = strings.TrimPrefix(info.Main.Version, "v")
-	}
-}
-
 type helpError struct {
 	s string
 }

cmd/nebula-cert/sign.go

@@ -43,7 +43,7 @@ type signFlags struct {
 func newSignFlags() *signFlags {
 	sf := signFlags{set: flag.NewFlagSet("sign", flag.ContinueOnError)}
 	sf.set.Usage = func() {}
-	sf.version = sf.set.Uint("version", 0, "Optional: version of the certificate format to use. The default is to match the version of the signing CA")
+	sf.version = sf.set.Uint("version", 0, "Optional: version of the certificate format to use, the default is to create both v1 and v2 certificates.")
 	sf.caKeyPath = sf.set.String("ca-key", "ca.key", "Optional: path to the signing CA key")
 	sf.caCertPath = sf.set.String("ca-crt", "ca.crt", "Optional: path to the signing CA cert")
 	sf.name = sf.set.String("name", "", "Required: name of the cert, usually a hostname")
@@ -116,28 +116,26 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		// naively attempt to decode the private key as though it is not encrypted
 		caKey, _, curve, err = cert.UnmarshalSigningPrivateKeyFromPEM(rawCAKey)
 		if errors.Is(err, cert.ErrPrivateKeyEncrypted) {
+			// ask for a passphrase until we get one
 			var passphrase []byte
-			passphrase = []byte(os.Getenv("NEBULA_CA_PASSPHRASE"))
+			for i := 0; i < 5; i++ {
-			if len(passphrase) == 0 {
+				out.Write([]byte("Enter passphrase: "))
-				// ask for a passphrase until we get one
+				passphrase, err = pr.ReadPassword()
-				for i := 0; i < 5; i++ {
-					out.Write([]byte("Enter passphrase: "))
-					passphrase, err = pr.ReadPassword()
 
-					if errors.Is(err, ErrNoTerminal) {
+				if errors.Is(err, ErrNoTerminal) {
-						return fmt.Errorf("ca-key is encrypted and must be decrypted interactively")
+					return fmt.Errorf("ca-key is encrypted and must be decrypted interactively")
-					} else if err != nil {
+				} else if err != nil {
-						return fmt.Errorf("error reading password: %s", err)
+					return fmt.Errorf("error reading password: %s", err)
-					}
-
-					if len(passphrase) > 0 {
-						break
-					}
 				}
-				if len(passphrase) == 0 {
+
-					return fmt.Errorf("cannot open encrypted ca-key without passphrase")
+				if len(passphrase) > 0 {
+					break
 				}
 			}
+			if len(passphrase) == 0 {
+				return fmt.Errorf("cannot open encrypted ca-key without passphrase")
+			}
+
 			curve, caKey, _, err = cert.DecryptAndUnmarshalSigningPrivateKey(passphrase, rawCAKey)
 			if err != nil {
 				return fmt.Errorf("error while parsing encrypted ca-key: %s", err)
@@ -167,10 +165,6 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		return fmt.Errorf("ca certificate is expired")
 	}
 
-	if version == 0 {
-		version = caCert.Version()
-	}
-
 	// if no duration is given, expire one second before the root expires
 	if *sf.duration <= 0 {
 		*sf.duration = time.Until(caCert.NotAfter()) - time.Second*1
@@ -283,19 +277,21 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 	notBefore := time.Now()
 	notAfter := notBefore.Add(*sf.duration)
 
-	switch version {
+	if version == 0 || version == cert.Version1 {
-	case cert.Version1:
+		// Make sure we at least have an ip
-		// Make sure we have only one ipv4 address
 		if len(v4Networks) != 1 {
 			return newHelpErrorf("invalid -networks definition: v1 certificates can only have a single ipv4 address")
 		}
 
-		if len(v6Networks) > 0 {
+		if version == cert.Version1 {
-			return newHelpErrorf("invalid -networks definition: v1 certificates can only contain ipv4 addresses")
+			// If we are asked to mint a v1 certificate only then we cant just ignore any v6 addresses
-		}
+			if len(v6Networks) > 0 {
+				return newHelpErrorf("invalid -networks definition: v1 certificates can only be ipv4")
+			}
 
-		if len(v6UnsafeNetworks) > 0 {
+			if len(v6UnsafeNetworks) > 0 {
-			return newHelpErrorf("invalid -unsafe-networks definition: v1 certificates can only contain ipv4 addresses")
+				return newHelpErrorf("invalid -unsafe-networks definition: v1 certificates can only be ipv4")
+			}
 		}
 
 		t := &cert.TBSCertificate{
@@ -325,8 +321,9 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		}
 
 		crts = append(crts, nc)
+	}
 
-	case cert.Version2:
+	if version == 0 || version == cert.Version2 {
 		t := &cert.TBSCertificate{
 			Version:        cert.Version2,
 			Name:           *sf.name,
@@ -354,9 +351,6 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		}
 
 		crts = append(crts, nc)
-	default:
-		// this should be unreachable
-		return fmt.Errorf("invalid version: %d", version)
 	}
 
 	if !isP11 && *sf.inPubPath == "" {

cmd/nebula-cert/sign_test.go

@@ -55,7 +55,7 @@ func Test_signHelp(t *testing.T) {
 			"  -unsafe-networks string\n"+
 			"    \tOptional: comma separated list of ip address and network in CIDR notation. Unsafe networks this cert can route for\n"+
 			"  -version uint\n"+
-			"    \tOptional: version of the certificate format to use. The default is to match the version of the signing CA\n",
+			"    \tOptional: version of the certificate format to use, the default is to create both v1 and v2 certificates.\n",
 		ob.String(),
 	)
 }
@@ -204,7 +204,7 @@ func Test_signCert(t *testing.T) {
 	ob.Reset()
 	eb.Reset()
 	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m", "-subnets", "100::100/100"}
-	assertHelpError(t, signCert(args, ob, eb, nopw), "invalid -unsafe-networks definition: v1 certificates can only contain ipv4 addresses")
+	assertHelpError(t, signCert(args, ob, eb, nopw), "invalid -unsafe-networks definition: v1 certificates can only be ipv4")
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
 
@@ -379,15 +379,6 @@ func Test_signCert(t *testing.T) {
 	assert.Equal(t, "Enter passphrase: ", ob.String())
 	assert.Empty(t, eb.String())
 
-	// test with the proper password in the environment
-	os.Remove(crtF.Name())
-	os.Remove(keyF.Name())
-	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
-	os.Setenv("NEBULA_CA_PASSPHRASE", string(passphrase))
-	require.NoError(t, signCert(args, ob, eb, testpw))
-	assert.Empty(t, eb.String())
-	os.Setenv("NEBULA_CA_PASSPHRASE", "")
-
 	// test with the wrong password
 	ob.Reset()
 	eb.Reset()
@@ -398,17 +389,6 @@ func Test_signCert(t *testing.T) {
 	assert.Equal(t, "Enter passphrase: ", ob.String())
 	assert.Empty(t, eb.String())
 
-	// test with the wrong password in environment
-	ob.Reset()
-	eb.Reset()
-
-	os.Setenv("NEBULA_CA_PASSPHRASE", "invalid password")
-	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
-	require.EqualError(t, signCert(args, ob, eb, nopw), "error while parsing encrypted ca-key: invalid passphrase or corrupt private key")
-	assert.Empty(t, ob.String())
-	assert.Empty(t, eb.String())
-	os.Setenv("NEBULA_CA_PASSPHRASE", "")
-
 	// test with the user not entering a password
 	ob.Reset()
 	eb.Reset()

cmd/nebula-service/main.go

@@ -4,8 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"runtime/debug"
-	"strings"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
@@ -20,17 +18,6 @@ import (
 // at compile-time.
 var Build string
 
-func init() {
-	if Build == "" {
-		info, ok := debug.ReadBuildInfo()
-		if !ok {
-			return
-		}
-
-		Build = strings.TrimPrefix(info.Main.Version, "v")
-	}
-}
-
 func main() {
 	serviceFlag := flag.String("service", "", "Control the system service.")
 	configPath := flag.String("config", "", "Path to either a file or directory to load configuration from")

cmd/nebula/main.go

@@ -3,12 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
-	"log"
-	"net/http"
-	_ "net/http/pprof"
 	"os"
-	"runtime/debug"
-	"strings"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
@@ -23,17 +18,6 @@ import (
 // at compile-time.
 var Build string
 
-func init() {
-	if Build == "" {
-		info, ok := debug.ReadBuildInfo()
-		if !ok {
-			return
-		}
-
-		Build = strings.TrimPrefix(info.Main.Version, "v")
-	}
-}
-
 func main() {
 	configPath := flag.String("config", "", "Path to either a file or directory to load configuration from")
 	configTest := flag.Bool("test", false, "Test the config and print the end result. Non zero exit indicates a faulty config")
@@ -74,10 +58,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	go func() {
-		log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
-	}()
-
 	if !*configTest {
 		ctrl.Start()
 		notifyReady(l)

config/config.go

@@ -17,7 +17,7 @@ import (
 
 	"dario.cat/mergo"
 	"github.com/sirupsen/logrus"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v3"
 )
 
 type C struct {

config/config_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v3"
 )
 
 func TestConfig_Load(t *testing.T) {

connection_state.go

@@ -13,7 +13,7 @@ import (
 	"github.com/slackhq/nebula/noiseutil"
 )
 
-const ReplayWindow = 4096
+const ReplayWindow = 1024
 
 type ConnectionState struct {
 	eKey           *NebulaCipherState
@@ -50,6 +50,11 @@ func NewConnectionState(l *logrus.Logger, cs *CertState, crt cert.Certificate, i
 	}
 
 	static := noise.DHKey{Private: cs.privateKey, Public: crt.PublicKey()}
+
+	b := NewBits(ReplayWindow)
+	// Clear out bit 0, we never transmit it, and we don't want it showing as packet loss
+	b.Update(l, 0)
+
 	hs, err := noise.NewHandshakeState(noise.Config{
 		CipherSuite:   ncs,
 		Random:        rand.Reader,
@@ -69,7 +74,7 @@ func NewConnectionState(l *logrus.Logger, cs *CertState, crt cert.Certificate, i
 	ci := &ConnectionState{
 		H:         hs,
 		initiator: initiator,
-		window:    NewBits(ReplayWindow),
+		window:    b,
 		myCert:    crt,
 	}
 	// always start the counter from 2, as packet 1 and packet 2 are handshake packets.

control_tester.go

@@ -80,8 +80,7 @@ func (c *Control) GetFromTun(block bool) []byte {
 
 // GetFromUDP will pull a udp packet off the udp side of nebula
 func (c *Control) GetFromUDP(block bool) *udp.Packet {
-	out := c.f.outside.(*udp.TesterConn).Get(block)
+	return c.f.outside.(*udp.TesterConn).Get(block)
-	return out
 }
 
 func (c *Control) GetUDPTxChan() <-chan *udp.Packet {
@@ -175,10 +174,6 @@ func (c *Control) GetHostmap() *HostMap {
 	return c.f.hostMap
 }
 
-func (c *Control) GetF() *Interface {
-	return c.f
-}
-
 func (c *Control) GetCertState() *CertState {
 	return c.f.pki.getCertState()
 }

e2e/handshakes_test.go

@@ -20,17 +20,16 @@ import (
 	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v3"
 )
 
 func BenchmarkHotPath(b *testing.B) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
 	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
 
 	// Put their info in our lighthouse
 	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
 
 	// Start the servers
 	myControl.Start()
@@ -39,9 +38,6 @@ func BenchmarkHotPath(b *testing.B) {
 	r := router.NewR(b, myControl, theirControl)
 	r.CancelFlowLogs()
 
-	assertTunnel(b, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	b.ResetTimer()
-
 	for n := 0; n < b.N; n++ {
 		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 		_ = r.RouteForAllUntilTxTun(theirControl)
@@ -51,39 +47,6 @@ func BenchmarkHotPath(b *testing.B) {
 	theirControl.Stop()
 }
 
-func BenchmarkHotPathRelay(b *testing.B) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay  ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them   ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(b, myControl, relayControl, theirControl)
-	r.CancelFlowLogs()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	assertTunnel(b, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
-	b.ResetTimer()
-
-	for n := 0; n < b.N; n++ {
-		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-		_ = r.RouteForAllUntilTxTun(theirControl)
-	}
-
-	myControl.Stop()
-	theirControl.Stop()
-	relayControl.Stop()
-}
-
 func TestGoodHandshake(t *testing.T) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
@@ -134,41 +97,6 @@ func TestGoodHandshake(t *testing.T) {
 	theirControl.Stop()
 }
 
-func TestGoodHandshakeNoOverlap(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "2001::69/24", nil) //look ma, cross-stack!
-
-	// Put their info in our lighthouse
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	empty := []byte{}
-	t.Log("do something to cause a handshake")
-	myControl.GetF().SendMessageToVpnAddr(header.Test, header.MessageNone, theirVpnIpNet[0].Addr(), empty, empty, empty)
-
-	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
-	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
-
-	t.Log("Get their stage 1 packet")
-	stage1Packet := theirControl.GetFromUDP(true)
-
-	t.Log("Have me consume their stage 1 packet. I have a tunnel now")
-	myControl.InjectUDPPacket(stage1Packet)
-
-	t.Log("Wait until we see a test packet come through to make sure we give the tunnel time to complete")
-	myControl.WaitForType(header.Test, 0, theirControl)
-
-	t.Log("Make sure our host infos are correct")
-	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
 func TestWrongResponderHandshake(t *testing.T) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 
@@ -536,35 +464,6 @@ func TestRelays(t *testing.T) {
 	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
 }
 
-func TestRelaysDontCareAboutIps(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay  ", "2001::9999/24", m{"relay": m{"am_relay": true}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them   ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-
-	p := r.RouteForAllUntilTxTun(theirControl)
-	r.Log("Assert the tunnel works")
-	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
-	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
-}
-
 func TestReestablishRelays(t *testing.T) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
@@ -1328,109 +1227,3 @@ func TestV2NonPrimaryWithLighthouse(t *testing.T) {
 	myControl.Stop()
 	theirControl.Stop()
 }
-
-func TestV2NonPrimaryWithOffNetLighthouse(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh  ", "2001::1/64", m{"lighthouse": m{"am_lighthouse": true}})
-
-	o := m{
-		"static_host_map": m{
-			lhVpnIpNet[0].Addr().String(): []string{lhUdpAddr.String()},
-		},
-		"lighthouse": m{
-			"hosts": []string{lhVpnIpNet[0].Addr().String()},
-			"local_allow_list": m{
-				// Try and block our lighthouse updates from using the actual addresses assigned to this computer
-				// If we start discovering addresses the test router doesn't know about then test traffic cant flow
-				"10.0.0.0/24": true,
-				"::/0":        false,
-			},
-		},
-	}
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me  ", "10.128.0.2/24, ff::2/64", o)
-	theirControl, theirVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24, ff::3/64", o)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, lhControl, myControl, theirControl)
-	defer r.RenderFlow()
-
-	// Start the servers
-	lhControl.Start()
-	myControl.Start()
-	theirControl.Start()
-
-	t.Log("Stand up an ipv6 tunnel between me and them")
-	assert.True(t, myVpnIpNet[1].Addr().Is6())
-	assert.True(t, theirVpnIpNet[1].Addr().Is6())
-	assertTunnel(t, myVpnIpNet[1].Addr(), theirVpnIpNet[1].Addr(), myControl, theirControl, r)
-
-	lhControl.Stop()
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestGoodHandshakeUnsafeDest(t *testing.T) {
-	unsafePrefix := "192.168.6.0/24"
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdpAndUnsafeNetworks(cert.Version2, ca, caKey, "spooky", "10.128.0.2/24", netip.MustParseAddrPort("10.64.0.2:4242"), unsafePrefix, nil)
-	route := m{"route": unsafePrefix, "via": theirVpnIpNet[0].Addr().String()}
-	myCfg := m{
-		"tun": m{
-			"unsafe_routes": []m{route},
-		},
-	}
-	myControl, myVpnIpNet, myUdpAddr, myConfig := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", myCfg)
-	t.Logf("my config %v", myConfig)
-	// Put their info in our lighthouse
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	spookyDest := netip.MustParseAddr("192.168.6.4")
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	t.Log("Send a udp packet through to begin standing up the tunnel, this should come out the other side")
-	myControl.InjectTunUDPPacket(spookyDest, 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-
-	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
-	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
-
-	t.Log("Get their stage 1 packet so that we can play with it")
-	stage1Packet := theirControl.GetFromUDP(true)
-
-	t.Log("I consume a garbage packet with a proper nebula header for our tunnel")
-	// this should log a statement and get ignored, allowing the real handshake packet to complete the tunnel
-	badPacket := stage1Packet.Copy()
-	badPacket.Data = badPacket.Data[:len(badPacket.Data)-header.Len]
-	myControl.InjectUDPPacket(badPacket)
-
-	t.Log("Have me consume their real stage 1 packet. I have a tunnel now")
-	myControl.InjectUDPPacket(stage1Packet)
-
-	t.Log("Wait until we see my cached packet come through")
-	myControl.WaitForType(1, 0, theirControl)
-
-	t.Log("Make sure our host infos are correct")
-	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl)
-
-	t.Log("Get that cached packet and make sure it looks right")
-	myCachedPacket := theirControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet[0].Addr(), spookyDest, 80, 80)
-
-	//reply
-	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, spookyDest, 80, []byte("Hi from the spookyman"))
-	//wait for reply
-	theirControl.WaitForType(1, 0, myControl)
-	theirCachedPacket := myControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi from the spookyman"), theirCachedPacket, spookyDest, myVpnIpNet[0].Addr(), 80, 80)
-
-	t.Log("Do a bidirectional tunnel test")
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-	myControl.Stop()
-	theirControl.Stop()
-}

e2e/helpers_test.go

@@ -22,14 +22,15 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/e2e/router"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
-	"go.yaml.in/yaml/v3"
 )
 
 type m = map[string]any
 
 // newSimpleServer creates a nebula instance with many assumptions
 func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
+	l := NewTestLogger()
+
 	var vpnNetworks []netip.Prefix
 	for _, sn := range strings.Split(sVpnNetworks, ",") {
 		vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn))
@@ -55,54 +56,7 @@ func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name
 		budpIp[3] = 239
 		udpAddr = netip.AddrPortFrom(netip.AddrFrom16(budpIp), 4242)
 	}
-	return newSimpleServerWithUdp(v, caCrt, caKey, name, sVpnNetworks, udpAddr, overrides)
+	_, _, myPrivKey, myPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, caCrt, caKey, name, time.Now(), time.Now().Add(5*time.Minute), vpnNetworks, nil, []string{})
-}
-
-func newSimpleServerWithUdp(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, udpAddr netip.AddrPort, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
-	return newSimpleServerWithUdpAndUnsafeNetworks(v, caCrt, caKey, name, sVpnNetworks, udpAddr, "", overrides)
-}
-
-func newSimpleServerWithUdpAndUnsafeNetworks(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, udpAddr netip.AddrPort, sUnsafeNetworks string, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
-	l := NewTestLogger()
-
-	var vpnNetworks []netip.Prefix
-	for _, sn := range strings.Split(sVpnNetworks, ",") {
-		vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn))
-		if err != nil {
-			panic(err)
-		}
-		vpnNetworks = append(vpnNetworks, vpnIpNet)
-	}
-
-	if len(vpnNetworks) == 0 {
-		panic("no vpn networks")
-	}
-
-	firewallInbound := []m{{
-		"proto": "any",
-		"port":  "any",
-		"host":  "any",
-	}}
-
-	var unsafeNetworks []netip.Prefix
-	if sUnsafeNetworks != "" {
-		firewallInbound = []m{{
-			"proto":      "any",
-			"port":       "any",
-			"host":       "any",
-			"local_cidr": "0.0.0.0/0",
-		}}
-
-		for _, sn := range strings.Split(sUnsafeNetworks, ",") {
-			x, err := netip.ParsePrefix(strings.TrimSpace(sn))
-			if err != nil {
-				panic(err)
-			}
-			unsafeNetworks = append(unsafeNetworks, x)
-		}
-	}
-
-	_, _, myPrivKey, myPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, caCrt, caKey, name, time.Now(), time.Now().Add(5*time.Minute), vpnNetworks, unsafeNetworks, []string{})
 
 	caB, err := caCrt.MarshalPEM()
 	if err != nil {
@@ -122,7 +76,11 @@ func newSimpleServerWithUdpAndUnsafeNetworks(v cert.Version, caCrt cert.Certific
 				"port":  "any",
 				"host":  "any",
 			}},
-			"inbound": firewallInbound,
+			"inbound": []m{{
+				"proto": "any",
+				"port":  "any",
+				"host":  "any",
+			}},
 		},
 		//"handshakes": m{
 		//	"try_interval": "1s",
@@ -292,7 +250,7 @@ func deadline(t *testing.T, seconds time.Duration) doneCb {
 	}
 }
 
-func assertTunnel(t testing.TB, vpnIpA, vpnIpB netip.Addr, controlA, controlB *nebula.Control, r *router.R) {
+func assertTunnel(t *testing.T, vpnIpA, vpnIpB netip.Addr, controlA, controlB *nebula.Control, r *router.R) {
 	// Send a packet from them to me
 	controlB.InjectTunUDPPacket(vpnIpA, 80, vpnIpB, 90, []byte("Hi from B"))
 	bPacket := r.RouteForAllUntilTxTun(controlA)
@@ -304,14 +262,14 @@ func assertTunnel(t testing.TB, vpnIpA, vpnIpB netip.Addr, controlA, controlB *n
 	assertUdpPacket(t, []byte("Hello from A"), aPacket, vpnIpA, vpnIpB, 90, 80)
 }
 
-func assertHostInfoPair(t testing.TB, addrA, addrB netip.AddrPort, vpnNetsA, vpnNetsB []netip.Prefix, controlA, controlB *nebula.Control) {
+func assertHostInfoPair(t *testing.T, addrA, addrB netip.AddrPort, vpnNetsA, vpnNetsB []netip.Prefix, controlA, controlB *nebula.Control) {
 	// Get both host infos
 	//TODO: CERT-V2 we may want to loop over each vpnAddr and assert all the things
 	hBinA := controlA.GetHostInfoByVpnAddr(vpnNetsB[0].Addr(), false)
-	require.NotNil(t, hBinA, "Host B was not found by vpnAddr in controlA")
+	assert.NotNil(t, hBinA, "Host B was not found by vpnAddr in controlA")
 
 	hAinB := controlB.GetHostInfoByVpnAddr(vpnNetsA[0].Addr(), false)
-	require.NotNil(t, hAinB, "Host A was not found by vpnAddr in controlB")
+	assert.NotNil(t, hAinB, "Host A was not found by vpnAddr in controlB")
 
 	// Check that both vpn and real addr are correct
 	assert.EqualValues(t, getAddrs(vpnNetsB), hBinA.VpnAddrs, "Host B VpnIp is wrong in control A")
@@ -325,7 +283,7 @@ func assertHostInfoPair(t testing.TB, addrA, addrB netip.AddrPort, vpnNetsA, vpn
 	assert.Equal(t, hBinA.RemoteIndex, hAinB.LocalIndex, "Host B remote index does not match host A local index")
 }
 
-func assertUdpPacket(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
+func assertUdpPacket(t *testing.T, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
 	if toIp.Is6() {
 		assertUdpPacket6(t, expected, b, fromIp, toIp, fromPort, toPort)
 	} else {
@@ -333,7 +291,7 @@ func assertUdpPacket(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr,
 	}
 }
 
-func assertUdpPacket6(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
+func assertUdpPacket6(t *testing.T, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
 	packet := gopacket.NewPacket(b, layers.LayerTypeIPv6, gopacket.Lazy)
 	v6 := packet.Layer(layers.LayerTypeIPv6).(*layers.IPv6)
 	assert.NotNil(t, v6, "No ipv6 data found")
@@ -352,7 +310,7 @@ func assertUdpPacket6(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr,
 	assert.Equal(t, expected, data.Payload(), "Data was incorrect")
 }
 
-func assertUdpPacket4(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
+func assertUdpPacket4(t *testing.T, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
 	packet := gopacket.NewPacket(b, layers.LayerTypeIPv4, gopacket.Lazy)
 	v4 := packet.Layer(layers.LayerTypeIPv4).(*layers.IPv4)
 	assert.NotNil(t, v4, "No ipv4 data found")

e2e/tunnels_test.go

@@ -318,50 +318,3 @@ func TestCertMismatchCorrection(t *testing.T) {
 	myControl.Stop()
 	theirControl.Stop()
 }
-
-func TestCrossStackRelaysWork(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24,fc00::1/64", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay  ", "10.128.0.128/24,fc00::128/64", m{"relay": m{"am_relay": true}})
-	theirUdp := netip.MustParseAddrPort("10.0.0.2:4242")
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, "them   ", "fc00::2/64", theirUdp, m{"relay": m{"use_relays": true}})
-
-	//myVpnV4 := myVpnIpNet[0]
-	myVpnV6 := myVpnIpNet[1]
-	relayVpnV4 := relayVpnIpNet[0]
-	relayVpnV6 := relayVpnIpNet[1]
-	theirVpnV6 := theirVpnIpNet[0]
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnV4.Addr(), relayUdpAddr)
-	myControl.InjectLightHouseAddr(relayVpnV6.Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnV6.Addr(), []netip.Addr{relayVpnV6.Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnV6.Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me"))
-
-	p := r.RouteForAllUntilTxTun(theirControl)
-	r.Log("Assert the tunnel works")
-	assertUdpPacket(t, []byte("Hi from me"), p, myVpnV6.Addr(), theirVpnV6.Addr(), 80, 80)
-
-	t.Log("reply?")
-	theirControl.InjectTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them"))
-	p = r.RouteForAllUntilTxTun(myControl)
-	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnV6.Addr(), myVpnV6.Addr(), 80, 80)
-
-	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
-	//t.Log("finish up")
-	//myControl.Stop()
-	//theirControl.Stop()
-	//relayControl.Stop()
-}

examples/config.yml

@@ -383,9 +383,8 @@ firewall:
   #   host: `any` or a literal hostname, ie `test-host`
   #   group: `any` or a literal group name, ie `default-group`
   #   groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass
-  #   cidr: a remote CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6. `any` means any ip family and address.
+  #   cidr: a remote CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6.
-  #   local_cidr: a local CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6. `any` means any ip family and address.
+  #   local_cidr: a local CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6. This can be used to filter destinations when using unsafe_routes.
-  #     This can be used to filter destinations when using unsafe_routes.
   #     By default, this is set to only the VPN (overlay) networks assigned via the certificate networks field unless `default_local_cidr_any` is set to true.
   #     If there are unsafe_routes present in this config file, `local_cidr` should be set appropriately for the intended us case.
   #   ca_name: An issuing CA name

firewall.go

@@ -8,7 +8,6 @@ import (
 	"hash/fnv"
 	"net/netip"
 	"reflect"
-	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -23,7 +22,7 @@ import (
 )
 
 type FirewallInterface interface {
-	AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
+	AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, addr, localAddr netip.Prefix, caName string, caSha string) error
 }
 
 type conn struct {
@@ -248,11 +247,22 @@ func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firew
 }
 
 // AddRule properly creates the in memory rule structure for a firewall table.
-func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
+func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip, localIp netip.Prefix, caName string, caSha string) error {
+	// Under gomobile, stringing a nil pointer with fmt causes an abort in debug mode for iOS
+	// https://github.com/golang/go/issues/14131
+	sIp := ""
+	if ip.IsValid() {
+		sIp = ip.String()
+	}
+	lIp := ""
+	if localIp.IsValid() {
+		lIp = localIp.String()
+	}
+
 	// We need this rule string because we generate a hash. Removing this will break firewall reload.
 	ruleString := fmt.Sprintf(
 		"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s",
-		incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha,
+		incoming, proto, startPort, endPort, groups, host, sIp, lIp, caName, caSha,
 	)
 	f.rules += ruleString + "\n"
 
@@ -260,7 +270,7 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 	if !incoming {
 		direction = "outgoing"
 	}
-	f.l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha}).
+	f.l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "ip": sIp, "localIp": lIp, "caName": caName, "caSha": caSha}).
 		Info("Firewall rule added")
 
 	var (
@@ -287,7 +297,7 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 		return fmt.Errorf("unknown protocol %v", proto)
 	}
 
-	return fp.addRule(f, startPort, endPort, groups, host, cidr, localCidr, caName, caSha)
+	return fp.addRule(f, startPort, endPort, groups, host, ip, localIp, caName, caSha)
 }
 
 // GetRuleHash returns a hash representation of all inbound and outbound rules
@@ -327,6 +337,7 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 	}
 
 	for i, t := range rs {
+		var groups []string
 		r, err := convertRule(l, t, table, i)
 		if err != nil {
 			return fmt.Errorf("%s rule #%v; %s", table, i, err)
@@ -336,10 +347,23 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 			return fmt.Errorf("%s rule #%v; only one of port or code should be provided", table, i)
 		}
 
-		if r.Host == "" && len(r.Groups) == 0 && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
+		if r.Host == "" && len(r.Groups) == 0 && r.Group == "" && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
 			return fmt.Errorf("%s rule #%v; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided", table, i)
 		}
 
+		if len(r.Groups) > 0 {
+			groups = r.Groups
+		}
+
+		if r.Group != "" {
+			// Check if we have both groups and group provided in the rule config
+			if len(groups) > 0 {
+				return fmt.Errorf("%s rule #%v; only one of group or groups should be defined, both provided", table, i)
+			}
+
+			groups = []string{r.Group}
+		}
+
 		var sPort, errPort string
 		if r.Code != "" {
 			errPort = "code"
@@ -368,25 +392,23 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 			return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
 		}
 
-		if r.Cidr != "" && r.Cidr != "any" {
+		var cidr netip.Prefix
-			_, err = netip.ParsePrefix(r.Cidr)
+		if r.Cidr != "" {
+			cidr, err = netip.ParsePrefix(r.Cidr)
 			if err != nil {
 				return fmt.Errorf("%s rule #%v; cidr did not parse; %s", table, i, err)
 			}
 		}
 
-		if r.LocalCidr != "" && r.LocalCidr != "any" {
+		var localCidr netip.Prefix
-			_, err = netip.ParsePrefix(r.LocalCidr)
+		if r.LocalCidr != "" {
+			localCidr, err = netip.ParsePrefix(r.LocalCidr)
 			if err != nil {
 				return fmt.Errorf("%s rule #%v; local_cidr did not parse; %s", table, i, err)
 			}
 		}
 
-		if warning := r.sanity(); warning != nil {
+		err = fw.AddRule(inbound, proto, startPort, endPort, groups, r.Host, cidr, localCidr, r.CAName, r.CASha)
-			l.Warnf("%s rule #%v; %s", table, i, warning)
-		}
-
-		err = fw.AddRule(inbound, proto, startPort, endPort, r.Groups, r.Host, r.Cidr, r.LocalCidr, r.CAName, r.CASha)
 		if err != nil {
 			return fmt.Errorf("%s rule #%v; `%s`", table, i, err)
 		}
@@ -395,45 +417,30 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 	return nil
 }
 
-var ErrUnknownNetworkType = errors.New("unknown network type")
+var ErrInvalidRemoteIP = errors.New("remote IP is not in remote certificate subnets")
-var ErrPeerRejected = errors.New("remote address is not within a network that we handle")
+var ErrInvalidLocalIP = errors.New("local IP is not in list of handled local IPs")
-var ErrInvalidRemoteIP = errors.New("remote address is not in remote certificate networks")
-var ErrInvalidLocalIP = errors.New("local address is not in list of handled local addresses")
 var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
 
 // Drop returns an error if the packet should be dropped, explaining why. It
 // returns nil if the packet should not be dropped.
-func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache, now time.Time) error {
+func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache *firewall.ConntrackCache) error {
 	// Check if we spoke to this tuple, if we did then allow this packet
-	if f.inConns(fp, h, caPool, localCache, now) {
+	if f.inConns(fp, h, caPool, localCache) {
 		return nil
 	}
 
-	// Make sure remote address matches nebula certificate, and determine how to treat it
+	// Make sure remote address matches nebula certificate
-	if h.networks == nil {
+	if h.networks != nil {
-		// Simple case: Certificate has one address and no unsafe networks
+		if !h.networks.Contains(fp.RemoteAddr) {
-		if h.vpnAddrs[0] != fp.RemoteAddr {
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
 			return ErrInvalidRemoteIP
 		}
 	} else {
-		nwType, ok := h.networks.Lookup(fp.RemoteAddr)
+		// Simple case: Certificate has one address and no unsafe networks
-		if !ok {
+		if h.vpnAddrs[0] != fp.RemoteAddr {
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
 			return ErrInvalidRemoteIP
 		}
-		switch nwType {
-		case NetworkTypeVPN:
-			break // nothing special
-		case NetworkTypeVPNPeer:
-			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-			return ErrPeerRejected // reject for now, one day this may have different FW rules
-		case NetworkTypeUnsafe:
-			break // nothing special, one day this may have different FW rules
-		default:
-			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-			return ErrUnknownNetworkType //should never happen
-		}
 	}
 
 	// Make sure we are supposed to be handling this local ip address
@@ -454,7 +461,7 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
 	}
 
 	// We always want to conntrack since it is a faster operation
-	f.addConn(fp, incoming, now)
+	f.addConn(fp, incoming)
 
 	return nil
 }
@@ -483,11 +490,9 @@ func (f *Firewall) EmitStats() {
 	metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
 }
 
-func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache, now time.Time) bool {
+func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache *firewall.ConntrackCache) bool {
-	if localCache != nil {
+	if localCache != nil && localCache.Has(fp) {
-		if _, ok := localCache[fp]; ok {
+		return true
-			return true
-		}
 	}
 	conntrack := f.Conntrack
 	conntrack.Lock()
@@ -495,7 +500,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 	// Purge every time we test
 	ep, has := conntrack.TimerWheel.Purge()
 	if has {
-		f.evict(ep, now)
+		f.evict(ep)
 	}
 
 	c, ok := conntrack.Conns[fp]
@@ -542,23 +547,23 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 
 	switch fp.Protocol {
 	case firewall.ProtoTCP:
-		c.Expires = now.Add(f.TCPTimeout)
+		c.Expires = time.Now().Add(f.TCPTimeout)
 	case firewall.ProtoUDP:
-		c.Expires = now.Add(f.UDPTimeout)
+		c.Expires = time.Now().Add(f.UDPTimeout)
 	default:
-		c.Expires = now.Add(f.DefaultTimeout)
+		c.Expires = time.Now().Add(f.DefaultTimeout)
 	}
 
 	conntrack.Unlock()
 
 	if localCache != nil {
-		localCache[fp] = struct{}{}
+		localCache.Add(fp)
 	}
 
 	return true
 }
 
-func (f *Firewall) addConn(fp firewall.Packet, incoming bool, now time.Time) {
+func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
 	var timeout time.Duration
 	c := &conn{}
 
@@ -574,7 +579,7 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool, now time.Time) {
 	conntrack := f.Conntrack
 	conntrack.Lock()
 	if _, ok := conntrack.Conns[fp]; !ok {
-		conntrack.TimerWheel.Advance(now)
+		conntrack.TimerWheel.Advance(time.Now())
 		conntrack.TimerWheel.Add(fp, timeout)
 	}
 
@@ -582,14 +587,14 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool, now time.Time) {
 	// firewall reload
 	c.incoming = incoming
 	c.rulesVersion = f.rulesVersion
-	c.Expires = now.Add(timeout)
+	c.Expires = time.Now().Add(timeout)
 	conntrack.Conns[fp] = c
 	conntrack.Unlock()
 }
 
 // Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel
 // Caller must own the connMutex lock!
-func (f *Firewall) evict(p firewall.Packet, now time.Time) {
+func (f *Firewall) evict(p firewall.Packet) {
 	// Are we still tracking this conn?
 	conntrack := f.Conntrack
 	t, ok := conntrack.Conns[p]
@@ -597,11 +602,11 @@ func (f *Firewall) evict(p firewall.Packet, now time.Time) {
 		return
 	}
 
-	newT := t.Expires.Sub(now)
+	newT := t.Expires.Sub(time.Now())
 
 	// Timeout is in the future, re-add the timer
 	if newT > 0 {
-		conntrack.TimerWheel.Advance(now)
+		conntrack.TimerWheel.Advance(time.Now())
 		conntrack.TimerWheel.Add(p, newT)
 		return
 	}
@@ -633,7 +638,7 @@ func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedC
 	return false
 }
 
-func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
+func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, ip, localIp netip.Prefix, caName string, caSha string) error {
 	if startPort > endPort {
 		return fmt.Errorf("start port was lower than end port")
 	}
@@ -646,7 +651,7 @@ func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, grou
 			}
 		}
 
-		if err := fp[i].addRule(f, groups, host, cidr, localCidr, caName, caSha); err != nil {
+		if err := fp[i].addRule(f, groups, host, ip, localIp, caName, caSha); err != nil {
 			return err
 		}
 	}
@@ -677,7 +682,7 @@ func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.CachedCer
 	return fp[firewall.PortAny].match(p, c, caPool)
 }
 
-func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, localCidr, caName, caSha string) error {
+func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, ip, localIp netip.Prefix, caName, caSha string) error {
 	fr := func() *FirewallRule {
 		return &FirewallRule{
 			Hosts:  make(map[string]*firewallLocalCIDR),
@@ -691,14 +696,14 @@ func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, l
 			fc.Any = fr()
 		}
 
-		return fc.Any.addRule(f, groups, host, cidr, localCidr)
+		return fc.Any.addRule(f, groups, host, ip, localIp)
 	}
 
 	if caSha != "" {
 		if _, ok := fc.CAShas[caSha]; !ok {
 			fc.CAShas[caSha] = fr()
 		}
-		err := fc.CAShas[caSha].addRule(f, groups, host, cidr, localCidr)
+		err := fc.CAShas[caSha].addRule(f, groups, host, ip, localIp)
 		if err != nil {
 			return err
 		}
@@ -708,7 +713,7 @@ func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, l
 		if _, ok := fc.CANames[caName]; !ok {
 			fc.CANames[caName] = fr()
 		}
-		err := fc.CANames[caName].addRule(f, groups, host, cidr, localCidr)
+		err := fc.CANames[caName].addRule(f, groups, host, ip, localIp)
 		if err != nil {
 			return err
 		}
@@ -740,24 +745,24 @@ func (fc *FirewallCA) match(p firewall.Packet, c *cert.CachedCertificate, caPool
 	return fc.CANames[s.Certificate.Name()].match(p, c)
 }
 
-func (fr *FirewallRule) addRule(f *Firewall, groups []string, host, cidr, localCidr string) error {
+func (fr *FirewallRule) addRule(f *Firewall, groups []string, host string, ip, localCIDR netip.Prefix) error {
 	flc := func() *firewallLocalCIDR {
 		return &firewallLocalCIDR{
 			LocalCIDR: new(bart.Lite),
 		}
 	}
 
-	if fr.isAny(groups, host, cidr) {
+	if fr.isAny(groups, host, ip) {
 		if fr.Any == nil {
 			fr.Any = flc()
 		}
 
-		return fr.Any.addRule(f, localCidr)
+		return fr.Any.addRule(f, localCIDR)
 	}
 
 	if len(groups) > 0 {
 		nlc := flc()
-		err := nlc.addRule(f, localCidr)
+		err := nlc.addRule(f, localCIDR)
 		if err != nil {
 			return err
 		}
@@ -773,34 +778,30 @@ func (fr *FirewallRule) addRule(f *Firewall, groups []string, host, cidr, localC
 		if nlc == nil {
 			nlc = flc()
 		}
-		err := nlc.addRule(f, localCidr)
+		err := nlc.addRule(f, localCIDR)
 		if err != nil {
 			return err
 		}
 		fr.Hosts[host] = nlc
 	}
 
-	if cidr != "" {
+	if ip.IsValid() {
-		c, err := netip.ParsePrefix(cidr)
+		nlc, _ := fr.CIDR.Get(ip)
-		if err != nil {
-			return err
-		}
-		nlc, _ := fr.CIDR.Get(c)
 		if nlc == nil {
 			nlc = flc()
 		}
-		err = nlc.addRule(f, localCidr)
+		err := nlc.addRule(f, localCIDR)
 		if err != nil {
 			return err
 		}
-		fr.CIDR.Insert(c, nlc)
+		fr.CIDR.Insert(ip, nlc)
 	}
 
 	return nil
 }
 
-func (fr *FirewallRule) isAny(groups []string, host string, cidr string) bool {
+func (fr *FirewallRule) isAny(groups []string, host string, ip netip.Prefix) bool {
-	if len(groups) == 0 && host == "" && cidr == "" {
+	if len(groups) == 0 && host == "" && !ip.IsValid() {
 		return true
 	}
 
@@ -814,7 +815,7 @@ func (fr *FirewallRule) isAny(groups []string, host string, cidr string) bool {
 		return true
 	}
 
-	if cidr == "any" {
+	if ip.IsValid() && ip.Bits() == 0 {
 		return true
 	}
 
@@ -866,13 +867,8 @@ func (fr *FirewallRule) match(p firewall.Packet, c *cert.CachedCertificate) bool
 	return false
 }
 
-func (flc *firewallLocalCIDR) addRule(f *Firewall, localCidr string) error {
+func (flc *firewallLocalCIDR) addRule(f *Firewall, localIp netip.Prefix) error {
-	if localCidr == "any" {
+	if !localIp.IsValid() {
-		flc.Any = true
-		return nil
-	}
-
-	if localCidr == "" {
 		if !f.hasUnsafeNetworks || f.defaultLocalCIDRAny {
 			flc.Any = true
 			return nil
@@ -883,13 +879,12 @@ func (flc *firewallLocalCIDR) addRule(f *Firewall, localCidr string) error {
 		}
 		return nil
 
+	} else if localIp.Bits() == 0 {
+		flc.Any = true
+		return nil
 	}
 
-	c, err := netip.ParsePrefix(localCidr)
+	flc.LocalCIDR.Insert(localIp)
-	if err != nil {
-		return err
-	}
-	flc.LocalCIDR.Insert(c)
 	return nil
 }
 
@@ -910,6 +905,7 @@ type rule struct {
 	Code      string
 	Proto     string
 	Host      string
+	Group     string
 	Groups    []string
 	Cidr      string
 	LocalCidr string
@@ -951,8 +947,7 @@ func convertRule(l *logrus.Logger, p any, table string, i int) (rule, error) {
 		l.Warnf("%s rule #%v; group was an array with a single value, converting to simple value", table, i)
 		m["group"] = v[0]
 	}
-
+	r.Group = toString("group", m)
-	singleGroup := toString("group", m)
 
 	if rg, ok := m["groups"]; ok {
 		switch reflect.TypeOf(rg).Kind() {
@@ -969,60 +964,9 @@ func convertRule(l *logrus.Logger, p any, table string, i int) (rule, error) {
 		}
 	}
 
-	//flatten group vs groups
-	if singleGroup != "" {
-		// Check if we have both groups and group provided in the rule config
-		if len(r.Groups) > 0 {
-			return r, fmt.Errorf("only one of group or groups should be defined, both provided")
-		}
-		r.Groups = []string{singleGroup}
-	}
-
 	return r, nil
 }
 
-// sanity returns an error if the rule would be evaluated in a way that would short-circuit a configured check on a wildcard value
-// rules are evaluated as "port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND local_cidr"
-func (r *rule) sanity() error {
-	//port, proto, local_cidr are AND, no need to check here
-	//ca_sha and ca_name don't have a wildcard value, no need to check here
-	groupsEmpty := len(r.Groups) == 0
-	hostEmpty := r.Host == ""
-	cidrEmpty := r.Cidr == ""
-
-	if (groupsEmpty && hostEmpty && cidrEmpty) == true {
-		return nil //no content!
-	}
-
-	groupsHasAny := slices.Contains(r.Groups, "any")
-	if groupsHasAny && len(r.Groups) > 1 {
-		return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the other groups specified", r.Groups)
-	}
-
-	if r.Host == "any" {
-		if !groupsEmpty {
-			return fmt.Errorf("groups specified as %s, but host=any will match any host, regardless of groups", r.Groups)
-		}
-
-		if !cidrEmpty {
-			return fmt.Errorf("cidr specified as %s, but host=any will match any host, regardless of cidr", r.Cidr)
-		}
-	}
-
-	if groupsHasAny {
-		if !hostEmpty && r.Host != "any" {
-			return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified host %s", r.Groups, r.Host)
-		}
-		if !cidrEmpty {
-			return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified cidr %s", r.Groups, r.Cidr)
-		}
-	}
-
-	//todo alert on cidr-any
-
-	return nil
-}
-
 func parsePort(s string) (startPort, endPort int32, err error) {
 	if s == "any" {
 		startPort = firewall.PortAny

firewall/cache.go

@@ -1,6 +1,7 @@
 package firewall
 
 import (
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -9,13 +10,58 @@ import (
 
 // ConntrackCache is used as a local routine cache to know if a given flow
 // has been seen in the conntrack table.
-type ConntrackCache map[Packet]struct{}
+type ConntrackCache struct {
+	mu      sync.Mutex
+	entries map[Packet]struct{}
+}
+
+func newConntrackCache() *ConntrackCache {
+	return &ConntrackCache{entries: make(map[Packet]struct{})}
+}
+
+func (c *ConntrackCache) Has(p Packet) bool {
+	if c == nil {
+		return false
+	}
+	c.mu.Lock()
+	_, ok := c.entries[p]
+	c.mu.Unlock()
+	return ok
+}
+
+func (c *ConntrackCache) Add(p Packet) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	c.entries[p] = struct{}{}
+	c.mu.Unlock()
+}
+
+func (c *ConntrackCache) Len() int {
+	if c == nil {
+		return 0
+	}
+	c.mu.Lock()
+	l := len(c.entries)
+	c.mu.Unlock()
+	return l
+}
+
+func (c *ConntrackCache) Reset(capHint int) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	c.entries = make(map[Packet]struct{}, capHint)
+	c.mu.Unlock()
+}
 
 type ConntrackCacheTicker struct {
 	cacheV    uint64
 	cacheTick atomic.Uint64
 
-	cache ConntrackCache
+	cache *ConntrackCache
 }
 
 func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
@@ -23,9 +69,7 @@ func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
 		return nil
 	}
 
-	c := &ConntrackCacheTicker{
+	c := &ConntrackCacheTicker{cache: newConntrackCache()}
-		cache: ConntrackCache{},
-	}
 
 	go c.tick(d)
 
@@ -41,17 +85,17 @@ func (c *ConntrackCacheTicker) tick(d time.Duration) {
 
 // Get checks if the cache ticker has moved to the next version before returning
 // the map. If it has moved, we reset the map.
-func (c *ConntrackCacheTicker) Get(l *logrus.Logger) ConntrackCache {
+func (c *ConntrackCacheTicker) Get(l *logrus.Logger) *ConntrackCache {
 	if c == nil {
 		return nil
 	}
 	if tick := c.cacheTick.Load(); tick != c.cacheV {
 		c.cacheV = tick
-		if ll := len(c.cache); ll > 0 {
+		if ll := c.cache.Len(); ll > 0 {
 			if l.Level == logrus.DebugLevel {
 				l.WithField("len", ll).Debug("resetting conntrack cache")
 			}
-			c.cache = make(ConntrackCache, ll)
+			c.cache.Reset(ll)
 		}
 	}
 

firewall_test.go

@@ -8,8 +8,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gaissmai/bart"
-	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
@@ -73,114 +71,85 @@ func TestFirewall_AddRule(t *testing.T) {
 	ti6, err := netip.ParsePrefix("fd12::34/128")
 	require.NoError(t, err)
 
-	require.NoError(t, fw.AddRule(true, firewall.ProtoTCP, 1, 1, []string{}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoTCP, 1, 1, []string{}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
 	// An empty rule is any
 	assert.True(t, fw.InRules.TCP[1].Any.Any.Any)
 	assert.Empty(t, fw.InRules.TCP[1].Any.Groups)
 	assert.Empty(t, fw.InRules.TCP[1].Any.Hosts)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
 	assert.Nil(t, fw.InRules.UDP[1].Any.Any)
 	assert.Contains(t, fw.InRules.UDP[1].Any.Groups[0].Groups, "g1")
 	assert.Empty(t, fw.InRules.UDP[1].Any.Hosts)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoICMP, 1, 1, []string{}, "h1", "", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoICMP, 1, 1, []string{}, "h1", netip.Prefix{}, netip.Prefix{}, "", ""))
 	assert.Nil(t, fw.InRules.ICMP[1].Any.Any)
 	assert.Empty(t, fw.InRules.ICMP[1].Any.Groups)
 	assert.Contains(t, fw.InRules.ICMP[1].Any.Hosts, "h1")
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", ti.String(), "", "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", ti, netip.Prefix{}, "", ""))
 	assert.Nil(t, fw.OutRules.AnyProto[1].Any.Any)
 	_, ok := fw.OutRules.AnyProto[1].Any.CIDR.Get(ti)
 	assert.True(t, ok)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", ti6.String(), "", "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", ti6, netip.Prefix{}, "", ""))
 	assert.Nil(t, fw.OutRules.AnyProto[1].Any.Any)
 	_, ok = fw.OutRules.AnyProto[1].Any.CIDR.Get(ti6)
 	assert.True(t, ok)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", "", ti.String(), "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", netip.Prefix{}, ti, "", ""))
 	assert.NotNil(t, fw.OutRules.AnyProto[1].Any.Any)
-	ok = fw.OutRules.AnyProto[1].Any.Any.LocalCIDR.Get(ti)
+	_, ok = fw.OutRules.AnyProto[1].Any.Any.LocalCIDR.Get(ti)
 	assert.True(t, ok)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", "", ti6.String(), "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", netip.Prefix{}, ti6, "", ""))
 	assert.NotNil(t, fw.OutRules.AnyProto[1].Any.Any)
-	ok = fw.OutRules.AnyProto[1].Any.Any.LocalCIDR.Get(ti6)
+	_, ok = fw.OutRules.AnyProto[1].Any.Any.LocalCIDR.Get(ti6)
 	assert.True(t, ok)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", "", "", "ca-name", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", netip.Prefix{}, netip.Prefix{}, "ca-name", ""))
 	assert.Contains(t, fw.InRules.UDP[1].CANames, "ca-name")
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", "", "", "", "ca-sha"))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", netip.Prefix{}, netip.Prefix{}, "", "ca-sha"))
 	assert.Contains(t, fw.InRules.UDP[1].CAShas, "ca-sha")
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "any", "", "", "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "any", netip.Prefix{}, netip.Prefix{}, "", ""))
 	assert.True(t, fw.OutRules.AnyProto[0].Any.Any.Any)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	anyIp, err := netip.ParsePrefix("0.0.0.0/0")
 	require.NoError(t, err)
 
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", anyIp.String(), "", "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", anyIp, netip.Prefix{}, "", ""))
-	assert.Nil(t, fw.OutRules.AnyProto[0].Any.Any)
+	assert.True(t, fw.OutRules.AnyProto[0].Any.Any.Any)
-	table, ok := fw.OutRules.AnyProto[0].Any.CIDR.Lookup(netip.MustParseAddr("1.1.1.1"))
-	assert.True(t, table.Any)
-	table, ok = fw.OutRules.AnyProto[0].Any.CIDR.Lookup(netip.MustParseAddr("9::9"))
-	assert.False(t, ok)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	anyIp6, err := netip.ParsePrefix("::/0")
 	require.NoError(t, err)
 
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", anyIp6.String(), "", "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", anyIp6, netip.Prefix{}, "", ""))
-	assert.Nil(t, fw.OutRules.AnyProto[0].Any.Any)
-	table, ok = fw.OutRules.AnyProto[0].Any.CIDR.Lookup(netip.MustParseAddr("9::9"))
-	assert.True(t, table.Any)
-	table, ok = fw.OutRules.AnyProto[0].Any.CIDR.Lookup(netip.MustParseAddr("1.1.1.1"))
-	assert.False(t, ok)
-
-	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", "any", "", "", ""))
-	assert.True(t, fw.OutRules.AnyProto[0].Any.Any.Any)
-
-	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", "", anyIp.String(), "", ""))
-	assert.False(t, fw.OutRules.AnyProto[0].Any.Any.Any)
-	assert.True(t, fw.OutRules.AnyProto[0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("1.1.1.1")))
-	assert.False(t, fw.OutRules.AnyProto[0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("9::9")))
-
-	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", "", anyIp6.String(), "", ""))
-	assert.False(t, fw.OutRules.AnyProto[0].Any.Any.Any)
-	assert.True(t, fw.OutRules.AnyProto[0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("9::9")))
-	assert.False(t, fw.OutRules.AnyProto[0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("1.1.1.1")))
-
-	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", "", "any", "", ""))
 	assert.True(t, fw.OutRules.AnyProto[0].Any.Any.Any)
 
 	// Test error conditions
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	require.Error(t, fw.AddRule(true, math.MaxUint8, 0, 0, []string{}, "", "", "", "", ""))
+	require.Error(t, fw.AddRule(true, math.MaxUint8, 0, 0, []string{}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
-	require.Error(t, fw.AddRule(true, firewall.ProtoAny, 10, 0, []string{}, "", "", "", "", ""))
+	require.Error(t, fw.AddRule(true, firewall.ProtoAny, 10, 0, []string{}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
 }
 
 func TestFirewall_Drop(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
+
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("1.2.3.4"),
 		RemoteAddr: netip.MustParseAddr("1.2.3.4"),
@@ -205,10 +174,10 @@ func TestFirewall_Drop(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
 	}
-	h.buildNetworks(myVpnNetworksTable, &c)
+	h.buildNetworks(c.networks, c.unsafeNetworks)
 
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
 	cp := cert.NewCAPool()
 
 	// Drop outbound
@@ -227,28 +196,28 @@ func TestFirewall_Drop(t *testing.T) {
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", netip.Prefix{}, netip.Prefix{}, "", "signer-shasum"))
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", netip.Prefix{}, netip.Prefix{}, "", "signer-shasum-bad"))
 	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", netip.Prefix{}, netip.Prefix{}, "", "signer-shasum-bad"))
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", netip.Prefix{}, netip.Prefix{}, "", "signer-shasum"))
 	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", netip.Prefix{}, netip.Prefix{}, "ca-good", ""))
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", netip.Prefix{}, netip.Prefix{}, "ca-good-bad", ""))
 	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", netip.Prefix{}, netip.Prefix{}, "ca-good-bad", ""))
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", netip.Prefix{}, netip.Prefix{}, "ca-good", ""))
 	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
 }
 
@@ -257,9 +226,6 @@ func TestFirewall_DropV6(t *testing.T) {
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
 
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("fd00::/7"))
-
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("fd12::34"),
 		RemoteAddr: netip.MustParseAddr("fd12::34"),
@@ -284,10 +250,10 @@ func TestFirewall_DropV6(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{netip.MustParseAddr("fd12::34")},
 	}
-	h.buildNetworks(myVpnNetworksTable, &c)
+	h.buildNetworks(c.networks, c.unsafeNetworks)
 
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
 	cp := cert.NewCAPool()
 
 	// Drop outbound
@@ -306,28 +272,28 @@ func TestFirewall_DropV6(t *testing.T) {
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", netip.Prefix{}, netip.Prefix{}, "", "signer-shasum"))
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", netip.Prefix{}, netip.Prefix{}, "", "signer-shasum-bad"))
 	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", netip.Prefix{}, netip.Prefix{}, "", "signer-shasum-bad"))
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", netip.Prefix{}, netip.Prefix{}, "", "signer-shasum"))
 	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", netip.Prefix{}, netip.Prefix{}, "ca-good", ""))
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", netip.Prefix{}, netip.Prefix{}, "ca-good-bad", ""))
 	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", netip.Prefix{}, netip.Prefix{}, "ca-good-bad", ""))
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", netip.Prefix{}, netip.Prefix{}, "ca-good", ""))
 	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
 }
 
@@ -338,12 +304,12 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 	}
 
 	pfix := netip.MustParsePrefix("172.1.1.1/32")
-	_ = ft.TCP.addRule(f, 10, 10, []string{"good-group"}, "good-host", pfix.String(), "", "", "")
+	_ = ft.TCP.addRule(f, 10, 10, []string{"good-group"}, "good-host", pfix, netip.Prefix{}, "", "")
-	_ = ft.TCP.addRule(f, 100, 100, []string{"good-group"}, "good-host", "", pfix.String(), "", "")
+	_ = ft.TCP.addRule(f, 100, 100, []string{"good-group"}, "good-host", netip.Prefix{}, pfix, "", "")
 
 	pfix6 := netip.MustParsePrefix("fd11::11/128")
-	_ = ft.TCP.addRule(f, 10, 10, []string{"good-group"}, "good-host", pfix6.String(), "", "", "")
+	_ = ft.TCP.addRule(f, 10, 10, []string{"good-group"}, "good-host", pfix6, netip.Prefix{}, "", "")
-	_ = ft.TCP.addRule(f, 100, 100, []string{"good-group"}, "good-host", "", pfix6.String(), "", "")
+	_ = ft.TCP.addRule(f, 100, 100, []string{"good-group"}, "good-host", netip.Prefix{}, pfix6, "", "")
 	cp := cert.NewCAPool()
 
 	b.Run("fail on proto", func(b *testing.B) {
@@ -487,8 +453,6 @@ func TestFirewall_Drop2(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("1.2.3.4"),
@@ -514,7 +478,7 @@ func TestFirewall_Drop2(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h.buildNetworks(myVpnNetworksTable, c.Certificate)
+	h.buildNetworks(c.Certificate.Networks(), c.Certificate.UnsafeNetworks())
 
 	c1 := cert.CachedCertificate{
 		Certificate: &dummyCert{
@@ -529,10 +493,10 @@ func TestFirewall_Drop2(t *testing.T) {
 			peerCert: &c1,
 		},
 	}
-	h1.buildNetworks(myVpnNetworksTable, c1.Certificate)
+	h1.buildNetworks(c1.Certificate.Networks(), c1.Certificate.UnsafeNetworks())
 
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group", "test-group"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group", "test-group"}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
 	cp := cert.NewCAPool()
 
 	// h1/c1 lacks the proper groups
@@ -546,8 +510,6 @@ func TestFirewall_Drop3(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("1.2.3.4"),
@@ -579,7 +541,7 @@ func TestFirewall_Drop3(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h1.buildNetworks(myVpnNetworksTable, c1.Certificate)
+	h1.buildNetworks(c1.Certificate.Networks(), c1.Certificate.UnsafeNetworks())
 
 	c2 := cert.CachedCertificate{
 		Certificate: &dummyCert{
@@ -594,7 +556,7 @@ func TestFirewall_Drop3(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h2.buildNetworks(myVpnNetworksTable, c2.Certificate)
+	h2.buildNetworks(c2.Certificate.Networks(), c2.Certificate.UnsafeNetworks())
 
 	c3 := cert.CachedCertificate{
 		Certificate: &dummyCert{
@@ -609,11 +571,11 @@ func TestFirewall_Drop3(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h3.buildNetworks(myVpnNetworksTable, c3.Certificate)
+	h3.buildNetworks(c3.Certificate.Networks(), c3.Certificate.UnsafeNetworks())
 
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "host1", "", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "host1", netip.Prefix{}, netip.Prefix{}, "", ""))
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "", "", "", "signer-sha"))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", netip.Prefix{}, netip.Prefix{}, "", "signer-sha"))
 	cp := cert.NewCAPool()
 
 	// c1 should pass because host match
@@ -627,7 +589,7 @@ func TestFirewall_Drop3(t *testing.T) {
 
 	// Test a remote address match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "1.2.3.4/24", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", netip.MustParsePrefix("1.2.3.4/24"), netip.Prefix{}, "", ""))
 	require.NoError(t, fw.Drop(p, true, &h1, cp, nil))
 }
 
@@ -635,8 +597,6 @@ func TestFirewall_Drop3V6(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("fd00::/7"))
 
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("fd12::34"),
@@ -660,12 +620,12 @@ func TestFirewall_Drop3V6(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h.buildNetworks(myVpnNetworksTable, c.Certificate)
+	h.buildNetworks(c.Certificate.Networks(), c.Certificate.UnsafeNetworks())
 
 	// Test a remote address match
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	cp := cert.NewCAPool()
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "fd12::34/120", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", netip.MustParsePrefix("fd12::34/120"), netip.Prefix{}, "", ""))
 	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
 }
 
@@ -673,8 +633,6 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
 
 	p := firewall.Packet{
 		LocalAddr:  netip.MustParseAddr("1.2.3.4"),
@@ -701,10 +659,10 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{network.Addr()},
 	}
-	h.buildNetworks(myVpnNetworksTable, c.Certificate)
+	h.buildNetworks(c.Certificate.Networks(), c.Certificate.UnsafeNetworks())
 
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
 	cp := cert.NewCAPool()
 
 	// Drop outbound
@@ -717,7 +675,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 
 	oldFw := fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 10, 10, []string{"any"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 10, 10, []string{"any"}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
 	fw.Conntrack = oldFw.Conntrack
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
@@ -726,7 +684,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 
 	oldFw = fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 11, 11, []string{"any"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 11, 11, []string{"any"}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
 	fw.Conntrack = oldFw.Conntrack
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
@@ -738,8 +696,6 @@ func TestFirewall_DropIPSpoofing(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}
 	l.SetOutput(ob)
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
 
 	c := cert.CachedCertificate{
 		Certificate: &dummyCert{
@@ -761,11 +717,11 @@ func TestFirewall_DropIPSpoofing(t *testing.T) {
 		},
 		vpnAddrs: []netip.Addr{c1.Certificate.Networks()[0].Addr()},
 	}
-	h1.buildNetworks(myVpnNetworksTable, c1.Certificate)
+	h1.buildNetworks(c1.Certificate.Networks(), c1.Certificate.UnsafeNetworks())
 
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", netip.Prefix{}, netip.Prefix{}, "", ""))
 	cp := cert.NewCAPool()
 
 	// Packet spoofed by `c1`. Note that the remote addr is not a valid one.
@@ -959,28 +915,28 @@ func TestAddFirewallRulesFromConfig(t *testing.T) {
 	mf := &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "tcp", "host": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoTCP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoTCP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: netip.Prefix{}, localIp: netip.Prefix{}}, mf.lastCall)
 
 	// Test adding udp rule
 	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "udp", "host": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoUDP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoUDP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: netip.Prefix{}, localIp: netip.Prefix{}}, mf.lastCall)
 
 	// Test adding icmp rule
 	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "icmp", "host": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoICMP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoICMP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: netip.Prefix{}, localIp: netip.Prefix{}}, mf.lastCall)
 
 	// Test adding any rule
 	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "host": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, host: "a", ip: netip.Prefix{}, localIp: netip.Prefix{}}, mf.lastCall)
 
 	// Test adding rule with cidr
 	cidr := netip.MustParsePrefix("10.0.0.0/8")
@@ -988,14 +944,14 @@ func TestAddFirewallRulesFromConfig(t *testing.T) {
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr.String()}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr.String(), localIp: ""}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr, localIp: netip.Prefix{}}, mf.lastCall)
 
 	// Test adding rule with local_cidr
 	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr.String()}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: cidr.String()}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: netip.Prefix{}, localIp: cidr}, mf.lastCall)
 
 	// Test adding rule with cidr ipv6
 	cidr6 := netip.MustParsePrefix("fd00::/8")
@@ -1003,75 +959,49 @@ func TestAddFirewallRulesFromConfig(t *testing.T) {
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr6.String()}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr6.String(), localIp: ""}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr6, localIp: netip.Prefix{}}, mf.lastCall)
-
-	// Test adding rule with any cidr
-	conf = config.NewC(l)
-	mf = &mockFirewall{}
-	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "any"}}}
-	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "any", localIp: ""}, mf.lastCall)
-
-	// Test adding rule with junk cidr
-	conf = config.NewC(l)
-	mf = &mockFirewall{}
-	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "junk/junk"}}}
-	require.EqualError(t, AddFirewallRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP")
 
 	// Test adding rule with local_cidr ipv6
 	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr6.String()}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: cidr6.String()}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: netip.Prefix{}, localIp: cidr6}, mf.lastCall)
-
-	// Test adding rule with any local_cidr
-	conf = config.NewC(l)
-	mf = &mockFirewall{}
-	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "any"}}}
-	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, localIp: "any"}, mf.lastCall)
-
-	// Test adding rule with junk local_cidr
-	conf = config.NewC(l)
-	mf = &mockFirewall{}
-	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "junk/junk"}}}
-	require.EqualError(t, AddFirewallRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; local_cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP")
 
 	// Test adding rule with ca_sha
 	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_sha": "12312313123"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caSha: "12312313123"}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: netip.Prefix{}, localIp: netip.Prefix{}, caSha: "12312313123"}, mf.lastCall)
 
 	// Test adding rule with ca_name
 	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_name": "root01"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caName: "root01"}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: netip.Prefix{}, localIp: netip.Prefix{}, caName: "root01"}, mf.lastCall)
 
 	// Test single group
 	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "group": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: netip.Prefix{}, localIp: netip.Prefix{}}, mf.lastCall)
 
 	// Test single groups
 	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: netip.Prefix{}, localIp: netip.Prefix{}}, mf.lastCall)
 
 	// Test multiple AND groups
 	conf = config.NewC(l)
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": []string{"a", "b"}}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a", "b"}, ip: "", localIp: ""}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a", "b"}, ip: netip.Prefix{}, localIp: netip.Prefix{}}, mf.lastCall)
 
 	// Test Add error
 	conf = config.NewC(l)
@@ -1094,7 +1024,7 @@ func TestFirewall_convertRule(t *testing.T) {
 	r, err := convertRule(l, c, "test", 1)
 	assert.Contains(t, ob.String(), "test rule #1; group was an array with a single value, converting to simple value")
 	require.NoError(t, err)
-	assert.Equal(t, []string{"group1"}, r.Groups)
+	assert.Equal(t, "group1", r.Group)
 
 	// Ensure group array of > 1 is errord
 	ob.Reset()
@@ -1114,228 +1044,7 @@ func TestFirewall_convertRule(t *testing.T) {
 
 	r, err = convertRule(l, c, "test", 1)
 	require.NoError(t, err)
-	assert.Equal(t, []string{"group1"}, r.Groups)
+	assert.Equal(t, "group1", r.Group)
-}
-
-func TestFirewall_convertRuleSanity(t *testing.T) {
-	l := test.NewLogger()
-	ob := &bytes.Buffer{}
-	l.SetOutput(ob)
-
-	noWarningPlease := []map[string]any{
-		{"group": "group1"},
-		{"groups": []any{"group2"}},
-		{"host": "bob"},
-		{"cidr": "1.1.1.1/1"},
-		{"groups": []any{"group2"}, "host": "bob"},
-		{"cidr": "1.1.1.1/1", "host": "bob"},
-		{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
-		{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
-	}
-	for _, c := range noWarningPlease {
-		r, err := convertRule(l, c, "test", 1)
-		require.NoError(t, err)
-		require.NoError(t, r.sanity(), "should not generate a sanity warning, %+v", c)
-	}
-
-	yesWarningPlease := []map[string]any{
-		{"group": "group1"},
-		{"groups": []any{"group2"}},
-		{"cidr": "1.1.1.1/1"},
-		{"groups": []any{"group2"}, "host": "bob"},
-		{"cidr": "1.1.1.1/1", "host": "bob"},
-		{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
-		{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
-	}
-	for _, c := range yesWarningPlease {
-		c["host"] = "any"
-		r, err := convertRule(l, c, "test", 1)
-		require.NoError(t, err)
-		err = r.sanity()
-		require.Error(t, err, "I wanted a warning: %+v", c)
-	}
-	//reset the list
-	yesWarningPlease = []map[string]any{
-		{"group": "group1"},
-		{"groups": []any{"group2"}},
-		{"cidr": "1.1.1.1/1"},
-		{"groups": []any{"group2"}, "host": "bob"},
-		{"cidr": "1.1.1.1/1", "host": "bob"},
-		{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
-		{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
-	}
-	for _, c := range yesWarningPlease {
-		r, err := convertRule(l, c, "test", 1)
-		require.NoError(t, err)
-		r.Groups = append(r.Groups, "any")
-		err = r.sanity()
-		require.Error(t, err, "I wanted a warning: %+v", c)
-	}
-}
-
-type testcase struct {
-	h   *HostInfo
-	p   firewall.Packet
-	c   cert.Certificate
-	err error
-}
-
-func (c *testcase) Test(t *testing.T, fw *Firewall) {
-	t.Helper()
-	cp := cert.NewCAPool()
-	resetConntrack(fw)
-	err := fw.Drop(c.p, true, c.h, cp, nil)
-	if c.err == nil {
-		require.NoError(t, err, "failed to not drop remote address %s", c.p.RemoteAddr)
-	} else {
-		require.ErrorIs(t, c.err, err, "failed to drop remote address %s", c.p.RemoteAddr)
-	}
-}
-
-func buildTestCase(setup testsetup, err error, theirPrefixes ...netip.Prefix) testcase {
-	c1 := dummyCert{
-		name:     "host1",
-		networks: theirPrefixes,
-		groups:   []string{"default-group"},
-		issuer:   "signer-shasum",
-	}
-	h := HostInfo{
-		ConnectionState: &ConnectionState{
-			peerCert: &cert.CachedCertificate{
-				Certificate:    &c1,
-				InvertedGroups: map[string]struct{}{"default-group": {}},
-			},
-		},
-		vpnAddrs: make([]netip.Addr, len(theirPrefixes)),
-	}
-	for i := range theirPrefixes {
-		h.vpnAddrs[i] = theirPrefixes[i].Addr()
-	}
-	h.buildNetworks(setup.myVpnNetworksTable, &c1)
-	p := firewall.Packet{
-		LocalAddr:  setup.c.Networks()[0].Addr(), //todo?
-		RemoteAddr: theirPrefixes[0].Addr(),
-		LocalPort:  10,
-		RemotePort: 90,
-		Protocol:   firewall.ProtoUDP,
-		Fragment:   false,
-	}
-	return testcase{
-		h:   &h,
-		p:   p,
-		c:   &c1,
-		err: err,
-	}
-}
-
-type testsetup struct {
-	c                  dummyCert
-	myVpnNetworksTable *bart.Lite
-	fw                 *Firewall
-}
-
-func newSetup(t *testing.T, l *logrus.Logger, myPrefixes ...netip.Prefix) testsetup {
-	c := dummyCert{
-		name:     "me",
-		networks: myPrefixes,
-		groups:   []string{"default-group"},
-		issuer:   "signer-shasum",
-	}
-
-	return newSetupFromCert(t, l, c)
-}
-
-func newSetupFromCert(t *testing.T, l *logrus.Logger, c dummyCert) testsetup {
-	myVpnNetworksTable := new(bart.Lite)
-	for _, prefix := range c.Networks() {
-		myVpnNetworksTable.Insert(prefix)
-	}
-	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
-
-	return testsetup{
-		c:                  c,
-		fw:                 fw,
-		myVpnNetworksTable: myVpnNetworksTable,
-	}
-}
-
-func TestFirewall_Drop_EnforceIPMatch(t *testing.T) {
-	t.Parallel()
-	l := test.NewLogger()
-	ob := &bytes.Buffer{}
-	l.SetOutput(ob)
-
-	myPrefix := netip.MustParsePrefix("1.1.1.1/8")
-	// for now, it's okay that these are all "incoming", the logic this test tries to check doesn't care about in/out
-	t.Run("allow inbound all matching", func(t *testing.T) {
-		t.Parallel()
-		setup := newSetup(t, l, myPrefix)
-		tc := buildTestCase(setup, nil, netip.MustParsePrefix("1.2.3.4/24"))
-		tc.Test(t, setup.fw)
-	})
-	t.Run("allow inbound local matching", func(t *testing.T) {
-		t.Parallel()
-		setup := newSetup(t, l, myPrefix)
-		tc := buildTestCase(setup, ErrInvalidLocalIP, netip.MustParsePrefix("1.2.3.4/24"))
-		tc.p.LocalAddr = netip.MustParseAddr("1.2.3.8")
-		tc.Test(t, setup.fw)
-	})
-	t.Run("block inbound remote mismatched", func(t *testing.T) {
-		t.Parallel()
-		setup := newSetup(t, l, myPrefix)
-		tc := buildTestCase(setup, ErrInvalidRemoteIP, netip.MustParsePrefix("1.2.3.4/24"))
-		tc.p.RemoteAddr = netip.MustParseAddr("9.9.9.9")
-		tc.Test(t, setup.fw)
-	})
-	t.Run("Block a vpn peer packet", func(t *testing.T) {
-		t.Parallel()
-		setup := newSetup(t, l, myPrefix)
-		tc := buildTestCase(setup, ErrPeerRejected, netip.MustParsePrefix("2.2.2.2/24"))
-		tc.Test(t, setup.fw)
-	})
-	twoPrefixes := []netip.Prefix{
-		netip.MustParsePrefix("1.2.3.4/24"), netip.MustParsePrefix("2.2.2.2/24"),
-	}
-	t.Run("allow inbound one matching", func(t *testing.T) {
-		t.Parallel()
-		setup := newSetup(t, l, myPrefix)
-		tc := buildTestCase(setup, nil, twoPrefixes...)
-		tc.Test(t, setup.fw)
-	})
-	t.Run("block inbound multimismatch", func(t *testing.T) {
-		t.Parallel()
-		setup := newSetup(t, l, myPrefix)
-		tc := buildTestCase(setup, ErrInvalidRemoteIP, twoPrefixes...)
-		tc.p.RemoteAddr = netip.MustParseAddr("9.9.9.9")
-		tc.Test(t, setup.fw)
-	})
-	t.Run("allow inbound 2nd one matching", func(t *testing.T) {
-		t.Parallel()
-		setup2 := newSetup(t, l, netip.MustParsePrefix("2.2.2.1/24"))
-		tc := buildTestCase(setup2, nil, twoPrefixes...)
-		tc.p.RemoteAddr = twoPrefixes[1].Addr()
-		tc.Test(t, setup2.fw)
-	})
-	t.Run("allow inbound unsafe route", func(t *testing.T) {
-		t.Parallel()
-		unsafePrefix := netip.MustParsePrefix("192.168.0.0/24")
-		c := dummyCert{
-			name:           "me",
-			networks:       []netip.Prefix{myPrefix},
-			unsafeNetworks: []netip.Prefix{unsafePrefix},
-			groups:         []string{"default-group"},
-			issuer:         "signer-shasum",
-		}
-		unsafeSetup := newSetupFromCert(t, l, c)
-		tc := buildTestCase(unsafeSetup, nil, twoPrefixes...)
-		tc.p.LocalAddr = netip.MustParseAddr("192.168.0.3")
-		tc.err = ErrNoMatchingRule
-		tc.Test(t, unsafeSetup.fw) //should hit firewall and bounce off
-		require.NoError(t, unsafeSetup.fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", unsafePrefix.String(), "", ""))
-		tc.err = nil
-		tc.Test(t, unsafeSetup.fw) //should pass
-	})
 }
 
 type addRuleCall struct {
@@ -1345,8 +1054,8 @@ type addRuleCall struct {
 	endPort   int32
 	groups    []string
 	host      string
-	ip        string
+	ip        netip.Prefix
-	localIp   string
+	localIp   netip.Prefix
 	caName    string
 	caSha     string
 }
@@ -1356,7 +1065,7 @@ type mockFirewall struct {
 	nextCallReturn error
 }
 
-func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip, localIp, caName string, caSha string) error {
+func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip netip.Prefix, localIp netip.Prefix, caName string, caSha string) error {
 	mf.lastCall = addRuleCall{
 		incoming:  incoming,
 		proto:     proto,

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/armon/go-radix v1.0.0
 	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
 	github.com/flynn/noise v1.1.0
-	github.com/gaissmai/bart v0.26.0
+	github.com/gaissmai/bart v0.25.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gopacket v1.1.19
 	github.com/kardianos/service v1.2.4
@@ -22,17 +22,16 @@ require (
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
 	github.com/stretchr/testify v1.11.1
 	github.com/vishvananda/netlink v1.3.1
-	go.yaml.in/yaml/v3 v3.0.4
+	golang.org/x/crypto v0.43.0
-	golang.org/x/crypto v0.45.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.47.0
+	golang.org/x/net v0.45.0
-	golang.org/x/sync v0.19.0
+	golang.org/x/sync v0.17.0
-	golang.org/x/sys v0.39.0
+	golang.org/x/sys v0.37.0
-	golang.org/x/term v0.38.0
+	golang.org/x/term v0.36.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
 	golang.zx2c4.com/wireguard/windows v0.5.3
-	google.golang.org/protobuf v1.36.10
+	google.golang.org/protobuf v1.36.8
 	gopkg.in/yaml.v3 v3.0.1
 	gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
 )
@@ -50,6 +49,6 @@ require (
 	github.com/vishvananda/netns v0.0.5 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/mod v0.24.0 // indirect
-	golang.org/x/time v0.7.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.33.0 // indirect
 )

go.sum

@@ -24,8 +24,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
 github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
-github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0=
+github.com/gaissmai/bart v0.25.0 h1:eqiokVPqM3F94vJ0bTHXHtH91S8zkKL+bKh+BsGOsJM=
-github.com/gaissmai/bart v0.26.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
+github.com/gaissmai/bart v0.25.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -155,15 +155,13 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
-go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
-go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
+golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
@@ -182,8 +180,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -191,8 +189,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -209,16 +207,16 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
-golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
+golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
-golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
+golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -244,8 +242,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
+google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
-google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

handshake_ix.go

@@ -2,6 +2,7 @@ package nebula
 
 import (
 	"net/netip"
+	"slices"
 	"time"
 
 	"github.com/flynn/noise"
@@ -99,11 +100,11 @@ func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
 	return true
 }
 
-func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H) {
+func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet []byte, h *header.H) {
 	cs := f.pki.getCertState()
 	crt := cs.GetDefaultCertificate()
 	if crt == nil {
-		f.l.WithField("from", via).
+		f.l.WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).
 			WithField("certVersion", cs.initiatingVersion).
 			Error("Unable to handshake with host because no certificate is available")
@@ -112,7 +113,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	ci, err := NewConnectionState(f.l, cs, crt, false, noise.HandshakeIX)
 	if err != nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Error("Failed to create connection state")
 		return
@@ -123,7 +124,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	msg, _, _, err := ci.H.ReadMessage(nil, packet[header.Len:])
 	if err != nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Error("Failed to call noise.ReadMessage")
 		return
@@ -132,7 +133,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	hs := &NebulaHandshake{}
 	err = hs.Unmarshal(msg)
 	if err != nil || hs.Details == nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Error("Failed unmarshal handshake message")
 		return
@@ -140,7 +141,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	rc, err := cert.Recombine(cert.Version(hs.Details.CertVersion), hs.Details.Cert, ci.H.PeerStatic(), ci.Curve())
 	if err != nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Info("Handshake did not contain a certificate")
 		return
@@ -153,7 +154,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 			fp = "<error generating certificate fingerprint>"
 		}
 
-		e := f.l.WithError(err).WithField("from", via).
+		e := f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			WithField("certVpnNetworks", rc.Networks()).
 			WithField("certFingerprint", fp)
@@ -172,7 +173,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 		if myCertOtherVersion == nil {
 			if f.l.Level >= logrus.DebugLevel {
 				f.l.WithError(err).WithFields(m{
-					"from":      via,
+					"udpAddr":   addr,
 					"handshake": m{"stage": 1, "style": "ix_psk0"},
 					"cert":      remoteCert,
 				}).Debug("Might be unable to handshake with host due to missing certificate version")
@@ -184,24 +185,24 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	}
 
 	if len(remoteCert.Certificate.Networks()) == 0 {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("cert", remoteCert).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Info("No networks in certificate")
 		return
 	}
 
+	var vpnAddrs []netip.Addr
+	var filteredNetworks []netip.Prefix
 	certName := remoteCert.Certificate.Name()
 	certVersion := remoteCert.Certificate.Version()
 	fingerprint := remoteCert.Fingerprint
 	issuer := remoteCert.Certificate.Issuer()
-	vpnNetworks := remoteCert.Certificate.Networks()
 
-	anyVpnAddrsInCommon := false
+	for _, network := range remoteCert.Certificate.Networks() {
-	vpnAddrs := make([]netip.Addr, len(vpnNetworks))
+		vpnAddr := network.Addr()
-	for i, network := range vpnNetworks {
+		if f.myVpnAddrsTable.Contains(vpnAddr) {
-		if f.myVpnAddrsTable.Contains(network.Addr()) {
+			f.l.WithField("vpnAddr", vpnAddr).WithField("udpAddr", addr).
-			f.l.WithField("vpnNetworks", vpnNetworks).WithField("from", via).
 				WithField("certName", certName).
 				WithField("certVersion", certVersion).
 				WithField("fingerprint", fingerprint).
@@ -209,24 +210,38 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Refusing to handshake with myself")
 			return
 		}
-		vpnAddrs[i] = network.Addr()
+
-		if f.myVpnNetworksTable.Contains(network.Addr()) {
+		// vpnAddrs outside our vpn networks are of no use to us, filter them out
-			anyVpnAddrsInCommon = true
+		if !f.myVpnNetworksTable.Contains(vpnAddr) {
+			continue
 		}
+
+		filteredNetworks = append(filteredNetworks, network)
+		vpnAddrs = append(vpnAddrs, vpnAddr)
 	}
 
-	if !via.IsRelayed {
+	if len(vpnAddrs) == 0 {
+		f.l.WithError(err).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("No usable vpn addresses from host, refusing handshake")
+		return
+	}
+
+	if addr.IsValid() {
+		// addr can be invalid when the tunnel is being relayed.
 		// We only want to apply the remote allow list for direct tunnels here
-		if !f.lightHouse.GetRemoteAllowList().AllowAll(vpnAddrs, via.UdpAddr.Addr()) {
+		if !f.lightHouse.GetRemoteAllowList().AllowAll(vpnAddrs, addr.Addr()) {
-			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
-				Debug("lighthouse.remote_allow_list denied incoming handshake")
 			return
 		}
 	}
 
 	myIndex, err := generateIndex(f.l)
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
 			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
@@ -249,30 +264,26 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 		},
 	}
 
-	msgRxL := f.l.WithFields(m{
+	f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
-		"vpnAddrs":       vpnAddrs,
+		WithField("certName", certName).
-		"from":           via,
+		WithField("certVersion", certVersion).
-		"certName":       certName,
+		WithField("fingerprint", fingerprint).
-		"certVersion":    certVersion,
+		WithField("issuer", issuer).
-		"fingerprint":    fingerprint,
+		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-		"issuer":         issuer,
+		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-		"initiatorIndex": hs.Details.InitiatorIndex,
+		Info("Handshake message received")
-		"responderIndex": hs.Details.ResponderIndex,
-		"remoteIndex":    h.RemoteIndex,
-		"handshake":      m{"stage": 1, "style": "ix_psk0"},
-	})
-
-	if anyVpnAddrsInCommon {
-		msgRxL.Info("Handshake message received")
-	} else {
-		//todo warn if not lighthouse or relay?
-		msgRxL.Info("Handshake message received, but no vpnNetworks in common.")
-	}
 
 	hs.Details.ResponderIndex = myIndex
 	hs.Details.Cert = cs.getHandshakeBytes(ci.myCert.Version())
 	if hs.Details.Cert == nil {
-		msgRxL.WithField("myCertVersion", ci.myCert.Version()).
+		f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			WithField("certVersion", ci.myCert.Version()).
 			Error("Unable to handshake with host because no certificate handshake bytes is available")
 		return
 	}
@@ -283,7 +294,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	hsBytes, err := hs.Marshal()
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
 			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
@@ -295,7 +306,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	nh := header.Encode(make([]byte, header.Len), header.Version, header.Handshake, header.HandshakeIXPSK0, hs.Details.InitiatorIndex, 2)
 	msg, dKey, eKey, err := ci.H.WriteMessage(nh, hsBytes)
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
 			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
@@ -303,7 +314,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage")
 		return
 	} else if dKey == nil || eKey == nil {
-		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
 			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
@@ -329,17 +340,15 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	ci.eKey = NewNebulaCipherState(eKey)
 
 	hostinfo.remotes = f.lightHouse.QueryCache(vpnAddrs)
-	if !via.IsRelayed {
+	hostinfo.SetRemote(addr)
-		hostinfo.SetRemote(via.UdpAddr)
+	hostinfo.buildNetworks(filteredNetworks, remoteCert.Certificate.UnsafeNetworks())
-	}
-	hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
 
 	existing, err := f.handshakeManager.CheckAndComplete(hostinfo, 0, f)
 	if err != nil {
 		switch err {
 		case ErrAlreadySeen:
 			// Update remote if preferred
-			if existing.SetRemoteIfPreferred(f.hostMap, via) {
+			if existing.SetRemoteIfPreferred(f.hostMap, addr) {
 				// Send a test packet to ensure the other side has also switched to
 				// the preferred remote
 				f.SendMessageToVpnAddr(header.Test, header.TestRequest, vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
@@ -347,21 +356,21 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 			msg = existing.HandshakePacket[2]
 			f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
-			if !via.IsRelayed {
+			if addr.IsValid() {
-				err := f.outside.WriteTo(msg, via.UdpAddr)
+				err := f.outside.WriteTo(msg, addr)
 				if err != nil {
-					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("from", via).
+					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("udpAddr", addr).
 						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
 						WithError(err).Error("Failed to send handshake message")
 				} else {
-					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("from", via).
+					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("udpAddr", addr).
 						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
 						Info("Handshake message sent")
 				}
 				return
 			} else {
-				if via.relay == nil {
+				if via == nil {
-					f.l.Error("Handshake send failed: both addr and via.relay are nil.")
+					f.l.Error("Handshake send failed: both addr and via are nil.")
 					return
 				}
 				hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
@@ -373,7 +382,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 			}
 		case ErrExistingHostInfo:
 			// This means there was an existing tunnel and this handshake was older than the one we are currently based on
-			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 				WithField("certName", certName).
 				WithField("certVersion", certVersion).
 				WithField("oldHandshakeTime", existing.lastHandshakeTime).
@@ -389,7 +398,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 			return
 		case ErrLocalIndexCollision:
 			// This means we failed to insert because of collision on localIndexId. Just let the next handshake packet retry
-			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 				WithField("certName", certName).
 				WithField("certVersion", certVersion).
 				WithField("fingerprint", fingerprint).
@@ -402,7 +411,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 		default:
 			// Shouldn't happen, but just in case someone adds a new error type to CheckAndComplete
 			// And we forget to update it here
-			f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 				WithField("certName", certName).
 				WithField("certVersion", certVersion).
 				WithField("fingerprint", fingerprint).
@@ -416,23 +425,30 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	// Do the send
 	f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
-	if !via.IsRelayed {
+	if addr.IsValid() {
-		err = f.outside.WriteTo(msg, via.UdpAddr)
+		err = f.outside.WriteTo(msg, addr)
-		log := f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
-			WithField("certName", certName).
-			WithField("certVersion", certVersion).
-			WithField("fingerprint", fingerprint).
-			WithField("issuer", issuer).
-			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"})
 		if err != nil {
-			log.WithError(err).Error("Failed to send handshake")
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("certVersion", certVersion).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+				WithError(err).Error("Failed to send handshake")
 		} else {
-			log.Info("Handshake message sent")
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("certVersion", certVersion).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+				Info("Handshake message sent")
 		}
 	} else {
-		if via.relay == nil {
+		if via == nil {
-			f.l.Error("Handshake send failed: both addr and via.relay are nil.")
+			f.l.Error("Handshake send failed: both addr and via are nil.")
 			return
 		}
 		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
@@ -457,7 +473,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	return
 }
 
-func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packet []byte, h *header.H) bool {
+func ixHandshakeStage2(f *Interface, addr netip.AddrPort, via *ViaSender, hh *HandshakeHostInfo, packet []byte, h *header.H) bool {
 	if hh == nil {
 		// Nothing here to tear down, got a bogus stage 2 packet
 		return true
@@ -467,10 +483,10 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	defer hh.Unlock()
 
 	hostinfo := hh.hostinfo
-	if !via.IsRelayed {
+	if addr.IsValid() {
 		// The vpnAddr we know about is the one we tried to handshake with, use it to apply the remote allow list.
-		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
+		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, addr.Addr()) {
-			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).Debug("lighthouse.remote_allow_list denied incoming handshake")
+			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
 			return false
 		}
 	}
@@ -478,7 +494,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	ci := hostinfo.ConnectionState
 	msg, eKey, dKey, err := ci.H.ReadMessage(nil, packet[header.Len:])
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("header", h).
 			Error("Failed to call noise.ReadMessage")
 
@@ -487,7 +503,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		// near future
 		return false
 	} else if dKey == nil || eKey == nil {
-		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			Error("Noise did not arrive at a key")
 
@@ -499,7 +515,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	hs := &NebulaHandshake{}
 	err = hs.Unmarshal(msg)
 	if err != nil || hs.Details == nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).Error("Failed unmarshal handshake message")
 
 		// The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again
@@ -508,7 +524,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 
 	rc, err := cert.Recombine(cert.Version(hs.Details.CertVersion), hs.Details.Cert, ci.H.PeerStatic(), ci.Curve())
 	if err != nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("vpnAddrs", hostinfo.vpnAddrs).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			Info("Handshake did not contain a certificate")
@@ -522,7 +538,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 			fp = "<error generating certificate fingerprint>"
 		}
 
-		e := f.l.WithError(err).WithField("from", via).
+		e := f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("vpnAddrs", hostinfo.vpnAddrs).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			WithField("certFingerprint", fp).
@@ -537,7 +553,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	}
 
 	if len(remoteCert.Certificate.Networks()) == 0 {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("vpnAddrs", hostinfo.vpnAddrs).
 			WithField("cert", remoteCert).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
@@ -560,30 +576,39 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	ci.eKey = NewNebulaCipherState(eKey)
 
 	// Make sure the current udpAddr being used is set for responding
-	if !via.IsRelayed {
+	if addr.IsValid() {
-		hostinfo.SetRemote(via.UdpAddr)
+		hostinfo.SetRemote(addr)
 	} else {
 		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
 	}
 
-	correctHostResponded := false
+	var vpnAddrs []netip.Addr
-	anyVpnAddrsInCommon := false
+	var filteredNetworks []netip.Prefix
-	vpnAddrs := make([]netip.Addr, len(vpnNetworks))
+	for _, network := range vpnNetworks {
-	for i, network := range vpnNetworks {
+		// vpnAddrs outside our vpn networks are of no use to us, filter them out
-		vpnAddrs[i] = network.Addr()
+		vpnAddr := network.Addr()
-		if f.myVpnNetworksTable.Contains(network.Addr()) {
+		if !f.myVpnNetworksTable.Contains(vpnAddr) {
-			anyVpnAddrsInCommon = true
+			continue
-		}
-		if hostinfo.vpnAddrs[0] == network.Addr() {
-			// todo is it more correct to see if any of hostinfo.vpnAddrs are in the cert? it should have len==1, but one day it might not?
-			correctHostResponded = true
 		}
+
+		filteredNetworks = append(filteredNetworks, network)
+		vpnAddrs = append(vpnAddrs, vpnAddr)
+	}
+
+	if len(vpnAddrs) == 0 {
+		f.l.WithError(err).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).Error("No usable vpn addresses from host, refusing handshake")
+		return true
 	}
 
 	// Ensure the right host responded
-	if !correctHostResponded {
+	if !slices.Contains(vpnAddrs, hostinfo.vpnAddrs[0]) {
 		f.l.WithField("intendedVpnAddrs", hostinfo.vpnAddrs).WithField("haveVpnNetworks", vpnNetworks).
-			WithField("from", via).
+			WithField("udpAddr", addr).
 			WithField("certName", certName).
 			WithField("certVersion", certVersion).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
@@ -593,11 +618,10 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		f.handshakeManager.DeleteHostInfo(hostinfo)
 
 		// Create a new hostinfo/handshake for the intended vpn ip
-		//TODO is hostinfo.vpnAddrs[0] always the address to use?
 		f.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) {
 			// Block the current used address
 			newHH.hostinfo.remotes = hostinfo.remotes
-			newHH.hostinfo.remotes.BlockRemote(via)
+			newHH.hostinfo.remotes.BlockRemote(addr)
 
 			f.l.WithField("blockedUdpAddrs", newHH.hostinfo.remotes.CopyBlockedRemotes()).
 				WithField("vpnNetworks", vpnNetworks).
@@ -620,7 +644,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	ci.window.Update(f.l, 2)
 
 	duration := time.Since(hh.startTime).Nanoseconds()
-	msgRxL := f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+	f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 		WithField("certName", certName).
 		WithField("certVersion", certVersion).
 		WithField("fingerprint", fingerprint).
@@ -628,17 +652,12 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
 		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 		WithField("durationNs", duration).
-		WithField("sentCachedPackets", len(hh.packetStore))
+		WithField("sentCachedPackets", len(hh.packetStore)).
-	if anyVpnAddrsInCommon {
+		Info("Handshake message received")
-		msgRxL.Info("Handshake message received")
-	} else {
-		//todo warn if not lighthouse or relay?
-		msgRxL.Info("Handshake message received, but no vpnNetworks in common.")
-	}
 
 	// Build up the radix for the firewall if we have subnets in the cert
 	hostinfo.vpnAddrs = vpnAddrs
-	hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
+	hostinfo.buildNetworks(filteredNetworks, remoteCert.Certificate.UnsafeNetworks())
 
 	// Complete our handshake and update metrics, this will replace any existing tunnels for the vpnAddrs here
 	f.handshakeManager.Complete(hostinfo, f)

handshake_manager.go

@@ -136,11 +136,11 @@ func (hm *HandshakeManager) Run(ctx context.Context) {
 	}
 }
 
-func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *header.H) {
+func (hm *HandshakeManager) HandleIncoming(addr netip.AddrPort, via *ViaSender, packet []byte, h *header.H) {
 	// First remote allow list check before we know the vpnIp
-	if !via.IsRelayed {
+	if addr.IsValid() {
-		if !hm.lightHouse.GetRemoteAllowList().AllowUnknownVpnAddr(via.UdpAddr.Addr()) {
+		if !hm.lightHouse.GetRemoteAllowList().AllowUnknownVpnAddr(addr.Addr()) {
-			hm.l.WithField("from", via).Debug("lighthouse.remote_allow_list denied incoming handshake")
+			hm.l.WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
 			return
 		}
 	}
@@ -149,11 +149,11 @@ func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *head
 	case header.HandshakeIXPSK0:
 		switch h.MessageCounter {
 		case 1:
-			ixHandshakeStage1(hm.f, via, packet, h)
+			ixHandshakeStage1(hm.f, addr, via, packet, h)
 
 		case 2:
 			newHostinfo := hm.queryIndex(h.RemoteIndex)
-			tearDown := ixHandshakeStage2(hm.f, via, newHostinfo, packet, h)
+			tearDown := ixHandshakeStage2(hm.f, addr, via, newHostinfo, packet, h)
 			if tearDown && newHostinfo != nil {
 				hm.DeleteHostInfo(newHostinfo.hostinfo)
 			}
@@ -269,12 +269,12 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 		hostinfo.logger(hm.l).WithField("relays", hostinfo.remotes.relays).Info("Attempt to relay through hosts")
 		// Send a RelayRequest to all known Relay IP's
 		for _, relay := range hostinfo.remotes.relays {
-			// Don't relay through the host I'm trying to connect to
+			// Don't relay to myself
 			if relay == vpnIp {
 				continue
 			}
 
-			// Don't relay to myself
+			// Don't relay through the host I'm trying to connect to
 			if hm.f.myVpnAddrsTable.Contains(relay) {
 				continue
 			}

hostmap.go

@@ -1,9 +1,7 @@
 package nebula
 
 import (
-	"encoding/json"
 	"errors"
-	"fmt"
 	"net"
 	"net/netip"
 	"slices"
@@ -214,18 +212,6 @@ func (rs *RelayState) InsertRelay(ip netip.Addr, idx uint32, r *Relay) {
 	rs.relayForByIdx[idx] = r
 }
 
-type NetworkType uint8
-
-const (
-	NetworkTypeUnknown NetworkType = iota
-	// NetworkTypeVPN is a network that overlaps one or more of the vpnNetworks in our certificate
-	NetworkTypeVPN
-	// NetworkTypeVPNPeer is a network that does not overlap one of our networks
-	NetworkTypeVPNPeer
-	// NetworkTypeUnsafe is a network from Certificate.UnsafeNetworks()
-	NetworkTypeUnsafe
-)
-
 type HostInfo struct {
 	remote          netip.AddrPort
 	remotes         *RemoteList
@@ -239,8 +225,8 @@ type HostInfo struct {
 	// vpn networks but were removed because they are not usable
 	vpnAddrs []netip.Addr
 
-	// networks is a combination of specific vpn addresses (not prefixes!) and full unsafe networks assigned to this host.
+	// networks are both all vpn and unsafe networks assigned to this host
-	networks   *bart.Table[NetworkType]
+	networks   *bart.Lite
 	relayState RelayState
 
 	// HandshakePacket records the packets used to create this hostinfo
@@ -278,25 +264,9 @@ type HostInfo struct {
 }
 
 type ViaSender struct {
-	UdpAddr   netip.AddrPort
 	relayHI   *HostInfo // relayHI is the host info object of the relay
 	remoteIdx uint32    // remoteIdx is the index included in the header of the received packet
 	relay     *Relay    // relay contains the rest of the relay information, including the PeerIP of the host trying to communicate with us.
-	IsRelayed bool      // IsRelayed is true if the packet was sent through a relay
-}
-
-func (v ViaSender) String() string {
-	if v.IsRelayed {
-		return fmt.Sprintf("%s (relayed)", v.UdpAddr)
-	}
-	return v.UdpAddr.String()
-}
-
-func (v ViaSender) MarshalJSON() ([]byte, error) {
-	if v.IsRelayed {
-		return json.Marshal(m{"relay": v.UdpAddr})
-	}
-	return json.Marshal(m{"direct": v.UdpAddr})
 }
 
 type cachedPacket struct {
@@ -712,7 +682,6 @@ func (i *HostInfo) GetCert() *cert.CachedCertificate {
 	return nil
 }
 
-// TODO: Maybe use ViaSender here?
 func (i *HostInfo) SetRemote(remote netip.AddrPort) {
 	// We copy here because we likely got this remote from a source that reuses the object
 	if i.remote != remote {
@@ -723,14 +692,14 @@ func (i *HostInfo) SetRemote(remote netip.AddrPort) {
 
 // SetRemoteIfPreferred returns true if the remote was changed. The lastRoam
 // time on the HostInfo will also be updated.
-func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
+func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, newRemote netip.AddrPort) bool {
-	if via.IsRelayed {
+	if !newRemote.IsValid() {
+		// relays have nil udp Addrs
 		return false
 	}
-
 	currentRemote := i.remote
 	if !currentRemote.IsValid() {
-		i.SetRemote(via.UdpAddr)
+		i.SetRemote(newRemote)
 		return true
 	}
 
@@ -743,7 +712,7 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
 			return false
 		}
 
-		if l.Contains(via.UdpAddr.Addr()) {
+		if l.Contains(newRemote.Addr()) {
 			newIsPreferred = true
 		}
 	}
@@ -753,7 +722,7 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
 		i.lastRoam = time.Now()
 		i.lastRoamRemote = currentRemote
 
-		i.SetRemote(via.UdpAddr)
+		i.SetRemote(newRemote)
 
 		return true
 	}
@@ -761,26 +730,20 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
 	return false
 }
 
-// buildNetworks fills in the networks field of HostInfo. It accepts a cert.Certificate so you never ever mix the network types up.
+func (i *HostInfo) buildNetworks(networks, unsafeNetworks []netip.Prefix) {
-func (i *HostInfo) buildNetworks(myVpnNetworksTable *bart.Lite, c cert.Certificate) {
+	if len(networks) == 1 && len(unsafeNetworks) == 0 {
-	if len(c.Networks()) == 1 && len(c.UnsafeNetworks()) == 0 {
+		// Simple case, no CIDRTree needed
-		if myVpnNetworksTable.Contains(c.Networks()[0].Addr()) {
+		return
-			return // Simple case, no BART needed
-		}
 	}
 
-	i.networks = new(bart.Table[NetworkType])
+	i.networks = new(bart.Lite)
-	for _, network := range c.Networks() {
+	for _, network := range networks {
 		nprefix := netip.PrefixFrom(network.Addr(), network.Addr().BitLen())
-		if myVpnNetworksTable.Contains(network.Addr()) {
+		i.networks.Insert(nprefix)
-			i.networks.Insert(nprefix, NetworkTypeVPN)
-		} else {
-			i.networks.Insert(nprefix, NetworkTypeVPNPeer)
-		}
 	}
 
-	for _, network := range c.UnsafeNetworks() {
+	for _, network := range unsafeNetworks {
-		i.networks.Insert(network, NetworkTypeUnsafe)
+		i.networks.Insert(network)
 	}
 }
 

inside.go

@@ -2,18 +2,18 @@ package nebula
 
 import (
 	"net/netip"
-	"time"
+	"unsafe"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
 	"github.com/slackhq/nebula/noiseutil"
-	"github.com/slackhq/nebula/packet"
+	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/routing"
 )
 
-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb []byte, out *packet.UDPPacket, q int, localCache firewall.ConntrackCache, now time.Time) {
+func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache *firewall.ConntrackCache) {
 	err := newPacket(packet, false, fwPacket)
 	if err != nil {
 		if f.l.Level >= logrus.DebugLevel {
@@ -55,7 +55,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 	})
 
 	if hostinfo == nil {
-		f.rejectInside(packet, out.Payload, q) //todo vector?
+		f.rejectInside(packet, out, q)
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("vpnAddr", fwPacket.RemoteAddr).
 				WithField("fwPacket", fwPacket).
@@ -68,11 +68,12 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache, now)
+	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
-		f.sendNoMetricsDelayed(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
+		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
+
 	} else {
-		f.rejectInside(packet, out.Payload, q) //todo vector?
+		f.rejectInside(packet, out, q)
 		if f.l.Level >= logrus.DebugLevel {
 			hostinfo.logger(f.l).
 				WithField("fwPacket", fwPacket).
@@ -121,10 +122,9 @@ func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *
 	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q)
 }
 
-// Handshake will attempt to initiate a tunnel with the provided vpn address. This is a no-op if the tunnel is already established or being established
+// Handshake will attempt to initiate a tunnel with the provided vpn address if it is within our vpn networks. This is a no-op if the tunnel is already established or being established
-// it does not check if it is within our vpn networks!
 func (f *Interface) Handshake(vpnAddr netip.Addr) {
-	f.handshakeManager.GetOrHandshake(vpnAddr, nil)
+	f.getOrHandshakeNoRouting(vpnAddr, nil)
 }
 
 // getOrHandshakeNoRouting returns nil if the vpnAddr is not routable.
@@ -140,6 +140,7 @@ func (f *Interface) getOrHandshakeNoRouting(vpnAddr netip.Addr, cacheCallback fu
 // getOrHandshakeConsiderRouting will try to find the HostInfo to handle this packet, starting a handshake if necessary.
 // If the 2nd return var is false then the hostinfo is not ready to be used in a tunnel.
 func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cacheCallback func(*HandshakeHostInfo)) (*HostInfo, bool) {
+
 	destinationAddr := fwPacket.RemoteAddr
 
 	hostinfo, ready := f.getOrHandshakeNoRouting(destinationAddr, cacheCallback)
@@ -219,7 +220,7 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
 	}
 
 	// check if packet is in outbound fw rules
-	dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil, time.Now())
+	dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil)
 	if dropReason != nil {
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("fwPacket", fp).
@@ -232,10 +233,9 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
 	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, 0)
 }
 
-// SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr.
+// SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr
-// This function ignores myVpnNetworksTable, and will always attempt to treat the address as a vpnAddr
 func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte) {
-	hostInfo, ready := f.handshakeManager.GetOrHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
+	hostInfo, ready := f.getOrHandshakeNoRouting(vpnAddr, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, t, st, p, f.SendMessageToHostInfo, f.cachedPacketMetrics)
 	})
 
@@ -337,9 +337,21 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 	if ci.eKey == nil {
 		return
 	}
-	useRelay := !remote.IsValid() && !hostinfo.remote.IsValid()
+	target := remote
+	if !target.IsValid() {
+		target = hostinfo.remote
+	}
+	useRelay := !target.IsValid()
 	fullOut := out
 
+	var pkt *overlay.Packet
+	if !useRelay && f.batches.Enabled() {
+		pkt = f.batches.newPacket()
+		if pkt != nil {
+			out = pkt.Payload()[:0]
+		}
+	}
+
 	if useRelay {
 		if len(out) < header.Len {
 			// out always has a capacity of mtu, but not always a length greater than the header.Len.
@@ -373,119 +385,85 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 	}
 
 	var err error
+	if len(p) > 0 && slicesOverlap(out, p) {
+		tmp := make([]byte, len(p))
+		copy(tmp, p)
+		p = tmp
+	}
 	out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
 	if noiseutil.EncryptLockNeeded {
 		ci.writeLock.Unlock()
 	}
 	if err != nil {
+		if pkt != nil {
+			pkt.Release()
+		}
 		hostinfo.logger(f.l).WithError(err).
-			WithField("udpAddr", remote).WithField("counter", c).
+			WithField("udpAddr", target).WithField("counter", c).
 			WithField("attemptedCounter", c).
 			Error("Failed to encrypt outgoing packet")
 		return
 	}
 
-	if remote.IsValid() {
+	if target.IsValid() {
-		err = f.writers[q].WriteTo(out, remote)
+		if pkt != nil {
-		if err != nil {
+			pkt.Len = len(out)
-			hostinfo.logger(f.l).WithError(err).
+			if f.l.Level >= logrus.DebugLevel {
-				WithField("udpAddr", remote).Error("Failed to write outgoing packet")
+				f.l.WithFields(logrus.Fields{
-		}
+					"queue":        q,
-	} else if hostinfo.remote.IsValid() {
+					"dest":         target,
-		err = f.writers[q].WriteTo(out, hostinfo.remote)
+					"payload_len":  pkt.Len,
-		if err != nil {
+					"use_batches":  true,
-			hostinfo.logger(f.l).WithError(err).
+					"remote_index": hostinfo.remoteIndexId,
-				WithField("udpAddr", remote).Error("Failed to write outgoing packet")
+				}).Debug("enqueueing packet to UDP batch queue")
-		}
-	} else {
-		// Try to send via a relay
-		for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
-			relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
-			if err != nil {
-				hostinfo.relayState.DeleteRelay(relayIP)
-				hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetrics failed to find HostInfo")
-				continue
 			}
-			f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
+			if f.tryQueuePacket(q, pkt, target) {
-			break
+				return
+			}
+			if f.l.Level >= logrus.DebugLevel {
+				f.l.WithFields(logrus.Fields{
+					"queue": q,
+					"dest":  target,
+				}).Debug("failed to enqueue packet; falling back to immediate send")
+			}
+			f.writeImmediatePacket(q, pkt, target, hostinfo)
+			return
 		}
-	}
+		if f.tryQueueDatagram(q, out, target) {
-}
+			return
-
-func (f *Interface) sendNoMetricsDelayed(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb []byte, out *packet.UDPPacket, q int) {
-	if ci.eKey == nil {
-		return
-	}
-	useRelay := !remote.IsValid() && !hostinfo.remote.IsValid()
-	fullOut := out.Payload
-
-	if useRelay {
-		if len(out.Payload) < header.Len {
-			// out always has a capacity of mtu, but not always a length greater than the header.Len.
-			// Grow it to make sure the next operation works.
-			out.Payload = out.Payload[:header.Len]
 		}
-		// Save a header's worth of data at the front of the 'out' buffer.
+		f.writeImmediate(q, out, target, hostinfo)
-		out.Payload = out.Payload[header.Len:]
-	}
-
-	if noiseutil.EncryptLockNeeded {
-		// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
-		ci.writeLock.Lock()
-	}
-	c := ci.messageCounter.Add(1)
-
-	//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
-	out.Payload = header.Encode(out.Payload, header.Version, t, st, hostinfo.remoteIndexId, c)
-	f.connectionManager.Out(hostinfo)
-
-	// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against
-	// all our addrs and enable a faster roaming.
-	if t != header.CloseTunnel && hostinfo.lastRebindCount != f.rebindCount {
-		//NOTE: there is an update hole if a tunnel isn't used and exactly 256 rebinds occur before the tunnel is
-		// finally used again. This tunnel would eventually be torn down and recreated if this action didn't help.
-		f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
-		hostinfo.lastRebindCount = f.rebindCount
-		if f.l.Level >= logrus.DebugLevel {
-			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).Debug("Lighthouse update triggered for punch due to rebind counter")
-		}
-	}
-
-	var err error
-	out.Payload, err = ci.eKey.EncryptDanger(out.Payload, out.Payload, p, c, nb)
-	if noiseutil.EncryptLockNeeded {
-		ci.writeLock.Unlock()
-	}
-	if err != nil {
-		hostinfo.logger(f.l).WithError(err).
-			WithField("udpAddr", remote).WithField("counter", c).
-			WithField("attemptedCounter", c).
-			Error("Failed to encrypt outgoing packet")
 		return
 	}
 
-	if remote.IsValid() {
+	// fall back to relay path
-		err = f.writers[q].Prep(out, remote)
+	if pkt != nil {
+		pkt.Release()
+	}
+
+	// Try to send via a relay
+	for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
+		relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", remote).Error("Failed to write outgoing packet")
+			hostinfo.relayState.DeleteRelay(relayIP)
-		}
+			hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetrics failed to find HostInfo")
-	} else if hostinfo.remote.IsValid() {
+			continue
-		err = f.writers[q].Prep(out, hostinfo.remote)
-		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", remote).Error("Failed to write outgoing packet")
-		}
-	} else {
-		// Try to send via a relay
-		for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
-			relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
-			if err != nil {
-				hostinfo.relayState.DeleteRelay(relayIP)
-				hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetrics failed to find HostInfo")
-				continue
-			}
-			//todo vector!!
-			f.SendVia(relayHostInfo, relay, out.Payload, nb, fullOut[:header.Len+len(out.Payload)], true)
-			break
 		}
+		f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
+		break
 	}
 }
+
+// slicesOverlap reports whether the two byte slices share any portion of memory.
+// cipher.AEAD.Seal requires plaintext and dst to live in disjoint regions.
+func slicesOverlap(a, b []byte) bool {
+	if len(a) == 0 || len(b) == 0 {
+		return false
+	}
+
+	aStart := uintptr(unsafe.Pointer(&a[0]))
+	aEnd := aStart + uintptr(len(a))
+	bStart := uintptr(unsafe.Pointer(&b[0]))
+	bEnd := bStart + uintptr(len(b))
+	return aStart < bEnd && bStart < aEnd
+}

interface.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net/netip"
 	"os"
 	"runtime"
+	"strings"
 	"sync/atomic"
 	"time"
 
@@ -17,12 +19,16 @@ import (
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/udp"
 )
 
-const mtu = 9001
+const (
-const batch = 1024 //todo config!
+	mtu                          = 9001
+	defaultGSOFlushInterval      = 150 * time.Microsecond
+	defaultBatchQueueDepthFactor = 4
+	defaultGSOMaxSegments        = 8
+	maxKernelGSOSegments         = 64
+)
 
 type InterfaceConfig struct {
 	HostMap            *HostMap
@@ -37,6 +43,9 @@ type InterfaceConfig struct {
 	connectionManager  *connectionManager
 	DropLocalBroadcast bool
 	DropMulticast      bool
+	EnableGSO          bool
+	EnableGRO          bool
+	GSOMaxSegments     int
 	routines           int
 	MessageMetrics     *MessageMetrics
 	version            string
@@ -48,6 +57,8 @@ type InterfaceConfig struct {
 	reQueryWait     time.Duration
 
 	ConntrackCacheTimeout time.Duration
+	BatchFlushInterval    time.Duration
+	BatchQueueDepth       int
 	l                     *logrus.Logger
 }
 
@@ -85,20 +96,25 @@ type Interface struct {
 	version     string
 
 	conntrackCacheTimeout time.Duration
+	batchQueueDepth       int
+	enableGSO             bool
+	enableGRO             bool
+	gsoMaxSegments        int
+	batchUDPQueueGauge    metrics.Gauge
+	batchUDPFlushCounter  metrics.Counter
+	batchTunQueueGauge    metrics.Gauge
+	batchTunFlushCounter  metrics.Counter
+	batchFlushInterval    atomic.Int64
+	sendSem               chan struct{}
 
 	writers []udp.Conn
-	readers []overlay.TunDev
+	readers []io.ReadWriteCloser
+	batches batchPipelines
 
 	metricHandshakes    metrics.Histogram
 	messageMetrics      *MessageMetrics
 	cachedPacketMetrics *cachedPacketMetrics
 
-	listenInN  int
-	listenOutN int
-
-	listenInMetric  metrics.Histogram
-	listenOutMetric metrics.Histogram
-
 	l *logrus.Logger
 }
 
@@ -168,6 +184,22 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		return nil, errors.New("no connection manager")
 	}
 
+	if c.GSOMaxSegments <= 0 {
+		c.GSOMaxSegments = defaultGSOMaxSegments
+	}
+	if c.GSOMaxSegments > maxKernelGSOSegments {
+		c.GSOMaxSegments = maxKernelGSOSegments
+	}
+	if c.BatchQueueDepth <= 0 {
+		c.BatchQueueDepth = c.routines * defaultBatchQueueDepthFactor
+	}
+	if c.BatchFlushInterval < 0 {
+		c.BatchFlushInterval = 0
+	}
+	if c.BatchFlushInterval == 0 && c.EnableGSO {
+		c.BatchFlushInterval = defaultGSOFlushInterval
+	}
+
 	cs := c.pki.getCertState()
 	ifce := &Interface{
 		pki:                   c.pki,
@@ -184,7 +216,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		routines:              c.routines,
 		version:               c.version,
 		writers:               make([]udp.Conn, c.routines),
-		readers:               make([]overlay.TunDev, c.routines),
+		readers:               make([]io.ReadWriteCloser, c.routines),
 		myVpnNetworks:         cs.myVpnNetworks,
 		myVpnNetworksTable:    cs.myVpnNetworksTable,
 		myVpnAddrs:            cs.myVpnAddrs,
@@ -193,6 +225,10 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		relayManager:          c.relayManager,
 		connectionManager:     c.connectionManager,
 		conntrackCacheTimeout: c.ConntrackCacheTimeout,
+		batchQueueDepth:       c.BatchQueueDepth,
+		enableGSO:             c.EnableGSO,
+		enableGRO:             c.EnableGRO,
+		gsoMaxSegments:        c.GSOMaxSegments,
 
 		metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
 		messageMetrics:   c.MessageMetrics,
@@ -203,12 +239,27 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 
 		l: c.l,
 	}
-	ifce.listenInMetric = metrics.GetOrRegisterHistogram("vhost.listenIn.n", nil, metrics.NewExpDecaySample(1028, 0.015))
-	ifce.listenOutMetric = metrics.GetOrRegisterHistogram("vhost.listenOut.n", nil, metrics.NewExpDecaySample(1028, 0.015))
 
 	ifce.tryPromoteEvery.Store(c.tryPromoteEvery)
+	ifce.batchUDPQueueGauge = metrics.GetOrRegisterGauge("batch.udp.queue_depth", nil)
+	ifce.batchUDPFlushCounter = metrics.GetOrRegisterCounter("batch.udp.flushes", nil)
+	ifce.batchTunQueueGauge = metrics.GetOrRegisterGauge("batch.tun.queue_depth", nil)
+	ifce.batchTunFlushCounter = metrics.GetOrRegisterCounter("batch.tun.flushes", nil)
+	ifce.batchFlushInterval.Store(int64(c.BatchFlushInterval))
+	ifce.sendSem = make(chan struct{}, c.routines)
+	ifce.batches.init(c.Inside, c.routines, c.BatchQueueDepth, c.GSOMaxSegments)
 	ifce.reQueryEvery.Store(c.reQueryEvery)
 	ifce.reQueryWait.Store(int64(c.reQueryWait))
+	if c.l.Level >= logrus.DebugLevel {
+		c.l.WithFields(logrus.Fields{
+			"enableGSO":       c.EnableGSO,
+			"enableGRO":       c.EnableGRO,
+			"gsoMaxSegments":  c.GSOMaxSegments,
+			"batchQueueDepth": c.BatchQueueDepth,
+			"batchFlush":      c.BatchFlushInterval,
+			"batching":        ifce.batches.Enabled(),
+		}).Debug("initialized batch pipelines")
+	}
 
 	ifce.connectionManager.intf = ifce
 
@@ -231,17 +282,10 @@ func (f *Interface) activate() {
 		WithField("boringcrypto", boringEnabled()).
 		Info("Nebula interface is active")
 
-	if f.routines > 1 {
-		if !f.inside.SupportsMultiqueue() || !f.outside.SupportsMultipleReaders() {
-			f.routines = 1
-			f.l.Warn("routines is not supported on this platform, falling back to a single routine")
-		}
-	}
-
 	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
 
 	// Prepare n tun queues
-	var reader overlay.TunDev = f.inside
+	var reader io.ReadWriteCloser = f.inside
 	for i := 0; i < f.routines; i++ {
 		if i > 0 {
 			reader, err = f.inside.NewMultiQueueReader()
@@ -264,87 +308,69 @@ func (f *Interface) run() {
 		go f.listenOut(i)
 	}
 
+	if f.l.Level >= logrus.DebugLevel {
+		f.l.WithField("batching", f.batches.Enabled()).Debug("starting interface run loops")
+	}
+
+	if f.batches.Enabled() {
+		for i := 0; i < f.routines; i++ {
+			go f.runInsideBatchWorker(i)
+			go f.runTunWriteQueue(i)
+			go f.runSendQueue(i)
+		}
+	}
+
 	// Launch n queues to read packets from tun dev
 	for i := 0; i < f.routines; i++ {
 		go f.listenIn(f.readers[i], i)
 	}
 }
 
-type Scratches struct {
+func (f *Interface) listenOut(i int) {
-	h        *header.H
-	nb       []byte
-	fwPacket *firewall.Packet
-	scratch  []byte
-}
-
-func NewScratches() *Scratches {
-	return &Scratches{
-		h:        &header.H{},
-		fwPacket: &firewall.Packet{},
-		nb:       make([]byte, 12),
-		scratch:  make([]byte, udp.MTU),
-	}
-}
-
-func (f *Interface) listenOut(q int) {
 	runtime.LockOSThread()
 
 	var li udp.Conn
-	if q > 0 {
+	if i > 0 {
-		li = f.writers[q]
+		li = f.writers[i]
 	} else {
 		li = f.outside
 	}
 
 	ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
 	lhh := f.lightHouse.NewRequestHandler()
+	plaintext := make([]byte, udp.MTU)
+	h := &header.H{}
+	fwPacket := &firewall.Packet{}
+	nb := make([]byte, 12, 12)
 
-	outPackets := make([]*packet.OutPacket, batch)
+	li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
-	for i := 0; i < batch; i++ {
+		f.readOutsidePackets(fromUdpAddr, nil, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get(f.l))
-		outPackets[i] = packet.NewOut()
-	}
-
-	scratches := NewScratches()
-
-	toSend := make([][]byte, batch)
-
-	li.ListenOut(func(pkts []*packet.UDPPacket) {
-		toSend = toSend[:0]
-
-		f.readOutsidePacketsMany(pkts, outPackets, lhh, scratches, q, ctCache.Get(f.l), time.Now())
-		//we opportunistically tx, but try to also send stragglers
-		if _, err := f.readers[q].WriteMany(outPackets, q); err != nil {
-			f.l.WithError(err).Error("Failed to send packets")
-		}
-		//todo I broke this
-		//n := len(toSend)
-		//if f.l.Level == logrus.DebugLevel {
-		//	f.listenOutMetric.Update(int64(n))
-		//}
-		//f.listenOutN = n
-
 	})
 }
 
-func (f *Interface) listenIn(reader overlay.TunDev, queueNum int) {
+func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
 	runtime.LockOSThread()
 
+	if f.batches.Enabled() {
+		if br, ok := reader.(overlay.BatchReader); ok {
+			f.listenInBatchLocked(reader, br, i)
+			return
+		}
+	}
+
+	f.listenInLegacyLocked(reader, i)
+}
+
+func (f *Interface) listenInLegacyLocked(reader io.ReadWriteCloser, i int) {
+	packet := make([]byte, mtu)
+	out := make([]byte, mtu)
 	fwPacket := &firewall.Packet{}
 	nb := make([]byte, 12, 12)
 
 	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
 
-	packets := reader.NewPacketArrays(batch)
-
-	outPackets := make([]*packet.UDPPacket, batch)
-	for i := 0; i < batch; i++ {
-		outPackets[i] = packet.New(false) //todo isv4?
-	}
-
 	for {
-		n, err := reader.ReadMany(packets, queueNum)
+		n, err := reader.Read(packet)
-
-		//todo!!
 		if err != nil {
 			if errors.Is(err, os.ErrClosed) && f.closed.Load() {
 				return
@@ -355,20 +381,581 @@ func (f *Interface) listenIn(reader overlay.TunDev, queueNum int) {
 			os.Exit(2)
 		}
 
-		if f.l.Level == logrus.DebugLevel {
+		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get(f.l))
-			f.listenInMetric.Update(int64(n))
+	}
-		}
+}
-		f.listenInN = n
 
-		now := time.Now()
+func (f *Interface) listenInBatchLocked(raw io.ReadWriteCloser, reader overlay.BatchReader, i int) {
-		for i, pkt := range packets[:n] {
+	pool := f.batches.Pool()
-			outPackets[i].ReadyToSend = false
+	if pool == nil {
-			f.consumeInsidePacket(pkt.GetPayload(), fwPacket, nb, outPackets[i], queueNum, conntrackCache.Get(f.l), now)
+		f.l.Warn("batch pipeline enabled without an allocated pool; falling back to single-packet reads")
-			reader.RecycleRxSeg(pkt, i == (n-1), queueNum) //todo handle err?
+		f.listenInLegacyLocked(raw, i)
-		}
+		return
-		_, err = f.writers[queueNum].WriteBatch(outPackets[:n])
+	}
+
+	for {
+		packets, err := reader.ReadIntoBatch(pool)
 		if err != nil {
-			f.l.WithError(err).Error("Error while writing outbound packets")
+			if errors.Is(err, os.ErrClosed) && f.closed.Load() {
+				return
+			}
+
+			if isVirtioHeadroomError(err) {
+				f.l.WithError(err).Warn("Batch reader fell back due to tun headroom issue")
+				f.listenInLegacyLocked(raw, i)
+				return
+			}
+
+			f.l.WithError(err).Error("Error while reading outbound packet batch")
+			os.Exit(2)
+		}
+
+		if len(packets) == 0 {
+			continue
+		}
+
+		for _, pkt := range packets {
+			if pkt == nil {
+				continue
+			}
+			if !f.batches.enqueueRx(i, pkt) {
+				pkt.Release()
+			}
+		}
+	}
+}
+
+func (f *Interface) runInsideBatchWorker(i int) {
+	queue := f.batches.rxQueue(i)
+	if queue == nil {
+		return
+	}
+
+	out := make([]byte, mtu)
+	fwPacket := &firewall.Packet{}
+	nb := make([]byte, 12, 12)
+	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+
+	for pkt := range queue {
+		if pkt == nil {
+			continue
+		}
+		f.consumeInsidePacket(pkt.Payload(), fwPacket, nb, out, i, conntrackCache.Get(f.l))
+		pkt.Release()
+	}
+}
+
+func (f *Interface) runSendQueue(i int) {
+	queue := f.batches.txQueue(i)
+	if queue == nil {
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("queue", i).Debug("tx queue not initialized; batching disabled for writer")
+		}
+		return
+	}
+	writer := f.writerForIndex(i)
+	if writer == nil {
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithField("queue", i).Debug("no UDP writer for batch queue")
+		}
+		return
+	}
+	if f.l.Level >= logrus.DebugLevel {
+		f.l.WithField("queue", i).Debug("send queue worker started")
+	}
+	defer func() {
+		if f.l.Level >= logrus.WarnLevel {
+			f.l.WithField("queue", i).Warn("send queue worker exited")
+		}
+	}()
+
+	batchCap := f.batches.batchSizeHint()
+	if batchCap <= 0 {
+		batchCap = 1
+	}
+	gsoLimit := f.effectiveGSOMaxSegments()
+	if gsoLimit > batchCap {
+		batchCap = gsoLimit
+	}
+	pending := make([]queuedDatagram, 0, batchCap)
+	var (
+		flushTimer *time.Timer
+		flushC     <-chan time.Time
+	)
+	dispatch := func(reason string, timerFired bool) {
+		if len(pending) == 0 {
+			return
+		}
+		batch := pending
+		f.flushAndReleaseBatch(i, writer, batch, reason)
+		for idx := range batch {
+			batch[idx] = queuedDatagram{}
+		}
+		pending = pending[:0]
+		if flushTimer != nil {
+			if !timerFired {
+				if !flushTimer.Stop() {
+					select {
+					case <-flushTimer.C:
+					default:
+					}
+				}
+			}
+			flushTimer = nil
+			flushC = nil
+		}
+	}
+	armTimer := func() {
+		delay := f.currentBatchFlushInterval()
+		if delay <= 0 {
+			dispatch("nogso", false)
+			return
+		}
+		if flushTimer == nil {
+			flushTimer = time.NewTimer(delay)
+			flushC = flushTimer.C
+		}
+	}
+
+	for {
+		select {
+		case d := <-queue:
+			if d.packet == nil {
+				continue
+			}
+			if f.l.Level >= logrus.DebugLevel {
+				f.l.WithFields(logrus.Fields{
+					"queue":       i,
+					"payload_len": d.packet.Len,
+					"dest":        d.addr,
+				}).Debug("send queue received packet")
+			}
+			pending = append(pending, d)
+			if gsoLimit > 0 && len(pending) >= gsoLimit {
+				dispatch("gso", false)
+				continue
+			}
+			if len(pending) >= cap(pending) {
+				dispatch("cap", false)
+				continue
+			}
+			armTimer()
+			f.observeUDPQueueLen(i)
+		case <-flushC:
+			dispatch("timer", true)
+		}
+	}
+}
+
+func (f *Interface) runTunWriteQueue(i int) {
+	queue := f.batches.tunQueue(i)
+	if queue == nil {
+		return
+	}
+	writer := f.batches.inside
+	if writer == nil {
+		return
+	}
+	requiredHeadroom := writer.BatchHeadroom()
+
+	batchCap := f.batches.batchSizeHint()
+	if batchCap <= 0 {
+		batchCap = 1
+	}
+	pending := make([]*overlay.Packet, 0, batchCap)
+	var (
+		flushTimer *time.Timer
+		flushC     <-chan time.Time
+	)
+	flush := func(reason string, timerFired bool) {
+		if len(pending) == 0 {
+			return
+		}
+		valid := pending[:0]
+		for idx := range pending {
+			if !f.ensurePacketHeadroom(&pending[idx], requiredHeadroom, i, reason) {
+				pending[idx] = nil
+				continue
+			}
+			if pending[idx] != nil {
+				valid = append(valid, pending[idx])
+			}
+		}
+		if len(valid) > 0 {
+			if _, err := writer.WriteBatch(valid); err != nil {
+				f.l.WithError(err).
+					WithField("queue", i).
+					WithField("reason", reason).
+					Warn("Failed to write tun batch")
+				for _, pkt := range valid {
+					if pkt != nil {
+						f.writePacketToTun(i, pkt)
+					}
+				}
+			}
+		}
+		pending = pending[:0]
+		if flushTimer != nil {
+			if !timerFired {
+				if !flushTimer.Stop() {
+					select {
+					case <-flushTimer.C:
+					default:
+					}
+				}
+			}
+			flushTimer = nil
+			flushC = nil
+		}
+	}
+	armTimer := func() {
+		delay := f.currentBatchFlushInterval()
+		if delay <= 0 {
+			return
+		}
+		if flushTimer == nil {
+			flushTimer = time.NewTimer(delay)
+			flushC = flushTimer.C
+		}
+	}
+
+	for {
+		select {
+		case pkt := <-queue:
+			if pkt == nil {
+				continue
+			}
+			if f.ensurePacketHeadroom(&pkt, requiredHeadroom, i, "queue") {
+				pending = append(pending, pkt)
+			}
+			if len(pending) >= cap(pending) {
+				flush("cap", false)
+				continue
+			}
+			armTimer()
+			f.observeTunQueueLen(i)
+		case <-flushC:
+			flush("timer", true)
+		}
+	}
+}
+
+func (f *Interface) flushAndReleaseBatch(index int, writer udp.Conn, batch []queuedDatagram, reason string) {
+	if len(batch) == 0 {
+		return
+	}
+	f.flushDatagrams(index, writer, batch, reason)
+	for idx := range batch {
+		if batch[idx].packet != nil {
+			batch[idx].packet.Release()
+			batch[idx].packet = nil
+		}
+	}
+	if f.batchUDPFlushCounter != nil {
+		f.batchUDPFlushCounter.Inc(int64(len(batch)))
+	}
+}
+
+func (f *Interface) flushDatagrams(index int, writer udp.Conn, batch []queuedDatagram, reason string) {
+	if len(batch) == 0 {
+		return
+	}
+	if f.l.Level >= logrus.DebugLevel {
+		f.l.WithFields(logrus.Fields{
+			"writer":  index,
+			"reason":  reason,
+			"pending": len(batch),
+		}).Debug("udp batch flush summary")
+	}
+	maxSeg := f.effectiveGSOMaxSegments()
+	if bw, ok := writer.(udp.BatchConn); ok {
+		chunkCap := maxSeg
+		if chunkCap <= 0 {
+			chunkCap = len(batch)
+		}
+		chunk := make([]udp.Datagram, 0, chunkCap)
+		var (
+			currentAddr netip.AddrPort
+			segments    int
+		)
+		flushChunk := func() {
+			if len(chunk) == 0 {
+				return
+			}
+			if f.l.Level >= logrus.DebugLevel {
+				f.l.WithFields(logrus.Fields{
+					"writer":        index,
+					"segments":      len(chunk),
+					"dest":          chunk[0].Addr,
+					"reason":        reason,
+					"pending_total": len(batch),
+				}).Debug("flushing UDP batch")
+			}
+			if err := bw.WriteBatch(chunk); err != nil {
+				f.l.WithError(err).
+					WithField("writer", index).
+					WithField("reason", reason).
+					Warn("Failed to write UDP batch")
+			}
+			chunk = chunk[:0]
+			segments = 0
+		}
+		for _, item := range batch {
+			if item.packet == nil || !item.addr.IsValid() {
+				continue
+			}
+			payload := item.packet.Payload()[:item.packet.Len]
+			if segments == 0 {
+				currentAddr = item.addr
+			}
+			if item.addr != currentAddr || (maxSeg > 0 && segments >= maxSeg) {
+				flushChunk()
+				currentAddr = item.addr
+			}
+			chunk = append(chunk, udp.Datagram{Payload: payload, Addr: item.addr})
+			segments++
+		}
+		flushChunk()
+		return
+	}
+	for _, item := range batch {
+		if item.packet == nil || !item.addr.IsValid() {
+			continue
+		}
+		if f.l.Level >= logrus.DebugLevel {
+			f.l.WithFields(logrus.Fields{
+				"writer":   index,
+				"reason":   reason,
+				"dest":     item.addr,
+				"segments": 1,
+			}).Debug("flushing UDP batch")
+		}
+		if err := writer.WriteTo(item.packet.Payload()[:item.packet.Len], item.addr); err != nil {
+			f.l.WithError(err).
+				WithField("writer", index).
+				WithField("udpAddr", item.addr).
+				WithField("reason", reason).
+				Warn("Failed to write UDP packet")
+		}
+	}
+}
+
+func (f *Interface) tryQueueDatagram(q int, buf []byte, addr netip.AddrPort) bool {
+	if !addr.IsValid() || !f.batches.Enabled() {
+		return false
+	}
+	pkt := f.batches.newPacket()
+	if pkt == nil {
+		return false
+	}
+	payload := pkt.Payload()
+	if len(payload) < len(buf) {
+		pkt.Release()
+		return false
+	}
+	copy(payload, buf)
+	pkt.Len = len(buf)
+	if f.batches.enqueueTx(q, pkt, addr) {
+		f.observeUDPQueueLen(q)
+		return true
+	}
+	pkt.Release()
+	return false
+}
+
+func (f *Interface) writerForIndex(i int) udp.Conn {
+	if i < 0 || i >= len(f.writers) {
+		return nil
+	}
+	return f.writers[i]
+}
+
+func (f *Interface) writeImmediate(q int, buf []byte, addr netip.AddrPort, hostinfo *HostInfo) {
+	writer := f.writerForIndex(q)
+	if writer == nil {
+		f.l.WithField("udpAddr", addr).
+			WithField("writer", q).
+			Error("Failed to write outgoing packet: no writer available")
+		return
+	}
+	if err := writer.WriteTo(buf, addr); err != nil {
+		hostinfo.logger(f.l).
+			WithError(err).
+			WithField("udpAddr", addr).
+			Error("Failed to write outgoing packet")
+	}
+}
+
+func (f *Interface) tryQueuePacket(q int, pkt *overlay.Packet, addr netip.AddrPort) bool {
+	if pkt == nil || !addr.IsValid() || !f.batches.Enabled() {
+		return false
+	}
+	if f.batches.enqueueTx(q, pkt, addr) {
+		f.observeUDPQueueLen(q)
+		return true
+	}
+	return false
+}
+
+func (f *Interface) writeImmediatePacket(q int, pkt *overlay.Packet, addr netip.AddrPort, hostinfo *HostInfo) {
+	if pkt == nil {
+		return
+	}
+	writer := f.writerForIndex(q)
+	if writer == nil {
+		f.l.WithField("udpAddr", addr).
+			WithField("writer", q).
+			Error("Failed to write outgoing packet: no writer available")
+		pkt.Release()
+		return
+	}
+	if err := writer.WriteTo(pkt.Payload()[:pkt.Len], addr); err != nil {
+		hostinfo.logger(f.l).
+			WithError(err).
+			WithField("udpAddr", addr).
+			Error("Failed to write outgoing packet")
+	}
+	pkt.Release()
+}
+
+func (f *Interface) writePacketToTun(q int, pkt *overlay.Packet) {
+	if pkt == nil {
+		return
+	}
+	writer := f.readers[q]
+	if writer == nil {
+		pkt.Release()
+		return
+	}
+	if bw, ok := writer.(interface {
+		WriteBatch([]*overlay.Packet) (int, error)
+	}); ok {
+		if _, err := bw.WriteBatch([]*overlay.Packet{pkt}); err != nil {
+			f.l.WithError(err).WithField("queue", q).Warn("Failed to write tun packet via batch writer")
+			pkt.Release()
+		}
+		return
+	}
+	if _, err := writer.Write(pkt.Payload()[:pkt.Len]); err != nil {
+		f.l.WithError(err).Error("Failed to write to tun")
+	}
+	pkt.Release()
+}
+
+func (f *Interface) clonePacketWithHeadroom(pkt *overlay.Packet, required int) *overlay.Packet {
+	if pkt == nil {
+		return nil
+	}
+	payload := pkt.Payload()[:pkt.Len]
+	if len(payload) == 0 && required <= 0 {
+		return pkt
+	}
+
+	pool := f.batches.Pool()
+	if pool != nil {
+		if clone := pool.Get(); clone != nil {
+			if len(clone.Payload()) >= len(payload) {
+				clone.Len = copy(clone.Payload(), payload)
+				pkt.Release()
+				return clone
+			}
+			clone.Release()
+		}
+	}
+
+	if required < 0 {
+		required = 0
+	}
+	buf := make([]byte, required+len(payload))
+	n := copy(buf[required:], payload)
+	pkt.Release()
+	return &overlay.Packet{
+		Buf:    buf,
+		Offset: required,
+		Len:    n,
+	}
+}
+
+func (f *Interface) observeUDPQueueLen(i int) {
+	if f.batchUDPQueueGauge == nil {
+		return
+	}
+	f.batchUDPQueueGauge.Update(int64(f.batches.txQueueLen(i)))
+}
+
+func (f *Interface) observeTunQueueLen(i int) {
+	if f.batchTunQueueGauge == nil {
+		return
+	}
+	f.batchTunQueueGauge.Update(int64(f.batches.tunQueueLen(i)))
+}
+
+func (f *Interface) currentBatchFlushInterval() time.Duration {
+	if v := f.batchFlushInterval.Load(); v > 0 {
+		return time.Duration(v)
+	}
+	return 0
+}
+
+func (f *Interface) ensurePacketHeadroom(pkt **overlay.Packet, required int, queue int, reason string) bool {
+	p := *pkt
+	if p == nil {
+		return false
+	}
+	if required <= 0 || p.Offset >= required {
+		return true
+	}
+	clone := f.clonePacketWithHeadroom(p, required)
+	if clone == nil {
+		f.l.WithFields(logrus.Fields{
+			"queue":  queue,
+			"reason": reason,
+		}).Warn("dropping packet lacking tun headroom")
+		return false
+	}
+	*pkt = clone
+	return true
+}
+
+func isVirtioHeadroomError(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := err.Error()
+	return strings.Contains(msg, "headroom") || strings.Contains(msg, "virtio")
+}
+
+func (f *Interface) effectiveGSOMaxSegments() int {
+	max := f.gsoMaxSegments
+	if max <= 0 {
+		max = defaultGSOMaxSegments
+	}
+	if max > maxKernelGSOSegments {
+		max = maxKernelGSOSegments
+	}
+	if !f.enableGSO {
+		return 1
+	}
+	return max
+}
+
+type udpOffloadConfigurator interface {
+	ConfigureOffload(enableGSO, enableGRO bool, maxSegments int)
+}
+
+func (f *Interface) applyOffloadConfig(enableGSO, enableGRO bool, maxSegments int) {
+	if maxSegments <= 0 {
+		maxSegments = defaultGSOMaxSegments
+	}
+	if maxSegments > maxKernelGSOSegments {
+		maxSegments = maxKernelGSOSegments
+	}
+	f.enableGSO = enableGSO
+	f.enableGRO = enableGRO
+	f.gsoMaxSegments = maxSegments
+	for _, writer := range f.writers {
+		if cfg, ok := writer.(udpOffloadConfigurator); ok {
+			cfg.ConfigureOffload(enableGSO, enableGRO, maxSegments)
 		}
 	}
 }
@@ -475,6 +1062,42 @@ func (f *Interface) reloadMisc(c *config.C) {
 		f.reQueryWait.Store(int64(n))
 		f.l.Info("timers.requery_wait_duration has changed")
 	}
+
+	if c.HasChanged("listen.gso_flush_timeout") {
+		d := c.GetDuration("listen.gso_flush_timeout", defaultGSOFlushInterval)
+		if d < 0 {
+			d = 0
+		}
+		f.batchFlushInterval.Store(int64(d))
+		f.l.WithField("duration", d).Info("listen.gso_flush_timeout has changed")
+	} else if c.HasChanged("batch.flush_interval") {
+		d := c.GetDuration("batch.flush_interval", defaultGSOFlushInterval)
+		if d < 0 {
+			d = 0
+		}
+		f.batchFlushInterval.Store(int64(d))
+		f.l.WithField("duration", d).Warn("batch.flush_interval is deprecated; use listen.gso_flush_timeout")
+	}
+
+	if c.HasChanged("batch.queue_depth") {
+		n := c.GetInt("batch.queue_depth", f.batchQueueDepth)
+		if n != f.batchQueueDepth {
+			f.batchQueueDepth = n
+			f.l.Warn("batch.queue_depth changes require a restart to take effect")
+		}
+	}
+
+	if c.HasChanged("listen.enable_gso") || c.HasChanged("listen.enable_gro") || c.HasChanged("listen.gso_max_segments") {
+		enableGSO := c.GetBool("listen.enable_gso", f.enableGSO)
+		enableGRO := c.GetBool("listen.enable_gro", f.enableGRO)
+		maxSeg := c.GetInt("listen.gso_max_segments", f.gsoMaxSegments)
+		f.applyOffloadConfig(enableGSO, enableGRO, maxSeg)
+		f.l.WithFields(logrus.Fields{
+			"enableGSO":      enableGSO,
+			"enableGRO":      enableGRO,
+			"gsoMaxSegments": maxSeg,
+		}).Info("listen GSO/GRO configuration updated")
+	}
 }
 
 func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
@@ -507,11 +1130,6 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
 			} else {
 				certMaxVersion.Update(int64(certState.v1Cert.Version()))
 			}
-			if f.l.Level != logrus.DebugLevel {
-				f.listenInMetric.Update(int64(f.listenInN))
-				f.listenOutMetric.Update(int64(f.listenOutN))
-			}
-
 		}
 	}
 }

lighthouse.go

@@ -360,8 +360,7 @@ func (lh *LightHouse) parseLighthouses(c *config.C) ([]netip.Addr, error) {
 		}
 
 		if !lh.myVpnNetworksTable.Contains(addr) {
-			lh.l.WithFields(m{"vpnAddr": addr, "networks": lh.myVpnNetworks}).
+			return nil, util.NewContextualError("lighthouse host is not in our networks, invalid", m{"vpnAddr": addr, "networks": lh.myVpnNetworks}, nil)
-				Warn("lighthouse host is not within our networks, lighthouse functionality will work but layer 3 network traffic to the lighthouse will not")
 		}
 		out[i] = addr
 	}
@@ -432,8 +431,7 @@ func (lh *LightHouse) loadStaticMap(c *config.C, staticList map[netip.Addr]struc
 		}
 
 		if !lh.myVpnNetworksTable.Contains(vpnAddr) {
-			lh.l.WithFields(m{"vpnAddr": vpnAddr, "networks": lh.myVpnNetworks, "entry": i + 1}).
+			return util.NewContextualError("static_host_map key is not in our network, invalid", m{"vpnAddr": vpnAddr, "networks": lh.myVpnNetworks, "entry": i + 1}, nil)
-				Warn("static_host_map key is not within our networks, layer 3 network traffic to this host will not work")
 		}
 
 		vals, ok := v.([]any)
@@ -1339,19 +1337,12 @@ func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpn
 		}
 	}
 
-	remoteAllowList := lhh.lh.GetRemoteAllowList()
 	for _, a := range n.Details.V4AddrPorts {
-		b := protoV4AddrPortToNetAddrPort(a)
+		punch(protoV4AddrPortToNetAddrPort(a), detailsVpnAddr)
-		if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) {
-			punch(b, detailsVpnAddr)
-		}
 	}
 
 	for _, a := range n.Details.V6AddrPorts {
-		b := protoV6AddrPortToNetAddrPort(a)
+		punch(protoV6AddrPortToNetAddrPort(a), detailsVpnAddr)
-		if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) {
-			punch(b, detailsVpnAddr)
-		}
 	}
 
 	// This sends a nebula test packet to the host trying to contact us. In the case

lighthouse_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v3"
 )
 
 func TestOldIPv4Only(t *testing.T) {

main.go

@@ -5,8 +5,7 @@ import (
 	"fmt"
 	"net"
 	"net/netip"
-	"runtime/debug"
+	"runtime"
-	"strings"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ import (
 	"github.com/slackhq/nebula/sshd"
 	"github.com/slackhq/nebula/udp"
 	"github.com/slackhq/nebula/util"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v3"
 )
 
 type m = map[string]any
@@ -29,10 +28,6 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		}
 	}()
 
-	if buildVersion == "" {
-		buildVersion = moduleVersion()
-	}
-
 	l := logger
 	l.Formatter = &logrus.TextFormatter{
 		FullTimestamp: true,
@@ -81,8 +76,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	if c.GetBool("sshd.enabled", false) {
 		sshStart, err = configSSH(l, ssh, c)
 		if err != nil {
-			l.WithError(err).Warn("Failed to configure sshd, ssh debugging will not be available")
+			return nil, util.ContextualizeIfNeeded("Error while configuring the sshd", err)
-			sshStart = nil
 		}
 	}
 
@@ -150,6 +144,20 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	// set up our UDP listener
 	udpConns := make([]udp.Conn, routines)
 	port := c.GetInt("listen.port", 0)
+	enableGSO := c.GetBool("listen.enable_gso", true)
+	enableGRO := c.GetBool("listen.enable_gro", true)
+	gsoMaxSegments := c.GetInt("listen.gso_max_segments", defaultGSOMaxSegments)
+	if gsoMaxSegments <= 0 {
+		gsoMaxSegments = defaultGSOMaxSegments
+	}
+	if gsoMaxSegments > maxKernelGSOSegments {
+		gsoMaxSegments = maxKernelGSOSegments
+	}
+	gsoFlushTimeout := c.GetDuration("listen.gso_flush_timeout", defaultGSOFlushInterval)
+	if gsoFlushTimeout < 0 {
+		gsoFlushTimeout = 0
+	}
+	batchQueueDepth := c.GetInt("batch.queue_depth", 0)
 
 	if !configTest {
 		rawListenHost := c.GetString("listen.host", "0.0.0.0")
@@ -169,13 +177,27 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 			listenHost = ips[0].Unmap()
 		}
 
+		useWGDefault := runtime.GOOS == "linux"
+		useWG := c.GetBool("listen.use_wireguard_stack", useWGDefault)
+		var mkListener func(*logrus.Logger, netip.Addr, int, bool, int) (udp.Conn, error)
+		if useWG {
+			mkListener = udp.NewWireguardListener
+		} else {
+			mkListener = udp.NewListener
+		}
+
 		for i := 0; i < routines; i++ {
 			l.Infof("listening on %v", netip.AddrPortFrom(listenHost, uint16(port)))
-			udpServer, err := udp.NewListener(l, listenHost, port, routines > 1, c.GetInt("listen.batch", 64))
+			udpServer, err := mkListener(l, listenHost, port, routines > 1, c.GetInt("listen.batch", 64))
 			if err != nil {
 				return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err)
 			}
 			udpServer.ReloadConfig(c)
+			if cfg, ok := udpServer.(interface {
+				ConfigureOffload(bool, bool, int)
+			}); ok {
+				cfg.ConfigureOffload(enableGSO, enableGRO, gsoMaxSegments)
+			}
 			udpConns[i] = udpServer
 
 			// If port is dynamic, discover it before the next pass through the for loop
@@ -243,12 +265,17 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		reQueryWait:           c.GetDuration("timers.requery_wait_duration", defaultReQueryWait),
 		DropLocalBroadcast:    c.GetBool("tun.drop_local_broadcast", false),
 		DropMulticast:         c.GetBool("tun.drop_multicast", false),
+		EnableGSO:             enableGSO,
+		EnableGRO:             enableGRO,
+		GSOMaxSegments:        gsoMaxSegments,
 		routines:              routines,
 		MessageMetrics:        messageMetrics,
 		version:               buildVersion,
 		relayManager:          NewRelayManager(ctx, l, hostMap, c),
 		punchy:                punchy,
 		ConntrackCacheTimeout: conntrackCacheTimeout,
+		BatchFlushInterval:    gsoFlushTimeout,
+		BatchQueueDepth:       batchQueueDepth,
 		l:                     l,
 	}
 
@@ -260,6 +287,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		}
 
 		ifce.writers = udpConns
+		ifce.applyOffloadConfig(enableGSO, enableGRO, gsoMaxSegments)
 		lightHouse.ifce = ifce
 
 		ifce.RegisterConfigChangeCallbacks(c)
@@ -302,18 +330,3 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		connManager.Start,
 	}, nil
 }
-
-func moduleVersion() string {
-	info, ok := debug.ReadBuildInfo()
-	if !ok {
-		return ""
-	}
-
-	for _, dep := range info.Deps {
-		if dep.Path == "github.com/slackhq/nebula" {
-			return strings.TrimPrefix(dep.Version, "v")
-		}
-	}
-
-	return ""
-}

outside.go

@@ -7,12 +7,12 @@ import (
 	"time"
 
 	"github.com/google/gopacket/layers"
-	"github.com/slackhq/nebula/packet"
 	"golang.org/x/net/ipv6"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/overlay"
 	"golang.org/x/net/ipv4"
 )
 
@@ -20,108 +20,30 @@ const (
 	minFwPacketLen = 4
 )
 
-// handleRelayPackets handles relay packets. Returns false if there's nothing left to do, true for continuing to process an unwrapped TerminalType packet
+func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache *firewall.ConntrackCache) {
-// scratch must be large enough to contain a packet to be relayed if needed
+	err := h.Parse(packet)
-func (f *Interface) handleRelayPackets(via ViaSender, hostinfo *HostInfo, segment []byte, scratch []byte, h *header.H, nb []byte) ([]byte, *ViaSender, bool) {
-	var err error
-	// The entire body is sent as AD, not encrypted.
-	// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
-	// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
-	// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
-	// which will gracefully fail in the DecryptDanger call.
-	signedPayload := segment[:len(segment)-hostinfo.ConnectionState.dKey.Overhead()]
-	signatureValue := segment[len(segment)-hostinfo.ConnectionState.dKey.Overhead():]
-	scratch, err = hostinfo.ConnectionState.dKey.DecryptDanger(scratch, signedPayload, signatureValue, h.MessageCounter, nb)
 	if err != nil {
-		return nil, nil, false
+		// Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
-	}
+		if len(packet) > 1 {
-	// Successfully validated the thing. Get rid of the Relay header.
+			f.l.WithField("packet", packet).Infof("Error while parsing inbound packet from %s: %s", ip, err)
-	signedPayload = signedPayload[header.Len:]
-	// Pull the Roaming parts up here, and return in all call paths.
-	f.handleHostRoaming(hostinfo, via)
-	// Track usage of both the HostInfo and the Relay for the received & authenticated packet
-	f.connectionManager.In(hostinfo)
-	f.connectionManager.RelayUsed(h.RemoteIndex)
-
-	relay, ok := hostinfo.relayState.QueryRelayForByIdx(h.RemoteIndex)
-	if !ok {
-		// The only way this happens is if hostmap has an index to the correct HostInfo, but the HostInfo is missing
-		// its internal mapping. This should never happen.
-		hostinfo.logger(f.l).WithFields(logrus.Fields{"vpnAddrs": hostinfo.vpnAddrs, "remoteIndex": h.RemoteIndex}).Error("HostInfo missing remote relay index")
-		return nil, nil, false
-	}
-
-	switch relay.Type {
-	case TerminalType:
-		// If I am the target of this relay, process the unwrapped packet
-		// We need to re-write our variables to ensure this segment is correctly parsed.
-		// We could set up for a recursive call here, but this makes it easier to prove that we'll never stack-overflow
-
-		//mirrors the top of readOutsideSegment
-		err = h.Parse(signedPayload)
-		if err != nil {
-			// Hole punch packets are 0 or 1 byte big, so let's ignore printing those errors
-			if len(signedPayload) > 1 {
-				f.l.WithField("packet", segment).Infof("Error while parsing inbound packet from %s: %s", via, err)
-			}
-			return nil, nil, false
-		}
-		newVia := &ViaSender{
-			UdpAddr:   via.UdpAddr,
-			relayHI:   hostinfo,
-			remoteIdx: relay.RemoteIndex,
-			relay:     relay,
-			IsRelayed: true,
-		}
-		//continue flowing through readOutsideSegment()
-		return signedPayload, newVia, true
-	case ForwardingType:
-		// Find the target HostInfo relay object
-		targetHI, targetRelay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relay.PeerAddr)
-		if err != nil {
-			hostinfo.logger(f.l).WithField("relayTo", relay.PeerAddr).WithError(err).WithField("hostinfo.vpnAddrs", hostinfo.vpnAddrs).Info("Failed to find target host info by ip")
-			return nil, nil, false
-		}
-
-		// If that relay is Established, forward the payload through it
-		if targetRelay.State == Established {
-			switch targetRelay.Type {
-			case ForwardingType:
-				// Forward this packet through the relay tunnel, and find the target HostInfo
-				f.SendVia(targetHI, targetRelay, signedPayload, nb, scratch[:0], false) //todo it would be nice to queue this up and do it later, or at least avoid a memcpy of signedPayload
-			case TerminalType:
-				hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
-			default:
-				hostinfo.logger(f.l).WithField("targetRelay.Type", targetRelay.Type).Error("Unexpected Relay Type")
-			}
-		} else {
-			hostinfo.logger(f.l).WithFields(logrus.Fields{"relayTo": relay.PeerAddr, "relayFrom": hostinfo.vpnAddrs[0], "targetRelayState": targetRelay.State}).Info("Unexpected target relay state")
-		}
-	}
-	return nil, nil, false
-}
-
-func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packet.OutPacket, lhf *LightHouseHandler, s *Scratches, q int, localCache firewall.ConntrackCache, now time.Time) {
-	h := s.h
-	err := h.Parse(segment)
-	if err != nil {
-		// Hole punch packets are 0 or 1 byte big, so let's ignore printing those errors
-		if len(segment) > 1 {
-			f.l.WithField("packet", segment).Infof("Error while parsing inbound packet from %s: %s", via, err)
 		}
 		return
 	}
 
+	//l.Error("in packet ", header, packet[HeaderLen:])
+	if ip.IsValid() {
+		if f.myVpnNetworksTable.Contains(ip.Addr()) {
+			if f.l.Level >= logrus.DebugLevel {
+				f.l.WithField("udpAddr", ip).Debug("Refusing to process double encrypted packet")
+			}
+			return
+		}
+	}
+
 	var hostinfo *HostInfo
 	// verify if we've seen this index before, otherwise respond to the handshake initiation
 	if h.Type == header.Message && h.Subtype == header.MessageRelay {
 		hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex)
-		newSegment, newVia, keepGoing := f.handleRelayPackets(via, hostinfo, segment, s.scratch, h, s.nb)
-		if !keepGoing {
-			return
-		}
-		via = *newVia
-		segment = newSegment
 	} else {
 		hostinfo = f.hostMap.QueryIndex(h.RemoteIndex)
 	}
@@ -133,49 +55,104 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 
 	switch h.Type {
 	case header.Message:
-		if !f.handleEncrypted(ci, via, h) {
+		// TODO handleEncrypted sends directly to addr on error. Handle this in the tunneling case.
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
 		switch h.Subtype {
 		case header.MessageNone:
-			if !f.decryptToTunDelayWrite(hostinfo, h.MessageCounter, out, segment, s.fwPacket, s.nb, q, localCache, now) {
+			if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, nb, q, localCache, ip, h.RemoteIndex) {
-				out.DestroyLastSegment() //prevent a rejected segment from being used
 				return
 			}
 		case header.MessageRelay:
-			f.l.Error("relayed messages cannot contain relay messages, dropping packet")
+			// The entire body is sent as AD, not encrypted.
-			return
+			// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
+			// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
+			// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
+			// which will gracefully fail in the DecryptDanger call.
+			signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
+			signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
+			out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
+			if err != nil {
+				return
+			}
+			// Successfully validated the thing. Get rid of the Relay header.
+			signedPayload = signedPayload[header.Len:]
+			// Pull the Roaming parts up here, and return in all call paths.
+			f.handleHostRoaming(hostinfo, ip)
+			// Track usage of both the HostInfo and the Relay for the received & authenticated packet
+			f.connectionManager.In(hostinfo)
+			f.connectionManager.RelayUsed(h.RemoteIndex)
+
+			relay, ok := hostinfo.relayState.QueryRelayForByIdx(h.RemoteIndex)
+			if !ok {
+				// The only way this happens is if hostmap has an index to the correct HostInfo, but the HostInfo is missing
+				// its internal mapping. This should never happen.
+				hostinfo.logger(f.l).WithFields(logrus.Fields{"vpnAddrs": hostinfo.vpnAddrs, "remoteIndex": h.RemoteIndex}).Error("HostInfo missing remote relay index")
+				return
+			}
+
+			switch relay.Type {
+			case TerminalType:
+				// If I am the target of this relay, process the unwrapped packet
+				// From this recursive point, all these variables are 'burned'. We shouldn't rely on them again.
+				f.readOutsidePackets(netip.AddrPort{}, &ViaSender{relayHI: hostinfo, remoteIdx: relay.RemoteIndex, relay: relay}, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
+				return
+			case ForwardingType:
+				// Find the target HostInfo relay object
+				targetHI, targetRelay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relay.PeerAddr)
+				if err != nil {
+					hostinfo.logger(f.l).WithField("relayTo", relay.PeerAddr).WithError(err).WithField("hostinfo.vpnAddrs", hostinfo.vpnAddrs).Info("Failed to find target host info by ip")
+					return
+				}
+
+				// If that relay is Established, forward the payload through it
+				if targetRelay.State == Established {
+					switch targetRelay.Type {
+					case ForwardingType:
+						// Forward this packet through the relay tunnel
+						// Find the target HostInfo
+						f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
+						return
+					case TerminalType:
+						hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
+					}
+				} else {
+					hostinfo.logger(f.l).WithFields(logrus.Fields{"relayTo": relay.PeerAddr, "relayFrom": hostinfo.vpnAddrs[0], "targetRelayState": targetRelay.State}).Info("Unexpected target relay state")
+					return
+				}
+			}
 		}
 
 	case header.LightHouse:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		if !f.handleEncrypted(ci, via, h) {
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, s.scratch, segment, h, s.nb)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", via.UdpAddr).
+			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
-				WithField("packet", segment).
+				WithField("packet", packet).
 				Error("Failed to decrypt lighthouse packet")
 			return
 		}
 
-		lhf.HandleRequest(via.UdpAddr, hostinfo.vpnAddrs, d, f)
+		lhf.HandleRequest(ip, hostinfo.vpnAddrs, d, f)
 
 		// Fallthrough to the bottom to record incoming traffic
 
 	case header.Test:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		if !f.handleEncrypted(ci, via, h) {
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, s.scratch, segment, h, s.nb)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", via).
+			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
-				WithField("packet", segment).
+				WithField("packet", packet).
 				Error("Failed to decrypt test packet")
 			return
 		}
@@ -183,8 +160,8 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 		if h.Subtype == header.TestRequest {
 			// This testRequest might be from TryPromoteBest, so we should roam
 			// to the new IP address before responding
-			f.handleHostRoaming(hostinfo, via)
+			f.handleHostRoaming(hostinfo, ip)
-			f.send(header.Test, header.TestReply, ci, hostinfo, d, s.nb, s.scratch)
+			f.send(header.Test, header.TestReply, ci, hostinfo, d, nb, out)
 		}
 
 		// Fallthrough to the bottom to record incoming traffic
@@ -194,35 +171,35 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 
 	case header.Handshake:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		f.handshakeManager.HandleIncoming(via, segment, h)
+		f.handshakeManager.HandleIncoming(ip, via, packet, h)
 		return
 
 	case header.RecvError:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		f.handleRecvError(via.UdpAddr, h)
+		f.handleRecvError(ip, h)
 		return
 
 	case header.CloseTunnel:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		if !f.handleEncrypted(ci, via, h) {
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
-		hostinfo.logger(f.l).WithField("udpAddr", via).
+		hostinfo.logger(f.l).WithField("udpAddr", ip).
 			Info("Close tunnel received, tearing down.")
 
 		f.closeTunnel(hostinfo)
 		return
 
 	case header.Control:
-		if !f.handleEncrypted(ci, via, h) {
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, s.scratch, segment, h, s.nb)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", via).
+			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
-				WithField("packet", segment).
+				WithField("packet", packet).
 				Error("Failed to decrypt Control packet")
 			return
 		}
@@ -231,37 +208,15 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 
 	default:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		hostinfo.logger(f.l).Debugf("Unexpected packet received from %s", via)
+		hostinfo.logger(f.l).Debugf("Unexpected packet received from %s", ip)
 		return
 	}
 
-	f.handleHostRoaming(hostinfo, via)
+	f.handleHostRoaming(hostinfo, ip)
 
 	f.connectionManager.In(hostinfo)
 }
 
-func (f *Interface) readOutsidePacketsMany(packets []*packet.UDPPacket, out []*packet.OutPacket, lhf *LightHouseHandler, s *Scratches, q int, localCache firewall.ConntrackCache, now time.Time) {
-	for i, pkt := range packets {
-		via := ViaSender{UdpAddr: pkt.AddrPort()}
-
-		//l.Error("in packet ", header, packet[HeaderLen:])
-		if f.myVpnNetworksTable.Contains(via.UdpAddr.Addr()) {
-			if f.l.Level >= logrus.DebugLevel {
-				f.l.WithField("from", via).Debug("Refusing to process double encrypted packet")
-			}
-			return
-		}
-
-		for segment := range pkt.Segments() {
-			f.readOutsideSegment(via, segment, out[i], lhf, s, q, localCache, now)
-		}
-		//_, err := f.readers[q].WriteOne(out[i], false, q)
-		//if err != nil {
-		//	f.l.WithError(err).Error("Failed to write packet")
-		//}
-	}
-}
-
 // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
 func (f *Interface) closeTunnel(hostInfo *HostInfo) {
 	final := f.hostMap.DeleteHostInfo(hostInfo)
@@ -276,36 +231,36 @@ func (f *Interface) sendCloseTunnel(h *HostInfo) {
 	f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
 }
 
-func (f *Interface) handleHostRoaming(hostinfo *HostInfo, via ViaSender) {
+func (f *Interface) handleHostRoaming(hostinfo *HostInfo, udpAddr netip.AddrPort) {
-	if !via.IsRelayed && hostinfo.remote != via.UdpAddr {
+	if udpAddr.IsValid() && hostinfo.remote != udpAddr {
-		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
+		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, udpAddr.Addr()) {
-			hostinfo.logger(f.l).WithField("newAddr", via.UdpAddr).Debug("lighthouse.remote_allow_list denied roaming")
+			hostinfo.logger(f.l).WithField("newAddr", udpAddr).Debug("lighthouse.remote_allow_list denied roaming")
 			return
 		}
 
-		if !hostinfo.lastRoam.IsZero() && via.UdpAddr == hostinfo.lastRoamRemote && time.Since(hostinfo.lastRoam) < RoamingSuppressSeconds*time.Second {
+		if !hostinfo.lastRoam.IsZero() && udpAddr == hostinfo.lastRoamRemote && time.Since(hostinfo.lastRoam) < RoamingSuppressSeconds*time.Second {
 			if f.l.Level >= logrus.DebugLevel {
-				hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", via.UdpAddr).
+				hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", udpAddr).
 					Debugf("Suppressing roam back to previous remote for %d seconds", RoamingSuppressSeconds)
 			}
 			return
 		}
 
-		hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", via.UdpAddr).
+		hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", udpAddr).
 			Info("Host roamed to new udp ip/port.")
 		hostinfo.lastRoam = time.Now()
 		hostinfo.lastRoamRemote = hostinfo.remote
-		hostinfo.SetRemote(via.UdpAddr)
+		hostinfo.SetRemote(udpAddr)
 	}
 
 }
 
 // handleEncrypted returns true if a packet should be processed, false otherwise
-func (f *Interface) handleEncrypted(ci *ConnectionState, via ViaSender, h *header.H) bool {
+func (f *Interface) handleEncrypted(ci *ConnectionState, addr netip.AddrPort, h *header.H) bool {
 	// If connectionstate does not exist, send a recv error, if possible, to encourage a fast reconnect
 	if ci == nil {
-		if !via.IsRelayed {
+		if addr.IsValid() {
-			f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex)
+			f.maybeSendRecvError(addr, h.RemoteIndex)
 		}
 		return false
 	}
@@ -332,11 +287,11 @@ func newPacket(data []byte, incoming bool, fp *firewall.Packet) error {
 		return ErrPacketTooShort
 	}
 
-	//version := int((data[0] >> 4) & 0x0f)
+	version := int((data[0] >> 4) & 0x0f)
-	switch data[0] & 0xf0 {
+	switch version {
-	case ipv4.Version << 4:
+	case ipv4.Version:
 		return parseV4(data, incoming, fp)
-	case ipv6.Version << 4:
+	case ipv6.Version:
 		return parseV6(data, incoming, fp)
 	}
 	return ErrUnknownIPVersion
@@ -511,40 +466,58 @@ func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []
 	return out, nil
 }
 
-func (f *Interface) decryptToTunDelayWrite(hostinfo *HostInfo, messageCounter uint64, out *packet.OutPacket, inSegment []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache, now time.Time) bool {
+func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache *firewall.ConntrackCache, addr netip.AddrPort, recvIndex uint32) bool {
-	var err error
+	var (
+		err error
+		pkt *overlay.Packet
+	)
 
-	seg, err := f.readers[q].AllocSeg(out, q)
+	if f.batches.tunQueue(q) != nil {
-	if err != nil {
+		pkt = f.batches.newPacket()
-		f.l.WithError(err).Errorln("decryptToTunDelayWrite: failed to allocate segment")
+		if pkt != nil {
-		return false
+			out = pkt.Payload()[:0]
+		}
 	}
 
-	out.SegmentPayloads[seg] = out.SegmentPayloads[seg][:0]
+	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
-	out.SegmentPayloads[seg], err = hostinfo.ConnectionState.dKey.DecryptDanger(out.SegmentPayloads[seg], inSegment[:header.Len], inSegment[header.Len:], messageCounter, nb)
 	if err != nil {
+		if pkt != nil {
+			pkt.Release()
+		}
 		hostinfo.logger(f.l).WithError(err).Error("Failed to decrypt packet")
+		if addr.IsValid() {
+			f.maybeSendRecvError(addr, recvIndex)
+		}
 		return false
 	}
 
-	err = newPacket(out.SegmentPayloads[seg], true, fwPacket)
+	err = newPacket(out, true, fwPacket)
 	if err != nil {
+		if pkt != nil {
+			pkt.Release()
+		}
 		hostinfo.logger(f.l).WithError(err).WithField("packet", out).
 			Warnf("Error while validating inbound packet")
 		return false
 	}
 
 	if !hostinfo.ConnectionState.window.Update(f.l, messageCounter) {
+		if pkt != nil {
+			pkt.Release()
+		}
 		hostinfo.logger(f.l).WithField("fwPacket", fwPacket).
 			Debugln("dropping out of window packet")
 		return false
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache, now)
+	dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason != nil {
+		if pkt != nil {
+			pkt.Release()
+		}
 		// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
 		// This gives us a buffer to build the reject packet in
-		f.rejectOutside(out.SegmentPayloads[seg], hostinfo.ConnectionState, hostinfo, nb, inSegment, q)
+		f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, nb, packet, q)
 		if f.l.Level >= logrus.DebugLevel {
 			hostinfo.logger(f.l).WithField("fwPacket", fwPacket).
 				WithField("reason", dropReason).
@@ -554,7 +527,19 @@ func (f *Interface) decryptToTunDelayWrite(hostinfo *HostInfo, messageCounter ui
 	}
 
 	f.connectionManager.In(hostinfo)
-	out.Segments[seg] = out.Segments[seg][:len(out.SegmentHeaders[seg])+len(out.SegmentPayloads[seg])]
+	if pkt != nil {
+		pkt.Len = len(out)
+		if f.batches.enqueueTun(q, pkt) {
+			f.observeTunQueueLen(q)
+			return true
+		}
+		f.writePacketToTun(q, pkt)
+		return true
+	}
+
+	if _, err = f.readers[q].Write(out); err != nil {
+		f.l.WithError(err).Error("Failed to write to tun")
+	}
 	return true
 }
 

overlay/device.go

@@ -1,17 +1,99 @@
 package overlay
 
 import (
+	"io"
 	"net/netip"
+	"sync"
 
 	"github.com/slackhq/nebula/routing"
 )
 
 type Device interface {
-	TunDev
+	io.ReadWriteCloser
 	Activate() error
 	Networks() []netip.Prefix
 	Name() string
 	RoutesFor(netip.Addr) routing.Gateways
-	SupportsMultiqueue() bool
+	NewMultiQueueReader() (io.ReadWriteCloser, error)
-	NewMultiQueueReader() (TunDev, error)
+}
+
+// Packet represents a single packet buffer with optional headroom to carry
+// metadata (for example virtio-net headers).
+type Packet struct {
+	Buf     []byte
+	Offset  int
+	Len     int
+	release func()
+}
+
+func (p *Packet) Payload() []byte {
+	return p.Buf[p.Offset : p.Offset+p.Len]
+}
+
+func (p *Packet) Reset() {
+	p.Len = 0
+	p.Offset = 0
+	p.release = nil
+}
+
+func (p *Packet) Release() {
+	if p.release != nil {
+		p.release()
+		p.release = nil
+	}
+}
+
+func (p *Packet) Capacity() int {
+	return len(p.Buf) - p.Offset
+}
+
+// PacketPool manages reusable buffers with headroom.
+type PacketPool struct {
+	headroom int
+	blksz    int
+	pool     sync.Pool
+}
+
+func NewPacketPool(headroom, payload int) *PacketPool {
+	p := &PacketPool{headroom: headroom, blksz: headroom + payload}
+	p.pool.New = func() any {
+		buf := make([]byte, p.blksz)
+		return &Packet{Buf: buf, Offset: headroom}
+	}
+	return p
+}
+
+func (p *PacketPool) Get() *Packet {
+	pkt := p.pool.Get().(*Packet)
+	pkt.Offset = p.headroom
+	pkt.Len = 0
+	pkt.release = func() { p.put(pkt) }
+	return pkt
+}
+
+func (p *PacketPool) put(pkt *Packet) {
+	pkt.Reset()
+	p.pool.Put(pkt)
+}
+
+// BatchReader allows reading multiple packets into a shared pool with
+// preallocated headroom (e.g. virtio-net headers).
+type BatchReader interface {
+	ReadIntoBatch(pool *PacketPool) ([]*Packet, error)
+}
+
+// BatchWriter writes a slice of packets that carry their own metadata.
+type BatchWriter interface {
+	WriteBatch(packets []*Packet) (int, error)
+}
+
+// BatchCapableDevice describes a device that can efficiently read and write
+// batches of packets with virtio headroom.
+type BatchCapableDevice interface {
+	Device
+	BatchReader
+	BatchWriter
+	BatchHeadroom() int
+	BatchPayloadCap() int
+	BatchSize() int
 }

overlay/eventfd/eventfd.go

@@ -1,91 +0,0 @@
-package eventfd
-
-import (
-	"encoding/binary"
-	"syscall"
-
-	"golang.org/x/sys/unix"
-)
-
-type EventFD struct {
-	fd  int
-	buf [8]byte
-}
-
-func New() (EventFD, error) {
-	fd, err := unix.Eventfd(0, unix.EFD_NONBLOCK)
-	if err != nil {
-		return EventFD{}, err
-	}
-	return EventFD{
-		fd:  fd,
-		buf: [8]byte{},
-	}, nil
-}
-
-func (e *EventFD) Kick() error {
-	binary.LittleEndian.PutUint64(e.buf[:], 1) //is this right???
-	_, err := syscall.Write(int(e.fd), e.buf[:])
-	return err
-}
-
-func (e *EventFD) Close() error {
-	if e.fd != 0 {
-		return unix.Close(e.fd)
-	}
-	return nil
-}
-
-func (e *EventFD) FD() int {
-	return e.fd
-}
-
-type Epoll struct {
-	fd     int
-	buf    [8]byte
-	events []syscall.EpollEvent
-}
-
-func NewEpoll() (Epoll, error) {
-	fd, err := unix.EpollCreate1(0)
-	if err != nil {
-		return Epoll{}, err
-	}
-	return Epoll{
-		fd:     fd,
-		buf:    [8]byte{},
-		events: make([]syscall.EpollEvent, 1),
-	}, nil
-}
-
-func (ep *Epoll) AddEvent(fdToAdd int) error {
-	event := syscall.EpollEvent{
-		Events: syscall.EPOLLIN,
-		Fd:     int32(fdToAdd),
-	}
-	return syscall.EpollCtl(ep.fd, syscall.EPOLL_CTL_ADD, fdToAdd, &event)
-}
-
-func (ep *Epoll) Block() (int, error) {
-	n, err := syscall.EpollWait(ep.fd, ep.events, -1)
-	if err != nil {
-		//goland:noinspection GoDirectComparisonOfErrors
-		if err == syscall.EINTR {
-			return 0, nil //??
-		}
-		return -1, err
-	}
-	return n, nil
-}
-
-func (ep *Epoll) Clear() error {
-	_, err := syscall.Read(int(ep.events[0].Fd), ep.buf[:])
-	return err
-}
-
-func (ep *Epoll) Close() error {
-	if ep.fd != 0 {
-		return unix.Close(ep.fd)
-	}
-	return nil
-}

overlay/packets.go

@@ -1,36 +0,0 @@
-package overlay
-
-//import (
-//	"github.com/slackhq/nebula/util/virtio"
-//)
-
-//type VirtIOPacket struct {
-//	Payload   []byte
-//	Header    virtio.NetHdr
-//	Chains    []uint16
-//	ChainRefs [][]byte
-//}
-//
-//func NewVIO() *VirtIOPacket {
-//	out := new(VirtIOPacket)
-//	out.Payload = nil
-//	out.ChainRefs = make([][]byte, 0, 4)
-//	out.Chains = make([]uint16, 0, 8)
-//	return out
-//}
-//
-//func (v *VirtIOPacket) Reset() {
-//	v.Payload = nil
-//	v.ChainRefs = v.ChainRefs[:0]
-//	v.Chains = v.Chains[:0]
-//}
-
-// TunPacket is formerly VirtIOPacket
-type TunPacket interface {
-	SetPayload([]byte)
-	GetPayload() []byte
-}
-type OutPacket interface {
-	SetPayload([]byte)
-	GetPayload() []byte
-}

overlay/tun.go

@@ -2,50 +2,35 @@ package overlay
 
 import (
 	"fmt"
-	"io"
 	"net"
 	"net/netip"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/util"
 )
 
 const DefaultMTU = 1300
 
-type TunDev interface {
-	io.WriteCloser
-	NewPacketArrays(batchSize int) []TunPacket
-
-	ReadMany(x []TunPacket, q int) (int, error)
-	RecycleRxSeg(pkt TunPacket, kick bool, q int) error
-
-	//todo this interface sux
-	AllocSeg(pkt *packet.OutPacket, q int) (int, error)
-	WriteOne(x *packet.OutPacket, kick bool, q int) (int, error)
-	WriteMany(x []*packet.OutPacket, q int) (int, error)
-}
-
 // TODO: We may be able to remove routines
 type DeviceFactory func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error)
 
 func NewDeviceFromConfig(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
 	switch {
 	case c.GetBool("tun.disabled", false):
-		t := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
+		tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
-		return t, nil
+		return tun, nil
 
 	default:
 		return newTun(c, l, vpnNetworks, routines > 1)
 	}
 }
 
-//func NewFdDeviceFromConfig(fd *int) DeviceFactory {
+func NewFdDeviceFromConfig(fd *int) DeviceFactory {
-//	return func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
+	return func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
-//		return newTunFromFd(c, l, *fd, vpnNetworks)
+		return newTunFromFd(c, l, *fd, vpnNetworks)
-//	}
+	}
-//}
+}
 
 func getAllRoutesFromConfig(c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) {
 	if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") {

overlay/tun_android.go

@@ -95,10 +95,6 @@ func (t *tun) Name() string {
 	return "android"
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
 }

overlay/tun_darwin.go

@@ -549,10 +549,6 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
 }

overlay/tun_disabled.go

@@ -9,8 +9,6 @@ import (
 	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/iputil"
-	"github.com/slackhq/nebula/overlay/virtqueue"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -24,14 +22,6 @@ type disabledTun struct {
 	l  *logrus.Logger
 }
 
-func (t *disabledTun) NewPacketArrays(batchSize int) []TunPacket {
-	panic("implement me") //TODO
-}
-
-func (*disabledTun) RecycleRxSeg(pkt TunPacket, kick bool, q int) error {
-	return nil
-}
-
 func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled bool, l *logrus.Logger) *disabledTun {
 	tun := &disabledTun{
 		vpnNetworks: vpnNetworks,
@@ -50,10 +40,6 @@ func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled boo
 	return tun
 }
 
-func (*disabledTun) GetQueues() []*virtqueue.SplitQueue {
-	return nil
-}
-
 func (*disabledTun) Activate() error {
 	return nil
 }
@@ -119,27 +105,7 @@ func (t *disabledTun) Write(b []byte) (int, error) {
 	return len(b), nil
 }
 
-func (t *disabledTun) SupportsMultiqueue() bool {
+func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return true
-}
-
-func (t *disabledTun) AllocSeg(pkt *packet.OutPacket, q int) (int, error) {
-	return 0, fmt.Errorf("tun_disabled: AllocSeg not implemented")
-}
-
-func (t *disabledTun) WriteOne(x *packet.OutPacket, kick bool, q int) (int, error) {
-	return 0, fmt.Errorf("tun_disabled: WriteOne not implemented")
-}
-
-func (t *disabledTun) WriteMany(x []*packet.OutPacket, q int) (int, error) {
-	return 0, fmt.Errorf("tun_disabled: WriteMany not implemented")
-}
-
-func (t *disabledTun) ReadMany(b []TunPacket, _ int) (int, error) {
-	return t.Read(b[0].GetPayload())
-}
-
-func (t *disabledTun) NewMultiQueueReader() (TunDev, error) {
 	return t, nil
 }
 

overlay/tun_freebsd.go

@@ -450,10 +450,6 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
 }

overlay/tun_ios.go

@@ -151,10 +151,6 @@ func (t *tun) Name() string {
 	return "iOS"
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
 }

overlay/tun_linux.go

@@ -4,11 +4,12 @@
 package overlay
 
 import (
-	"context"
 	"fmt"
+	"io"
 	"net"
 	"net/netip"
 	"os"
+	"runtime"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -17,19 +18,16 @@ import (
 	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/vhostnet"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/util/virtio"
+	wgtun "github.com/slackhq/nebula/wgstack/tun"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
 )
 
 type tun struct {
-	file        *os.File
+	io.ReadWriteCloser
 	fd          int
-	vdev        []*vhostnet.Device
 	Device      string
 	vpnNetworks []netip.Prefix
 	MaxMTU      int
@@ -37,6 +35,7 @@ type tun struct {
 	TXQueueLen  int
 	deviceIndex int
 	ioctlFd     uintptr
+	wgDevice    wgtun.Device
 
 	Routes                    atomic.Pointer[[]Route]
 	routeTree                 atomic.Pointer[bart.Table[routing.Gateways]]
@@ -44,8 +43,7 @@ type tun struct {
 	useSystemRoutes           bool
 	useSystemRoutesBufferSize int
 
-	isV6 bool
+	l *logrus.Logger
-	l    *logrus.Logger
 }
 
 func (t *tun) Networks() []netip.Prefix {
@@ -73,7 +71,9 @@ type ifreqQLEN struct {
 func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
 	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
 
-	t, err := newTunGeneric(c, l, file, vpnNetworks)
+	useWGDefault := runtime.GOOS == "linux"
+	useWG := c.GetBool("tun.use_wireguard_stack", c.GetBool("listen.use_wireguard_stack", useWGDefault))
+	t, err := newTunGeneric(c, l, file, vpnNetworks, useWG)
 	if err != nil {
 		return nil, err
 	}
@@ -107,7 +107,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueu
 	}
 
 	var req ifReq
-	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_TUN_EXCL | unix.IFF_VNET_HDR | unix.IFF_NAPI)
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
 	if multiqueue {
 		req.Flags |= unix.IFF_MULTI_QUEUE
 	}
@@ -117,56 +117,57 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueu
 	}
 	name := strings.Trim(string(req.Name[:]), "\x00")
 
-	if err = unix.SetNonblock(fd, true); err != nil {
-		_ = unix.Close(fd)
-		return nil, fmt.Errorf("make file descriptor non-blocking: %w", err)
-	}
-
 	file := os.NewFile(uintptr(fd), "/dev/net/tun")
-
+	useWGDefault := runtime.GOOS == "linux"
-	err = unix.IoctlSetPointerInt(fd, unix.TUNSETVNETHDRSZ, virtio.NetHdrSize)
+	useWG := c.GetBool("tun.use_wireguard_stack", c.GetBool("listen.use_wireguard_stack", useWGDefault))
-	if err != nil {
+	t, err := newTunGeneric(c, l, file, vpnNetworks, useWG)
-		return nil, fmt.Errorf("set vnethdr size: %w", err)
-	}
-
-	flags := 0
-	//flags = //unix.TUN_F_CSUM //| unix.TUN_F_TSO4 | unix.TUN_F_USO4 | unix.TUN_F_TSO6 | unix.TUN_F_USO6
-	err = unix.IoctlSetInt(fd, unix.TUNSETOFFLOAD, flags)
-	if err != nil {
-		return nil, fmt.Errorf("set offloads: %w", err)
-	}
-
-	t, err := newTunGeneric(c, l, file, vpnNetworks)
 	if err != nil {
 		return nil, err
 	}
-	t.fd = fd
+
 	t.Device = name
 
-	vdev, err := vhostnet.NewDevice(
-		vhostnet.WithBackendFD(fd),
-		vhostnet.WithQueueSize(8192), //todo config
-	)
-	if err != nil {
-		return nil, err
-	}
-	t.vdev = []*vhostnet.Device{vdev}
-
 	return t, nil
 }
 
-func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []netip.Prefix) (*tun, error) {
+func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []netip.Prefix, useWireguard bool) (*tun, error) {
+	var (
+		rw    io.ReadWriteCloser = file
+		fd                       = int(file.Fd())
+		wgDev wgtun.Device
+	)
+
+	if useWireguard {
+		dev, err := wgtun.CreateTUNFromFile(file, c.GetInt("tun.mtu", DefaultMTU))
+		if err != nil {
+			return nil, fmt.Errorf("failed to initialize wireguard tun device: %w", err)
+		}
+		wgDev = dev
+		rw = newWireguardTunIO(dev, c.GetInt("tun.mtu", DefaultMTU))
+		fd = int(dev.File().Fd())
+	}
+
 	t := &tun{
-		file:                      file,
+		ReadWriteCloser:           rw,
-		fd:                        int(file.Fd()),
+		fd:                        fd,
 		vpnNetworks:               vpnNetworks,
 		TXQueueLen:                c.GetInt("tun.tx_queue", 500),
 		useSystemRoutes:           c.GetBool("tun.use_system_route_table", false),
 		useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
 		l:                         l,
 	}
-	if len(vpnNetworks) != 0 {
+	if wgDev != nil {
-		t.isV6 = vpnNetworks[0].Addr().Is6() //todo what about multi-IP?
+		t.wgDevice = wgDev
+	}
+	if wgDev != nil {
+		// replace ioctl fd with device file descriptor to keep route management working
+		file = wgDev.File()
+		t.fd = int(file.Fd())
+		t.ioctlFd = file.Fd()
+	}
+
+	if t.ioctlFd == 0 {
+		t.ioctlFd = file.Fd()
 	}
 
 	err := t.reload(c, true)
@@ -184,14 +185,6 @@ func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []n
 	return t, nil
 }
 
-func (t *tun) NewPacketArrays(batchSize int) []TunPacket {
-	inPackets := make([]TunPacket, batchSize)
-	for i := 0; i < batchSize; i++ {
-		inPackets[i] = vhostnet.NewVIO()
-	}
-	return inPackets
-}
-
 func (t *tun) reload(c *config.C, initial bool) error {
 	routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
 	if err != nil {
@@ -259,11 +252,7 @@ func (t *tun) reload(c *config.C, initial bool) error {
 	return nil
 }
 
-func (t *tun) SupportsMultiqueue() bool {
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return true
-}
-
-func (t *tun) NewMultiQueueReader() (TunDev, error) {
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
@@ -276,17 +265,9 @@ func (t *tun) NewMultiQueueReader() (TunDev, error) {
 		return nil, err
 	}
 
-	vdev, err := vhostnet.NewDevice(
+	file := os.NewFile(uintptr(fd), "/dev/net/tun")
-		vhostnet.WithBackendFD(fd),
-		vhostnet.WithQueueSize(8192), //todo config
-	)
-	if err != nil {
-		return nil, err
-	}
 
-	t.vdev = append(t.vdev, vdev)
+	return file, nil
-
-	return t, nil
 }
 
 func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
@@ -294,6 +275,29 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	return r
 }
 
+func (t *tun) Write(b []byte) (int, error) {
+	var nn int
+	maximum := len(b)
+
+	for {
+		n, err := unix.Write(t.fd, b[nn:maximum])
+		if n > 0 {
+			nn += n
+		}
+		if nn == len(b) {
+			return nn, err
+		}
+
+		if err != nil {
+			return nn, err
+		}
+
+		if n == 0 {
+			return nn, io.ErrUnexpectedEOF
+		}
+	}
+}
+
 func (t *tun) deviceBytes() (o [16]byte) {
 	for i, c := range t.Device {
 		o[i] = byte(c)
@@ -614,42 +618,48 @@ func (t *tun) isGatewayInVpnNetworks(gwAddr netip.Addr) bool {
 }
 
 func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways {
+
 	var gateways routing.Gateways
 
 	link, err := netlink.LinkByName(t.Device)
 	if err != nil {
-		t.l.WithField("deviceName", t.Device).Error("Ignoring route update: failed to get link by name")
+		t.l.WithField("Devicename", t.Device).Error("Ignoring route update: failed to get link by name")
 		return gateways
 	}
 
 	// If this route is relevant to our interface and there is a gateway then add it
-	if r.LinkIndex == link.Attrs().Index {
+	if r.LinkIndex == link.Attrs().Index && len(r.Gw) > 0 {
-		gwAddr, ok := getGatewayAddr(r.Gw, r.Via)
+		gwAddr, ok := netip.AddrFromSlice(r.Gw)
-		if ok {
+		if !ok {
-			if t.isGatewayInVpnNetworks(gwAddr) {
+			t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway address")
-				gateways = append(gateways, routing.NewGateway(gwAddr, 1))
-			} else {
-				// Gateway isn't in our overlay network, ignore
-				t.l.WithField("route", r).Debug("Ignoring route update, gateway is not in our network")
-			}
 		} else {
-			t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway or via address")
+			gwAddr = gwAddr.Unmap()
+
+			if !t.isGatewayInVpnNetworks(gwAddr) {
+				// Gateway isn't in our overlay network, ignore
+				t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
+			} else {
+				gateways = append(gateways, routing.NewGateway(gwAddr, 1))
+			}
 		}
 	}
 
 	for _, p := range r.MultiPath {
 		// If this route is relevant to our interface and there is a gateway then add it
-		if p.LinkIndex == link.Attrs().Index {
+		if p.LinkIndex == link.Attrs().Index && len(p.Gw) > 0 {
-			gwAddr, ok := getGatewayAddr(p.Gw, p.Via)
+			gwAddr, ok := netip.AddrFromSlice(p.Gw)
-			if ok {
+			if !ok {
-				if t.isGatewayInVpnNetworks(gwAddr) {
+				t.l.WithField("route", r).Debug("Ignoring multipath route update, invalid gateway address")
-					gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1))
-				} else {
-					// Gateway isn't in our overlay network, ignore
-					t.l.WithField("route", r).Debug("Ignoring route update, gateway is not in our network")
-				}
 			} else {
-				t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway or via address")
+				gwAddr = gwAddr.Unmap()
+
+				if !t.isGatewayInVpnNetworks(gwAddr) {
+					// Gateway isn't in our overlay network, ignore
+					t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
+				} else {
+					// p.Hops+1 = weight of the route
+					gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1))
+				}
 			}
 		}
 	}
@@ -658,27 +668,10 @@ func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways {
 	return gateways
 }
 
-func getGatewayAddr(gw net.IP, via netlink.Destination) (netip.Addr, bool) {
-	// Try to use the old RTA_GATEWAY first
-	gwAddr, ok := netip.AddrFromSlice(gw)
-	if !ok {
-		// Fallback to the new RTA_VIA
-		rVia, ok := via.(*netlink.Via)
-		if ok {
-			gwAddr, ok = netip.AddrFromSlice(rVia.Addr)
-		}
-	}
-
-	if gwAddr.IsValid() {
-		gwAddr = gwAddr.Unmap()
-		return gwAddr, true
-	}
-
-	return netip.Addr{}, false
-}
-
 func (t *tun) updateRoutes(r netlink.RouteUpdate) {
+
 	gateways := t.getGatewaysFromRoute(&r.Route)
+
 	if len(gateways) == 0 {
 		// No gateways relevant to our network, no routing changes required.
 		t.l.WithField("route", r).Debug("Ignoring route update, no gateways")
@@ -717,14 +710,16 @@ func (t *tun) Close() error {
 		close(t.routeChan)
 	}
 
-	for _, v := range t.vdev {
+	if t.ReadWriteCloser != nil {
-		if v != nil {
+		_ = t.ReadWriteCloser.Close()
-			_ = v.Close()
-		}
 	}
 
-	if t.file != nil {
+	if t.wgDevice != nil {
-		_ = t.file.Close()
+		_ = t.wgDevice.Close()
+		if t.ioctlFd > 0 {
+			// underlying fd already closed by the device
+			t.ioctlFd = 0
+		}
 	}
 
 	if t.ioctlFd > 0 {
@@ -733,84 +728,3 @@ func (t *tun) Close() error {
 
 	return nil
 }
-
-func (t *tun) ReadMany(p []TunPacket, q int) (int, error) {
-	err := t.vdev[q].ReceiveQueue.WaitForUsedElements(context.TODO())
-	if err != nil {
-		return 0, err
-	}
-	i := 0
-	for i = 0; i < len(p); i++ {
-		item, ok := t.vdev[q].ReceiveQueue.TakeSingleNoBlock()
-		if !ok {
-			break
-		}
-		pkt := p[i].(*vhostnet.VirtIOPacket) //todo I'm not happy about this but I don't want to change how memory is "owned" rn
-		_, err = t.vdev[q].ProcessRxChain(pkt, item)
-		if err != nil {
-			return i, err
-		}
-		i++
-	}
-	return i, nil
-}
-
-func (t *tun) Write(b []byte) (int, error) {
-	maximum := len(b) //we are RXing
-
-	//todo garbagey
-	out := packet.NewOut()
-	x, err := t.AllocSeg(out, 0)
-	if err != nil {
-		return 0, err
-	}
-	copy(out.SegmentPayloads[x], b)
-	err = t.vdev[0].TransmitPacket(out, true)
-
-	if err != nil {
-		t.l.WithError(err).Error("Transmitting packet")
-		return 0, err
-	}
-	return maximum, nil
-}
-
-func (t *tun) AllocSeg(pkt *packet.OutPacket, q int) (int, error) {
-	idx, buf, err := t.vdev[q].GetPacketForTx()
-	if err != nil {
-		return 0, err
-	}
-	x := pkt.UseSegment(idx, buf, t.isV6)
-	return x, nil
-}
-
-func (t *tun) WriteOne(x *packet.OutPacket, kick bool, q int) (int, error) {
-	if err := t.vdev[q].TransmitPacket(x, kick); err != nil {
-		t.l.WithError(err).Error("Transmitting packet")
-		return 0, err
-	}
-	return 1, nil
-}
-
-func (t *tun) WriteMany(x []*packet.OutPacket, q int) (int, error) {
-	maximum := len(x) //we are RXing
-	if maximum == 0 {
-		return 0, nil
-	}
-
-	err := t.vdev[q].TransmitPackets(x)
-	if err != nil {
-		t.l.WithError(err).Error("Transmitting packet")
-		return 0, err
-	}
-	return maximum, nil
-}
-
-func (t *tun) RecycleRxSeg(pkt TunPacket, kick bool, q int) error {
-	if pkt.GetPayload() == nil {
-		return nil
-	}
-	vpkt := pkt.(*vhostnet.VirtIOPacket)
-	err := t.vdev[q].ReceiveQueue.OfferDescriptorChains([]uint16{vpkt.Chain}, kick)
-	vpkt.Reset() //intentionally ignoring err!
-	return err
-}

overlay/tun_linux_batch.go

@@ -0,0 +1,56 @@
+//go:build linux && !android && !e2e_testing
+
+package overlay
+
+import "fmt"
+
+func (t *tun) batchIO() (*wireguardTunIO, bool) {
+	io, ok := t.ReadWriteCloser.(*wireguardTunIO)
+	return io, ok
+}
+
+func (t *tun) ReadIntoBatch(pool *PacketPool) ([]*Packet, error) {
+	io, ok := t.batchIO()
+	if !ok {
+		return nil, fmt.Errorf("wireguard batch I/O not enabled")
+	}
+	return io.ReadIntoBatch(pool)
+}
+
+func (t *tun) WriteBatch(packets []*Packet) (int, error) {
+	io, ok := t.batchIO()
+	if ok {
+		return io.WriteBatch(packets)
+	}
+	for _, pkt := range packets {
+		if pkt == nil {
+			continue
+		}
+		if _, err := t.Write(pkt.Payload()[:pkt.Len]); err != nil {
+			return 0, err
+		}
+		pkt.Release()
+	}
+	return len(packets), nil
+}
+
+func (t *tun) BatchHeadroom() int {
+	if io, ok := t.batchIO(); ok {
+		return io.BatchHeadroom()
+	}
+	return 0
+}
+
+func (t *tun) BatchPayloadCap() int {
+	if io, ok := t.batchIO(); ok {
+		return io.BatchPayloadCap()
+	}
+	return 0
+}
+
+func (t *tun) BatchSize() int {
+	if io, ok := t.batchIO(); ok {
+		return io.BatchSize()
+	}
+	return 1
+}

overlay/tun_netbsd.go

@@ -390,10 +390,6 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd")
 }

overlay/tun_openbsd.go

@@ -310,10 +310,6 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd")
 }

overlay/tun_tester.go

@@ -13,7 +13,6 @@ import (
 	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -27,7 +26,6 @@ type TestTun struct {
 	closed    atomic.Bool
 	rxPackets chan []byte // Packets to receive into nebula
 	TxPackets chan []byte // Packets transmitted outside by nebula
-	buffers   [][]byte
 }
 
 func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) {
@@ -106,68 +104,15 @@ func (t *TestTun) Name() string {
 	return t.Device
 }
 
-func (t *TestTun) ReadMany(x []TunPacket, q int) (int, error) {
+func (t *TestTun) Write(b []byte) (n int, err error) {
-	p, ok := <-t.rxPackets
-	if !ok {
-		return 0, os.ErrClosed
-	}
-	x[0].Payload = p
-	return 1, nil
-}
-
-func (t *TestTun) AllocSeg(pkt *packet.OutPacket, q int) (int, error) {
-	buf := make([]byte, 9000)
-	t.buffers = append(t.buffers, buf)
-	idx := len(t.buffers) - 1
-	isV6 := false //todo?
-	x := pkt.UseSegment(uint16(idx), buf, isV6)
-	return x, nil
-}
-
-func (t *TestTun) Write(b []byte) (int, error) {
-	//todo garbagey
-	out := packet.NewOut()
-	x, err := t.AllocSeg(out, 0)
-	if err != nil {
-		return 0, err
-	}
-	copy(out.SegmentPayloads[x], b)
-	return t.WriteOne(out, true, 0)
-}
-
-func (t *TestTun) WriteOne(x *packet.OutPacket, kick bool, q int) (int, error) {
 	if t.closed.Load() {
 		return 0, io.ErrClosedPipe
 	}
-	if len(x.SegmentIDs) == 0 {
-		return 0, nil
-	}
-	for i, _ := range x.SegmentIDs {
-		t.TxPackets <- x.SegmentPayloads[i]
-	}
-	//todo if kick, delete alloced seg
 
-	return 1, nil
+	packet := make([]byte, len(b), len(b))
-}
+	copy(packet, b)
-
+	t.TxPackets <- packet
-func (t *TestTun) WriteMany(x []*packet.OutPacket, q int) (int, error) {
+	return len(b), nil
-	if len(x) == 0 {
-		return 0, nil
-	}
-
-	for _, pkt := range x {
-		_, err := t.WriteOne(pkt, true, q)
-		if err != nil {
-			return 0, err
-		}
-	}
-
-	return len(x), nil
-}
-
-func (t *TestTun) RecycleRxSeg(pkt *TunPacket, kick bool, q int) error {
-	//todo this ought to maybe track something
-	return nil
 }
 
 func (t *TestTun) Close() error {
@@ -178,10 +123,15 @@ func (t *TestTun) Close() error {
 	return nil
 }
 
-func (t *TestTun) SupportsMultiqueue() bool {
+func (t *TestTun) Read(b []byte) (int, error) {
-	return false
+	p, ok := <-t.rxPackets
+	if !ok {
+		return 0, os.ErrClosed
+	}
+	copy(b, p)
+	return len(p), nil
 }
 
-func (t *TestTun) NewMultiQueueReader() (TunDev, error) {
+func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented")
 }

overlay/tun_windows.go

@@ -234,10 +234,6 @@ func (t *winTun) Write(b []byte) (int, error) {
 	return t.tun.Write(b, 0)
 }
 
-func (t *winTun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
 }

overlay/user.go

@@ -1,13 +1,11 @@
 package overlay
 
 import (
-	"fmt"
 	"io"
 	"net/netip"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -38,21 +36,6 @@ type UserDevice struct {
 	inboundWriter *io.PipeWriter
 }
 
-func (d *UserDevice) NewPacketArrays(batchSize int) []TunPacket {
-	//inPackets := make([]TunPacket, batchSize)
-	//outPackets := make([]OutPacket, batchSize)
-	panic("not implemented") //todo!
-	//for i := 0; i < batchSize; i++ {
-	//	inPackets[i] = vhostnet.NewVIO()
-	//	outPackets[i] = packet.New(false)
-	//}
-	//return inPackets, outPackets
-}
-
-func (d *UserDevice) RecycleRxSeg(pkt TunPacket, kick bool, q int) error {
-	return nil
-}
-
 func (d *UserDevice) Activate() error {
 	return nil
 }
@@ -63,11 +46,7 @@ func (d *UserDevice) RoutesFor(ip netip.Addr) routing.Gateways {
 	return routing.Gateways{routing.NewGateway(ip, 1)}
 }
 
-func (d *UserDevice) SupportsMultiqueue() bool {
+func (d *UserDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	return true
-}
-
-func (d *UserDevice) NewMultiQueueReader() (TunDev, error) {
 	return d, nil
 }
 
@@ -86,23 +65,3 @@ func (d *UserDevice) Close() error {
 	d.outboundWriter.Close()
 	return nil
 }
-
-func (d *UserDevice) ReadMany(b []TunPacket, _ int) (int, error) {
-	_, err := d.Read(b[0].GetPayload())
-	if err != nil {
-		return 0, err
-	}
-	return 1, nil
-}
-
-func (d *UserDevice) AllocSeg(pkt *packet.OutPacket, q int) (int, error) {
-	return 0, fmt.Errorf("user: AllocSeg not implemented")
-}
-
-func (d *UserDevice) WriteOne(x *packet.OutPacket, kick bool, q int) (int, error) {
-	return 0, fmt.Errorf("user: WriteOne not implemented")
-}
-
-func (d *UserDevice) WriteMany(x []*packet.OutPacket, q int) (int, error) {
-	return 0, fmt.Errorf("user: WriteMany not implemented")
-}

overlay/vhost/README.md

@@ -1,23 +0,0 @@
-Significant portions of this code are derived from https://pkg.go.dev/github.com/hetznercloud/virtio-go
-
-MIT License
-
-Copyright (c) 2025 Hetzner Cloud GmbH
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

overlay/vhost/doc.go

@@ -1,4 +0,0 @@
-// Package vhost implements the basic ioctl requests needed to interact with the
-// kernel-level virtio server that provides accelerated virtio devices for
-// networking and more.
-package vhost

overlay/vhost/ioctl.go

@@ -1,218 +0,0 @@
-package vhost
-
-import (
-	"fmt"
-	"unsafe"
-
-	"github.com/slackhq/nebula/overlay/virtqueue"
-	"github.com/slackhq/nebula/util/virtio"
-	"golang.org/x/sys/unix"
-)
-
-const (
-	// vhostIoctlGetFeatures can be used to retrieve the features supported by
-	// the vhost implementation in the kernel.
-	//
-	// Response payload: [virtio.Feature]
-	// Kernel name: VHOST_GET_FEATURES
-	vhostIoctlGetFeatures = 0x8008af00
-
-	// vhostIoctlSetFeatures can be used to communicate the features supported
-	// by this virtio implementation to the kernel.
-	//
-	// Request payload: [virtio.Feature]
-	// Kernel name: VHOST_SET_FEATURES
-	vhostIoctlSetFeatures = 0x4008af00
-
-	// vhostIoctlSetOwner can be used to set the current process as the
-	// exclusive owner of a control file descriptor.
-	//
-	// Request payload: none
-	// Kernel name: VHOST_SET_OWNER
-	vhostIoctlSetOwner = 0x0000af01
-
-	// vhostIoctlSetMemoryLayout can be used to set up or modify the memory
-	// layout which describes the IOTLB mappings in the kernel.
-	//
-	// Request payload: [MemoryLayout] with custom serialization
-	// Kernel name: VHOST_SET_MEM_TABLE
-	vhostIoctlSetMemoryLayout = 0x4008af03
-
-	// vhostIoctlSetQueueSize can be used to set the size of the virtqueue.
-	//
-	// Request payload: [QueueState]
-	// Kernel name: VHOST_SET_VRING_NUM
-	vhostIoctlSetQueueSize = 0x4008af10
-
-	// vhostIoctlSetQueueAddress can be used to set the addresses of the
-	// different parts of the virtqueue.
-	//
-	// Request payload: [QueueAddresses]
-	// Kernel name: VHOST_SET_VRING_ADDR
-	vhostIoctlSetQueueAddress = 0x4028af11
-
-	// vhostIoctlSetAvailableRingBase can be used to set the index of the next
-	// available ring entry the device will process.
-	//
-	// Request payload: [QueueState]
-	// Kernel name: VHOST_SET_VRING_BASE
-	vhostIoctlSetAvailableRingBase = 0x4008af12
-
-	// vhostIoctlSetQueueKickEventFD can be used to set the event file
-	// descriptor to signal the device when descriptor chains were added to the
-	// available ring.
-	//
-	// Request payload: [QueueFile]
-	// Kernel name: VHOST_SET_VRING_KICK
-	vhostIoctlSetQueueKickEventFD = 0x4008af20
-
-	// vhostIoctlSetQueueCallEventFD can be used to set the event file
-	// descriptor that gets signaled by the device when descriptor chains have
-	// been used by it.
-	//
-	// Request payload: [QueueFile]
-	// Kernel name: VHOST_SET_VRING_CALL
-	vhostIoctlSetQueueCallEventFD = 0x4008af21
-)
-
-// QueueState is an ioctl request payload that can hold a queue index and any
-// 32-bit number.
-//
-// Kernel name: vhost_vring_state
-type QueueState struct {
-	// QueueIndex is the index of the virtqueue.
-	QueueIndex uint32
-	// Num is any 32-bit number, depending on the request.
-	Num uint32
-}
-
-// QueueAddresses is an ioctl request payload that can hold the addresses of the
-// different parts of a virtqueue.
-//
-// Kernel name: vhost_vring_addr
-type QueueAddresses struct {
-	// QueueIndex is the index of the virtqueue.
-	QueueIndex uint32
-	// Flags that are not used in this implementation.
-	Flags uint32
-	// DescriptorTableAddress is the address of the descriptor table in user
-	// space memory. It must be 16-byte aligned.
-	DescriptorTableAddress uintptr
-	// UsedRingAddress is the address of the used ring in user space memory. It
-	// must be 4-byte aligned.
-	UsedRingAddress uintptr
-	// AvailableRingAddress is the address of the available ring in user space
-	// memory. It must be 2-byte aligned.
-	AvailableRingAddress uintptr
-	// LogAddress is used for an optional logging support, not supported by this
-	// implementation.
-	LogAddress uintptr
-}
-
-// QueueFile is an ioctl request payload that can hold a queue index and a file
-// descriptor.
-//
-// Kernel name: vhost_vring_file
-type QueueFile struct {
-	// QueueIndex is the index of the virtqueue.
-	QueueIndex uint32
-	// FD is the file descriptor of the file. Pass -1 to unbind from a file.
-	FD int32
-}
-
-// IoctlPtr is a copy of the similarly named unexported function from the Go
-// unix package. This is needed to do custom ioctl requests not supported by the
-// standard library.
-func IoctlPtr(fd int, req uint, arg unsafe.Pointer) error {
-	_, _, err := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg))
-	if err != 0 {
-		return fmt.Errorf("ioctl request %d: %w", req, err)
-	}
-	return nil
-}
-
-// GetFeatures requests the supported feature bits from the virtio device
-// associated with the given control file descriptor.
-func GetFeatures(controlFD int) (virtio.Feature, error) {
-	var features virtio.Feature
-	if err := IoctlPtr(controlFD, vhostIoctlGetFeatures, unsafe.Pointer(&features)); err != nil {
-		return 0, fmt.Errorf("get features: %w", err)
-	}
-	return features, nil
-}
-
-// SetFeatures communicates the feature bits supported by this implementation
-// to the virtio device associated with the given control file descriptor.
-func SetFeatures(controlFD int, features virtio.Feature) error {
-	if err := IoctlPtr(controlFD, vhostIoctlSetFeatures, unsafe.Pointer(&features)); err != nil {
-		return fmt.Errorf("set features: %w", err)
-	}
-	return nil
-}
-
-// OwnControlFD sets the current process as the exclusive owner for the
-// given control file descriptor. This must be called before interacting with
-// the control file descriptor in any other way.
-func OwnControlFD(controlFD int) error {
-	if err := IoctlPtr(controlFD, vhostIoctlSetOwner, unsafe.Pointer(nil)); err != nil {
-		return fmt.Errorf("set control file descriptor owner: %w", err)
-	}
-	return nil
-}
-
-// SetMemoryLayout sets up or modifies the memory layout for the kernel-level
-// virtio device associated with the given control file descriptor.
-func SetMemoryLayout(controlFD int, layout MemoryLayout) error {
-	payload := layout.serializePayload()
-	if err := IoctlPtr(controlFD, vhostIoctlSetMemoryLayout, unsafe.Pointer(&payload[0])); err != nil {
-		return fmt.Errorf("set memory layout: %w", err)
-	}
-	return nil
-}
-
-// RegisterQueue registers a virtio queue with the kernel-level virtio server.
-// The virtqueue will be linked to the given control file descriptor and will
-// have the given index. The kernel will use this queue until the control file
-// descriptor is closed.
-func RegisterQueue(controlFD int, queueIndex uint32, queue *virtqueue.SplitQueue) error {
-	if err := IoctlPtr(controlFD, vhostIoctlSetQueueSize, unsafe.Pointer(&QueueState{
-		QueueIndex: queueIndex,
-		Num:        uint32(queue.Size()),
-	})); err != nil {
-		return fmt.Errorf("set queue size: %w", err)
-	}
-
-	if err := IoctlPtr(controlFD, vhostIoctlSetQueueAddress, unsafe.Pointer(&QueueAddresses{
-		QueueIndex:             queueIndex,
-		Flags:                  0,
-		DescriptorTableAddress: queue.DescriptorTable().Address(),
-		UsedRingAddress:        queue.UsedRing().Address(),
-		AvailableRingAddress:   queue.AvailableRing().Address(),
-		LogAddress:             0,
-	})); err != nil {
-		return fmt.Errorf("set queue addresses: %w", err)
-	}
-
-	if err := IoctlPtr(controlFD, vhostIoctlSetAvailableRingBase, unsafe.Pointer(&QueueState{
-		QueueIndex: queueIndex,
-		Num:        0,
-	})); err != nil {
-		return fmt.Errorf("set available ring base: %w", err)
-	}
-
-	if err := IoctlPtr(controlFD, vhostIoctlSetQueueKickEventFD, unsafe.Pointer(&QueueFile{
-		QueueIndex: queueIndex,
-		FD:         int32(queue.KickEventFD()),
-	})); err != nil {
-		return fmt.Errorf("set kick event file descriptor: %w", err)
-	}
-
-	if err := IoctlPtr(controlFD, vhostIoctlSetQueueCallEventFD, unsafe.Pointer(&QueueFile{
-		QueueIndex: queueIndex,
-		FD:         int32(queue.CallEventFD()),
-	})); err != nil {
-		return fmt.Errorf("set call event file descriptor: %w", err)
-	}
-
-	return nil
-}

overlay/vhost/ioctl_test.go

@@ -1,21 +0,0 @@
-package vhost_test
-
-import (
-	"testing"
-	"unsafe"
-
-	"github.com/slackhq/nebula/overlay/vhost"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestQueueState_Size(t *testing.T) {
-	assert.EqualValues(t, 8, unsafe.Sizeof(vhost.QueueState{}))
-}
-
-func TestQueueAddresses_Size(t *testing.T) {
-	assert.EqualValues(t, 40, unsafe.Sizeof(vhost.QueueAddresses{}))
-}
-
-func TestQueueFile_Size(t *testing.T) {
-	assert.EqualValues(t, 8, unsafe.Sizeof(vhost.QueueFile{}))
-}

overlay/vhost/memory.go

@@ -1,73 +0,0 @@
-package vhost
-
-import (
-	"encoding/binary"
-	"fmt"
-	"unsafe"
-
-	"github.com/slackhq/nebula/overlay/virtqueue"
-)
-
-// MemoryRegion describes a region of userspace memory which is being made
-// accessible to a vhost device.
-//
-// Kernel name: vhost_memory_region
-type MemoryRegion struct {
-	// GuestPhysicalAddress is the physical address of the memory region within
-	// the guest, when virtualization is used. When no virtualization is used,
-	// this should be the same as UserspaceAddress.
-	GuestPhysicalAddress uintptr
-	// Size is the size of the memory region.
-	Size uint64
-	// UserspaceAddress is the virtual address in the userspace of the host
-	// where the memory region can be found.
-	UserspaceAddress uintptr
-	// Padding and room for flags. Currently unused.
-	_ uint64
-}
-
-// MemoryLayout is a list of [MemoryRegion]s.
-type MemoryLayout []MemoryRegion
-
-// NewMemoryLayoutForQueues returns a new [MemoryLayout] that describes the
-// memory pages used by the descriptor tables of the given queues.
-func NewMemoryLayoutForQueues(queues []*virtqueue.SplitQueue) MemoryLayout {
-	regions := make([]MemoryRegion, 0)
-	for _, queue := range queues {
-		for address, size := range queue.DescriptorTable().BufferAddresses() {
-			regions = append(regions, MemoryRegion{
-				// There is no virtualization in play here, so the guest address
-				// is the same as in the host's userspace.
-				GuestPhysicalAddress: address,
-				Size:                 uint64(size),
-				UserspaceAddress:     address,
-			})
-		}
-	}
-	return regions
-}
-
-// serializePayload serializes the list of memory regions into a format that is
-// compatible to the vhost_memory kernel struct. The returned byte slice can be
-// used as a payload for the vhostIoctlSetMemoryLayout ioctl.
-func (regions MemoryLayout) serializePayload() []byte {
-	regionCount := len(regions)
-	regionSize := int(unsafe.Sizeof(MemoryRegion{}))
-	payload := make([]byte, 8+regionCount*regionSize)
-
-	// The first 32 bits contain the number of memory regions. The following 32
-	// bits are padding.
-	binary.LittleEndian.PutUint32(payload[0:4], uint32(regionCount))
-
-	if regionCount > 0 {
-		// The underlying byte array of the slice should already have the correct
-		// format, so just copy that.
-		copied := copy(payload[8:], unsafe.Slice((*byte)(unsafe.Pointer(&regions[0])), regionCount*regionSize))
-		if copied != regionCount*regionSize {
-			panic(fmt.Sprintf("copied only %d bytes of the memory regions, but expected %d",
-				copied, regionCount*regionSize))
-		}
-	}
-
-	return payload
-}

overlay/vhost/memory_internal_test.go

@@ -1,42 +0,0 @@
-package vhost
-
-import (
-	"testing"
-	"unsafe"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestMemoryRegion_Size(t *testing.T) {
-	assert.EqualValues(t, 32, unsafe.Sizeof(MemoryRegion{}))
-}
-
-func TestMemoryLayout_SerializePayload(t *testing.T) {
-	layout := MemoryLayout([]MemoryRegion{
-		{
-			GuestPhysicalAddress: 42,
-			Size:                 100,
-			UserspaceAddress:     142,
-		}, {
-			GuestPhysicalAddress: 99,
-			Size:                 100,
-			UserspaceAddress:     99,
-		},
-	})
-	payload := layout.serializePayload()
-
-	assert.Equal(t, []byte{
-		0x02, 0x00, 0x00, 0x00, // nregions
-		0x00, 0x00, 0x00, 0x00, // padding
-		// region 0
-		0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // guest_phys_addr
-		0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // memory_size
-		0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // userspace_addr
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // flags_padding
-		// region 1
-		0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // guest_phys_addr
-		0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // memory_size
-		0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // userspace_addr
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // flags_padding
-	}, payload)
-}

overlay/vhostnet/README.md

@@ -1,23 +0,0 @@
-Significant portions of this code are derived from https://pkg.go.dev/github.com/hetznercloud/virtio-go
-
-MIT License
-
-Copyright (c) 2025 Hetzner Cloud GmbH
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

overlay/vhostnet/device.go

@@ -1,339 +0,0 @@
-package vhostnet
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"os"
-	"runtime"
-
-	"github.com/slackhq/nebula/overlay/vhost"
-	"github.com/slackhq/nebula/overlay/virtqueue"
-	"github.com/slackhq/nebula/packet"
-	"github.com/slackhq/nebula/util/virtio"
-	"golang.org/x/sys/unix"
-)
-
-// ErrDeviceClosed is returned when the [Device] is closed while operations are
-// still running.
-var ErrDeviceClosed = errors.New("device was closed")
-
-// The indexes for the receive and transmit queues.
-const (
-	receiveQueueIndex  = 0
-	transmitQueueIndex = 1
-)
-
-// Device represents a vhost networking device within the kernel-level virtio
-// implementation and provides methods to interact with it.
-type Device struct {
-	controlFD int
-
-	ReceiveQueue  *virtqueue.SplitQueue
-	TransmitQueue *virtqueue.SplitQueue
-}
-
-// NewDevice initializes a new vhost networking device within the
-// kernel-level virtio implementation, sets up the virtqueues and returns a
-// [Device] instance that can be used to communicate with that vhost device.
-//
-// There are multiple options that can be passed to this constructor to
-// influence device creation:
-//   - [WithQueueSize]
-//   - [WithBackendFD]
-//   - [WithBackendDevice]
-//
-// Remember to call [Device.Close] after use to free up resources.
-func NewDevice(options ...Option) (*Device, error) {
-	var err error
-	opts := optionDefaults
-	opts.apply(options)
-	if err = opts.validate(); err != nil {
-		return nil, fmt.Errorf("invalid options: %w", err)
-	}
-
-	dev := Device{
-		controlFD: -1,
-	}
-
-	// Clean up a partially initialized device when something fails.
-	defer func() {
-		if err != nil {
-			_ = dev.Close()
-		}
-	}()
-
-	// Retrieve a new control file descriptor. This will be used to configure
-	// the vhost networking device in the kernel.
-	dev.controlFD, err = unix.Open("/dev/vhost-net", os.O_RDWR, 0666)
-	if err != nil {
-		return nil, fmt.Errorf("get control file descriptor: %w", err)
-	}
-	if err = vhost.OwnControlFD(dev.controlFD); err != nil {
-		return nil, fmt.Errorf("own control file descriptor: %w", err)
-	}
-
-	// Advertise the supported features. This isn't much for now.
-	// TODO: Add feature options and implement proper feature negotiation.
-	getFeatures, err := vhost.GetFeatures(dev.controlFD) //0x1033D008000 but why
-	if err != nil {
-		return nil, fmt.Errorf("get features: %w", err)
-	}
-	if getFeatures == 0 {
-
-	}
-	//const funky = virtio.Feature(1 << 27)
-	//features := virtio.FeatureVersion1 | funky // | todo virtio.FeatureNetMergeRXBuffers
-	features := virtio.FeatureVersion1 | virtio.FeatureNetMergeRXBuffers
-	if err = vhost.SetFeatures(dev.controlFD, features); err != nil {
-		return nil, fmt.Errorf("set features: %w", err)
-	}
-
-	itemSize := os.Getpagesize() * 4 //todo config
-
-	// Initialize and register the queues needed for the networking device.
-	if dev.ReceiveQueue, err = createQueue(dev.controlFD, receiveQueueIndex, opts.queueSize, itemSize); err != nil {
-		return nil, fmt.Errorf("create receive queue: %w", err)
-	}
-	if dev.TransmitQueue, err = createQueue(dev.controlFD, transmitQueueIndex, opts.queueSize, itemSize); err != nil {
-		return nil, fmt.Errorf("create transmit queue: %w", err)
-	}
-
-	// Set up memory mappings for all buffers used by the queues. This has to
-	// happen before a backend for the queues can be registered.
-	memoryLayout := vhost.NewMemoryLayoutForQueues(
-		[]*virtqueue.SplitQueue{dev.ReceiveQueue, dev.TransmitQueue},
-	)
-	if err = vhost.SetMemoryLayout(dev.controlFD, memoryLayout); err != nil {
-		return nil, fmt.Errorf("setup memory layout: %w", err)
-	}
-
-	// Set the queue backends. This activates the queues within the kernel.
-	if err = SetQueueBackend(dev.controlFD, receiveQueueIndex, opts.backendFD); err != nil {
-		return nil, fmt.Errorf("set receive queue backend: %w", err)
-	}
-	if err = SetQueueBackend(dev.controlFD, transmitQueueIndex, opts.backendFD); err != nil {
-		return nil, fmt.Errorf("set transmit queue backend: %w", err)
-	}
-
-	// Fully populate the rx queue with available buffers which the device
-	// can write new packets into.
-	if err = dev.refillReceiveQueue(); err != nil {
-		return nil, fmt.Errorf("refill receive queue: %w", err)
-	}
-	if err = dev.prefillTxQueue(); err != nil {
-		return nil, fmt.Errorf("prefill tx queue: %w", err)
-	}
-
-	// Make sure to clean up even when the device gets garbage collected without
-	// Close being called first.
-	devPtr := &dev
-	runtime.SetFinalizer(devPtr, (*Device).Close)
-
-	return devPtr, nil
-}
-
-// refillReceiveQueue offers as many new device-writable buffers to the device
-// as the queue can fit. The device will then use these to write received
-// packets.
-func (dev *Device) refillReceiveQueue() error {
-	for {
-		_, err := dev.ReceiveQueue.OfferInDescriptorChains()
-		if err != nil {
-			if errors.Is(err, virtqueue.ErrNotEnoughFreeDescriptors) {
-				// Queue is full, job is done.
-				return nil
-			}
-			return fmt.Errorf("offer descriptor chain: %w", err)
-		}
-	}
-}
-
-func (dev *Device) prefillTxQueue() error {
-	for {
-		dt := dev.TransmitQueue.DescriptorTable()
-		for {
-			x, _, err := dt.CreateDescriptorForOutputs()
-			if err != nil {
-				if errors.Is(err, virtqueue.ErrNotEnoughFreeDescriptors) {
-					// Queue is full, job is done.
-					return nil
-				}
-				return err
-			}
-			err = dev.TransmitQueue.OfferDescriptorChains([]uint16{x}, false)
-			if err != nil {
-				return err
-			}
-		}
-
-	}
-}
-
-// Close cleans up the vhost networking device within the kernel and releases
-// all resources used for it.
-// The implementation will try to release as many resources as possible and
-// collect potential errors before returning them.
-func (dev *Device) Close() error {
-	// Closing the control file descriptor will unregister all queues from the
-	// kernel.
-	if dev.controlFD >= 0 {
-		if err := unix.Close(dev.controlFD); err != nil {
-			// Return an error and do not continue, because the memory used for
-			// the queues should not be released before they were unregistered
-			// from the kernel.
-			return fmt.Errorf("close control file descriptor: %w", err)
-		}
-		dev.controlFD = -1
-	}
-
-	var errs []error
-
-	if dev.ReceiveQueue != nil {
-		if err := dev.ReceiveQueue.Close(); err == nil {
-			dev.ReceiveQueue = nil
-		} else {
-			errs = append(errs, fmt.Errorf("close receive queue: %w", err))
-		}
-	}
-
-	if dev.TransmitQueue != nil {
-		if err := dev.TransmitQueue.Close(); err == nil {
-			dev.TransmitQueue = nil
-		} else {
-			errs = append(errs, fmt.Errorf("close transmit queue: %w", err))
-		}
-	}
-
-	if len(errs) == 0 {
-		// Everything was cleaned up. No need to run the finalizer anymore.
-		runtime.SetFinalizer(dev, nil)
-	}
-
-	return errors.Join(errs...)
-}
-
-// createQueue creates a new virtqueue and registers it with the vhost device
-// using the given index.
-func createQueue(controlFD int, queueIndex int, queueSize int, itemSize int) (*virtqueue.SplitQueue, error) {
-	queue, err := virtqueue.NewSplitQueue(queueSize, itemSize)
-	if err != nil {
-		return nil, fmt.Errorf("create virtqueue: %w", err)
-	}
-	if err = vhost.RegisterQueue(controlFD, uint32(queueIndex), queue); err != nil {
-		return nil, fmt.Errorf("register virtqueue with index %d: %w", queueIndex, err)
-	}
-	return queue, nil
-}
-
-func (dev *Device) GetPacketForTx() (uint16, []byte, error) {
-	idx, err := dev.TransmitQueue.TakeSingleIndex(context.TODO())
-	if err != nil {
-		return 0, nil, fmt.Errorf("transmit queue: %w", err)
-	}
-	buf := dev.TransmitQueue.GetDescriptorItem(idx)
-	return idx, buf, nil
-}
-
-func (dev *Device) TransmitPacket(pkt *packet.OutPacket, kick bool) error {
-	if len(pkt.SegmentIDs) == 0 {
-		return nil
-	}
-	for idx := range pkt.SegmentIDs {
-		segmentID := pkt.SegmentIDs[idx]
-		dev.TransmitQueue.SetDescSize(segmentID, len(pkt.Segments[idx]))
-	}
-	err := dev.TransmitQueue.OfferDescriptorChains(pkt.SegmentIDs, false)
-	if err != nil {
-		return fmt.Errorf("offer descriptor chains: %w", err)
-	}
-	pkt.Reset()
-	if kick {
-		return dev.TransmitQueue.Kick()
-	}
-
-	return nil
-}
-
-func (dev *Device) TransmitPackets(pkts []*packet.OutPacket) error {
-	if len(pkts) == 0 {
-		return nil
-	}
-
-	for i := range pkts {
-		if err := dev.TransmitPacket(pkts[i], false); err != nil {
-			return err
-		}
-	}
-	if err := dev.TransmitQueue.Kick(); err != nil {
-		return err
-	}
-	return nil
-}
-
-// ProcessRxChain processes a single chain to create one packet. The number of processed chains is returned.
-func (dev *Device) ProcessRxChain(pkt *VirtIOPacket, chain virtqueue.UsedElement) (int, error) {
-	//read first element to see how many descriptors we need:
-	pkt.Chain = uint16(chain.DescriptorIndex)
-	buf := dev.ReceiveQueue.GetDescriptorItem(pkt.Chain)
-
-	// The specification requires that the first descriptor chain starts
-	// with a virtio-net header. It is not clear, whether it is also
-	// required to be fully contained in the first buffer of that
-	// descriptor chain, but it is reasonable to assume that this is
-	// always the case.
-	// The decode method already does the buffer length check.
-
-	//HACK: we only want the last bit of the header, the NumBuffers field. So, let's grab just that:
-	//numBuffers := binary.BigEndian.Uint16(buf[virtio.NetHdrSize-3:])
-	//even bigger hack: apparently this is hitting some kind of memory access pitfall? Let's only grab the last byte:
-	//numBuffers := buf[virtio.NetHdrSize-2]
-
-	//if err = pkt.header.Decode(buf); err != nil {
-	//	// The device misbehaved. There is no way we can gracefully
-	//	// recover from this, because we don't know how many of the
-	//	// following descriptor chains belong to this packet.
-	//	return 0, fmt.Errorf("decode vnethdr: %w", err)
-	//}
-
-	//we have the header now: what do we need to do?
-	//todo we're ignoring the header lol
-	//if int(numBuffers) != 1 {
-	//	return 0, fmt.Errorf("too smol-brain to handle more than one buffer per Chain item right now: %d chains, %d bufs", 1, int(numBuffers))
-	//}
-
-	if chain.Length > 16000 {
-		//todo!
-		pkt.payload = nil
-		return 1, fmt.Errorf("too big packet length: %d", chain.Length)
-	}
-
-	//shift the buffer out of out:
-	pkt.payload = buf[virtio.NetHdrSize:chain.Length]
-	return 1, nil
-}
-
-type VirtIOPacket struct {
-	payload []byte
-	//header  virtio.NetHdr
-	Chain uint16
-}
-
-func NewVIO() *VirtIOPacket {
-	out := new(VirtIOPacket)
-	out.payload = nil
-	out.Chain = 0
-	return out
-}
-
-func (v *VirtIOPacket) Reset() {
-	v.payload = nil
-	v.Chain = 0
-}
-
-func (v *VirtIOPacket) GetPayload() []byte {
-	return v.payload //todo this could be dev.ReceiveQueue.GetDescriptorItem(idx)
-}
-func (v *VirtIOPacket) SetPayload(x []byte) {
-	v.payload = x
-}

overlay/vhostnet/doc.go

@@ -1,3 +0,0 @@
-// Package vhostnet implements methods to initialize vhost networking devices
-// within the kernel-level virtio implementation and communicate with them.
-package vhostnet

overlay/vhostnet/ioctl.go

@@ -1,31 +0,0 @@
-package vhostnet
-
-import (
-	"fmt"
-	"unsafe"
-
-	"github.com/slackhq/nebula/overlay/vhost"
-)
-
-const (
-	// vhostNetIoctlSetBackend can be used to attach a virtqueue to a RAW socket
-	// or TAP device.
-	//
-	// Request payload: [vhost.QueueFile]
-	// Kernel name: VHOST_NET_SET_BACKEND
-	vhostNetIoctlSetBackend = 0x4008af30
-)
-
-// SetQueueBackend attaches a virtqueue of the vhost networking device
-// described by controlFD to the given backend file descriptor.
-// The backend file descriptor can either be a RAW socket or a TAP device. When
-// it is -1, the queue will be detached.
-func SetQueueBackend(controlFD int, queueIndex uint32, backendFD int) error {
-	if err := vhost.IoctlPtr(controlFD, vhostNetIoctlSetBackend, unsafe.Pointer(&vhost.QueueFile{
-		QueueIndex: queueIndex,
-		FD:         int32(backendFD),
-	})); err != nil {
-		return fmt.Errorf("set queue backend file descriptor: %w", err)
-	}
-	return nil
-}

overlay/vhostnet/options.go

@@ -1,69 +0,0 @@
-package vhostnet
-
-import (
-	"errors"
-
-	"github.com/slackhq/nebula/overlay/virtqueue"
-)
-
-type optionValues struct {
-	queueSize int
-	backendFD int
-}
-
-func (o *optionValues) apply(options []Option) {
-	for _, option := range options {
-		option(o)
-	}
-}
-
-func (o *optionValues) validate() error {
-	if o.queueSize == -1 {
-		return errors.New("queue size is required")
-	}
-	if err := virtqueue.CheckQueueSize(o.queueSize); err != nil {
-		return err
-	}
-	if o.backendFD == -1 {
-		return errors.New("backend file descriptor is required")
-	}
-	return nil
-}
-
-var optionDefaults = optionValues{
-	// Required.
-	queueSize: -1,
-	// Required.
-	backendFD: -1,
-}
-
-// Option can be passed to [NewDevice] to influence device creation.
-type Option func(*optionValues)
-
-// WithQueueSize returns an [Option] that sets the size of the TX and RX queues
-// that are to be created for the device. It specifies the number of
-// entries/buffers each queue can hold. This also affects the memory
-// consumption.
-// This is required and must be an integer from 1 to 32768 that is also a power
-// of 2.
-func WithQueueSize(queueSize int) Option {
-	return func(o *optionValues) { o.queueSize = queueSize }
-}
-
-// WithBackendFD returns an [Option] that sets the file descriptor of the
-// backend that will be used for the queues of the device. The device will write
-// and read packets to/from that backend. The file descriptor can either be of a
-// RAW socket or TUN/TAP device.
-// Either this or [WithBackendDevice] is required.
-func WithBackendFD(backendFD int) Option {
-	return func(o *optionValues) { o.backendFD = backendFD }
-}
-
-//// WithBackendDevice returns an [Option] that sets the given TAP device as the
-//// backend that will be used for the queues of the device. The device will
-//// write and read packets to/from that backend. The TAP device should have been
-//// created with the [tuntap.WithVirtioNetHdr] option enabled.
-//// Either this or [WithBackendFD] is required.
-//func WithBackendDevice(dev *tuntap.Device) Option {
-//	return func(o *optionValues) { o.backendFD = int(dev.File().Fd()) }
-//}

overlay/virtqueue/README.md

@@ -1,23 +0,0 @@
-Significant portions of this code are derived from https://pkg.go.dev/github.com/hetznercloud/virtio-go
-
-MIT License
-
-Copyright (c) 2025 Hetzner Cloud GmbH
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

overlay/virtqueue/available_ring.go

@@ -1,140 +0,0 @@
-package virtqueue
-
-import (
-	"fmt"
-	"unsafe"
-)
-
-// availableRingFlag is a flag that describes an [AvailableRing].
-type availableRingFlag uint16
-
-const (
-	// availableRingFlagNoInterrupt is used by the guest to advise the host to
-	// not interrupt it when consuming a buffer. It's unreliable, so it's simply
-	// an optimization.
-	availableRingFlagNoInterrupt availableRingFlag = 1 << iota
-)
-
-// availableRingSize is the number of bytes needed to store an [AvailableRing]
-// with the given queue size in memory.
-func availableRingSize(queueSize int) int {
-	return 6 + 2*queueSize
-}
-
-// availableRingAlignment is the minimum alignment of an [AvailableRing]
-// in memory, as required by the virtio spec.
-const availableRingAlignment = 2
-
-// AvailableRing is used by the driver to offer descriptor chains to the device.
-// Each ring entry refers to the head of a descriptor chain. It is only written
-// to by the driver and read by the device.
-//
-// Because the size of the ring depends on the queue size, we cannot define a
-// Go struct with a static size that maps to the memory of the ring. Instead,
-// this struct only contains pointers to the corresponding memory areas.
-type AvailableRing struct {
-	initialized bool
-
-	// flags that describe this ring.
-	flags *availableRingFlag
-	// ringIndex indicates where the driver would put the next entry into the
-	// ring (modulo the queue size).
-	ringIndex *uint16
-	// ring references buffers using the index of the head of the descriptor
-	// chain in the [DescriptorTable]. It wraps around at queue size.
-	ring []uint16
-	// usedEvent is not used by this implementation, but we reserve it anyway to
-	// avoid issues in case a device may try to access it, contrary to the
-	// virtio specification.
-	usedEvent *uint16
-}
-
-// newAvailableRing creates an available ring that uses the given underlying
-// memory. The length of the memory slice must match the size needed for the
-// ring (see [availableRingSize]) for the given queue size.
-func newAvailableRing(queueSize int, mem []byte) *AvailableRing {
-	ringSize := availableRingSize(queueSize)
-	if len(mem) != ringSize {
-		panic(fmt.Sprintf("memory size (%v) does not match required size "+
-			"for available ring: %v", len(mem), ringSize))
-	}
-
-	return &AvailableRing{
-		initialized: true,
-		flags:       (*availableRingFlag)(unsafe.Pointer(&mem[0])),
-		ringIndex:   (*uint16)(unsafe.Pointer(&mem[2])),
-		ring:        unsafe.Slice((*uint16)(unsafe.Pointer(&mem[4])), queueSize),
-		usedEvent:   (*uint16)(unsafe.Pointer(&mem[ringSize-2])),
-	}
-}
-
-// Address returns the pointer to the beginning of the ring in memory.
-// Do not modify the memory directly to not interfere with this implementation.
-func (r *AvailableRing) Address() uintptr {
-	if !r.initialized {
-		panic("available ring is not initialized")
-	}
-	return uintptr(unsafe.Pointer(r.flags))
-}
-
-// offer adds the given descriptor chain heads to the available ring and
-// advances the ring index accordingly to make the device process the new
-// descriptor chains.
-func (r *AvailableRing) offerElements(chains []UsedElement) {
-	//always called under lock
-	//r.mu.Lock()
-	//defer r.mu.Unlock()
-
-	// Add descriptor chain heads to the ring.
-	for offset, x := range chains {
-		// The 16-bit ring index may overflow. This is expected and is not an
-		// issue because the size of the ring array (which equals the queue
-		// size) is always a power of 2 and smaller than the highest possible
-		// 16-bit value.
-		insertIndex := int(*r.ringIndex+uint16(offset)) % len(r.ring)
-		r.ring[insertIndex] = x.GetHead()
-	}
-
-	// Increase the ring index by the number of descriptor chains added to the
-	// ring.
-	*r.ringIndex += uint16(len(chains))
-}
-
-func (r *AvailableRing) offer(chains []uint16) {
-	//always called under lock
-	//r.mu.Lock()
-	//defer r.mu.Unlock()
-
-	// Add descriptor chain heads to the ring.
-	for offset, x := range chains {
-		// The 16-bit ring index may overflow. This is expected and is not an
-		// issue because the size of the ring array (which equals the queue
-		// size) is always a power of 2 and smaller than the highest possible
-		// 16-bit value.
-		insertIndex := int(*r.ringIndex+uint16(offset)) % len(r.ring)
-		r.ring[insertIndex] = x
-	}
-
-	// Increase the ring index by the number of descriptor chains added to the
-	// ring.
-	*r.ringIndex += uint16(len(chains))
-}
-
-func (r *AvailableRing) offerSingle(x uint16) {
-	//always called under lock
-	//r.mu.Lock()
-	//defer r.mu.Unlock()
-
-	offset := 0
-	// Add descriptor chain heads to the ring.
-
-	// The 16-bit ring index may overflow. This is expected and is not an
-	// issue because the size of the ring array (which equals the queue
-	// size) is always a power of 2 and smaller than the highest possible
-	// 16-bit value.
-	insertIndex := int(*r.ringIndex+uint16(offset)) % len(r.ring)
-	r.ring[insertIndex] = x
-
-	// Increase the ring index by the number of descriptor chains added to the ring.
-	*r.ringIndex += 1
-}

overlay/virtqueue/available_ring_internal_test.go

@@ -1,71 +0,0 @@
-package virtqueue
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestAvailableRing_MemoryLayout(t *testing.T) {
-	const queueSize = 2
-
-	memory := make([]byte, availableRingSize(queueSize))
-	r := newAvailableRing(queueSize, memory)
-
-	*r.flags = 0x01ff
-	*r.ringIndex = 1
-	r.ring[0] = 0x1234
-	r.ring[1] = 0x5678
-
-	assert.Equal(t, []byte{
-		0xff, 0x01,
-		0x01, 0x00,
-		0x34, 0x12,
-		0x78, 0x56,
-		0x00, 0x00,
-	}, memory)
-}
-
-func TestAvailableRing_Offer(t *testing.T) {
-	const queueSize = 8
-
-	chainHeads := []uint16{42, 33, 69}
-
-	tests := []struct {
-		name              string
-		startRingIndex    uint16
-		expectedRingIndex uint16
-		expectedRing      []uint16
-	}{
-		{
-			name:              "no overflow",
-			startRingIndex:    0,
-			expectedRingIndex: 3,
-			expectedRing:      []uint16{42, 33, 69, 0, 0, 0, 0, 0},
-		},
-		{
-			name:              "ring overflow",
-			startRingIndex:    6,
-			expectedRingIndex: 9,
-			expectedRing:      []uint16{69, 0, 0, 0, 0, 0, 42, 33},
-		},
-		{
-			name:              "index overflow",
-			startRingIndex:    65535,
-			expectedRingIndex: 2,
-			expectedRing:      []uint16{33, 69, 0, 0, 0, 0, 0, 42},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			memory := make([]byte, availableRingSize(queueSize))
-			r := newAvailableRing(queueSize, memory)
-			*r.ringIndex = tt.startRingIndex
-
-			r.offer(chainHeads)
-
-			assert.Equal(t, tt.expectedRingIndex, *r.ringIndex)
-			assert.Equal(t, tt.expectedRing, r.ring)
-		})
-	}
-}

overlay/virtqueue/descriptor.go

@@ -1,43 +0,0 @@
-package virtqueue
-
-// descriptorFlag is a flag that describes a [Descriptor].
-type descriptorFlag uint16
-
-const (
-	// descriptorFlagHasNext marks a descriptor chain as continuing via the next
-	// field.
-	descriptorFlagHasNext descriptorFlag = 1 << iota
-	// descriptorFlagWritable marks a buffer as device write-only (otherwise
-	// device read-only).
-	descriptorFlagWritable
-	// descriptorFlagIndirect means the buffer contains a list of buffer
-	// descriptors to provide an additional layer of indirection.
-	// Only allowed when the [virtio.FeatureIndirectDescriptors] feature was
-	// negotiated.
-	descriptorFlagIndirect
-)
-
-// descriptorSize is the number of bytes needed to store a [Descriptor] in
-// memory.
-const descriptorSize = 16
-
-// Descriptor describes (a part of) a buffer which is either read-only for the
-// device or write-only for the device (depending on [descriptorFlagWritable]).
-// Multiple descriptors can be chained to produce a "descriptor chain" that can
-// contain both device-readable and device-writable buffers. Device-readable
-// descriptors always come first in a chain. A single, large buffer may be
-// split up by chaining multiple similar descriptors that reference different
-// memory pages. This is required, because buffers may exceed a single page size
-// and the memory accessed by the device is expected to be continuous.
-type Descriptor struct {
-	// address is the address to the continuous memory holding the data for this
-	// descriptor.
-	address uintptr
-	// length is the amount of bytes stored at address.
-	length uint32
-	// flags that describe this descriptor.
-	flags descriptorFlag
-	// next contains the index of the next descriptor continuing this descriptor
-	// chain when the [descriptorFlagHasNext] flag is set.
-	next uint16
-}

overlay/virtqueue/descriptor_internal_test.go

@@ -1,12 +0,0 @@
-package virtqueue
-
-import (
-	"testing"
-	"unsafe"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestDescriptor_Size(t *testing.T) {
-	assert.EqualValues(t, descriptorSize, unsafe.Sizeof(Descriptor{}))
-}

overlay/virtqueue/descriptor_table.go

@@ -1,288 +0,0 @@
-package virtqueue
-
-import (
-	"errors"
-	"fmt"
-	"math"
-	"unsafe"
-
-	"golang.org/x/sys/unix"
-)
-
-var (
-	// ErrNotEnoughFreeDescriptors is returned when the free descriptors are
-	// exhausted, meaning that the queue is full.
-	ErrNotEnoughFreeDescriptors = errors.New("not enough free descriptors, queue is full")
-
-	// ErrInvalidDescriptorChain is returned when a descriptor chain is not
-	// valid for a given operation.
-	ErrInvalidDescriptorChain = errors.New("invalid descriptor chain")
-)
-
-// noFreeHead is used to mark when all descriptors are in use and we have no
-// free chain. This value is impossible to occur as an index naturally, because
-// it exceeds the maximum queue size.
-const noFreeHead = uint16(math.MaxUint16)
-
-// descriptorTableSize is the number of bytes needed to store a
-// [DescriptorTable] with the given queue size in memory.
-func descriptorTableSize(queueSize int) int {
-	return descriptorSize * queueSize
-}
-
-// descriptorTableAlignment is the minimum alignment of a [DescriptorTable]
-// in memory, as required by the virtio spec.
-const descriptorTableAlignment = 16
-
-// DescriptorTable is a table that holds [Descriptor]s, addressed via their
-// index in the slice.
-type DescriptorTable struct {
-	descriptors []Descriptor
-
-	// freeHeadIndex is the index of the head of the descriptor chain which
-	// contains all currently unused descriptors. When all descriptors are in
-	// use, this has the special value of noFreeHead.
-	freeHeadIndex uint16
-	// freeNum tracks the number of descriptors which are currently not in use.
-	freeNum uint16
-
-	bufferBase uintptr
-	bufferSize int
-	itemSize   int
-}
-
-// newDescriptorTable creates a descriptor table that uses the given underlying
-// memory. The Length of the memory slice must match the size needed for the
-// descriptor table (see [descriptorTableSize]) for the given queue size.
-//
-// Before this descriptor table can be used, [initialize] must be called.
-func newDescriptorTable(queueSize int, mem []byte, itemSize int) *DescriptorTable {
-	dtSize := descriptorTableSize(queueSize)
-	if len(mem) != dtSize {
-		panic(fmt.Sprintf("memory size (%v) does not match required size "+
-			"for descriptor table: %v", len(mem), dtSize))
-	}
-
-	return &DescriptorTable{
-		descriptors: unsafe.Slice((*Descriptor)(unsafe.Pointer(&mem[0])), queueSize),
-		// We have no free descriptors until they were initialized.
-		freeHeadIndex: noFreeHead,
-		freeNum:       0,
-		itemSize:      itemSize, //todo configurable? needs to be page-aligned
-	}
-}
-
-// Address returns the pointer to the beginning of the descriptor table in
-// memory. Do not modify the memory directly to not interfere with this
-// implementation.
-func (dt *DescriptorTable) Address() uintptr {
-	if dt.descriptors == nil {
-		panic("descriptor table is not initialized")
-	}
-	//should be same as dt.bufferBase
-	return uintptr(unsafe.Pointer(&dt.descriptors[0]))
-}
-
-func (dt *DescriptorTable) Size() uintptr {
-	if dt.descriptors == nil {
-		panic("descriptor table is not initialized")
-	}
-	return uintptr(dt.bufferSize)
-}
-
-// BufferAddresses returns a map of pointer->size for all allocations used by the table
-func (dt *DescriptorTable) BufferAddresses() map[uintptr]int {
-	if dt.descriptors == nil {
-		panic("descriptor table is not initialized")
-	}
-
-	return map[uintptr]int{dt.bufferBase: dt.bufferSize}
-}
-
-// initializeDescriptors allocates buffers with the size of a full memory page
-// for each descriptor in the table. While this may be a bit wasteful, it makes
-// dealing with descriptors way easier. Without this preallocation, we would
-// have to allocate and free memory on demand, increasing complexity.
-//
-// All descriptors will be marked as free and will form a free chain. The
-// addresses of all descriptors will be populated while their length remains
-// zero.
-func (dt *DescriptorTable) initializeDescriptors() error {
-	numDescriptors := len(dt.descriptors)
-
-	// Allocate ONE large region for all buffers
-	totalSize := dt.itemSize * numDescriptors
-	basePtr, err := unix.MmapPtr(-1, 0, nil, uintptr(totalSize),
-		unix.PROT_READ|unix.PROT_WRITE,
-		unix.MAP_PRIVATE|unix.MAP_ANONYMOUS)
-	if err != nil {
-		return fmt.Errorf("allocate buffer memory for descriptors: %w", err)
-	}
-
-	// Store the base for cleanup later
-	dt.bufferBase = uintptr(basePtr)
-	dt.bufferSize = totalSize
-
-	for i := range dt.descriptors {
-		dt.descriptors[i] = Descriptor{
-			address: dt.bufferBase + uintptr(i*dt.itemSize),
-			length:  0,
-			// All descriptors should form a free chain that loops around.
-			flags: descriptorFlagHasNext,
-			next:  uint16((i + 1) % len(dt.descriptors)),
-		}
-	}
-
-	// All descriptors are free to use now.
-	dt.freeHeadIndex = 0
-	dt.freeNum = uint16(len(dt.descriptors))
-
-	return nil
-}
-
-// releaseBuffers releases all allocated buffers for this descriptor table.
-// The implementation will try to release as many buffers as possible and
-// collect potential errors before returning them.
-// The descriptor table should no longer be used after calling this.
-func (dt *DescriptorTable) releaseBuffers() error {
-	for i := range dt.descriptors {
-		descriptor := &dt.descriptors[i]
-		descriptor.address = 0
-	}
-
-	// As a safety measure, make sure no descriptors can be used anymore.
-	dt.freeHeadIndex = noFreeHead
-	dt.freeNum = 0
-
-	if dt.bufferBase != 0 {
-		// The pointer points to memory not managed by Go, so this conversion
-		// is safe. See https://github.com/golang/go/issues/58625
-		dt.bufferBase = 0
-		//goland:noinspection GoVetUnsafePointer
-		err := unix.MunmapPtr(unsafe.Pointer(dt.bufferBase), uintptr(dt.bufferSize))
-		if err != nil {
-			return fmt.Errorf("release buffer memory: %w", err)
-		}
-	}
-
-	return nil
-}
-
-func (dt *DescriptorTable) CreateDescriptorForOutputs() (uint16, uint32, error) {
-	//todo just fill the damn table
-	// Do we still have enough free descriptors?
-
-	if 1 > dt.freeNum {
-		return 0, 0, ErrNotEnoughFreeDescriptors
-	}
-
-	// Above validation ensured that there is at least one free descriptor, so
-	// the free descriptor chain head should be valid.
-	if dt.freeHeadIndex == noFreeHead {
-		panic("free descriptor chain head is unset but there should be free descriptors")
-	}
-
-	// To avoid having to iterate over the whole table to find the descriptor
-	// pointing to the head just to replace the free head, we instead always
-	// create descriptor chains from the descriptors coming after the head.
-	// This way we only have to touch the head as a last resort, when all other
-	// descriptors are already used.
-	head := dt.descriptors[dt.freeHeadIndex].next
-	desc := &dt.descriptors[head]
-	next := desc.next
-
-	checkUnusedDescriptorLength(head, desc)
-
-	// Give the device the maximum available number of bytes to write into.
-	desc.length = uint32(dt.itemSize)
-	desc.flags = 0 // descriptorFlagWritable
-	desc.next = 0  // Not necessary to clear this, it's just for looks.
-
-	dt.freeNum -= 1
-
-	if dt.freeNum == 0 {
-		// The last descriptor in the chain should be the free chain head
-		// itself.
-		if next != dt.freeHeadIndex {
-			panic("descriptor chain takes up all free descriptors but does not end with the free chain head")
-		}
-
-		// When this new chain takes up all remaining descriptors, we no longer
-		// have a free chain.
-		dt.freeHeadIndex = noFreeHead
-	} else {
-		// We took some descriptors out of the free chain, so make sure to close
-		// the circle again.
-		dt.descriptors[dt.freeHeadIndex].next = next
-	}
-
-	return head, desc.length, nil
-}
-
-func (dt *DescriptorTable) createDescriptorForInputs() (uint16, error) {
-	// Do we still have enough free descriptors?
-	if 1 > dt.freeNum {
-		return 0, ErrNotEnoughFreeDescriptors
-	}
-
-	// Above validation ensured that there is at least one free descriptor, so
-	// the free descriptor chain head should be valid.
-	if dt.freeHeadIndex == noFreeHead {
-		panic("free descriptor chain head is unset but there should be free descriptors")
-	}
-
-	// To avoid having to iterate over the whole table to find the descriptor
-	// pointing to the head just to replace the free head, we instead always
-	// create descriptor chains from the descriptors coming after the head.
-	// This way we only have to touch the head as a last resort, when all other
-	// descriptors are already used.
-	head := dt.descriptors[dt.freeHeadIndex].next
-	desc := &dt.descriptors[head]
-	next := desc.next
-
-	checkUnusedDescriptorLength(head, desc)
-
-	// Give the device the maximum available number of bytes to write into.
-	desc.length = uint32(dt.itemSize)
-	desc.flags = descriptorFlagWritable
-	desc.next = 0 // Not necessary to clear this, it's just for looks.
-
-	dt.freeNum -= 1
-
-	if dt.freeNum == 0 {
-		// The last descriptor in the chain should be the free chain head
-		// itself.
-		if next != dt.freeHeadIndex {
-			panic("descriptor chain takes up all free descriptors but does not end with the free chain head")
-		}
-
-		// When this new chain takes up all remaining descriptors, we no longer
-		// have a free chain.
-		dt.freeHeadIndex = noFreeHead
-	} else {
-		// We took some descriptors out of the free chain, so make sure to close
-		// the circle again.
-		dt.descriptors[dt.freeHeadIndex].next = next
-	}
-
-	return head, nil
-}
-
-func (dt *DescriptorTable) getDescriptorItem(head uint16) []byte {
-	desc := &dt.descriptors[head] //todo this is a pretty nasty hack with no checks
-
-	// The descriptor address points to memory not managed by Go, so this
-	// conversion is safe. See https://github.com/golang/go/issues/58625
-	//goland:noinspection GoVetUnsafePointer
-	return unsafe.Slice((*byte)(unsafe.Pointer(desc.address)), desc.length)
-}
-
-// checkUnusedDescriptorLength asserts that the length of an unused descriptor
-// is zero, as it should be.
-// This is not a requirement by the virtio spec but rather a thing we do to
-// notice when our algorithm goes sideways.
-func checkUnusedDescriptorLength(index uint16, desc *Descriptor) {
-	if desc.length != 0 {
-		panic(fmt.Sprintf("descriptor %d should be unused but has a non-zero length", index))
-	}
-}

overlay/virtqueue/doc.go

@@ -1,7 +0,0 @@
-// Package virtqueue implements the driver-side for a virtio queue as described
-// in the specification:
-// https://docs.oasis-open.org/virtio/virtio/v1.2/csd01/virtio-v1.2-csd01.html#x1-270006
-// This package does not make assumptions about the device that consumes the
-// queue. It rather just allocates the queue structures in memory and provides
-// methods to interact with it.
-package virtqueue

overlay/virtqueue/eventfd_test.go

@@ -1,45 +0,0 @@
-package virtqueue
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"gvisor.dev/gvisor/pkg/eventfd"
-)
-
-// Tests how an eventfd and a waiting goroutine can be gracefully closed.
-// Extends the eventfd test suite:
-// https://github.com/google/gvisor/blob/0799336d64be65eb97d330606c30162dc3440cab/pkg/eventfd/eventfd_test.go
-func TestEventFD_CancelWait(t *testing.T) {
-	efd, err := eventfd.Create()
-	require.NoError(t, err)
-	t.Cleanup(func() {
-		assert.NoError(t, efd.Close())
-	})
-
-	var stop bool
-
-	done := make(chan struct{})
-	go func() {
-		for !stop {
-			_ = efd.Wait()
-		}
-		close(done)
-	}()
-	select {
-	case <-done:
-		t.Fatalf("goroutine ended early")
-	case <-time.After(500 * time.Millisecond):
-	}
-
-	stop = true
-	assert.NoError(t, efd.Notify())
-	select {
-	case <-done:
-		break
-	case <-time.After(5 * time.Second):
-		t.Error("goroutine did not end")
-	}
-}

overlay/virtqueue/size.go

@@ -1,33 +0,0 @@
-package virtqueue
-
-import (
-	"errors"
-	"fmt"
-)
-
-// ErrQueueSizeInvalid is returned when a queue size is invalid.
-var ErrQueueSizeInvalid = errors.New("queue size is invalid")
-
-// CheckQueueSize checks if the given value would be a valid size for a
-// virtqueue and returns an [ErrQueueSizeInvalid], if not.
-func CheckQueueSize(queueSize int) error {
-	if queueSize <= 0 {
-		return fmt.Errorf("%w: %d is too small", ErrQueueSizeInvalid, queueSize)
-	}
-
-	// The queue size must always be a power of 2.
-	// This ensures that ring indexes wrap correctly when the 16-bit integers
-	// overflow.
-	if queueSize&(queueSize-1) != 0 {
-		return fmt.Errorf("%w: %d is not a power of 2", ErrQueueSizeInvalid, queueSize)
-	}
-
-	// The largest power of 2 that fits into a 16-bit integer is 32768.
-	// 2 * 32768 would be 65536 which no longer fits.
-	if queueSize > 32768 {
-		return fmt.Errorf("%w: %d is larger than the maximum possible queue size 32768",
-			ErrQueueSizeInvalid, queueSize)
-	}
-
-	return nil
-}

overlay/virtqueue/size_test.go

@@ -1,59 +0,0 @@
-package virtqueue
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestCheckQueueSize(t *testing.T) {
-	tests := []struct {
-		name        string
-		queueSize   int
-		containsErr string
-	}{
-		{
-			name:        "negative",
-			queueSize:   -1,
-			containsErr: "too small",
-		},
-		{
-			name:        "zero",
-			queueSize:   0,
-			containsErr: "too small",
-		},
-		{
-			name:        "not a power of 2",
-			queueSize:   24,
-			containsErr: "not a power of 2",
-		},
-		{
-			name:        "too large",
-			queueSize:   65536,
-			containsErr: "larger than the maximum",
-		},
-		{
-			name:      "valid 1",
-			queueSize: 1,
-		},
-		{
-			name:      "valid 256",
-			queueSize: 256,
-		},
-
-		{
-			name:      "valid 32768",
-			queueSize: 32768,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			err := CheckQueueSize(tt.queueSize)
-			if tt.containsErr != "" {
-				assert.ErrorContains(t, err, tt.containsErr)
-			} else {
-				assert.NoError(t, err)
-			}
-		})
-	}
-}

overlay/virtqueue/split_virtqueue.go

@@ -1,415 +0,0 @@
-package virtqueue
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"os"
-
-	"github.com/slackhq/nebula/overlay/eventfd"
-	"golang.org/x/sys/unix"
-)
-
-// SplitQueue is a virtqueue that consists of several parts, where each part is
-// writeable by either the driver or the device, but not both.
-type SplitQueue struct {
-	// size is the size of the queue.
-	size int
-	// buf is the underlying memory used for the queue.
-	buf []byte
-
-	descriptorTable *DescriptorTable
-	availableRing   *AvailableRing
-	usedRing        *UsedRing
-
-	// kickEventFD is used to signal the device when descriptor chains were
-	// added to the available ring.
-	kickEventFD eventfd.EventFD
-	// callEventFD is used by the device to signal when it has used descriptor
-	// chains and put them in the used ring.
-	callEventFD eventfd.EventFD
-
-	// stop is used by [SplitQueue.Close] to cancel the goroutine that handles
-	// used buffer notifications. It blocks until the goroutine ended.
-	stop func() error
-
-	itemSize int
-
-	epoll eventfd.Epoll
-	more  int
-}
-
-// NewSplitQueue allocates a new [SplitQueue] in memory. The given queue size
-// specifies the number of entries/buffers the queue can hold. This also affects
-// the memory consumption.
-func NewSplitQueue(queueSize int, itemSize int) (_ *SplitQueue, err error) {
-	if err = CheckQueueSize(queueSize); err != nil {
-		return nil, err
-	}
-
-	if itemSize%os.Getpagesize() != 0 {
-		return nil, errors.New("split queue size must be multiple of os.Getpagesize()")
-	}
-
-	sq := SplitQueue{
-		size:     queueSize,
-		itemSize: itemSize,
-	}
-
-	// Clean up a partially initialized queue when something fails.
-	defer func() {
-		if err != nil {
-			_ = sq.Close()
-		}
-	}()
-
-	// There are multiple ways for how the memory for the virtqueue could be
-	// allocated. We could use Go native structs with arrays inside them, but
-	// this wouldn't allow us to make the queue size configurable. And including
-	// a slice in the Go structs wouldn't work, because this would just put the
-	// Go slice descriptor into the memory region which the virtio device will
-	// not understand.
-	// Additionally, Go does not allow us to ensure a correct alignment of the
-	// parts of the virtqueue, as it is required by the virtio specification.
-	//
-	// To resolve this, let's just allocate the memory manually by allocating
-	// one or more memory pages, depending on the queue size. Making the
-	// virtqueue start at the beginning of a page is not strictly necessary, as
-	// the virtio specification does not require it to be continuous in the
-	// physical memory of the host (e.g. the vhost implementation in the kernel
-	// always uses copy_from_user to access it), but this makes it very easy to
-	// guarantee the alignment. Also, it is not required for the virtqueue parts
-	// to be in the same memory region, as we pass separate pointers to them to
-	// the device, but this design just makes things easier to implement.
-	//
-	// One added benefit of allocating the memory manually is, that we have full
-	// control over its lifetime and don't risk the garbage collector to collect
-	// our valuable structures while the device still works with them.
-
-	// The descriptor table is at the start of the page, so alignment is not an
-	// issue here.
-	descriptorTableStart := 0
-	descriptorTableEnd := descriptorTableStart + descriptorTableSize(queueSize)
-	availableRingStart := align(descriptorTableEnd, availableRingAlignment)
-	availableRingEnd := availableRingStart + availableRingSize(queueSize)
-	usedRingStart := align(availableRingEnd, usedRingAlignment)
-	usedRingEnd := usedRingStart + usedRingSize(queueSize)
-
-	sq.buf, err = unix.Mmap(-1, 0, usedRingEnd,
-		unix.PROT_READ|unix.PROT_WRITE,
-		unix.MAP_PRIVATE|unix.MAP_ANONYMOUS)
-	if err != nil {
-		return nil, fmt.Errorf("allocate virtqueue buffer: %w", err)
-	}
-
-	sq.descriptorTable = newDescriptorTable(queueSize, sq.buf[descriptorTableStart:descriptorTableEnd], sq.itemSize)
-	sq.availableRing = newAvailableRing(queueSize, sq.buf[availableRingStart:availableRingEnd])
-	sq.usedRing = newUsedRing(queueSize, sq.buf[usedRingStart:usedRingEnd])
-
-	sq.kickEventFD, err = eventfd.New()
-	if err != nil {
-		return nil, fmt.Errorf("create kick event file descriptor: %w", err)
-	}
-	sq.callEventFD, err = eventfd.New()
-	if err != nil {
-		return nil, fmt.Errorf("create call event file descriptor: %w", err)
-	}
-
-	if err = sq.descriptorTable.initializeDescriptors(); err != nil {
-		return nil, fmt.Errorf("initialize descriptors: %w", err)
-	}
-
-	sq.epoll, err = eventfd.NewEpoll()
-	if err != nil {
-		return nil, err
-	}
-	err = sq.epoll.AddEvent(sq.callEventFD.FD())
-	if err != nil {
-		return nil, err
-	}
-
-	sq.stop = sq.kickSelfToExit()
-
-	return &sq, nil
-}
-
-// Size returns the size of this queue, which is the number of entries/buffers
-// this queue can hold.
-func (sq *SplitQueue) Size() int {
-	return sq.size
-}
-
-// DescriptorTable returns the [DescriptorTable] behind this queue.
-func (sq *SplitQueue) DescriptorTable() *DescriptorTable {
-	return sq.descriptorTable
-}
-
-// AvailableRing returns the [AvailableRing] behind this queue.
-func (sq *SplitQueue) AvailableRing() *AvailableRing {
-	return sq.availableRing
-}
-
-// UsedRing returns the [UsedRing] behind this queue.
-func (sq *SplitQueue) UsedRing() *UsedRing {
-	return sq.usedRing
-}
-
-// KickEventFD returns the kick event file descriptor behind this queue.
-// The returned file descriptor should be used with great care to not interfere
-// with this implementation.
-func (sq *SplitQueue) KickEventFD() int {
-	return sq.kickEventFD.FD()
-}
-
-// CallEventFD returns the call event file descriptor behind this queue.
-// The returned file descriptor should be used with great care to not interfere
-// with this implementation.
-func (sq *SplitQueue) CallEventFD() int {
-	return sq.callEventFD.FD()
-}
-
-func (sq *SplitQueue) kickSelfToExit() func() error {
-	return func() error {
-
-		// The goroutine blocks until it receives a signal on the event file
-		// descriptor, so it will never notice the context being canceled.
-		// To resolve this, we can just produce a fake-signal ourselves to wake
-		// it up.
-		if err := sq.callEventFD.Kick(); err != nil {
-			return fmt.Errorf("wake up goroutine: %w", err)
-		}
-		return nil
-	}
-}
-
-func (sq *SplitQueue) TakeSingleIndex(ctx context.Context) (uint16, error) {
-	element, err := sq.TakeSingle(ctx)
-	if err != nil {
-		return 0xffff, err
-	}
-	return element.GetHead(), nil
-}
-
-func (sq *SplitQueue) TakeSingle(ctx context.Context) (UsedElement, error) {
-	var n int
-	var err error
-	for ctx.Err() == nil {
-		out, ok := sq.usedRing.takeOne()
-		if ok {
-			return out, nil
-		}
-		// Wait for a signal from the device.
-		if n, err = sq.epoll.Block(); err != nil {
-			return UsedElement{}, fmt.Errorf("wait: %w", err)
-		}
-
-		if n > 0 {
-			out, ok = sq.usedRing.takeOne()
-			if ok {
-				_ = sq.epoll.Clear() //???
-				return out, nil
-			} else {
-				continue //???
-			}
-		}
-	}
-	return UsedElement{}, ctx.Err()
-}
-
-func (sq *SplitQueue) TakeSingleNoBlock() (UsedElement, bool) {
-	return sq.usedRing.takeOne()
-}
-
-func (sq *SplitQueue) WaitForUsedElements(ctx context.Context) error {
-	if sq.usedRing.availableToTake() != 0 {
-		return nil
-	}
-	for ctx.Err() == nil {
-		// Wait for a signal from the device.
-		n, err := sq.epoll.Block()
-		if err != nil {
-			return fmt.Errorf("wait: %w", err)
-		}
-		if n > 0 {
-			_ = sq.epoll.Clear()
-			if sq.usedRing.availableToTake() != 0 {
-				return nil
-			}
-		}
-	}
-	return ctx.Err()
-}
-
-func (sq *SplitQueue) BlockAndGetHeadsCapped(ctx context.Context, maxToTake int) ([]UsedElement, error) {
-	var n int
-	var err error
-	for ctx.Err() == nil {
-
-		//we have leftovers in the fridge
-		if sq.more > 0 {
-			stillNeedToTake, out := sq.usedRing.take(maxToTake)
-			sq.more = stillNeedToTake
-			return out, nil
-		}
-		//look inside the fridge
-		stillNeedToTake, out := sq.usedRing.take(maxToTake)
-		if len(out) > 0 {
-			sq.more = stillNeedToTake
-			return out, nil
-		}
-		//fridge is empty I guess
-
-		// Wait for a signal from the device.
-		if n, err = sq.epoll.Block(); err != nil {
-			return nil, fmt.Errorf("wait: %w", err)
-		}
-		if n > 0 {
-			_ = sq.epoll.Clear()
-			stillNeedToTake, out = sq.usedRing.take(maxToTake)
-			sq.more = stillNeedToTake
-			return out, nil
-		}
-	}
-
-	return nil, ctx.Err()
-}
-
-// OfferDescriptorChain offers a descriptor chain to the device which contains a
-// number of device-readable buffers (out buffers) and device-writable buffers
-// (in buffers).
-//
-// All buffers in the outBuffers slice will be concatenated by chaining
-// descriptors, one for each buffer in the slice. When a buffer is too large to
-// fit into a single descriptor (limited by the system's page size), it will be
-// split up into multiple descriptors within the chain.
-// When numInBuffers is greater than zero, the given number of device-writable
-// descriptors will be appended to the end of the chain, each referencing a
-// whole memory page (see [os.Getpagesize]).
-//
-// When the queue is full and no more descriptor chains can be added, a wrapped
-// [ErrNotEnoughFreeDescriptors] will be returned. If you set waitFree to true,
-// this method will handle this error and will block instead until there are
-// enough free descriptors again.
-//
-// After defining the descriptor chain in the [DescriptorTable], the index of
-// the head of the chain will be made available to the device using the
-// [AvailableRing] and will be returned by this method.
-// Callers should read from the [SplitQueue.UsedDescriptorChains] channel to be
-// notified when the descriptor chain was used by the device and should free the
-// used descriptor chains again using [SplitQueue.FreeDescriptorChain] when
-// they're done with them. When this does not happen, the queue will run full
-// and any further calls to [SplitQueue.OfferDescriptorChain] will stall.
-
-func (sq *SplitQueue) OfferInDescriptorChains() (uint16, error) {
-	var (
-		head uint16
-		err  error
-	)
-	for {
-		head, err = sq.descriptorTable.createDescriptorForInputs()
-		if err == nil {
-			break
-		}
-
-		// I don't wanna use errors.Is, it's slow
-		//goland:noinspection GoDirectComparisonOfErrors
-		if err == ErrNotEnoughFreeDescriptors {
-			return 0, err
-		} else {
-			return 0, fmt.Errorf("create descriptor chain: %w", err)
-		}
-	}
-
-	// Make the descriptor chain available to the device.
-	sq.availableRing.offerSingle(head)
-
-	// Notify the device to make it process the updated available ring.
-	if err = sq.kickEventFD.Kick(); err != nil {
-		return head, fmt.Errorf("notify device: %w", err)
-	}
-
-	return head, nil
-}
-
-// GetDescriptorItem returns the buffer of a given index
-// The head index must be one that was returned by a previous call to
-// [SplitQueue.OfferDescriptorChain] and the descriptor chain must not have been
-// freed yet.
-//
-// Be careful to only access the returned buffer slices when the device is no
-// longer using them. They must not be accessed after
-// [SplitQueue.FreeDescriptorChain] has been called.
-func (sq *SplitQueue) GetDescriptorItem(head uint16) []byte {
-	sq.descriptorTable.descriptors[head].length = uint32(sq.descriptorTable.itemSize)
-	return sq.descriptorTable.getDescriptorItem(head)
-}
-
-func (sq *SplitQueue) SetDescSize(head uint16, sz int) {
-	//not called under lock
-	sq.descriptorTable.descriptors[int(head)].length = uint32(sz)
-}
-
-func (sq *SplitQueue) OfferDescriptorChains(chains []uint16, kick bool) error {
-	// Make the descriptor chain available to the device.
-	sq.availableRing.offer(chains)
-
-	// Notify the device to make it process the updated available ring.
-	if kick {
-		return sq.Kick()
-	}
-
-	return nil
-}
-
-func (sq *SplitQueue) Kick() error {
-	if err := sq.kickEventFD.Kick(); err != nil {
-		return fmt.Errorf("notify device: %w", err)
-	}
-	return nil
-}
-
-// Close releases all resources used for this queue.
-// The implementation will try to release as many resources as possible and
-// collect potential errors before returning them.
-func (sq *SplitQueue) Close() error {
-	var errs []error
-
-	if sq.stop != nil {
-		// This has to happen before the event file descriptors may be closed.
-		if err := sq.stop(); err != nil {
-			errs = append(errs, fmt.Errorf("stop consume used ring: %w", err))
-		}
-
-		// Make sure that this code block is executed only once.
-		sq.stop = nil
-	}
-
-	if err := sq.kickEventFD.Close(); err != nil {
-		errs = append(errs, fmt.Errorf("close kick event file descriptor: %w", err))
-	}
-	if err := sq.callEventFD.Close(); err != nil {
-		errs = append(errs, fmt.Errorf("close call event file descriptor: %w", err))
-	}
-
-	if err := sq.descriptorTable.releaseBuffers(); err != nil {
-		errs = append(errs, fmt.Errorf("release descriptor buffers: %w", err))
-	}
-
-	if sq.buf != nil {
-		if err := unix.Munmap(sq.buf); err == nil {
-			sq.buf = nil
-		} else {
-			errs = append(errs, fmt.Errorf("unmap virtqueue buffer: %w", err))
-		}
-	}
-
-	return errors.Join(errs...)
-}
-
-func align(index, alignment int) int {
-	remainder := index % alignment
-	if remainder == 0 {
-		return index
-	}
-	return index + alignment - remainder
-}

overlay/virtqueue/used_element.go

@@ -1,21 +0,0 @@
-package virtqueue
-
-// usedElementSize is the number of bytes needed to store a [UsedElement] in
-// memory.
-const usedElementSize = 8
-
-// UsedElement is an element of the [UsedRing] and describes a descriptor chain
-// that was used by the device.
-type UsedElement struct {
-	// DescriptorIndex is the index of the head of the used descriptor chain in
-	// the [DescriptorTable].
-	// The index is 32-bit here for padding reasons.
-	DescriptorIndex uint32
-	// Length is the number of bytes written into the device writable portion of
-	// the buffer described by the descriptor chain.
-	Length uint32
-}
-
-func (u *UsedElement) GetHead() uint16 {
-	return uint16(u.DescriptorIndex)
-}

overlay/virtqueue/used_element_internal_test.go

@@ -1,12 +0,0 @@
-package virtqueue
-
-import (
-	"testing"
-	"unsafe"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestUsedElement_Size(t *testing.T) {
-	assert.EqualValues(t, usedElementSize, unsafe.Sizeof(UsedElement{}))
-}

overlay/virtqueue/used_ring.go

@@ -1,170 +0,0 @@
-package virtqueue
-
-import (
-	"fmt"
-	"unsafe"
-)
-
-// usedRingFlag is a flag that describes a [UsedRing].
-type usedRingFlag uint16
-
-const (
-	// usedRingFlagNoNotify is used by the host to advise the guest to not
-	// kick it when adding a buffer. It's unreliable, so it's simply an
-	// optimization. Guest will still kick when it's out of buffers.
-	usedRingFlagNoNotify usedRingFlag = 1 << iota
-)
-
-// usedRingSize is the number of bytes needed to store a [UsedRing] with the
-// given queue size in memory.
-func usedRingSize(queueSize int) int {
-	return 6 + usedElementSize*queueSize
-}
-
-// usedRingAlignment is the minimum alignment of a [UsedRing] in memory, as
-// required by the virtio spec.
-const usedRingAlignment = 4
-
-// UsedRing is where the device returns descriptor chains once it is done with
-// them. Each ring entry is a [UsedElement]. It is only written to by the device
-// and read by the driver.
-//
-// Because the size of the ring depends on the queue size, we cannot define a
-// Go struct with a static size that maps to the memory of the ring. Instead,
-// this struct only contains pointers to the corresponding memory areas.
-type UsedRing struct {
-	initialized bool
-
-	// flags that describe this ring.
-	flags *usedRingFlag
-	// ringIndex indicates where the device would put the next entry into the
-	// ring (modulo the queue size).
-	ringIndex *uint16
-	// ring contains the [UsedElement]s. It wraps around at queue size.
-	ring []UsedElement
-	// availableEvent is not used by this implementation, but we reserve it
-	// anyway to avoid issues in case a device may try to write to it, contrary
-	// to the virtio specification.
-	availableEvent *uint16
-
-	// lastIndex is the internal ringIndex up to which all [UsedElement]s were
-	// processed.
-	lastIndex uint16
-
-	//mu sync.Mutex
-}
-
-// newUsedRing creates a used ring that uses the given underlying memory. The
-// length of the memory slice must match the size needed for the ring (see
-// [usedRingSize]) for the given queue size.
-func newUsedRing(queueSize int, mem []byte) *UsedRing {
-	ringSize := usedRingSize(queueSize)
-	if len(mem) != ringSize {
-		panic(fmt.Sprintf("memory size (%v) does not match required size "+
-			"for used ring: %v", len(mem), ringSize))
-	}
-
-	r := UsedRing{
-		initialized:    true,
-		flags:          (*usedRingFlag)(unsafe.Pointer(&mem[0])),
-		ringIndex:      (*uint16)(unsafe.Pointer(&mem[2])),
-		ring:           unsafe.Slice((*UsedElement)(unsafe.Pointer(&mem[4])), queueSize),
-		availableEvent: (*uint16)(unsafe.Pointer(&mem[ringSize-2])),
-	}
-	r.lastIndex = *r.ringIndex
-	return &r
-}
-
-// Address returns the pointer to the beginning of the ring in memory.
-// Do not modify the memory directly to not interfere with this implementation.
-func (r *UsedRing) Address() uintptr {
-	if !r.initialized {
-		panic("used ring is not initialized")
-	}
-	return uintptr(unsafe.Pointer(r.flags))
-}
-
-func (r *UsedRing) availableToTake() int {
-	ringIndex := *r.ringIndex
-	if ringIndex == r.lastIndex {
-		// Nothing new.
-		return 0
-	}
-
-	// Calculate the number new used elements that we can read from the ring.
-	// The ring index may wrap, so special handling for that case is needed.
-	count := int(ringIndex - r.lastIndex)
-	if count < 0 {
-		count += 0xffff
-	}
-	return count
-}
-
-// take returns all new [UsedElement]s that the device put into the ring and
-// that weren't already returned by a previous call to this method.
-func (r *UsedRing) take(maxToTake int) (int, []UsedElement) {
-	count := r.availableToTake()
-	if count == 0 {
-		return 0, nil
-	}
-
-	stillNeedToTake := 0
-
-	if maxToTake > 0 {
-		stillNeedToTake = count - maxToTake
-		if stillNeedToTake < 0 {
-			stillNeedToTake = 0
-		}
-		count = min(count, maxToTake)
-	}
-
-	// The number of new elements can never exceed the queue size.
-	if count > len(r.ring) {
-		panic("used ring contains more new elements than the ring is long")
-	}
-
-	elems := make([]UsedElement, count)
-	for i := range count {
-		elems[i] = r.ring[r.lastIndex%uint16(len(r.ring))]
-		r.lastIndex++
-	}
-
-	return stillNeedToTake, elems
-}
-
-func (r *UsedRing) takeOne() (UsedElement, bool) {
-	//r.mu.Lock()
-	//defer r.mu.Unlock()
-
-	count := r.availableToTake()
-	if count == 0 {
-		return UsedElement{}, false
-	}
-
-	// The number of new elements can never exceed the queue size.
-	if count > len(r.ring) {
-		panic("used ring contains more new elements than the ring is long")
-	}
-
-	out := r.ring[r.lastIndex%uint16(len(r.ring))]
-	r.lastIndex++
-
-	return out, true
-}
-
-// InitOfferSingle is only used to pre-fill the used queue at startup, and should not be used if the device is running!
-func (r *UsedRing) InitOfferSingle(x uint16, size uint32) {
-	offset := 0
-	// Add descriptor chain heads to the ring.
-
-	// The 16-bit ring index may overflow. This is expected and is not an
-	// issue because the size of the ring array (which equals the queue
-	// size) is always a power of 2 and smaller than the highest possible
-	// 16-bit value.
-	insertIndex := int(*r.ringIndex+uint16(offset)) % len(r.ring)
-	r.ring[insertIndex].DescriptorIndex = uint32(x)
-	r.ring[insertIndex].Length = size
-
-	// Increase the ring index by the number of descriptor chains added to the ring.
-	*r.ringIndex += 1
-}

overlay/virtqueue/used_ring_internal_test.go

@@ -1,136 +0,0 @@
-package virtqueue
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestUsedRing_MemoryLayout(t *testing.T) {
-	const queueSize = 2
-
-	memory := make([]byte, usedRingSize(queueSize))
-	r := newUsedRing(queueSize, memory)
-
-	*r.flags = 0x01ff
-	*r.ringIndex = 1
-	r.ring[0] = UsedElement{
-		DescriptorIndex: 0x0123,
-		Length:          0x4567,
-	}
-	r.ring[1] = UsedElement{
-		DescriptorIndex: 0x89ab,
-		Length:          0xcdef,
-	}
-
-	assert.Equal(t, []byte{
-		0xff, 0x01,
-		0x01, 0x00,
-		0x23, 0x01, 0x00, 0x00,
-		0x67, 0x45, 0x00, 0x00,
-		0xab, 0x89, 0x00, 0x00,
-		0xef, 0xcd, 0x00, 0x00,
-		0x00, 0x00,
-	}, memory)
-}
-
-//func TestUsedRing_Take(t *testing.T) {
-//	const queueSize = 8
-//
-//	tests := []struct {
-//		name      string
-//		ring      []UsedElement
-//		ringIndex uint16
-//		lastIndex uint16
-//		expected  []UsedElement
-//	}{
-//		{
-//			name: "nothing new",
-//			ring: []UsedElement{
-//				{DescriptorIndex: 1},
-//				{DescriptorIndex: 2},
-//				{DescriptorIndex: 3},
-//				{DescriptorIndex: 4},
-//				{},
-//				{},
-//				{},
-//				{},
-//			},
-//			ringIndex: 4,
-//			lastIndex: 4,
-//			expected:  nil,
-//		},
-//		{
-//			name: "no overflow",
-//			ring: []UsedElement{
-//				{DescriptorIndex: 1},
-//				{DescriptorIndex: 2},
-//				{DescriptorIndex: 3},
-//				{DescriptorIndex: 4},
-//				{},
-//				{},
-//				{},
-//				{},
-//			},
-//			ringIndex: 4,
-//			lastIndex: 1,
-//			expected: []UsedElement{
-//				{DescriptorIndex: 2},
-//				{DescriptorIndex: 3},
-//				{DescriptorIndex: 4},
-//			},
-//		},
-//		{
-//			name: "ring overflow",
-//			ring: []UsedElement{
-//				{DescriptorIndex: 9},
-//				{DescriptorIndex: 10},
-//				{DescriptorIndex: 3},
-//				{DescriptorIndex: 4},
-//				{DescriptorIndex: 5},
-//				{DescriptorIndex: 6},
-//				{DescriptorIndex: 7},
-//				{DescriptorIndex: 8},
-//			},
-//			ringIndex: 10,
-//			lastIndex: 7,
-//			expected: []UsedElement{
-//				{DescriptorIndex: 8},
-//				{DescriptorIndex: 9},
-//				{DescriptorIndex: 10},
-//			},
-//		},
-//		{
-//			name: "index overflow",
-//			ring: []UsedElement{
-//				{DescriptorIndex: 9},
-//				{DescriptorIndex: 10},
-//				{DescriptorIndex: 3},
-//				{DescriptorIndex: 4},
-//				{DescriptorIndex: 5},
-//				{DescriptorIndex: 6},
-//				{DescriptorIndex: 7},
-//				{DescriptorIndex: 8},
-//			},
-//			ringIndex: 2,
-//			lastIndex: 65535,
-//			expected: []UsedElement{
-//				{DescriptorIndex: 8},
-//				{DescriptorIndex: 9},
-//				{DescriptorIndex: 10},
-//			},
-//		},
-//	}
-//	for _, tt := range tests {
-//		t.Run(tt.name, func(t *testing.T) {
-//			memory := make([]byte, usedRingSize(queueSize))
-//			r := newUsedRing(queueSize, memory)
-//
-//			copy(r.ring, tt.ring)
-//			*r.ringIndex = tt.ringIndex
-//			r.lastIndex = tt.lastIndex
-//
-//			assert.Equal(t, tt.expected, r.take())
-//		})
-//	}
-//}

overlay/wireguard_tun_linux.go

@@ -0,0 +1,220 @@
+//go:build linux && !android && !e2e_testing
+
+package overlay
+
+import (
+	"fmt"
+	"sync"
+
+	wgtun "github.com/slackhq/nebula/wgstack/tun"
+)
+
+type wireguardTunIO struct {
+	dev       wgtun.Device
+	mtu       int
+	batchSize int
+
+	readMu      sync.Mutex
+	readBuffers [][]byte
+	readLens    []int
+	legacyBuf   []byte
+
+	writeMu      sync.Mutex
+	writeBuf     []byte
+	writeWrap    [][]byte
+	writeBuffers [][]byte
+}
+
+func newWireguardTunIO(dev wgtun.Device, mtu int) *wireguardTunIO {
+	batch := dev.BatchSize()
+	if batch <= 0 {
+		batch = 1
+	}
+	if mtu <= 0 {
+		mtu = DefaultMTU
+	}
+	return &wireguardTunIO{
+		dev:       dev,
+		mtu:       mtu,
+		batchSize: batch,
+		readLens:  make([]int, batch),
+		legacyBuf: make([]byte, wgtun.VirtioNetHdrLen+mtu),
+		writeBuf:  make([]byte, wgtun.VirtioNetHdrLen+mtu),
+		writeWrap: make([][]byte, 1),
+	}
+}
+
+func (w *wireguardTunIO) Read(p []byte) (int, error) {
+	w.readMu.Lock()
+	defer w.readMu.Unlock()
+
+	bufs := w.readBuffers
+	if len(bufs) == 0 {
+		bufs = [][]byte{w.legacyBuf}
+		w.readBuffers = bufs
+	}
+	n, err := w.dev.Read(bufs[:1], w.readLens[:1], wgtun.VirtioNetHdrLen)
+	if err != nil {
+		return 0, err
+	}
+	if n == 0 {
+		return 0, nil
+	}
+	length := w.readLens[0]
+	copy(p, w.legacyBuf[wgtun.VirtioNetHdrLen:wgtun.VirtioNetHdrLen+length])
+	return length, nil
+}
+
+func (w *wireguardTunIO) Write(p []byte) (int, error) {
+	if len(p) > w.mtu {
+		return 0, fmt.Errorf("wireguard tun: payload exceeds MTU (%d > %d)", len(p), w.mtu)
+	}
+	w.writeMu.Lock()
+	defer w.writeMu.Unlock()
+	buf := w.writeBuf[:wgtun.VirtioNetHdrLen+len(p)]
+	for i := 0; i < wgtun.VirtioNetHdrLen; i++ {
+		buf[i] = 0
+	}
+	copy(buf[wgtun.VirtioNetHdrLen:], p)
+	w.writeWrap[0] = buf
+	n, err := w.dev.Write(w.writeWrap, wgtun.VirtioNetHdrLen)
+	if err != nil {
+		return n, err
+	}
+	return len(p), nil
+}
+
+func (w *wireguardTunIO) ReadIntoBatch(pool *PacketPool) ([]*Packet, error) {
+	if pool == nil {
+		return nil, fmt.Errorf("wireguard tun: packet pool is nil")
+	}
+
+	w.readMu.Lock()
+	defer w.readMu.Unlock()
+
+	if len(w.readBuffers) < w.batchSize {
+		w.readBuffers = make([][]byte, w.batchSize)
+	}
+	if len(w.readLens) < w.batchSize {
+		w.readLens = make([]int, w.batchSize)
+	}
+
+	packets := make([]*Packet, w.batchSize)
+	requiredHeadroom := w.BatchHeadroom()
+	requiredPayload := w.BatchPayloadCap()
+	headroom := 0
+	for i := 0; i < w.batchSize; i++ {
+		pkt := pool.Get()
+		if pkt == nil {
+			releasePackets(packets[:i])
+			return nil, fmt.Errorf("wireguard tun: packet pool returned nil packet")
+		}
+		if pkt.Capacity() < requiredPayload {
+			pkt.Release()
+			releasePackets(packets[:i])
+			return nil, fmt.Errorf("wireguard tun: packet capacity %d below required %d", pkt.Capacity(), requiredPayload)
+		}
+		if i == 0 {
+			headroom = pkt.Offset
+			if headroom < requiredHeadroom {
+				pkt.Release()
+				releasePackets(packets[:i])
+				return nil, fmt.Errorf("wireguard tun: packet headroom %d below virtio requirement %d", headroom, requiredHeadroom)
+			}
+		} else if pkt.Offset != headroom {
+			pkt.Release()
+			releasePackets(packets[:i])
+			return nil, fmt.Errorf("wireguard tun: inconsistent packet headroom (%d != %d)", pkt.Offset, headroom)
+		}
+		packets[i] = pkt
+		w.readBuffers[i] = pkt.Buf
+	}
+
+	n, err := w.dev.Read(w.readBuffers[:w.batchSize], w.readLens[:w.batchSize], headroom)
+	if err != nil {
+		releasePackets(packets)
+		return nil, err
+	}
+	if n == 0 {
+		releasePackets(packets)
+		return nil, nil
+	}
+	for i := 0; i < n; i++ {
+		packets[i].Len = w.readLens[i]
+	}
+	for i := n; i < w.batchSize; i++ {
+		packets[i].Release()
+		packets[i] = nil
+	}
+	return packets[:n], nil
+}
+
+func (w *wireguardTunIO) WriteBatch(packets []*Packet) (int, error) {
+	if len(packets) == 0 {
+		return 0, nil
+	}
+	requiredHeadroom := w.BatchHeadroom()
+	offset := packets[0].Offset
+	if offset < requiredHeadroom {
+		releasePackets(packets)
+		return 0, fmt.Errorf("wireguard tun: packet offset %d smaller than required headroom %d", offset, requiredHeadroom)
+	}
+	for _, pkt := range packets {
+		if pkt == nil {
+			continue
+		}
+		if pkt.Offset != offset {
+			releasePackets(packets)
+			return 0, fmt.Errorf("wireguard tun: mixed packet offsets not supported")
+		}
+		limit := pkt.Offset + pkt.Len
+		if limit > len(pkt.Buf) {
+			releasePackets(packets)
+			return 0, fmt.Errorf("wireguard tun: packet length %d exceeds buffer capacity %d", pkt.Len, len(pkt.Buf)-pkt.Offset)
+		}
+	}
+	w.writeMu.Lock()
+	defer w.writeMu.Unlock()
+
+	if len(w.writeBuffers) < len(packets) {
+		w.writeBuffers = make([][]byte, len(packets))
+	}
+	for i, pkt := range packets {
+		if pkt == nil {
+			w.writeBuffers[i] = nil
+			continue
+		}
+		limit := pkt.Offset + pkt.Len
+		w.writeBuffers[i] = pkt.Buf[:limit]
+	}
+	n, err := w.dev.Write(w.writeBuffers[:len(packets)], offset)
+	if err != nil {
+		return n, err
+	}
+	releasePackets(packets)
+	return n, nil
+}
+
+func (w *wireguardTunIO) BatchHeadroom() int {
+	return wgtun.VirtioNetHdrLen
+}
+
+func (w *wireguardTunIO) BatchPayloadCap() int {
+	return w.mtu
+}
+
+func (w *wireguardTunIO) BatchSize() int {
+	return w.batchSize
+}
+
+func (w *wireguardTunIO) Close() error {
+	return nil
+}
+
+func releasePackets(pkts []*Packet) {
+	for _, pkt := range pkts {
+		if pkt != nil {
+			pkt.Release()
+		}
+	}
+}

packet/outpacket.go

@@ -1,75 +0,0 @@
-package packet
-
-import (
-	"github.com/slackhq/nebula/util/virtio"
-	"golang.org/x/sys/unix"
-)
-
-type OutPacket struct {
-	Segments [][]byte
-	// SegmentHeaders maps to the first virtio.NetHdrSize+14 bytes of Segments[n]
-	SegmentHeaders [][]byte
-	// SegmentPayloads maps to the remaining bytes of Segments[n]
-	SegmentPayloads [][]byte
-	// SegmentIDs is the list of underlying buffer IDs of Segments.
-	// SegmentIDs, Segments, SegmentHeaders, SegmentPayloads should all have the same length at all times!
-	SegmentIDs []uint16
-}
-
-func NewOut() *OutPacket {
-	out := new(OutPacket)
-	out.Segments = make([][]byte, 0, 64)
-	out.SegmentHeaders = make([][]byte, 0, 64)
-	out.SegmentPayloads = make([][]byte, 0, 64)
-	out.SegmentIDs = make([]uint16, 0, 64)
-	return out
-}
-
-func (pkt *OutPacket) Reset() {
-	pkt.Segments = pkt.Segments[:0]
-	pkt.SegmentPayloads = pkt.SegmentPayloads[:0]
-	pkt.SegmentHeaders = pkt.SegmentHeaders[:0]
-	pkt.SegmentIDs = pkt.SegmentIDs[:0]
-}
-
-// DestroyLastSegment removes the contents of the last segment in the list.
-// Use this to handle firewall drops or similar, but still hand the segment buffer back to the underlying driver.
-// Implementations shall discard zero-length segments internally.
-func (pkt *OutPacket) DestroyLastSegment() {
-	if len(pkt.Segments) == 0 {
-		return
-	}
-	lastSeg := len(pkt.SegmentIDs) - 1
-	pkt.SegmentPayloads[lastSeg] = pkt.SegmentPayloads[lastSeg][:0]
-	pkt.SegmentHeaders[lastSeg] = pkt.SegmentHeaders[lastSeg][:0]
-	pkt.Segments[lastSeg] = pkt.Segments[lastSeg][:0]
-}
-
-func (pkt *OutPacket) UseSegment(segID uint16, seg []byte, isV6 bool) int {
-	pkt.SegmentIDs = append(pkt.SegmentIDs, segID)
-	pkt.Segments = append(pkt.Segments, seg) //todo do we need this?
-
-	vhdr := virtio.NetHdr{ //todo
-		Flags:      unix.VIRTIO_NET_HDR_F_DATA_VALID,
-		GSOType:    unix.VIRTIO_NET_HDR_GSO_NONE,
-		HdrLen:     0,
-		GSOSize:    0,
-		CsumStart:  0,
-		CsumOffset: 0,
-		NumBuffers: 0,
-	}
-
-	hdr := seg[0 : virtio.NetHdrSize+14]
-	_ = vhdr.Encode(hdr)
-	if isV6 {
-		hdr[virtio.NetHdrSize+14-2] = 0x86
-		hdr[virtio.NetHdrSize+14-1] = 0xdd
-	} else {
-		hdr[virtio.NetHdrSize+14-2] = 0x08
-		hdr[virtio.NetHdrSize+14-1] = 0x00
-	}
-
-	pkt.SegmentHeaders = append(pkt.SegmentHeaders, hdr)
-	pkt.SegmentPayloads = append(pkt.SegmentPayloads, seg[virtio.NetHdrSize+14:])
-	return len(pkt.SegmentIDs) - 1
-}

packet/packet.go

@@ -1,148 +0,0 @@
-package packet
-
-import (
-	"encoding/binary"
-	"fmt"
-	"iter"
-	"net/netip"
-	"slices"
-	"syscall"
-	"unsafe"
-
-	"golang.org/x/sys/unix"
-)
-
-const Size = 0xffff
-
-type UDPPacket struct {
-	Payload []byte
-	Control []byte
-	Name    []byte
-	SegSize int
-
-	ReadyToSend  bool
-	wasSegmented bool
-	isV4         bool
-}
-
-func New(isV4 bool) *UDPPacket {
-	return &UDPPacket{
-		Payload: make([]byte, Size),
-		Control: make([]byte, unix.CmsgSpace(2)),
-		Name:    make([]byte, unix.SizeofSockaddrInet6),
-		isV4:    isV4,
-	}
-}
-
-func (p *UDPPacket) AddrPort() netip.AddrPort {
-	var ip netip.Addr
-	// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
-	if p.isV4 {
-		ip, _ = netip.AddrFromSlice(p.Name[4:8])
-	} else {
-		ip, _ = netip.AddrFromSlice(p.Name[8:24])
-	}
-	return netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(p.Name[2:4]))
-}
-
-func (p *UDPPacket) encodeSockaddr(dst []byte, addr netip.AddrPort) (uint32, error) {
-	//todo no chance this works on windows?
-	if p.isV4 {
-		if !addr.Addr().Is4() {
-			return 0, fmt.Errorf("Listener is IPv4, but writing to IPv6 remote")
-		}
-		var sa unix.RawSockaddrInet4
-		sa.Family = unix.AF_INET
-		sa.Addr = addr.Addr().As4()
-		binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
-		size := unix.SizeofSockaddrInet4
-		copy(dst[:size], (*(*[unix.SizeofSockaddrInet4]byte)(unsafe.Pointer(&sa)))[:])
-		return uint32(size), nil
-	}
-
-	var sa unix.RawSockaddrInet6
-	sa.Family = unix.AF_INET6
-	sa.Addr = addr.Addr().As16()
-	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
-	size := unix.SizeofSockaddrInet6
-	copy(dst[:size], (*(*[unix.SizeofSockaddrInet6]byte)(unsafe.Pointer(&sa)))[:])
-	return uint32(size), nil
-}
-
-func (p *UDPPacket) SetAddrPort(addr netip.AddrPort) error {
-	nl, err := p.encodeSockaddr(p.Name, addr)
-	if err != nil {
-		return err
-	}
-	p.Name = p.Name[:nl]
-	return nil
-}
-
-func (p *UDPPacket) updateCtrl(ctrlLen int) {
-	p.SegSize = len(p.Payload)
-	p.wasSegmented = false
-	if ctrlLen == 0 {
-		return
-	}
-	if len(p.Control) == 0 {
-		return
-	}
-	header, data, _ /*remain*/, err := unix.ParseOneSocketControlMessage(p.Control)
-	if err != nil {
-		return // oh well
-	}
-
-	if header.Level == unix.SOL_UDP && header.Type == unix.UDP_GRO && len(data) >= 2 {
-		p.wasSegmented = true
-		p.SegSize = int(binary.LittleEndian.Uint16(data[:2]))
-		return
-	}
-}
-
-// Update sets a UDPPacket into "just received, not processed" state
-func (p *UDPPacket) Update(ctrlLen int) {
-	p.updateCtrl(ctrlLen)
-}
-
-func (p *UDPPacket) SetSegSizeForTX() {
-	p.SegSize = len(p.Payload)
-	hdr := (*unix.Cmsghdr)(unsafe.Pointer(&p.Control[0]))
-	hdr.Level = unix.SOL_UDP
-	hdr.Type = unix.UDP_SEGMENT
-	hdr.SetLen(syscall.CmsgLen(2))
-	binary.NativeEndian.PutUint16(p.Control[unix.CmsgLen(0):unix.CmsgLen(0)+2], uint16(p.SegSize))
-}
-
-func (p *UDPPacket) CompatibleForSegmentationWith(otherP *UDPPacket, currentTotalSize int) bool {
-	//same dest
-	if !slices.Equal(p.Name, otherP.Name) {
-		return false
-	}
-
-	//don't get too big
-	if len(p.Payload)+currentTotalSize >= 0xffff {
-		return false
-	}
-
-	//same body len
-	//todo allow single different size at end
-	if len(p.Payload) != len(otherP.Payload) {
-		return false //todo technically you can cram one extra in
-	}
-	return true
-}
-
-func (p *UDPPacket) Segments() iter.Seq[[]byte] {
-	return func(yield func([]byte) bool) {
-		//cursor := 0
-		for offset := 0; offset < len(p.Payload); offset += p.SegSize {
-			end := offset + p.SegSize
-			if end > len(p.Payload) {
-				end = len(p.Payload)
-			}
-			if !yield(p.Payload[offset:end]) {
-				return
-			}
-		}
-	}
-}

pki.go

@@ -523,13 +523,9 @@ func loadCAPoolFromConfig(l *logrus.Logger, c *config.C) (*cert.CAPool, error) {
 		return nil, fmt.Errorf("error while adding CA certificate to CA trust store: %s", err)
 	}
 
-	bl := c.GetStringSlice("pki.blocklist", []string{})
+	for _, fp := range c.GetStringSlice("pki.blocklist", []string{}) {
-	if len(bl) > 0 {
+		l.WithField("fingerprint", fp).Info("Blocklisting cert")
-		for _, fp := range bl {
+		caPool.BlocklistFingerprint(fp)
-			caPool.BlocklistFingerprint(fp)
-		}
-
-		l.WithField("fingerprintCount", len(bl)).Info("Blocklisted certificates")
 	}
 
 	return caPool, nil

remote_list.go

@@ -338,21 +338,21 @@ func (r *RemoteList) CopyCache() *CacheMap {
 }
 
 // BlockRemote locks and records the address as bad, it will be excluded from the deduplicated address list
-func (r *RemoteList) BlockRemote(bad ViaSender) {
+func (r *RemoteList) BlockRemote(bad netip.AddrPort) {
-	if bad.IsRelayed {
+	if !bad.IsValid() {
+		// relays can have nil udp Addrs
 		return
 	}
-
 	r.Lock()
 	defer r.Unlock()
 
 	// Check if we already blocked this addr
-	if r.unlockedIsBad(bad.UdpAddr) {
+	if r.unlockedIsBad(bad) {
 		return
 	}
 
 	// We copy here because we are taking something else's memory and we can't trust everything
-	r.badRemotes = append(r.badRemotes, bad.UdpAddr)
+	r.badRemotes = append(r.badRemotes, bad)
 
 	// Mark the next interaction must recollect/dedupe
 	r.shouldRebuild = true

service/service_test.go

@@ -16,8 +16,8 @@ import (
 	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/overlay"
-	"go.yaml.in/yaml/v3"
 	"golang.org/x/sync/errgroup"
+	"gopkg.in/yaml.v3"
 )
 
 type m = map[string]any

test/tun.go

@@ -34,10 +34,6 @@ func (NoopTun) Write([]byte) (int, error) {
 	return 0, nil
 }
 
-func (NoopTun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (NoopTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, errors.New("unsupported")
 }

udp/conn.go

@@ -4,13 +4,13 @@ import (
 	"net/netip"
 
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/packet"
 )
 
 const MTU = 9001
 
 type EncReader func(
-	[]*packet.UDPPacket,
+	addr netip.AddrPort,
+	payload []byte,
 )
 
 type Conn interface {
@@ -19,12 +19,21 @@ type Conn interface {
 	ListenOut(r EncReader)
 	WriteTo(b []byte, addr netip.AddrPort) error
 	ReloadConfig(c *config.C)
-	Prep(pkt *packet.UDPPacket, addr netip.AddrPort) error
-	WriteBatch(pkt []*packet.UDPPacket) (int, error)
-	SupportsMultipleReaders() bool
 	Close() error
 }
 
+// Datagram represents a UDP payload destined to a specific address.
+type Datagram struct {
+	Payload []byte
+	Addr    netip.AddrPort
+}
+
+// BatchConn can send multiple datagrams in one syscall.
+type BatchConn interface {
+	Conn
+	WriteBatch(pkts []Datagram) error
+}
+
 type NoopConn struct{}
 
 func (NoopConn) Rebind() error {
@@ -36,9 +45,6 @@ func (NoopConn) LocalAddr() (netip.AddrPort, error) {
 func (NoopConn) ListenOut(_ EncReader) {
 	return
 }
-func (NoopConn) SupportsMultipleReaders() bool {
-	return false
-}
 func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
 	return nil
 }

udp/udp_darwin.go

@@ -98,9 +98,9 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
 			return ErrInvalidIPv6RemoteForSocket
 		}
 
-		var rsa unix.RawSockaddrInet4
+		var rsa unix.RawSockaddrInet6
-		rsa.Family = unix.AF_INET
+		rsa.Family = unix.AF_INET6
-		rsa.Addr = ap.Addr().As4()
+		rsa.Addr = ap.Addr().As16()
 		binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ap.Port())
 		sa = unsafe.Pointer(&rsa)
 		addrLen = syscall.SizeofSockaddrInet4
@@ -184,10 +184,6 @@ func (u *StdConn) ListenOut(r EncReader) {
 	}
 }
 
-func (u *StdConn) SupportsMultipleReaders() bool {
-	return false
-}
-
 func (u *StdConn) Rebind() error {
 	var err error
 	if u.isV4 {

udp/udp_generic.go

@@ -85,7 +85,3 @@ func (u *GenericConn) ListenOut(r EncReader) {
 		r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n])
 	}
 }
-
-func (u *GenericConn) SupportsMultipleReaders() bool {
-	return false
-}

udp/udp_linux.go

@@ -14,22 +14,22 @@ import (
 	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/packet"
 	"golang.org/x/sys/unix"
 )
 
-const iovMax = 128 //1024 //no unix constant for this? from limits.h
-//todo I'd like this to be 1024 but we seem to hit errors around ~130?
-
 type StdConn struct {
-	sysFd     int
+	sysFd int
-	isV4      bool
+	isV4  bool
-	l         *logrus.Logger
+	l     *logrus.Logger
-	batch     int
+	batch int
-	enableGRO bool
+}
 
-	msgs []rawMessage
+func maybeIPV4(ip net.IP) (net.IP, bool) {
-	iovs [][]iovec
+	ip4 := ip.To4()
+	if ip4 != nil {
+		return ip4, true
+	}
+	return ip, false
 }
 
 func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
@@ -69,24 +69,7 @@ func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch in
 		return nil, fmt.Errorf("unable to bind to socket: %s", err)
 	}
 
-	const batchSize = 8192
+	return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, err
-	msgs := make([]rawMessage, 0, batchSize) //todo configure
-	iovs := make([][]iovec, batchSize)
-	for i := range iovs {
-		iovs[i] = make([]iovec, iovMax)
-	}
-	return &StdConn{
-		sysFd: fd,
-		isV4:  ip.Is4(),
-		l:     l,
-		batch: batch,
-		msgs:  msgs,
-		iovs:  iovs,
-	}, err
-}
-
-func (u *StdConn) SupportsMultipleReaders() bool {
-	return true
 }
 
 func (u *StdConn) Rebind() error {
@@ -136,7 +119,9 @@ func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
 }
 
 func (u *StdConn) ListenOut(r EncReader) {
-	msgs, packets := u.PrepareRawMessages(u.batch, u.isV4)
+	var ip netip.Addr
+
+	msgs, buffers, names := u.PrepareRawMessages(u.batch)
 	read := u.ReadMulti
 	if u.batch == 1 {
 		read = u.ReadSingle
@@ -150,12 +135,13 @@ func (u *StdConn) ListenOut(r EncReader) {
 		}
 
 		for i := 0; i < n; i++ {
-			packets[i].Payload = packets[i].Payload[:msgs[i].Len]
+			// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
-			packets[i].Update(getRawMessageControlLen(&msgs[i]))
+			if u.isV4 {
-		}
+				ip, _ = netip.AddrFromSlice(names[i][4:8])
-		r(packets[:n])
+			} else {
-		for i := 0; i < n; i++ { //todo reset this in prev loop, but this makes debug ez
+				ip, _ = netip.AddrFromSlice(names[i][8:24])
-			msgs[i].Hdr.Controllen = uint64(unix.CmsgSpace(2))
+			}
+			r(netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4])), buffers[i][:msgs[i].Len])
 		}
 	}
 }
@@ -208,148 +194,6 @@ func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
 	return u.writeTo6(b, ip)
 }
 
-func (u *StdConn) WriteToBatch(b []byte, ip netip.AddrPort) error {
-	if u.isV4 {
-		return u.writeTo4(b, ip)
-	}
-	return u.writeTo6(b, ip)
-}
-
-func (u *StdConn) Prep(pkt *packet.UDPPacket, addr netip.AddrPort) error {
-	//todo move this into pkt
-	nl, err := u.encodeSockaddr(pkt.Name, addr)
-	if err != nil {
-		return err
-	}
-	pkt.ReadyToSend = true
-	pkt.Name = pkt.Name[:nl]
-	return nil
-}
-
-func (u *StdConn) WriteBatch(pkts []*packet.UDPPacket) (int, error) {
-	if len(pkts) == 0 {
-		return 0, nil
-	}
-
-	u.msgs = u.msgs[:0]
-	//u.iovs = u.iovs[:0]
-
-	sent := 0
-	var mostRecentPkt *packet.UDPPacket
-	mostRecentPktSize := 0
-	//segmenting := false
-	idx := 0
-	for _, pkt := range pkts {
-		if !pkt.ReadyToSend || len(pkt.Payload) == 0 {
-			sent++
-			continue
-		}
-		lastIdx := idx - 1
-		if mostRecentPkt != nil && pkt.CompatibleForSegmentationWith(mostRecentPkt, mostRecentPktSize) && u.msgs[lastIdx].Hdr.Iovlen < iovMax {
-			u.msgs[lastIdx].Hdr.Controllen = uint64(len(mostRecentPkt.Control))
-			u.msgs[lastIdx].Hdr.Control = &mostRecentPkt.Control[0]
-
-			u.iovs[lastIdx][u.msgs[lastIdx].Hdr.Iovlen].Base = &pkt.Payload[0]
-			u.iovs[lastIdx][u.msgs[lastIdx].Hdr.Iovlen].Len = uint64(len(pkt.Payload))
-			u.msgs[lastIdx].Hdr.Iovlen++
-
-			mostRecentPktSize += len(pkt.Payload)
-			mostRecentPkt.SetSegSizeForTX()
-		} else {
-			u.msgs = append(u.msgs, rawMessage{})
-			u.iovs[idx][0] = iovec{
-				Base: &pkt.Payload[0],
-				Len:  uint64(len(pkt.Payload)),
-			}
-
-			msg := &u.msgs[idx]
-			iov := &u.iovs[idx][0]
-			idx++
-
-			msg.Hdr.Iov = iov
-			msg.Hdr.Iovlen = 1
-			setRawMessageControl(msg, nil)
-			msg.Hdr.Flags = 0
-
-			msg.Hdr.Name = &pkt.Name[0]
-			msg.Hdr.Namelen = uint32(len(pkt.Name))
-			mostRecentPkt = pkt
-			mostRecentPktSize = len(pkt.Payload)
-		}
-	}
-
-	if len(u.msgs) == 0 {
-		return sent, nil
-	}
-
-	offset := 0
-	for offset < len(u.msgs) {
-		n, _, errno := unix.Syscall6(
-			unix.SYS_SENDMMSG,
-			uintptr(u.sysFd),
-			uintptr(unsafe.Pointer(&u.msgs[offset])),
-			uintptr(len(u.msgs)-offset),
-			0,
-			0,
-			0,
-		)
-
-		if errno != 0 {
-			if errno == unix.EINTR {
-				continue
-			}
-			//for i := 0; i < len(u.msgs); i++ {
-			//	for j := 0; j < int(u.msgs[i].Hdr.Iovlen); j++ {
-			//		u.l.WithFields(logrus.Fields{
-			//			"msg_index": i,
-			//			"iov idx":   j,
-			//			"iov":       fmt.Sprintf("%+v", u.iovs[i][j]),
-			//		}).Warn("failed to send message")
-			//	}
-			//
-			//}
-			u.l.WithFields(logrus.Fields{
-				"errno":   errno,
-				"idx":     idx,
-				"len":     len(u.msgs),
-				"deets":   fmt.Sprintf("%+v", u.msgs),
-				"lastIOV": fmt.Sprintf("%+v", u.iovs[len(u.msgs)-1][u.msgs[len(u.msgs)-1].Hdr.Iovlen-1]),
-			}).Error("failed to send message")
-			return sent + offset, &net.OpError{Op: "sendmmsg", Err: errno}
-		}
-
-		if n == 0 {
-			break
-		}
-		offset += int(n)
-	}
-
-	return sent + len(u.msgs), nil
-}
-
-func (u *StdConn) encodeSockaddr(dst []byte, addr netip.AddrPort) (uint32, error) {
-	if u.isV4 {
-		if !addr.Addr().Is4() {
-			return 0, fmt.Errorf("Listener is IPv4, but writing to IPv6 remote")
-		}
-		var sa unix.RawSockaddrInet4
-		sa.Family = unix.AF_INET
-		sa.Addr = addr.Addr().As4()
-		binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
-		size := unix.SizeofSockaddrInet4
-		copy(dst[:size], (*(*[unix.SizeofSockaddrInet4]byte)(unsafe.Pointer(&sa)))[:])
-		return uint32(size), nil
-	}
-
-	var sa unix.RawSockaddrInet6
-	sa.Family = unix.AF_INET6
-	sa.Addr = addr.Addr().As16()
-	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
-	size := unix.SizeofSockaddrInet6
-	copy(dst[:size], (*(*[unix.SizeofSockaddrInet6]byte)(unsafe.Pointer(&sa)))[:])
-	return uint32(size), nil
-}
-
 func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
 	var rsa unix.RawSockaddrInet6
 	rsa.Family = unix.AF_INET6
@@ -450,27 +294,6 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 			u.l.WithError(err).Error("Failed to set listen.so_mark")
 		}
 	}
-	u.configureGRO(true)
-}
-
-func (u *StdConn) configureGRO(enable bool) {
-	if enable == u.enableGRO {
-		return
-	}
-
-	if enable {
-		if err := unix.SetsockoptInt(u.sysFd, unix.SOL_UDP, unix.UDP_GRO, 1); err != nil {
-			u.l.WithError(err).Warn("Failed to enable UDP GRO")
-			return
-		}
-		u.enableGRO = true
-		u.l.Info("UDP GRO enabled")
-	} else {
-		if err := unix.SetsockoptInt(u.sysFd, unix.SOL_UDP, unix.UDP_GRO, 0); err != nil && err != unix.ENOPROTOOPT {
-			u.l.WithError(err).Warn("Failed to disable UDP GRO")
-		}
-		u.enableGRO = false
-	}
 }
 
 func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
@@ -487,31 +310,51 @@ func (u *StdConn) Close() error {
 }
 
 func NewUDPStatsEmitter(udpConns []Conn) func() {
-	// Check if our kernel supports SO_MEMINFO before registering the gauges
+	if len(udpConns) == 0 {
-	var udpGauges [][unix.SK_MEMINFO_VARS]metrics.Gauge
+		return func() {}
+	}
+
+	type statsProvider struct {
+		index int
+		conn  *StdConn
+	}
+
+	providers := make([]statsProvider, 0, len(udpConns))
+	for i, c := range udpConns {
+		if sc, ok := c.(*StdConn); ok {
+			providers = append(providers, statsProvider{index: i, conn: sc})
+		}
+	}
+
+	if len(providers) == 0 {
+		return func() {}
+	}
+
 	var meminfo [unix.SK_MEMINFO_VARS]uint32
-	if err := udpConns[0].(*StdConn).getMemInfo(&meminfo); err == nil {
+	if err := providers[0].conn.getMemInfo(&meminfo); err != nil {
-		udpGauges = make([][unix.SK_MEMINFO_VARS]metrics.Gauge, len(udpConns))
+		return func() {}
-		for i := range udpConns {
+	}
-			udpGauges[i] = [unix.SK_MEMINFO_VARS]metrics.Gauge{
+
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.rmem_alloc", i), nil),
+	udpGauges := make([][unix.SK_MEMINFO_VARS]metrics.Gauge, len(providers))
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.rcvbuf", i), nil),
+	for i, provider := range providers {
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.wmem_alloc", i), nil),
+		udpGauges[i] = [unix.SK_MEMINFO_VARS]metrics.Gauge{
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.sndbuf", i), nil),
+			metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.rmem_alloc", provider.index), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.fwd_alloc", i), nil),
+			metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.rcvbuf", provider.index), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.wmem_queued", i), nil),
+			metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.wmem_alloc", provider.index), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.optmem", i), nil),
+			metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.sndbuf", provider.index), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.backlog", i), nil),
+			metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.fwd_alloc", provider.index), nil),
-				metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.drops", i), nil),
+			metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.wmem_queued", provider.index), nil),
-			}
+			metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.optmem", provider.index), nil),
+			metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.backlog", provider.index), nil),
+			metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.drops", provider.index), nil),
 		}
 	}
 
 	return func() {
-		for i, gauges := range udpGauges {
+		for i, provider := range providers {
-			if err := udpConns[i].(*StdConn).getMemInfo(&meminfo); err == nil {
+			if err := provider.conn.getMemInfo(&meminfo); err == nil {
 				for j := 0; j < unix.SK_MEMINFO_VARS; j++ {
-					gauges[j].Update(int64(meminfo[j]))
+					udpGauges[i][j].Update(int64(meminfo[j]))
 				}
 			}
 		}

udp/udp_linux_64.go

@@ -7,7 +7,6 @@
 package udp
 
 import (
-	"github.com/slackhq/nebula/packet"
 	"golang.org/x/sys/unix"
 )
 
@@ -34,59 +33,25 @@ type rawMessage struct {
 	Pad0 [4]byte
 }
 
-func setRawMessageControl(msg *rawMessage, buf []byte) {
+func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
-	if len(buf) == 0 {
-		msg.Hdr.Control = nil
-		msg.Hdr.Controllen = 0
-		return
-	}
-	msg.Hdr.Control = &buf[0]
-	msg.Hdr.Controllen = uint64(len(buf))
-}
-
-func getRawMessageControlLen(msg *rawMessage) int {
-	return int(msg.Hdr.Controllen)
-}
-
-func setCmsgLen(h *unix.Cmsghdr, l int) {
-	h.Len = uint64(l)
-}
-
-func (u *StdConn) PrepareRawMessages(n int, isV4 bool) ([]rawMessage, []*packet.UDPPacket) {
 	msgs := make([]rawMessage, n)
-	packets := make([]*packet.UDPPacket, n)
+	buffers := make([][]byte, n)
+	names := make([][]byte, n)
 
 	for i := range msgs {
-		packets[i] = packet.New(isV4)
+		buffers[i] = make([]byte, MTU)
+		names[i] = make([]byte, unix.SizeofSockaddrInet6)
 
 		vs := []iovec{
-			{Base: &packets[i].Payload[0], Len: uint64(packet.Size)},
+			{Base: &buffers[i][0], Len: uint64(len(buffers[i]))},
 		}
 
 		msgs[i].Hdr.Iov = &vs[0]
 		msgs[i].Hdr.Iovlen = uint64(len(vs))
 
-		msgs[i].Hdr.Name = &packets[i].Name[0]
+		msgs[i].Hdr.Name = &names[i][0]
-		msgs[i].Hdr.Namelen = uint32(len(packets[i].Name))
+		msgs[i].Hdr.Namelen = uint32(len(names[i]))
-
-		if u.enableGRO {
-			msgs[i].Hdr.Control = &packets[i].Control[0]
-			msgs[i].Hdr.Controllen = uint64(len(packets[i].Control))
-		} else {
-			msgs[i].Hdr.Control = nil
-			msgs[i].Hdr.Controllen = 0
-		}
 	}
 
-	return msgs, packets
+	return msgs, buffers, names
-}
-
-func setIovecSlice(iov *iovec, b []byte) {
-	if len(b) == 0 {
-		iov.Base = nil
-		iov.Len = 0
-		return
-	}
-	iov.Base = &b[0]
-	iov.Len = uint64(len(b))
 }

udp/udp_rio_windows.go

@@ -315,10 +315,6 @@ func (u *RIOConn) LocalAddr() (netip.AddrPort, error) {
 
 }
 
-func (u *RIOConn) SupportsMultipleReaders() bool {
-	return false
-}
-
 func (u *RIOConn) Rebind() error {
 	return nil
 }

udp/udp_tester.go

@@ -11,7 +11,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/packet"
 )
 
 type Packet struct {
@@ -41,11 +40,6 @@ type TesterConn struct {
 	l      *logrus.Logger
 }
 
-func (u *TesterConn) Prep(pkt *packet.UDPPacket, addr netip.AddrPort) error {
-	pkt.ReadyToSend = true
-	return pkt.SetAddrPort(addr)
-}
-
 func NewListener(l *logrus.Logger, ip netip.Addr, port int, _ bool, _ int) (Conn, error) {
 	return &TesterConn{
 		Addr:      netip.AddrPortFrom(ip, uint16(port)),
@@ -96,19 +90,6 @@ func (u *TesterConn) Get(block bool) *Packet {
 // Below this is boilerplate implementation to make nebula actually work
 //********************************************************************************************************************//
 
-func (u *TesterConn) WriteBatch(pkts []*packet.UDPPacket) (int, error) {
-	for _, pkt := range pkts {
-		if !pkt.ReadyToSend {
-			continue
-		}
-		err := u.WriteTo(pkt.Payload, pkt.AddrPort())
-		if err != nil {
-			return 0, err
-		}
-	}
-	return len(pkts), nil
-}
-
 func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
 	if u.closed.Load() {
 		return io.ErrClosedPipe
@@ -119,9 +100,6 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
 		From: u.Addr,
 		To:   addr,
 	}
-	if addr.Addr().IsUnspecified() {
-		panic("invalid address")
-	}
 
 	copy(p.Data, b)
 	u.TxPackets <- p
@@ -134,15 +112,7 @@ func (u *TesterConn) ListenOut(r EncReader) {
 		if !ok {
 			return
 		}
-		x := packet.New(p.From.Addr().Is4())
+		r(p.From, p.Data)
-		x.Payload = p.Data
-		x.SetSegSizeForTX()
-		err := x.SetAddrPort(p.From)
-		if err != nil {
-			panic(err)
-		}
-		y := []*packet.UDPPacket{x}
-		r(y)
 	}
 }
 
@@ -157,10 +127,6 @@ func (u *TesterConn) LocalAddr() (netip.AddrPort, error) {
 	return u.Addr, nil
 }
 
-func (u *TesterConn) SupportsMultipleReaders() bool {
-	return false
-}
-
 func (u *TesterConn) Rebind() error {
 	return nil
 }

udp/wireguard_conn_linux.go

@@ -0,0 +1,225 @@
+//go:build linux && !android && !e2e_testing
+
+package udp
+
+import (
+	"errors"
+	"net"
+	"net/netip"
+	"sync"
+	"sync/atomic"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/config"
+	wgconn "github.com/slackhq/nebula/wgstack/conn"
+)
+
+// WGConn adapts WireGuard's batched UDP bind implementation to Nebula's udp.Conn interface.
+type WGConn struct {
+	l         *logrus.Logger
+	bind      *wgconn.StdNetBind
+	recvers   []wgconn.ReceiveFunc
+	batch     int
+	reqBatch  int
+	localIP   netip.Addr
+	localPort uint16
+	enableGSO bool
+	enableGRO bool
+	gsoMaxSeg int
+	closed    atomic.Bool
+
+	closeOnce sync.Once
+}
+
+// NewWireguardListener creates a UDP listener backed by WireGuard's StdNetBind.
+func NewWireguardListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
+	bind := wgconn.NewStdNetBindForAddr(ip, multi)
+	recvers, actualPort, err := bind.Open(uint16(port))
+	if err != nil {
+		return nil, err
+	}
+	if batch <= 0 {
+		batch = bind.BatchSize()
+	} else if batch > bind.BatchSize() {
+		batch = bind.BatchSize()
+	}
+	return &WGConn{
+		l:         l,
+		bind:      bind,
+		recvers:   recvers,
+		batch:     batch,
+		reqBatch:  batch,
+		localIP:   ip,
+		localPort: actualPort,
+	}, nil
+}
+
+func (c *WGConn) Rebind() error {
+	// WireGuard's bind does not support rebinding in place.
+	return nil
+}
+
+func (c *WGConn) LocalAddr() (netip.AddrPort, error) {
+	if !c.localIP.IsValid() || c.localIP.IsUnspecified() {
+		// Fallback to wildcard IPv4 for display purposes.
+		return netip.AddrPortFrom(netip.IPv4Unspecified(), c.localPort), nil
+	}
+	return netip.AddrPortFrom(c.localIP, c.localPort), nil
+}
+
+func (c *WGConn) listen(fn wgconn.ReceiveFunc, r EncReader) {
+	batchSize := c.batch
+	packets := make([][]byte, batchSize)
+	for i := range packets {
+		packets[i] = make([]byte, MTU)
+	}
+	sizes := make([]int, batchSize)
+	endpoints := make([]wgconn.Endpoint, batchSize)
+
+	for {
+		if c.closed.Load() {
+			return
+		}
+		n, err := fn(packets, sizes, endpoints)
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
+				return
+			}
+			if c.l != nil {
+				c.l.WithError(err).Debug("wireguard UDP listener receive error")
+			}
+			continue
+		}
+		for i := 0; i < n; i++ {
+			if sizes[i] == 0 {
+				continue
+			}
+			stdEp, ok := endpoints[i].(*wgconn.StdNetEndpoint)
+			if !ok {
+				if c.l != nil {
+					c.l.Warn("wireguard UDP listener received unexpected endpoint type")
+				}
+				continue
+			}
+			addr := stdEp.AddrPort
+			r(addr, packets[i][:sizes[i]])
+			endpoints[i] = nil
+		}
+	}
+}
+
+func (c *WGConn) ListenOut(r EncReader) {
+	for _, fn := range c.recvers {
+		go c.listen(fn, r)
+	}
+}
+
+func (c *WGConn) WriteTo(b []byte, addr netip.AddrPort) error {
+	if len(b) == 0 {
+		return nil
+	}
+	if c.closed.Load() {
+		return net.ErrClosed
+	}
+	ep := &wgconn.StdNetEndpoint{AddrPort: addr}
+	return c.bind.Send([][]byte{b}, ep)
+}
+
+func (c *WGConn) WriteBatch(datagrams []Datagram) error {
+	if len(datagrams) == 0 {
+		return nil
+	}
+	if c.closed.Load() {
+		return net.ErrClosed
+	}
+	max := c.batch
+	if max <= 0 {
+		max = len(datagrams)
+		if max == 0 {
+			max = 1
+		}
+	}
+	bufs := make([][]byte, 0, max)
+	var (
+		current  netip.AddrPort
+		endpoint *wgconn.StdNetEndpoint
+		haveAddr bool
+	)
+	flush := func() error {
+		if len(bufs) == 0 || endpoint == nil {
+			bufs = bufs[:0]
+			return nil
+		}
+		err := c.bind.Send(bufs, endpoint)
+		bufs = bufs[:0]
+		return err
+	}
+
+	for _, d := range datagrams {
+		if len(d.Payload) == 0 || !d.Addr.IsValid() {
+			continue
+		}
+		if !haveAddr || d.Addr != current {
+			if err := flush(); err != nil {
+				return err
+			}
+			current = d.Addr
+			endpoint = &wgconn.StdNetEndpoint{AddrPort: current}
+			haveAddr = true
+		}
+		bufs = append(bufs, d.Payload)
+		if len(bufs) >= max {
+			if err := flush(); err != nil {
+				return err
+			}
+		}
+	}
+	return flush()
+}
+
+func (c *WGConn) ConfigureOffload(enableGSO, enableGRO bool, maxSegments int) {
+	c.enableGSO = enableGSO
+	c.enableGRO = enableGRO
+	if maxSegments <= 0 {
+		maxSegments = 1
+	} else if maxSegments > wgconn.IdealBatchSize {
+		maxSegments = wgconn.IdealBatchSize
+	}
+	c.gsoMaxSeg = maxSegments
+
+	effectiveBatch := c.reqBatch
+	if enableGSO && c.bind != nil {
+		bindBatch := c.bind.BatchSize()
+		if effectiveBatch < bindBatch {
+			if c.l != nil {
+				c.l.WithFields(logrus.Fields{
+					"requested": c.reqBatch,
+					"effective": bindBatch,
+				}).Warn("listen.batch below wireguard minimum; using bind batch size for UDP GSO support")
+			}
+			effectiveBatch = bindBatch
+		}
+	}
+	c.batch = effectiveBatch
+
+	if c.l != nil {
+		c.l.WithFields(logrus.Fields{
+			"enableGSO":      enableGSO,
+			"enableGRO":      enableGRO,
+			"gsoMaxSegments": maxSegments,
+		}).Debug("configured wireguard UDP offload")
+	}
+}
+
+func (c *WGConn) ReloadConfig(*config.C) {
+	// WireGuard bind currently does not expose runtime configuration knobs.
+}
+
+func (c *WGConn) Close() error {
+	var err error
+	c.closeOnce.Do(func() {
+		c.closed.Store(true)
+		err = c.bind.Close()
+	})
+	return err
+}

udp/wireguard_conn_unsupported.go

@@ -0,0 +1,15 @@
+//go:build !linux || android || e2e_testing
+
+package udp
+
+import (
+	"fmt"
+	"net/netip"
+
+	"github.com/sirupsen/logrus"
+)
+
+// NewWireguardListener is only available on Linux builds.
+func NewWireguardListener(*logrus.Logger, netip.Addr, int, bool, int) (Conn, error) {
+	return nil, fmt.Errorf("wireguard experimental UDP listener is only supported on Linux")
+}

util/virtio/doc.go

@@ -1,3 +0,0 @@
-// Package virtio contains some generic types and concepts related to the virtio
-// protocol.
-package virtio