mirror of
https://github.com/slackhq/nebula.git
synced 2026-08-15 13:16:58 +02:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ef1739bec4 | |||
| 030b7e2763 | |||
| 6b6a4bc1cc | |||
| 30db76ed79 | |||
| 15333f9fed |
@@ -1,4 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
const NoSuchFileError = "no such file or directory"
|
|
||||||
const NoSuchDirError = "no such file or directory"
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
const NoSuchFileError = "no such file or directory"
|
|
||||||
const NoSuchDirError = "no such file or directory"
|
|
||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
"github.com/slackhq/nebula/cert"
|
"github.com/slackhq/nebula/cert"
|
||||||
"github.com/slackhq/nebula/config"
|
"github.com/slackhq/nebula/config"
|
||||||
"github.com/slackhq/nebula/test"
|
"github.com/slackhq/nebula/test"
|
||||||
"github.com/slackhq/nebula/test/device"
|
|
||||||
"github.com/slackhq/nebula/udp"
|
"github.com/slackhq/nebula/udp"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -53,7 +52,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
|
|||||||
lh := newTestLighthouse()
|
lh := newTestLighthouse()
|
||||||
ifce := &Interface{
|
ifce := &Interface{
|
||||||
hostMap: hostMap,
|
hostMap: hostMap,
|
||||||
inside: &device.NoopTun{},
|
inside: &test.NoopTun{},
|
||||||
outside: &udp.NoopConn{},
|
outside: &udp.NoopConn{},
|
||||||
firewall: &Firewall{},
|
firewall: &Firewall{},
|
||||||
lightHouse: lh,
|
lightHouse: lh,
|
||||||
@@ -136,7 +135,7 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
|
|||||||
lh := newTestLighthouse()
|
lh := newTestLighthouse()
|
||||||
ifce := &Interface{
|
ifce := &Interface{
|
||||||
hostMap: hostMap,
|
hostMap: hostMap,
|
||||||
inside: &device.NoopTun{},
|
inside: &test.NoopTun{},
|
||||||
outside: &udp.NoopConn{},
|
outside: &udp.NoopConn{},
|
||||||
firewall: &Firewall{},
|
firewall: &Firewall{},
|
||||||
lightHouse: lh,
|
lightHouse: lh,
|
||||||
@@ -221,7 +220,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
|
|||||||
lh := newTestLighthouse()
|
lh := newTestLighthouse()
|
||||||
ifce := &Interface{
|
ifce := &Interface{
|
||||||
hostMap: hostMap,
|
hostMap: hostMap,
|
||||||
inside: &device.NoopTun{},
|
inside: &test.NoopTun{},
|
||||||
outside: &udp.NoopConn{},
|
outside: &udp.NoopConn{},
|
||||||
firewall: &Firewall{},
|
firewall: &Firewall{},
|
||||||
lightHouse: lh,
|
lightHouse: lh,
|
||||||
@@ -348,7 +347,7 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
|
|||||||
lh := newTestLighthouse()
|
lh := newTestLighthouse()
|
||||||
ifce := &Interface{
|
ifce := &Interface{
|
||||||
hostMap: hostMap,
|
hostMap: hostMap,
|
||||||
inside: &device.NoopTun{},
|
inside: &test.NoopTun{},
|
||||||
outside: &udp.NoopConn{},
|
outside: &udp.NoopConn{},
|
||||||
firewall: &Firewall{},
|
firewall: &Firewall{},
|
||||||
lightHouse: lh,
|
lightHouse: lh,
|
||||||
|
|||||||
+5
-14
@@ -81,14 +81,8 @@ type FirewallConntrack struct {
|
|||||||
|
|
||||||
Conns map[firewall.Packet]*conn
|
Conns map[firewall.Packet]*conn
|
||||||
TimerWheel *TimerWheel[firewall.Packet]
|
TimerWheel *TimerWheel[firewall.Packet]
|
||||||
|
|
||||||
// purgeCounter tracks lookups to trigger periodic purge instead of every lookup
|
|
||||||
purgeCounter uint32
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// purgeInterval defines how many lookups between purge attempts
|
|
||||||
const conntrackPurgeInterval = 1024
|
|
||||||
|
|
||||||
// FirewallTable is the entry point for a rule, the evaluation order is:
|
// FirewallTable is the entry point for a rule, the evaluation order is:
|
||||||
// Proto AND port AND (CA SHA or CA name) AND local CIDR AND (group OR groups OR name OR remote CIDR)
|
// Proto AND port AND (CA SHA or CA name) AND local CIDR AND (group OR groups OR name OR remote CIDR)
|
||||||
type FirewallTable struct {
|
type FirewallTable struct {
|
||||||
@@ -498,17 +492,14 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
|
|||||||
conntrack := f.Conntrack
|
conntrack := f.Conntrack
|
||||||
conntrack.Lock()
|
conntrack.Lock()
|
||||||
|
|
||||||
// Periodic purge instead of every lookup (major CPU savings)
|
// Purge every time we test
|
||||||
conntrack.purgeCounter++
|
ep, has := conntrack.TimerWheel.Purge()
|
||||||
if conntrack.purgeCounter >= conntrackPurgeInterval {
|
if has {
|
||||||
conntrack.purgeCounter = 0
|
f.evict(ep)
|
||||||
ep, has := conntrack.TimerWheel.Purge()
|
|
||||||
if has {
|
|
||||||
f.evict(ep)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
c, ok := conntrack.Conns[fp]
|
c, ok := conntrack.Conns[fp]
|
||||||
|
|
||||||
if !ok {
|
if !ok {
|
||||||
conntrack.Unlock()
|
conntrack.Unlock()
|
||||||
return false
|
return false
|
||||||
|
|||||||
@@ -30,11 +30,11 @@ require (
|
|||||||
golang.org/x/sys v0.40.0
|
golang.org/x/sys v0.40.0
|
||||||
golang.org/x/term v0.39.0
|
golang.org/x/term v0.39.0
|
||||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
|
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
|
||||||
golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb
|
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
|
||||||
golang.zx2c4.com/wireguard/windows v0.5.3
|
golang.zx2c4.com/wireguard/windows v0.5.3
|
||||||
google.golang.org/protobuf v1.36.11
|
google.golang.org/protobuf v1.36.11
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c
|
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@@ -50,6 +50,6 @@ require (
|
|||||||
github.com/vishvananda/netns v0.0.5 // indirect
|
github.com/vishvananda/netns v0.0.5 // indirect
|
||||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||||
golang.org/x/mod v0.31.0 // indirect
|
golang.org/x/mod v0.31.0 // indirect
|
||||||
golang.org/x/time v0.7.0 // indirect
|
golang.org/x/time v0.5.0 // indirect
|
||||||
golang.org/x/tools v0.40.0 // indirect
|
golang.org/x/tools v0.40.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -216,8 +216,8 @@ golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
|
|||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
|
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
||||||
golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
@@ -231,8 +231,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
|
|||||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
|
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
|
||||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
|
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
|
||||||
golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb h1:whnFRlWMcXI9d+ZbWg+4sHnLp52d5yiIPUxMBSt4X9A=
|
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
|
||||||
golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb/go.mod h1:rpwXGsirqLqN2L0JDJQlwOboGHmptD5ZD6T2VmcqhTw=
|
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
|
||||||
golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
|
golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
|
||||||
golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
|
golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
|
||||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||||
@@ -258,5 +258,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c h1:m/r7OM+Y2Ty1sgBQ7Qb27VgIMBW8ZZhT4gLnUyDIhzI=
|
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g=
|
||||||
gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c/go.mod h1:3r5CMtNQMKIvBlrmM9xWUNamjKBYPOWyXOjmg5Kts3g=
|
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
|
||||||
|
|||||||
@@ -2,8 +2,6 @@ package nebula
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
"github.com/slackhq/nebula/firewall"
|
"github.com/slackhq/nebula/firewall"
|
||||||
@@ -11,258 +9,73 @@ import (
|
|||||||
"github.com/slackhq/nebula/iputil"
|
"github.com/slackhq/nebula/iputil"
|
||||||
"github.com/slackhq/nebula/noiseutil"
|
"github.com/slackhq/nebula/noiseutil"
|
||||||
"github.com/slackhq/nebula/routing"
|
"github.com/slackhq/nebula/routing"
|
||||||
|
"github.com/slackhq/nebula/udp"
|
||||||
)
|
)
|
||||||
|
|
||||||
// preEncryptionPacket holds packet data before batch encryption
|
// consumeInsidePacketBatched is a variant of consumeInsidePacket that queues
|
||||||
type preEncryptionPacket struct {
|
// outgoing packets into pendingPackets instead of sending them immediately.
|
||||||
hostinfo *HostInfo
|
// The caller is responsible for flushing pendingPackets with WriteBatch.
|
||||||
ci *ConnectionState
|
func (f *Interface) consumeInsidePacketBatched(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache, pendingPackets *[]udp.BatchPacket) {
|
||||||
packet []byte
|
err := newPacket(packet, false, fwPacket)
|
||||||
out []byte
|
if err != nil {
|
||||||
}
|
if f.l.Level >= logrus.DebugLevel {
|
||||||
|
f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
|
||||||
// Pool for preEncryptionBatch slices to reduce allocations
|
|
||||||
var preEncryptionBatchPool = sync.Pool{
|
|
||||||
New: func() any {
|
|
||||||
// Pre-allocate with reasonable capacity
|
|
||||||
batch := make([]preEncryptionPacket, 0, 128)
|
|
||||||
return &batch
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// consumeInsidePackets processes multiple packets in a batch for improved performance
|
|
||||||
// packets: slice of packet buffers to process
|
|
||||||
// sizes: slice of packet sizes
|
|
||||||
// count: number of packets to process
|
|
||||||
// outs: slice of output buffers (one per packet) with virtio headroom
|
|
||||||
// q: queue index
|
|
||||||
// localCache: firewall conntrack cache
|
|
||||||
// batchPackets: pre-allocated slice for accumulating encrypted packets
|
|
||||||
// batchAddrs: pre-allocated slice for accumulating destination addresses
|
|
||||||
func (f *Interface) consumeInsidePackets(packets [][]byte, sizes []int, count int, outs [][]byte, nb []byte, q int, localCache firewall.ConntrackCache, batchPackets *[][]byte, batchAddrs *[]netip.AddrPort) {
|
|
||||||
// Reusable per-packet state
|
|
||||||
fwPacket := &firewall.Packet{}
|
|
||||||
|
|
||||||
// Reset batch accumulation slices (reuse capacity)
|
|
||||||
*batchPackets = (*batchPackets)[:0]
|
|
||||||
*batchAddrs = (*batchAddrs)[:0]
|
|
||||||
|
|
||||||
// Collect packets for batched encryption
|
|
||||||
preEncryptionBatch := make([]preEncryptionPacket, 0, count)
|
|
||||||
|
|
||||||
// Process each packet in the batch
|
|
||||||
for i := 0; i < count; i++ {
|
|
||||||
packet := packets[i][:sizes[i]]
|
|
||||||
out := outs[i]
|
|
||||||
|
|
||||||
// Inline the consumeInsidePacket logic for better performance
|
|
||||||
err := newPacket(packet, false, fwPacket)
|
|
||||||
if err != nil {
|
|
||||||
if f.l.Level >= logrus.DebugLevel {
|
|
||||||
f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
return
|
||||||
// Ignore local broadcast packets
|
|
||||||
if f.dropLocalBroadcast {
|
|
||||||
if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if f.myVpnAddrsTable.Contains(fwPacket.RemoteAddr) {
|
|
||||||
// Immediately forward packets from self to self.
|
|
||||||
if immediatelyForwardToSelf {
|
|
||||||
_, err := f.readers[q].Write(packet)
|
|
||||||
if err != nil {
|
|
||||||
f.l.WithError(err).Error("Failed to forward to tun")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ignore multicast packets
|
|
||||||
if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
|
|
||||||
hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
|
|
||||||
})
|
|
||||||
|
|
||||||
if hostinfo == nil {
|
|
||||||
f.rejectInside(packet, out, q)
|
|
||||||
if f.l.Level >= logrus.DebugLevel {
|
|
||||||
f.l.WithField("vpnAddr", fwPacket.RemoteAddr).
|
|
||||||
WithField("fwPacket", fwPacket).
|
|
||||||
Debugln("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks")
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if !ready {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
|
|
||||||
if dropReason != nil {
|
|
||||||
f.rejectInside(packet, out, q)
|
|
||||||
if f.l.Level >= logrus.DebugLevel {
|
|
||||||
hostinfo.logger(f.l).
|
|
||||||
WithField("fwPacket", fwPacket).
|
|
||||||
WithField("reason", dropReason).
|
|
||||||
Debugln("dropping outbound packet")
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prepare packet for batch encryption
|
|
||||||
ci := hostinfo.ConnectionState
|
|
||||||
if ci.eKey == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if this needs relay - if so, send immediately and skip batching
|
|
||||||
useRelay := !hostinfo.remote.IsValid()
|
|
||||||
if useRelay {
|
|
||||||
// Handle relay sends individually (less common path)
|
|
||||||
f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, packet, nb, out, q)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Collect for batched encryption
|
|
||||||
preEncryptionBatch = append(preEncryptionBatch, preEncryptionPacket{
|
|
||||||
hostinfo: hostinfo,
|
|
||||||
ci: ci,
|
|
||||||
packet: packet,
|
|
||||||
out: out,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BATCH ENCRYPTION: Process all collected packets
|
// Ignore local broadcast packets
|
||||||
if len(preEncryptionBatch) > 0 {
|
if f.dropLocalBroadcast {
|
||||||
f.encryptBatch(preEncryptionBatch, nb, batchPackets, batchAddrs)
|
if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
|
||||||
}
|
return
|
||||||
|
|
||||||
// Send all accumulated packets in one batch
|
|
||||||
if len(*batchPackets) > 0 {
|
|
||||||
batchSize := len(*batchPackets)
|
|
||||||
n, err := f.writers[q].WriteMulti(*batchPackets, *batchAddrs)
|
|
||||||
if err != nil {
|
|
||||||
f.l.WithError(err).WithField("sent", n).WithField("total", batchSize).Error("Failed to send batch")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// encryptBatch processes multiple packets, grouping by ConnectionState to reduce lock acquisitions
|
if f.myVpnAddrsTable.Contains(fwPacket.RemoteAddr) {
|
||||||
func (f *Interface) encryptBatch(batch []preEncryptionPacket, nb []byte, batchPackets *[][]byte, batchAddrs *[]netip.AddrPort) {
|
if immediatelyForwardToSelf {
|
||||||
lockStart := time.Now()
|
_, err := f.readers[q].Write(packet)
|
||||||
lockAcquisitions := int64(0)
|
|
||||||
|
|
||||||
if noiseutil.EncryptLockNeeded {
|
|
||||||
// Group packets by ConnectionState to minimize lock acquisitions
|
|
||||||
// Process packets in order but batch lock acquisitions per CI
|
|
||||||
var currentCI *ConnectionState
|
|
||||||
var lockHeld bool
|
|
||||||
|
|
||||||
for i := range batch {
|
|
||||||
ci := batch[i].ci
|
|
||||||
hostinfo := batch[i].hostinfo
|
|
||||||
|
|
||||||
// Validate packet data to prevent nil pointer dereference
|
|
||||||
if ci == nil || hostinfo == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Switch locks if we're moving to a different ConnectionState
|
|
||||||
if ci != currentCI {
|
|
||||||
if lockHeld {
|
|
||||||
currentCI.writeLock.Unlock()
|
|
||||||
}
|
|
||||||
ci.writeLock.Lock()
|
|
||||||
lockAcquisitions++
|
|
||||||
currentCI = ci
|
|
||||||
lockHeld = true
|
|
||||||
}
|
|
||||||
|
|
||||||
c := ci.messageCounter.Add(1)
|
|
||||||
out := header.Encode(batch[i].out, header.Version, header.Message, 0, hostinfo.remoteIndexId, c)
|
|
||||||
f.connectionManager.Out(hostinfo)
|
|
||||||
|
|
||||||
// Query lighthouse if needed
|
|
||||||
if hostinfo.lastRebindCount != f.rebindCount {
|
|
||||||
f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
|
|
||||||
hostinfo.lastRebindCount = f.rebindCount
|
|
||||||
if f.l.Level >= logrus.DebugLevel {
|
|
||||||
f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).Debug("Lighthouse update triggered for punch due to rebind counter")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var err error
|
|
||||||
out, err = ci.eKey.EncryptDanger(out, out, batch[i].packet, c, nb)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
hostinfo.logger(f.l).WithError(err).
|
f.l.WithError(err).Error("Failed to forward to tun")
|
||||||
WithField("counter", c).
|
|
||||||
Error("Failed to encrypt outgoing packet")
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add to output batches
|
|
||||||
*batchPackets = append(*batchPackets, out)
|
|
||||||
*batchAddrs = append(*batchAddrs, hostinfo.remote)
|
|
||||||
}
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Release final lock
|
// Ignore multicast packets
|
||||||
if lockHeld {
|
if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() {
|
||||||
currentCI.writeLock.Unlock()
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
|
||||||
|
hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
|
||||||
|
})
|
||||||
|
|
||||||
|
if hostinfo == nil {
|
||||||
|
f.rejectInside(packet, out, q)
|
||||||
|
if f.l.Level >= logrus.DebugLevel {
|
||||||
|
f.l.WithField("vpnAddr", fwPacket.RemoteAddr).
|
||||||
|
WithField("fwPacket", fwPacket).
|
||||||
|
Debugln("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks")
|
||||||
}
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if !ready {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
|
||||||
|
if dropReason == nil {
|
||||||
|
f.sendNoMetricsBatched(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q, pendingPackets)
|
||||||
} else {
|
} else {
|
||||||
// No locks needed - process directly
|
f.rejectInside(packet, out, q)
|
||||||
for i := range batch {
|
if f.l.Level >= logrus.DebugLevel {
|
||||||
ci := batch[i].ci
|
hostinfo.logger(f.l).
|
||||||
hostinfo := batch[i].hostinfo
|
WithField("fwPacket", fwPacket).
|
||||||
|
WithField("reason", dropReason).
|
||||||
// Validate packet data to prevent nil pointer dereference
|
Debugln("dropping outbound packet")
|
||||||
if ci == nil || hostinfo == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
c := ci.messageCounter.Add(1)
|
|
||||||
out := header.Encode(batch[i].out, header.Version, header.Message, 0, hostinfo.remoteIndexId, c)
|
|
||||||
f.connectionManager.Out(hostinfo)
|
|
||||||
|
|
||||||
// Query lighthouse if needed
|
|
||||||
if hostinfo.lastRebindCount != f.rebindCount {
|
|
||||||
f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
|
|
||||||
hostinfo.lastRebindCount = f.rebindCount
|
|
||||||
if f.l.Level >= logrus.DebugLevel {
|
|
||||||
f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).Debug("Lighthouse update triggered for punch due to rebind counter")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var err error
|
|
||||||
out, err = ci.eKey.EncryptDanger(out, out, batch[i].packet, c, nb)
|
|
||||||
if err != nil {
|
|
||||||
hostinfo.logger(f.l).WithError(err).
|
|
||||||
WithField("counter", c).
|
|
||||||
Error("Failed to encrypt outgoing packet")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add to output batches
|
|
||||||
*batchPackets = append(*batchPackets, out)
|
|
||||||
*batchAddrs = append(*batchAddrs, hostinfo.remote)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record metrics
|
|
||||||
encryptionTime := time.Since(lockStart)
|
|
||||||
if noiseutil.EncryptLockNeeded {
|
|
||||||
f.batchMetrics.lockAcquisitions.Inc(lockAcquisitions)
|
|
||||||
}
|
|
||||||
f.batchMetrics.encryptionTime.Update(encryptionTime.Nanoseconds())
|
|
||||||
f.batchMetrics.batchSize.Update(int64(len(batch)))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
|
func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
|
||||||
@@ -323,7 +136,6 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
|
|||||||
dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
|
dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
|
||||||
if dropReason == nil {
|
if dropReason == nil {
|
||||||
f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
|
f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
f.rejectInside(packet, out, q)
|
f.rejectInside(packet, out, q)
|
||||||
if f.l.Level >= logrus.DebugLevel {
|
if f.l.Level >= logrus.DebugLevel {
|
||||||
@@ -664,3 +476,75 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sendNoMetricsBatched is like sendNoMetrics but queues the packet for batched sending
|
||||||
|
// instead of sending immediately. The caller must flush pendingPackets with WriteBatch.
|
||||||
|
func (f *Interface) sendNoMetricsBatched(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, q int, pendingPackets *[]udp.BatchPacket) {
|
||||||
|
if ci.eKey == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
useRelay := !remote.IsValid() && !hostinfo.remote.IsValid()
|
||||||
|
fullOut := out
|
||||||
|
|
||||||
|
if useRelay {
|
||||||
|
if len(out) < header.Len {
|
||||||
|
out = out[:header.Len]
|
||||||
|
}
|
||||||
|
out = out[header.Len:]
|
||||||
|
}
|
||||||
|
|
||||||
|
if noiseutil.EncryptLockNeeded {
|
||||||
|
ci.writeLock.Lock()
|
||||||
|
}
|
||||||
|
c := ci.messageCounter.Add(1)
|
||||||
|
|
||||||
|
out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
|
||||||
|
f.connectionManager.Out(hostinfo)
|
||||||
|
|
||||||
|
if t != header.CloseTunnel && hostinfo.lastRebindCount != f.rebindCount {
|
||||||
|
f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
|
||||||
|
hostinfo.lastRebindCount = f.rebindCount
|
||||||
|
if f.l.Level >= logrus.DebugLevel {
|
||||||
|
f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).Debug("Lighthouse update triggered for punch due to rebind counter")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var err error
|
||||||
|
out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
|
||||||
|
if noiseutil.EncryptLockNeeded {
|
||||||
|
ci.writeLock.Unlock()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
hostinfo.logger(f.l).WithError(err).
|
||||||
|
WithField("udpAddr", remote).WithField("counter", c).
|
||||||
|
WithField("attemptedCounter", c).
|
||||||
|
Error("Failed to encrypt outgoing packet")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Queue the packet for batched sending
|
||||||
|
var addr netip.AddrPort
|
||||||
|
if remote.IsValid() {
|
||||||
|
addr = remote
|
||||||
|
} else if hostinfo.remote.IsValid() {
|
||||||
|
addr = hostinfo.remote
|
||||||
|
} else {
|
||||||
|
// Relay path - send immediately, not batched
|
||||||
|
for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
|
||||||
|
relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
|
||||||
|
if err != nil {
|
||||||
|
hostinfo.relayState.DeleteRelay(relayIP)
|
||||||
|
hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetricsBatched failed to find HostInfo")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy the payload since the buffer will be reused
|
||||||
|
payload := make([]byte, len(out))
|
||||||
|
copy(payload, out)
|
||||||
|
*pendingPackets = append(*pendingPackets, udp.BatchPacket{Payload: payload, Addr: addr})
|
||||||
|
}
|
||||||
|
|||||||
+86
-60
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
@@ -21,7 +22,6 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const mtu = 9001
|
const mtu = 9001
|
||||||
const virtioNetHdrLen = overlay.VirtioNetHdrLen
|
|
||||||
|
|
||||||
type InterfaceConfig struct {
|
type InterfaceConfig struct {
|
||||||
HostMap *HostMap
|
HostMap *HostMap
|
||||||
@@ -48,13 +48,8 @@ type InterfaceConfig struct {
|
|||||||
|
|
||||||
ConntrackCacheTimeout time.Duration
|
ConntrackCacheTimeout time.Duration
|
||||||
l *logrus.Logger
|
l *logrus.Logger
|
||||||
}
|
|
||||||
|
|
||||||
type batchMetrics struct {
|
tunBatchSize int // batch size for TUN read/write batching, 0 to disable
|
||||||
udpReadSize metrics.Histogram
|
|
||||||
encryptionTime metrics.Histogram // Time spent in encryption (including lock waits)
|
|
||||||
batchSize metrics.Histogram // Dynamic batch sizes being used
|
|
||||||
lockAcquisitions metrics.Counter // Number of lock acquisitions (should be minimal)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Interface struct {
|
type Interface struct {
|
||||||
@@ -93,13 +88,13 @@ type Interface struct {
|
|||||||
|
|
||||||
conntrackCacheTimeout time.Duration
|
conntrackCacheTimeout time.Duration
|
||||||
|
|
||||||
writers []udp.Conn
|
writers []udp.Conn
|
||||||
readers []overlay.BatchReadWriter
|
readers []io.ReadWriteCloser
|
||||||
|
tunBatchSize int // batch size for TUN read/write batching
|
||||||
|
|
||||||
metricHandshakes metrics.Histogram
|
metricHandshakes metrics.Histogram
|
||||||
messageMetrics *MessageMetrics
|
messageMetrics *MessageMetrics
|
||||||
cachedPacketMetrics *cachedPacketMetrics
|
cachedPacketMetrics *cachedPacketMetrics
|
||||||
batchMetrics *batchMetrics
|
|
||||||
|
|
||||||
l *logrus.Logger
|
l *logrus.Logger
|
||||||
}
|
}
|
||||||
@@ -186,7 +181,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
|
|||||||
routines: c.routines,
|
routines: c.routines,
|
||||||
version: c.version,
|
version: c.version,
|
||||||
writers: make([]udp.Conn, c.routines),
|
writers: make([]udp.Conn, c.routines),
|
||||||
readers: make([]overlay.BatchReadWriter, c.routines),
|
readers: make([]io.ReadWriteCloser, c.routines),
|
||||||
myVpnNetworks: cs.myVpnNetworks,
|
myVpnNetworks: cs.myVpnNetworks,
|
||||||
myVpnNetworksTable: cs.myVpnNetworksTable,
|
myVpnNetworksTable: cs.myVpnNetworksTable,
|
||||||
myVpnAddrs: cs.myVpnAddrs,
|
myVpnAddrs: cs.myVpnAddrs,
|
||||||
@@ -195,6 +190,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
|
|||||||
relayManager: c.relayManager,
|
relayManager: c.relayManager,
|
||||||
connectionManager: c.connectionManager,
|
connectionManager: c.connectionManager,
|
||||||
conntrackCacheTimeout: c.ConntrackCacheTimeout,
|
conntrackCacheTimeout: c.ConntrackCacheTimeout,
|
||||||
|
tunBatchSize: c.tunBatchSize,
|
||||||
|
|
||||||
metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
|
metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
|
||||||
messageMetrics: c.MessageMetrics,
|
messageMetrics: c.MessageMetrics,
|
||||||
@@ -202,12 +198,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
|
|||||||
sent: metrics.GetOrRegisterCounter("hostinfo.cached_packets.sent", nil),
|
sent: metrics.GetOrRegisterCounter("hostinfo.cached_packets.sent", nil),
|
||||||
dropped: metrics.GetOrRegisterCounter("hostinfo.cached_packets.dropped", nil),
|
dropped: metrics.GetOrRegisterCounter("hostinfo.cached_packets.dropped", nil),
|
||||||
},
|
},
|
||||||
batchMetrics: &batchMetrics{
|
|
||||||
udpReadSize: metrics.GetOrRegisterHistogram("batch.udp_read_size", nil, metrics.NewUniformSample(1024)),
|
|
||||||
encryptionTime: metrics.GetOrRegisterHistogram("batch.encryption_time_ns", nil, metrics.NewUniformSample(1024)),
|
|
||||||
batchSize: metrics.GetOrRegisterHistogram("batch.size", nil, metrics.NewUniformSample(1024)),
|
|
||||||
lockAcquisitions: metrics.GetOrRegisterCounter("batch.lock_acquisitions", nil),
|
|
||||||
},
|
|
||||||
|
|
||||||
l: c.l,
|
l: c.l,
|
||||||
}
|
}
|
||||||
@@ -247,7 +237,7 @@ func (f *Interface) activate() {
|
|||||||
metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
|
metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
|
||||||
|
|
||||||
// Prepare n tun queues
|
// Prepare n tun queues
|
||||||
var reader overlay.BatchReadWriter = f.inside
|
var reader io.ReadWriteCloser = f.inside
|
||||||
for i := 0; i < f.routines; i++ {
|
for i := 0; i < f.routines; i++ {
|
||||||
if i > 0 {
|
if i > 0 {
|
||||||
reader, err = f.inside.NewMultiQueueReader()
|
reader, err = f.inside.NewMultiQueueReader()
|
||||||
@@ -258,6 +248,15 @@ func (f *Interface) activate() {
|
|||||||
f.readers[i] = reader
|
f.readers[i] = reader
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Enable batch reading on all readers if batch size > 1
|
||||||
|
if f.tunBatchSize > 1 {
|
||||||
|
for i := 0; i < f.routines; i++ {
|
||||||
|
if err := overlay.EnableBatchReading(f.readers[i]); err != nil {
|
||||||
|
f.l.WithError(err).WithField("routine", i).Warn("Failed to enable batch reading, falling back to single reads")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err := f.inside.Activate(); err != nil {
|
if err := f.inside.Activate(); err != nil {
|
||||||
f.inside.Close()
|
f.inside.Close()
|
||||||
f.l.Fatal(err)
|
f.l.Fatal(err)
|
||||||
@@ -288,68 +287,95 @@ func (f *Interface) listenOut(i int) {
|
|||||||
|
|
||||||
ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
|
ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
|
||||||
lhh := f.lightHouse.NewRequestHandler()
|
lhh := f.lightHouse.NewRequestHandler()
|
||||||
|
plaintext := make([]byte, udp.MTU)
|
||||||
// Pre-allocate output buffers for batch processing
|
|
||||||
batchSize := li.BatchSize()
|
|
||||||
outs := make([][]byte, batchSize)
|
|
||||||
for idx := range outs {
|
|
||||||
// Allocate full buffer with virtio header space
|
|
||||||
outs[idx] = make([]byte, virtioNetHdrLen, virtioNetHdrLen+udp.MTU)
|
|
||||||
}
|
|
||||||
|
|
||||||
h := &header.H{}
|
h := &header.H{}
|
||||||
fwPacket := &firewall.Packet{}
|
fwPacket := &firewall.Packet{}
|
||||||
nb := make([]byte, 12)
|
nb := make([]byte, 12, 12)
|
||||||
|
|
||||||
li.ListenOutBatch(func(addrs []netip.AddrPort, payloads [][]byte, count int) {
|
|
||||||
f.readOutsidePacketsBatch(addrs, payloads, count, outs[:count], nb, i, h, fwPacket, lhh, ctCache.Get(f.l))
|
|
||||||
|
|
||||||
|
li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
|
||||||
|
f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get(f.l))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Interface) listenIn(reader overlay.BatchReadWriter, i int) {
|
func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
|
||||||
runtime.LockOSThread()
|
runtime.LockOSThread()
|
||||||
|
|
||||||
batchSize := reader.BatchSize()
|
|
||||||
|
|
||||||
// Allocate buffers for batch reading
|
|
||||||
bufs := make([][]byte, batchSize)
|
|
||||||
for idx := range bufs {
|
|
||||||
bufs[idx] = make([]byte, mtu)
|
|
||||||
}
|
|
||||||
sizes := make([]int, batchSize)
|
|
||||||
|
|
||||||
// Allocate output buffers for batch processing (one per packet)
|
|
||||||
// Each has virtio header headroom to avoid copies on write
|
|
||||||
outs := make([][]byte, batchSize)
|
|
||||||
for idx := range outs {
|
|
||||||
outBuf := make([]byte, virtioNetHdrLen+mtu)
|
|
||||||
outs[idx] = outBuf[virtioNetHdrLen:] // Slice starting after headroom
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pre-allocate batch accumulation buffers for sending
|
|
||||||
batchPackets := make([][]byte, 0, batchSize)
|
|
||||||
batchAddrs := make([]netip.AddrPort, 0, batchSize)
|
|
||||||
|
|
||||||
// Pre-allocate nonce buffer (reused for all encryptions)
|
|
||||||
nb := make([]byte, 12)
|
|
||||||
|
|
||||||
conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
|
conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
|
||||||
|
|
||||||
|
// Check if batch reading is available and enabled
|
||||||
|
batchReader := overlay.AsBatchReader(reader)
|
||||||
|
if batchReader != nil && f.tunBatchSize > 1 {
|
||||||
|
f.listenInBatched(reader, batchReader, i, conntrackCache)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to single-packet reading
|
||||||
|
packet := make([]byte, mtu)
|
||||||
|
out := make([]byte, mtu)
|
||||||
|
fwPacket := &firewall.Packet{}
|
||||||
|
nb := make([]byte, 12, 12)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
n, err := reader.BatchRead(bufs, sizes)
|
n, err := reader.Read(packet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, os.ErrClosed) && f.closed.Load() {
|
if errors.Is(err, os.ErrClosed) && f.closed.Load() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
f.l.WithError(err).Error("Error while batch reading outbound packets")
|
f.l.WithError(err).Error("Error while reading outbound packet")
|
||||||
// This only seems to happen when something fatal happens to the fd, so exit.
|
// This only seems to happen when something fatal happens to the fd, so exit.
|
||||||
os.Exit(2)
|
os.Exit(2)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process all packets in the batch at once
|
f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get(f.l))
|
||||||
f.consumeInsidePackets(bufs, sizes, n, outs, nb, i, conntrackCache.Get(f.l), &batchPackets, &batchAddrs)
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Interface) listenInBatched(reader io.ReadWriteCloser, batchReader overlay.BatchReader, i int, conntrackCache *firewall.ConntrackCacheTicker) {
|
||||||
|
batchSize := f.tunBatchSize
|
||||||
|
|
||||||
|
// Pre-allocate buffers for batch reading
|
||||||
|
packets := make([][]byte, batchSize)
|
||||||
|
for j := range packets {
|
||||||
|
packets[j] = make([]byte, mtu)
|
||||||
|
}
|
||||||
|
sizes := make([]int, batchSize)
|
||||||
|
|
||||||
|
// Pre-allocate buffers for packet processing
|
||||||
|
out := make([]byte, mtu)
|
||||||
|
fwPacket := &firewall.Packet{}
|
||||||
|
nb := make([]byte, 12, 12)
|
||||||
|
|
||||||
|
// Pre-allocate buffer for batched UDP writes
|
||||||
|
pendingPackets := make([]udp.BatchPacket, 0, batchSize)
|
||||||
|
|
||||||
|
for {
|
||||||
|
// Read a batch of packets from TUN
|
||||||
|
n, err := batchReader.ReadBatch(packets, sizes)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrClosed) && f.closed.Load() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
f.l.WithError(err).Error("Error while reading outbound packets")
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process all packets in the batch
|
||||||
|
cache := conntrackCache.Get(f.l)
|
||||||
|
for j := 0; j < n; j++ {
|
||||||
|
f.consumeInsidePacketBatched(packets[j][:sizes[j]], fwPacket, nb, out, i, cache, &pendingPackets)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flush all pending UDP writes
|
||||||
|
if len(pendingPackets) > 0 {
|
||||||
|
f.writers[i].WriteBatch(pendingPackets)
|
||||||
|
pendingPackets = pendingPackets[:0]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -171,7 +171,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
|
|||||||
|
|
||||||
for i := 0; i < routines; i++ {
|
for i := 0; i < routines; i++ {
|
||||||
l.Infof("listening on %v", netip.AddrPortFrom(listenHost, uint16(port)))
|
l.Infof("listening on %v", netip.AddrPortFrom(listenHost, uint16(port)))
|
||||||
udpServer, err := udp.NewListener(l, listenHost, port, routines > 1, c.GetInt("listen.batch", 128))
|
udpServer, err := udp.NewListener(l, listenHost, port, routines > 1, c.GetInt("listen.batch", 64))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err)
|
return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err)
|
||||||
}
|
}
|
||||||
@@ -250,6 +250,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
|
|||||||
punchy: punchy,
|
punchy: punchy,
|
||||||
ConntrackCacheTimeout: conntrackCacheTimeout,
|
ConntrackCacheTimeout: conntrackCacheTimeout,
|
||||||
l: l,
|
l: l,
|
||||||
|
tunBatchSize: c.GetInt("listen.batch", 64),
|
||||||
}
|
}
|
||||||
|
|
||||||
var ifce *Interface
|
var ifce *Interface
|
||||||
|
|||||||
+7
-114
@@ -102,7 +102,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
|
|||||||
relay: relay,
|
relay: relay,
|
||||||
IsRelayed: true,
|
IsRelayed: true,
|
||||||
}
|
}
|
||||||
f.readOutsidePackets(via, out[:virtioNetHdrLen], signedPayload, h, fwPacket, lhf, nb, q, localCache)
|
f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
|
||||||
return
|
return
|
||||||
case ForwardingType:
|
case ForwardingType:
|
||||||
// Find the target HostInfo relay object
|
// Find the target HostInfo relay object
|
||||||
@@ -145,7 +145,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
|
|||||||
}
|
}
|
||||||
|
|
||||||
//TODO: assert via is not relayed
|
//TODO: assert via is not relayed
|
||||||
lhf.HandleRequest(via.UdpAddr, hostinfo.vpnAddrs, d[virtioNetHdrLen:], f)
|
lhf.HandleRequest(via.UdpAddr, hostinfo.vpnAddrs, d, f)
|
||||||
|
|
||||||
// Fallthrough to the bottom to record incoming traffic
|
// Fallthrough to the bottom to record incoming traffic
|
||||||
|
|
||||||
@@ -167,7 +167,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
|
|||||||
// This testRequest might be from TryPromoteBest, so we should roam
|
// This testRequest might be from TryPromoteBest, so we should roam
|
||||||
// to the new IP address before responding
|
// to the new IP address before responding
|
||||||
f.handleHostRoaming(hostinfo, via)
|
f.handleHostRoaming(hostinfo, via)
|
||||||
f.send(header.Test, header.TestReply, ci, hostinfo, d[virtioNetHdrLen:], nb, out)
|
f.send(header.Test, header.TestReply, ci, hostinfo, d, nb, out)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallthrough to the bottom to record incoming traffic
|
// Fallthrough to the bottom to record incoming traffic
|
||||||
@@ -210,7 +210,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
f.relayManager.HandleControlMsg(hostinfo, d[virtioNetHdrLen:], f)
|
f.relayManager.HandleControlMsg(hostinfo, d, f)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
f.messageMetrics.Rx(h.Type, h.Subtype, 1)
|
f.messageMetrics.Rx(h.Type, h.Subtype, 1)
|
||||||
@@ -481,11 +481,9 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
packetData := out[virtioNetHdrLen:]
|
err = newPacket(out, true, fwPacket)
|
||||||
|
|
||||||
err = newPacket(packetData, true, fwPacket)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
hostinfo.logger(f.l).WithError(err).WithField("packet", packetData).
|
hostinfo.logger(f.l).WithError(err).WithField("packet", out).
|
||||||
Warnf("Error while validating inbound packet")
|
Warnf("Error while validating inbound packet")
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@@ -500,7 +498,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
|
|||||||
if dropReason != nil {
|
if dropReason != nil {
|
||||||
// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
|
// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
|
||||||
// This gives us a buffer to build the reject packet in
|
// This gives us a buffer to build the reject packet in
|
||||||
f.rejectOutside(packetData, hostinfo.ConnectionState, hostinfo, nb, packet, q)
|
f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, nb, packet, q)
|
||||||
if f.l.Level >= logrus.DebugLevel {
|
if f.l.Level >= logrus.DebugLevel {
|
||||||
hostinfo.logger(f.l).WithField("fwPacket", fwPacket).
|
hostinfo.logger(f.l).WithField("fwPacket", fwPacket).
|
||||||
WithField("reason", dropReason).
|
WithField("reason", dropReason).
|
||||||
@@ -564,108 +562,3 @@ func (f *Interface) handleRecvError(addr netip.AddrPort, h *header.H) {
|
|||||||
// We also delete it from pending hostmap to allow for fast reconnect.
|
// We also delete it from pending hostmap to allow for fast reconnect.
|
||||||
f.handshakeManager.DeleteHostInfo(hostinfo)
|
f.handshakeManager.DeleteHostInfo(hostinfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
// readOutsidePacketsBatch processes multiple packets received from UDP in a batch
|
|
||||||
// and writes all successfully decrypted packets to TUN in a single operation
|
|
||||||
func (f *Interface) readOutsidePacketsBatch(addrs []netip.AddrPort, payloads [][]byte, count int, outs [][]byte, nb []byte, q int, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, localCache firewall.ConntrackCache) {
|
|
||||||
// Pre-allocate slice for accumulating successful decryptions
|
|
||||||
tunPackets := make([][]byte, 0, count)
|
|
||||||
|
|
||||||
for i := 0; i < count; i++ {
|
|
||||||
payload := payloads[i]
|
|
||||||
addr := addrs[i]
|
|
||||||
out := outs[i]
|
|
||||||
|
|
||||||
// Parse header
|
|
||||||
err := h.Parse(payload)
|
|
||||||
if err != nil {
|
|
||||||
if len(payload) > 1 {
|
|
||||||
f.l.WithField("packet", payload).Infof("Error while parsing inbound packet from %s: %s", addr, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if addr.IsValid() {
|
|
||||||
if f.myVpnNetworksTable.Contains(addr.Addr()) {
|
|
||||||
if f.l.Level >= logrus.DebugLevel {
|
|
||||||
f.l.WithField("udpAddr", addr).Debug("Refusing to process double encrypted packet")
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var hostinfo *HostInfo
|
|
||||||
if h.Type == header.Message && h.Subtype == header.MessageRelay {
|
|
||||||
hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex)
|
|
||||||
} else {
|
|
||||||
hostinfo = f.hostMap.QueryIndex(h.RemoteIndex)
|
|
||||||
}
|
|
||||||
|
|
||||||
var ci *ConnectionState
|
|
||||||
if hostinfo != nil {
|
|
||||||
ci = hostinfo.ConnectionState
|
|
||||||
}
|
|
||||||
|
|
||||||
switch h.Type {
|
|
||||||
case header.Message:
|
|
||||||
if !f.handleEncrypted(ci, ViaSender{UdpAddr: addr}, h) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
switch h.Subtype {
|
|
||||||
case header.MessageNone:
|
|
||||||
// Decrypt packet
|
|
||||||
out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, payload[:header.Len], payload[header.Len:], h.MessageCounter, nb)
|
|
||||||
if err != nil {
|
|
||||||
hostinfo.logger(f.l).WithError(err).Error("Failed to decrypt packet")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
packetData := out[virtioNetHdrLen:]
|
|
||||||
|
|
||||||
err = newPacket(packetData, true, fwPacket)
|
|
||||||
if err != nil {
|
|
||||||
hostinfo.logger(f.l).WithError(err).WithField("packet", packetData).Warnf("Error while validating inbound packet")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if !hostinfo.ConnectionState.window.Update(f.l, h.MessageCounter) {
|
|
||||||
hostinfo.logger(f.l).WithField("fwPacket", fwPacket).Debugln("dropping out of window packet")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
|
|
||||||
if dropReason != nil {
|
|
||||||
f.rejectOutside(packetData, hostinfo.ConnectionState, hostinfo, nb, payload, q)
|
|
||||||
if f.l.Level >= logrus.DebugLevel {
|
|
||||||
hostinfo.logger(f.l).WithField("fwPacket", fwPacket).WithField("reason", dropReason).Debugln("dropping inbound packet")
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
f.connectionManager.In(hostinfo)
|
|
||||||
// Add to batch for TUN write
|
|
||||||
tunPackets = append(tunPackets, out)
|
|
||||||
|
|
||||||
case header.MessageRelay:
|
|
||||||
// Skip relay packets in batch mode for now (less common path)
|
|
||||||
f.readOutsidePackets(ViaSender{UdpAddr: addr}, out[:virtioNetHdrLen], payload, h, fwPacket, lhf, nb, q, localCache)
|
|
||||||
|
|
||||||
default:
|
|
||||||
hostinfo.logger(f.l).Debugf("unexpected message subtype %d", h.Subtype)
|
|
||||||
}
|
|
||||||
|
|
||||||
default:
|
|
||||||
// Handle non-Message types using single-packet path
|
|
||||||
f.readOutsidePackets(ViaSender{UdpAddr: addr}, out[:virtioNetHdrLen], payload, h, fwPacket, lhf, nb, q, localCache)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(tunPackets) > 0 {
|
|
||||||
n, err := f.readers[q].WriteBatch(tunPackets, virtioNetHdrLen)
|
|
||||||
if err != nil {
|
|
||||||
f.l.WithError(err).WithField("sent", n).WithField("total", len(tunPackets)).Error("Failed to batch write to tun")
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
+37
-16
@@ -7,26 +7,47 @@ import (
|
|||||||
"github.com/slackhq/nebula/routing"
|
"github.com/slackhq/nebula/routing"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BatchReadWriter extends io.ReadWriteCloser with batch I/O operations
|
|
||||||
type BatchReadWriter interface {
|
|
||||||
io.ReadWriteCloser
|
|
||||||
|
|
||||||
// BatchRead reads multiple packets at once
|
|
||||||
BatchRead(bufs [][]byte, sizes []int) (int, error)
|
|
||||||
|
|
||||||
// WriteBatch writes multiple packets at once
|
|
||||||
WriteBatch(bufs [][]byte, offset int) (int, error)
|
|
||||||
|
|
||||||
// BatchSize returns the optimal batch size for this device
|
|
||||||
BatchSize() int
|
|
||||||
}
|
|
||||||
|
|
||||||
type Device interface {
|
type Device interface {
|
||||||
BatchReadWriter
|
io.ReadWriteCloser
|
||||||
Activate() error
|
Activate() error
|
||||||
Networks() []netip.Prefix
|
Networks() []netip.Prefix
|
||||||
Name() string
|
Name() string
|
||||||
RoutesFor(netip.Addr) routing.Gateways
|
RoutesFor(netip.Addr) routing.Gateways
|
||||||
SupportsMultiqueue() bool
|
SupportsMultiqueue() bool
|
||||||
NewMultiQueueReader() (BatchReadWriter, error)
|
NewMultiQueueReader() (io.ReadWriteCloser, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchReader is an optional interface that devices can implement
|
||||||
|
// to support reading multiple packets in a single batch operation.
|
||||||
|
// This can significantly reduce syscall overhead under high load.
|
||||||
|
type BatchReader interface {
|
||||||
|
// ReadBatch reads up to len(packets) packets into the provided buffers.
|
||||||
|
// Each packet is read into packets[i] and its length is stored in sizes[i].
|
||||||
|
// Returns the number of packets read, or an error.
|
||||||
|
// A return of (0, nil) indicates no packets were available (non-blocking).
|
||||||
|
ReadBatch(packets [][]byte, sizes []int) (int, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AsBatchReader returns a BatchReader if the reader supports batch operations,
|
||||||
|
// otherwise returns nil.
|
||||||
|
func AsBatchReader(r io.ReadWriteCloser) BatchReader {
|
||||||
|
if br, ok := r.(BatchReader); ok {
|
||||||
|
return br
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchEnabler is an optional interface for devices that need explicit
|
||||||
|
// enabling of batch read support (e.g., setting non-blocking mode).
|
||||||
|
type BatchEnabler interface {
|
||||||
|
EnableBatchReading() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnableBatchReading enables batch reading on the device if supported.
|
||||||
|
// Returns nil if the device doesn't support or need explicit enabling.
|
||||||
|
func EnableBatchReading(d interface{}) error {
|
||||||
|
if be, ok := d.(BatchEnabler); ok {
|
||||||
|
return be.EnableBatchReading()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const DefaultMTU = 1300
|
const DefaultMTU = 1300
|
||||||
const VirtioNetHdrLen = 10 // Size of virtio_net_hdr structure
|
|
||||||
|
|
||||||
type NameError struct {
|
type NameError struct {
|
||||||
Name string
|
Name string
|
||||||
|
|||||||
+1
-24
@@ -99,29 +99,6 @@ func (t *tun) SupportsMultiqueue() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
|
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
|
return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
n, err := t.Read(bufs[0])
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
sizes[0] = n
|
|
||||||
return 1, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
for i, buf := range bufs {
|
|
||||||
_, err := t.Write(buf[offset:])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(bufs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|||||||
+1
-27
@@ -553,32 +553,6 @@ func (t *tun) SupportsMultiqueue() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
|
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
|
return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
|
||||||
}
|
}
|
||||||
|
|
||||||
// BatchRead reads a single packet (batch size 1 for non-Linux platforms)
|
|
||||||
func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
n, err := t.Read(bufs[0])
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
sizes[0] = n
|
|
||||||
return 1, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteBatch writes packets individually (no batching for non-Linux platforms)
|
|
||||||
func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
for i, buf := range bufs {
|
|
||||||
_, err := t.Write(buf[offset:])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(bufs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// BatchSize returns 1 for non-Linux platforms (no batching)
|
|
||||||
func (t *tun) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|||||||
+1
-27
@@ -109,36 +109,10 @@ func (t *disabledTun) SupportsMultiqueue() bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *disabledTun) NewMultiQueueReader() (BatchReadWriter, error) {
|
func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
return t, nil
|
return t, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// BatchRead reads a single packet (batch size 1 for disabled tun)
|
|
||||||
func (t *disabledTun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
n, err := t.Read(bufs[0])
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
sizes[0] = n
|
|
||||||
return 1, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteBatch writes packets individually (no batching for disabled tun)
|
|
||||||
func (t *disabledTun) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
for i, buf := range bufs {
|
|
||||||
_, err := t.Write(buf[offset:])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(bufs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// BatchSize returns 1 for disabled tun (no batching)
|
|
||||||
func (t *disabledTun) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *disabledTun) Close() error {
|
func (t *disabledTun) Close() error {
|
||||||
if t.read != nil {
|
if t.read != nil {
|
||||||
close(t.read)
|
close(t.read)
|
||||||
|
|||||||
+2
-27
@@ -7,6 +7,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -453,36 +454,10 @@ func (t *tun) SupportsMultiqueue() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
|
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
|
return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
|
||||||
}
|
}
|
||||||
|
|
||||||
// BatchRead reads a single packet (batch size 1 for FreeBSD)
|
|
||||||
func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
n, err := t.Read(bufs[0])
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
sizes[0] = n
|
|
||||||
return 1, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteBatch writes packets individually (no batching for FreeBSD)
|
|
||||||
func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
for i, buf := range bufs {
|
|
||||||
_, err := t.Write(buf[offset:])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(bufs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// BatchSize returns 1 for FreeBSD (no batching)
|
|
||||||
func (t *tun) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) addRoutes(logErrors bool) error {
|
func (t *tun) addRoutes(logErrors bool) error {
|
||||||
routes := *t.Routes.Load()
|
routes := *t.Routes.Load()
|
||||||
for _, r := range routes {
|
for _, r := range routes {
|
||||||
|
|||||||
+1
-24
@@ -155,29 +155,6 @@ func (t *tun) SupportsMultiqueue() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
|
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
|
return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
n, err := t.Read(bufs[0])
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
sizes[0] = n
|
|
||||||
return 1, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
for i, buf := range bufs {
|
|
||||||
_, err := t.Write(buf[offset:])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(bufs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|||||||
+404
-206
@@ -9,6 +9,7 @@ import (
|
|||||||
"net"
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
@@ -21,12 +22,15 @@ import (
|
|||||||
"github.com/slackhq/nebula/util"
|
"github.com/slackhq/nebula/util"
|
||||||
"github.com/vishvananda/netlink"
|
"github.com/vishvananda/netlink"
|
||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
wgtun "golang.zx2c4.com/wireguard/tun"
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// virtioNetHdrLen is the length of virtio_net_hdr (without mergeable buffers)
|
||||||
|
virtioNetHdrLen = 10
|
||||||
)
|
)
|
||||||
|
|
||||||
type tun struct {
|
type tun struct {
|
||||||
io.ReadWriteCloser
|
io.ReadWriteCloser
|
||||||
wgDevice wgtun.Device
|
|
||||||
fd int
|
fd int
|
||||||
Device string
|
Device string
|
||||||
vpnNetworks []netip.Prefix
|
vpnNetworks []netip.Prefix
|
||||||
@@ -35,6 +39,13 @@ type tun struct {
|
|||||||
TXQueueLen int
|
TXQueueLen int
|
||||||
deviceIndex int
|
deviceIndex int
|
||||||
ioctlFd uintptr
|
ioctlFd uintptr
|
||||||
|
nonBlocking bool // true if fd is in non-blocking mode
|
||||||
|
vnetHdr bool // true if IFF_VNET_HDR is enabled on the TUN device
|
||||||
|
|
||||||
|
// readBuf is used when vnetHdr is enabled to read the full packet+header
|
||||||
|
// before stripping the header. This is needed because caller-provided
|
||||||
|
// buffers are sized for MTU but kernel writes MTU+10 with virtio header.
|
||||||
|
readBuf []byte
|
||||||
|
|
||||||
Routes atomic.Pointer[[]Route]
|
Routes atomic.Pointer[[]Route]
|
||||||
routeTree atomic.Pointer[bart.Table[routing.Gateways]]
|
routeTree atomic.Pointer[bart.Table[routing.Gateways]]
|
||||||
@@ -54,6 +65,23 @@ func (t *tun) Networks() []netip.Prefix {
|
|||||||
return t.vpnNetworks
|
return t.vpnNetworks
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// tunVnetHdrSupported checks if the kernel supports IFF_VNET_HDR on TUN devices
|
||||||
|
func tunVnetHdrSupported() bool {
|
||||||
|
fd, err := unix.Open("/dev/net/tun", unix.O_RDONLY, 0)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer unix.Close(fd)
|
||||||
|
|
||||||
|
var features uint32
|
||||||
|
err = ioctl(uintptr(fd), uintptr(unix.TUNGETFEATURES), uintptr(unsafe.Pointer(&features)))
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return features&unix.IFF_VNET_HDR != 0
|
||||||
|
}
|
||||||
|
|
||||||
type ifReq struct {
|
type ifReq struct {
|
||||||
Name [16]byte
|
Name [16]byte
|
||||||
Flags uint16
|
Flags uint16
|
||||||
@@ -72,155 +100,85 @@ type ifreqQLEN struct {
|
|||||||
pad [8]byte
|
pad [8]byte
|
||||||
}
|
}
|
||||||
|
|
||||||
// wgDeviceWrapper wraps a wireguard Device to implement io.ReadWriteCloser
|
|
||||||
// This allows multiqueue readers to use the same wireguard Device batching as the main device
|
|
||||||
type wgDeviceWrapper struct {
|
|
||||||
dev wgtun.Device
|
|
||||||
buf []byte // Reusable buffer for single packet reads
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *wgDeviceWrapper) Read(b []byte) (int, error) {
|
|
||||||
// Use wireguard Device's batch API for single packet
|
|
||||||
bufs := [][]byte{b}
|
|
||||||
sizes := make([]int, 1)
|
|
||||||
n, err := w.dev.Read(bufs, sizes, 0)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if n == 0 {
|
|
||||||
return 0, io.EOF
|
|
||||||
}
|
|
||||||
return sizes[0], nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *wgDeviceWrapper) Write(b []byte) (int, error) {
|
|
||||||
// Buffer b should have virtio header space (10 bytes) at the beginning
|
|
||||||
// The decrypted packet data starts at offset 10
|
|
||||||
// Pass the full buffer to WireGuard with offset=virtioNetHdrLen
|
|
||||||
bufs := [][]byte{b}
|
|
||||||
n, err := w.dev.Write(bufs, VirtioNetHdrLen)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if n == 0 {
|
|
||||||
return 0, io.ErrShortWrite
|
|
||||||
}
|
|
||||||
return len(b), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *wgDeviceWrapper) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
// Pass all buffers to WireGuard's batch write
|
|
||||||
return w.dev.Write(bufs, offset)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *wgDeviceWrapper) Close() error {
|
|
||||||
return w.dev.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// BatchRead implements batching for multiqueue readers
|
|
||||||
func (w *wgDeviceWrapper) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
// The zero here is offset.
|
|
||||||
return w.dev.Read(bufs, sizes, 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// BatchSize returns the optimal batch size
|
|
||||||
func (w *wgDeviceWrapper) BatchSize() int {
|
|
||||||
return w.dev.BatchSize()
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
|
func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
|
||||||
wgDev, name, err := wgtun.CreateUnmonitoredTUNFromFD(deviceFd)
|
file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to create TUN from FD: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
file := wgDev.File()
|
|
||||||
t, err := newTunGeneric(c, l, file, vpnNetworks)
|
t, err := newTunGeneric(c, l, file, vpnNetworks)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = wgDev.Close()
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
t.wgDevice = wgDev
|
t.Device = "tun0"
|
||||||
t.Device = name
|
|
||||||
|
|
||||||
return t, nil
|
return t, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
|
func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
|
||||||
// Check if /dev/net/tun exists, create if needed (for docker containers)
|
|
||||||
if _, err := os.Stat("/dev/net/tun"); os.IsNotExist(err) {
|
|
||||||
if err := os.MkdirAll("/dev/net", 0755); err != nil {
|
|
||||||
return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
|
|
||||||
}
|
|
||||||
if err := unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200))); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
devName := c.GetString("tun.dev", "")
|
|
||||||
mtu := c.GetInt("tun.mtu", DefaultMTU)
|
|
||||||
|
|
||||||
// Create TUN device manually to support multiqueue
|
|
||||||
fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
|
fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
// If /dev/net/tun doesn't exist, try to create it (will happen in docker)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
err = os.MkdirAll("/dev/net", 0755)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
|
||||||
|
}
|
||||||
|
err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200)))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if VNET_HDR is supported before trying to use it
|
||||||
|
useVnetHdr := tunVnetHdrSupported()
|
||||||
|
|
||||||
var req ifReq
|
var req ifReq
|
||||||
req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_VNET_HDR)
|
req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
|
||||||
if multiqueue {
|
if multiqueue {
|
||||||
req.Flags |= unix.IFF_MULTI_QUEUE
|
req.Flags |= unix.IFF_MULTI_QUEUE
|
||||||
}
|
}
|
||||||
copy(req.Name[:], devName)
|
if useVnetHdr {
|
||||||
|
req.Flags |= unix.IFF_VNET_HDR
|
||||||
|
}
|
||||||
|
|
||||||
|
nameStr := c.GetString("tun.dev", "")
|
||||||
|
copy(req.Name[:], nameStr)
|
||||||
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
|
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
|
||||||
unix.Close(fd)
|
return nil, &NameError{
|
||||||
return nil, err
|
Name: nameStr,
|
||||||
|
Underlying: err,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
name := strings.Trim(string(req.Name[:]), "\x00")
|
||||||
|
|
||||||
// Set nonblocking
|
// Track if VNET_HDR is in use
|
||||||
if err = unix.SetNonblock(fd, true); err != nil {
|
// Note: We don't call TUNSETOFFLOAD - just handle the headers manually
|
||||||
unix.Close(fd)
|
vnetHdrEnabled := useVnetHdr
|
||||||
return nil, err
|
if vnetHdrEnabled {
|
||||||
}
|
l.Info("TUN VNET_HDR enabled")
|
||||||
|
|
||||||
// Enable TCP and UDP offload (TSO/GRO) for performance
|
|
||||||
// This allows the kernel to handle segmentation/coalescing
|
|
||||||
const (
|
|
||||||
tunTCPOffloads = unix.TUN_F_CSUM | unix.TUN_F_TSO4 | unix.TUN_F_TSO6
|
|
||||||
tunUDPOffloads = unix.TUN_F_USO4 | unix.TUN_F_USO6
|
|
||||||
)
|
|
||||||
offloads := tunTCPOffloads | tunUDPOffloads
|
|
||||||
if err = unix.IoctlSetInt(fd, unix.TUNSETOFFLOAD, offloads); err != nil {
|
|
||||||
// Log warning but don't fail - offload is optional
|
|
||||||
l.WithError(err).Warn("Failed to enable TUN offload (TSO/GRO), performance may be reduced")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
file := os.NewFile(uintptr(fd), "/dev/net/tun")
|
file := os.NewFile(uintptr(fd), "/dev/net/tun")
|
||||||
|
|
||||||
// Create wireguard device from file descriptor
|
|
||||||
wgDev, err := wgtun.CreateTUNFromFile(file, mtu)
|
|
||||||
if err != nil {
|
|
||||||
file.Close()
|
|
||||||
return nil, fmt.Errorf("failed to create TUN from file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
name, err := wgDev.Name()
|
|
||||||
if err != nil {
|
|
||||||
_ = wgDev.Close()
|
|
||||||
return nil, fmt.Errorf("failed to get TUN device name: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// file is now owned by wgDev, get a new reference
|
|
||||||
file = wgDev.File()
|
|
||||||
t, err := newTunGeneric(c, l, file, vpnNetworks)
|
t, err := newTunGeneric(c, l, file, vpnNetworks)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = wgDev.Close()
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
t.wgDevice = wgDev
|
|
||||||
t.Device = name
|
t.Device = name
|
||||||
|
t.vnetHdr = vnetHdrEnabled
|
||||||
|
|
||||||
|
// Allocate read buffer for virtio header handling
|
||||||
|
// Buffer needs to be large enough for virtio header + max packet
|
||||||
|
if t.vnetHdr {
|
||||||
|
t.readBuf = make([]byte, t.MaxMTU+virtioNetHdrLen)
|
||||||
|
}
|
||||||
|
|
||||||
return t, nil
|
return t, nil
|
||||||
}
|
}
|
||||||
@@ -330,44 +288,173 @@ func (t *tun) SupportsMultiqueue() bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
|
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
|
fd, err := unix.Open("/dev/net/tun", os.O_RDWR|unix.O_NONBLOCK, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var req ifReq
|
var req ifReq
|
||||||
// MUST match the flags used in newTun - includes IFF_VNET_HDR
|
req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
|
||||||
req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_VNET_HDR | unix.IFF_MULTI_QUEUE)
|
if t.vnetHdr {
|
||||||
|
req.Flags |= unix.IFF_VNET_HDR
|
||||||
|
}
|
||||||
copy(req.Name[:], t.Device)
|
copy(req.Name[:], t.Device)
|
||||||
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
|
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
|
||||||
unix.Close(fd)
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set nonblocking mode - CRITICAL for proper netpoller integration
|
reader := &tunBatchReader{fd: fd, device: t.Device, vnetHdr: t.vnetHdr}
|
||||||
if err = unix.SetNonblock(fd, true); err != nil {
|
if t.vnetHdr {
|
||||||
unix.Close(fd)
|
reader.readBuf = make([]byte, t.MaxMTU+virtioNetHdrLen)
|
||||||
return nil, err
|
}
|
||||||
|
return reader, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// tunBatchReader implements BatchReader for efficient batch packet reading
|
||||||
|
type tunBatchReader struct {
|
||||||
|
fd int
|
||||||
|
device string
|
||||||
|
vnetHdr bool
|
||||||
|
readBuf []byte // internal buffer for virtio header handling
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *tunBatchReader) Read(b []byte) (int, error) {
|
||||||
|
// Choose buffer: use internal buffer for vnetHdr, caller's buffer otherwise
|
||||||
|
readBuf := b
|
||||||
|
if r.vnetHdr {
|
||||||
|
readBuf = r.readBuf
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get MTU from main device
|
// Use poll to wait for data, then read
|
||||||
mtu := t.MaxMTU
|
for {
|
||||||
if mtu == 0 {
|
n, err := unix.Read(r.fd, readBuf)
|
||||||
mtu = DefaultMTU
|
if err == nil {
|
||||||
|
if r.vnetHdr && n > virtioNetHdrLen {
|
||||||
|
packetLen := n - virtioNetHdrLen
|
||||||
|
copy(b, readBuf[virtioNetHdrLen:n])
|
||||||
|
return packetLen, nil
|
||||||
|
}
|
||||||
|
if r.vnetHdr {
|
||||||
|
return 0, nil // No packet data
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
if err == unix.EAGAIN || err == unix.EWOULDBLOCK {
|
||||||
|
// Wait for data
|
||||||
|
pfds := []unix.PollFd{{Fd: int32(r.fd), Events: unix.POLLIN}}
|
||||||
|
_, err = unix.Poll(pfds, -1)
|
||||||
|
if err != nil {
|
||||||
|
if err == unix.EINTR {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *tunBatchReader) Write(b []byte) (int, error) {
|
||||||
|
if !r.vnetHdr {
|
||||||
|
return unix.Write(r.fd, b)
|
||||||
}
|
}
|
||||||
|
|
||||||
file := os.NewFile(uintptr(fd), "/dev/net/tun")
|
// Use writev to prepend virtio header without copying the packet data
|
||||||
|
// Header is all zeros = no GSO, no checksum offload
|
||||||
|
var hdr [virtioNetHdrLen]byte
|
||||||
|
bufs := [][]byte{hdr[:], b}
|
||||||
|
|
||||||
// Create wireguard Device from the file descriptor (just like the main device)
|
n, err := unix.Writev(r.fd, bufs)
|
||||||
wgDev, err := wgtun.CreateTUNFromFile(file, mtu)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
file.Close()
|
return 0, err
|
||||||
return nil, fmt.Errorf("failed to create multiqueue TUN device: %w", err)
|
}
|
||||||
|
// Return only the packet bytes written (exclude header)
|
||||||
|
if n > virtioNetHdrLen {
|
||||||
|
return n - virtioNetHdrLen, nil
|
||||||
|
}
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *tunBatchReader) Close() error {
|
||||||
|
return unix.Close(r.fd)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadBatch reads up to len(packets) packets from the TUN device.
|
||||||
|
// It drains all available packets without blocking, using poll() only
|
||||||
|
// when no packets have been read yet.
|
||||||
|
func (r *tunBatchReader) ReadBatch(packets [][]byte, sizes []int) (int, error) {
|
||||||
|
count := 0
|
||||||
|
maxPackets := len(packets)
|
||||||
|
if len(sizes) < maxPackets {
|
||||||
|
maxPackets = len(sizes)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return a wrapper that uses the wireguard Device for all I/O
|
// Choose read buffer based on vnetHdr
|
||||||
return &wgDeviceWrapper{dev: wgDev}, nil
|
readBuf := packets[0] // Will be updated in loop for non-vnetHdr
|
||||||
|
if r.vnetHdr {
|
||||||
|
readBuf = r.readBuf
|
||||||
|
}
|
||||||
|
|
||||||
|
for count < maxPackets {
|
||||||
|
if !r.vnetHdr {
|
||||||
|
readBuf = packets[count]
|
||||||
|
}
|
||||||
|
|
||||||
|
n, err := unix.Read(r.fd, readBuf)
|
||||||
|
if err == nil && n > 0 {
|
||||||
|
if r.vnetHdr {
|
||||||
|
if n > virtioNetHdrLen {
|
||||||
|
packetLen := n - virtioNetHdrLen
|
||||||
|
copy(packets[count], readBuf[virtioNetHdrLen:n])
|
||||||
|
sizes[count] = packetLen
|
||||||
|
} else {
|
||||||
|
// Malformed packet (no data after header), skip
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sizes[count] = n
|
||||||
|
}
|
||||||
|
count++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err == unix.EAGAIN || err == unix.EWOULDBLOCK {
|
||||||
|
// No more packets available
|
||||||
|
if count > 0 {
|
||||||
|
// We have some packets, return them
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
// No packets yet, wait for at least one
|
||||||
|
pfds := []unix.PollFd{{Fd: int32(r.fd), Events: unix.POLLIN}}
|
||||||
|
_, err = unix.Poll(pfds, -1)
|
||||||
|
if err != nil {
|
||||||
|
if err == unix.EINTR {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
if count > 0 {
|
||||||
|
// Return what we have
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if n == 0 {
|
||||||
|
if count > 0 {
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
|
func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
|
||||||
@@ -375,68 +462,28 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
|
|||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) Read(b []byte) (int, error) {
|
func (t *tun) Write(b []byte) (int, error) {
|
||||||
if t.wgDevice != nil {
|
if !t.vnetHdr {
|
||||||
// Use wireguard device which handles virtio headers internally
|
return t.writeSimple(b)
|
||||||
bufs := [][]byte{b}
|
|
||||||
sizes := make([]int, 1)
|
|
||||||
n, err := t.wgDevice.Read(bufs, sizes, 0)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if n == 0 {
|
|
||||||
return 0, io.EOF
|
|
||||||
}
|
|
||||||
return sizes[0], nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: direct read from file (shouldn't happen in normal operation)
|
// Use writev to prepend virtio header without copying the packet data
|
||||||
return t.ReadWriteCloser.Read(b)
|
// Header is all zeros = no GSO, no checksum offload
|
||||||
}
|
var hdr [virtioNetHdrLen]byte
|
||||||
|
bufs := [][]byte{hdr[:], b}
|
||||||
|
|
||||||
// BatchRead reads multiple packets at once for improved performance
|
n, err := unix.Writev(t.fd, bufs)
|
||||||
// bufs: slice of buffers to read into
|
|
||||||
// sizes: slice that will be filled with packet sizes
|
|
||||||
// Returns number of packets read
|
|
||||||
func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
if t.wgDevice != nil {
|
|
||||||
return t.wgDevice.Read(bufs, sizes, 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: single packet read
|
|
||||||
n, err := t.ReadWriteCloser.Read(bufs[0])
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
sizes[0] = n
|
// Return only the packet bytes written (exclude header)
|
||||||
return 1, nil
|
if n > virtioNetHdrLen {
|
||||||
|
return n - virtioNetHdrLen, nil
|
||||||
|
}
|
||||||
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// BatchSize returns the optimal number of packets to read/write in a batch
|
func (t *tun) writeSimple(b []byte) (int, error) {
|
||||||
func (t *tun) BatchSize() int {
|
|
||||||
if t.wgDevice != nil {
|
|
||||||
return t.wgDevice.BatchSize()
|
|
||||||
}
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) Write(b []byte) (int, error) {
|
|
||||||
if t.wgDevice != nil {
|
|
||||||
// Buffer b should have virtio header space (10 bytes) at the beginning
|
|
||||||
// The decrypted packet data starts at offset 10
|
|
||||||
// Pass the full buffer to WireGuard with offset=virtioNetHdrLen
|
|
||||||
bufs := [][]byte{b}
|
|
||||||
n, err := t.wgDevice.Write(bufs, VirtioNetHdrLen)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if n == 0 {
|
|
||||||
return 0, io.ErrShortWrite
|
|
||||||
}
|
|
||||||
return len(b), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: direct write (shouldn't happen in normal operation)
|
|
||||||
var nn int
|
var nn int
|
||||||
maximum := len(b)
|
maximum := len(b)
|
||||||
|
|
||||||
@@ -459,20 +506,175 @@ func (t *tun) Write(b []byte) (int, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteBatch writes multiple packets to the TUN device in a single syscall
|
// EnableBatchReading sets the TUN fd to non-blocking mode to enable batch reading.
|
||||||
func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
// This should be called before using ReadBatch.
|
||||||
if t.wgDevice != nil {
|
func (t *tun) EnableBatchReading() error {
|
||||||
return t.wgDevice.Write(bufs, offset)
|
if t.nonBlocking {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
err := unix.SetNonblock(t.fd, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
t.nonBlocking = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read overrides the default Read to handle non-blocking mode and virtio headers
|
||||||
|
func (t *tun) Read(b []byte) (int, error) {
|
||||||
|
if !t.vnetHdr {
|
||||||
|
return t.readSimple(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: write individually (shouldn't happen in normal operation)
|
// With VNET_HDR, read into internal buffer (which has space for header)
|
||||||
for i, buf := range bufs {
|
// then copy packet data to caller's buffer
|
||||||
_, err := t.Write(buf)
|
if !t.nonBlocking {
|
||||||
|
n, err := t.ReadWriteCloser.Read(t.readBuf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return i, err
|
return 0, err
|
||||||
|
}
|
||||||
|
if n <= virtioNetHdrLen {
|
||||||
|
return 0, nil // No packet data
|
||||||
|
}
|
||||||
|
packetLen := n - virtioNetHdrLen
|
||||||
|
copy(b, t.readBuf[virtioNetHdrLen:n])
|
||||||
|
return packetLen, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Non-blocking read with poll
|
||||||
|
for {
|
||||||
|
n, err := unix.Read(t.fd, t.readBuf)
|
||||||
|
if err == nil {
|
||||||
|
if n <= virtioNetHdrLen {
|
||||||
|
return 0, nil // No packet data
|
||||||
|
}
|
||||||
|
packetLen := n - virtioNetHdrLen
|
||||||
|
copy(b, t.readBuf[virtioNetHdrLen:n])
|
||||||
|
return packetLen, nil
|
||||||
|
}
|
||||||
|
if err == unix.EAGAIN || err == unix.EWOULDBLOCK {
|
||||||
|
pfds := []unix.PollFd{{Fd: int32(t.fd), Events: unix.POLLIN}}
|
||||||
|
_, err = unix.Poll(pfds, -1)
|
||||||
|
if err != nil {
|
||||||
|
if err == unix.EINTR {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tun) readSimple(b []byte) (int, error) {
|
||||||
|
if !t.nonBlocking {
|
||||||
|
return t.ReadWriteCloser.Read(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
for {
|
||||||
|
n, err := unix.Read(t.fd, b)
|
||||||
|
if err == nil {
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
if err == unix.EAGAIN || err == unix.EWOULDBLOCK {
|
||||||
|
pfds := []unix.PollFd{{Fd: int32(t.fd), Events: unix.POLLIN}}
|
||||||
|
_, err = unix.Poll(pfds, -1)
|
||||||
|
if err != nil {
|
||||||
|
if err == unix.EINTR {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadBatch reads up to len(packets) packets from the TUN device.
|
||||||
|
// EnableBatchReading must be called first.
|
||||||
|
func (t *tun) ReadBatch(packets [][]byte, sizes []int) (int, error) {
|
||||||
|
if !t.nonBlocking {
|
||||||
|
// Fallback to single read if non-blocking not enabled
|
||||||
|
n, err := t.Read(packets[0])
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
sizes[0] = n
|
||||||
|
return 1, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
count := 0
|
||||||
|
maxPackets := len(packets)
|
||||||
|
if len(sizes) < maxPackets {
|
||||||
|
maxPackets = len(sizes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Choose read buffer based on vnetHdr
|
||||||
|
// With vnetHdr, we need to read into internal buffer (has space for header)
|
||||||
|
// then copy packet data to caller's buffer
|
||||||
|
readBuf := packets[0] // Will be updated in the loop
|
||||||
|
if t.vnetHdr {
|
||||||
|
readBuf = t.readBuf
|
||||||
|
}
|
||||||
|
|
||||||
|
for count < maxPackets {
|
||||||
|
if !t.vnetHdr {
|
||||||
|
readBuf = packets[count]
|
||||||
|
}
|
||||||
|
|
||||||
|
n, err := unix.Read(t.fd, readBuf)
|
||||||
|
if err == nil && n > 0 {
|
||||||
|
if t.vnetHdr {
|
||||||
|
if n > virtioNetHdrLen {
|
||||||
|
packetLen := n - virtioNetHdrLen
|
||||||
|
copy(packets[count], readBuf[virtioNetHdrLen:n])
|
||||||
|
sizes[count] = packetLen
|
||||||
|
} else {
|
||||||
|
// Malformed packet (no data after header), skip
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sizes[count] = n
|
||||||
|
}
|
||||||
|
count++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err == unix.EAGAIN || err == unix.EWOULDBLOCK {
|
||||||
|
// No more packets available
|
||||||
|
if count > 0 {
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
// No packets yet, wait for at least one
|
||||||
|
pfds := []unix.PollFd{{Fd: int32(t.fd), Events: unix.POLLIN}}
|
||||||
|
_, err = unix.Poll(pfds, -1)
|
||||||
|
if err != nil {
|
||||||
|
if err == unix.EINTR {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
if count > 0 {
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if n == 0 {
|
||||||
|
if count > 0 {
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
return 0, io.EOF
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return len(bufs), nil
|
|
||||||
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) deviceBytes() (o [16]byte) {
|
func (t *tun) deviceBytes() (o [16]byte) {
|
||||||
@@ -902,10 +1104,6 @@ func (t *tun) Close() error {
|
|||||||
close(t.routeChan)
|
close(t.routeChan)
|
||||||
}
|
}
|
||||||
|
|
||||||
if t.wgDevice != nil {
|
|
||||||
_ = t.wgDevice.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
if t.ReadWriteCloser != nil {
|
if t.ReadWriteCloser != nil {
|
||||||
_ = t.ReadWriteCloser.Close()
|
_ = t.ReadWriteCloser.Close()
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-24
@@ -6,6 +6,7 @@ package overlay
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"regexp"
|
"regexp"
|
||||||
@@ -393,33 +394,10 @@ func (t *tun) SupportsMultiqueue() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
|
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd")
|
return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
n, err := t.Read(bufs[0])
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
sizes[0] = n
|
|
||||||
return 1, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
for i, buf := range bufs {
|
|
||||||
_, err := t.Write(buf[offset:])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(bufs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) addRoutes(logErrors bool) error {
|
func (t *tun) addRoutes(logErrors bool) error {
|
||||||
routes := *t.Routes.Load()
|
routes := *t.Routes.Load()
|
||||||
|
|
||||||
|
|||||||
+2
-24
@@ -6,6 +6,7 @@ package overlay
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"regexp"
|
"regexp"
|
||||||
@@ -313,33 +314,10 @@ func (t *tun) SupportsMultiqueue() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) NewMultiQueueReader() (BatchReadWriter, error) {
|
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd")
|
return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
n, err := t.Read(bufs[0])
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
sizes[0] = n
|
|
||||||
return 1, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
for i, buf := range bufs {
|
|
||||||
_, err := t.Write(buf[offset:])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(bufs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tun) addRoutes(logErrors bool) error {
|
func (t *tun) addRoutes(logErrors bool) error {
|
||||||
routes := *t.Routes.Load()
|
routes := *t.Routes.Load()
|
||||||
|
|
||||||
|
|||||||
+3
-35
@@ -109,12 +109,8 @@ func (t *TestTun) Write(b []byte) (n int, err error) {
|
|||||||
return 0, io.ErrClosedPipe
|
return 0, io.ErrClosedPipe
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip virtio header (consistent with production Linux tun)
|
packet := make([]byte, len(b), len(b))
|
||||||
// The buffer b has VirtioNetHdrLen bytes of header followed by the actual packet
|
copy(packet, b)
|
||||||
data := b[VirtioNetHdrLen:]
|
|
||||||
|
|
||||||
packet := make([]byte, len(data))
|
|
||||||
copy(packet, data)
|
|
||||||
t.TxPackets <- packet
|
t.TxPackets <- packet
|
||||||
return len(b), nil
|
return len(b), nil
|
||||||
}
|
}
|
||||||
@@ -140,34 +136,6 @@ func (t *TestTun) SupportsMultiqueue() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestTun) NewMultiQueueReader() (BatchReadWriter, error) {
|
func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
return nil, fmt.Errorf("TODO: multiqueue not implemented")
|
return nil, fmt.Errorf("TODO: multiqueue not implemented")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestTun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
n, err := t.Read(bufs[0])
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
sizes[0] = n
|
|
||||||
return 1, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TestTun) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
if t.closed.Load() {
|
|
||||||
return 0, io.ErrClosedPipe
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, buf := range bufs {
|
|
||||||
// Strip the header at offset and send directly to channel
|
|
||||||
data := buf[offset:]
|
|
||||||
packet := make([]byte, len(data))
|
|
||||||
copy(packet, data)
|
|
||||||
t.TxPackets <- packet
|
|
||||||
}
|
|
||||||
return len(bufs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TestTun) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|||||||
+2
-27
@@ -6,6 +6,7 @@ package overlay
|
|||||||
import (
|
import (
|
||||||
"crypto"
|
"crypto"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -240,36 +241,10 @@ func (t *winTun) SupportsMultiqueue() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *winTun) NewMultiQueueReader() (BatchReadWriter, error) {
|
func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
|
return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
|
||||||
}
|
}
|
||||||
|
|
||||||
// BatchRead reads a single packet (batch size 1 for Windows)
|
|
||||||
func (t *winTun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
n, err := t.Read(bufs[0])
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
sizes[0] = n
|
|
||||||
return 1, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteBatch writes packets individually (no batching for Windows)
|
|
||||||
func (t *winTun) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
for i, buf := range bufs {
|
|
||||||
_, err := t.Write(buf[offset:])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(bufs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// BatchSize returns 1 for Windows (no batching)
|
|
||||||
func (t *winTun) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *winTun) Close() error {
|
func (t *winTun) Close() error {
|
||||||
// It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes,
|
// It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes,
|
||||||
// so to be certain, just remove everything before destroying.
|
// so to be certain, just remove everything before destroying.
|
||||||
|
|||||||
+1
-27
@@ -50,36 +50,10 @@ func (d *UserDevice) SupportsMultiqueue() bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *UserDevice) NewMultiQueueReader() (BatchReadWriter, error) {
|
func (d *UserDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
return d, nil
|
return d, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// BatchRead reads a single packet (batch size 1 for UserDevice)
|
|
||||||
func (d *UserDevice) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
n, err := d.Read(bufs[0])
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
sizes[0] = n
|
|
||||||
return 1, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteBatch writes packets individually (no batching for UserDevice)
|
|
||||||
func (d *UserDevice) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
for i, buf := range bufs {
|
|
||||||
_, err := d.Write(buf[offset:])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(bufs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// BatchSize returns 1 for UserDevice (no batching)
|
|
||||||
func (d *UserDevice) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *UserDevice) Pipe() (*io.PipeReader, *io.PipeWriter) {
|
func (d *UserDevice) Pipe() (*io.PipeReader, *io.PipeWriter) {
|
||||||
return d.inboundReader, d.outboundWriter
|
return d.inboundReader, d.outboundWriter
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
_ "net/http/pprof"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|||||||
@@ -1,11 +1,10 @@
|
|||||||
package device
|
package test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
|
|
||||||
"github.com/slackhq/nebula/overlay"
|
|
||||||
"github.com/slackhq/nebula/routing"
|
"github.com/slackhq/nebula/routing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -39,25 +38,10 @@ func (NoopTun) SupportsMultiqueue() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (NoopTun) NewMultiQueueReader() (overlay.BatchReadWriter, error) {
|
func (NoopTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||||
return nil, errors.New("unsupported")
|
return nil, errors.New("unsupported")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (NoopTun) Close() error {
|
func (NoopTun) Close() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// BatchRead implements BatchReadWriter interface
|
|
||||||
func (NoopTun) BatchRead(bufs [][]byte, sizes []int) (int, error) {
|
|
||||||
return 0, io.EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteBatch implements BatchReadWriter interface
|
|
||||||
func (NoopTun) WriteBatch(bufs [][]byte, offset int) (int, error) {
|
|
||||||
return len(bufs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// BatchSize implements BatchReadWriter interface
|
|
||||||
func (NoopTun) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
+22
-20
@@ -13,21 +13,22 @@ type EncReader func(
|
|||||||
payload []byte,
|
payload []byte,
|
||||||
)
|
)
|
||||||
|
|
||||||
type EncBatchReader func(
|
// BatchPacket represents a single packet in a batch write operation
|
||||||
addrs []netip.AddrPort,
|
type BatchPacket struct {
|
||||||
payloads [][]byte,
|
Payload []byte
|
||||||
count int,
|
Addr netip.AddrPort
|
||||||
)
|
}
|
||||||
|
|
||||||
type Conn interface {
|
type Conn interface {
|
||||||
Rebind() error
|
Rebind() error
|
||||||
LocalAddr() (netip.AddrPort, error)
|
LocalAddr() (netip.AddrPort, error)
|
||||||
ListenOutBatch(r EncBatchReader)
|
ListenOut(r EncReader)
|
||||||
WriteTo(b []byte, addr netip.AddrPort) error
|
WriteTo(b []byte, addr netip.AddrPort) error
|
||||||
WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error)
|
WriteBatch(pkts []BatchPacket) (int, error)
|
||||||
ReloadConfig(c *config.C)
|
ReloadConfig(c *config.C)
|
||||||
SupportsMultipleReaders() bool
|
SupportsMultipleReaders() bool
|
||||||
BatchSize() int
|
SupportsGSO() bool
|
||||||
|
SupportsGRO() bool
|
||||||
Close() error
|
Close() error
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -39,25 +40,26 @@ func (NoopConn) Rebind() error {
|
|||||||
func (NoopConn) LocalAddr() (netip.AddrPort, error) {
|
func (NoopConn) LocalAddr() (netip.AddrPort, error) {
|
||||||
return netip.AddrPort{}, nil
|
return netip.AddrPort{}, nil
|
||||||
}
|
}
|
||||||
|
func (NoopConn) ListenOut(_ EncReader) {
|
||||||
func (NoopConn) ListenOut(_ EncReader) {}
|
return
|
||||||
|
}
|
||||||
func (NoopConn) SupportsMultipleReaders() bool {
|
func (NoopConn) SupportsMultipleReaders() bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
func (NoopConn) SupportsGSO() bool {
|
||||||
func (NoopConn) ListenOutBatch(_ EncBatchReader) {}
|
return false
|
||||||
|
}
|
||||||
|
func (NoopConn) SupportsGRO() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
|
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (NoopConn) WriteMulti(_ [][]byte, _ []netip.AddrPort) (int, error) {
|
func (NoopConn) WriteBatch(pkts []BatchPacket) (int, error) {
|
||||||
return 0, nil
|
return len(pkts), nil
|
||||||
}
|
}
|
||||||
func (NoopConn) ReloadConfig(_ *config.C) {}
|
func (NoopConn) ReloadConfig(_ *config.C) {
|
||||||
|
return
|
||||||
func (NoopConn) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
}
|
||||||
func (NoopConn) Close() error {
|
func (NoopConn) Close() error {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
+13
-35
@@ -140,17 +140,6 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteMulti sends multiple packets - fallback implementation without sendmmsg
|
|
||||||
func (u *StdConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
|
|
||||||
for i := range packets {
|
|
||||||
err := u.WriteTo(packets[i], addrs[i])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(packets), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
|
func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
|
||||||
a := u.UDPConn.LocalAddr()
|
a := u.UDPConn.LocalAddr()
|
||||||
|
|
||||||
@@ -199,32 +188,12 @@ func (u *StdConn) SupportsMultipleReaders() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListenOutBatch - fallback to single-packet reads for Darwin
|
func (u *StdConn) SupportsGSO() bool {
|
||||||
func (u *StdConn) ListenOutBatch(r EncBatchReader) {
|
return false
|
||||||
buffer := make([]byte, MTU)
|
|
||||||
addrs := make([]netip.AddrPort, 1)
|
|
||||||
payloads := make([][]byte, 1)
|
|
||||||
|
|
||||||
for {
|
|
||||||
// Just read one packet at a time and call batch callback with count=1
|
|
||||||
n, rua, err := u.ReadFromUDPAddrPort(buffer)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, net.ErrClosed) {
|
|
||||||
u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
u.l.WithError(err).Error("unexpected udp socket receive error")
|
|
||||||
}
|
|
||||||
|
|
||||||
addrs[0] = netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port())
|
|
||||||
payloads[0] = buffer[:n]
|
|
||||||
r(addrs, payloads, 1)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) BatchSize() int {
|
func (u *StdConn) SupportsGRO() bool {
|
||||||
return 1
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) Rebind() error {
|
func (u *StdConn) Rebind() error {
|
||||||
@@ -241,3 +210,12 @@ func (u *StdConn) Rebind() error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (u *StdConn) WriteBatch(pkts []BatchPacket) (int, error) {
|
||||||
|
for i := range pkts {
|
||||||
|
if err := u.WriteTo(pkts[i].Payload, pkts[i].Addr); err != nil {
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(pkts), nil
|
||||||
|
}
|
||||||
|
|||||||
+10
-28
@@ -102,37 +102,19 @@ func (u *GenericConn) SupportsMultipleReaders() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListenOutBatch - fallback to single-packet reads for generic platforms
|
func (u *GenericConn) SupportsGSO() bool {
|
||||||
func (u *GenericConn) ListenOutBatch(r EncBatchReader) {
|
return false
|
||||||
buffer := make([]byte, MTU)
|
|
||||||
addrs := make([]netip.AddrPort, 1)
|
|
||||||
payloads := make([][]byte, 1)
|
|
||||||
|
|
||||||
for {
|
|
||||||
// Just read one packet at a time and call batch callback with count=1
|
|
||||||
n, rua, err := u.ReadFromUDPAddrPort(buffer)
|
|
||||||
if err != nil {
|
|
||||||
u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
addrs[0] = netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port())
|
|
||||||
payloads[0] = buffer[:n]
|
|
||||||
r(addrs, payloads, 1)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteMulti sends multiple packets - fallback implementation
|
func (u *GenericConn) SupportsGRO() bool {
|
||||||
func (u *GenericConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
|
return false
|
||||||
for i := range packets {
|
}
|
||||||
err := u.WriteTo(packets[i], addrs[i])
|
|
||||||
if err != nil {
|
func (u *GenericConn) WriteBatch(pkts []BatchPacket) (int, error) {
|
||||||
|
for i := range pkts {
|
||||||
|
if err := u.WriteTo(pkts[i].Payload, pkts[i].Addr); err != nil {
|
||||||
return i, err
|
return i, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return len(packets), nil
|
return len(pkts), nil
|
||||||
}
|
|
||||||
|
|
||||||
func (u *GenericConn) BatchSize() int {
|
|
||||||
return 1
|
|
||||||
}
|
}
|
||||||
|
|||||||
+356
-176
@@ -5,6 +5,7 @@ package udp
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
@@ -18,15 +19,12 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type StdConn struct {
|
type StdConn struct {
|
||||||
sysFd int
|
sysFd int
|
||||||
isV4 bool
|
isV4 bool
|
||||||
l *logrus.Logger
|
l *logrus.Logger
|
||||||
batch int
|
batch int
|
||||||
|
gsoSupported bool
|
||||||
// Pre-allocated buffers for batch writes (sized for IPv6, works for both)
|
groSupported bool
|
||||||
writeMsgs []rawMessage
|
|
||||||
writeIovecs []iovec
|
|
||||||
writeNames [][]byte
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func maybeIPV4(ip net.IP) (net.IP, bool) {
|
func maybeIPV4(ip net.IP) (net.IP, bool) {
|
||||||
@@ -37,6 +35,78 @@ func maybeIPV4(ip net.IP) (net.IP, bool) {
|
|||||||
return ip, false
|
return ip, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// supportsUDPOffload checks if the kernel supports UDP GSO (Generic Segmentation Offload)
|
||||||
|
// by attempting to get the UDP_SEGMENT socket option.
|
||||||
|
func supportsUDPOffload(fd int) bool {
|
||||||
|
_, err := unix.GetsockoptInt(fd, unix.IPPROTO_UDP, unix.UDP_SEGMENT)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// supportsUDPGRO checks if the kernel supports UDP GRO (Generic Receive Offload)
|
||||||
|
// and attempts to enable it on the socket.
|
||||||
|
func supportsUDPGRO(fd int) bool {
|
||||||
|
// Try to enable UDP_GRO
|
||||||
|
err := unix.SetsockoptInt(fd, unix.IPPROTO_UDP, unix.UDP_GRO, 1)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
// Maximum number of datagrams that can be coalesced with GSO/GRO
|
||||||
|
udpSegmentMaxDatagrams = 64
|
||||||
|
|
||||||
|
// Maximum size of a GRO coalesced packet (64KB is the practical limit)
|
||||||
|
// This is udpSegmentMaxDatagrams * MTU but capped at 65535
|
||||||
|
groMaxPacketSize = 65535
|
||||||
|
)
|
||||||
|
|
||||||
|
// setGSOSize writes a UDP_SEGMENT control message to the provided buffer
|
||||||
|
// with the given segment size. Returns the actual control message length.
|
||||||
|
func setGSOSize(control []byte, gsoSize uint16) int {
|
||||||
|
// Build the cmsghdr structure
|
||||||
|
cmsgLen := unix.CmsgLen(2) // 2 bytes for uint16 segment size
|
||||||
|
cmsg := (*unix.Cmsghdr)(unsafe.Pointer(&control[0]))
|
||||||
|
cmsg.Level = unix.IPPROTO_UDP
|
||||||
|
cmsg.Type = unix.UDP_SEGMENT
|
||||||
|
cmsg.SetLen(cmsgLen)
|
||||||
|
|
||||||
|
// Write the segment size after the header (after cmsghdr)
|
||||||
|
binary.NativeEndian.PutUint16(control[unix.SizeofCmsghdr:], gsoSize)
|
||||||
|
|
||||||
|
return unix.CmsgSpace(2) // aligned size
|
||||||
|
}
|
||||||
|
|
||||||
|
// getGROSize parses a control message buffer to extract the UDP_GRO segment size.
|
||||||
|
// Returns 0 if no GRO control message is present (meaning the packet is not coalesced).
|
||||||
|
func getGROSize(control []byte, controlLen int) uint16 {
|
||||||
|
if controlLen < unix.SizeofCmsghdr {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse control messages
|
||||||
|
for offset := 0; offset < controlLen; {
|
||||||
|
if offset+unix.SizeofCmsghdr > controlLen {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
cmsg := (*unix.Cmsghdr)(unsafe.Pointer(&control[offset]))
|
||||||
|
cmsgDataLen := int(cmsg.Len) - unix.SizeofCmsghdr
|
||||||
|
if cmsgDataLen < 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
if cmsg.Level == unix.IPPROTO_UDP && cmsg.Type == unix.UDP_GRO {
|
||||||
|
if cmsgDataLen >= 2 {
|
||||||
|
return binary.NativeEndian.Uint16(control[offset+unix.SizeofCmsghdr:])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Move to next control message (aligned)
|
||||||
|
offset += unix.CmsgSpace(cmsgDataLen)
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
|
func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
|
||||||
af := unix.AF_INET6
|
af := unix.AF_INET6
|
||||||
if ip.Is4() {
|
if ip.Is4() {
|
||||||
@@ -74,32 +144,31 @@ func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch in
|
|||||||
return nil, fmt.Errorf("unable to bind to socket: %s", err)
|
return nil, fmt.Errorf("unable to bind to socket: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
c := &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}
|
gsoSupported := supportsUDPOffload(fd)
|
||||||
|
if gsoSupported {
|
||||||
// Pre-allocate write message structures for batching (sized for IPv6, works for both)
|
l.Info("UDP GSO offload is supported")
|
||||||
c.writeMsgs = make([]rawMessage, batch)
|
|
||||||
c.writeIovecs = make([]iovec, batch)
|
|
||||||
c.writeNames = make([][]byte, batch)
|
|
||||||
|
|
||||||
for i := range c.writeMsgs {
|
|
||||||
// Allocate for IPv6 size (larger than IPv4, works for both)
|
|
||||||
c.writeNames[i] = make([]byte, unix.SizeofSockaddrInet6)
|
|
||||||
|
|
||||||
// Point to the iovec in the slice
|
|
||||||
c.writeMsgs[i].Hdr.Iov = &c.writeIovecs[i]
|
|
||||||
c.writeMsgs[i].Hdr.Iovlen = 1
|
|
||||||
|
|
||||||
c.writeMsgs[i].Hdr.Name = &c.writeNames[i][0]
|
|
||||||
// Namelen will be set appropriately in writeMulti4/writeMulti6
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return c, err
|
groSupported := supportsUDPGRO(fd)
|
||||||
|
if groSupported {
|
||||||
|
l.Info("UDP GRO offload is supported and enabled")
|
||||||
|
}
|
||||||
|
|
||||||
|
return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch, gsoSupported: gsoSupported, groSupported: groSupported}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) SupportsMultipleReaders() bool {
|
func (u *StdConn) SupportsMultipleReaders() bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (u *StdConn) SupportsGSO() bool {
|
||||||
|
return u.gsoSupported
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *StdConn) SupportsGRO() bool {
|
||||||
|
return u.groSupported
|
||||||
|
}
|
||||||
|
|
||||||
func (u *StdConn) Rebind() error {
|
func (u *StdConn) Rebind() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -146,43 +215,68 @@ func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) ListenOutBatch(r EncBatchReader) {
|
func (u *StdConn) ListenOut(r EncReader) {
|
||||||
var ip netip.Addr
|
var ip netip.Addr
|
||||||
|
|
||||||
msgs, buffers, names := u.PrepareRawMessages(u.batch)
|
msgs, buffers, names, controls := u.PrepareRawMessages(u.batch)
|
||||||
read := u.ReadMulti
|
read := u.ReadMulti
|
||||||
if u.batch == 1 {
|
if u.batch == 1 {
|
||||||
read = u.ReadSingle
|
read = u.ReadSingle
|
||||||
}
|
}
|
||||||
|
|
||||||
udpBatchHist := metrics.GetOrRegisterHistogram("batch.udp_read_size", nil, metrics.NewUniformSample(1024))
|
// Store the original control buffer size for resetting after each read
|
||||||
|
controlLen := 0
|
||||||
// Pre-allocate slices for batch callback
|
if u.groSupported && len(controls) > 0 && len(controls[0]) > 0 {
|
||||||
addrs := make([]netip.AddrPort, u.batch)
|
controlLen = len(controls[0])
|
||||||
payloads := make([][]byte, u.batch)
|
}
|
||||||
|
|
||||||
for {
|
for {
|
||||||
|
// Reset Controllen before each read - the kernel updates this field
|
||||||
|
// after recvmsg to indicate actual received control data length
|
||||||
|
if controlLen > 0 {
|
||||||
|
for i := range msgs {
|
||||||
|
setMsghdrControllen(&msgs[i].Hdr, controlLen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
n, err := read(msgs)
|
n, err := read(msgs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
|
u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
udpBatchHist.Update(int64(n))
|
|
||||||
|
|
||||||
// Prepare batch data
|
|
||||||
for i := 0; i < n; i++ {
|
for i := 0; i < n; i++ {
|
||||||
|
// Extract source address
|
||||||
if u.isV4 {
|
if u.isV4 {
|
||||||
ip, _ = netip.AddrFromSlice(names[i][4:8])
|
ip, _ = netip.AddrFromSlice(names[i][4:8])
|
||||||
} else {
|
} else {
|
||||||
ip, _ = netip.AddrFromSlice(names[i][8:24])
|
ip, _ = netip.AddrFromSlice(names[i][8:24])
|
||||||
}
|
}
|
||||||
addrs[i] = netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4]))
|
srcAddr := netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4]))
|
||||||
payloads[i] = buffers[i][:msgs[i].Len]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Call batch callback with all packets
|
// Check for GRO coalesced packet
|
||||||
r(addrs, payloads, n)
|
totalLen := int(msgs[i].Len)
|
||||||
|
segmentSize := uint16(0)
|
||||||
|
if controlLen > 0 {
|
||||||
|
segmentSize = getGROSize(controls[i], getMsghdrControllen(&msgs[i].Hdr))
|
||||||
|
}
|
||||||
|
|
||||||
|
if segmentSize > 0 && totalLen > int(segmentSize) {
|
||||||
|
// This is a GRO coalesced packet - split it into individual datagrams
|
||||||
|
for offset := 0; offset < totalLen; {
|
||||||
|
packetLen := int(segmentSize)
|
||||||
|
if offset+packetLen > totalLen {
|
||||||
|
// Last packet may be smaller
|
||||||
|
packetLen = totalLen - offset
|
||||||
|
}
|
||||||
|
r(srcAddr, buffers[i][offset:offset+packetLen])
|
||||||
|
offset += packetLen
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Single packet, no coalescing
|
||||||
|
r(srcAddr, buffers[i][:totalLen])
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -234,19 +328,6 @@ func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
|
|||||||
return u.writeTo6(b, ip)
|
return u.writeTo6(b, ip)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
|
|
||||||
if len(packets) != len(addrs) {
|
|
||||||
return 0, fmt.Errorf("packets and addrs length mismatch")
|
|
||||||
}
|
|
||||||
if len(packets) == 0 {
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
if u.isV4 {
|
|
||||||
return u.writeMulti4(packets, addrs)
|
|
||||||
}
|
|
||||||
return u.writeMulti6(packets, addrs)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
|
func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
|
||||||
var rsa unix.RawSockaddrInet6
|
var rsa unix.RawSockaddrInet6
|
||||||
rsa.Family = unix.AF_INET6
|
rsa.Family = unix.AF_INET6
|
||||||
@@ -301,123 +382,6 @@ func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) writeMulti4(packets [][]byte, addrs []netip.AddrPort) (int, error) {
|
|
||||||
sent := 0
|
|
||||||
for sent < len(packets) {
|
|
||||||
// Determine batch size based on remaining packets and buffer capacity
|
|
||||||
batchSize := len(packets) - sent
|
|
||||||
if batchSize > len(u.writeMsgs) {
|
|
||||||
batchSize = len(u.writeMsgs)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use pre-allocated buffers
|
|
||||||
msgs := u.writeMsgs[:batchSize]
|
|
||||||
iovecs := u.writeIovecs[:batchSize]
|
|
||||||
names := u.writeNames[:batchSize]
|
|
||||||
|
|
||||||
// Setup message structures for this batch
|
|
||||||
for i := 0; i < batchSize; i++ {
|
|
||||||
pktIdx := sent + i
|
|
||||||
if !addrs[pktIdx].Addr().Is4() {
|
|
||||||
return sent + i, ErrInvalidIPv6RemoteForSocket
|
|
||||||
}
|
|
||||||
|
|
||||||
// Setup the packet buffer
|
|
||||||
iovecs[i].Base = &packets[pktIdx][0]
|
|
||||||
iovecs[i].Len = uint(len(packets[pktIdx]))
|
|
||||||
|
|
||||||
// Setup the destination address
|
|
||||||
rsa := (*unix.RawSockaddrInet4)(unsafe.Pointer(&names[i][0]))
|
|
||||||
rsa.Family = unix.AF_INET
|
|
||||||
rsa.Addr = addrs[pktIdx].Addr().As4()
|
|
||||||
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], addrs[pktIdx].Port())
|
|
||||||
|
|
||||||
// Set the appropriate address length for IPv4
|
|
||||||
msgs[i].Hdr.Namelen = unix.SizeofSockaddrInet4
|
|
||||||
}
|
|
||||||
|
|
||||||
// Send this batch
|
|
||||||
nsent, _, err := unix.Syscall6(
|
|
||||||
unix.SYS_SENDMMSG,
|
|
||||||
uintptr(u.sysFd),
|
|
||||||
uintptr(unsafe.Pointer(&msgs[0])),
|
|
||||||
uintptr(batchSize),
|
|
||||||
0,
|
|
||||||
0,
|
|
||||||
0,
|
|
||||||
)
|
|
||||||
|
|
||||||
if err != 0 {
|
|
||||||
return sent + int(nsent), &net.OpError{Op: "sendmmsg", Err: err}
|
|
||||||
}
|
|
||||||
|
|
||||||
sent += int(nsent)
|
|
||||||
if int(nsent) < batchSize {
|
|
||||||
// Couldn't send all packets in batch, return what we sent
|
|
||||||
return sent, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return sent, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *StdConn) writeMulti6(packets [][]byte, addrs []netip.AddrPort) (int, error) {
|
|
||||||
sent := 0
|
|
||||||
for sent < len(packets) {
|
|
||||||
// Determine batch size based on remaining packets and buffer capacity
|
|
||||||
batchSize := len(packets) - sent
|
|
||||||
if batchSize > len(u.writeMsgs) {
|
|
||||||
batchSize = len(u.writeMsgs)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use pre-allocated buffers
|
|
||||||
msgs := u.writeMsgs[:batchSize]
|
|
||||||
iovecs := u.writeIovecs[:batchSize]
|
|
||||||
names := u.writeNames[:batchSize]
|
|
||||||
|
|
||||||
// Setup message structures for this batch
|
|
||||||
for i := 0; i < batchSize; i++ {
|
|
||||||
pktIdx := sent + i
|
|
||||||
|
|
||||||
// Setup the packet buffer
|
|
||||||
iovecs[i].Base = &packets[pktIdx][0]
|
|
||||||
iovecs[i].Len = uint(len(packets[pktIdx]))
|
|
||||||
|
|
||||||
// Setup the destination address
|
|
||||||
rsa := (*unix.RawSockaddrInet6)(unsafe.Pointer(&names[i][0]))
|
|
||||||
rsa.Family = unix.AF_INET6
|
|
||||||
rsa.Addr = addrs[pktIdx].Addr().As16()
|
|
||||||
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], addrs[pktIdx].Port())
|
|
||||||
|
|
||||||
// Set the appropriate address length for IPv6
|
|
||||||
msgs[i].Hdr.Namelen = unix.SizeofSockaddrInet6
|
|
||||||
}
|
|
||||||
|
|
||||||
// Send this batch
|
|
||||||
nsent, _, err := unix.Syscall6(
|
|
||||||
unix.SYS_SENDMMSG,
|
|
||||||
uintptr(u.sysFd),
|
|
||||||
uintptr(unsafe.Pointer(&msgs[0])),
|
|
||||||
uintptr(batchSize),
|
|
||||||
0,
|
|
||||||
0,
|
|
||||||
0,
|
|
||||||
)
|
|
||||||
|
|
||||||
if err != 0 {
|
|
||||||
return sent + int(nsent), &net.OpError{Op: "sendmmsg", Err: err}
|
|
||||||
}
|
|
||||||
|
|
||||||
sent += int(nsent)
|
|
||||||
if int(nsent) < batchSize {
|
|
||||||
// Couldn't send all packets in batch, return what we sent
|
|
||||||
return sent, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return sent, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *StdConn) ReloadConfig(c *config.C) {
|
func (u *StdConn) ReloadConfig(c *config.C) {
|
||||||
b := c.GetInt("listen.read_buffer", 0)
|
b := c.GetInt("listen.read_buffer", 0)
|
||||||
if b > 0 {
|
if b > 0 {
|
||||||
@@ -475,14 +439,230 @@ func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) BatchSize() int {
|
|
||||||
return u.batch
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *StdConn) Close() error {
|
func (u *StdConn) Close() error {
|
||||||
return syscall.Close(u.sysFd)
|
return syscall.Close(u.sysFd)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (u *StdConn) WriteBatch(pkts []BatchPacket) (int, error) {
|
||||||
|
if len(pkts) == 0 {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// If GSO is supported, try to coalesce packets to the same destination
|
||||||
|
if u.gsoSupported {
|
||||||
|
return u.writeBatchGSO(pkts)
|
||||||
|
}
|
||||||
|
|
||||||
|
return u.writeBatchSendmmsg(pkts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeBatchSendmmsg sends packets using sendmmsg without GSO coalescing
|
||||||
|
func (u *StdConn) writeBatchSendmmsg(pkts []BatchPacket) (int, error) {
|
||||||
|
msgs := make([]rawMessage, len(pkts))
|
||||||
|
iovecs := make([]iovec, len(pkts))
|
||||||
|
var names4 []unix.RawSockaddrInet4
|
||||||
|
var names6 []unix.RawSockaddrInet6
|
||||||
|
|
||||||
|
if u.isV4 {
|
||||||
|
names4 = make([]unix.RawSockaddrInet4, len(pkts))
|
||||||
|
} else {
|
||||||
|
names6 = make([]unix.RawSockaddrInet6, len(pkts))
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range pkts {
|
||||||
|
setIovecBase(&iovecs[i], &pkts[i].Payload[0])
|
||||||
|
setIovecLen(&iovecs[i], len(pkts[i].Payload))
|
||||||
|
msgs[i].Hdr.Iov = &iovecs[i]
|
||||||
|
setMsghdrIovlen(&msgs[i].Hdr, 1)
|
||||||
|
|
||||||
|
if u.isV4 {
|
||||||
|
names4[i].Family = unix.AF_INET
|
||||||
|
names4[i].Addr = pkts[i].Addr.Addr().As4()
|
||||||
|
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&names4[i].Port))[:], pkts[i].Addr.Port())
|
||||||
|
msgs[i].Hdr.Name = (*byte)(unsafe.Pointer(&names4[i]))
|
||||||
|
msgs[i].Hdr.Namelen = unix.SizeofSockaddrInet4
|
||||||
|
} else {
|
||||||
|
names6[i].Family = unix.AF_INET6
|
||||||
|
names6[i].Addr = pkts[i].Addr.Addr().As16()
|
||||||
|
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&names6[i].Port))[:], pkts[i].Addr.Port())
|
||||||
|
msgs[i].Hdr.Name = (*byte)(unsafe.Pointer(&names6[i]))
|
||||||
|
msgs[i].Hdr.Namelen = unix.SizeofSockaddrInet6
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var sent int
|
||||||
|
for sent < len(msgs) {
|
||||||
|
n, _, errno := unix.Syscall6(
|
||||||
|
unix.SYS_SENDMMSG,
|
||||||
|
uintptr(u.sysFd),
|
||||||
|
uintptr(unsafe.Pointer(&msgs[sent])),
|
||||||
|
uintptr(len(msgs)-sent),
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
|
||||||
|
if errno == unix.EINTR {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if errno != 0 {
|
||||||
|
return sent, &net.OpError{Op: "sendmmsg", Err: errno}
|
||||||
|
}
|
||||||
|
|
||||||
|
sent += int(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
return sent, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeBatchGSO sends packets using GSO coalescing when possible.
|
||||||
|
// Packets to the same destination with the same size are coalesced into a single
|
||||||
|
// GSO message. Mixed destinations or sizes fall back to individual sendmmsg calls.
|
||||||
|
func (u *StdConn) writeBatchGSO(pkts []BatchPacket) (int, error) {
|
||||||
|
// Group packets by destination and try to coalesce
|
||||||
|
totalSent := 0
|
||||||
|
i := 0
|
||||||
|
|
||||||
|
for i < len(pkts) {
|
||||||
|
// Find a run of packets to the same destination with compatible sizes
|
||||||
|
startIdx := i
|
||||||
|
dst := pkts[i].Addr
|
||||||
|
segmentSize := len(pkts[i].Payload)
|
||||||
|
|
||||||
|
// Count how many packets we can coalesce (same destination, same size except possibly last)
|
||||||
|
coalescedCount := 1
|
||||||
|
totalSize := segmentSize
|
||||||
|
for i+coalescedCount < len(pkts) && coalescedCount < udpSegmentMaxDatagrams {
|
||||||
|
next := pkts[i+coalescedCount]
|
||||||
|
if next.Addr != dst {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
nextSize := len(next.Payload)
|
||||||
|
// For GSO, all packets except the last must have the same size
|
||||||
|
// The last packet can be smaller (but not larger)
|
||||||
|
if nextSize != segmentSize {
|
||||||
|
// Check if this could be the last packet (smaller is ok)
|
||||||
|
if nextSize < segmentSize && i+coalescedCount == len(pkts)-1 {
|
||||||
|
coalescedCount++
|
||||||
|
totalSize += nextSize
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
coalescedCount++
|
||||||
|
totalSize += nextSize
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we have multiple packets to coalesce, use GSO
|
||||||
|
if coalescedCount > 1 {
|
||||||
|
err := u.sendGSO(pkts[startIdx:startIdx+coalescedCount], dst, segmentSize, totalSize)
|
||||||
|
if err != nil {
|
||||||
|
// If GSO fails (e.g., EIO due to NIC not supporting checksum offload),
|
||||||
|
// disable GSO and fall back to sendmmsg for the rest
|
||||||
|
if isGSOError(err) {
|
||||||
|
u.l.WithError(err).Warn("GSO send failed, disabling GSO for this connection")
|
||||||
|
u.gsoSupported = false
|
||||||
|
// Send remaining packets with sendmmsg
|
||||||
|
remaining, rerr := u.writeBatchSendmmsg(pkts[startIdx:])
|
||||||
|
return totalSent + remaining, rerr
|
||||||
|
}
|
||||||
|
return totalSent, err
|
||||||
|
}
|
||||||
|
totalSent += coalescedCount
|
||||||
|
i += coalescedCount
|
||||||
|
} else {
|
||||||
|
// Single packet, send without GSO overhead
|
||||||
|
err := u.WriteTo(pkts[i].Payload, pkts[i].Addr)
|
||||||
|
if err != nil {
|
||||||
|
return totalSent, err
|
||||||
|
}
|
||||||
|
totalSent++
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return totalSent, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendGSO sends coalesced packets using UDP GSO
|
||||||
|
func (u *StdConn) sendGSO(pkts []BatchPacket, dst netip.AddrPort, segmentSize, totalSize int) error {
|
||||||
|
// Allocate a buffer large enough for all packet payloads
|
||||||
|
coalescedBuf := make([]byte, totalSize)
|
||||||
|
offset := 0
|
||||||
|
for _, pkt := range pkts {
|
||||||
|
copy(coalescedBuf[offset:], pkt.Payload)
|
||||||
|
offset += len(pkt.Payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prepare control message with GSO segment size
|
||||||
|
control := make([]byte, unix.CmsgSpace(2))
|
||||||
|
controlLen := setGSOSize(control, uint16(segmentSize))
|
||||||
|
|
||||||
|
// Prepare the iovec
|
||||||
|
iov := iovec{}
|
||||||
|
setIovecBase(&iov, &coalescedBuf[0])
|
||||||
|
setIovecLen(&iov, totalSize)
|
||||||
|
|
||||||
|
// Prepare the msghdr
|
||||||
|
var hdr msghdr
|
||||||
|
hdr.Iov = &iov
|
||||||
|
setMsghdrIovlen(&hdr, 1)
|
||||||
|
hdr.Control = &control[0]
|
||||||
|
setMsghdrControllen(&hdr, controlLen)
|
||||||
|
|
||||||
|
// Declare sockaddr at function scope so it remains valid for the syscall
|
||||||
|
// (must not go out of scope before the syscall is made)
|
||||||
|
var rsa4 unix.RawSockaddrInet4
|
||||||
|
var rsa6 unix.RawSockaddrInet6
|
||||||
|
|
||||||
|
// Set destination address
|
||||||
|
if u.isV4 {
|
||||||
|
rsa4.Family = unix.AF_INET
|
||||||
|
rsa4.Addr = dst.Addr().As4()
|
||||||
|
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa4.Port))[:], dst.Port())
|
||||||
|
hdr.Name = (*byte)(unsafe.Pointer(&rsa4))
|
||||||
|
hdr.Namelen = unix.SizeofSockaddrInet4
|
||||||
|
} else {
|
||||||
|
rsa6.Family = unix.AF_INET6
|
||||||
|
rsa6.Addr = dst.Addr().As16()
|
||||||
|
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa6.Port))[:], dst.Port())
|
||||||
|
hdr.Name = (*byte)(unsafe.Pointer(&rsa6))
|
||||||
|
hdr.Namelen = unix.SizeofSockaddrInet6
|
||||||
|
}
|
||||||
|
|
||||||
|
for {
|
||||||
|
_, _, errno := unix.Syscall6(
|
||||||
|
unix.SYS_SENDMSG,
|
||||||
|
uintptr(u.sysFd),
|
||||||
|
uintptr(unsafe.Pointer(&hdr)),
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
|
||||||
|
if errno == unix.EINTR {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if errno != 0 {
|
||||||
|
return &net.OpError{Op: "sendmsg", Err: errno}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// isGSOError returns true if the error indicates GSO is not supported by the NIC
|
||||||
|
func isGSOError(err error) bool {
|
||||||
|
var opErr *net.OpError
|
||||||
|
if !errors.As(err, &opErr) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// EIO typically means the NIC doesn't support checksum offload required for GSO
|
||||||
|
return errors.Is(opErr.Err, unix.EIO)
|
||||||
|
}
|
||||||
|
|
||||||
func NewUDPStatsEmitter(udpConns []Conn) func() {
|
func NewUDPStatsEmitter(udpConns []Conn) func() {
|
||||||
// Check if our kernel supports SO_MEMINFO before registering the gauges
|
// Check if our kernel supports SO_MEMINFO before registering the gauges
|
||||||
var udpGauges [][unix.SK_MEMINFO_VARS]metrics.Gauge
|
var udpGauges [][unix.SK_MEMINFO_VARS]metrics.Gauge
|
||||||
|
|||||||
+45
-5
@@ -12,7 +12,7 @@ import (
|
|||||||
|
|
||||||
type iovec struct {
|
type iovec struct {
|
||||||
Base *byte
|
Base *byte
|
||||||
Len uint
|
Len uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
type msghdr struct {
|
type msghdr struct {
|
||||||
@@ -30,17 +30,30 @@ type rawMessage struct {
|
|||||||
Len uint32
|
Len uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
|
func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte, [][]byte) {
|
||||||
msgs := make([]rawMessage, n)
|
msgs := make([]rawMessage, n)
|
||||||
buffers := make([][]byte, n)
|
buffers := make([][]byte, n)
|
||||||
names := make([][]byte, n)
|
names := make([][]byte, n)
|
||||||
|
controls := make([][]byte, n)
|
||||||
|
|
||||||
|
// Use larger buffers if GRO is enabled to hold coalesced packets
|
||||||
|
bufSize := MTU
|
||||||
|
if u.groSupported {
|
||||||
|
bufSize = groMaxPacketSize
|
||||||
|
}
|
||||||
|
|
||||||
|
// Control buffer size for receiving UDP_GRO segment size
|
||||||
|
controlSize := 0
|
||||||
|
if u.groSupported {
|
||||||
|
controlSize = unix.CmsgSpace(2) // space for uint16 segment size
|
||||||
|
}
|
||||||
|
|
||||||
for i := range msgs {
|
for i := range msgs {
|
||||||
buffers[i] = make([]byte, MTU)
|
buffers[i] = make([]byte, bufSize)
|
||||||
names[i] = make([]byte, unix.SizeofSockaddrInet6)
|
names[i] = make([]byte, unix.SizeofSockaddrInet6)
|
||||||
|
|
||||||
vs := []iovec{
|
vs := []iovec{
|
||||||
{Base: &buffers[i][0], Len: uint(len(buffers[i]))},
|
{Base: &buffers[i][0], Len: uint32(len(buffers[i]))},
|
||||||
}
|
}
|
||||||
|
|
||||||
msgs[i].Hdr.Iov = &vs[0]
|
msgs[i].Hdr.Iov = &vs[0]
|
||||||
@@ -48,7 +61,34 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
|
|||||||
|
|
||||||
msgs[i].Hdr.Name = &names[i][0]
|
msgs[i].Hdr.Name = &names[i][0]
|
||||||
msgs[i].Hdr.Namelen = uint32(len(names[i]))
|
msgs[i].Hdr.Namelen = uint32(len(names[i]))
|
||||||
|
|
||||||
|
// Set up control message buffer for GRO
|
||||||
|
if controlSize > 0 {
|
||||||
|
controls[i] = make([]byte, controlSize)
|
||||||
|
msgs[i].Hdr.Control = &controls[i][0]
|
||||||
|
msgs[i].Hdr.Controllen = uint32(controlSize)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return msgs, buffers, names
|
return msgs, buffers, names, controls
|
||||||
|
}
|
||||||
|
|
||||||
|
func setIovecBase(iov *iovec, base *byte) {
|
||||||
|
iov.Base = base
|
||||||
|
}
|
||||||
|
|
||||||
|
func setIovecLen(iov *iovec, l int) {
|
||||||
|
iov.Len = uint32(l)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setMsghdrIovlen(hdr *msghdr, l int) {
|
||||||
|
hdr.Iovlen = uint32(l)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setMsghdrControllen(hdr *msghdr, l int) {
|
||||||
|
hdr.Controllen = uint32(l)
|
||||||
|
}
|
||||||
|
|
||||||
|
func getMsghdrControllen(hdr *msghdr) int {
|
||||||
|
return int(hdr.Controllen)
|
||||||
}
|
}
|
||||||
|
|||||||
+45
-5
@@ -12,7 +12,7 @@ import (
|
|||||||
|
|
||||||
type iovec struct {
|
type iovec struct {
|
||||||
Base *byte
|
Base *byte
|
||||||
Len uint
|
Len uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
type msghdr struct {
|
type msghdr struct {
|
||||||
@@ -33,17 +33,30 @@ type rawMessage struct {
|
|||||||
Pad0 [4]byte
|
Pad0 [4]byte
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
|
func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte, [][]byte) {
|
||||||
msgs := make([]rawMessage, n)
|
msgs := make([]rawMessage, n)
|
||||||
buffers := make([][]byte, n)
|
buffers := make([][]byte, n)
|
||||||
names := make([][]byte, n)
|
names := make([][]byte, n)
|
||||||
|
controls := make([][]byte, n)
|
||||||
|
|
||||||
|
// Use larger buffers if GRO is enabled to hold coalesced packets
|
||||||
|
bufSize := MTU
|
||||||
|
if u.groSupported {
|
||||||
|
bufSize = groMaxPacketSize
|
||||||
|
}
|
||||||
|
|
||||||
|
// Control buffer size for receiving UDP_GRO segment size
|
||||||
|
controlSize := 0
|
||||||
|
if u.groSupported {
|
||||||
|
controlSize = unix.CmsgSpace(2) // space for uint16 segment size
|
||||||
|
}
|
||||||
|
|
||||||
for i := range msgs {
|
for i := range msgs {
|
||||||
buffers[i] = make([]byte, MTU)
|
buffers[i] = make([]byte, bufSize)
|
||||||
names[i] = make([]byte, unix.SizeofSockaddrInet6)
|
names[i] = make([]byte, unix.SizeofSockaddrInet6)
|
||||||
|
|
||||||
vs := []iovec{
|
vs := []iovec{
|
||||||
{Base: &buffers[i][0], Len: uint(len(buffers[i]))},
|
{Base: &buffers[i][0], Len: uint64(len(buffers[i]))},
|
||||||
}
|
}
|
||||||
|
|
||||||
msgs[i].Hdr.Iov = &vs[0]
|
msgs[i].Hdr.Iov = &vs[0]
|
||||||
@@ -51,7 +64,34 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
|
|||||||
|
|
||||||
msgs[i].Hdr.Name = &names[i][0]
|
msgs[i].Hdr.Name = &names[i][0]
|
||||||
msgs[i].Hdr.Namelen = uint32(len(names[i]))
|
msgs[i].Hdr.Namelen = uint32(len(names[i]))
|
||||||
|
|
||||||
|
// Set up control message buffer for GRO
|
||||||
|
if controlSize > 0 {
|
||||||
|
controls[i] = make([]byte, controlSize)
|
||||||
|
msgs[i].Hdr.Control = &controls[i][0]
|
||||||
|
msgs[i].Hdr.Controllen = uint64(controlSize)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return msgs, buffers, names
|
return msgs, buffers, names, controls
|
||||||
|
}
|
||||||
|
|
||||||
|
func setIovecBase(iov *iovec, base *byte) {
|
||||||
|
iov.Base = base
|
||||||
|
}
|
||||||
|
|
||||||
|
func setIovecLen(iov *iovec, l int) {
|
||||||
|
iov.Len = uint64(l)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setMsghdrIovlen(hdr *msghdr, l int) {
|
||||||
|
hdr.Iovlen = uint64(l)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setMsghdrControllen(hdr *msghdr, l int) {
|
||||||
|
hdr.Controllen = uint64(l)
|
||||||
|
}
|
||||||
|
|
||||||
|
func getMsghdrControllen(hdr *msghdr) int {
|
||||||
|
return int(hdr.Controllen)
|
||||||
}
|
}
|
||||||
|
|||||||
+12
-39
@@ -332,54 +332,27 @@ func (u *RIOConn) SupportsMultipleReaders() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (u *RIOConn) SupportsGSO() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *RIOConn) SupportsGRO() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func (u *RIOConn) Rebind() error {
|
func (u *RIOConn) Rebind() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *RIOConn) ReloadConfig(*config.C) {}
|
func (u *RIOConn) ReloadConfig(*config.C) {}
|
||||||
|
|
||||||
// BatchSize returns 1 since RIO reads packets one at a time
|
func (u *RIOConn) WriteBatch(pkts []BatchPacket) (int, error) {
|
||||||
func (u *RIOConn) BatchSize() int {
|
for i := range pkts {
|
||||||
return 1
|
if err := u.WriteTo(pkts[i].Payload, pkts[i].Addr); err != nil {
|
||||||
}
|
|
||||||
|
|
||||||
// ListenOutBatch - fallback to single-packet reads for RIO
|
|
||||||
func (u *RIOConn) ListenOutBatch(r EncBatchReader) {
|
|
||||||
buffer := make([]byte, MTU)
|
|
||||||
addrs := make([]netip.AddrPort, 1)
|
|
||||||
payloads := make([][]byte, 1)
|
|
||||||
|
|
||||||
var lastRecvErr time.Time
|
|
||||||
|
|
||||||
for {
|
|
||||||
n, rua, err := u.receive(buffer)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, net.ErrClosed) {
|
|
||||||
u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute {
|
|
||||||
lastRecvErr = time.Now()
|
|
||||||
u.l.WithError(err).Warn("unexpected udp socket receive error")
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
addrs[0] = netip.AddrPortFrom(netip.AddrFrom16(rua.Addr).Unmap(), (rua.Port>>8)|((rua.Port&0xff)<<8))
|
|
||||||
payloads[0] = buffer[:n]
|
|
||||||
r(addrs, payloads, 1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteMulti sends multiple packets - fallback implementation
|
|
||||||
func (u *RIOConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
|
|
||||||
for i := range packets {
|
|
||||||
err := u.WriteTo(packets[i], addrs[i])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
return i, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return len(packets), nil
|
return len(pkts), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *RIOConn) Close() error {
|
func (u *RIOConn) Close() error {
|
||||||
|
|||||||
+15
-27
@@ -116,31 +116,6 @@ func (u *TesterConn) ListenOut(r EncReader) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *TesterConn) ListenOutBatch(r EncBatchReader) {
|
|
||||||
addrs := make([]netip.AddrPort, 1)
|
|
||||||
payloads := make([][]byte, 1)
|
|
||||||
|
|
||||||
for {
|
|
||||||
p, ok := <-u.RxPackets
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
addrs[0] = p.From
|
|
||||||
payloads[0] = p.Data
|
|
||||||
r(addrs, payloads, 1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *TesterConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
|
|
||||||
for i := range packets {
|
|
||||||
err := u.WriteTo(packets[i], addrs[i])
|
|
||||||
if err != nil {
|
|
||||||
return i, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(packets), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *TesterConn) ReloadConfig(*config.C) {}
|
func (u *TesterConn) ReloadConfig(*config.C) {}
|
||||||
|
|
||||||
func NewUDPStatsEmitter(_ []Conn) func() {
|
func NewUDPStatsEmitter(_ []Conn) func() {
|
||||||
@@ -156,8 +131,12 @@ func (u *TesterConn) SupportsMultipleReaders() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *TesterConn) BatchSize() int {
|
func (u *TesterConn) SupportsGSO() bool {
|
||||||
return 1
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *TesterConn) SupportsGRO() bool {
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *TesterConn) Rebind() error {
|
func (u *TesterConn) Rebind() error {
|
||||||
@@ -171,3 +150,12 @@ func (u *TesterConn) Close() error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (u *TesterConn) WriteBatch(pkts []BatchPacket) (int, error) {
|
||||||
|
for i := range pkts {
|
||||||
|
if err := u.WriteTo(pkts[i].Payload, pkts[i].Addr); err != nil {
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(pkts), nil
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user