Compare commits

...

7 Commits

Author SHA1 Message Date
Jay Wren ef1739bec4 claude does TUN virtio header support 2026-02-04 13:13:41 -05:00
Jay Wren 030b7e2763 claude implements UDP GRO 2026-02-04 11:02:06 -05:00
Jay Wren 6b6a4bc1cc claude implements UDP GSO 2026-02-04 10:29:37 -05:00
Jay Wren 30db76ed79 batch tun reads 2026-02-03 17:12:44 -05:00
Jay Wren 15333f9fed batch udp packet sending 2026-02-03 16:56:21 -05:00
Jack Doan 42bee7cf17 Report if Nebula start fails because of tun device name (#1588)
gofmt / Run gofmt (push) Failing after 2s
smoke-extra / Run extra smoke tests (push) Failing after 2s
smoke / Run multi node smoke test (push) Failing after 2s
Build and test / Build all and test on ubuntu-linux (push) Failing after 2s
Build and test / Build and test on linux with boringcrypto (push) Failing after 2s
Build and test / Build and test on linux with pkcs11 (push) Failing after 2s
Build and test / Build and test on macos-latest (push) Has been cancelled
Build and test / Build and test on windows-latest (push) Has been cancelled
* specifically report if nebula start fails because of tun device name

* close all routines when closing the tun
2026-01-28 10:03:36 -06:00
Caleb Jasik 02d8bcac68 Remove lighthouse goroutine leaks in lighthouse_test.go (#1589)
gofmt / Run gofmt (push) Failing after 3s
smoke-extra / Run extra smoke tests (push) Failing after 2s
smoke / Run multi node smoke test (push) Failing after 2s
Build and test / Build all and test on ubuntu-linux (push) Failing after 3s
Build and test / Build and test on linux with boringcrypto (push) Failing after 2s
Build and test / Build and test on linux with pkcs11 (push) Failing after 2s
Build and test / Build and test on macos-latest (push) Has been cancelled
Build and test / Build and test on windows-latest (push) Has been cancelled
Using <https://go.dev/doc/go1.26#goroutineleak-profiles> + Claude, I was able to run nebula's unit tests and e2e tests with the leak detector enabled.

Added a TestMain that queries pprof to see if there are any reported goroutine leaks.
I'd love to get some form of this in CI whenever go 1.26 comes out, though I'd also like to prove this is properly useful past the just five detections it got here.

<details>
<summary>TestMain</summary>


```go
package nebula

import (
    "fmt"
    "os"
    "runtime/pprof"
    "strings"
    "testing"
)

// TestMain runs after all tests and checks for goroutine leaks
func TestMain(m *testing.M) {
    // Run all tests
    exitCode := m.Run()

    // Check for goroutine leaks after all tests complete
    prof := pprof.Lookup("goroutineleak")
    if prof != nil {
        var sb strings.Builder
        if err := prof.WriteTo(&sb, 2); err != nil {
            fmt.Fprintf(os.Stderr, "Failed to write goroutineleak profile: %v\n", err)
            os.Exit(1)
        }

        content := sb.String()
        leakedCount := strings.Count(content, "(leaked)")

        if leakedCount > 0 {
            fmt.Fprintf(os.Stderr, "\n=== GOROUTINE LEAK DETECTED ===\n")
            fmt.Fprintf(os.Stderr, "Found %d leaked goroutine(s) in package nebula\n\n", leakedCount)

            goros := strings.Split(content, "\n\n")
            for _, goro := range goros {
                if strings.Contains(goro, "(leaked)") {
                    fmt.Fprintln(os.Stderr, goro)
                    fmt.Fprintln(os.Stderr)
                }
            }
            os.Exit(1)
        } else {
            fmt.Println("✓ No goroutine leaks detected in package nebula")
        }
    }

    os.Exit(exitCode)
}
```

</details>

Also had to install go1.26rc2 and update the makefile to use that go binary + set ex:

```makefile
test-goroutineleak:
	GOEXPERIMENT=goroutineleakprofile go1.26rc2 test -v ./...
```
2026-01-27 23:44:43 -06:00
17 changed files with 1211 additions and 35 deletions
+139 -1
View File
@@ -9,8 +9,75 @@ import (
"github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/noiseutil" "github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/udp"
) )
// consumeInsidePacketBatched is a variant of consumeInsidePacket that queues
// outgoing packets into pendingPackets instead of sending them immediately.
// The caller is responsible for flushing pendingPackets with WriteBatch.
func (f *Interface) consumeInsidePacketBatched(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache, pendingPackets *[]udp.BatchPacket) {
err := newPacket(packet, false, fwPacket)
if err != nil {
if f.l.Level >= logrus.DebugLevel {
f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
}
return
}
// Ignore local broadcast packets
if f.dropLocalBroadcast {
if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
return
}
}
if f.myVpnAddrsTable.Contains(fwPacket.RemoteAddr) {
if immediatelyForwardToSelf {
_, err := f.readers[q].Write(packet)
if err != nil {
f.l.WithError(err).Error("Failed to forward to tun")
}
}
return
}
// Ignore multicast packets
if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() {
return
}
hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
})
if hostinfo == nil {
f.rejectInside(packet, out, q)
if f.l.Level >= logrus.DebugLevel {
f.l.WithField("vpnAddr", fwPacket.RemoteAddr).
WithField("fwPacket", fwPacket).
Debugln("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks")
}
return
}
if !ready {
return
}
dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
if dropReason == nil {
f.sendNoMetricsBatched(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q, pendingPackets)
} else {
f.rejectInside(packet, out, q)
if f.l.Level >= logrus.DebugLevel {
hostinfo.logger(f.l).
WithField("fwPacket", fwPacket).
WithField("reason", dropReason).
Debugln("dropping outbound packet")
}
}
}
func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) { func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
err := newPacket(packet, false, fwPacket) err := newPacket(packet, false, fwPacket)
if err != nil { if err != nil {
@@ -69,7 +136,6 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache) dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
if dropReason == nil { if dropReason == nil {
f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q) f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
} else { } else {
f.rejectInside(packet, out, q) f.rejectInside(packet, out, q)
if f.l.Level >= logrus.DebugLevel { if f.l.Level >= logrus.DebugLevel {
@@ -410,3 +476,75 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
} }
} }
} }
// sendNoMetricsBatched is like sendNoMetrics but queues the packet for batched sending
// instead of sending immediately. The caller must flush pendingPackets with WriteBatch.
func (f *Interface) sendNoMetricsBatched(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, q int, pendingPackets *[]udp.BatchPacket) {
if ci.eKey == nil {
return
}
useRelay := !remote.IsValid() && !hostinfo.remote.IsValid()
fullOut := out
if useRelay {
if len(out) < header.Len {
out = out[:header.Len]
}
out = out[header.Len:]
}
if noiseutil.EncryptLockNeeded {
ci.writeLock.Lock()
}
c := ci.messageCounter.Add(1)
out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
f.connectionManager.Out(hostinfo)
if t != header.CloseTunnel && hostinfo.lastRebindCount != f.rebindCount {
f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
hostinfo.lastRebindCount = f.rebindCount
if f.l.Level >= logrus.DebugLevel {
f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).Debug("Lighthouse update triggered for punch due to rebind counter")
}
}
var err error
out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
if noiseutil.EncryptLockNeeded {
ci.writeLock.Unlock()
}
if err != nil {
hostinfo.logger(f.l).WithError(err).
WithField("udpAddr", remote).WithField("counter", c).
WithField("attemptedCounter", c).
Error("Failed to encrypt outgoing packet")
return
}
// Queue the packet for batched sending
var addr netip.AddrPort
if remote.IsValid() {
addr = remote
} else if hostinfo.remote.IsValid() {
addr = hostinfo.remote
} else {
// Relay path - send immediately, not batched
for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
if err != nil {
hostinfo.relayState.DeleteRelay(relayIP)
hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetricsBatched failed to find HostInfo")
continue
}
f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
break
}
return
}
// Copy the payload since the buffer will be reused
payload := make([]byte, len(out))
copy(payload, out)
*pendingPackets = append(*pendingPackets, udp.BatchPacket{Payload: payload, Addr: addr})
}
+81 -4
View File
@@ -48,6 +48,8 @@ type InterfaceConfig struct {
ConntrackCacheTimeout time.Duration ConntrackCacheTimeout time.Duration
l *logrus.Logger l *logrus.Logger
tunBatchSize int // batch size for TUN read/write batching, 0 to disable
} }
type Interface struct { type Interface struct {
@@ -86,8 +88,9 @@ type Interface struct {
conntrackCacheTimeout time.Duration conntrackCacheTimeout time.Duration
writers []udp.Conn writers []udp.Conn
readers []io.ReadWriteCloser readers []io.ReadWriteCloser
tunBatchSize int // batch size for TUN read/write batching
metricHandshakes metrics.Histogram metricHandshakes metrics.Histogram
messageMetrics *MessageMetrics messageMetrics *MessageMetrics
@@ -187,6 +190,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
relayManager: c.relayManager, relayManager: c.relayManager,
connectionManager: c.connectionManager, connectionManager: c.connectionManager,
conntrackCacheTimeout: c.ConntrackCacheTimeout, conntrackCacheTimeout: c.ConntrackCacheTimeout,
tunBatchSize: c.tunBatchSize,
metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)), metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
messageMetrics: c.MessageMetrics, messageMetrics: c.MessageMetrics,
@@ -244,6 +248,15 @@ func (f *Interface) activate() {
f.readers[i] = reader f.readers[i] = reader
} }
// Enable batch reading on all readers if batch size > 1
if f.tunBatchSize > 1 {
for i := 0; i < f.routines; i++ {
if err := overlay.EnableBatchReading(f.readers[i]); err != nil {
f.l.WithError(err).WithField("routine", i).Warn("Failed to enable batch reading, falling back to single reads")
}
}
}
if err := f.inside.Activate(); err != nil { if err := f.inside.Activate(); err != nil {
f.inside.Close() f.inside.Close()
f.l.Fatal(err) f.l.Fatal(err)
@@ -287,13 +300,21 @@ func (f *Interface) listenOut(i int) {
func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) { func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
runtime.LockOSThread() runtime.LockOSThread()
conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
// Check if batch reading is available and enabled
batchReader := overlay.AsBatchReader(reader)
if batchReader != nil && f.tunBatchSize > 1 {
f.listenInBatched(reader, batchReader, i, conntrackCache)
return
}
// Fallback to single-packet reading
packet := make([]byte, mtu) packet := make([]byte, mtu)
out := make([]byte, mtu) out := make([]byte, mtu)
fwPacket := &firewall.Packet{} fwPacket := &firewall.Packet{}
nb := make([]byte, 12, 12) nb := make([]byte, 12, 12)
conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
for { for {
n, err := reader.Read(packet) n, err := reader.Read(packet)
if err != nil { if err != nil {
@@ -310,6 +331,54 @@ func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
} }
} }
func (f *Interface) listenInBatched(reader io.ReadWriteCloser, batchReader overlay.BatchReader, i int, conntrackCache *firewall.ConntrackCacheTicker) {
batchSize := f.tunBatchSize
// Pre-allocate buffers for batch reading
packets := make([][]byte, batchSize)
for j := range packets {
packets[j] = make([]byte, mtu)
}
sizes := make([]int, batchSize)
// Pre-allocate buffers for packet processing
out := make([]byte, mtu)
fwPacket := &firewall.Packet{}
nb := make([]byte, 12, 12)
// Pre-allocate buffer for batched UDP writes
pendingPackets := make([]udp.BatchPacket, 0, batchSize)
for {
// Read a batch of packets from TUN
n, err := batchReader.ReadBatch(packets, sizes)
if err != nil {
if errors.Is(err, os.ErrClosed) && f.closed.Load() {
return
}
f.l.WithError(err).Error("Error while reading outbound packets")
os.Exit(2)
}
if n == 0 {
continue
}
// Process all packets in the batch
cache := conntrackCache.Get(f.l)
for j := 0; j < n; j++ {
f.consumeInsidePacketBatched(packets[j][:sizes[j]], fwPacket, nb, out, i, cache, &pendingPackets)
}
// Flush all pending UDP writes
if len(pendingPackets) > 0 {
f.writers[i].WriteBatch(pendingPackets)
pendingPackets = pendingPackets[:0]
}
}
}
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) { func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
c.RegisterReloadCallback(f.reloadFirewall) c.RegisterReloadCallback(f.reloadFirewall)
c.RegisterReloadCallback(f.reloadSendRecvError) c.RegisterReloadCallback(f.reloadSendRecvError)
@@ -490,6 +559,14 @@ func (f *Interface) Close() error {
f.l.WithError(err).Error("Error while closing udp socket") f.l.WithError(err).Error("Error while closing udp socket")
} }
} }
for i, r := range f.readers {
if i == 0 {
continue // f.readers[0] is f.inside, which we want to save for last
}
if err := r.Close(); err != nil {
f.l.WithError(err).Error("Error while closing tun reader")
}
}
// Release the tun device // Release the tun device
return f.inside.Close() return f.inside.Close()
+8 -9
View File
@@ -1,7 +1,6 @@
package nebula package nebula
import ( import (
"context"
"encoding/binary" "encoding/binary"
"fmt" "fmt"
"net/netip" "net/netip"
@@ -42,14 +41,14 @@ func Test_lhStaticMapping(t *testing.T) {
c := config.NewC(l) c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}} c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}}
c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}} c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
_, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil) _, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err) require.NoError(t, err)
lh2 := "10.128.0.3" lh2 := "10.128.0.3"
c = config.NewC(l) c = config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1, lh2}} c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1, lh2}}
c.Settings["static_host_map"] = map[string]any{lh1: []any{"100.1.1.1:4242"}} c.Settings["static_host_map"] = map[string]any{lh1: []any{"100.1.1.1:4242"}}
_, err = NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil) _, err = NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.EqualError(t, err, "lighthouse 10.128.0.3 does not have a static_host_map entry") require.EqualError(t, err, "lighthouse 10.128.0.3 does not have a static_host_map entry")
} }
@@ -71,7 +70,7 @@ func TestReloadLighthouseInterval(t *testing.T) {
} }
c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}} c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil) lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err) require.NoError(t, err)
lh.ifce = &mockEncWriter{} lh.ifce = &mockEncWriter{}
@@ -99,7 +98,7 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) {
} }
c := config.NewC(l) c := config.NewC(l)
lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil) lh, err := NewLightHouseFromConfig(b.Context(), l, c, cs, nil, nil)
require.NoError(b, err) require.NoError(b, err)
hAddr := netip.MustParseAddrPort("4.5.6.7:12345") hAddr := netip.MustParseAddrPort("4.5.6.7:12345")
@@ -202,7 +201,7 @@ func TestLighthouse_Memory(t *testing.T) {
myVpnNetworks: []netip.Prefix{myVpnNet}, myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt, myVpnNetworksTable: nt,
} }
lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil) lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
lh.ifce = &mockEncWriter{} lh.ifce = &mockEncWriter{}
require.NoError(t, err) require.NoError(t, err)
lhh := lh.NewRequestHandler() lhh := lh.NewRequestHandler()
@@ -288,7 +287,7 @@ func TestLighthouse_reload(t *testing.T) {
myVpnNetworksTable: nt, myVpnNetworksTable: nt,
} }
lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil) lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err) require.NoError(t, err)
nc := map[string]any{ nc := map[string]any{
@@ -523,7 +522,7 @@ func TestLighthouse_Dont_Delete_Static_Hosts(t *testing.T) {
myVpnNetworks: []netip.Prefix{myVpnNet}, myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt, myVpnNetworksTable: nt,
} }
lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil) lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err) require.NoError(t, err)
lh.ifce = &mockEncWriter{} lh.ifce = &mockEncWriter{}
@@ -589,7 +588,7 @@ func TestLighthouse_DeletesWork(t *testing.T) {
myVpnNetworks: []netip.Prefix{myVpnNet}, myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt, myVpnNetworksTable: nt,
} }
lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil) lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err) require.NoError(t, err)
lh.ifce = &mockEncWriter{} lh.ifce = &mockEncWriter{}
+1
View File
@@ -250,6 +250,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
punchy: punchy, punchy: punchy,
ConntrackCacheTimeout: conntrackCacheTimeout, ConntrackCacheTimeout: conntrackCacheTimeout,
l: l, l: l,
tunBatchSize: c.GetInt("listen.batch", 64),
} }
var ifce *Interface var ifce *Interface
+35
View File
@@ -16,3 +16,38 @@ type Device interface {
SupportsMultiqueue() bool SupportsMultiqueue() bool
NewMultiQueueReader() (io.ReadWriteCloser, error) NewMultiQueueReader() (io.ReadWriteCloser, error)
} }
// BatchReader is an optional interface that devices can implement
// to support reading multiple packets in a single batch operation.
// This can significantly reduce syscall overhead under high load.
type BatchReader interface {
// ReadBatch reads up to len(packets) packets into the provided buffers.
// Each packet is read into packets[i] and its length is stored in sizes[i].
// Returns the number of packets read, or an error.
// A return of (0, nil) indicates no packets were available (non-blocking).
ReadBatch(packets [][]byte, sizes []int) (int, error)
}
// AsBatchReader returns a BatchReader if the reader supports batch operations,
// otherwise returns nil.
func AsBatchReader(r io.ReadWriteCloser) BatchReader {
if br, ok := r.(BatchReader); ok {
return br
}
return nil
}
// BatchEnabler is an optional interface for devices that need explicit
// enabling of batch read support (e.g., setting non-blocking mode).
type BatchEnabler interface {
EnableBatchReading() error
}
// EnableBatchReading enables batch reading on the device if supported.
// Returns nil if the device doesn't support or need explicit enabling.
func EnableBatchReading(d interface{}) error {
if be, ok := d.(BatchEnabler); ok {
return be.EnableBatchReading()
}
return nil
}
+9
View File
@@ -12,6 +12,15 @@ import (
const DefaultMTU = 1300 const DefaultMTU = 1300
type NameError struct {
Name string
Underlying error
}
func (e *NameError) Error() string {
return fmt.Sprintf("could not set tun device name: %s because %s", e.Name, e.Underlying)
}
// TODO: We may be able to remove routines // TODO: We may be able to remove routines
type DeviceFactory func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) type DeviceFactory func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error)
+1 -1
View File
@@ -266,7 +266,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
} }
// Set the device name // Set the device name
ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr))) _ = ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
} }
t := &tun{ t := &tun{
+403 -5
View File
@@ -24,6 +24,11 @@ import (
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
const (
// virtioNetHdrLen is the length of virtio_net_hdr (without mergeable buffers)
virtioNetHdrLen = 10
)
type tun struct { type tun struct {
io.ReadWriteCloser io.ReadWriteCloser
fd int fd int
@@ -34,6 +39,13 @@ type tun struct {
TXQueueLen int TXQueueLen int
deviceIndex int deviceIndex int
ioctlFd uintptr ioctlFd uintptr
nonBlocking bool // true if fd is in non-blocking mode
vnetHdr bool // true if IFF_VNET_HDR is enabled on the TUN device
// readBuf is used when vnetHdr is enabled to read the full packet+header
// before stripping the header. This is needed because caller-provided
// buffers are sized for MTU but kernel writes MTU+10 with virtio header.
readBuf []byte
Routes atomic.Pointer[[]Route] Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]] routeTree atomic.Pointer[bart.Table[routing.Gateways]]
@@ -53,6 +65,23 @@ func (t *tun) Networks() []netip.Prefix {
return t.vpnNetworks return t.vpnNetworks
} }
// tunVnetHdrSupported checks if the kernel supports IFF_VNET_HDR on TUN devices
func tunVnetHdrSupported() bool {
fd, err := unix.Open("/dev/net/tun", unix.O_RDONLY, 0)
if err != nil {
return false
}
defer unix.Close(fd)
var features uint32
err = ioctl(uintptr(fd), uintptr(unix.TUNGETFEATURES), uintptr(unsafe.Pointer(&features)))
if err != nil {
return false
}
return features&unix.IFF_VNET_HDR != 0
}
type ifReq struct { type ifReq struct {
Name [16]byte Name [16]byte
Flags uint16 Flags uint16
@@ -107,17 +136,35 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueu
} }
} }
// Check if VNET_HDR is supported before trying to use it
useVnetHdr := tunVnetHdrSupported()
var req ifReq var req ifReq
req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI) req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
if multiqueue { if multiqueue {
req.Flags |= unix.IFF_MULTI_QUEUE req.Flags |= unix.IFF_MULTI_QUEUE
} }
copy(req.Name[:], c.GetString("tun.dev", "")) if useVnetHdr {
req.Flags |= unix.IFF_VNET_HDR
}
nameStr := c.GetString("tun.dev", "")
copy(req.Name[:], nameStr)
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil { if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
return nil, err return nil, &NameError{
Name: nameStr,
Underlying: err,
}
} }
name := strings.Trim(string(req.Name[:]), "\x00") name := strings.Trim(string(req.Name[:]), "\x00")
// Track if VNET_HDR is in use
// Note: We don't call TUNSETOFFLOAD - just handle the headers manually
vnetHdrEnabled := useVnetHdr
if vnetHdrEnabled {
l.Info("TUN VNET_HDR enabled")
}
file := os.NewFile(uintptr(fd), "/dev/net/tun") file := os.NewFile(uintptr(fd), "/dev/net/tun")
t, err := newTunGeneric(c, l, file, vpnNetworks) t, err := newTunGeneric(c, l, file, vpnNetworks)
if err != nil { if err != nil {
@@ -125,6 +172,13 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueu
} }
t.Device = name t.Device = name
t.vnetHdr = vnetHdrEnabled
// Allocate read buffer for virtio header handling
// Buffer needs to be large enough for virtio header + max packet
if t.vnetHdr {
t.readBuf = make([]byte, t.MaxMTU+virtioNetHdrLen)
}
return t, nil return t, nil
} }
@@ -235,21 +289,172 @@ func (t *tun) SupportsMultiqueue() bool {
} }
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0) fd, err := unix.Open("/dev/net/tun", os.O_RDWR|unix.O_NONBLOCK, 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var req ifReq var req ifReq
req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE) req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
if t.vnetHdr {
req.Flags |= unix.IFF_VNET_HDR
}
copy(req.Name[:], t.Device) copy(req.Name[:], t.Device)
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil { if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
return nil, err return nil, err
} }
file := os.NewFile(uintptr(fd), "/dev/net/tun") reader := &tunBatchReader{fd: fd, device: t.Device, vnetHdr: t.vnetHdr}
if t.vnetHdr {
reader.readBuf = make([]byte, t.MaxMTU+virtioNetHdrLen)
}
return reader, nil
}
return file, nil // tunBatchReader implements BatchReader for efficient batch packet reading
type tunBatchReader struct {
fd int
device string
vnetHdr bool
readBuf []byte // internal buffer for virtio header handling
}
func (r *tunBatchReader) Read(b []byte) (int, error) {
// Choose buffer: use internal buffer for vnetHdr, caller's buffer otherwise
readBuf := b
if r.vnetHdr {
readBuf = r.readBuf
}
// Use poll to wait for data, then read
for {
n, err := unix.Read(r.fd, readBuf)
if err == nil {
if r.vnetHdr && n > virtioNetHdrLen {
packetLen := n - virtioNetHdrLen
copy(b, readBuf[virtioNetHdrLen:n])
return packetLen, nil
}
if r.vnetHdr {
return 0, nil // No packet data
}
return n, nil
}
if err == unix.EAGAIN || err == unix.EWOULDBLOCK {
// Wait for data
pfds := []unix.PollFd{{Fd: int32(r.fd), Events: unix.POLLIN}}
_, err = unix.Poll(pfds, -1)
if err != nil {
if err == unix.EINTR {
continue
}
return 0, err
}
continue
}
return n, err
}
}
func (r *tunBatchReader) Write(b []byte) (int, error) {
if !r.vnetHdr {
return unix.Write(r.fd, b)
}
// Use writev to prepend virtio header without copying the packet data
// Header is all zeros = no GSO, no checksum offload
var hdr [virtioNetHdrLen]byte
bufs := [][]byte{hdr[:], b}
n, err := unix.Writev(r.fd, bufs)
if err != nil {
return 0, err
}
// Return only the packet bytes written (exclude header)
if n > virtioNetHdrLen {
return n - virtioNetHdrLen, nil
}
return 0, nil
}
func (r *tunBatchReader) Close() error {
return unix.Close(r.fd)
}
// ReadBatch reads up to len(packets) packets from the TUN device.
// It drains all available packets without blocking, using poll() only
// when no packets have been read yet.
func (r *tunBatchReader) ReadBatch(packets [][]byte, sizes []int) (int, error) {
count := 0
maxPackets := len(packets)
if len(sizes) < maxPackets {
maxPackets = len(sizes)
}
// Choose read buffer based on vnetHdr
readBuf := packets[0] // Will be updated in loop for non-vnetHdr
if r.vnetHdr {
readBuf = r.readBuf
}
for count < maxPackets {
if !r.vnetHdr {
readBuf = packets[count]
}
n, err := unix.Read(r.fd, readBuf)
if err == nil && n > 0 {
if r.vnetHdr {
if n > virtioNetHdrLen {
packetLen := n - virtioNetHdrLen
copy(packets[count], readBuf[virtioNetHdrLen:n])
sizes[count] = packetLen
} else {
// Malformed packet (no data after header), skip
continue
}
} else {
sizes[count] = n
}
count++
continue
}
if err == unix.EAGAIN || err == unix.EWOULDBLOCK {
// No more packets available
if count > 0 {
// We have some packets, return them
return count, nil
}
// No packets yet, wait for at least one
pfds := []unix.PollFd{{Fd: int32(r.fd), Events: unix.POLLIN}}
_, err = unix.Poll(pfds, -1)
if err != nil {
if err == unix.EINTR {
continue
}
return 0, err
}
continue
}
if err != nil {
if count > 0 {
// Return what we have
return count, nil
}
return 0, err
}
if n == 0 {
if count > 0 {
return count, nil
}
return 0, io.EOF
}
}
return count, nil
} }
func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
@@ -258,6 +463,27 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
} }
func (t *tun) Write(b []byte) (int, error) { func (t *tun) Write(b []byte) (int, error) {
if !t.vnetHdr {
return t.writeSimple(b)
}
// Use writev to prepend virtio header without copying the packet data
// Header is all zeros = no GSO, no checksum offload
var hdr [virtioNetHdrLen]byte
bufs := [][]byte{hdr[:], b}
n, err := unix.Writev(t.fd, bufs)
if err != nil {
return 0, err
}
// Return only the packet bytes written (exclude header)
if n > virtioNetHdrLen {
return n - virtioNetHdrLen, nil
}
return 0, nil
}
func (t *tun) writeSimple(b []byte) (int, error) {
var nn int var nn int
maximum := len(b) maximum := len(b)
@@ -280,6 +506,177 @@ func (t *tun) Write(b []byte) (int, error) {
} }
} }
// EnableBatchReading sets the TUN fd to non-blocking mode to enable batch reading.
// This should be called before using ReadBatch.
func (t *tun) EnableBatchReading() error {
if t.nonBlocking {
return nil
}
err := unix.SetNonblock(t.fd, true)
if err != nil {
return err
}
t.nonBlocking = true
return nil
}
// Read overrides the default Read to handle non-blocking mode and virtio headers
func (t *tun) Read(b []byte) (int, error) {
if !t.vnetHdr {
return t.readSimple(b)
}
// With VNET_HDR, read into internal buffer (which has space for header)
// then copy packet data to caller's buffer
if !t.nonBlocking {
n, err := t.ReadWriteCloser.Read(t.readBuf)
if err != nil {
return 0, err
}
if n <= virtioNetHdrLen {
return 0, nil // No packet data
}
packetLen := n - virtioNetHdrLen
copy(b, t.readBuf[virtioNetHdrLen:n])
return packetLen, nil
}
// Non-blocking read with poll
for {
n, err := unix.Read(t.fd, t.readBuf)
if err == nil {
if n <= virtioNetHdrLen {
return 0, nil // No packet data
}
packetLen := n - virtioNetHdrLen
copy(b, t.readBuf[virtioNetHdrLen:n])
return packetLen, nil
}
if err == unix.EAGAIN || err == unix.EWOULDBLOCK {
pfds := []unix.PollFd{{Fd: int32(t.fd), Events: unix.POLLIN}}
_, err = unix.Poll(pfds, -1)
if err != nil {
if err == unix.EINTR {
continue
}
return 0, err
}
continue
}
return n, err
}
}
func (t *tun) readSimple(b []byte) (int, error) {
if !t.nonBlocking {
return t.ReadWriteCloser.Read(b)
}
for {
n, err := unix.Read(t.fd, b)
if err == nil {
return n, nil
}
if err == unix.EAGAIN || err == unix.EWOULDBLOCK {
pfds := []unix.PollFd{{Fd: int32(t.fd), Events: unix.POLLIN}}
_, err = unix.Poll(pfds, -1)
if err != nil {
if err == unix.EINTR {
continue
}
return 0, err
}
continue
}
return n, err
}
}
// ReadBatch reads up to len(packets) packets from the TUN device.
// EnableBatchReading must be called first.
func (t *tun) ReadBatch(packets [][]byte, sizes []int) (int, error) {
if !t.nonBlocking {
// Fallback to single read if non-blocking not enabled
n, err := t.Read(packets[0])
if err != nil {
return 0, err
}
sizes[0] = n
return 1, nil
}
count := 0
maxPackets := len(packets)
if len(sizes) < maxPackets {
maxPackets = len(sizes)
}
// Choose read buffer based on vnetHdr
// With vnetHdr, we need to read into internal buffer (has space for header)
// then copy packet data to caller's buffer
readBuf := packets[0] // Will be updated in the loop
if t.vnetHdr {
readBuf = t.readBuf
}
for count < maxPackets {
if !t.vnetHdr {
readBuf = packets[count]
}
n, err := unix.Read(t.fd, readBuf)
if err == nil && n > 0 {
if t.vnetHdr {
if n > virtioNetHdrLen {
packetLen := n - virtioNetHdrLen
copy(packets[count], readBuf[virtioNetHdrLen:n])
sizes[count] = packetLen
} else {
// Malformed packet (no data after header), skip
continue
}
} else {
sizes[count] = n
}
count++
continue
}
if err == unix.EAGAIN || err == unix.EWOULDBLOCK {
// No more packets available
if count > 0 {
return count, nil
}
// No packets yet, wait for at least one
pfds := []unix.PollFd{{Fd: int32(t.fd), Events: unix.POLLIN}}
_, err = unix.Poll(pfds, -1)
if err != nil {
if err == unix.EINTR {
continue
}
return 0, err
}
continue
}
if err != nil {
if count > 0 {
return count, nil
}
return 0, err
}
if n == 0 {
if count > 0 {
return count, nil
}
return 0, io.EOF
}
}
return count, nil
}
func (t *tun) deviceBytes() (o [16]byte) { func (t *tun) deviceBytes() (o [16]byte) {
for i, c := range t.Device { for i, c := range t.Device {
o[i] = byte(c) o[i] = byte(c)
@@ -713,6 +1110,7 @@ func (t *tun) Close() error {
if t.ioctlFd > 0 { if t.ioctlFd > 0 {
_ = os.NewFile(t.ioctlFd, "ioctlFd").Close() _ = os.NewFile(t.ioctlFd, "ioctlFd").Close()
t.ioctlFd = 0
} }
return nil return nil
+4 -1
View File
@@ -74,7 +74,10 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
l.WithError(err).Debug("Failed to create wintun device, retrying") l.WithError(err).Debug("Failed to create wintun device, retrying")
tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU) tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU)
if err != nil { if err != nil {
return nil, fmt.Errorf("create TUN device failed: %w", err) return nil, &NameError{
Name: deviceName,
Underlying: fmt.Errorf("create TUN device failed: %w", err),
}
} }
} }
t.tun = tunDevice.(*wintun.NativeTun) t.tun = tunDevice.(*wintun.NativeTun)
+18
View File
@@ -13,13 +13,22 @@ type EncReader func(
payload []byte, payload []byte,
) )
// BatchPacket represents a single packet in a batch write operation
type BatchPacket struct {
Payload []byte
Addr netip.AddrPort
}
type Conn interface { type Conn interface {
Rebind() error Rebind() error
LocalAddr() (netip.AddrPort, error) LocalAddr() (netip.AddrPort, error)
ListenOut(r EncReader) ListenOut(r EncReader)
WriteTo(b []byte, addr netip.AddrPort) error WriteTo(b []byte, addr netip.AddrPort) error
WriteBatch(pkts []BatchPacket) (int, error)
ReloadConfig(c *config.C) ReloadConfig(c *config.C)
SupportsMultipleReaders() bool SupportsMultipleReaders() bool
SupportsGSO() bool
SupportsGRO() bool
Close() error Close() error
} }
@@ -37,9 +46,18 @@ func (NoopConn) ListenOut(_ EncReader) {
func (NoopConn) SupportsMultipleReaders() bool { func (NoopConn) SupportsMultipleReaders() bool {
return false return false
} }
func (NoopConn) SupportsGSO() bool {
return false
}
func (NoopConn) SupportsGRO() bool {
return false
}
func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error { func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
return nil return nil
} }
func (NoopConn) WriteBatch(pkts []BatchPacket) (int, error) {
return len(pkts), nil
}
func (NoopConn) ReloadConfig(_ *config.C) { func (NoopConn) ReloadConfig(_ *config.C) {
return return
} }
+17
View File
@@ -188,6 +188,14 @@ func (u *StdConn) SupportsMultipleReaders() bool {
return false return false
} }
func (u *StdConn) SupportsGSO() bool {
return false
}
func (u *StdConn) SupportsGRO() bool {
return false
}
func (u *StdConn) Rebind() error { func (u *StdConn) Rebind() error {
var err error var err error
if u.isV4 { if u.isV4 {
@@ -202,3 +210,12 @@ func (u *StdConn) Rebind() error {
return nil return nil
} }
func (u *StdConn) WriteBatch(pkts []BatchPacket) (int, error) {
for i := range pkts {
if err := u.WriteTo(pkts[i].Payload, pkts[i].Addr); err != nil {
return i, err
}
}
return len(pkts), nil
}
+17
View File
@@ -101,3 +101,20 @@ func (u *GenericConn) ListenOut(r EncReader) {
func (u *GenericConn) SupportsMultipleReaders() bool { func (u *GenericConn) SupportsMultipleReaders() bool {
return false return false
} }
func (u *GenericConn) SupportsGSO() bool {
return false
}
func (u *GenericConn) SupportsGRO() bool {
return false
}
func (u *GenericConn) WriteBatch(pkts []BatchPacket) (int, error) {
for i := range pkts {
if err := u.WriteTo(pkts[i].Payload, pkts[i].Addr); err != nil {
return i, err
}
}
return len(pkts), nil
}
+358 -8
View File
@@ -5,6 +5,7 @@ package udp
import ( import (
"encoding/binary" "encoding/binary"
"errors"
"fmt" "fmt"
"net" "net"
"net/netip" "net/netip"
@@ -18,10 +19,12 @@ import (
) )
type StdConn struct { type StdConn struct {
sysFd int sysFd int
isV4 bool isV4 bool
l *logrus.Logger l *logrus.Logger
batch int batch int
gsoSupported bool
groSupported bool
} }
func maybeIPV4(ip net.IP) (net.IP, bool) { func maybeIPV4(ip net.IP) (net.IP, bool) {
@@ -32,6 +35,78 @@ func maybeIPV4(ip net.IP) (net.IP, bool) {
return ip, false return ip, false
} }
// supportsUDPOffload checks if the kernel supports UDP GSO (Generic Segmentation Offload)
// by attempting to get the UDP_SEGMENT socket option.
func supportsUDPOffload(fd int) bool {
_, err := unix.GetsockoptInt(fd, unix.IPPROTO_UDP, unix.UDP_SEGMENT)
return err == nil
}
// supportsUDPGRO checks if the kernel supports UDP GRO (Generic Receive Offload)
// and attempts to enable it on the socket.
func supportsUDPGRO(fd int) bool {
// Try to enable UDP_GRO
err := unix.SetsockoptInt(fd, unix.IPPROTO_UDP, unix.UDP_GRO, 1)
return err == nil
}
const (
// Maximum number of datagrams that can be coalesced with GSO/GRO
udpSegmentMaxDatagrams = 64
// Maximum size of a GRO coalesced packet (64KB is the practical limit)
// This is udpSegmentMaxDatagrams * MTU but capped at 65535
groMaxPacketSize = 65535
)
// setGSOSize writes a UDP_SEGMENT control message to the provided buffer
// with the given segment size. Returns the actual control message length.
func setGSOSize(control []byte, gsoSize uint16) int {
// Build the cmsghdr structure
cmsgLen := unix.CmsgLen(2) // 2 bytes for uint16 segment size
cmsg := (*unix.Cmsghdr)(unsafe.Pointer(&control[0]))
cmsg.Level = unix.IPPROTO_UDP
cmsg.Type = unix.UDP_SEGMENT
cmsg.SetLen(cmsgLen)
// Write the segment size after the header (after cmsghdr)
binary.NativeEndian.PutUint16(control[unix.SizeofCmsghdr:], gsoSize)
return unix.CmsgSpace(2) // aligned size
}
// getGROSize parses a control message buffer to extract the UDP_GRO segment size.
// Returns 0 if no GRO control message is present (meaning the packet is not coalesced).
func getGROSize(control []byte, controlLen int) uint16 {
if controlLen < unix.SizeofCmsghdr {
return 0
}
// Parse control messages
for offset := 0; offset < controlLen; {
if offset+unix.SizeofCmsghdr > controlLen {
break
}
cmsg := (*unix.Cmsghdr)(unsafe.Pointer(&control[offset]))
cmsgDataLen := int(cmsg.Len) - unix.SizeofCmsghdr
if cmsgDataLen < 0 {
break
}
if cmsg.Level == unix.IPPROTO_UDP && cmsg.Type == unix.UDP_GRO {
if cmsgDataLen >= 2 {
return binary.NativeEndian.Uint16(control[offset+unix.SizeofCmsghdr:])
}
}
// Move to next control message (aligned)
offset += unix.CmsgSpace(cmsgDataLen)
}
return 0
}
func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) { func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
af := unix.AF_INET6 af := unix.AF_INET6
if ip.Is4() { if ip.Is4() {
@@ -69,13 +144,31 @@ func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch in
return nil, fmt.Errorf("unable to bind to socket: %s", err) return nil, fmt.Errorf("unable to bind to socket: %s", err)
} }
return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, err gsoSupported := supportsUDPOffload(fd)
if gsoSupported {
l.Info("UDP GSO offload is supported")
}
groSupported := supportsUDPGRO(fd)
if groSupported {
l.Info("UDP GRO offload is supported and enabled")
}
return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch, gsoSupported: gsoSupported, groSupported: groSupported}, err
} }
func (u *StdConn) SupportsMultipleReaders() bool { func (u *StdConn) SupportsMultipleReaders() bool {
return true return true
} }
func (u *StdConn) SupportsGSO() bool {
return u.gsoSupported
}
func (u *StdConn) SupportsGRO() bool {
return u.groSupported
}
func (u *StdConn) Rebind() error { func (u *StdConn) Rebind() error {
return nil return nil
} }
@@ -125,13 +218,27 @@ func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
func (u *StdConn) ListenOut(r EncReader) { func (u *StdConn) ListenOut(r EncReader) {
var ip netip.Addr var ip netip.Addr
msgs, buffers, names := u.PrepareRawMessages(u.batch) msgs, buffers, names, controls := u.PrepareRawMessages(u.batch)
read := u.ReadMulti read := u.ReadMulti
if u.batch == 1 { if u.batch == 1 {
read = u.ReadSingle read = u.ReadSingle
} }
// Store the original control buffer size for resetting after each read
controlLen := 0
if u.groSupported && len(controls) > 0 && len(controls[0]) > 0 {
controlLen = len(controls[0])
}
for { for {
// Reset Controllen before each read - the kernel updates this field
// after recvmsg to indicate actual received control data length
if controlLen > 0 {
for i := range msgs {
setMsghdrControllen(&msgs[i].Hdr, controlLen)
}
}
n, err := read(msgs) n, err := read(msgs)
if err != nil { if err != nil {
u.l.WithError(err).Debug("udp socket is closed, exiting read loop") u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
@@ -139,13 +246,36 @@ func (u *StdConn) ListenOut(r EncReader) {
} }
for i := 0; i < n; i++ { for i := 0; i < n; i++ {
// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic // Extract source address
if u.isV4 { if u.isV4 {
ip, _ = netip.AddrFromSlice(names[i][4:8]) ip, _ = netip.AddrFromSlice(names[i][4:8])
} else { } else {
ip, _ = netip.AddrFromSlice(names[i][8:24]) ip, _ = netip.AddrFromSlice(names[i][8:24])
} }
r(netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4])), buffers[i][:msgs[i].Len]) srcAddr := netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4]))
// Check for GRO coalesced packet
totalLen := int(msgs[i].Len)
segmentSize := uint16(0)
if controlLen > 0 {
segmentSize = getGROSize(controls[i], getMsghdrControllen(&msgs[i].Hdr))
}
if segmentSize > 0 && totalLen > int(segmentSize) {
// This is a GRO coalesced packet - split it into individual datagrams
for offset := 0; offset < totalLen; {
packetLen := int(segmentSize)
if offset+packetLen > totalLen {
// Last packet may be smaller
packetLen = totalLen - offset
}
r(srcAddr, buffers[i][offset:offset+packetLen])
offset += packetLen
}
} else {
// Single packet, no coalescing
r(srcAddr, buffers[i][:totalLen])
}
} }
} }
} }
@@ -313,6 +443,226 @@ func (u *StdConn) Close() error {
return syscall.Close(u.sysFd) return syscall.Close(u.sysFd)
} }
func (u *StdConn) WriteBatch(pkts []BatchPacket) (int, error) {
if len(pkts) == 0 {
return 0, nil
}
// If GSO is supported, try to coalesce packets to the same destination
if u.gsoSupported {
return u.writeBatchGSO(pkts)
}
return u.writeBatchSendmmsg(pkts)
}
// writeBatchSendmmsg sends packets using sendmmsg without GSO coalescing
func (u *StdConn) writeBatchSendmmsg(pkts []BatchPacket) (int, error) {
msgs := make([]rawMessage, len(pkts))
iovecs := make([]iovec, len(pkts))
var names4 []unix.RawSockaddrInet4
var names6 []unix.RawSockaddrInet6
if u.isV4 {
names4 = make([]unix.RawSockaddrInet4, len(pkts))
} else {
names6 = make([]unix.RawSockaddrInet6, len(pkts))
}
for i := range pkts {
setIovecBase(&iovecs[i], &pkts[i].Payload[0])
setIovecLen(&iovecs[i], len(pkts[i].Payload))
msgs[i].Hdr.Iov = &iovecs[i]
setMsghdrIovlen(&msgs[i].Hdr, 1)
if u.isV4 {
names4[i].Family = unix.AF_INET
names4[i].Addr = pkts[i].Addr.Addr().As4()
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&names4[i].Port))[:], pkts[i].Addr.Port())
msgs[i].Hdr.Name = (*byte)(unsafe.Pointer(&names4[i]))
msgs[i].Hdr.Namelen = unix.SizeofSockaddrInet4
} else {
names6[i].Family = unix.AF_INET6
names6[i].Addr = pkts[i].Addr.Addr().As16()
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&names6[i].Port))[:], pkts[i].Addr.Port())
msgs[i].Hdr.Name = (*byte)(unsafe.Pointer(&names6[i]))
msgs[i].Hdr.Namelen = unix.SizeofSockaddrInet6
}
}
var sent int
for sent < len(msgs) {
n, _, errno := unix.Syscall6(
unix.SYS_SENDMMSG,
uintptr(u.sysFd),
uintptr(unsafe.Pointer(&msgs[sent])),
uintptr(len(msgs)-sent),
0,
0,
0,
)
if errno == unix.EINTR {
continue
}
if errno != 0 {
return sent, &net.OpError{Op: "sendmmsg", Err: errno}
}
sent += int(n)
}
return sent, nil
}
// writeBatchGSO sends packets using GSO coalescing when possible.
// Packets to the same destination with the same size are coalesced into a single
// GSO message. Mixed destinations or sizes fall back to individual sendmmsg calls.
func (u *StdConn) writeBatchGSO(pkts []BatchPacket) (int, error) {
// Group packets by destination and try to coalesce
totalSent := 0
i := 0
for i < len(pkts) {
// Find a run of packets to the same destination with compatible sizes
startIdx := i
dst := pkts[i].Addr
segmentSize := len(pkts[i].Payload)
// Count how many packets we can coalesce (same destination, same size except possibly last)
coalescedCount := 1
totalSize := segmentSize
for i+coalescedCount < len(pkts) && coalescedCount < udpSegmentMaxDatagrams {
next := pkts[i+coalescedCount]
if next.Addr != dst {
break
}
nextSize := len(next.Payload)
// For GSO, all packets except the last must have the same size
// The last packet can be smaller (but not larger)
if nextSize != segmentSize {
// Check if this could be the last packet (smaller is ok)
if nextSize < segmentSize && i+coalescedCount == len(pkts)-1 {
coalescedCount++
totalSize += nextSize
}
break
}
coalescedCount++
totalSize += nextSize
}
// If we have multiple packets to coalesce, use GSO
if coalescedCount > 1 {
err := u.sendGSO(pkts[startIdx:startIdx+coalescedCount], dst, segmentSize, totalSize)
if err != nil {
// If GSO fails (e.g., EIO due to NIC not supporting checksum offload),
// disable GSO and fall back to sendmmsg for the rest
if isGSOError(err) {
u.l.WithError(err).Warn("GSO send failed, disabling GSO for this connection")
u.gsoSupported = false
// Send remaining packets with sendmmsg
remaining, rerr := u.writeBatchSendmmsg(pkts[startIdx:])
return totalSent + remaining, rerr
}
return totalSent, err
}
totalSent += coalescedCount
i += coalescedCount
} else {
// Single packet, send without GSO overhead
err := u.WriteTo(pkts[i].Payload, pkts[i].Addr)
if err != nil {
return totalSent, err
}
totalSent++
i++
}
}
return totalSent, nil
}
// sendGSO sends coalesced packets using UDP GSO
func (u *StdConn) sendGSO(pkts []BatchPacket, dst netip.AddrPort, segmentSize, totalSize int) error {
// Allocate a buffer large enough for all packet payloads
coalescedBuf := make([]byte, totalSize)
offset := 0
for _, pkt := range pkts {
copy(coalescedBuf[offset:], pkt.Payload)
offset += len(pkt.Payload)
}
// Prepare control message with GSO segment size
control := make([]byte, unix.CmsgSpace(2))
controlLen := setGSOSize(control, uint16(segmentSize))
// Prepare the iovec
iov := iovec{}
setIovecBase(&iov, &coalescedBuf[0])
setIovecLen(&iov, totalSize)
// Prepare the msghdr
var hdr msghdr
hdr.Iov = &iov
setMsghdrIovlen(&hdr, 1)
hdr.Control = &control[0]
setMsghdrControllen(&hdr, controlLen)
// Declare sockaddr at function scope so it remains valid for the syscall
// (must not go out of scope before the syscall is made)
var rsa4 unix.RawSockaddrInet4
var rsa6 unix.RawSockaddrInet6
// Set destination address
if u.isV4 {
rsa4.Family = unix.AF_INET
rsa4.Addr = dst.Addr().As4()
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa4.Port))[:], dst.Port())
hdr.Name = (*byte)(unsafe.Pointer(&rsa4))
hdr.Namelen = unix.SizeofSockaddrInet4
} else {
rsa6.Family = unix.AF_INET6
rsa6.Addr = dst.Addr().As16()
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa6.Port))[:], dst.Port())
hdr.Name = (*byte)(unsafe.Pointer(&rsa6))
hdr.Namelen = unix.SizeofSockaddrInet6
}
for {
_, _, errno := unix.Syscall6(
unix.SYS_SENDMSG,
uintptr(u.sysFd),
uintptr(unsafe.Pointer(&hdr)),
0,
0,
0,
0,
)
if errno == unix.EINTR {
continue
}
if errno != 0 {
return &net.OpError{Op: "sendmsg", Err: errno}
}
return nil
}
}
// isGSOError returns true if the error indicates GSO is not supported by the NIC
func isGSOError(err error) bool {
var opErr *net.OpError
if !errors.As(err, &opErr) {
return false
}
// EIO typically means the NIC doesn't support checksum offload required for GSO
return errors.Is(opErr.Err, unix.EIO)
}
func NewUDPStatsEmitter(udpConns []Conn) func() { func NewUDPStatsEmitter(udpConns []Conn) func() {
// Check if our kernel supports SO_MEMINFO before registering the gauges // Check if our kernel supports SO_MEMINFO before registering the gauges
var udpGauges [][unix.SK_MEMINFO_VARS]metrics.Gauge var udpGauges [][unix.SK_MEMINFO_VARS]metrics.Gauge
+43 -3
View File
@@ -30,13 +30,26 @@ type rawMessage struct {
Len uint32 Len uint32
} }
func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) { func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte, [][]byte) {
msgs := make([]rawMessage, n) msgs := make([]rawMessage, n)
buffers := make([][]byte, n) buffers := make([][]byte, n)
names := make([][]byte, n) names := make([][]byte, n)
controls := make([][]byte, n)
// Use larger buffers if GRO is enabled to hold coalesced packets
bufSize := MTU
if u.groSupported {
bufSize = groMaxPacketSize
}
// Control buffer size for receiving UDP_GRO segment size
controlSize := 0
if u.groSupported {
controlSize = unix.CmsgSpace(2) // space for uint16 segment size
}
for i := range msgs { for i := range msgs {
buffers[i] = make([]byte, MTU) buffers[i] = make([]byte, bufSize)
names[i] = make([]byte, unix.SizeofSockaddrInet6) names[i] = make([]byte, unix.SizeofSockaddrInet6)
vs := []iovec{ vs := []iovec{
@@ -48,7 +61,34 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
msgs[i].Hdr.Name = &names[i][0] msgs[i].Hdr.Name = &names[i][0]
msgs[i].Hdr.Namelen = uint32(len(names[i])) msgs[i].Hdr.Namelen = uint32(len(names[i]))
// Set up control message buffer for GRO
if controlSize > 0 {
controls[i] = make([]byte, controlSize)
msgs[i].Hdr.Control = &controls[i][0]
msgs[i].Hdr.Controllen = uint32(controlSize)
}
} }
return msgs, buffers, names return msgs, buffers, names, controls
}
func setIovecBase(iov *iovec, base *byte) {
iov.Base = base
}
func setIovecLen(iov *iovec, l int) {
iov.Len = uint32(l)
}
func setMsghdrIovlen(hdr *msghdr, l int) {
hdr.Iovlen = uint32(l)
}
func setMsghdrControllen(hdr *msghdr, l int) {
hdr.Controllen = uint32(l)
}
func getMsghdrControllen(hdr *msghdr) int {
return int(hdr.Controllen)
} }
+43 -3
View File
@@ -33,13 +33,26 @@ type rawMessage struct {
Pad0 [4]byte Pad0 [4]byte
} }
func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) { func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte, [][]byte) {
msgs := make([]rawMessage, n) msgs := make([]rawMessage, n)
buffers := make([][]byte, n) buffers := make([][]byte, n)
names := make([][]byte, n) names := make([][]byte, n)
controls := make([][]byte, n)
// Use larger buffers if GRO is enabled to hold coalesced packets
bufSize := MTU
if u.groSupported {
bufSize = groMaxPacketSize
}
// Control buffer size for receiving UDP_GRO segment size
controlSize := 0
if u.groSupported {
controlSize = unix.CmsgSpace(2) // space for uint16 segment size
}
for i := range msgs { for i := range msgs {
buffers[i] = make([]byte, MTU) buffers[i] = make([]byte, bufSize)
names[i] = make([]byte, unix.SizeofSockaddrInet6) names[i] = make([]byte, unix.SizeofSockaddrInet6)
vs := []iovec{ vs := []iovec{
@@ -51,7 +64,34 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
msgs[i].Hdr.Name = &names[i][0] msgs[i].Hdr.Name = &names[i][0]
msgs[i].Hdr.Namelen = uint32(len(names[i])) msgs[i].Hdr.Namelen = uint32(len(names[i]))
// Set up control message buffer for GRO
if controlSize > 0 {
controls[i] = make([]byte, controlSize)
msgs[i].Hdr.Control = &controls[i][0]
msgs[i].Hdr.Controllen = uint64(controlSize)
}
} }
return msgs, buffers, names return msgs, buffers, names, controls
}
func setIovecBase(iov *iovec, base *byte) {
iov.Base = base
}
func setIovecLen(iov *iovec, l int) {
iov.Len = uint64(l)
}
func setMsghdrIovlen(hdr *msghdr, l int) {
hdr.Iovlen = uint64(l)
}
func setMsghdrControllen(hdr *msghdr, l int) {
hdr.Controllen = uint64(l)
}
func getMsghdrControllen(hdr *msghdr) int {
return int(hdr.Controllen)
} }
+17
View File
@@ -332,12 +332,29 @@ func (u *RIOConn) SupportsMultipleReaders() bool {
return false return false
} }
func (u *RIOConn) SupportsGSO() bool {
return false
}
func (u *RIOConn) SupportsGRO() bool {
return false
}
func (u *RIOConn) Rebind() error { func (u *RIOConn) Rebind() error {
return nil return nil
} }
func (u *RIOConn) ReloadConfig(*config.C) {} func (u *RIOConn) ReloadConfig(*config.C) {}
func (u *RIOConn) WriteBatch(pkts []BatchPacket) (int, error) {
for i := range pkts {
if err := u.WriteTo(pkts[i].Payload, pkts[i].Addr); err != nil {
return i, err
}
}
return len(pkts), nil
}
func (u *RIOConn) Close() error { func (u *RIOConn) Close() error {
if !u.isOpen.CompareAndSwap(true, false) { if !u.isOpen.CompareAndSwap(true, false) {
return nil return nil
+17
View File
@@ -131,6 +131,14 @@ func (u *TesterConn) SupportsMultipleReaders() bool {
return false return false
} }
func (u *TesterConn) SupportsGSO() bool {
return false
}
func (u *TesterConn) SupportsGRO() bool {
return false
}
func (u *TesterConn) Rebind() error { func (u *TesterConn) Rebind() error {
return nil return nil
} }
@@ -142,3 +150,12 @@ func (u *TesterConn) Close() error {
} }
return nil return nil
} }
func (u *TesterConn) WriteBatch(pkts []BatchPacket) (int, error) {
for i := range pkts {
if err := u.WriteTo(pkts[i].Payload, pkts[i].Addr); err != nil {
return i, err
}
}
return len(pkts), nil
}