overlay: replace per-fd tun readers with a batched Queue interface

Device loses io.ReadWriteCloser + NewMultiQueueReader in favor of
Queues(n), which returns up to n tio.Queue objects; platforms without
multiqueue hand back their single queue and the interface sizes its
reader routines to what it actually got. Queue.Read returns a batch of
borrowed packets (single-element for every current backend) so a future
backend can deliver more than one packet per syscall without another
interface change.

The Linux poll/eventfd machinery moves out of tun_linux.go into the new
overlay/tio package: nonblocking fds, a shared shutdown eventfd owned by
the queue set, and pollfd arrays built on the stack so concurrent
writers parked in blockOnWrite no longer share Revents storage. Other
platforms wrap their existing one-datagram Read/Write in a singleQueue
adapter that owns a private scratch buffer, so multiqueue-by-sharing
devices (user, disabled) no longer race concurrent readers on one
buffer.

This is the tun-interface subset of better-tun-interface-ordering,
extracted at 18dc13b with none of the GSO/GRO offload mechanics and no
udp/sendmmsg changes.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
JackDoan
2026-07-16 12:18:38 -05:00
parent 6c3972f464
commit 913a37cfee
25 changed files with 930 additions and 501 deletions
+26 -24
View File
@@ -10,6 +10,7 @@ import (
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/routing"
)
@@ -23,6 +24,23 @@ type disabledTun struct {
l *slog.Logger
}
// Read hands the next queued packet to a reader, copying it into b. Reads
// from concurrent queues are safe: the channel receive serializes them and
// each queue copies into its own private scratch buffer.
func (t *disabledTun) Read(b []byte) (int, error) {
r, ok := <-t.read
if !ok {
return 0, io.EOF
}
t.tx.Inc(1)
if t.l.Enabled(context.Background(), slog.LevelDebug) {
t.l.Debug("Write payload", "raw", prettyPacket(r))
}
return copy(b, r), nil
}
func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled bool, l *slog.Logger) *disabledTun {
tun := &disabledTun{
vpnNetworks: vpnNetworks,
@@ -57,24 +75,6 @@ func (*disabledTun) Name() string {
return "disabled"
}
func (t *disabledTun) Read(b []byte) (int, error) {
r, ok := <-t.read
if !ok {
return 0, io.EOF
}
if len(r) > len(b) {
return 0, fmt.Errorf("packet larger than mtu: %d > %d bytes", len(r), len(b))
}
t.tx.Inc(1)
if t.l.Enabled(context.Background(), slog.LevelDebug) {
t.l.Debug("Write payload", "raw", prettyPacket(r))
}
return copy(b, r), nil
}
func (t *disabledTun) handleICMPEchoRequest(b []byte) bool {
out := make([]byte, len(b))
out = iputil.CreateICMPEchoResponse(b, out)
@@ -106,12 +106,14 @@ func (t *disabledTun) Write(b []byte) (int, error) {
return len(b), nil
}
func (t *disabledTun) SupportsMultiqueue() bool {
return true
}
func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return t, nil
func (t *disabledTun) Queues(n int) ([]tio.Queue, error) {
out := make([]tio.Queue, n)
for i := range out {
// NoClose: the shared channel and metrics are owned by the
// disabledTun; Close on the device tears them down once for everybody.
out[i] = tio.NewSingleQueueNoClose(t, defaultBatchBufSize)
}
return out, nil
}
func (t *disabledTun) Close() error {