mirror of
https://github.com/slackhq/nebula.git
synced 2026-08-16 05:37:00 +02:00
overlay/tio: log dropped tun reads with bad virtio headers
decodeRead failures were silently swallowed; a kernel emitting an unnegotiated GSO type would blackhole all tun traffic with nothing in the logs. Debug-gated per the usual idiom so the happy path pays nothing, which means plumbing the logger down through the offload queueset. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014ugV2edVqoz3tBvq9J6yWp
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
|||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
@@ -21,11 +22,13 @@ type offloadQueueSet struct {
|
|||||||
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
|
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
|
||||||
usoEnabled bool
|
usoEnabled bool
|
||||||
closed atomic.Bool
|
closed atomic.Bool
|
||||||
|
// l is handed to each queue for its bad-vnet-header drop logging.
|
||||||
|
l *slog.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewOffloadQueueSet creates a QueueSet that uses virtio_net_hdr to do TSO segmentation.
|
// NewOffloadQueueSet creates a QueueSet that uses virtio_net_hdr to do TSO segmentation.
|
||||||
// usoEnabled tells downstream queues whether the kernel agreed to deliver/accept GSO_UDP_L4 superpackets.
|
// usoEnabled tells downstream queues whether the kernel agreed to deliver/accept GSO_UDP_L4 superpackets.
|
||||||
func NewOffloadQueueSet(usoEnabled bool) (QueueSet, error) {
|
func NewOffloadQueueSet(usoEnabled bool, l *slog.Logger) (QueueSet, error) {
|
||||||
shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
|
shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to create eventfd: %w", err)
|
return nil, fmt.Errorf("failed to create eventfd: %w", err)
|
||||||
@@ -36,6 +39,7 @@ func NewOffloadQueueSet(usoEnabled bool) (QueueSet, error) {
|
|||||||
pqi: []Queue{},
|
pqi: []Queue{},
|
||||||
shutdownFd: shutdownFd,
|
shutdownFd: shutdownFd,
|
||||||
usoEnabled: usoEnabled,
|
usoEnabled: usoEnabled,
|
||||||
|
l: l,
|
||||||
}
|
}
|
||||||
|
|
||||||
return out, nil
|
return out, nil
|
||||||
@@ -46,7 +50,7 @@ func (c *offloadQueueSet) Queues() []Queue {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *offloadQueueSet) Add(fd int) error {
|
func (c *offloadQueueSet) Add(fd int) error {
|
||||||
x, err := newOffload(fd, c.shutdownFd, c.usoEnabled)
|
x, err := newOffload(fd, c.shutdownFd, c.usoEnabled, c.l)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,8 +4,10 @@
|
|||||||
package tio
|
package tio
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"syscall"
|
"syscall"
|
||||||
@@ -87,9 +89,13 @@ type Offload struct {
|
|||||||
// iovec[0] points at readVnetScratch
|
// iovec[0] points at readVnetScratch
|
||||||
// iovec[1].Base/Len is updated per read to address the current rxBuf slot.
|
// iovec[1].Base/Len is updated per read to address the current rxBuf slot.
|
||||||
readIovs [2]unix.Iovec
|
readIovs [2]unix.Iovec
|
||||||
|
|
||||||
|
// l is only consulted on the rare bad-vnet-header drop path; it lives
|
||||||
|
// after the hot state on purpose. May be nil (tests); drops go unlogged then.
|
||||||
|
l *slog.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
func newOffload(fd int, shutdownFd int, usoEnabled bool) (*Offload, error) {
|
func newOffload(fd int, shutdownFd int, usoEnabled bool, l *slog.Logger) (*Offload, error) {
|
||||||
if err := unix.SetNonblock(fd, true); err != nil {
|
if err := unix.SetNonblock(fd, true); err != nil {
|
||||||
return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
|
return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
|
||||||
}
|
}
|
||||||
@@ -99,6 +105,7 @@ func newOffload(fd int, shutdownFd int, usoEnabled bool) (*Offload, error) {
|
|||||||
shutdownFd: shutdownFd,
|
shutdownFd: shutdownFd,
|
||||||
usoEnabled: usoEnabled,
|
usoEnabled: usoEnabled,
|
||||||
closed: atomic.Bool{},
|
closed: atomic.Bool{},
|
||||||
|
l: l,
|
||||||
|
|
||||||
rxBuf: make([]byte, tunRxBufCap),
|
rxBuf: make([]byte, tunRxBufCap),
|
||||||
gsoIovs: make([]unix.Iovec, 2, gsoMaxIovs),
|
gsoIovs: make([]unix.Iovec, 2, gsoMaxIovs),
|
||||||
@@ -178,7 +185,9 @@ func (r *Offload) Read() ([]Packet, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := r.decodeRead(n); err != nil {
|
if err := r.decodeRead(n); err != nil {
|
||||||
// Drop and read again. A bad packet should not kill the reader.
|
// Drop and read again. A bad packet should not kill the reader,
|
||||||
|
// but a systematic decode failure must not be invisible either.
|
||||||
|
r.logDroppedRead(err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
@@ -200,6 +209,7 @@ func (r *Offload) Read() ([]Packet, error) {
|
|||||||
if err := r.decodeRead(n); err != nil {
|
if err := r.decodeRead(n); err != nil {
|
||||||
// Drop this packet and stop the drain; we'd rather hand off
|
// Drop this packet and stop the drain; we'd rather hand off
|
||||||
// what we have than keep spinning here.
|
// what we have than keep spinning here.
|
||||||
|
r.logDroppedRead(err)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -207,6 +217,14 @@ func (r *Offload) Read() ([]Packet, error) {
|
|||||||
return r.pending, nil
|
return r.pending, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// logDroppedRead reports a tun packet dropped for a bad/unsupported virtio
|
||||||
|
// header. Debug-gated so the happy path never pays for attribute assembly.
|
||||||
|
func (r *Offload) logDroppedRead(err error) {
|
||||||
|
if r.l != nil && r.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||||
|
r.l.Debug("dropping tun packet with bad virtio header", "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// decodeRead processes the packet sitting in rxBuf at rxOff (length pktLen).
|
// decodeRead processes the packet sitting in rxBuf at rxOff (length pktLen).
|
||||||
// The bytes stay in rxBuf:
|
// The bytes stay in rxBuf:
|
||||||
// - for GSO_NONE we slice them as a regular IP datagram (running finishChecksum if NEEDS_CSUM is set);
|
// - for GSO_NONE we slice them as a regular IP datagram (running finishChecksum if NEEDS_CSUM is set);
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ package tio
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -210,7 +211,7 @@ func TestPollQueueSet_Close_ClosesShutdownFd(t *testing.T) {
|
|||||||
// TestOffloadQueueSet_Close_ClosesShutdownFd mirrors the poll regression test
|
// TestOffloadQueueSet_Close_ClosesShutdownFd mirrors the poll regression test
|
||||||
// for the GSO/offload queueset.
|
// for the GSO/offload queueset.
|
||||||
func TestOffloadQueueSet_Close_ClosesShutdownFd(t *testing.T) {
|
func TestOffloadQueueSet_Close_ClosesShutdownFd(t *testing.T) {
|
||||||
qs, err := NewOffloadQueueSet(false)
|
qs, err := NewOffloadQueueSet(false, slog.New(slog.DiscardHandler))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
c, ok := qs.(*offloadQueueSet)
|
c, ok := qs.(*offloadQueueSet)
|
||||||
require.True(t, ok)
|
require.True(t, ok)
|
||||||
|
|||||||
@@ -231,7 +231,7 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
|
|||||||
var qs tio.QueueSet
|
var qs tio.QueueSet
|
||||||
var err error
|
var err error
|
||||||
if vnetHdr {
|
if vnetHdr {
|
||||||
qs, err = tio.NewOffloadQueueSet(offloadUSOEnabled(offloadFlags))
|
qs, err = tio.NewOffloadQueueSet(offloadUSOEnabled(offloadFlags), l)
|
||||||
} else {
|
} else {
|
||||||
qs, err = tio.NewPollQueueSet()
|
qs, err = tio.NewPollQueueSet()
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user