mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 22:36:52 +01:00
Wait for readers to exit before closing their fds on linux (#1979)
smoke-extra / freebsd-amd64 (push) Failing after 16s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 17s
smoke-extra / netbsd-amd64 (push) Failing after 17s
smoke-extra / openbsd-amd64 (push) Failing after 16s
smoke-extra / linux-386 (push) Failing after 15s
smoke / Run multi node smoke test (push) Failing after 1m40s
Build and test / Static checks (push) Successful in 47s
Build and test / Test linux (push) Failing after 1m52s
Build and test / Test linux-pkcs11 (push) Failing after 2m7s
Build and test / Test linux-boringcrypto (push) Failing after 3m3s
Build and test / Test linux-fips140 (push) Failing after 2m57s
Build and test / Cross-build linux-arm (push) Successful in 3m2s
Build and test / Cross-build linux-mips (push) Successful in 3m42s
Build and test / Cross-build linux-other (push) Successful in 3m12s
Build and test / Cross-build windows (push) Successful in 1m4s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m32s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m24s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
smoke-extra / freebsd-amd64 (push) Failing after 16s
smoke-extra / linux-amd64-ipv6disable (push) Failing after 17s
smoke-extra / netbsd-amd64 (push) Failing after 17s
smoke-extra / openbsd-amd64 (push) Failing after 16s
smoke-extra / linux-386 (push) Failing after 15s
smoke / Run multi node smoke test (push) Failing after 1m40s
Build and test / Static checks (push) Successful in 47s
Build and test / Test linux (push) Failing after 1m52s
Build and test / Test linux-pkcs11 (push) Failing after 2m7s
Build and test / Test linux-boringcrypto (push) Failing after 3m3s
Build and test / Test linux-fips140 (push) Failing after 2m57s
Build and test / Cross-build linux-arm (push) Successful in 3m2s
Build and test / Cross-build linux-mips (push) Successful in 3m42s
Build and test / Cross-build linux-other (push) Successful in 3m12s
Build and test / Cross-build windows (push) Successful in 1m4s
Build and test / Cross-build freebsd (push) Successful in 1m32s
Build and test / Cross-build netbsd (push) Successful in 1m32s
Build and test / Cross-build openbsd (push) Successful in 1m32s
Build and test / Cross-build mobile (push) Successful in 3m24s
smoke-extra / Run windows smoke test (push) Canceled after 0s
smoke / Run self traffic smoke test on macOS (push) Canceled after 0s
Build and test / Test macos (push) Canceled after 0s
Build and test / Test windows (push) Canceled after 0s
Build and test / CI status (push) Canceled after 0s
This commit is contained in:
1 parent
fb2c153dfa
commit
48ba879198
10 files changed
+216
-26
No files matched your search
@@ -20,6 +20,7 @@ type offloadQueueSet struct {
|
||||
// usoEnabled is true when newTun successfully negotiated TUN_F_USO4|6 with the kernel.
|
||||
// Queues created by Add inherit this and surface it via Offload.USOSupported so coalescers can gate USO emission.
|
||||
usoEnabled bool
|
||||
shutdown atomic.Bool
|
||||
closed atomic.Bool
|
||||
// l is handed to each queue for its bad-vnet-header drop logging.
|
||||
l *slog.Logger
|
||||
@@ -69,6 +70,17 @@ func (c *offloadQueueSet) wakeForShutdown() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
|
||||
func (c *offloadQueueSet) Shutdown() error {
|
||||
if c.shutdown.Swap(true) {
|
||||
return nil
|
||||
}
|
||||
for _, x := range c.pq {
|
||||
x.shutdown.Store(true)
|
||||
}
|
||||
return c.wakeForShutdown()
|
||||
}
|
||||
|
||||
func (c *offloadQueueSet) Close() error {
|
||||
if c.closed.Swap(true) {
|
||||
return nil
|
||||
@@ -76,9 +88,7 @@ func (c *offloadQueueSet) Close() error {
|
||||
|
||||
errs := []error{}
|
||||
|
||||
// Signal all readers blocked in poll to wake up and exit.
|
||||
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
|
||||
if err := c.wakeForShutdown(); err != nil {
|
||||
if err := c.Shutdown(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ type pollQueueSet struct {
|
||||
// pqi is exactly the same as pq, but stored as the interface type
|
||||
pqi []Queue
|
||||
shutdownFd int
|
||||
shutdown atomic.Bool
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
@@ -59,6 +60,17 @@ func (c *pollQueueSet) wakeForShutdown() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Shutdown wakes every reader without releasing any fds, they see POLLIN on the eventfd and return os.ErrClosed
|
||||
func (c *pollQueueSet) Shutdown() error {
|
||||
if c.shutdown.Swap(true) {
|
||||
return nil
|
||||
}
|
||||
for _, x := range c.pq {
|
||||
x.shutdown.Store(true)
|
||||
}
|
||||
return c.wakeForShutdown()
|
||||
}
|
||||
|
||||
func (c *pollQueueSet) Close() error {
|
||||
if c.closed.Swap(true) {
|
||||
return nil
|
||||
@@ -66,9 +78,7 @@ func (c *pollQueueSet) Close() error {
|
||||
|
||||
errs := []error{}
|
||||
|
||||
// Signal all readers blocked in poll to wake up and exit.
|
||||
// They observe POLLIN on the shutdown eventfd and return os.ErrClosed.
|
||||
if err := c.wakeForShutdown(); err != nil {
|
||||
if err := c.Shutdown(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
// QueueSet holds one or many Queue objects and helps close them in an orderly way.
|
||||
type QueueSet interface {
|
||||
io.Closer
|
||||
// Shutdown wakes every reader without releasing any fds, Close only once they have exited
|
||||
Shutdown() error
|
||||
Queues() []Queue
|
||||
|
||||
// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
|
||||
|
||||
@@ -66,7 +66,9 @@ type Offload struct {
|
||||
// usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD,
|
||||
// so writers can decide whether emitting GSO_UDP_L4 superpackets is safe.
|
||||
usoEnabled bool
|
||||
closed atomic.Bool
|
||||
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
|
||||
shutdown atomic.Bool
|
||||
closed atomic.Bool
|
||||
|
||||
// gsoHdrBuf is a per-queue 10-byte scratch for the virtio_net_hdr emitted
|
||||
// by WriteGSO. Kept separate from the read-only package-level validVnetHdr
|
||||
@@ -176,6 +178,9 @@ func (r *Offload) readPacket(block bool) (int, error) {
|
||||
// This amortizes the poll wake over bursts of small packets (e.g. TCP ACKs).
|
||||
// Packet.Bytes slices point into the Offload's internal buffer and are only valid until the next Read or Close on this Queue.
|
||||
func (r *Offload) Read() ([]Packet, error) {
|
||||
if r.shutdown.Load() {
|
||||
return nil, os.ErrClosed
|
||||
}
|
||||
r.pending = r.pending[:0]
|
||||
r.rxOff = 0
|
||||
|
||||
|
||||
@@ -13,7 +13,9 @@ import (
|
||||
type Poll struct {
|
||||
fd int
|
||||
shutdownFd int
|
||||
closed atomic.Bool
|
||||
// shutdown is set before shutdownFd is signaled, a busy reader never polls to see that
|
||||
shutdown atomic.Bool
|
||||
closed atomic.Bool
|
||||
|
||||
readBuf []byte
|
||||
batchRet [1]Packet
|
||||
@@ -52,6 +54,9 @@ func (t *Poll) blockOnWrite() error {
|
||||
// config currently lose that batching. blockOn and the EAGAIN plumbing are
|
||||
// already shared; kept one-packet-per-Read for now to preserve behavior.
|
||||
func (t *Poll) Read() ([]Packet, error) {
|
||||
if t.shutdown.Load() {
|
||||
return nil, os.ErrClosed
|
||||
}
|
||||
n, err := t.readOne(t.readBuf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -223,3 +223,30 @@ func TestOffloadQueueSet_Close_ClosesShutdownFd(t *testing.T) {
|
||||
// Second Close must not touch fds (shutdownFd is now -1) and must return nil.
|
||||
require.NoError(t, qs.Close())
|
||||
}
|
||||
|
||||
// A busy reader never polls the shutdown eventfd, so Read has to stop on its own with data still waiting
|
||||
func TestQueueSet_ShutdownStopsReaderWithDataWaiting(t *testing.T) {
|
||||
sets := map[string]func() (QueueSet, error){
|
||||
"poll": NewPollQueueSet,
|
||||
"offload": func() (QueueSet, error) { return NewOffloadQueueSet(false, slog.New(slog.DiscardHandler)) },
|
||||
}
|
||||
for name, newSet := range sets {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var fds [2]int
|
||||
require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
|
||||
t.Cleanup(func() { _ = unix.Close(fds[1]) })
|
||||
|
||||
qs, err := newSet()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, qs.Add(fds[0]))
|
||||
t.Cleanup(func() { _ = qs.Close() })
|
||||
|
||||
_, err = unix.Write(fds[1], make([]byte, 64))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, qs.Shutdown())
|
||||
|
||||
_, err = qs.Queues()[0].Read()
|
||||
require.ErrorIs(t, err, os.ErrClosed)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -832,6 +832,11 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
|
||||
t.routeTree.Store(newTree)
|
||||
}
|
||||
|
||||
// Shutdown wakes the tun readers without releasing their fds, Close releases them
|
||||
func (t *tun) Shutdown() error {
|
||||
return t.readers.Shutdown()
|
||||
}
|
||||
|
||||
func (t *tun) Close() error {
|
||||
t.closeLock.Lock()
|
||||
defer t.closeLock.Unlock()
|
||||
|
||||
Reference in new issue
Block a user