Swap back to a blocking udp socket, test shutdown(2) (#1806)

Co-authored-by: Jack Doan <me@jackdoan.com>
This commit is contained in:
Nate Brown
2026-07-17 15:16:45 -05:00
committed by GitHub
parent e290a6892f
commit 147c202c27
3 changed files with 443 additions and 167 deletions
+96
View File
@@ -0,0 +1,96 @@
//go:build linux && !android && !e2e_testing
package main
import (
"fmt"
"net/netip"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/cert"
cert_test "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/require"
)
// TestControlStopClosesOnTimer reproduces the dnclient lifecycle: nebula runs as
// a library, and on a config update dnclient calls Stop() in-process to tear the
// old instance down before starting a new one. This boots a real nebula (real
// blocking UDP sockets, tun disabled), lets it run, then Stop()s it on a timer
// and asserts it actually closes. If the reader goroutines parked in recvmmsg
// don't wake on Close(), Wait() blocks forever and this fails with a goroutine
// dump instead of relying on a process signal to unstick them.
func TestControlStopClosesOnTimer(t *testing.T) {
l := test.NewLogger()
dir := t.TempDir()
before := time.Now().Add(-time.Hour)
after := time.Now().Add(time.Hour)
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "close-on-timer", before, after, networks, nil, nil)
caPath := filepath.Join(dir, "ca.pem")
certPath := filepath.Join(dir, "cert.pem")
keyPath := filepath.Join(dir, "key.pem")
require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
// tun disabled so no device/root is needed; routines: 2 so we exercise the
// multi-socket (SO_REUSEPORT) teardown, which is where dnclient runs.
configBody := fmt.Sprintf(`
pki:
ca: %s
cert: %s
key: %s
listen:
host: 127.0.0.1
port: 0
tun:
disabled: true
firewall:
outbound:
- port: any
proto: any
host: any
inbound:
- port: any
proto: any
host: any
routines: 2
`, caPath, certPath, keyPath)
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
c := config.NewC(l)
require.NoError(t, c.Load(dir))
ctrl, err := nebula.Main(c, false, "close-on-timer", l, nil)
require.NoError(t, err)
require.NoError(t, ctrl.Start())
// Run like a live nebula, then close on a timer, exactly as dnclient does.
<-time.NewTimer(5 * time.Second).C
stopped := make(chan struct{})
go func() {
ctrl.Stop() // closes the udp sockets (shutdown(2)) and the tun
ctrl.Wait() // blocks until every reader goroutine has returned
close(stopped)
}()
select {
case <-stopped:
t.Log("nebula closed cleanly on timer")
case <-time.After(10 * time.Second):
buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true)
t.Fatalf("nebula did NOT close within 10s of Stop(): a blocking reader never woke\n%s", buf[:n])
}
}
+168 -167
View File
@@ -4,12 +4,13 @@
package udp package udp
import ( import (
"context"
"encoding/binary" "encoding/binary"
"errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net" "net"
"net/netip" "net/netip"
"sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -19,58 +20,51 @@ import (
) )
type StdConn struct { type StdConn struct {
udpConn *net.UDPConn sysFd int
rawConn syscall.RawConn closed atomic.Bool
isV4 bool isV4 bool
l *slog.Logger l *slog.Logger
batch int batch int
}
func setReusePort(network, address string, c syscall.RawConn) error {
var opErr error
err := c.Control(func(fd uintptr) {
opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
//CloseOnExec already set by the runtime
})
if err != nil {
return err
}
return opErr
} }
func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) { func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
listen := netip.AddrPortFrom(ip, uint16(port)) af := unix.AF_INET6
lc := net.ListenConfig{} if ip.Is4() {
af = unix.AF_INET
}
syscall.ForkLock.RLock()
fd, err := unix.Socket(af, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
if err == nil {
unix.CloseOnExec(fd)
}
syscall.ForkLock.RUnlock()
if err != nil {
return nil, fmt.Errorf("unable to open socket: %w", err)
}
if multi { if multi {
lc.Control = setReusePort if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
} _ = unix.Close(fd)
//this context is only used during the bind operation, you can't cancel it to kill the socket return nil, fmt.Errorf("unable to set SO_REUSEPORT: %w", err)
pc, err := lc.ListenPacket(context.Background(), "udp", listen.String()) }
if err != nil {
return nil, fmt.Errorf("unable to open socket: %s", err)
}
udpConn := pc.(*net.UDPConn)
rawConn, err := udpConn.SyscallConn()
if err != nil {
_ = udpConn.Close()
return nil, err
}
//gotta find out if we got an AF_INET6 socket or not:
out := &StdConn{
udpConn: udpConn,
rawConn: rawConn,
l: l,
batch: batch,
} }
af, err := out.getSockOptInt(unix.SO_DOMAIN) var sa unix.Sockaddr
if err != nil { if ip.Is4() {
_ = out.Close() sa4 := &unix.SockaddrInet4{Port: port}
return nil, err sa4.Addr = ip.As4()
sa = sa4
} else {
sa6 := &unix.SockaddrInet6{Port: port}
sa6.Addr = ip.As16()
sa = sa6
}
if err = unix.Bind(fd, sa); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("unable to bind to socket: %w", err)
} }
out.isV4 = af == unix.AF_INET
return out, nil return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, nil
} }
func (u *StdConn) SupportsMultipleReaders() bool { func (u *StdConn) SupportsMultipleReaders() bool {
@@ -81,134 +75,111 @@ func (u *StdConn) Rebind() error {
return nil return nil
} }
func (u *StdConn) getSockOptInt(opt int) (int, error) {
if u.rawConn == nil {
return 0, fmt.Errorf("no UDP connection")
}
var out int
var opErr error
err := u.rawConn.Control(func(fd uintptr) {
out, opErr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, opt)
})
if err != nil {
return 0, err
}
return out, opErr
}
func (u *StdConn) setSockOptInt(opt int, n int) error {
if u.rawConn == nil {
return fmt.Errorf("no UDP connection")
}
var opErr error
err := u.rawConn.Control(func(fd uintptr) {
opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, opt, n)
})
if err != nil {
return err
}
return opErr
}
func (u *StdConn) SetRecvBuffer(n int) error { func (u *StdConn) SetRecvBuffer(n int) error {
return u.setSockOptInt(unix.SO_RCVBUFFORCE, n) return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n)
} }
func (u *StdConn) SetSendBuffer(n int) error { func (u *StdConn) SetSendBuffer(n int) error {
return u.setSockOptInt(unix.SO_SNDBUFFORCE, n) return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
} }
func (u *StdConn) SetSoMark(mark int) error { func (u *StdConn) SetSoMark(mark int) error {
return u.setSockOptInt(unix.SO_MARK, mark) return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK, mark)
} }
func (u *StdConn) GetRecvBuffer() (int, error) { func (u *StdConn) GetRecvBuffer() (int, error) {
return u.getSockOptInt(unix.SO_RCVBUF) return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUF)
} }
func (u *StdConn) GetSendBuffer() (int, error) { func (u *StdConn) GetSendBuffer() (int, error) {
return u.getSockOptInt(unix.SO_SNDBUF) return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF)
} }
func (u *StdConn) GetSoMark() (int, error) { func (u *StdConn) GetSoMark() (int, error) {
return u.getSockOptInt(unix.SO_MARK) return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK)
} }
func (u *StdConn) LocalAddr() (netip.AddrPort, error) { func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
a := u.udpConn.LocalAddr() sa, err := unix.Getsockname(u.sysFd)
if err != nil {
switch v := a.(type) { return netip.AddrPort{}, err
case *net.UDPAddr: }
addr, ok := netip.AddrFromSlice(v.IP) switch sa := sa.(type) {
if !ok { case *unix.SockaddrInet4:
return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP) return netip.AddrPortFrom(netip.AddrFrom4(sa.Addr), uint16(sa.Port)), nil
} case *unix.SockaddrInet6:
return netip.AddrPortFrom(addr, uint16(v.Port)), nil return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr), uint16(sa.Port)), nil
default: default:
return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a) return netip.AddrPort{}, fmt.Errorf("unsupported sock type: %T", sa)
} }
} }
func recvmmsg(fd uintptr, msgs []rawMessage) (int, bool, error) { // recvmmsg does one blocking recvmmsg (MSG_WAITFORONE), reading up to len(msgs) datagrams
var errno syscall.Errno func (u *StdConn) recvmmsg(msgs []rawMessage) (int, error) {
n, _, errno := unix.Syscall6( r, _, errno := unix.Syscall6(
unix.SYS_RECVMMSG, unix.SYS_RECVMMSG,
fd, uintptr(u.sysFd),
uintptr(unsafe.Pointer(&msgs[0])), uintptr(unsafe.Pointer(&msgs[0])),
uintptr(len(msgs)), uintptr(len(msgs)),
unix.MSG_WAITFORONE, unix.MSG_WAITFORONE,
0, 0,
0, 0,
) )
if errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK {
// No data available, block for I/O and try again.
return int(n), false, nil
}
if errno != 0 { if errno != 0 {
return int(n), true, &net.OpError{Op: "recvmmsg", Err: errno} if u.closed.Load() {
} return 0, net.ErrClosed
return int(n), true, nil
}
func (u *StdConn) listenOutSingle(r EncReader) error {
var err error
var n int
var from netip.AddrPort
buffer := make([]byte, MTU)
for {
n, from, err = u.udpConn.ReadFromUDPAddrPort(buffer)
if err != nil {
return err
} }
from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port()) return 0, &net.OpError{Op: "recvmmsg", Err: errno}
r(from, buffer[:n])
} }
n := int(r)
if (n == 0 || msgs[0].Len == 0) && u.closed.Load() {
return 0, net.ErrClosed
}
return n, nil
} }
func (u *StdConn) listenOutBatch(r EncReader) error { // recvmsg does one blocking recvmsg into msgs[0]
func (u *StdConn) recvmsg(msgs []rawMessage) (int, error) {
r, _, errno := unix.Syscall6(
unix.SYS_RECVMSG,
uintptr(u.sysFd),
uintptr(unsafe.Pointer(&msgs[0].Hdr)),
0,
0,
0,
0,
)
if errno != 0 {
if u.closed.Load() {
return 0, net.ErrClosed
}
return 0, &net.OpError{Op: "recvmsg", Err: errno}
}
if r == 0 && u.closed.Load() {
return 0, net.ErrClosed
}
msgs[0].Len = uint32(r)
return 1, nil
}
func (u *StdConn) ListenOut(r EncReader) error {
var ip netip.Addr var ip netip.Addr
var n int
var operr error
msgs, buffers, names := u.PrepareRawMessages(u.batch) msgs, buffers, names := u.PrepareRawMessages(u.batch)
read := u.recvmmsg
//reader needs to capture variables from this function, since it's used as a lambda with rawConn.Read if u.batch == 1 {
//defining it outside the loop so it gets re-used read = u.recvmsg
reader := func(fd uintptr) (done bool) {
n, done, operr = recvmmsg(fd, msgs)
return done
} }
for { for {
err := u.rawConn.Read(reader) n, err := read(msgs)
if err != nil { if err != nil {
if errors.Is(err, unix.EINTR) {
continue // interrupted by a signal, retry the read
}
// net.ErrClosed after Close() is teardown, absorbed by the caller's
// closed flag like the other platforms; anything else is a real error.
return err return err
} }
if operr != nil {
return operr
}
for i := 0; i < n; i++ { for i := 0; i < n; i++ {
// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic // Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
@@ -222,26 +193,68 @@ func (u *StdConn) listenOutBatch(r EncReader) error {
} }
} }
func (u *StdConn) ListenOut(r EncReader) error { func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
if u.batch == 1 { if u.isV4 {
return u.listenOutSingle(r) return u.writeTo4(b, ip)
} else { }
return u.listenOutBatch(r) return u.writeTo6(b, ip)
}
func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
var rsa unix.RawSockaddrInet6
rsa.Family = unix.AF_INET6
rsa.Addr = ip.Addr().As16()
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
for {
_, _, err := unix.Syscall6(
unix.SYS_SENDTO,
uintptr(u.sysFd),
uintptr(unsafe.Pointer(&b[0])),
uintptr(len(b)),
uintptr(0),
uintptr(unsafe.Pointer(&rsa)),
uintptr(unix.SizeofSockaddrInet6),
)
if err != 0 {
return &net.OpError{Op: "sendto", Err: err}
}
return nil
} }
} }
func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error { func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
_, err := u.udpConn.WriteToUDPAddrPort(b, ip) if !ip.Addr().Is4() {
return err return ErrInvalidIPv6RemoteForSocket
}
var rsa unix.RawSockaddrInet4
rsa.Family = unix.AF_INET
rsa.Addr = ip.Addr().As4()
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
for {
_, _, err := unix.Syscall6(
unix.SYS_SENDTO,
uintptr(u.sysFd),
uintptr(unsafe.Pointer(&b[0])),
uintptr(len(b)),
uintptr(0),
uintptr(unsafe.Pointer(&rsa)),
uintptr(unix.SizeofSockaddrInet4),
)
if err != 0 {
return &net.OpError{Op: "sendto", Err: err}
}
return nil
}
} }
func (u *StdConn) ReloadConfig(c *config.C) { func (u *StdConn) ReloadConfig(c *config.C) {
b := c.GetInt("listen.read_buffer", 0) b := c.GetInt("listen.read_buffer", 0)
if b > 0 { if b > 0 {
err := u.SetRecvBuffer(b) if err := u.SetRecvBuffer(b); err == nil {
if err == nil { if s, err := u.GetRecvBuffer(); err == nil {
s, err := u.GetRecvBuffer()
if err == nil {
u.l.Info("listen.read_buffer was set", "size", s) u.l.Info("listen.read_buffer was set", "size", s)
} else { } else {
u.l.Warn("Failed to get listen.read_buffer", "error", err) u.l.Warn("Failed to get listen.read_buffer", "error", err)
@@ -253,10 +266,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
b = c.GetInt("listen.write_buffer", 0) b = c.GetInt("listen.write_buffer", 0)
if b > 0 { if b > 0 {
err := u.SetSendBuffer(b) if err := u.SetSendBuffer(b); err == nil {
if err == nil { if s, err := u.GetSendBuffer(); err == nil {
s, err := u.GetSendBuffer()
if err == nil {
u.l.Info("listen.write_buffer was set", "size", s) u.l.Info("listen.write_buffer was set", "size", s)
} else { } else {
u.l.Warn("Failed to get listen.write_buffer", "error", err) u.l.Warn("Failed to get listen.write_buffer", "error", err)
@@ -269,10 +280,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
b = c.GetInt("listen.so_mark", 0) b = c.GetInt("listen.so_mark", 0)
s, err := u.GetSoMark() s, err := u.GetSoMark()
if b > 0 || (err == nil && s != 0) { if b > 0 || (err == nil && s != 0) {
err := u.SetSoMark(b) if err := u.SetSoMark(b); err == nil {
if err == nil { if s, err := u.GetSoMark(); err == nil {
s, err := u.GetSoMark()
if err == nil {
u.l.Info("listen.so_mark was set", "mark", s) u.l.Info("listen.so_mark was set", "mark", s)
} else { } else {
u.l.Warn("Failed to get listen.so_mark", "error", err) u.l.Warn("Failed to get listen.so_mark", "error", err)
@@ -285,28 +294,20 @@ func (u *StdConn) ReloadConfig(c *config.C) {
func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error { func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
var vallen uint32 = 4 * unix.SK_MEMINFO_VARS var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
if u.rawConn == nil { if err != 0 {
return fmt.Errorf("no UDP connection")
}
var opErr error
err := u.rawConn.Control(func(fd uintptr) {
_, _, syserr := unix.Syscall6(unix.SYS_GETSOCKOPT, fd, uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
if syserr != 0 {
opErr = syserr
}
})
if err != nil {
return err return err
} }
return opErr return nil
} }
func (u *StdConn) Close() error { func (u *StdConn) Close() error {
if u.udpConn != nil { u.closed.Store(true)
return u.udpConn.Close() // Wake the reader parked in recvmmsg/recvmsg. shutdown(2) on an unconnected socket
} // returns ENOTCONN but still wakes it, so ignore the error.
return nil // The reader then sees closed and stops touching the fd, making the Close below safe.
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
return unix.Close(u.sysFd)
} }
func NewUDPStatsEmitter(udpConns []Conn) func() { func NewUDPStatsEmitter(udpConns []Conn) func() {
+179
View File
@@ -0,0 +1,179 @@
//go:build linux && !android && !e2e_testing
package udp
import (
"errors"
"fmt"
"log/slog"
"net"
"net/netip"
"os"
"runtime"
"sync/atomic"
"testing"
"time"
"golang.org/x/sys/unix"
)
func testLogger() *slog.Logger {
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
}
// TestShutdownWakesAfterRx_Mechanism exercises the kernel quirk our teardown
// relies on: once a socket has received a packet, shutdown(2) wakes a blocked
// recvmmsg with n>=1/Len==0 (not n==0). recvmmsg must turn that into net.ErrClosed
// once Close set closed, so a parked reader exits instead of spinning.
func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, 64)
if err != nil {
t.Fatalf("NewListener: %v", err)
}
sc := c.(*StdConn)
addr, err := sc.LocalAddr()
if err != nil {
t.Fatalf("LocalAddr: %v", err)
}
msgs, _, _ := sc.PrepareRawMessages(sc.batch)
// Receive a real packet so the socket has carried data.
send, err := net.Dial("udp", addr.String())
if err != nil {
t.Fatalf("dial: %v", err)
}
if _, err := send.Write([]byte("hello")); err != nil {
t.Fatalf("write: %v", err)
}
time.Sleep(50 * time.Millisecond)
n, err := sc.recvmmsg(msgs)
t.Logf("drain of real packet: n=%d err=%v msgs[0].Len=%d", n, err, msgs[0].Len)
_ = send.Close()
// Block a reader on the now-empty queue, then tear down as Close() does.
// recvmmsg must return net.ErrClosed (not hang, not spin) even post-rx.
done := make(chan error, 1)
go func() {
_, err := sc.recvmmsg(msgs)
done <- err
}()
time.Sleep(150 * time.Millisecond) // let it park in recvmmsg
sc.closed.Store(true)
if serr := unix.Shutdown(sc.sysFd, unix.SHUT_RDWR); serr != nil {
t.Logf("shutdown returned %v (expected ENOTCONN on unconnected UDP)", serr)
}
select {
case err := <-done:
if !errors.Is(err, net.ErrClosed) {
t.Errorf("recvmmsg after post-rx shutdown returned %v, want net.ErrClosed", err)
}
case <-time.After(2 * time.Second):
t.Fatalf("HANG: recvmmsg did not return after shutdown following a received packet")
}
_ = unix.Close(sc.sysFd)
}
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
// reader must tear down cleanly on Close() regardless of what the socket has
// carried. The three cases the report called out:
//
// no traffic ever -> works (shutdown wakes recvmmsg with n==0)
// ping once, then idle -> historically HUNG: once the socket has received a
// packet, shutdown(2) wakes recvmmsg with n>=1/Len==0,
// which an n==0-only teardown check misses
// continuous traffic -> works (a real packet is always arriving)
//
// All three must return within the deadline; a hang dumps goroutines so the
// stuck reader is visible.
func TestListenOutTeardown_TrafficPatterns(t *testing.T) {
cases := []struct {
name string
traffic func(send net.Conn, stop <-chan struct{})
}{
{"no_traffic_ever", func(net.Conn, <-chan struct{}) {}},
{"ping_once_then_idle", func(send net.Conn, _ <-chan struct{}) {
_, _ = send.Write([]byte("hello"))
}},
{"continuous", func(send net.Conn, stop <-chan struct{}) {
for {
select {
case <-stop:
return
default:
_, _ = send.Write([]byte("hello"))
time.Sleep(2 * time.Millisecond)
}
}
}},
}
// batch 1 exercises the recvmsg path, batch 64 the recvmmsg path; both must
// tear down cleanly.
for _, batch := range []int{1, 64} {
for _, tc := range cases {
t.Run(fmt.Sprintf("batch%d/%s", batch, tc.name), func(t *testing.T) {
runTeardownCase(t, batch, tc.name, tc.traffic)
})
}
}
}
func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net.Conn, stop <-chan struct{})) {
c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, batch)
if err != nil {
t.Fatalf("NewListener: %v", err)
}
sc := c.(*StdConn)
addr, err := sc.LocalAddr()
if err != nil {
t.Fatalf("LocalAddr: %v", err)
}
var received atomic.Int64
loopDone := make(chan error, 1)
go func() {
loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
received.Add(1)
})
}()
send, err := net.Dial("udp", addr.String())
if err != nil {
t.Fatalf("dial: %v", err)
}
defer send.Close()
stop := make(chan struct{})
trafficDone := make(chan struct{})
go func() {
traffic(send, stop)
close(trafficDone)
}()
// Let the pattern run and, for the idle case, the reader park again on an
// empty queue with the socket already having received a packet.
time.Sleep(500 * time.Millisecond)
start := time.Now()
if err := sc.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
close(stop)
select {
case err := <-loopDone:
// Clean teardown surfaces as net.ErrClosed (propagated like the other
// platforms); the caller absorbs it via its closed flag.
if err != nil && !errors.Is(err, net.ErrClosed) {
t.Fatalf("%s: ListenOut returned unexpected error on teardown: %v", name, err)
}
t.Logf("%s: closed in %v (received %d packets)", name, time.Since(start), received.Load())
case <-time.After(3 * time.Second):
buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true)
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
}
<-trafficDone
}