mirror of
https://github.com/slackhq/nebula.git
synced 2026-08-15 06:17:03 +02:00
Swap back to a blocking udp socket, test shutdown(2) (#1806)
Co-authored-by: Jack Doan <me@jackdoan.com>
This commit is contained in:
@@ -0,0 +1,96 @@
|
|||||||
|
//go:build linux && !android && !e2e_testing
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/netip"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/slackhq/nebula"
|
||||||
|
"github.com/slackhq/nebula/cert"
|
||||||
|
cert_test "github.com/slackhq/nebula/cert_test"
|
||||||
|
"github.com/slackhq/nebula/config"
|
||||||
|
"github.com/slackhq/nebula/test"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestControlStopClosesOnTimer reproduces the dnclient lifecycle: nebula runs as
|
||||||
|
// a library, and on a config update dnclient calls Stop() in-process to tear the
|
||||||
|
// old instance down before starting a new one. This boots a real nebula (real
|
||||||
|
// blocking UDP sockets, tun disabled), lets it run, then Stop()s it on a timer
|
||||||
|
// and asserts it actually closes. If the reader goroutines parked in recvmmsg
|
||||||
|
// don't wake on Close(), Wait() blocks forever and this fails with a goroutine
|
||||||
|
// dump instead of relying on a process signal to unstick them.
|
||||||
|
func TestControlStopClosesOnTimer(t *testing.T) {
|
||||||
|
l := test.NewLogger()
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
before := time.Now().Add(-time.Hour)
|
||||||
|
after := time.Now().Add(time.Hour)
|
||||||
|
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
|
||||||
|
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
|
||||||
|
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "close-on-timer", before, after, networks, nil, nil)
|
||||||
|
|
||||||
|
caPath := filepath.Join(dir, "ca.pem")
|
||||||
|
certPath := filepath.Join(dir, "cert.pem")
|
||||||
|
keyPath := filepath.Join(dir, "key.pem")
|
||||||
|
require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
|
||||||
|
require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
|
||||||
|
require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
|
||||||
|
|
||||||
|
// tun disabled so no device/root is needed; routines: 2 so we exercise the
|
||||||
|
// multi-socket (SO_REUSEPORT) teardown, which is where dnclient runs.
|
||||||
|
configBody := fmt.Sprintf(`
|
||||||
|
pki:
|
||||||
|
ca: %s
|
||||||
|
cert: %s
|
||||||
|
key: %s
|
||||||
|
listen:
|
||||||
|
host: 127.0.0.1
|
||||||
|
port: 0
|
||||||
|
tun:
|
||||||
|
disabled: true
|
||||||
|
firewall:
|
||||||
|
outbound:
|
||||||
|
- port: any
|
||||||
|
proto: any
|
||||||
|
host: any
|
||||||
|
inbound:
|
||||||
|
- port: any
|
||||||
|
proto: any
|
||||||
|
host: any
|
||||||
|
routines: 2
|
||||||
|
`, caPath, certPath, keyPath)
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
|
||||||
|
|
||||||
|
c := config.NewC(l)
|
||||||
|
require.NoError(t, c.Load(dir))
|
||||||
|
|
||||||
|
ctrl, err := nebula.Main(c, false, "close-on-timer", l, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, ctrl.Start())
|
||||||
|
|
||||||
|
// Run like a live nebula, then close on a timer, exactly as dnclient does.
|
||||||
|
<-time.NewTimer(5 * time.Second).C
|
||||||
|
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
ctrl.Stop() // closes the udp sockets (shutdown(2)) and the tun
|
||||||
|
ctrl.Wait() // blocks until every reader goroutine has returned
|
||||||
|
close(stopped)
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-stopped:
|
||||||
|
t.Log("nebula closed cleanly on timer")
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
buf := make([]byte, 1<<20)
|
||||||
|
n := runtime.Stack(buf, true)
|
||||||
|
t.Fatalf("nebula did NOT close within 10s of Stop(): a blocking reader never woke\n%s", buf[:n])
|
||||||
|
}
|
||||||
|
}
|
||||||
+168
-167
@@ -4,12 +4,13 @@
|
|||||||
package udp
|
package udp
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
|
"sync/atomic"
|
||||||
"syscall"
|
"syscall"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
@@ -19,58 +20,51 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type StdConn struct {
|
type StdConn struct {
|
||||||
udpConn *net.UDPConn
|
sysFd int
|
||||||
rawConn syscall.RawConn
|
closed atomic.Bool
|
||||||
isV4 bool
|
isV4 bool
|
||||||
l *slog.Logger
|
l *slog.Logger
|
||||||
batch int
|
batch int
|
||||||
}
|
|
||||||
|
|
||||||
func setReusePort(network, address string, c syscall.RawConn) error {
|
|
||||||
var opErr error
|
|
||||||
err := c.Control(func(fd uintptr) {
|
|
||||||
opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
|
|
||||||
//CloseOnExec already set by the runtime
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return opErr
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
|
func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
|
||||||
listen := netip.AddrPortFrom(ip, uint16(port))
|
af := unix.AF_INET6
|
||||||
lc := net.ListenConfig{}
|
if ip.Is4() {
|
||||||
|
af = unix.AF_INET
|
||||||
|
}
|
||||||
|
syscall.ForkLock.RLock()
|
||||||
|
fd, err := unix.Socket(af, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
|
||||||
|
if err == nil {
|
||||||
|
unix.CloseOnExec(fd)
|
||||||
|
}
|
||||||
|
syscall.ForkLock.RUnlock()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("unable to open socket: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
if multi {
|
if multi {
|
||||||
lc.Control = setReusePort
|
if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
|
||||||
}
|
_ = unix.Close(fd)
|
||||||
//this context is only used during the bind operation, you can't cancel it to kill the socket
|
return nil, fmt.Errorf("unable to set SO_REUSEPORT: %w", err)
|
||||||
pc, err := lc.ListenPacket(context.Background(), "udp", listen.String())
|
}
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("unable to open socket: %s", err)
|
|
||||||
}
|
|
||||||
udpConn := pc.(*net.UDPConn)
|
|
||||||
rawConn, err := udpConn.SyscallConn()
|
|
||||||
if err != nil {
|
|
||||||
_ = udpConn.Close()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
//gotta find out if we got an AF_INET6 socket or not:
|
|
||||||
out := &StdConn{
|
|
||||||
udpConn: udpConn,
|
|
||||||
rawConn: rawConn,
|
|
||||||
l: l,
|
|
||||||
batch: batch,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
af, err := out.getSockOptInt(unix.SO_DOMAIN)
|
var sa unix.Sockaddr
|
||||||
if err != nil {
|
if ip.Is4() {
|
||||||
_ = out.Close()
|
sa4 := &unix.SockaddrInet4{Port: port}
|
||||||
return nil, err
|
sa4.Addr = ip.As4()
|
||||||
|
sa = sa4
|
||||||
|
} else {
|
||||||
|
sa6 := &unix.SockaddrInet6{Port: port}
|
||||||
|
sa6.Addr = ip.As16()
|
||||||
|
sa = sa6
|
||||||
|
}
|
||||||
|
if err = unix.Bind(fd, sa); err != nil {
|
||||||
|
_ = unix.Close(fd)
|
||||||
|
return nil, fmt.Errorf("unable to bind to socket: %w", err)
|
||||||
}
|
}
|
||||||
out.isV4 = af == unix.AF_INET
|
|
||||||
|
|
||||||
return out, nil
|
return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) SupportsMultipleReaders() bool {
|
func (u *StdConn) SupportsMultipleReaders() bool {
|
||||||
@@ -81,134 +75,111 @@ func (u *StdConn) Rebind() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) getSockOptInt(opt int) (int, error) {
|
|
||||||
if u.rawConn == nil {
|
|
||||||
return 0, fmt.Errorf("no UDP connection")
|
|
||||||
}
|
|
||||||
var out int
|
|
||||||
var opErr error
|
|
||||||
err := u.rawConn.Control(func(fd uintptr) {
|
|
||||||
out, opErr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, opt)
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return out, opErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *StdConn) setSockOptInt(opt int, n int) error {
|
|
||||||
if u.rawConn == nil {
|
|
||||||
return fmt.Errorf("no UDP connection")
|
|
||||||
}
|
|
||||||
var opErr error
|
|
||||||
err := u.rawConn.Control(func(fd uintptr) {
|
|
||||||
opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, opt, n)
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return opErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *StdConn) SetRecvBuffer(n int) error {
|
func (u *StdConn) SetRecvBuffer(n int) error {
|
||||||
return u.setSockOptInt(unix.SO_RCVBUFFORCE, n)
|
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) SetSendBuffer(n int) error {
|
func (u *StdConn) SetSendBuffer(n int) error {
|
||||||
return u.setSockOptInt(unix.SO_SNDBUFFORCE, n)
|
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) SetSoMark(mark int) error {
|
func (u *StdConn) SetSoMark(mark int) error {
|
||||||
return u.setSockOptInt(unix.SO_MARK, mark)
|
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK, mark)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) GetRecvBuffer() (int, error) {
|
func (u *StdConn) GetRecvBuffer() (int, error) {
|
||||||
return u.getSockOptInt(unix.SO_RCVBUF)
|
return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUF)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) GetSendBuffer() (int, error) {
|
func (u *StdConn) GetSendBuffer() (int, error) {
|
||||||
return u.getSockOptInt(unix.SO_SNDBUF)
|
return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) GetSoMark() (int, error) {
|
func (u *StdConn) GetSoMark() (int, error) {
|
||||||
return u.getSockOptInt(unix.SO_MARK)
|
return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
|
func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
|
||||||
a := u.udpConn.LocalAddr()
|
sa, err := unix.Getsockname(u.sysFd)
|
||||||
|
if err != nil {
|
||||||
switch v := a.(type) {
|
return netip.AddrPort{}, err
|
||||||
case *net.UDPAddr:
|
}
|
||||||
addr, ok := netip.AddrFromSlice(v.IP)
|
switch sa := sa.(type) {
|
||||||
if !ok {
|
case *unix.SockaddrInet4:
|
||||||
return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP)
|
return netip.AddrPortFrom(netip.AddrFrom4(sa.Addr), uint16(sa.Port)), nil
|
||||||
}
|
case *unix.SockaddrInet6:
|
||||||
return netip.AddrPortFrom(addr, uint16(v.Port)), nil
|
return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr), uint16(sa.Port)), nil
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a)
|
return netip.AddrPort{}, fmt.Errorf("unsupported sock type: %T", sa)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func recvmmsg(fd uintptr, msgs []rawMessage) (int, bool, error) {
|
// recvmmsg does one blocking recvmmsg (MSG_WAITFORONE), reading up to len(msgs) datagrams
|
||||||
var errno syscall.Errno
|
func (u *StdConn) recvmmsg(msgs []rawMessage) (int, error) {
|
||||||
n, _, errno := unix.Syscall6(
|
r, _, errno := unix.Syscall6(
|
||||||
unix.SYS_RECVMMSG,
|
unix.SYS_RECVMMSG,
|
||||||
fd,
|
uintptr(u.sysFd),
|
||||||
uintptr(unsafe.Pointer(&msgs[0])),
|
uintptr(unsafe.Pointer(&msgs[0])),
|
||||||
uintptr(len(msgs)),
|
uintptr(len(msgs)),
|
||||||
unix.MSG_WAITFORONE,
|
unix.MSG_WAITFORONE,
|
||||||
0,
|
0,
|
||||||
0,
|
0,
|
||||||
)
|
)
|
||||||
if errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK {
|
|
||||||
// No data available, block for I/O and try again.
|
|
||||||
return int(n), false, nil
|
|
||||||
}
|
|
||||||
if errno != 0 {
|
if errno != 0 {
|
||||||
return int(n), true, &net.OpError{Op: "recvmmsg", Err: errno}
|
if u.closed.Load() {
|
||||||
}
|
return 0, net.ErrClosed
|
||||||
return int(n), true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *StdConn) listenOutSingle(r EncReader) error {
|
|
||||||
var err error
|
|
||||||
var n int
|
|
||||||
var from netip.AddrPort
|
|
||||||
buffer := make([]byte, MTU)
|
|
||||||
|
|
||||||
for {
|
|
||||||
n, from, err = u.udpConn.ReadFromUDPAddrPort(buffer)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port())
|
return 0, &net.OpError{Op: "recvmmsg", Err: errno}
|
||||||
r(from, buffer[:n])
|
|
||||||
}
|
}
|
||||||
|
n := int(r)
|
||||||
|
if (n == 0 || msgs[0].Len == 0) && u.closed.Load() {
|
||||||
|
return 0, net.ErrClosed
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) listenOutBatch(r EncReader) error {
|
// recvmsg does one blocking recvmsg into msgs[0]
|
||||||
|
func (u *StdConn) recvmsg(msgs []rawMessage) (int, error) {
|
||||||
|
r, _, errno := unix.Syscall6(
|
||||||
|
unix.SYS_RECVMSG,
|
||||||
|
uintptr(u.sysFd),
|
||||||
|
uintptr(unsafe.Pointer(&msgs[0].Hdr)),
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
if errno != 0 {
|
||||||
|
if u.closed.Load() {
|
||||||
|
return 0, net.ErrClosed
|
||||||
|
}
|
||||||
|
return 0, &net.OpError{Op: "recvmsg", Err: errno}
|
||||||
|
}
|
||||||
|
if r == 0 && u.closed.Load() {
|
||||||
|
return 0, net.ErrClosed
|
||||||
|
}
|
||||||
|
msgs[0].Len = uint32(r)
|
||||||
|
return 1, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *StdConn) ListenOut(r EncReader) error {
|
||||||
var ip netip.Addr
|
var ip netip.Addr
|
||||||
var n int
|
|
||||||
var operr error
|
|
||||||
|
|
||||||
msgs, buffers, names := u.PrepareRawMessages(u.batch)
|
msgs, buffers, names := u.PrepareRawMessages(u.batch)
|
||||||
|
read := u.recvmmsg
|
||||||
//reader needs to capture variables from this function, since it's used as a lambda with rawConn.Read
|
if u.batch == 1 {
|
||||||
//defining it outside the loop so it gets re-used
|
read = u.recvmsg
|
||||||
reader := func(fd uintptr) (done bool) {
|
|
||||||
n, done, operr = recvmmsg(fd, msgs)
|
|
||||||
return done
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for {
|
for {
|
||||||
err := u.rawConn.Read(reader)
|
n, err := read(msgs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, unix.EINTR) {
|
||||||
|
continue // interrupted by a signal, retry the read
|
||||||
|
}
|
||||||
|
// net.ErrClosed after Close() is teardown, absorbed by the caller's
|
||||||
|
// closed flag like the other platforms; anything else is a real error.
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if operr != nil {
|
|
||||||
return operr
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := 0; i < n; i++ {
|
for i := 0; i < n; i++ {
|
||||||
// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
|
// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
|
||||||
@@ -222,26 +193,68 @@ func (u *StdConn) listenOutBatch(r EncReader) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) ListenOut(r EncReader) error {
|
func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
|
||||||
if u.batch == 1 {
|
if u.isV4 {
|
||||||
return u.listenOutSingle(r)
|
return u.writeTo4(b, ip)
|
||||||
} else {
|
}
|
||||||
return u.listenOutBatch(r)
|
return u.writeTo6(b, ip)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
|
||||||
|
var rsa unix.RawSockaddrInet6
|
||||||
|
rsa.Family = unix.AF_INET6
|
||||||
|
rsa.Addr = ip.Addr().As16()
|
||||||
|
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
|
||||||
|
|
||||||
|
for {
|
||||||
|
_, _, err := unix.Syscall6(
|
||||||
|
unix.SYS_SENDTO,
|
||||||
|
uintptr(u.sysFd),
|
||||||
|
uintptr(unsafe.Pointer(&b[0])),
|
||||||
|
uintptr(len(b)),
|
||||||
|
uintptr(0),
|
||||||
|
uintptr(unsafe.Pointer(&rsa)),
|
||||||
|
uintptr(unix.SizeofSockaddrInet6),
|
||||||
|
)
|
||||||
|
if err != 0 {
|
||||||
|
return &net.OpError{Op: "sendto", Err: err}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
|
func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
|
||||||
_, err := u.udpConn.WriteToUDPAddrPort(b, ip)
|
if !ip.Addr().Is4() {
|
||||||
return err
|
return ErrInvalidIPv6RemoteForSocket
|
||||||
|
}
|
||||||
|
|
||||||
|
var rsa unix.RawSockaddrInet4
|
||||||
|
rsa.Family = unix.AF_INET
|
||||||
|
rsa.Addr = ip.Addr().As4()
|
||||||
|
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
|
||||||
|
|
||||||
|
for {
|
||||||
|
_, _, err := unix.Syscall6(
|
||||||
|
unix.SYS_SENDTO,
|
||||||
|
uintptr(u.sysFd),
|
||||||
|
uintptr(unsafe.Pointer(&b[0])),
|
||||||
|
uintptr(len(b)),
|
||||||
|
uintptr(0),
|
||||||
|
uintptr(unsafe.Pointer(&rsa)),
|
||||||
|
uintptr(unix.SizeofSockaddrInet4),
|
||||||
|
)
|
||||||
|
if err != 0 {
|
||||||
|
return &net.OpError{Op: "sendto", Err: err}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) ReloadConfig(c *config.C) {
|
func (u *StdConn) ReloadConfig(c *config.C) {
|
||||||
b := c.GetInt("listen.read_buffer", 0)
|
b := c.GetInt("listen.read_buffer", 0)
|
||||||
if b > 0 {
|
if b > 0 {
|
||||||
err := u.SetRecvBuffer(b)
|
if err := u.SetRecvBuffer(b); err == nil {
|
||||||
if err == nil {
|
if s, err := u.GetRecvBuffer(); err == nil {
|
||||||
s, err := u.GetRecvBuffer()
|
|
||||||
if err == nil {
|
|
||||||
u.l.Info("listen.read_buffer was set", "size", s)
|
u.l.Info("listen.read_buffer was set", "size", s)
|
||||||
} else {
|
} else {
|
||||||
u.l.Warn("Failed to get listen.read_buffer", "error", err)
|
u.l.Warn("Failed to get listen.read_buffer", "error", err)
|
||||||
@@ -253,10 +266,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
|
|||||||
|
|
||||||
b = c.GetInt("listen.write_buffer", 0)
|
b = c.GetInt("listen.write_buffer", 0)
|
||||||
if b > 0 {
|
if b > 0 {
|
||||||
err := u.SetSendBuffer(b)
|
if err := u.SetSendBuffer(b); err == nil {
|
||||||
if err == nil {
|
if s, err := u.GetSendBuffer(); err == nil {
|
||||||
s, err := u.GetSendBuffer()
|
|
||||||
if err == nil {
|
|
||||||
u.l.Info("listen.write_buffer was set", "size", s)
|
u.l.Info("listen.write_buffer was set", "size", s)
|
||||||
} else {
|
} else {
|
||||||
u.l.Warn("Failed to get listen.write_buffer", "error", err)
|
u.l.Warn("Failed to get listen.write_buffer", "error", err)
|
||||||
@@ -269,10 +280,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
|
|||||||
b = c.GetInt("listen.so_mark", 0)
|
b = c.GetInt("listen.so_mark", 0)
|
||||||
s, err := u.GetSoMark()
|
s, err := u.GetSoMark()
|
||||||
if b > 0 || (err == nil && s != 0) {
|
if b > 0 || (err == nil && s != 0) {
|
||||||
err := u.SetSoMark(b)
|
if err := u.SetSoMark(b); err == nil {
|
||||||
if err == nil {
|
if s, err := u.GetSoMark(); err == nil {
|
||||||
s, err := u.GetSoMark()
|
|
||||||
if err == nil {
|
|
||||||
u.l.Info("listen.so_mark was set", "mark", s)
|
u.l.Info("listen.so_mark was set", "mark", s)
|
||||||
} else {
|
} else {
|
||||||
u.l.Warn("Failed to get listen.so_mark", "error", err)
|
u.l.Warn("Failed to get listen.so_mark", "error", err)
|
||||||
@@ -285,28 +294,20 @@ func (u *StdConn) ReloadConfig(c *config.C) {
|
|||||||
|
|
||||||
func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
|
func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
|
||||||
var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
|
var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
|
||||||
|
_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
|
||||||
if u.rawConn == nil {
|
if err != 0 {
|
||||||
return fmt.Errorf("no UDP connection")
|
|
||||||
}
|
|
||||||
var opErr error
|
|
||||||
err := u.rawConn.Control(func(fd uintptr) {
|
|
||||||
_, _, syserr := unix.Syscall6(unix.SYS_GETSOCKOPT, fd, uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
|
|
||||||
if syserr != 0 {
|
|
||||||
opErr = syserr
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return opErr
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *StdConn) Close() error {
|
func (u *StdConn) Close() error {
|
||||||
if u.udpConn != nil {
|
u.closed.Store(true)
|
||||||
return u.udpConn.Close()
|
// Wake the reader parked in recvmmsg/recvmsg. shutdown(2) on an unconnected socket
|
||||||
}
|
// returns ENOTCONN but still wakes it, so ignore the error.
|
||||||
return nil
|
// The reader then sees closed and stops touching the fd, making the Close below safe.
|
||||||
|
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
|
||||||
|
return unix.Close(u.sysFd)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewUDPStatsEmitter(udpConns []Conn) func() {
|
func NewUDPStatsEmitter(udpConns []Conn) func() {
|
||||||
|
|||||||
@@ -0,0 +1,179 @@
|
|||||||
|
//go:build linux && !android && !e2e_testing
|
||||||
|
|
||||||
|
package udp
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"net/netip"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
func testLogger() *slog.Logger {
|
||||||
|
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestShutdownWakesAfterRx_Mechanism exercises the kernel quirk our teardown
|
||||||
|
// relies on: once a socket has received a packet, shutdown(2) wakes a blocked
|
||||||
|
// recvmmsg with n>=1/Len==0 (not n==0). recvmmsg must turn that into net.ErrClosed
|
||||||
|
// once Close set closed, so a parked reader exits instead of spinning.
|
||||||
|
func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
|
||||||
|
c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, 64)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewListener: %v", err)
|
||||||
|
}
|
||||||
|
sc := c.(*StdConn)
|
||||||
|
addr, err := sc.LocalAddr()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LocalAddr: %v", err)
|
||||||
|
}
|
||||||
|
msgs, _, _ := sc.PrepareRawMessages(sc.batch)
|
||||||
|
|
||||||
|
// Receive a real packet so the socket has carried data.
|
||||||
|
send, err := net.Dial("udp", addr.String())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dial: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := send.Write([]byte("hello")); err != nil {
|
||||||
|
t.Fatalf("write: %v", err)
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
n, err := sc.recvmmsg(msgs)
|
||||||
|
t.Logf("drain of real packet: n=%d err=%v msgs[0].Len=%d", n, err, msgs[0].Len)
|
||||||
|
_ = send.Close()
|
||||||
|
|
||||||
|
// Block a reader on the now-empty queue, then tear down as Close() does.
|
||||||
|
// recvmmsg must return net.ErrClosed (not hang, not spin) even post-rx.
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := sc.recvmmsg(msgs)
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
time.Sleep(150 * time.Millisecond) // let it park in recvmmsg
|
||||||
|
|
||||||
|
sc.closed.Store(true)
|
||||||
|
if serr := unix.Shutdown(sc.sysFd, unix.SHUT_RDWR); serr != nil {
|
||||||
|
t.Logf("shutdown returned %v (expected ENOTCONN on unconnected UDP)", serr)
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if !errors.Is(err, net.ErrClosed) {
|
||||||
|
t.Errorf("recvmmsg after post-rx shutdown returned %v, want net.ErrClosed", err)
|
||||||
|
}
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatalf("HANG: recvmmsg did not return after shutdown following a received packet")
|
||||||
|
}
|
||||||
|
_ = unix.Close(sc.sysFd)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
|
||||||
|
// reader must tear down cleanly on Close() regardless of what the socket has
|
||||||
|
// carried. The three cases the report called out:
|
||||||
|
//
|
||||||
|
// no traffic ever -> works (shutdown wakes recvmmsg with n==0)
|
||||||
|
// ping once, then idle -> historically HUNG: once the socket has received a
|
||||||
|
// packet, shutdown(2) wakes recvmmsg with n>=1/Len==0,
|
||||||
|
// which an n==0-only teardown check misses
|
||||||
|
// continuous traffic -> works (a real packet is always arriving)
|
||||||
|
//
|
||||||
|
// All three must return within the deadline; a hang dumps goroutines so the
|
||||||
|
// stuck reader is visible.
|
||||||
|
func TestListenOutTeardown_TrafficPatterns(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
traffic func(send net.Conn, stop <-chan struct{})
|
||||||
|
}{
|
||||||
|
{"no_traffic_ever", func(net.Conn, <-chan struct{}) {}},
|
||||||
|
{"ping_once_then_idle", func(send net.Conn, _ <-chan struct{}) {
|
||||||
|
_, _ = send.Write([]byte("hello"))
|
||||||
|
}},
|
||||||
|
{"continuous", func(send net.Conn, stop <-chan struct{}) {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
_, _ = send.Write([]byte("hello"))
|
||||||
|
time.Sleep(2 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
// batch 1 exercises the recvmsg path, batch 64 the recvmmsg path; both must
|
||||||
|
// tear down cleanly.
|
||||||
|
for _, batch := range []int{1, 64} {
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(fmt.Sprintf("batch%d/%s", batch, tc.name), func(t *testing.T) {
|
||||||
|
runTeardownCase(t, batch, tc.name, tc.traffic)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net.Conn, stop <-chan struct{})) {
|
||||||
|
c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, batch)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewListener: %v", err)
|
||||||
|
}
|
||||||
|
sc := c.(*StdConn)
|
||||||
|
addr, err := sc.LocalAddr()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LocalAddr: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var received atomic.Int64
|
||||||
|
loopDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
|
||||||
|
received.Add(1)
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
|
||||||
|
send, err := net.Dial("udp", addr.String())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dial: %v", err)
|
||||||
|
}
|
||||||
|
defer send.Close()
|
||||||
|
|
||||||
|
stop := make(chan struct{})
|
||||||
|
trafficDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
traffic(send, stop)
|
||||||
|
close(trafficDone)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Let the pattern run and, for the idle case, the reader park again on an
|
||||||
|
// empty queue with the socket already having received a packet.
|
||||||
|
time.Sleep(500 * time.Millisecond)
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
if err := sc.Close(); err != nil {
|
||||||
|
t.Fatalf("Close: %v", err)
|
||||||
|
}
|
||||||
|
close(stop)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-loopDone:
|
||||||
|
// Clean teardown surfaces as net.ErrClosed (propagated like the other
|
||||||
|
// platforms); the caller absorbs it via its closed flag.
|
||||||
|
if err != nil && !errors.Is(err, net.ErrClosed) {
|
||||||
|
t.Fatalf("%s: ListenOut returned unexpected error on teardown: %v", name, err)
|
||||||
|
}
|
||||||
|
t.Logf("%s: closed in %v (received %d packets)", name, time.Since(start), received.Load())
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
buf := make([]byte, 1<<20)
|
||||||
|
n := runtime.Stack(buf, true)
|
||||||
|
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
|
||||||
|
}
|
||||||
|
<-trafficDone
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user