gso and gro with uring on send/receive for udp

udp/udp_linux_32.go

@@ -7,6 +7,9 @@
 package udp
 
 import (
+	"errors"
+	"fmt"
+
 	"golang.org/x/sys/unix"
 )
 
@@ -30,17 +33,29 @@ type rawMessage struct {
 	Len uint32
 }
 
-func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
+func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte, [][]byte) {
+	controlLen := int(u.controlLen.Load())
+
 	msgs := make([]rawMessage, n)
 	buffers := make([][]byte, n)
 	names := make([][]byte, n)
 
+	var controls [][]byte
+	if controlLen > 0 {
+		controls = make([][]byte, n)
+	}
+
 	for i := range msgs {
-		buffers[i] = make([]byte, MTU)
+		size := int(u.groBufSize.Load())
+		if size < MTU {
+			size = MTU
+		}
+		buf := u.borrowRxBuffer(size)
+		buffers[i] = buf
 		names[i] = make([]byte, unix.SizeofSockaddrInet6)
 
 		vs := []iovec{
-			{Base: &buffers[i][0], Len: uint32(len(buffers[i]))},
+			{Base: &buf[0], Len: uint32(len(buf))},
 		}
 
 		msgs[i].Hdr.Iov = &vs[0]
@@ -48,7 +63,71 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
 
 		msgs[i].Hdr.Name = &names[i][0]
 		msgs[i].Hdr.Namelen = uint32(len(names[i]))
+
+		if controlLen > 0 {
+			controls[i] = make([]byte, controlLen)
+			msgs[i].Hdr.Control = &controls[i][0]
+			msgs[i].Hdr.Controllen = controllen(len(controls[i]))
+		} else {
+			msgs[i].Hdr.Control = nil
+			msgs[i].Hdr.Controllen = controllen(0)
+		}
 	}
 
-	return msgs, buffers, names
+	return msgs, buffers, names, controls
+}
+
+func setIovecBase(msg *rawMessage, buf []byte) {
+	iov := (*iovec)(msg.Hdr.Iov)
+	iov.Base = &buf[0]
+	iov.Len = uint32(len(buf))
+}
+
+func rawMessageToUnixMsghdr(msg *rawMessage) (unix.Msghdr, unix.Iovec, error) {
+	var hdr unix.Msghdr
+	var iov unix.Iovec
+	if msg == nil {
+		return hdr, iov, errors.New("nil rawMessage")
+	}
+	if msg.Hdr.Iov == nil || msg.Hdr.Iov.Base == nil {
+		return hdr, iov, errors.New("rawMessage missing payload buffer")
+	}
+	payloadLen := int(msg.Hdr.Iov.Len)
+	if payloadLen < 0 {
+		return hdr, iov, fmt.Errorf("invalid payload length: %d", payloadLen)
+	}
+	iov.Base = msg.Hdr.Iov.Base
+	iov.Len = uint32(payloadLen)
+	hdr.Iov = &iov
+	hdr.Iovlen = 1
+	hdr.Name = msg.Hdr.Name
+	// CRITICAL: Always set to full buffer size for receive, not what kernel wrote last time
+	if hdr.Name != nil {
+		hdr.Namelen = uint32(unix.SizeofSockaddrInet6)
+	} else {
+		hdr.Namelen = 0
+	}
+	hdr.Control = msg.Hdr.Control
+	// CRITICAL: Use the allocated size, not what was previously returned
+	if hdr.Control != nil {
+		// Control buffer size is stored in Controllen from PrepareRawMessages
+		hdr.Controllen = msg.Hdr.Controllen
+	} else {
+		hdr.Controllen = 0
+	}
+	hdr.Flags = 0 // Reset flags for new receive
+	return hdr, iov, nil
+}
+
+func updateRawMessageFromUnixMsghdr(msg *rawMessage, hdr *unix.Msghdr, n int) {
+	if msg == nil || hdr == nil {
+		return
+	}
+	msg.Hdr.Namelen = hdr.Namelen
+	msg.Hdr.Controllen = hdr.Controllen
+	msg.Hdr.Flags = hdr.Flags
+	if n < 0 {
+		n = 0
+	}
+	msg.Len = uint32(n)
 }