use wg tun library; batching & locking improvements

udp/conn.go

@@ -13,13 +13,21 @@ type EncReader func(
 	payload []byte,
 )
 
+type EncBatchReader func(
+	addrs []netip.AddrPort,
+	payloads [][]byte,
+	count int,
+)
+
 type Conn interface {
 	Rebind() error
 	LocalAddr() (netip.AddrPort, error)
-	ListenOut(r EncReader)
+	ListenOutBatch(r EncBatchReader)
 	WriteTo(b []byte, addr netip.AddrPort) error
+	WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error)
 	ReloadConfig(c *config.C)
 	SupportsMultipleReaders() bool
+	BatchSize() int
 	Close() error
 }
 
@@ -31,17 +39,25 @@ func (NoopConn) Rebind() error {
 func (NoopConn) LocalAddr() (netip.AddrPort, error) {
 	return netip.AddrPort{}, nil
 }
-func (NoopConn) ListenOut(_ EncReader) {
-	return
-}
+
+func (NoopConn) ListenOut(_ EncReader) {}
+
 func (NoopConn) SupportsMultipleReaders() bool {
 	return false
 }
+
+func (NoopConn) ListenOutBatch(_ EncBatchReader) {}
+
 func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
 	return nil
 }
-func (NoopConn) ReloadConfig(_ *config.C) {
-	return
+func (NoopConn) WriteMulti(_ [][]byte, _ []netip.AddrPort) (int, error) {
+	return 0, nil
+}
+func (NoopConn) ReloadConfig(_ *config.C) {}
+
+func (NoopConn) BatchSize() int {
+	return 1
 }
 func (NoopConn) Close() error {
 	return nil

udp/udp_darwin.go

@@ -140,6 +140,17 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
 	}
 }
 
+// WriteMulti sends multiple packets - fallback implementation without sendmmsg
+func (u *StdConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
+	for i := range packets {
+		err := u.WriteTo(packets[i], addrs[i])
+		if err != nil {
+			return i, err
+		}
+	}
+	return len(packets), nil
+}
+
 func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
 	a := u.UDPConn.LocalAddr()
 
@@ -188,6 +199,34 @@ func (u *StdConn) SupportsMultipleReaders() bool {
 	return false
 }
 
+// ListenOutBatch - fallback to single-packet reads for Darwin
+func (u *StdConn) ListenOutBatch(r EncBatchReader) {
+	buffer := make([]byte, MTU)
+	addrs := make([]netip.AddrPort, 1)
+	payloads := make([][]byte, 1)
+
+	for {
+		// Just read one packet at a time and call batch callback with count=1
+		n, rua, err := u.ReadFromUDPAddrPort(buffer)
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
+				u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+				return
+			}
+
+			u.l.WithError(err).Error("unexpected udp socket receive error")
+		}
+
+		addrs[0] = netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port())
+		payloads[0] = buffer[:n]
+		r(addrs, payloads, 1)
+	}
+}
+
+func (u *StdConn) BatchSize() int {
+	return 1
+}
+
 func (u *StdConn) Rebind() error {
 	var err error
 	if u.isV4 {

udp/udp_generic.go

@@ -101,3 +101,38 @@ func (u *GenericConn) ListenOut(r EncReader) {
 func (u *GenericConn) SupportsMultipleReaders() bool {
 	return false
 }
+
+// ListenOutBatch - fallback to single-packet reads for generic platforms
+func (u *GenericConn) ListenOutBatch(r EncBatchReader) {
+	buffer := make([]byte, MTU)
+	addrs := make([]netip.AddrPort, 1)
+	payloads := make([][]byte, 1)
+
+	for {
+		// Just read one packet at a time and call batch callback with count=1
+		n, rua, err := u.ReadFromUDPAddrPort(buffer)
+		if err != nil {
+			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+			return
+		}
+
+		addrs[0] = netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port())
+		payloads[0] = buffer[:n]
+		r(addrs, payloads, 1)
+	}
+}
+
+// WriteMulti sends multiple packets - fallback implementation
+func (u *GenericConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
+	for i := range packets {
+		err := u.WriteTo(packets[i], addrs[i])
+		if err != nil {
+			return i, err
+		}
+	}
+	return len(packets), nil
+}
+
+func (u *GenericConn) BatchSize() int {
+	return 1
+}

udp/udp_linux.go

@@ -22,6 +22,11 @@ type StdConn struct {
 	isV4  bool
 	l     *logrus.Logger
 	batch int
+
+	// Pre-allocated buffers for batch writes (sized for IPv6, works for both)
+	writeMsgs   []rawMessage
+	writeIovecs []iovec
+	writeNames  [][]byte
 }
 
 func maybeIPV4(ip net.IP) (net.IP, bool) {
@@ -69,7 +74,26 @@ func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch in
 		return nil, fmt.Errorf("unable to bind to socket: %s", err)
 	}
 
-	return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, err
+	c := &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}
+
+	// Pre-allocate write message structures for batching (sized for IPv6, works for both)
+	c.writeMsgs = make([]rawMessage, batch)
+	c.writeIovecs = make([]iovec, batch)
+	c.writeNames = make([][]byte, batch)
+
+	for i := range c.writeMsgs {
+		// Allocate for IPv6 size (larger than IPv4, works for both)
+		c.writeNames[i] = make([]byte, unix.SizeofSockaddrInet6)
+
+		// Point to the iovec in the slice
+		c.writeMsgs[i].Hdr.Iov = &c.writeIovecs[i]
+		c.writeMsgs[i].Hdr.Iovlen = 1
+
+		c.writeMsgs[i].Hdr.Name = &c.writeNames[i][0]
+		// Namelen will be set appropriately in writeMulti4/writeMulti6
+	}
+
+	return c, err
 }
 
 func (u *StdConn) SupportsMultipleReaders() bool {
@@ -122,7 +146,7 @@ func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
 	}
 }
 
-func (u *StdConn) ListenOut(r EncReader) {
+func (u *StdConn) ListenOutBatch(r EncBatchReader) {
 	var ip netip.Addr
 
 	msgs, buffers, names := u.PrepareRawMessages(u.batch)
@@ -131,6 +155,12 @@ func (u *StdConn) ListenOut(r EncReader) {
 		read = u.ReadSingle
 	}
 
+	udpBatchHist := metrics.GetOrRegisterHistogram("batch.udp_read_size", nil, metrics.NewUniformSample(1024))
+
+	// Pre-allocate slices for batch callback
+	addrs := make([]netip.AddrPort, u.batch)
+	payloads := make([][]byte, u.batch)
+
 	for {
 		n, err := read(msgs)
 		if err != nil {
@@ -138,15 +168,21 @@ func (u *StdConn) ListenOut(r EncReader) {
 			return
 		}
 
+		udpBatchHist.Update(int64(n))
+
+		// Prepare batch data
 		for i := 0; i < n; i++ {
-			// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
 			if u.isV4 {
 				ip, _ = netip.AddrFromSlice(names[i][4:8])
 			} else {
 				ip, _ = netip.AddrFromSlice(names[i][8:24])
 			}
-			r(netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4])), buffers[i][:msgs[i].Len])
+			addrs[i] = netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4]))
+			payloads[i] = buffers[i][:msgs[i].Len]
 		}
+
+		// Call batch callback with all packets
+		r(addrs, payloads, n)
 	}
 }
 
@@ -198,6 +234,19 @@ func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
 	return u.writeTo6(b, ip)
 }
 
+func (u *StdConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
+	if len(packets) != len(addrs) {
+		return 0, fmt.Errorf("packets and addrs length mismatch")
+	}
+	if len(packets) == 0 {
+		return 0, nil
+	}
+	if u.isV4 {
+		return u.writeMulti4(packets, addrs)
+	}
+	return u.writeMulti6(packets, addrs)
+}
+
 func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
 	var rsa unix.RawSockaddrInet6
 	rsa.Family = unix.AF_INET6
@@ -252,6 +301,123 @@ func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
 	}
 }
 
+func (u *StdConn) writeMulti4(packets [][]byte, addrs []netip.AddrPort) (int, error) {
+	sent := 0
+	for sent < len(packets) {
+		// Determine batch size based on remaining packets and buffer capacity
+		batchSize := len(packets) - sent
+		if batchSize > len(u.writeMsgs) {
+			batchSize = len(u.writeMsgs)
+		}
+
+		// Use pre-allocated buffers
+		msgs := u.writeMsgs[:batchSize]
+		iovecs := u.writeIovecs[:batchSize]
+		names := u.writeNames[:batchSize]
+
+		// Setup message structures for this batch
+		for i := 0; i < batchSize; i++ {
+			pktIdx := sent + i
+			if !addrs[pktIdx].Addr().Is4() {
+				return sent + i, ErrInvalidIPv6RemoteForSocket
+			}
+
+			// Setup the packet buffer
+			iovecs[i].Base = &packets[pktIdx][0]
+			iovecs[i].Len = uint(len(packets[pktIdx]))
+
+			// Setup the destination address
+			rsa := (*unix.RawSockaddrInet4)(unsafe.Pointer(&names[i][0]))
+			rsa.Family = unix.AF_INET
+			rsa.Addr = addrs[pktIdx].Addr().As4()
+			binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], addrs[pktIdx].Port())
+
+			// Set the appropriate address length for IPv4
+			msgs[i].Hdr.Namelen = unix.SizeofSockaddrInet4
+		}
+
+		// Send this batch
+		nsent, _, err := unix.Syscall6(
+			unix.SYS_SENDMMSG,
+			uintptr(u.sysFd),
+			uintptr(unsafe.Pointer(&msgs[0])),
+			uintptr(batchSize),
+			0,
+			0,
+			0,
+		)
+
+		if err != 0 {
+			return sent + int(nsent), &net.OpError{Op: "sendmmsg", Err: err}
+		}
+
+		sent += int(nsent)
+		if int(nsent) < batchSize {
+			// Couldn't send all packets in batch, return what we sent
+			return sent, nil
+		}
+	}
+
+	return sent, nil
+}
+
+func (u *StdConn) writeMulti6(packets [][]byte, addrs []netip.AddrPort) (int, error) {
+	sent := 0
+	for sent < len(packets) {
+		// Determine batch size based on remaining packets and buffer capacity
+		batchSize := len(packets) - sent
+		if batchSize > len(u.writeMsgs) {
+			batchSize = len(u.writeMsgs)
+		}
+
+		// Use pre-allocated buffers
+		msgs := u.writeMsgs[:batchSize]
+		iovecs := u.writeIovecs[:batchSize]
+		names := u.writeNames[:batchSize]
+
+		// Setup message structures for this batch
+		for i := 0; i < batchSize; i++ {
+			pktIdx := sent + i
+
+			// Setup the packet buffer
+			iovecs[i].Base = &packets[pktIdx][0]
+			iovecs[i].Len = uint(len(packets[pktIdx]))
+
+			// Setup the destination address
+			rsa := (*unix.RawSockaddrInet6)(unsafe.Pointer(&names[i][0]))
+			rsa.Family = unix.AF_INET6
+			rsa.Addr = addrs[pktIdx].Addr().As16()
+			binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], addrs[pktIdx].Port())
+
+			// Set the appropriate address length for IPv6
+			msgs[i].Hdr.Namelen = unix.SizeofSockaddrInet6
+		}
+
+		// Send this batch
+		nsent, _, err := unix.Syscall6(
+			unix.SYS_SENDMMSG,
+			uintptr(u.sysFd),
+			uintptr(unsafe.Pointer(&msgs[0])),
+			uintptr(batchSize),
+			0,
+			0,
+			0,
+		)
+
+		if err != 0 {
+			return sent + int(nsent), &net.OpError{Op: "sendmmsg", Err: err}
+		}
+
+		sent += int(nsent)
+		if int(nsent) < batchSize {
+			// Couldn't send all packets in batch, return what we sent
+			return sent, nil
+		}
+	}
+
+	return sent, nil
+}
+
 func (u *StdConn) ReloadConfig(c *config.C) {
 	b := c.GetInt("listen.read_buffer", 0)
 	if b > 0 {
@@ -309,6 +475,10 @@ func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
 	return nil
 }
 
+func (u *StdConn) BatchSize() int {
+	return u.batch
+}
+
 func (u *StdConn) Close() error {
 	return syscall.Close(u.sysFd)
 }

udp/udp_linux_32.go

@@ -12,7 +12,7 @@ import (
 
 type iovec struct {
 	Base *byte
-	Len  uint32
+	Len  uint
 }
 
 type msghdr struct {
@@ -40,7 +40,7 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
 		names[i] = make([]byte, unix.SizeofSockaddrInet6)
 
 		vs := []iovec{
-			{Base: &buffers[i][0], Len: uint32(len(buffers[i]))},
+			{Base: &buffers[i][0], Len: uint(len(buffers[i]))},
 		}
 
 		msgs[i].Hdr.Iov = &vs[0]

udp/udp_linux_64.go

@@ -12,7 +12,7 @@ import (
 
 type iovec struct {
 	Base *byte
-	Len  uint64
+	Len  uint
 }
 
 type msghdr struct {
@@ -43,7 +43,7 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) {
 		names[i] = make([]byte, unix.SizeofSockaddrInet6)
 
 		vs := []iovec{
-			{Base: &buffers[i][0], Len: uint64(len(buffers[i]))},
+			{Base: &buffers[i][0], Len: uint(len(buffers[i]))},
 		}
 
 		msgs[i].Hdr.Iov = &vs[0]

udp/udp_rio_windows.go

@@ -338,6 +338,50 @@ func (u *RIOConn) Rebind() error {
 
 func (u *RIOConn) ReloadConfig(*config.C) {}
 
+// BatchSize returns 1 since RIO reads packets one at a time
+func (u *RIOConn) BatchSize() int {
+	return 1
+}
+
+// ListenOutBatch - fallback to single-packet reads for RIO
+func (u *RIOConn) ListenOutBatch(r EncBatchReader) {
+	buffer := make([]byte, MTU)
+	addrs := make([]netip.AddrPort, 1)
+	payloads := make([][]byte, 1)
+
+	var lastRecvErr time.Time
+
+	for {
+		n, rua, err := u.receive(buffer)
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
+				u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+				return
+			}
+			if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute {
+				lastRecvErr = time.Now()
+				u.l.WithError(err).Warn("unexpected udp socket receive error")
+			}
+			continue
+		}
+
+		addrs[0] = netip.AddrPortFrom(netip.AddrFrom16(rua.Addr).Unmap(), (rua.Port>>8)|((rua.Port&0xff)<<8))
+		payloads[0] = buffer[:n]
+		r(addrs, payloads, 1)
+	}
+}
+
+// WriteMulti sends multiple packets - fallback implementation
+func (u *RIOConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
+	for i := range packets {
+		err := u.WriteTo(packets[i], addrs[i])
+		if err != nil {
+			return i, err
+		}
+	}
+	return len(packets), nil
+}
+
 func (u *RIOConn) Close() error {
 	if !u.isOpen.CompareAndSwap(true, false) {
 		return nil

udp/udp_tester.go

@@ -116,6 +116,31 @@ func (u *TesterConn) ListenOut(r EncReader) {
 	}
 }
 
+func (u *TesterConn) ListenOutBatch(r EncBatchReader) {
+	addrs := make([]netip.AddrPort, 1)
+	payloads := make([][]byte, 1)
+
+	for {
+		p, ok := <-u.RxPackets
+		if !ok {
+			return
+		}
+		addrs[0] = p.From
+		payloads[0] = p.Data
+		r(addrs, payloads, 1)
+	}
+}
+
+func (u *TesterConn) WriteMulti(packets [][]byte, addrs []netip.AddrPort) (int, error) {
+	for i := range packets {
+		err := u.WriteTo(packets[i], addrs[i])
+		if err != nil {
+			return i, err
+		}
+	}
+	return len(packets), nil
+}
+
 func (u *TesterConn) ReloadConfig(*config.C) {}
 
 func NewUDPStatsEmitter(_ []Conn) func() {
@@ -131,6 +156,10 @@ func (u *TesterConn) SupportsMultipleReaders() bool {
 	return false
 }
 
+func (u *TesterConn) BatchSize() int {
+	return 1
+}
+
 func (u *TesterConn) Rebind() error {
 	return nil
 }