pull deps in for optimization, maybe slice back out later

overlay/virtqueue/available_ring.go

@@ -0,0 +1,103 @@
+package virtqueue
+
+import (
+	"fmt"
+	"sync"
+	"unsafe"
+)
+
+// availableRingFlag is a flag that describes an [AvailableRing].
+type availableRingFlag uint16
+
+const (
+	// availableRingFlagNoInterrupt is used by the guest to advise the host to
+	// not interrupt it when consuming a buffer. It's unreliable, so it's simply
+	// an optimization.
+	availableRingFlagNoInterrupt availableRingFlag = 1 << iota
+)
+
+// availableRingSize is the number of bytes needed to store an [AvailableRing]
+// with the given queue size in memory.
+func availableRingSize(queueSize int) int {
+	return 6 + 2*queueSize
+}
+
+// availableRingAlignment is the minimum alignment of an [AvailableRing]
+// in memory, as required by the virtio spec.
+const availableRingAlignment = 2
+
+// AvailableRing is used by the driver to offer descriptor chains to the device.
+// Each ring entry refers to the head of a descriptor chain. It is only written
+// to by the driver and read by the device.
+//
+// Because the size of the ring depends on the queue size, we cannot define a
+// Go struct with a static size that maps to the memory of the ring. Instead,
+// this struct only contains pointers to the corresponding memory areas.
+type AvailableRing struct {
+	initialized bool
+
+	// flags that describe this ring.
+	flags *availableRingFlag
+	// ringIndex indicates where the driver would put the next entry into the
+	// ring (modulo the queue size).
+	ringIndex *uint16
+	// ring references buffers using the index of the head of the descriptor
+	// chain in the [DescriptorTable]. It wraps around at queue size.
+	ring []uint16
+	// usedEvent is not used by this implementation, but we reserve it anyway to
+	// avoid issues in case a device may try to access it, contrary to the
+	// virtio specification.
+	usedEvent *uint16
+
+	mu sync.Mutex
+}
+
+// newAvailableRing creates an available ring that uses the given underlying
+// memory. The length of the memory slice must match the size needed for the
+// ring (see [availableRingSize]) for the given queue size.
+func newAvailableRing(queueSize int, mem []byte) *AvailableRing {
+	ringSize := availableRingSize(queueSize)
+	if len(mem) != ringSize {
+		panic(fmt.Sprintf("memory size (%v) does not match required size "+
+			"for available ring: %v", len(mem), ringSize))
+	}
+
+	return &AvailableRing{
+		initialized: true,
+		flags:       (*availableRingFlag)(unsafe.Pointer(&mem[0])),
+		ringIndex:   (*uint16)(unsafe.Pointer(&mem[2])),
+		ring:        unsafe.Slice((*uint16)(unsafe.Pointer(&mem[4])), queueSize),
+		usedEvent:   (*uint16)(unsafe.Pointer(&mem[ringSize-2])),
+	}
+}
+
+// Address returns the pointer to the beginning of the ring in memory.
+// Do not modify the memory directly to not interfere with this implementation.
+func (r *AvailableRing) Address() uintptr {
+	if !r.initialized {
+		panic("available ring is not initialized")
+	}
+	return uintptr(unsafe.Pointer(r.flags))
+}
+
+// offer adds the given descriptor chain heads to the available ring and
+// advances the ring index accordingly to make the device process the new
+// descriptor chains.
+func (r *AvailableRing) offer(chainHeads []uint16) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	// Add descriptor chain heads to the ring.
+	for offset, head := range chainHeads {
+		// The 16-bit ring index may overflow. This is expected and is not an
+		// issue because the size of the ring array (which equals the queue
+		// size) is always a power of 2 and smaller than the highest possible
+		// 16-bit value.
+		insertIndex := int(*r.ringIndex+uint16(offset)) % len(r.ring)
+		r.ring[insertIndex] = head
+	}
+
+	// Increase the ring index by the number of descriptor chains added to the
+	// ring.
+	*r.ringIndex += uint16(len(chainHeads))
+}

overlay/virtqueue/available_ring_internal_test.go

@@ -0,0 +1,71 @@
+package virtqueue
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAvailableRing_MemoryLayout(t *testing.T) {
+	const queueSize = 2
+
+	memory := make([]byte, availableRingSize(queueSize))
+	r := newAvailableRing(queueSize, memory)
+
+	*r.flags = 0x01ff
+	*r.ringIndex = 1
+	r.ring[0] = 0x1234
+	r.ring[1] = 0x5678
+
+	assert.Equal(t, []byte{
+		0xff, 0x01,
+		0x01, 0x00,
+		0x34, 0x12,
+		0x78, 0x56,
+		0x00, 0x00,
+	}, memory)
+}
+
+func TestAvailableRing_Offer(t *testing.T) {
+	const queueSize = 8
+
+	chainHeads := []uint16{42, 33, 69}
+
+	tests := []struct {
+		name              string
+		startRingIndex    uint16
+		expectedRingIndex uint16
+		expectedRing      []uint16
+	}{
+		{
+			name:              "no overflow",
+			startRingIndex:    0,
+			expectedRingIndex: 3,
+			expectedRing:      []uint16{42, 33, 69, 0, 0, 0, 0, 0},
+		},
+		{
+			name:              "ring overflow",
+			startRingIndex:    6,
+			expectedRingIndex: 9,
+			expectedRing:      []uint16{69, 0, 0, 0, 0, 0, 42, 33},
+		},
+		{
+			name:              "index overflow",
+			startRingIndex:    65535,
+			expectedRingIndex: 2,
+			expectedRing:      []uint16{33, 69, 0, 0, 0, 0, 0, 42},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			memory := make([]byte, availableRingSize(queueSize))
+			r := newAvailableRing(queueSize, memory)
+			*r.ringIndex = tt.startRingIndex
+
+			r.offer(chainHeads)
+
+			assert.Equal(t, tt.expectedRingIndex, *r.ringIndex)
+			assert.Equal(t, tt.expectedRing, r.ring)
+		})
+	}
+}

overlay/virtqueue/descriptor.go

@@ -0,0 +1,43 @@
+package virtqueue
+
+// descriptorFlag is a flag that describes a [Descriptor].
+type descriptorFlag uint16
+
+const (
+	// descriptorFlagHasNext marks a descriptor chain as continuing via the next
+	// field.
+	descriptorFlagHasNext descriptorFlag = 1 << iota
+	// descriptorFlagWritable marks a buffer as device write-only (otherwise
+	// device read-only).
+	descriptorFlagWritable
+	// descriptorFlagIndirect means the buffer contains a list of buffer
+	// descriptors to provide an additional layer of indirection.
+	// Only allowed when the [virtio.FeatureIndirectDescriptors] feature was
+	// negotiated.
+	descriptorFlagIndirect
+)
+
+// descriptorSize is the number of bytes needed to store a [Descriptor] in
+// memory.
+const descriptorSize = 16
+
+// Descriptor describes (a part of) a buffer which is either read-only for the
+// device or write-only for the device (depending on [descriptorFlagWritable]).
+// Multiple descriptors can be chained to produce a "descriptor chain" that can
+// contain both device-readable and device-writable buffers. Device-readable
+// descriptors always come first in a chain. A single, large buffer may be
+// split up by chaining multiple similar descriptors that reference different
+// memory pages. This is required, because buffers may exceed a single page size
+// and the memory accessed by the device is expected to be continuous.
+type Descriptor struct {
+	// address is the address to the continuous memory holding the data for this
+	// descriptor.
+	address uintptr
+	// length is the amount of bytes stored at address.
+	length uint32
+	// flags that describe this descriptor.
+	flags descriptorFlag
+	// next contains the index of the next descriptor continuing this descriptor
+	// chain when the [descriptorFlagHasNext] flag is set.
+	next uint16
+}

overlay/virtqueue/descriptor_internal_test.go

@@ -0,0 +1,12 @@
+package virtqueue
+
+import (
+	"testing"
+	"unsafe"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDescriptor_Size(t *testing.T) {
+	assert.EqualValues(t, descriptorSize, unsafe.Sizeof(Descriptor{}))
+}

overlay/virtqueue/descriptor_table.go

@@ -0,0 +1,437 @@
+package virtqueue
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"os"
+	"sync"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+var (
+	// ErrDescriptorChainEmpty is returned when a descriptor chain would contain
+	// no buffers, which is not allowed.
+	ErrDescriptorChainEmpty = errors.New("empty descriptor chains are not allowed")
+
+	// ErrNotEnoughFreeDescriptors is returned when the free descriptors are
+	// exhausted, meaning that the queue is full.
+	ErrNotEnoughFreeDescriptors = errors.New("not enough free descriptors, queue is full")
+
+	// ErrInvalidDescriptorChain is returned when a descriptor chain is not
+	// valid for a given operation.
+	ErrInvalidDescriptorChain = errors.New("invalid descriptor chain")
+)
+
+// noFreeHead is used to mark when all descriptors are in use and we have no
+// free chain. This value is impossible to occur as an index naturally, because
+// it exceeds the maximum queue size.
+const noFreeHead = uint16(math.MaxUint16)
+
+// descriptorTableSize is the number of bytes needed to store a
+// [DescriptorTable] with the given queue size in memory.
+func descriptorTableSize(queueSize int) int {
+	return descriptorSize * queueSize
+}
+
+// descriptorTableAlignment is the minimum alignment of a [DescriptorTable]
+// in memory, as required by the virtio spec.
+const descriptorTableAlignment = 16
+
+// DescriptorTable is a table that holds [Descriptor]s, addressed via their
+// index in the slice.
+type DescriptorTable struct {
+	descriptors []Descriptor
+
+	// freeHeadIndex is the index of the head of the descriptor chain which
+	// contains all currently unused descriptors. When all descriptors are in
+	// use, this has the special value of noFreeHead.
+	freeHeadIndex uint16
+	// freeNum tracks the number of descriptors which are currently not in use.
+	freeNum uint16
+
+	mu sync.Mutex
+}
+
+// newDescriptorTable creates a descriptor table that uses the given underlying
+// memory. The Length of the memory slice must match the size needed for the
+// descriptor table (see [descriptorTableSize]) for the given queue size.
+//
+// Before this descriptor table can be used, [initialize] must be called.
+func newDescriptorTable(queueSize int, mem []byte) *DescriptorTable {
+	dtSize := descriptorTableSize(queueSize)
+	if len(mem) != dtSize {
+		panic(fmt.Sprintf("memory size (%v) does not match required size "+
+			"for descriptor table: %v", len(mem), dtSize))
+	}
+
+	return &DescriptorTable{
+		descriptors: unsafe.Slice((*Descriptor)(unsafe.Pointer(&mem[0])), queueSize),
+		// We have no free descriptors until they were initialized.
+		freeHeadIndex: noFreeHead,
+		freeNum:       0,
+	}
+}
+
+// Address returns the pointer to the beginning of the descriptor table in
+// memory. Do not modify the memory directly to not interfere with this
+// implementation.
+func (dt *DescriptorTable) Address() uintptr {
+	if dt.descriptors == nil {
+		panic("descriptor table is not initialized")
+	}
+	return uintptr(unsafe.Pointer(&dt.descriptors[0]))
+}
+
+// BufferAddresses returns pointers to all memory pages used by the descriptor
+// table to store buffers, independent of whether descriptors are currently in
+// use or not. These pointers can be helpful to set up memory mappings. Do not
+// use them to access or modify the memory in any way.
+// Each pointer points to a whole memory page. Use [os.Getpagesize] to get the
+// page size.
+func (dt *DescriptorTable) BufferAddresses() []uintptr {
+	if dt.descriptors == nil {
+		panic("descriptor table is not initialized")
+	}
+
+	dt.mu.Lock()
+	defer dt.mu.Unlock()
+
+	ptrs := make([]uintptr, len(dt.descriptors))
+	for i, desc := range dt.descriptors {
+		ptrs[i] = desc.address
+	}
+
+	return ptrs
+}
+
+// initializeDescriptors allocates buffers with the size of a full memory page
+// for each descriptor in the table. While this may be a bit wasteful, it makes
+// dealing with descriptors way easier. Without this preallocation, we would
+// have to allocate and free memory on demand, increasing complexity.
+//
+// All descriptors will be marked as free and will form a free chain. The
+// addresses of all descriptors will be populated while their length remains
+// zero.
+func (dt *DescriptorTable) initializeDescriptors() error {
+	pageSize := os.Getpagesize()
+
+	dt.mu.Lock()
+	defer dt.mu.Unlock()
+
+	for i := range dt.descriptors {
+		// Allocate a full memory page for this descriptor.
+		pagePtr, err := unix.MmapPtr(-1, 0, nil, uintptr(pageSize),
+			unix.PROT_READ|unix.PROT_WRITE,
+			unix.MAP_PRIVATE|unix.MAP_ANONYMOUS)
+		if err != nil {
+			return fmt.Errorf("allocate page for descriptor %d: %w", i, err)
+		}
+
+		dt.descriptors[i] = Descriptor{
+			address: uintptr(pagePtr),
+			length:  0,
+			// All descriptors should form a free chain that loops around.
+			flags: descriptorFlagHasNext,
+			next:  uint16((i + 1) % len(dt.descriptors)),
+		}
+	}
+
+	// All descriptors are free to use now.
+	dt.freeHeadIndex = 0
+	dt.freeNum = uint16(len(dt.descriptors))
+
+	return nil
+}
+
+// releaseBuffers releases all allocated buffers for this descriptor table.
+// The implementation will try to release as many buffers as possible and
+// collect potential errors before returning them.
+// The descriptor table should no longer be used after calling this.
+func (dt *DescriptorTable) releaseBuffers() error {
+	dt.mu.Lock()
+	defer dt.mu.Unlock()
+
+	var errs []error
+	pageSize := os.Getpagesize()
+	for i := range dt.descriptors {
+		descriptor := &dt.descriptors[i]
+		if descriptor.address == 0 {
+			continue
+		}
+
+		// The pointer points to memory not managed by Go, so this conversion
+		// is safe. See https://github.com/golang/go/issues/58625
+		//goland:noinspection GoVetUnsafePointer
+		err := unix.MunmapPtr(unsafe.Pointer(descriptor.address), uintptr(pageSize))
+		if err == nil {
+			descriptor.address = 0
+		} else {
+			errs = append(errs, fmt.Errorf("release page for descriptor %d: %w", i, err))
+		}
+	}
+
+	// As a safety measure, make sure no descriptors can be used anymore.
+	dt.freeHeadIndex = noFreeHead
+	dt.freeNum = 0
+
+	return errors.Join(errs...)
+}
+
+// createDescriptorChain creates a new descriptor chain within the descriptor
+// table which contains a number of device-readable buffers (out buffers) and
+// device-writable buffers (in buffers).
+//
+// All buffers in the outBuffers slice will be concatenated by chaining
+// descriptors, one for each buffer in the slice. The size of the single buffers
+// must not exceed the size of a memory page (see [os.Getpagesize]).
+// When numInBuffers is greater than zero, the given number of device-writable
+// descriptors will be appended to the end of the chain, each referencing a
+// whole memory page.
+//
+// The index of the head of the new descriptor chain will be returned. Callers
+// should make sure to free the descriptor chain using [freeDescriptorChain]
+// after it was used by the device.
+//
+// When there are not enough free descriptors to hold the given number of
+// buffers, an [ErrNotEnoughFreeDescriptors] will be returned. In this case, the
+// caller should try again after some descriptor chains were used by the device
+// and returned back into the free chain.
+func (dt *DescriptorTable) createDescriptorChain(outBuffers [][]byte, numInBuffers int) (uint16, error) {
+	pageSize := os.Getpagesize()
+
+	// Calculate the number of descriptors needed to build the chain.
+	numDesc := uint16(len(outBuffers) + numInBuffers)
+
+	// Descriptor chains must always contain at least one descriptor.
+	if numDesc < 1 {
+		return 0, ErrDescriptorChainEmpty
+	}
+
+	dt.mu.Lock()
+	defer dt.mu.Unlock()
+
+	// Do we still have enough free descriptors?
+	if numDesc > dt.freeNum {
+		return 0, fmt.Errorf("%w: %d free but needed %d", ErrNotEnoughFreeDescriptors, dt.freeNum, numDesc)
+	}
+
+	// Above validation ensured that there is at least one free descriptor, so
+	// the free descriptor chain head should be valid.
+	if dt.freeHeadIndex == noFreeHead {
+		panic("free descriptor chain head is unset but there should be free descriptors")
+	}
+
+	// To avoid having to iterate over the whole table to find the descriptor
+	// pointing to the head just to replace the free head, we instead always
+	// create descriptor chains from the descriptors coming after the head.
+	// This way we only have to touch the head as a last resort, when all other
+	// descriptors are already used.
+	head := dt.descriptors[dt.freeHeadIndex].next
+	next := head
+	tail := head
+	for i, buffer := range outBuffers {
+		desc := &dt.descriptors[next]
+		checkUnusedDescriptorLength(next, desc)
+
+		if len(buffer) > pageSize {
+			// The caller should already prevent that from happening.
+			panic(fmt.Sprintf("out buffer %d has size %d which exceeds page size %d", i, len(buffer), pageSize))
+		}
+
+		// Copy the buffer to the memory referenced by the descriptor.
+		// The descriptor address points to memory not managed by Go, so this
+		// conversion is safe. See https://github.com/golang/go/issues/58625
+		//goland:noinspection GoVetUnsafePointer
+		copy(unsafe.Slice((*byte)(unsafe.Pointer(desc.address)), pageSize), buffer)
+		desc.length = uint32(len(buffer))
+
+		// Clear the flags in case there were any others set.
+		desc.flags = descriptorFlagHasNext
+
+		tail = next
+		next = desc.next
+	}
+	for range numInBuffers {
+		desc := &dt.descriptors[next]
+		checkUnusedDescriptorLength(next, desc)
+
+		// Give the device the maximum available number of bytes to write into.
+		desc.length = uint32(pageSize)
+
+		// Mark the descriptor as device-writable.
+		desc.flags = descriptorFlagHasNext | descriptorFlagWritable
+
+		tail = next
+		next = desc.next
+	}
+
+	// The last descriptor should end the chain.
+	tailDesc := &dt.descriptors[tail]
+	tailDesc.flags &= ^descriptorFlagHasNext
+	tailDesc.next = 0 // Not necessary to clear this, it's just for looks.
+
+	dt.freeNum -= numDesc
+
+	if dt.freeNum == 0 {
+		// The last descriptor in the chain should be the free chain head
+		// itself.
+		if tail != dt.freeHeadIndex {
+			panic("descriptor chain takes up all free descriptors but does not end with the free chain head")
+		}
+
+		// When this new chain takes up all remaining descriptors, we no longer
+		// have a free chain.
+		dt.freeHeadIndex = noFreeHead
+	} else {
+		// We took some descriptors out of the free chain, so make sure to close
+		// the circle again.
+		dt.descriptors[dt.freeHeadIndex].next = next
+	}
+
+	return head, nil
+}
+
+// TODO: Implement a zero-copy variant of createDescriptorChain?
+
+// getDescriptorChain returns the device-readable buffers (out buffers) and
+// device-writable buffers (in buffers) of the descriptor chain that starts with
+// the given head index. The descriptor chain must have been created using
+// [createDescriptorChain] and must not have been freed yet (meaning that the
+// head index must not be contained in the free chain).
+//
+// Be careful to only access the returned buffer slices when the device has not
+// yet or is no longer using them. They must not be accessed after
+// [freeDescriptorChain] has been called.
+func (dt *DescriptorTable) getDescriptorChain(head uint16) (outBuffers, inBuffers [][]byte, err error) {
+	if int(head) > len(dt.descriptors) {
+		return nil, nil, fmt.Errorf("%w: index out of range", ErrInvalidDescriptorChain)
+	}
+
+	dt.mu.Lock()
+	defer dt.mu.Unlock()
+
+	// Iterate over the chain. The iteration is limited to the queue size to
+	// avoid ending up in an endless loop when things go very wrong.
+	next := head
+	for range len(dt.descriptors) {
+		if next == dt.freeHeadIndex {
+			return nil, nil, fmt.Errorf("%w: must not be part of the free chain", ErrInvalidDescriptorChain)
+		}
+
+		desc := &dt.descriptors[next]
+
+		// The descriptor address points to memory not managed by Go, so this
+		// conversion is safe. See https://github.com/golang/go/issues/58625
+		//goland:noinspection GoVetUnsafePointer
+		bs := unsafe.Slice((*byte)(unsafe.Pointer(desc.address)), desc.length)
+
+		if desc.flags&descriptorFlagWritable == 0 {
+			outBuffers = append(outBuffers, bs)
+		} else {
+			inBuffers = append(inBuffers, bs)
+		}
+
+		// Is this the tail of the chain?
+		if desc.flags&descriptorFlagHasNext == 0 {
+			break
+		}
+
+		// Detect loops.
+		if desc.next == head {
+			return nil, nil, fmt.Errorf("%w: contains a loop", ErrInvalidDescriptorChain)
+		}
+
+		next = desc.next
+	}
+
+	return
+}
+
+// freeDescriptorChain can be used to free a descriptor chain when it is no
+// longer in use. The descriptor chain that starts with the given index will be
+// put back into the free chain, so the descriptors can be used for later calls
+// of [createDescriptorChain].
+// The descriptor chain must have been created using [createDescriptorChain] and
+// must not have been freed yet (meaning that the head index must not be
+// contained in the free chain).
+func (dt *DescriptorTable) freeDescriptorChain(head uint16) error {
+	if int(head) > len(dt.descriptors) {
+		return fmt.Errorf("%w: index out of range", ErrInvalidDescriptorChain)
+	}
+
+	dt.mu.Lock()
+	defer dt.mu.Unlock()
+
+	// Iterate over the chain. The iteration is limited to the queue size to
+	// avoid ending up in an endless loop when things go very wrong.
+	next := head
+	var tailDesc *Descriptor
+	var chainLen uint16
+	for range len(dt.descriptors) {
+		if next == dt.freeHeadIndex {
+			return fmt.Errorf("%w: must not be part of the free chain", ErrInvalidDescriptorChain)
+		}
+
+		desc := &dt.descriptors[next]
+		chainLen++
+
+		// Set the length of all unused descriptors back to zero.
+		desc.length = 0
+
+		// Unset all flags except the next flag.
+		desc.flags &= descriptorFlagHasNext
+
+		// Is this the tail of the chain?
+		if desc.flags&descriptorFlagHasNext == 0 {
+			tailDesc = desc
+			break
+		}
+
+		// Detect loops.
+		if desc.next == head {
+			return fmt.Errorf("%w: contains a loop", ErrInvalidDescriptorChain)
+		}
+
+		next = desc.next
+	}
+	if tailDesc == nil {
+		// A descriptor chain longer than the queue size but without loops
+		// should be impossible.
+		panic(fmt.Sprintf("could not find a tail for descriptor chain starting at %d", head))
+	}
+
+	// The tail descriptor does not have the next flag set, but when it comes
+	// back into the free chain, it should have.
+	tailDesc.flags = descriptorFlagHasNext
+
+	if dt.freeHeadIndex == noFreeHead {
+		// The whole free chain was used up, so we turn this returned descriptor
+		// chain into the new free chain by completing the circle and using its
+		// head.
+		tailDesc.next = head
+		dt.freeHeadIndex = head
+	} else {
+		// Attach the returned chain at the beginning of the free chain but
+		// right after the free chain head.
+		freeHeadDesc := &dt.descriptors[dt.freeHeadIndex]
+		tailDesc.next = freeHeadDesc.next
+		freeHeadDesc.next = head
+	}
+
+	dt.freeNum += chainLen
+
+	return nil
+}
+
+// checkUnusedDescriptorLength asserts that the length of an unused descriptor
+// is zero, as it should be.
+// This is not a requirement by the virtio spec but rather a thing we do to
+// notice when our algorithm goes sideways.
+func checkUnusedDescriptorLength(index uint16, desc *Descriptor) {
+	if desc.length != 0 {
+		panic(fmt.Sprintf("descriptor %d should be unused but has a non-zero length", index))
+	}
+}

overlay/virtqueue/descriptor_table_internal_test.go

@@ -0,0 +1,407 @@
+package virtqueue
+
+import (
+	"os"
+	"testing"
+	"unsafe"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDescriptorTable_InitializeDescriptors(t *testing.T) {
+	const queueSize = 32
+
+	dt := DescriptorTable{
+		descriptors: make([]Descriptor, queueSize),
+	}
+
+	assert.NoError(t, dt.initializeDescriptors())
+	t.Cleanup(func() {
+		assert.NoError(t, dt.releaseBuffers())
+	})
+
+	for i, descriptor := range dt.descriptors {
+		assert.NotZero(t, descriptor.address)
+		assert.Zero(t, descriptor.length)
+		assert.EqualValues(t, descriptorFlagHasNext, descriptor.flags)
+		assert.EqualValues(t, (i+1)%queueSize, descriptor.next)
+	}
+}
+
+func TestDescriptorTable_DescriptorChains(t *testing.T) {
+	// Use a very short queue size to not make this test overly verbose.
+	const queueSize = 8
+
+	pageSize := os.Getpagesize()
+
+	// Initialize descriptor table.
+	dt := DescriptorTable{
+		descriptors: make([]Descriptor, queueSize),
+	}
+	assert.NoError(t, dt.initializeDescriptors())
+	t.Cleanup(func() {
+		assert.NoError(t, dt.releaseBuffers())
+	})
+
+	// Some utilities for easier checking if the descriptor table looks as
+	// expected.
+	type desc struct {
+		buffer []byte
+		flags  descriptorFlag
+		next   uint16
+	}
+	assertDescriptorTable := func(expected [queueSize]desc) {
+		for i := 0; i < queueSize; i++ {
+			actualDesc := &dt.descriptors[i]
+			expectedDesc := &expected[i]
+			assert.Equal(t, uint32(len(expectedDesc.buffer)), actualDesc.length)
+			if len(expectedDesc.buffer) > 0 {
+				//goland:noinspection GoVetUnsafePointer
+				assert.EqualValues(t,
+					unsafe.Slice((*byte)(unsafe.Pointer(actualDesc.address)), actualDesc.length),
+					expectedDesc.buffer)
+			}
+			assert.Equal(t, expectedDesc.flags, actualDesc.flags)
+			if expectedDesc.flags&descriptorFlagHasNext != 0 {
+				assert.Equal(t, expectedDesc.next, actualDesc.next)
+			}
+		}
+	}
+
+	// Initial state: All descriptors are in the free chain.
+	assert.Equal(t, uint16(0), dt.freeHeadIndex)
+	assert.Equal(t, uint16(8), dt.freeNum)
+	assertDescriptorTable([queueSize]desc{
+		{
+			// Free head.
+			flags: descriptorFlagHasNext,
+			next:  1,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  2,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  3,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  4,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  5,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  6,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  7,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  0,
+		},
+	})
+
+	// Create the first chain.
+	firstChain, err := dt.createDescriptorChain([][]byte{
+		makeTestBuffer(t, 26),
+		makeTestBuffer(t, 256),
+	}, 1)
+	assert.NoError(t, err)
+	assert.Equal(t, uint16(1), firstChain)
+
+	// Now there should be a new chain next to the free chain.
+	assert.Equal(t, uint16(0), dt.freeHeadIndex)
+	assert.Equal(t, uint16(5), dt.freeNum)
+	assertDescriptorTable([queueSize]desc{
+		{
+			// Free head.
+			flags: descriptorFlagHasNext,
+			next:  4,
+		},
+		{
+			// Head of first chain.
+			buffer: makeTestBuffer(t, 26),
+			flags:  descriptorFlagHasNext,
+			next:   2,
+		},
+		{
+			buffer: makeTestBuffer(t, 256),
+			flags:  descriptorFlagHasNext,
+			next:   3,
+		},
+		{
+			// Tail of first chain.
+			buffer: make([]byte, pageSize),
+			flags:  descriptorFlagWritable,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  5,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  6,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  7,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  0,
+		},
+	})
+
+	// Create a second chain with only a single in buffer.
+	secondChain, err := dt.createDescriptorChain(nil, 1)
+	assert.NoError(t, err)
+	assert.Equal(t, uint16(4), secondChain)
+
+	// Now there should be two chains next to the free chain.
+	assert.Equal(t, uint16(0), dt.freeHeadIndex)
+	assert.Equal(t, uint16(4), dt.freeNum)
+	assertDescriptorTable([queueSize]desc{
+		{
+			// Free head.
+			flags: descriptorFlagHasNext,
+			next:  5,
+		},
+		{
+			// Head of the first chain.
+			buffer: makeTestBuffer(t, 26),
+			flags:  descriptorFlagHasNext,
+			next:   2,
+		},
+		{
+			buffer: makeTestBuffer(t, 256),
+			flags:  descriptorFlagHasNext,
+			next:   3,
+		},
+		{
+			// Tail of the first chain.
+			buffer: make([]byte, pageSize),
+			flags:  descriptorFlagWritable,
+		},
+		{
+			// Head and tail of the second chain.
+			buffer: make([]byte, pageSize),
+			flags:  descriptorFlagWritable,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  6,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  7,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  0,
+		},
+	})
+
+	// Create a third chain taking up all remaining descriptors.
+	thirdChain, err := dt.createDescriptorChain([][]byte{
+		makeTestBuffer(t, 42),
+		makeTestBuffer(t, 96),
+		makeTestBuffer(t, 33),
+		makeTestBuffer(t, 222),
+	}, 0)
+	assert.NoError(t, err)
+	assert.Equal(t, uint16(5), thirdChain)
+
+	// Now there should be three chains and no free chain.
+	assert.Equal(t, noFreeHead, dt.freeHeadIndex)
+	assert.Equal(t, uint16(0), dt.freeNum)
+	assertDescriptorTable([queueSize]desc{
+		{
+			// Tail of the third chain.
+			buffer: makeTestBuffer(t, 222),
+		},
+		{
+			// Head of the first chain.
+			buffer: makeTestBuffer(t, 26),
+			flags:  descriptorFlagHasNext,
+			next:   2,
+		},
+		{
+			buffer: makeTestBuffer(t, 256),
+			flags:  descriptorFlagHasNext,
+			next:   3,
+		},
+		{
+			// Tail of the first chain.
+			buffer: make([]byte, pageSize),
+			flags:  descriptorFlagWritable,
+		},
+		{
+			// Head and tail of the second chain.
+			buffer: make([]byte, pageSize),
+			flags:  descriptorFlagWritable,
+		},
+		{
+			// Head of the third chain.
+			buffer: makeTestBuffer(t, 42),
+			flags:  descriptorFlagHasNext,
+			next:   6,
+		},
+		{
+			buffer: makeTestBuffer(t, 96),
+			flags:  descriptorFlagHasNext,
+			next:   7,
+		},
+		{
+			buffer: makeTestBuffer(t, 33),
+			flags:  descriptorFlagHasNext,
+			next:   0,
+		},
+	})
+
+	// Free the third chain.
+	assert.NoError(t, dt.freeDescriptorChain(thirdChain))
+
+	// Now there should be two chains and a free chain again.
+	assert.Equal(t, uint16(5), dt.freeHeadIndex)
+	assert.Equal(t, uint16(4), dt.freeNum)
+	assertDescriptorTable([queueSize]desc{
+		{
+			flags: descriptorFlagHasNext,
+			next:  5,
+		},
+		{
+			// Head of the first chain.
+			buffer: makeTestBuffer(t, 26),
+			flags:  descriptorFlagHasNext,
+			next:   2,
+		},
+		{
+			buffer: makeTestBuffer(t, 256),
+			flags:  descriptorFlagHasNext,
+			next:   3,
+		},
+		{
+			// Tail of the first chain.
+			buffer: make([]byte, pageSize),
+			flags:  descriptorFlagWritable,
+		},
+		{
+			// Head and tail of the second chain.
+			buffer: make([]byte, pageSize),
+			flags:  descriptorFlagWritable,
+		},
+		{
+			// Free head.
+			flags: descriptorFlagHasNext,
+			next:  6,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  7,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  0,
+		},
+	})
+
+	// Free the first chain.
+	assert.NoError(t, dt.freeDescriptorChain(firstChain))
+
+	// Now there should be only a single chain next to the free chain.
+	assert.Equal(t, uint16(5), dt.freeHeadIndex)
+	assert.Equal(t, uint16(7), dt.freeNum)
+	assertDescriptorTable([queueSize]desc{
+		{
+			flags: descriptorFlagHasNext,
+			next:  5,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  2,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  3,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  6,
+		},
+		{
+			// Head and tail of the second chain.
+			buffer: make([]byte, pageSize),
+			flags:  descriptorFlagWritable,
+		},
+		{
+			// Free head.
+			flags: descriptorFlagHasNext,
+			next:  1,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  7,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  0,
+		},
+	})
+
+	// Free the second chain.
+	assert.NoError(t, dt.freeDescriptorChain(secondChain))
+
+	// Now all descriptors should be in the free chain again.
+	assert.Equal(t, uint16(5), dt.freeHeadIndex)
+	assert.Equal(t, uint16(8), dt.freeNum)
+	assertDescriptorTable([queueSize]desc{
+		{
+			flags: descriptorFlagHasNext,
+			next:  5,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  2,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  3,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  6,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  1,
+		},
+		{
+			// Free head.
+			flags: descriptorFlagHasNext,
+			next:  4,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  7,
+		},
+		{
+			flags: descriptorFlagHasNext,
+			next:  0,
+		},
+	})
+}
+
+func makeTestBuffer(t *testing.T, length int) []byte {
+	t.Helper()
+	buf := make([]byte, length)
+	for i := 0; i < length; i++ {
+		buf[i] = byte(length - i)
+	}
+	return buf
+}

overlay/virtqueue/doc.go

@@ -0,0 +1,7 @@
+// Package virtqueue implements the driver-side for a virtio queue as described
+// in the specification:
+// https://docs.oasis-open.org/virtio/virtio/v1.2/csd01/virtio-v1.2-csd01.html#x1-270006
+// This package does not make assumptions about the device that consumes the
+// queue. It rather just allocates the queue structures in memory and provides
+// methods to interact with it.
+package virtqueue

overlay/virtqueue/eventfd_test.go

@@ -0,0 +1,45 @@
+package virtqueue
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gvisor.dev/gvisor/pkg/eventfd"
+)
+
+// Tests how an eventfd and a waiting goroutine can be gracefully closed.
+// Extends the eventfd test suite:
+// https://github.com/google/gvisor/blob/0799336d64be65eb97d330606c30162dc3440cab/pkg/eventfd/eventfd_test.go
+func TestEventFD_CancelWait(t *testing.T) {
+	efd, err := eventfd.Create()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		assert.NoError(t, efd.Close())
+	})
+
+	var stop bool
+
+	done := make(chan struct{})
+	go func() {
+		for !stop {
+			_ = efd.Wait()
+		}
+		close(done)
+	}()
+	select {
+	case <-done:
+		t.Fatalf("goroutine ended early")
+	case <-time.After(500 * time.Millisecond):
+	}
+
+	stop = true
+	assert.NoError(t, efd.Notify())
+	select {
+	case <-done:
+		break
+	case <-time.After(5 * time.Second):
+		t.Error("goroutine did not end")
+	}
+}

overlay/virtqueue/size.go

@@ -0,0 +1,33 @@
+package virtqueue
+
+import (
+	"errors"
+	"fmt"
+)
+
+// ErrQueueSizeInvalid is returned when a queue size is invalid.
+var ErrQueueSizeInvalid = errors.New("queue size is invalid")
+
+// CheckQueueSize checks if the given value would be a valid size for a
+// virtqueue and returns an [ErrQueueSizeInvalid], if not.
+func CheckQueueSize(queueSize int) error {
+	if queueSize <= 0 {
+		return fmt.Errorf("%w: %d is too small", ErrQueueSizeInvalid, queueSize)
+	}
+
+	// The queue size must always be a power of 2.
+	// This ensures that ring indexes wrap correctly when the 16-bit integers
+	// overflow.
+	if queueSize&(queueSize-1) != 0 {
+		return fmt.Errorf("%w: %d is not a power of 2", ErrQueueSizeInvalid, queueSize)
+	}
+
+	// The largest power of 2 that fits into a 16-bit integer is 32768.
+	// 2 * 32768 would be 65536 which no longer fits.
+	if queueSize > 32768 {
+		return fmt.Errorf("%w: %d is larger than the maximum possible queue size 32768",
+			ErrQueueSizeInvalid, queueSize)
+	}
+
+	return nil
+}

overlay/virtqueue/size_test.go

@@ -0,0 +1,59 @@
+package virtqueue
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCheckQueueSize(t *testing.T) {
+	tests := []struct {
+		name        string
+		queueSize   int
+		containsErr string
+	}{
+		{
+			name:        "negative",
+			queueSize:   -1,
+			containsErr: "too small",
+		},
+		{
+			name:        "zero",
+			queueSize:   0,
+			containsErr: "too small",
+		},
+		{
+			name:        "not a power of 2",
+			queueSize:   24,
+			containsErr: "not a power of 2",
+		},
+		{
+			name:        "too large",
+			queueSize:   65536,
+			containsErr: "larger than the maximum",
+		},
+		{
+			name:      "valid 1",
+			queueSize: 1,
+		},
+		{
+			name:      "valid 256",
+			queueSize: 256,
+		},
+
+		{
+			name:      "valid 32768",
+			queueSize: 32768,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := CheckQueueSize(tt.queueSize)
+			if tt.containsErr != "" {
+				assert.ErrorContains(t, err, tt.containsErr)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

overlay/virtqueue/split_virtqueue.go

@@ -0,0 +1,427 @@
+package virtqueue
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"sync"
+
+	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/eventfd"
+)
+
+// SplitQueue is a virtqueue that consists of several parts, where each part is
+// writeable by either the driver or the device, but not both.
+type SplitQueue struct {
+	// size is the size of the queue.
+	size int
+	// buf is the underlying memory used for the queue.
+	buf []byte
+
+	descriptorTable *DescriptorTable
+	availableRing   *AvailableRing
+	usedRing        *UsedRing
+
+	// kickEventFD is used to signal the device when descriptor chains were
+	// added to the available ring.
+	kickEventFD eventfd.Eventfd
+	// callEventFD is used by the device to signal when it has used descriptor
+	// chains and put them in the used ring.
+	callEventFD eventfd.Eventfd
+
+	// usedChains is a chanel that receives [UsedElement]s for descriptor chains
+	// that were used by the device.
+	usedChains chan UsedElement
+
+	// moreFreeDescriptors is a channel that signals when any descriptors were
+	// put back into the free chain of the descriptor table. This is used to
+	// unblock methods waiting for available room in the queue to create new
+	// descriptor chains again.
+	moreFreeDescriptors chan struct{}
+
+	// stop is used by [SplitQueue.Close] to cancel the goroutine that handles
+	// used buffer notifications. It blocks until the goroutine ended.
+	stop func() error
+
+	// offerMutex is used to synchronize calls to
+	// [SplitQueue.OfferDescriptorChain].
+	offerMutex sync.Mutex
+}
+
+// NewSplitQueue allocates a new [SplitQueue] in memory. The given queue size
+// specifies the number of entries/buffers the queue can hold. This also affects
+// the memory consumption.
+func NewSplitQueue(queueSize int) (_ *SplitQueue, err error) {
+	if err = CheckQueueSize(queueSize); err != nil {
+		return nil, err
+	}
+
+	sq := SplitQueue{
+		size: queueSize,
+	}
+
+	// Clean up a partially initialized queue when something fails.
+	defer func() {
+		if err != nil {
+			_ = sq.Close()
+		}
+	}()
+
+	// There are multiple ways for how the memory for the virtqueue could be
+	// allocated. We could use Go native structs with arrays inside them, but
+	// this wouldn't allow us to make the queue size configurable. And including
+	// a slice in the Go structs wouldn't work, because this would just put the
+	// Go slice descriptor into the memory region which the virtio device will
+	// not understand.
+	// Additionally, Go does not allow us to ensure a correct alignment of the
+	// parts of the virtqueue, as it is required by the virtio specification.
+	//
+	// To resolve this, let's just allocate the memory manually by allocating
+	// one or more memory pages, depending on the queue size. Making the
+	// virtqueue start at the beginning of a page is not strictly necessary, as
+	// the virtio specification does not require it to be continuous in the
+	// physical memory of the host (e.g. the vhost implementation in the kernel
+	// always uses copy_from_user to access it), but this makes it very easy to
+	// guarantee the alignment. Also, it is not required for the virtqueue parts
+	// to be in the same memory region, as we pass separate pointers to them to
+	// the device, but this design just makes things easier to implement.
+	//
+	// One added benefit of allocating the memory manually is, that we have full
+	// control over its lifetime and don't risk the garbage collector to collect
+	// our valuable structures while the device still works with them.
+
+	// The descriptor table is at the start of the page, so alignment is not an
+	// issue here.
+	descriptorTableStart := 0
+	descriptorTableEnd := descriptorTableStart + descriptorTableSize(queueSize)
+	availableRingStart := align(descriptorTableEnd, availableRingAlignment)
+	availableRingEnd := availableRingStart + availableRingSize(queueSize)
+	usedRingStart := align(availableRingEnd, usedRingAlignment)
+	usedRingEnd := usedRingStart + usedRingSize(queueSize)
+
+	sq.buf, err = unix.Mmap(-1, 0, usedRingEnd,
+		unix.PROT_READ|unix.PROT_WRITE,
+		unix.MAP_PRIVATE|unix.MAP_ANONYMOUS)
+	if err != nil {
+		return nil, fmt.Errorf("allocate virtqueue buffer: %w", err)
+	}
+
+	sq.descriptorTable = newDescriptorTable(queueSize, sq.buf[descriptorTableStart:descriptorTableEnd])
+	sq.availableRing = newAvailableRing(queueSize, sq.buf[availableRingStart:availableRingEnd])
+	sq.usedRing = newUsedRing(queueSize, sq.buf[usedRingStart:usedRingEnd])
+
+	sq.kickEventFD, err = eventfd.Create()
+	if err != nil {
+		return nil, fmt.Errorf("create kick event file descriptor: %w", err)
+	}
+	sq.callEventFD, err = eventfd.Create()
+	if err != nil {
+		return nil, fmt.Errorf("create call event file descriptor: %w", err)
+	}
+
+	if err = sq.descriptorTable.initializeDescriptors(); err != nil {
+		return nil, fmt.Errorf("initialize descriptors: %w", err)
+	}
+
+	// Initialize channels.
+	sq.usedChains = make(chan UsedElement, queueSize)
+	sq.moreFreeDescriptors = make(chan struct{})
+
+	// Consume used buffer notifications in the background.
+	sq.stop = sq.startConsumeUsedRing()
+
+	return &sq, nil
+}
+
+// Size returns the size of this queue, which is the number of entries/buffers
+// this queue can hold.
+func (sq *SplitQueue) Size() int {
+	sq.ensureInitialized()
+	return sq.size
+}
+
+// DescriptorTable returns the [DescriptorTable] behind this queue.
+func (sq *SplitQueue) DescriptorTable() *DescriptorTable {
+	sq.ensureInitialized()
+	return sq.descriptorTable
+}
+
+// AvailableRing returns the [AvailableRing] behind this queue.
+func (sq *SplitQueue) AvailableRing() *AvailableRing {
+	sq.ensureInitialized()
+	return sq.availableRing
+}
+
+// UsedRing returns the [UsedRing] behind this queue.
+func (sq *SplitQueue) UsedRing() *UsedRing {
+	sq.ensureInitialized()
+	return sq.usedRing
+}
+
+// KickEventFD returns the kick event file descriptor behind this queue.
+// The returned file descriptor should be used with great care to not interfere
+// with this implementation.
+func (sq *SplitQueue) KickEventFD() int {
+	sq.ensureInitialized()
+	return sq.kickEventFD.FD()
+}
+
+// CallEventFD returns the call event file descriptor behind this queue.
+// The returned file descriptor should be used with great care to not interfere
+// with this implementation.
+func (sq *SplitQueue) CallEventFD() int {
+	sq.ensureInitialized()
+	return sq.callEventFD.FD()
+}
+
+// UsedDescriptorChains returns the channel that receives [UsedElement]s for all
+// descriptor chains that were used by the device.
+//
+// Users of the [SplitQueue] should read from this channel, handle the used
+// descriptor chains and free them using [SplitQueue.FreeDescriptorChain] when
+// they're done with them. When this does not happen, the queue will run full
+// and any further calls to [SplitQueue.OfferDescriptorChain] will stall.
+//
+// When [SplitQueue.Close] is called, this channel will be closed as well.
+func (sq *SplitQueue) UsedDescriptorChains() chan UsedElement {
+	sq.ensureInitialized()
+	return sq.usedChains
+}
+
+// startConsumeUsedRing starts a goroutine that runs [consumeUsedRing].
+// A function is returned that can be used to gracefully cancel it.
+func (sq *SplitQueue) startConsumeUsedRing() func() error {
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error)
+	go func() {
+		done <- sq.consumeUsedRing(ctx)
+	}()
+	return func() error {
+		cancel()
+
+		// The goroutine blocks until it receives a signal on the event file
+		// descriptor, so it will never notice the context being canceled.
+		// To resolve this, we can just produce a fake-signal ourselves to wake
+		// it up.
+		if err := sq.callEventFD.Notify(); err != nil {
+			return fmt.Errorf("wake up goroutine: %w", err)
+		}
+
+		// Wait for the goroutine to end. This prevents the event file
+		// descriptor to be closed while it's still being used.
+		// If the goroutine failed, this is the last chance to propagate the
+		// error so it at least doesn't go unnoticed, even though the error may
+		// be older already.
+		if err := <-done; err != nil {
+			return fmt.Errorf("goroutine: consume used ring: %w", err)
+		}
+		return nil
+	}
+}
+
+// consumeUsedRing runs in a goroutine, waits for the device to signal that it
+// has used descriptor chains and puts all new [UsedElement]s into the channel
+// for them.
+func (sq *SplitQueue) consumeUsedRing(ctx context.Context) error {
+	for ctx.Err() == nil {
+		// Wait for a signal from the device.
+		if err := sq.callEventFD.Wait(); err != nil {
+			return fmt.Errorf("wait: %w", err)
+		}
+
+		// Process all new used elements.
+		for _, usedElement := range sq.usedRing.take() {
+			sq.usedChains <- usedElement
+		}
+	}
+
+	return nil
+}
+
+// OfferDescriptorChain offers a descriptor chain to the device which contains a
+// number of device-readable buffers (out buffers) and device-writable buffers
+// (in buffers).
+//
+// All buffers in the outBuffers slice will be concatenated by chaining
+// descriptors, one for each buffer in the slice. When a buffer is too large to
+// fit into a single descriptor (limited by the system's page size), it will be
+// split up into multiple descriptors within the chain.
+// When numInBuffers is greater than zero, the given number of device-writable
+// descriptors will be appended to the end of the chain, each referencing a
+// whole memory page (see [os.Getpagesize]).
+//
+// When the queue is full and no more descriptor chains can be added, a wrapped
+// [ErrNotEnoughFreeDescriptors] will be returned. If you set waitFree to true,
+// this method will handle this error and will block instead until there are
+// enough free descriptors again.
+//
+// After defining the descriptor chain in the [DescriptorTable], the index of
+// the head of the chain will be made available to the device using the
+// [AvailableRing] and will be returned by this method.
+// Callers should read from the [SplitQueue.UsedDescriptorChains] channel to be
+// notified when the descriptor chain was used by the device and should free the
+// used descriptor chains again using [SplitQueue.FreeDescriptorChain] when
+// they're done with them. When this does not happen, the queue will run full
+// and any further calls to [SplitQueue.OfferDescriptorChain] will stall.
+func (sq *SplitQueue) OfferDescriptorChain(outBuffers [][]byte, numInBuffers int, waitFree bool) (uint16, error) {
+	sq.ensureInitialized()
+
+	// Each descriptor can only hold a whole memory page, so split large out
+	// buffers into multiple smaller ones.
+	outBuffers = splitBuffers(outBuffers, os.Getpagesize())
+
+	// Synchronize the offering of descriptor chains. While the descriptor table
+	// and available ring are synchronized on their own as well, this does not
+	// protect us from interleaved calls which could cause reordering.
+	// By locking here, we can ensure that all descriptor chains are made
+	// available to the device in the same order as this method was called.
+	sq.offerMutex.Lock()
+	defer sq.offerMutex.Unlock()
+
+	// Create a descriptor chain for the given buffers.
+	var (
+		head uint16
+		err  error
+	)
+	for {
+		head, err = sq.descriptorTable.createDescriptorChain(outBuffers, numInBuffers)
+		if err == nil {
+			break
+		}
+		if waitFree && errors.Is(err, ErrNotEnoughFreeDescriptors) {
+			// Wait for more free descriptors to be put back into the queue.
+			// If the number of free descriptors is still not sufficient, we'll
+			// land here again.
+			<-sq.moreFreeDescriptors
+			continue
+		}
+		return 0, fmt.Errorf("create descriptor chain: %w", err)
+	}
+
+	// Make the descriptor chain available to the device.
+	sq.availableRing.offer([]uint16{head})
+
+	// Notify the device to make it process the updated available ring.
+	if err := sq.kickEventFD.Notify(); err != nil {
+		return head, fmt.Errorf("notify device: %w", err)
+	}
+
+	return head, nil
+}
+
+// GetDescriptorChain returns the device-readable buffers (out buffers) and
+// device-writable buffers (in buffers) of the descriptor chain with the given
+// head index.
+// The head index must be one that was returned by a previous call to
+// [SplitQueue.OfferDescriptorChain] and the descriptor chain must not have been
+// freed yet.
+//
+// Be careful to only access the returned buffer slices when the device is no
+// longer using them. They must not be accessed after
+// [SplitQueue.FreeDescriptorChain] has been called.
+func (sq *SplitQueue) GetDescriptorChain(head uint16) (outBuffers, inBuffers [][]byte, err error) {
+	sq.ensureInitialized()
+	return sq.descriptorTable.getDescriptorChain(head)
+}
+
+// FreeDescriptorChain frees the descriptor chain with the given head index.
+// The head index must be one that was returned by a previous call to
+// [SplitQueue.OfferDescriptorChain] and the descriptor chain must not have been
+// freed yet.
+//
+// This creates new room in the queue which can be used by following
+// [SplitQueue.OfferDescriptorChain] calls.
+// When there are outstanding calls for [SplitQueue.OfferDescriptorChain] that
+// are waiting for free room in the queue, they may become unblocked by this.
+func (sq *SplitQueue) FreeDescriptorChain(head uint16) error {
+	sq.ensureInitialized()
+
+	if err := sq.descriptorTable.freeDescriptorChain(head); err != nil {
+		return fmt.Errorf("free: %w", err)
+	}
+
+	// There is more free room in the descriptor table now.
+	// This is a fire-and-forget signal, so do not block when nobody listens.
+	select {
+	case sq.moreFreeDescriptors <- struct{}{}:
+	default:
+	}
+
+	return nil
+}
+
+// Close releases all resources used for this queue.
+// The implementation will try to release as many resources as possible and
+// collect potential errors before returning them.
+func (sq *SplitQueue) Close() error {
+	var errs []error
+
+	if sq.stop != nil {
+		// This has to happen before the event file descriptors may be closed.
+		if err := sq.stop(); err != nil {
+			errs = append(errs, fmt.Errorf("stop consume used ring: %w", err))
+		}
+
+		// The stop function blocked until the goroutine ended, so the channel
+		// can now safely be closed.
+		close(sq.usedChains)
+
+		// Make sure that this code block is executed only once.
+		sq.stop = nil
+	}
+
+	if err := sq.kickEventFD.Close(); err != nil {
+		errs = append(errs, fmt.Errorf("close kick event file descriptor: %w", err))
+	}
+	if err := sq.callEventFD.Close(); err != nil {
+		errs = append(errs, fmt.Errorf("close call event file descriptor: %w", err))
+	}
+
+	if err := sq.descriptorTable.releaseBuffers(); err != nil {
+		errs = append(errs, fmt.Errorf("release descriptor buffers: %w", err))
+	}
+
+	if sq.buf != nil {
+		if err := unix.Munmap(sq.buf); err == nil {
+			sq.buf = nil
+		} else {
+			errs = append(errs, fmt.Errorf("unmap virtqueue buffer: %w", err))
+		}
+	}
+
+	return errors.Join(errs...)
+}
+
+// ensureInitialized is used as a guard to prevent methods to be called on an
+// uninitialized instance.
+func (sq *SplitQueue) ensureInitialized() {
+	if sq.buf == nil {
+		panic("used ring is not initialized")
+	}
+}
+
+func align(index, alignment int) int {
+	remainder := index % alignment
+	if remainder == 0 {
+		return index
+	}
+	return index + alignment - remainder
+}
+
+// splitBuffers processes a list of buffers and splits each buffer that is
+// larger than the size limit into multiple smaller buffers.
+func splitBuffers(buffers [][]byte, sizeLimit int) [][]byte {
+	result := make([][]byte, 0, len(buffers))
+	for _, buffer := range buffers {
+		for added := 0; added < len(buffer); added += sizeLimit {
+			if len(buffer)-added <= sizeLimit {
+				result = append(result, buffer[added:])
+				break
+			}
+			result = append(result, buffer[added:added+sizeLimit])
+		}
+	}
+
+	return result
+}

overlay/virtqueue/split_virtqueue_internal_test.go

@@ -0,0 +1,105 @@
+package virtqueue
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSplitQueue_MemoryAlignment(t *testing.T) {
+	tests := []struct {
+		name      string
+		queueSize int
+	}{
+		{
+			name:      "minimal queue size",
+			queueSize: 1,
+		},
+		{
+			name:      "small queue size",
+			queueSize: 8,
+		},
+		{
+			name:      "large queue size",
+			queueSize: 256,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sq, err := NewSplitQueue(tt.queueSize)
+			require.NoError(t, err)
+
+			assert.Zero(t, sq.descriptorTable.Address()%descriptorTableAlignment)
+			assert.Zero(t, sq.availableRing.Address()%availableRingAlignment)
+			assert.Zero(t, sq.usedRing.Address()%usedRingAlignment)
+		})
+	}
+}
+
+func TestSplitBuffers(t *testing.T) {
+	const sizeLimit = 16
+	tests := []struct {
+		name     string
+		buffers  [][]byte
+		expected [][]byte
+	}{
+		{
+			name:     "no buffers",
+			buffers:  make([][]byte, 0),
+			expected: make([][]byte, 0),
+		},
+		{
+			name: "small",
+			buffers: [][]byte{
+				make([]byte, 11),
+			},
+			expected: [][]byte{
+				make([]byte, 11),
+			},
+		},
+		{
+			name: "exact size",
+			buffers: [][]byte{
+				make([]byte, sizeLimit),
+			},
+			expected: [][]byte{
+				make([]byte, sizeLimit),
+			},
+		},
+		{
+			name: "large",
+			buffers: [][]byte{
+				make([]byte, 42),
+			},
+			expected: [][]byte{
+				make([]byte, 16),
+				make([]byte, 16),
+				make([]byte, 10),
+			},
+		},
+		{
+			name: "mixed",
+			buffers: [][]byte{
+				make([]byte, 7),
+				make([]byte, 30),
+				make([]byte, 15),
+				make([]byte, 32),
+			},
+			expected: [][]byte{
+				make([]byte, 7),
+				make([]byte, 16),
+				make([]byte, 14),
+				make([]byte, 15),
+				make([]byte, 16),
+				make([]byte, 16),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := splitBuffers(tt.buffers, sizeLimit)
+			assert.Equal(t, tt.expected, actual)
+		})
+	}
+}

overlay/virtqueue/used_element.go

@@ -0,0 +1,17 @@
+package virtqueue
+
+// usedElementSize is the number of bytes needed to store a [UsedElement] in
+// memory.
+const usedElementSize = 8
+
+// UsedElement is an element of the [UsedRing] and describes a descriptor chain
+// that was used by the device.
+type UsedElement struct {
+	// DescriptorIndex is the index of the head of the used descriptor chain in
+	// the [DescriptorTable].
+	// The index is 32-bit here for padding reasons.
+	DescriptorIndex uint32
+	// Length is the number of bytes written into the device writable portion of
+	// the buffer described by the descriptor chain.
+	Length uint32
+}

overlay/virtqueue/used_element_internal_test.go

@@ -0,0 +1,12 @@
+package virtqueue
+
+import (
+	"testing"
+	"unsafe"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUsedElement_Size(t *testing.T) {
+	assert.EqualValues(t, usedElementSize, unsafe.Sizeof(UsedElement{}))
+}

overlay/virtqueue/used_ring.go

@@ -0,0 +1,119 @@
+package virtqueue
+
+import (
+	"fmt"
+	"sync"
+	"unsafe"
+)
+
+// usedRingFlag is a flag that describes a [UsedRing].
+type usedRingFlag uint16
+
+const (
+	// usedRingFlagNoNotify is used by the host to advise the guest to not
+	// kick it when adding a buffer. It's unreliable, so it's simply an
+	// optimization. Guest will still kick when it's out of buffers.
+	usedRingFlagNoNotify usedRingFlag = 1 << iota
+)
+
+// usedRingSize is the number of bytes needed to store a [UsedRing] with the
+// given queue size in memory.
+func usedRingSize(queueSize int) int {
+	return 6 + usedElementSize*queueSize
+}
+
+// usedRingAlignment is the minimum alignment of a [UsedRing] in memory, as
+// required by the virtio spec.
+const usedRingAlignment = 4
+
+// UsedRing is where the device returns descriptor chains once it is done with
+// them. Each ring entry is a [UsedElement]. It is only written to by the device
+// and read by the driver.
+//
+// Because the size of the ring depends on the queue size, we cannot define a
+// Go struct with a static size that maps to the memory of the ring. Instead,
+// this struct only contains pointers to the corresponding memory areas.
+type UsedRing struct {
+	initialized bool
+
+	// flags that describe this ring.
+	flags *usedRingFlag
+	// ringIndex indicates where the device would put the next entry into the
+	// ring (modulo the queue size).
+	ringIndex *uint16
+	// ring contains the [UsedElement]s. It wraps around at queue size.
+	ring []UsedElement
+	// availableEvent is not used by this implementation, but we reserve it
+	// anyway to avoid issues in case a device may try to write to it, contrary
+	// to the virtio specification.
+	availableEvent *uint16
+
+	// lastIndex is the internal ringIndex up to which all [UsedElement]s were
+	// processed.
+	lastIndex uint16
+
+	mu sync.Mutex
+}
+
+// newUsedRing creates a used ring that uses the given underlying memory. The
+// length of the memory slice must match the size needed for the ring (see
+// [usedRingSize]) for the given queue size.
+func newUsedRing(queueSize int, mem []byte) *UsedRing {
+	ringSize := usedRingSize(queueSize)
+	if len(mem) != ringSize {
+		panic(fmt.Sprintf("memory size (%v) does not match required size "+
+			"for used ring: %v", len(mem), ringSize))
+	}
+
+	r := UsedRing{
+		initialized:    true,
+		flags:          (*usedRingFlag)(unsafe.Pointer(&mem[0])),
+		ringIndex:      (*uint16)(unsafe.Pointer(&mem[2])),
+		ring:           unsafe.Slice((*UsedElement)(unsafe.Pointer(&mem[4])), queueSize),
+		availableEvent: (*uint16)(unsafe.Pointer(&mem[ringSize-2])),
+	}
+	r.lastIndex = *r.ringIndex
+	return &r
+}
+
+// Address returns the pointer to the beginning of the ring in memory.
+// Do not modify the memory directly to not interfere with this implementation.
+func (r *UsedRing) Address() uintptr {
+	if !r.initialized {
+		panic("used ring is not initialized")
+	}
+	return uintptr(unsafe.Pointer(r.flags))
+}
+
+// take returns all new [UsedElement]s that the device put into the ring and
+// that weren't already returned by a previous call to this method.
+func (r *UsedRing) take() []UsedElement {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	ringIndex := *r.ringIndex
+	if ringIndex == r.lastIndex {
+		// Nothing new.
+		return nil
+	}
+
+	// Calculate the number new used elements that we can read from the ring.
+	// The ring index may wrap, so special handling for that case is needed.
+	count := int(ringIndex - r.lastIndex)
+	if count < 0 {
+		count += 0xffff
+	}
+
+	// The number of new elements can never exceed the queue size.
+	if count > len(r.ring) {
+		panic("used ring contains more new elements than the ring is long")
+	}
+
+	elems := make([]UsedElement, count)
+	for i := range count {
+		elems[i] = r.ring[r.lastIndex%uint16(len(r.ring))]
+		r.lastIndex++
+	}
+
+	return elems
+}

overlay/virtqueue/used_ring_internal_test.go

@@ -0,0 +1,136 @@
+package virtqueue
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUsedRing_MemoryLayout(t *testing.T) {
+	const queueSize = 2
+
+	memory := make([]byte, usedRingSize(queueSize))
+	r := newUsedRing(queueSize, memory)
+
+	*r.flags = 0x01ff
+	*r.ringIndex = 1
+	r.ring[0] = UsedElement{
+		DescriptorIndex: 0x0123,
+		Length:          0x4567,
+	}
+	r.ring[1] = UsedElement{
+		DescriptorIndex: 0x89ab,
+		Length:          0xcdef,
+	}
+
+	assert.Equal(t, []byte{
+		0xff, 0x01,
+		0x01, 0x00,
+		0x23, 0x01, 0x00, 0x00,
+		0x67, 0x45, 0x00, 0x00,
+		0xab, 0x89, 0x00, 0x00,
+		0xef, 0xcd, 0x00, 0x00,
+		0x00, 0x00,
+	}, memory)
+}
+
+func TestUsedRing_Take(t *testing.T) {
+	const queueSize = 8
+
+	tests := []struct {
+		name      string
+		ring      []UsedElement
+		ringIndex uint16
+		lastIndex uint16
+		expected  []UsedElement
+	}{
+		{
+			name: "nothing new",
+			ring: []UsedElement{
+				{DescriptorIndex: 1},
+				{DescriptorIndex: 2},
+				{DescriptorIndex: 3},
+				{DescriptorIndex: 4},
+				{},
+				{},
+				{},
+				{},
+			},
+			ringIndex: 4,
+			lastIndex: 4,
+			expected:  nil,
+		},
+		{
+			name: "no overflow",
+			ring: []UsedElement{
+				{DescriptorIndex: 1},
+				{DescriptorIndex: 2},
+				{DescriptorIndex: 3},
+				{DescriptorIndex: 4},
+				{},
+				{},
+				{},
+				{},
+			},
+			ringIndex: 4,
+			lastIndex: 1,
+			expected: []UsedElement{
+				{DescriptorIndex: 2},
+				{DescriptorIndex: 3},
+				{DescriptorIndex: 4},
+			},
+		},
+		{
+			name: "ring overflow",
+			ring: []UsedElement{
+				{DescriptorIndex: 9},
+				{DescriptorIndex: 10},
+				{DescriptorIndex: 3},
+				{DescriptorIndex: 4},
+				{DescriptorIndex: 5},
+				{DescriptorIndex: 6},
+				{DescriptorIndex: 7},
+				{DescriptorIndex: 8},
+			},
+			ringIndex: 10,
+			lastIndex: 7,
+			expected: []UsedElement{
+				{DescriptorIndex: 8},
+				{DescriptorIndex: 9},
+				{DescriptorIndex: 10},
+			},
+		},
+		{
+			name: "index overflow",
+			ring: []UsedElement{
+				{DescriptorIndex: 9},
+				{DescriptorIndex: 10},
+				{DescriptorIndex: 3},
+				{DescriptorIndex: 4},
+				{DescriptorIndex: 5},
+				{DescriptorIndex: 6},
+				{DescriptorIndex: 7},
+				{DescriptorIndex: 8},
+			},
+			ringIndex: 2,
+			lastIndex: 65535,
+			expected: []UsedElement{
+				{DescriptorIndex: 8},
+				{DescriptorIndex: 9},
+				{DescriptorIndex: 10},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			memory := make([]byte, usedRingSize(queueSize))
+			r := newUsedRing(queueSize, memory)
+
+			copy(r.ring, tt.ring)
+			*r.ringIndex = tt.ringIndex
+			r.lastIndex = tt.lastIndex
+
+			assert.Equal(t, tt.expected, r.take())
+		})
+	}
+}