BAM! a hit from my spice-weasel. Now TX is zero-copy, at the expense of my sanity!

overlay/virtqueue/descriptor_table.go

@@ -281,6 +281,106 @@ func (dt *DescriptorTable) createDescriptorChain(outBuffers [][]byte, numInBuffe
 	return head, nil
 }
 
+func (dt *DescriptorTable) CreateDescriptorForOutputs() (uint16, error) {
+	//todo just fill the damn table
+	// Do we still have enough free descriptors?
+
+	if 1 > dt.freeNum {
+		return 0, ErrNotEnoughFreeDescriptors
+	}
+
+	// Above validation ensured that there is at least one free descriptor, so
+	// the free descriptor chain head should be valid.
+	if dt.freeHeadIndex == noFreeHead {
+		panic("free descriptor chain head is unset but there should be free descriptors")
+	}
+
+	// To avoid having to iterate over the whole table to find the descriptor
+	// pointing to the head just to replace the free head, we instead always
+	// create descriptor chains from the descriptors coming after the head.
+	// This way we only have to touch the head as a last resort, when all other
+	// descriptors are already used.
+	head := dt.descriptors[dt.freeHeadIndex].next
+	desc := &dt.descriptors[head]
+	next := desc.next
+
+	checkUnusedDescriptorLength(head, desc)
+
+	// Give the device the maximum available number of bytes to write into.
+	desc.length = uint32(dt.itemSize)
+	desc.flags = 0 // descriptorFlagWritable
+	desc.next = 0  // Not necessary to clear this, it's just for looks.
+
+	dt.freeNum -= 1
+
+	if dt.freeNum == 0 {
+		// The last descriptor in the chain should be the free chain head
+		// itself.
+		if next != dt.freeHeadIndex {
+			panic("descriptor chain takes up all free descriptors but does not end with the free chain head")
+		}
+
+		// When this new chain takes up all remaining descriptors, we no longer
+		// have a free chain.
+		dt.freeHeadIndex = noFreeHead
+	} else {
+		// We took some descriptors out of the free chain, so make sure to close
+		// the circle again.
+		dt.descriptors[dt.freeHeadIndex].next = next
+	}
+
+	return head, nil
+}
+
+func (dt *DescriptorTable) createDescriptorForInputs() (uint16, error) {
+	// Do we still have enough free descriptors?
+	if 1 > dt.freeNum {
+		return 0, ErrNotEnoughFreeDescriptors
+	}
+
+	// Above validation ensured that there is at least one free descriptor, so
+	// the free descriptor chain head should be valid.
+	if dt.freeHeadIndex == noFreeHead {
+		panic("free descriptor chain head is unset but there should be free descriptors")
+	}
+
+	// To avoid having to iterate over the whole table to find the descriptor
+	// pointing to the head just to replace the free head, we instead always
+	// create descriptor chains from the descriptors coming after the head.
+	// This way we only have to touch the head as a last resort, when all other
+	// descriptors are already used.
+	head := dt.descriptors[dt.freeHeadIndex].next
+	desc := &dt.descriptors[head]
+	next := desc.next
+
+	checkUnusedDescriptorLength(head, desc)
+
+	// Give the device the maximum available number of bytes to write into.
+	desc.length = uint32(dt.itemSize)
+	desc.flags = descriptorFlagWritable
+	desc.next = 0 // Not necessary to clear this, it's just for looks.
+
+	dt.freeNum -= 1
+
+	if dt.freeNum == 0 {
+		// The last descriptor in the chain should be the free chain head
+		// itself.
+		if next != dt.freeHeadIndex {
+			panic("descriptor chain takes up all free descriptors but does not end with the free chain head")
+		}
+
+		// When this new chain takes up all remaining descriptors, we no longer
+		// have a free chain.
+		dt.freeHeadIndex = noFreeHead
+	} else {
+		// We took some descriptors out of the free chain, so make sure to close
+		// the circle again.
+		dt.descriptors[dt.freeHeadIndex].next = next
+	}
+
+	return head, nil
+}
+
 // TODO: Implement a zero-copy variant of createDescriptorChain?
 
 // getDescriptorChain returns the device-readable buffers (out buffers) and
@@ -334,6 +434,20 @@ func (dt *DescriptorTable) getDescriptorChain(head uint16) (outBuffers, inBuffer
 	return
 }
 
+func (dt *DescriptorTable) getDescriptorItem(head uint16) ([]byte, error) {
+	if int(head) > len(dt.descriptors) {
+		return nil, fmt.Errorf("%w: index out of range", ErrInvalidDescriptorChain)
+	}
+
+	desc := &dt.descriptors[head] //todo this is a pretty nasty hack with no checks
+
+	// The descriptor address points to memory not managed by Go, so this
+	// conversion is safe. See https://github.com/golang/go/issues/58625
+	//goland:noinspection GoVetUnsafePointer
+	bs := unsafe.Slice((*byte)(unsafe.Pointer(desc.address)), desc.length)
+	return bs, nil
+}
+
 func (dt *DescriptorTable) getDescriptorInbuffers(head uint16, inBuffers *[][]byte) error {
 	if int(head) > len(dt.descriptors) {
 		return fmt.Errorf("%w: index out of range", ErrInvalidDescriptorChain)

overlay/virtqueue/split_virtqueue.go

@@ -208,6 +208,32 @@ func (sq *SplitQueue) BlockAndGetHeads(ctx context.Context) ([]UsedElement, erro
 	return nil, ctx.Err()
 }
 
+func (sq *SplitQueue) TakeSingle(ctx context.Context) (uint16, error) {
+	var n int
+	var err error
+	for ctx.Err() == nil {
+		out, ok := sq.usedRing.takeOne()
+		if ok {
+			return out, nil
+		}
+		// Wait for a signal from the device.
+		if n, err = sq.epoll.Block(); err != nil {
+			return 0, fmt.Errorf("wait: %w", err)
+		}
+
+		if n > 0 {
+			out, ok = sq.usedRing.takeOne()
+			if ok {
+				_ = sq.epoll.Clear() //???
+				return out, nil
+			} else {
+				continue //???
+			}
+		}
+	}
+	return 0, ctx.Err()
+}
+
 func (sq *SplitQueue) BlockAndGetHeadsCapped(ctx context.Context, maxToTake int) ([]UsedElement, error) {
 	var n int
 	var err error
@@ -268,14 +294,14 @@ func (sq *SplitQueue) BlockAndGetHeadsCapped(ctx context.Context, maxToTake int)
 // they're done with them. When this does not happen, the queue will run full
 // and any further calls to [SplitQueue.OfferDescriptorChain] will stall.
 
-func (sq *SplitQueue) OfferInDescriptorChains(numInBuffers int) (uint16, error) {
+func (sq *SplitQueue) OfferInDescriptorChains() (uint16, error) {
 	// Create a descriptor chain for the given buffers.
 	var (
 		head uint16
 		err  error
 	)
 	for {
-		head, err = sq.descriptorTable.createDescriptorChain(nil, numInBuffers)
+		head, err = sq.descriptorTable.createDescriptorForInputs()
 		if err == nil {
 			break
 		}
@@ -361,6 +387,11 @@ func (sq *SplitQueue) GetDescriptorChain(head uint16) (outBuffers, inBuffers [][
 	return sq.descriptorTable.getDescriptorChain(head)
 }
 
+func (sq *SplitQueue) GetDescriptorItem(head uint16) ([]byte, error) {
+	sq.descriptorTable.descriptors[head].length = uint32(sq.descriptorTable.itemSize)
+	return sq.descriptorTable.getDescriptorItem(head)
+}
+
 func (sq *SplitQueue) GetDescriptorChainContents(head uint16, out []byte, maxLen int) (int, error) {
 	return sq.descriptorTable.getDescriptorChainContents(head, out, maxLen)
 }
@@ -387,7 +418,12 @@ func (sq *SplitQueue) FreeDescriptorChain(head uint16) error {
 	return nil
 }
 
-func (sq *SplitQueue) RecycleDescriptorChains(chains []uint16, kick bool) error {
+func (sq *SplitQueue) SetDescSize(head uint16, sz int) {
+	//not called under lock
+	sq.descriptorTable.descriptors[int(head)].length = uint32(sz)
+}
+
+func (sq *SplitQueue) OfferDescriptorChains(chains []uint16, kick bool) error {
 	//todo not doing this may break eventually?
 	//not called under lock
 	//if err := sq.descriptorTable.freeDescriptorChain(head); err != nil {
@@ -399,14 +435,19 @@ func (sq *SplitQueue) RecycleDescriptorChains(chains []uint16, kick bool) error
 
 	// Notify the device to make it process the updated available ring.
 	if kick {
-		if err := sq.kickEventFD.Kick(); err != nil {
-			return fmt.Errorf("notify device: %w", err)
-		}
+		return sq.Kick()
 	}
 
 	return nil
 }
 
+func (sq *SplitQueue) Kick() error {
+	if err := sq.kickEventFD.Kick(); err != nil {
+		return fmt.Errorf("notify device: %w", err)
+	}
+	return nil
+}
+
 // Close releases all resources used for this queue.
 // The implementation will try to release as many resources as possible and
 // collect potential errors before returning them.

overlay/virtqueue/used_ring.go

@@ -127,3 +127,58 @@ func (r *UsedRing) take(maxToTake int) (int, []UsedElement) {
 
 	return stillNeedToTake, elems
 }
+
+func (r *UsedRing) takeOne() (uint16, bool) {
+	//r.mu.Lock()
+	//defer r.mu.Unlock()
+
+	ringIndex := *r.ringIndex
+	if ringIndex == r.lastIndex {
+		// Nothing new.
+		return 0xffff, false
+	}
+
+	// Calculate the number new used elements that we can read from the ring.
+	// The ring index may wrap, so special handling for that case is needed.
+	count := int(ringIndex - r.lastIndex)
+	if count < 0 {
+		count += 0xffff
+	}
+
+	// The number of new elements can never exceed the queue size.
+	if count > len(r.ring) {
+		panic("used ring contains more new elements than the ring is long")
+	}
+
+	if count == 0 {
+		return 0xffff, false
+	}
+
+	out := r.ring[r.lastIndex%uint16(len(r.ring))].GetHead()
+	r.lastIndex++
+
+	return out, true
+}
+
+// InitOfferSingle is only used to pre-fill the used queue at startup, and should not be used if the device is running!
+func (r *UsedRing) InitOfferSingle(x uint16, size int) {
+	//always called under lock
+	//r.mu.Lock()
+	//defer r.mu.Unlock()
+
+	offset := 0
+	// Add descriptor chain heads to the ring.
+
+	// The 16-bit ring index may overflow. This is expected and is not an
+	// issue because the size of the ring array (which equals the queue
+	// size) is always a power of 2 and smaller than the highest possible
+	// 16-bit value.
+	insertIndex := int(*r.ringIndex+uint16(offset)) % len(r.ring)
+	r.ring[insertIndex] = UsedElement{
+		DescriptorIndex: uint32(x),
+		Length:          uint32(size),
+	}
+
+	// Increase the ring index by the number of descriptor chains added to the ring.
+	*r.ringIndex += 1
+}