more ram -> more speed

cmd/nebula-service/main.go

@@ -98,14 +98,15 @@ func main() {
 	}
 
 	if !*configTest {
-		if err := ctrl.Start(); err != nil {
+		wait, err := ctrl.Start()
+		if err != nil {
 			util.LogWithContextIfNeeded("Error while running", err, l)
 			os.Exit(1)
 		}
 
 		go ctrl.ShutdownBlock()
 
-		if err := ctrl.Wait(); err != nil {
+		if err := wait(); err != nil {
 			l.Error("Nebula stopped due to fatal error", "error", err)
 			os.Exit(2)
 		}

interface.go

@@ -435,6 +435,14 @@ func (f *Interface) listenIn(reader tio.Queue, i int) {
 
 		for _, pkt := range pkts {
 			f.consumeInsidePacket(pkt, fwPacket, nb, sb, rejectBuf, i, conntrackCache.Get())
+			// Flush incrementally once a full sendmmsg batch has
+			// accumulated so the first packets of a deep read drain
+			// hit the wire while the rest are still being encrypted.
+			if sb.Len() >= batch.SendBatchCap {
+				if err := sb.Flush(); err != nil {
+					f.l.Error("Failed to write outgoing batch", "error", err, "writer", i)
+				}
+			}
 		}
 		if err := sb.Flush(); err != nil {
 			f.l.Error("Failed to write outgoing batch", "error", err, "writer", i)

overlay/batch/tx_batch.go

@@ -45,6 +45,11 @@ func (b *SendBatch) Reserve(sz int) []byte {
 	return b.backing[start : start+sz : start+sz]
 }
 
+// Len reports how many packets are queued for the next Flush. Callers use
+// it to flush incrementally once a full sendmmsg batch has accumulated,
+// bounding how long the first packet of a large read batch waits.
+func (b *SendBatch) Len() int { return len(b.bufs) }
+
 func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort, outerECN byte) {
 	b.bufs = append(b.bufs, pkt)
 	b.dsts = append(b.dsts, dst)

overlay/tio/tio_gso_linux.go

@@ -28,9 +28,12 @@ const tunRxBufSize = 64 * 1024
 // tunRxBufCap is the total size we allocate for the per-reader rx
 // buffer. With reads landing directly in rxBuf, each drain iteration
 // consumes up to tunRxBufSize of headroom for the kernel-supplied bytes.
-// Sized to two such iterations so the initial blocking read plus one
+// Sized to eight such iterations so a single poll wake can drain several
-// drain read both fit without partial-drop.
+// TSO/USO superpackets under bulk load, amortizing the wake and giving
-const tunRxBufCap = tunRxBufSize * 2
+// the sendmmsg planner longer same-destination runs. Hold latency stays
+// bounded because listenIn flushes its send batch incrementally rather
+// than only at end-of-drain.
+const tunRxBufCap = tunRxBufSize * 8
 
 // tunDrainCap caps how many packets a single Read will accumulate via
 // the post-wake drain loop. Sized to soak up a burst of small ACKs while