block less I guess

cmd/nebula/main.go

@@ -3,9 +3,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"log"
-	"net/http"
-	_ "net/http/pprof"
 	"os"
 	"runtime/debug"
 	"strings"
@@ -74,10 +71,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	go func() {
-		log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
-	}()
-
 	if !*configTest {
 		wait, err := ctrl.Start()
 		if err != nil {

interface.go

@@ -507,12 +507,14 @@ func (f *Interface) Close() error {
 	}
 
 	// Release the tun readers
-	for i, u := range f.readers {
-		err := u.Close()
-		if err != nil {
-			f.l.WithError(err).WithField("i", i).Error("Error while closing tun device")
+	for i, r := range f.readers {
+		if i == 0 {
+			continue // f.readers[0] is f.inside, which we want to save for last, since it closes other stuff too
+		}
+		if err := r.Close(); err != nil {
+			f.l.WithError(err).Error("Error while closing tun reader")
 		}
 	}
 
-	return nil
+	return f.inside.Close()
 }

overlay/tun_linux.go

@@ -72,6 +72,11 @@ type ifreqQLEN struct {
 }
 
 func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
+	err := unix.SetNonblock(deviceFd, true)
+	if err != nil {
+		return nil, err
+	}
+
 	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
 
 	t, err := newTunGeneric(c, l, file, vpnNetworks)
@@ -122,6 +127,11 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueu
 	}
 	name := strings.Trim(string(req.Name[:]), "\x00")
 
+	err = unix.SetNonblock(fd, true)
+	if err != nil {
+		return nil, err
+	}
+
 	file := os.NewFile(uintptr(fd), "/dev/net/tun")
 	t, err := newTunGeneric(c, l, file, vpnNetworks)
 	if err != nil {
@@ -145,7 +155,12 @@ func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []n
 		l:                         l,
 	}
 
-	err := t.reload(c, true)
+	err := unix.SetNonblock(t.fd, true)
+	if err != nil {
+		return nil, err
+	}
+
+	err = t.reload(c, true)
 	if err != nil {
 		return nil, err
 	}
@@ -251,6 +266,11 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 		return nil, err
 	}
 
+	err = unix.SetNonblock(fd, true)
+	if err != nil {
+		return nil, err
+	}
+
 	file := os.NewFile(uintptr(fd), "/dev/net/tun")
 
 	return file, nil
@@ -261,29 +281,6 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	return r
 }
 
-func (t *tun) Write(b []byte) (int, error) {
-	var nn int
-	maximum := len(b)
-
-	for {
-		n, err := unix.Write(t.fd, b[nn:maximum])
-		if n > 0 {
-			nn += n
-		}
-		if nn == len(b) {
-			return nn, err
-		}
-
-		if err != nil {
-			return nn, err
-		}
-
-		if n == 0 {
-			return nn, io.ErrUnexpectedEOF
-		}
-	}
-}
-
 func (t *tun) deviceBytes() (o [16]byte) {
 	for i, c := range t.Device {
 		o[i] = byte(c)
@@ -712,11 +709,17 @@ func (t *tun) Close() error {
 	}
 
 	if t.ReadWriteCloser != nil {
-		_ = t.ReadWriteCloser.Close()
+		err := t.ReadWriteCloser.Close()
+		if err != nil {
+			t.l.WithField("error", err).Error("Failed to close read/write connection")
+		}
 	}
 
 	if t.ioctlFd > 0 {
-		_ = os.NewFile(t.ioctlFd, "ioctlFd").Close()
+		err := os.NewFile(t.ioctlFd, "ioctlFd").Close()
+		if err != nil {
+			t.l.WithField("error", err).Error("Failed to close ioctl fd")
+		}
 		t.ioctlFd = 0
 	}
 

udp/udp_generic.go

@@ -10,11 +10,9 @@ package udp
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net"
 	"net/netip"
-	"time"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
@@ -76,22 +74,11 @@ type rawMessage struct {
 func (u *GenericConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
-	var lastRecvErr time.Time
-
 	for {
 		// Just read one packet at a time
 		n, rua, err := u.ReadFromUDPAddrPort(buffer)
 		if err != nil {
 			return err
-			if errors.Is(err, net.ErrClosed) {
-				return err
-			}
-			// Dampen unexpected message warns to once per minute
-			if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute {
-				lastRecvErr = time.Now()
-				u.l.WithError(err).Warn("unexpected udp socket receive error")
-			}
-			continue
 		}
 
 		r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n])

udp/udp_linux.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"net/netip"
 	"syscall"
+	"time"
 	"unsafe"
 
 	"github.com/rcrowley/go-metrics"
@@ -17,6 +18,8 @@ import (
 	"golang.org/x/sys/unix"
 )
 
+var readTimeout = unix.NsecToTimeval(int64(3 * time.Second))
+
 type StdConn struct {
 	sysFd int
 	isV4  bool
@@ -47,6 +50,11 @@ func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch in
 		}
 	}
 
+	// Set a read timeout
+	if err = unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &readTimeout); err != nil {
+		return nil, fmt.Errorf("unable to set SO_RCVTIMEO: %s", err)
+	}
+
 	var sa unix.Sockaddr
 	if ip.Is4() {
 		sa4 := &unix.SockaddrInet4{Port: port}
@@ -154,6 +162,9 @@ func (u *StdConn) ReadSingle(msgs []rawMessage) (int, error) {
 		)
 
 		if err != 0 {
+			if err == unix.EAGAIN || err == unix.EINTR || err == unix.EWOULDBLOCK {
+				continue
+			}
 			return 0, &net.OpError{Op: "recvmsg", Err: err}
 		}
 
@@ -173,8 +184,14 @@ func (u *StdConn) ReadMulti(msgs []rawMessage) (int, error) {
 			0,
 			0,
 		)
-
-		if err != 0 {
+		if err == unix.EAGAIN || err == unix.EINTR || err == unix.EWOULDBLOCK {
+			if int64(n) > 0 {
+				//ran out of time, but have some messages to return
+				return int(n), nil
+			} else {
+				continue
+			}
+		} else if err != 0 {
 			return 0, &net.OpError{Op: "recvmmsg", Err: err}
 		}
 
@@ -301,7 +318,6 @@ func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
 }
 
 func (u *StdConn) Close() error {
-	_ = syscall.Shutdown(u.sysFd, syscall.SHUT_RDWR)
 	return syscall.Close(u.sysFd)
 }