diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 9920992..5d1a40c 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -53,4 +53,12 @@ jobs: working-directory: ./.github/workflows/smoke run: ./smoke-relay.sh + - name: setup docker image for multiport + working-directory: ./.github/workflows/smoke + run: NAME="smoke-multiport" MULTIPORT_TX=true MULTIPORT_RX=true MULTIPORT_HANDSHAKE=true ./build.sh + + - name: run smoke + working-directory: ./.github/workflows/smoke + run: ./smoke.sh + timeout-minutes: 10 diff --git a/.github/workflows/smoke/build.sh b/.github/workflows/smoke/build.sh index 0c20b3f..49cb6ce 100755 --- a/.github/workflows/smoke/build.sh +++ b/.github/workflows/smoke/build.sh @@ -36,4 +36,4 @@ mkdir ./build ../../../../nebula-cert sign -name "host4" -groups "host,host4" -ip "192.168.100.4/24" ) -sudo docker build -t nebula:smoke . +sudo docker build -t "nebula:${NAME:-smoke}" . diff --git a/.github/workflows/smoke/genconfig.sh b/.github/workflows/smoke/genconfig.sh index 005734c..54297a5 100755 --- a/.github/workflows/smoke/genconfig.sh +++ b/.github/workflows/smoke/genconfig.sh @@ -48,6 +48,10 @@ listen: tun: dev: ${TUN_DEV:-nebula1} + multiport: + tx_enabled: ${MULTIPORT_TX:-false} + rx_enabled: ${MULTIPORT_RX:-false} + tx_handshake: ${MULTIPORT_HANDSHAKE:-false} firewall: outbound: ${OUTBOUND:-$FIREWALL_ALL} diff --git a/.github/workflows/smoke/smoke.sh b/.github/workflows/smoke/smoke.sh index 213add3..b003f9d 100755 --- a/.github/workflows/smoke/smoke.sh +++ b/.github/workflows/smoke/smoke.sh @@ -20,18 +20,20 @@ cleanup() { trap cleanup EXIT -sudo docker run --name lighthouse1 --rm nebula:smoke -config lighthouse1.yml -test -sudo docker run --name host2 --rm nebula:smoke -config host2.yml -test -sudo docker run --name host3 --rm nebula:smoke -config host3.yml -test -sudo docker run --name host4 --rm nebula:smoke -config host4.yml -test +CONTAINER="nebula:${NAME:-smoke}" -sudo docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' & +sudo docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test +sudo docker run --name host2 --rm "$CONTAINER" -config host2.yml -test +sudo docker run --name host3 --rm "$CONTAINER" -config host3.yml -test +sudo docker run --name host4 --rm "$CONTAINER" -config host4.yml -test + +sudo docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' & sleep 1 -sudo docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' & +sudo docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' & sleep 1 -sudo docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' & +sudo docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' & sleep 1 -sudo docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' & +sudo docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' & sleep 1 set +x diff --git a/Makefile b/Makefile index 188ffea..8915d09 100644 --- a/Makefile +++ b/Makefile @@ -175,6 +175,10 @@ smoke-relay-docker: bin-docker cd .github/workflows/smoke/ && ./build-relay.sh cd .github/workflows/smoke/ && ./smoke-relay.sh +smoke-multiport-docker: bin-docker + cd .github/workflows/smoke/ && NAME="smoke-multiport" MULTIPORT_TX=true MULTIPORT_RX=true MULTIPORT_HANDSHAKE=true ./build.sh + cd .github/workflows/smoke/ && NAME="smoke-multiport" ./smoke.sh + smoke-docker-race: BUILD_ARGS = -race smoke-docker-race: smoke-docker diff --git a/examples/config.yml b/examples/config.yml index 0e3d3e1..824c7ad 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -202,6 +202,46 @@ tun: # mtu: 1300 # metric: 100 + # EXPERIMENTAL: This option may change or disappear in the future. + # Multiport spreads outgoing UDP packets across multiple UDP send ports, + # which allows nebula to work around any issues on the underlay network. + # Some example issues this could work around: + # - UDP rate limits on a per flow basis. + # - Partial underlay network failure in which some flows work and some don't + # Agreement is done during the handshake to decide if multiport mode will + # be used for a given tunnel (one side must have tx_enabled set, the other + # side must have rx_enabled set) + # + # NOTE: you cannot use multiport on a host if you are relying on UDP hole + # punching to get through a NAT or firewall. + # + # NOTE: Linux only (uses raw sockets to send). Also currently only works + # with IPv4 underlay network remotes. + # + # The default values are listed below: + #multiport: + # This host support sending via multiple UDP ports. + #tx_enabled: false + # + # This host supports receiving packets sent from multiple UDP ports. + #rx_enabled: false + # + # How many UDP ports to use when sending. The lowest source port will be + # listen.port and go up to (but not including) listen.port + tx_ports. + #tx_ports: 100 + # + # NOTE: All of your hosts must be running a version of Nebula that supports + # multiport if you want to enable this feature. Older versions of Nebula + # will be confused by these multiport handshakes. + # + # If handshakes are not getting a response, attempt to transmit handshakes + # using random UDP source ports (to get around partial underlay network + # failures). + #tx_handshake: false + # + # How many unresponded handshakes we should send before we attempt to + # send multiport handshakes. + #tx_handshake_delay: 2 # TODO # Configure logging level diff --git a/firewall/packet.go b/firewall/packet.go index 1c4affd..dc3270e 100644 --- a/firewall/packet.go +++ b/firewall/packet.go @@ -3,6 +3,7 @@ package firewall import ( "encoding/json" "fmt" + mathrand "math/rand" "github.com/slackhq/nebula/iputil" ) @@ -60,3 +61,30 @@ func (fp Packet) MarshalJSON() ([]byte, error) { "Fragment": fp.Fragment, }) } + +// UDPSendPort calculates the UDP port to send from when using multiport mode. +// The result will be from [0, numBuckets) +func (fp Packet) UDPSendPort(numBuckets int) uint16 { + if numBuckets <= 1 { + return 0 + } + + // If there is no port (like an ICMP packet), pick a random UDP send port + if fp.LocalPort == 0 { + return uint16(mathrand.Intn(numBuckets)) + } + + // A decent enough 32bit hash function + // Prospecting for Hash Functions + // - https://nullprogram.com/blog/2018/07/31/ + // - https://github.com/skeeto/hash-prospector + // [16 21f0aaad 15 d35a2d97 15] = 0.10760229515479501 + x := (uint32(fp.LocalPort) << 16) | uint32(fp.RemotePort) + x ^= x >> 16 + x *= 0x21f0aaad + x ^= x >> 15 + x *= 0xd35a2d97 + x ^= x >> 15 + + return uint16(x) % uint16(numBuckets) +} diff --git a/handshake_ix.go b/handshake_ix.go index fd1a908..cd5add4 100644 --- a/handshake_ix.go +++ b/handshake_ix.go @@ -37,6 +37,15 @@ func ixHandshakeStage0(f *Interface, vpnIp iputil.VpnIp, hostinfo *HostInfo) { Cert: ci.certState.rawCertificateNoKey, } + if f.multiPort.Tx || f.multiPort.Rx { + hsProto.InitiatorMultiPort = &MultiPortDetails{ + RxSupported: f.multiPort.Rx, + TxSupported: f.multiPort.Tx, + BasePort: uint32(f.multiPort.TxBasePort), + TotalPorts: uint32(f.multiPort.TxPorts), + } + } + hsBytes := []byte{} hs := &NebulaHandshake{ @@ -130,6 +139,29 @@ func ixHandshakeStage1(f *Interface, addr *udp.Addr, via interface{}, packet []b return } + var multiportTx, multiportRx bool + if f.multiPort.Rx || f.multiPort.Tx { + if hs.Details.InitiatorMultiPort != nil { + multiportTx = hs.Details.InitiatorMultiPort.RxSupported && f.multiPort.Tx + multiportRx = hs.Details.InitiatorMultiPort.TxSupported && f.multiPort.Rx + } + + hs.Details.ResponderMultiPort = &MultiPortDetails{ + TxSupported: f.multiPort.Tx, + RxSupported: f.multiPort.Rx, + BasePort: uint32(f.multiPort.TxBasePort), + TotalPorts: uint32(f.multiPort.TxPorts), + } + } + if hs.Details.InitiatorMultiPort != nil && hs.Details.InitiatorMultiPort.BasePort != uint32(addr.Port) { + // The other side sent us a handshake from a different port, make sure + // we send responses back to the BasePort + addr = &udp.Addr{ + IP: addr.IP, + Port: uint16(hs.Details.InitiatorMultiPort.BasePort), + } + } + hostinfo := &HostInfo{ ConnectionState: ci, localIndexId: myIndex, @@ -137,6 +169,8 @@ func ixHandshakeStage1(f *Interface, addr *udp.Addr, via interface{}, packet []b vpnIp: vpnIp, HandshakePacket: make(map[uint8][]byte, 0), lastHandshakeTime: hs.Details.Time, + multiportTx: multiportTx, + multiportRx: multiportRx, relayState: RelayState{ relays: map[iputil.VpnIp]struct{}{}, relayForByIp: map[iputil.VpnIp]*Relay{}, @@ -153,6 +187,7 @@ func ixHandshakeStage1(f *Interface, addr *udp.Addr, via interface{}, packet []b WithField("issuer", issuer). WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex). WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). + WithField("multiportTx", multiportTx).WithField("multiportRx", multiportRx). Info("Handshake message received") hs.Details.ResponderIndex = myIndex @@ -231,7 +266,14 @@ func ixHandshakeStage1(f *Interface, addr *udp.Addr, via interface{}, packet []b msg = existing.HandshakePacket[2] f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1) if addr != nil { - err := f.outside.WriteTo(msg, addr) + if multiportTx { + // TODO remove alloc here + raw := make([]byte, len(msg)+udp.RawOverhead) + copy(raw[udp.RawOverhead:], msg) + err = f.udpRaw.WriteTo(raw, udp.RandomSendPort.UDPSendPort(f.multiPort.TxPorts), addr) + } else { + err = f.outside.WriteTo(msg, addr) + } if err != nil { f.l.WithField("vpnIp", existing.vpnIp).WithField("udpAddr", addr). WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true). @@ -308,7 +350,14 @@ func ixHandshakeStage1(f *Interface, addr *udp.Addr, via interface{}, packet []b // Do the send f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1) if addr != nil { - err = f.outside.WriteTo(msg, addr) + if multiportTx { + // TODO remove alloc here + raw := make([]byte, len(msg)+udp.RawOverhead) + copy(raw[udp.RawOverhead:], msg) + err = f.udpRaw.WriteTo(raw, udp.RandomSendPort.UDPSendPort(f.multiPort.TxPorts), addr) + } else { + err = f.outside.WriteTo(msg, addr) + } if err != nil { f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr). WithField("certName", certName). @@ -368,6 +417,11 @@ func ixHandshakeStage2(f *Interface, addr *udp.Addr, via interface{}, hostinfo * ci := hostinfo.ConnectionState if ci.ready { + if hostinfo.multiportRx { + // The other host is sending to us with multiport, so only grab the IP + addr.Port = hostinfo.remote.Port + } + f.l.WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr). WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("header", h). Info("Handshake is already complete") @@ -413,6 +467,20 @@ func ixHandshakeStage2(f *Interface, addr *udp.Addr, via interface{}, hostinfo * return true } + if (f.multiPort.Tx || f.multiPort.Rx) && hs.Details.ResponderMultiPort != nil { + hostinfo.multiportTx = hs.Details.ResponderMultiPort.RxSupported && f.multiPort.Tx + hostinfo.multiportRx = hs.Details.ResponderMultiPort.TxSupported && f.multiPort.Rx + } + + if hs.Details.ResponderMultiPort != nil && hs.Details.ResponderMultiPort.BasePort != uint32(addr.Port) { + // The other side sent us a handshake from a different port, make sure + // we send responses back to the BasePort + addr = &udp.Addr{ + IP: addr.IP, + Port: uint16(hs.Details.ResponderMultiPort.BasePort), + } + } + remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.caPool) if err != nil { f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr). @@ -480,6 +548,7 @@ func ixHandshakeStage2(f *Interface, addr *udp.Addr, via interface{}, hostinfo * WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}). WithField("durationNs", duration). WithField("sentCachedPackets", len(hostinfo.packetStore)). + WithField("multiportTx", hostinfo.multiportTx).WithField("multiportRx", hostinfo.multiportRx). Info("Handshake message received") hostinfo.remoteIndexId = hs.Details.ResponderIndex diff --git a/handshake_manager.go b/handshake_manager.go index cda1509..89d078b 100644 --- a/handshake_manager.go +++ b/handshake_manager.go @@ -53,6 +53,9 @@ type HandshakeManager struct { metricTimedOut metrics.Counter l *logrus.Logger + multiPort MultiPortConfig + udpRaw *udp.RawConn + // can be used to trigger outbound handshake for the given vpnIp trigger chan iputil.VpnIp } @@ -163,6 +166,7 @@ func (c *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, f udp.EncWriter, l // Send a the handshake to all known ips, stage 2 takes care of assigning the hostinfo.remote based on the first to reply var sentTo []*udp.Addr + var sentMultiport bool hostinfo.remotes.ForEach(c.pendingHostMap.preferredRanges, func(addr *udp.Addr, _ bool) { c.messageMetrics.Tx(header.Handshake, header.MessageSubType(hostinfo.HandshakePacket[0][1]), 1) err = c.outside.WriteTo(hostinfo.HandshakePacket[0], addr) @@ -175,6 +179,27 @@ func (c *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, f udp.EncWriter, l } else { sentTo = append(sentTo, addr) } + + // Attempt a multiport handshake if we are past the TxHandshakeDelay attempts + if c.multiPort.TxHandshake && c.udpRaw != nil && hostinfo.HandshakeCounter >= c.multiPort.TxHandshakeDelay { + sentMultiport = true + // We need to re-allocate with 8 bytes at the start of SOCK_RAW + raw := hostinfo.HandshakePacket[0x80] + if raw == nil { + raw = make([]byte, len(hostinfo.HandshakePacket[0])+udp.RawOverhead) + copy(raw[udp.RawOverhead:], hostinfo.HandshakePacket[0]) + hostinfo.HandshakePacket[0x80] = raw + } + + c.messageMetrics.Tx(header.Handshake, header.MessageSubType(hostinfo.HandshakePacket[0][1]), 1) + err = c.udpRaw.WriteTo(raw, udp.RandomSendPort.UDPSendPort(c.multiPort.TxPorts), addr) + if err != nil { + hostinfo.logger(c.l).WithField("udpAddr", addr). + WithField("initiatorIndex", hostinfo.localIndexId). + WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). + WithError(err).Error("Failed to send handshake message") + } + } }) // Don't be too noisy or confusing if we fail to send a handshake - if we don't get through we'll eventually log a timeout @@ -182,6 +207,7 @@ func (c *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, f udp.EncWriter, l hostinfo.logger(c.l).WithField("udpAddrs", sentTo). WithField("initiatorIndex", hostinfo.localIndexId). WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). + WithField("multiportHandshake", sentMultiport). Info("Handshake message sent") } diff --git a/hostmap.go b/hostmap.go index 402c1a8..f840c88 100644 --- a/hostmap.go +++ b/hostmap.go @@ -154,6 +154,8 @@ type HostInfo struct { remote *udp.Addr remotes *RemoteList promoteCounter uint32 + multiportTx bool + multiportRx bool ConnectionState *ConnectionState handshakeStart time.Time //todo: this an entry in the handshake manager HandshakeReady bool //todo: being in the manager means you are ready diff --git a/inside.go b/inside.go index 39c5a43..f412e75 100644 --- a/inside.go +++ b/inside.go @@ -70,7 +70,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet dropReason := f.firewall.Drop(packet, *fwPacket, false, hostinfo, f.caPool, localCache) if dropReason == nil { - f.sendNoMetrics(header.Message, 0, ci, hostinfo, nil, packet, nb, out, q) + f.sendNoMetrics(header.Message, 0, ci, hostinfo, nil, packet, nb, out, q, fwPacket) } else if f.l.Level >= logrus.DebugLevel { hostinfo.logger(f.l). @@ -161,7 +161,7 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp return } - f.sendNoMetrics(header.Message, st, hostInfo.ConnectionState, hostInfo, nil, p, nb, out, 0) + f.sendNoMetrics(header.Message, st, hostInfo.ConnectionState, hostInfo, nil, p, nb, out, 0, nil) } // SendMessageToVpnIp handles real ip:port lookup and sends to the current best known address for vpnIp @@ -197,12 +197,12 @@ func (f *Interface) sendMessageToVpnIp(t header.MessageType, st header.MessageSu func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p, nb, out []byte) { f.messageMetrics.Tx(t, st, 1) - f.sendNoMetrics(t, st, ci, hostinfo, nil, p, nb, out, 0) + f.sendNoMetrics(t, st, ci, hostinfo, nil, p, nb, out, 0, nil) } func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote *udp.Addr, p, nb, out []byte) { f.messageMetrics.Tx(t, st, 1) - f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0) + f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0, nil) } // sendVia sends a payload through a Relay tunnel. No authentication or encryption is done @@ -261,11 +261,28 @@ func (f *Interface) SendVia(viaIfc interface{}, } } -func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote *udp.Addr, p, nb, out []byte, q int) { +func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote *udp.Addr, p, nb, out []byte, q int, udpPortGetter udp.SendPortGetter) { if ci.eKey == nil { //TODO: log warning return } + + multiport := f.multiPort.Tx && hostinfo.multiportTx + rawOut := out + if multiport { + if len(out) < udp.RawOverhead { + // NOTE: This is because some spots in the code send us `out[:0]`, so + // we need to expand the slice back out to get our 8 bytes back. + out = out[:udp.RawOverhead] + } + // Preserve bytes needed for the raw socket + out = out[udp.RawOverhead:] + + if udpPortGetter == nil { + udpPortGetter = udp.RandomSendPort + } + } + useRelay := remote == nil && hostinfo.remote == nil fullOut := out @@ -312,13 +329,25 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType } if remote != nil { - err = f.writers[q].WriteTo(out, remote) + if multiport { + rawOut = rawOut[:len(out)+udp.RawOverhead] + port := udpPortGetter.UDPSendPort(f.multiPort.TxPorts) + err = f.udpRaw.WriteTo(rawOut, port, remote) + } else { + err = f.writers[q].WriteTo(out, remote) + } if err != nil { hostinfo.logger(f.l).WithError(err). WithField("udpAddr", remote).Error("Failed to write outgoing packet") } } else if hostinfo.remote != nil { - err = f.writers[q].WriteTo(out, hostinfo.remote) + if multiport { + rawOut = rawOut[:len(out)+udp.RawOverhead] + port := udpPortGetter.UDPSendPort(f.multiPort.TxPorts) + err = f.udpRaw.WriteTo(rawOut, port, hostinfo.remote) + } else { + err = f.writers[q].WriteTo(out, hostinfo.remote) + } if err != nil { hostinfo.logger(f.l).WithError(err). WithField("udpAddr", remote).Error("Failed to write outgoing packet") diff --git a/interface.go b/interface.go index a84eb7f..df5cadc 100644 --- a/interface.go +++ b/interface.go @@ -80,6 +80,9 @@ type Interface struct { writers []*udp.Conn readers []io.ReadWriteCloser + udpRaw *udp.RawConn + + multiPort MultiPortConfig metricHandshakes metrics.Histogram messageMetrics *MessageMetrics @@ -88,6 +91,15 @@ type Interface struct { l *logrus.Logger } +type MultiPortConfig struct { + Tx bool + Rx bool + TxBasePort uint16 + TxPorts int + TxHandshake bool + TxHandshakeDelay int +} + type sendRecvErrorConfig uint8 const ( @@ -194,6 +206,8 @@ func (f *Interface) activate() { metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines)) + metrics.GetOrRegisterGauge("multiport.tx_ports", nil).Update(int64(f.multiPort.TxPorts)) + // Prepare n tun queues var reader io.ReadWriteCloser = f.inside for i := 0; i < f.routines; i++ { @@ -378,6 +392,8 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) { udpStats := udp.NewUDPStatsEmitter(f.writers) + var rawStats func() + for { select { case <-ctx.Done(): @@ -386,6 +402,12 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) { f.firewall.EmitStats() f.handshakeManager.EmitStats() udpStats() + if f.udpRaw != nil { + if rawStats == nil { + rawStats = udp.NewRawStatsEmitter(f.udpRaw) + } + rawStats() + } } } } diff --git a/main.go b/main.go index 71bd233..c08082c 100644 --- a/main.go +++ b/main.go @@ -304,6 +304,39 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg // I don't want to make this initial commit too far-reaching though ifce.writers = udpConns + loadMultiPortConfig := func(c *config.C) { + ifce.multiPort.Rx = c.GetBool("tun.multiport.rx_enabled", false) + + tx := c.GetBool("tun.multiport.tx_enabled", false) + + if tx && ifce.udpRaw == nil { + ifce.udpRaw, err = udp.NewRawConn(l, c.GetString("listen.host", "0.0.0.0"), port, uint16(port)) + if err != nil { + l.WithError(err).Error("Failed to get raw socket for tun.multiport.tx_enabled") + ifce.udpRaw = nil + tx = false + } + } + + if tx { + ifce.multiPort.TxBasePort = uint16(port) + ifce.multiPort.TxPorts = c.GetInt("tun.multiport.tx_ports", 100) + ifce.multiPort.TxHandshake = c.GetBool("tun.multiport.tx_handshake", false) + ifce.multiPort.TxHandshakeDelay = c.GetInt("tun.multiport.tx_handshake_delay", 2) + ifce.udpRaw.ReloadConfig(c) + } + ifce.multiPort.Tx = tx + + // TODO: if we upstream this, make this cleaner + handshakeManager.udpRaw = ifce.udpRaw + handshakeManager.multiPort = ifce.multiPort + + l.WithField("multiPort", ifce.multiPort).Info("Multiport configured") + } + + loadMultiPortConfig(c) + c.RegisterReloadCallback(loadMultiPortConfig) + ifce.RegisterConfigChangeCallbacks(c) ifce.reloadSendRecvError(c) diff --git a/nebula.pb.go b/nebula.pb.go index 649b7cb..0844c83 100644 --- a/nebula.pb.go +++ b/nebula.pb.go @@ -121,7 +121,7 @@ func (x NebulaControl_MessageType) String() string { } func (NebulaControl_MessageType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_2d65afa7693df5ef, []int{7, 0} + return fileDescriptor_2d65afa7693df5ef, []int{8, 0} } type NebulaMeta struct { @@ -468,19 +468,89 @@ func (m *NebulaHandshake) GetHmac() []byte { return nil } +type MultiPortDetails struct { + RxSupported bool `protobuf:"varint,1,opt,name=RxSupported,proto3" json:"RxSupported,omitempty"` + TxSupported bool `protobuf:"varint,2,opt,name=TxSupported,proto3" json:"TxSupported,omitempty"` + BasePort uint32 `protobuf:"varint,3,opt,name=BasePort,proto3" json:"BasePort,omitempty"` + TotalPorts uint32 `protobuf:"varint,4,opt,name=TotalPorts,proto3" json:"TotalPorts,omitempty"` +} + +func (m *MultiPortDetails) Reset() { *m = MultiPortDetails{} } +func (m *MultiPortDetails) String() string { return proto.CompactTextString(m) } +func (*MultiPortDetails) ProtoMessage() {} +func (*MultiPortDetails) Descriptor() ([]byte, []int) { + return fileDescriptor_2d65afa7693df5ef, []int{6} +} +func (m *MultiPortDetails) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *MultiPortDetails) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_MultiPortDetails.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *MultiPortDetails) XXX_Merge(src proto.Message) { + xxx_messageInfo_MultiPortDetails.Merge(m, src) +} +func (m *MultiPortDetails) XXX_Size() int { + return m.Size() +} +func (m *MultiPortDetails) XXX_DiscardUnknown() { + xxx_messageInfo_MultiPortDetails.DiscardUnknown(m) +} + +var xxx_messageInfo_MultiPortDetails proto.InternalMessageInfo + +func (m *MultiPortDetails) GetRxSupported() bool { + if m != nil { + return m.RxSupported + } + return false +} + +func (m *MultiPortDetails) GetTxSupported() bool { + if m != nil { + return m.TxSupported + } + return false +} + +func (m *MultiPortDetails) GetBasePort() uint32 { + if m != nil { + return m.BasePort + } + return 0 +} + +func (m *MultiPortDetails) GetTotalPorts() uint32 { + if m != nil { + return m.TotalPorts + } + return 0 +} + type NebulaHandshakeDetails struct { - Cert []byte `protobuf:"bytes,1,opt,name=Cert,proto3" json:"Cert,omitempty"` - InitiatorIndex uint32 `protobuf:"varint,2,opt,name=InitiatorIndex,proto3" json:"InitiatorIndex,omitempty"` - ResponderIndex uint32 `protobuf:"varint,3,opt,name=ResponderIndex,proto3" json:"ResponderIndex,omitempty"` - Cookie uint64 `protobuf:"varint,4,opt,name=Cookie,proto3" json:"Cookie,omitempty"` - Time uint64 `protobuf:"varint,5,opt,name=Time,proto3" json:"Time,omitempty"` + Cert []byte `protobuf:"bytes,1,opt,name=Cert,proto3" json:"Cert,omitempty"` + InitiatorIndex uint32 `protobuf:"varint,2,opt,name=InitiatorIndex,proto3" json:"InitiatorIndex,omitempty"` + ResponderIndex uint32 `protobuf:"varint,3,opt,name=ResponderIndex,proto3" json:"ResponderIndex,omitempty"` + Cookie uint64 `protobuf:"varint,4,opt,name=Cookie,proto3" json:"Cookie,omitempty"` + Time uint64 `protobuf:"varint,5,opt,name=Time,proto3" json:"Time,omitempty"` + InitiatorMultiPort *MultiPortDetails `protobuf:"bytes,6,opt,name=InitiatorMultiPort,proto3" json:"InitiatorMultiPort,omitempty"` + ResponderMultiPort *MultiPortDetails `protobuf:"bytes,7,opt,name=ResponderMultiPort,proto3" json:"ResponderMultiPort,omitempty"` } func (m *NebulaHandshakeDetails) Reset() { *m = NebulaHandshakeDetails{} } func (m *NebulaHandshakeDetails) String() string { return proto.CompactTextString(m) } func (*NebulaHandshakeDetails) ProtoMessage() {} func (*NebulaHandshakeDetails) Descriptor() ([]byte, []int) { - return fileDescriptor_2d65afa7693df5ef, []int{6} + return fileDescriptor_2d65afa7693df5ef, []int{7} } func (m *NebulaHandshakeDetails) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -544,6 +614,20 @@ func (m *NebulaHandshakeDetails) GetTime() uint64 { return 0 } +func (m *NebulaHandshakeDetails) GetInitiatorMultiPort() *MultiPortDetails { + if m != nil { + return m.InitiatorMultiPort + } + return nil +} + +func (m *NebulaHandshakeDetails) GetResponderMultiPort() *MultiPortDetails { + if m != nil { + return m.ResponderMultiPort + } + return nil +} + type NebulaControl struct { Type NebulaControl_MessageType `protobuf:"varint,1,opt,name=Type,proto3,enum=nebula.NebulaControl_MessageType" json:"Type,omitempty"` InitiatorRelayIndex uint32 `protobuf:"varint,2,opt,name=InitiatorRelayIndex,proto3" json:"InitiatorRelayIndex,omitempty"` @@ -556,7 +640,7 @@ func (m *NebulaControl) Reset() { *m = NebulaControl{} } func (m *NebulaControl) String() string { return proto.CompactTextString(m) } func (*NebulaControl) ProtoMessage() {} func (*NebulaControl) Descriptor() ([]byte, []int) { - return fileDescriptor_2d65afa7693df5ef, []int{7} + return fileDescriptor_2d65afa7693df5ef, []int{8} } func (m *NebulaControl) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -630,6 +714,7 @@ func init() { proto.RegisterType((*Ip6AndPort)(nil), "nebula.Ip6AndPort") proto.RegisterType((*NebulaPing)(nil), "nebula.NebulaPing") proto.RegisterType((*NebulaHandshake)(nil), "nebula.NebulaHandshake") + proto.RegisterType((*MultiPortDetails)(nil), "nebula.MultiPortDetails") proto.RegisterType((*NebulaHandshakeDetails)(nil), "nebula.NebulaHandshakeDetails") proto.RegisterType((*NebulaControl)(nil), "nebula.NebulaControl") } @@ -637,51 +722,56 @@ func init() { func init() { proto.RegisterFile("nebula.proto", fileDescriptor_2d65afa7693df5ef) } var fileDescriptor_2d65afa7693df5ef = []byte{ - // 696 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x54, 0xcd, 0x6e, 0xd3, 0x4a, - 0x14, 0x8e, 0x1d, 0xe7, 0xef, 0xa4, 0x49, 0x7d, 0x4f, 0xef, 0xcd, 0x4d, 0xaf, 0xae, 0xac, 0xe0, - 0x05, 0xca, 0x2a, 0xad, 0xd2, 0x52, 0xb1, 0x04, 0x82, 0x50, 0x52, 0xb5, 0x55, 0x18, 0x15, 0x90, - 0xd8, 0xa0, 0x69, 0x32, 0xd4, 0x56, 0x12, 0x8f, 0x6b, 0x4f, 0x50, 0xf3, 0x16, 0x3c, 0x4c, 0x1f, - 0x82, 0x05, 0x12, 0x5d, 0xb0, 0x60, 0x89, 0xda, 0x17, 0x41, 0x33, 0x76, 0x6c, 0x27, 0x0d, 0xec, - 0xce, 0xcf, 0xf7, 0xcd, 0x7c, 0xe7, 0x9b, 0x63, 0xc3, 0x96, 0xc7, 0x2e, 0xe6, 0x53, 0xda, 0xf1, - 0x03, 0x2e, 0x38, 0x16, 0xa3, 0xcc, 0xfe, 0xaa, 0x03, 0x9c, 0xa9, 0xf0, 0x94, 0x09, 0x8a, 0x5d, - 0x30, 0xce, 0x17, 0x3e, 0x6b, 0x6a, 0x2d, 0xad, 0x5d, 0xef, 0x5a, 0x9d, 0x98, 0x93, 0x22, 0x3a, - 0xa7, 0x2c, 0x0c, 0xe9, 0x25, 0x93, 0x28, 0xa2, 0xb0, 0x78, 0x00, 0xa5, 0x97, 0x4c, 0x50, 0x77, - 0x1a, 0x36, 0xf5, 0x96, 0xd6, 0xae, 0x76, 0x77, 0x1f, 0xd2, 0x62, 0x00, 0x59, 0x22, 0xed, 0xef, - 0x1a, 0x54, 0x33, 0x47, 0x61, 0x19, 0x8c, 0x33, 0xee, 0x31, 0x33, 0x87, 0x35, 0xa8, 0xf4, 0x79, - 0x28, 0x5e, 0xcf, 0x59, 0xb0, 0x30, 0x35, 0x44, 0xa8, 0x27, 0x29, 0x61, 0xfe, 0x74, 0x61, 0xea, - 0xf8, 0x1f, 0x34, 0x64, 0xed, 0x8d, 0x3f, 0xa6, 0x82, 0x9d, 0x71, 0xe1, 0x7e, 0x74, 0x47, 0x54, - 0xb8, 0xdc, 0x33, 0xf3, 0xb8, 0x0b, 0xff, 0xc8, 0xde, 0x29, 0xff, 0xc4, 0xc6, 0x2b, 0x2d, 0x63, - 0xd9, 0x1a, 0xce, 0xbd, 0x91, 0xb3, 0xd2, 0x2a, 0x60, 0x1d, 0x40, 0xb6, 0xde, 0x39, 0x9c, 0xce, - 0x5c, 0xb3, 0x88, 0x3b, 0xb0, 0x9d, 0xe6, 0xd1, 0xb5, 0x25, 0xa9, 0x6c, 0x48, 0x85, 0xd3, 0x73, - 0xd8, 0x68, 0x62, 0x96, 0xa5, 0xb2, 0x24, 0x8d, 0x20, 0x15, 0xfb, 0x9b, 0x06, 0x7f, 0x3d, 0x98, - 0x1a, 0xff, 0x86, 0xc2, 0x5b, 0xdf, 0x1b, 0xf8, 0xca, 0xd6, 0x1a, 0x89, 0x12, 0x3c, 0x84, 0xea, - 0xc0, 0x3f, 0x7c, 0xee, 0x8d, 0x87, 0x3c, 0x10, 0xd2, 0xbb, 0x7c, 0xbb, 0xda, 0xc5, 0xa5, 0x77, - 0x69, 0x8b, 0x64, 0x61, 0x11, 0xeb, 0x28, 0x61, 0x19, 0xeb, 0xac, 0xa3, 0x0c, 0x2b, 0x81, 0xa1, - 0x05, 0x40, 0xd8, 0x94, 0x2e, 0x22, 0x19, 0x85, 0x56, 0xbe, 0x5d, 0x23, 0x99, 0x0a, 0x36, 0xa1, - 0x34, 0xe2, 0x73, 0x4f, 0xb0, 0xa0, 0x99, 0x57, 0x1a, 0x97, 0xa9, 0xbd, 0x0f, 0x90, 0x5e, 0x8f, - 0x75, 0xd0, 0x93, 0x31, 0xf4, 0x81, 0x8f, 0x08, 0x86, 0xac, 0xab, 0x87, 0xaf, 0x11, 0x15, 0xdb, - 0xcf, 0x24, 0xe3, 0x28, 0xc3, 0xe8, 0xbb, 0x8a, 0x61, 0x10, 0xbd, 0xef, 0xca, 0xfc, 0x84, 0x2b, - 0xbc, 0x41, 0xf4, 0x13, 0x9e, 0x9c, 0x90, 0xcf, 0x9c, 0x70, 0xbd, 0xdc, 0xc9, 0xa1, 0xeb, 0x5d, - 0xfe, 0x79, 0x27, 0x25, 0x62, 0xc3, 0x4e, 0x22, 0x18, 0xe7, 0xee, 0x8c, 0xc5, 0xf7, 0xa8, 0xd8, - 0xb6, 0x1f, 0x6c, 0x9c, 0x24, 0x9b, 0x39, 0xac, 0x40, 0x21, 0x7a, 0x3f, 0xcd, 0xfe, 0x00, 0xdb, - 0xd1, 0xb9, 0x7d, 0xea, 0x8d, 0x43, 0x87, 0x4e, 0x18, 0x3e, 0x4d, 0xd7, 0x5b, 0x53, 0xeb, 0xbd, - 0xa6, 0x20, 0x41, 0xae, 0xef, 0xb8, 0x14, 0xd1, 0x9f, 0xd1, 0x91, 0x12, 0xb1, 0x45, 0x54, 0x6c, - 0xdf, 0x68, 0xd0, 0xd8, 0xcc, 0x93, 0xf0, 0x1e, 0x0b, 0x84, 0xba, 0x65, 0x8b, 0xa8, 0x18, 0x1f, - 0x43, 0x7d, 0xe0, 0xb9, 0xc2, 0xa5, 0x82, 0x07, 0x03, 0x6f, 0xcc, 0xae, 0x63, 0xa7, 0xd7, 0xaa, - 0x12, 0x47, 0x58, 0xe8, 0x73, 0x6f, 0xcc, 0x62, 0x5c, 0xe4, 0xe7, 0x5a, 0x15, 0x1b, 0x50, 0xec, - 0x71, 0x3e, 0x71, 0x59, 0xd3, 0x50, 0xce, 0xc4, 0x59, 0xe2, 0x57, 0x21, 0xf5, 0xeb, 0xd8, 0x28, - 0x17, 0xcd, 0xd2, 0xb1, 0x51, 0x2e, 0x99, 0x65, 0xfb, 0x46, 0x87, 0x5a, 0x24, 0xbb, 0xc7, 0x3d, - 0x11, 0xf0, 0x29, 0x3e, 0x59, 0x79, 0x95, 0x47, 0xab, 0x9e, 0xc4, 0xa0, 0x0d, 0x0f, 0xb3, 0x0f, - 0x3b, 0x89, 0x74, 0xb5, 0x7f, 0xd9, 0xa9, 0x36, 0xb5, 0x24, 0x23, 0x19, 0x22, 0xc3, 0x88, 0xe6, - 0xdb, 0xd4, 0xc2, 0xff, 0xa1, 0xa2, 0xb2, 0x73, 0x3e, 0xf0, 0xd5, 0x9c, 0x35, 0x92, 0x16, 0xb0, - 0x05, 0x55, 0x95, 0xbc, 0x0a, 0xf8, 0x4c, 0x7d, 0x0b, 0xb2, 0x9f, 0x2d, 0xd9, 0xfd, 0xdf, 0xfd, - 0x9a, 0x1a, 0x80, 0xbd, 0x80, 0x51, 0xc1, 0x14, 0x9a, 0xb0, 0xab, 0x39, 0x0b, 0x85, 0xa9, 0xe1, - 0xbf, 0xb0, 0xb3, 0x52, 0x97, 0x92, 0x42, 0x66, 0xea, 0x2f, 0x0e, 0xbe, 0xdc, 0x59, 0xda, 0xed, - 0x9d, 0xa5, 0xfd, 0xbc, 0xb3, 0xb4, 0xcf, 0xf7, 0x56, 0xee, 0xf6, 0xde, 0xca, 0xfd, 0xb8, 0xb7, - 0x72, 0xef, 0x77, 0x2f, 0x5d, 0xe1, 0xcc, 0x2f, 0x3a, 0x23, 0x3e, 0xdb, 0x0b, 0xa7, 0x74, 0x34, - 0x71, 0xae, 0xf6, 0x22, 0x0b, 0x2f, 0x8a, 0xea, 0x0f, 0x7d, 0xf0, 0x2b, 0x00, 0x00, 0xff, 0xff, - 0xcd, 0xd7, 0xbe, 0xd5, 0xb1, 0x05, 0x00, 0x00, + // 775 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x55, 0xcd, 0x6e, 0xd3, 0x4a, + 0x14, 0x8e, 0x1d, 0xe7, 0xef, 0xa4, 0x49, 0x7d, 0xa7, 0xf7, 0xe6, 0xa6, 0xd5, 0x95, 0x95, 0xeb, + 0xc5, 0x55, 0x56, 0x69, 0x95, 0xf6, 0x56, 0x2c, 0xa1, 0x41, 0x28, 0x91, 0x9a, 0x2a, 0x0c, 0x01, + 0x24, 0x36, 0x68, 0x9a, 0x0c, 0x8d, 0x15, 0xc7, 0xe3, 0xda, 0x63, 0xd4, 0xbc, 0x05, 0xe2, 0x59, + 0x58, 0xf2, 0x00, 0x2c, 0x90, 0xe8, 0x82, 0x05, 0x4b, 0xd4, 0xbe, 0x08, 0x9a, 0xf1, 0x6f, 0x7e, + 0x80, 0xdd, 0x9c, 0x73, 0xbe, 0xef, 0xcc, 0x37, 0xe7, 0x7c, 0x71, 0x60, 0xc7, 0xa1, 0x97, 0x81, + 0x4d, 0x3a, 0xae, 0xc7, 0x38, 0x43, 0xc5, 0x30, 0x32, 0x3f, 0xab, 0x00, 0x17, 0xf2, 0x38, 0xa4, + 0x9c, 0xa0, 0x2e, 0x68, 0xe3, 0xa5, 0x4b, 0x9b, 0x4a, 0x4b, 0x69, 0xd7, 0xbb, 0x46, 0x27, 0xe2, + 0xa4, 0x88, 0xce, 0x90, 0xfa, 0x3e, 0xb9, 0xa2, 0x02, 0x85, 0x25, 0x16, 0x1d, 0x43, 0xe9, 0x31, + 0xe5, 0xc4, 0xb2, 0xfd, 0xa6, 0xda, 0x52, 0xda, 0xd5, 0xee, 0xfe, 0x26, 0x2d, 0x02, 0xe0, 0x18, + 0x69, 0x7e, 0x55, 0xa0, 0x9a, 0x69, 0x85, 0xca, 0xa0, 0x5d, 0x30, 0x87, 0xea, 0x39, 0x54, 0x83, + 0x4a, 0x9f, 0xf9, 0xfc, 0x69, 0x40, 0xbd, 0xa5, 0xae, 0x20, 0x04, 0xf5, 0x24, 0xc4, 0xd4, 0xb5, + 0x97, 0xba, 0x8a, 0x0e, 0xa0, 0x21, 0x72, 0xcf, 0xdd, 0x29, 0xe1, 0xf4, 0x82, 0x71, 0xeb, 0x8d, + 0x35, 0x21, 0xdc, 0x62, 0x8e, 0x9e, 0x47, 0xfb, 0xf0, 0x97, 0xa8, 0x0d, 0xd9, 0x5b, 0x3a, 0x5d, + 0x29, 0x69, 0x71, 0x69, 0x14, 0x38, 0x93, 0xd9, 0x4a, 0xa9, 0x80, 0xea, 0x00, 0xa2, 0xf4, 0x72, + 0xc6, 0xc8, 0xc2, 0xd2, 0x8b, 0x68, 0x0f, 0x76, 0xd3, 0x38, 0xbc, 0xb6, 0x24, 0x94, 0x8d, 0x08, + 0x9f, 0xf5, 0x66, 0x74, 0x32, 0xd7, 0xcb, 0x42, 0x59, 0x12, 0x86, 0x90, 0x8a, 0xf9, 0x45, 0x81, + 0x3f, 0x36, 0x5e, 0x8d, 0xfe, 0x84, 0xc2, 0x0b, 0xd7, 0x19, 0xb8, 0x72, 0xac, 0x35, 0x1c, 0x06, + 0xe8, 0x04, 0xaa, 0x03, 0xf7, 0xe4, 0x91, 0x33, 0x1d, 0x31, 0x8f, 0x8b, 0xd9, 0xe5, 0xdb, 0xd5, + 0x2e, 0x8a, 0x67, 0x97, 0x96, 0x70, 0x16, 0x16, 0xb2, 0x4e, 0x13, 0x96, 0xb6, 0xce, 0x3a, 0xcd, + 0xb0, 0x12, 0x18, 0x32, 0x00, 0x30, 0xb5, 0xc9, 0x32, 0x94, 0x51, 0x68, 0xe5, 0xdb, 0x35, 0x9c, + 0xc9, 0xa0, 0x26, 0x94, 0x26, 0x2c, 0x70, 0x38, 0xf5, 0x9a, 0x79, 0xa9, 0x31, 0x0e, 0xcd, 0x23, + 0x80, 0xf4, 0x7a, 0x54, 0x07, 0x35, 0x79, 0x86, 0x3a, 0x70, 0x11, 0x02, 0x4d, 0xe4, 0xe5, 0xe2, + 0x6b, 0x58, 0x9e, 0xcd, 0x87, 0x82, 0x71, 0x9a, 0x61, 0xf4, 0x2d, 0xc9, 0xd0, 0xb0, 0xda, 0xb7, + 0x44, 0x7c, 0xce, 0x24, 0x5e, 0xc3, 0xea, 0x39, 0x4b, 0x3a, 0xe4, 0x33, 0x1d, 0x6e, 0x62, 0x4f, + 0x8e, 0x2c, 0xe7, 0xea, 0xd7, 0x9e, 0x14, 0x88, 0x2d, 0x9e, 0x44, 0xa0, 0x8d, 0xad, 0x05, 0x8d, + 0xee, 0x91, 0x67, 0xd3, 0xdc, 0x70, 0x9c, 0x20, 0xeb, 0x39, 0x54, 0x81, 0x42, 0xb8, 0x3f, 0xc5, + 0x7c, 0x0d, 0xbb, 0x61, 0xdf, 0x3e, 0x71, 0xa6, 0xfe, 0x8c, 0xcc, 0x29, 0x7a, 0x90, 0xda, 0x5b, + 0x91, 0xf6, 0x5e, 0x53, 0x90, 0x20, 0xd7, 0x3d, 0x2e, 0x44, 0xf4, 0x17, 0x64, 0x22, 0x45, 0xec, + 0x60, 0x79, 0x36, 0xdf, 0x2b, 0xa0, 0x0f, 0x03, 0x9b, 0x5b, 0xe2, 0xa1, 0x31, 0xb0, 0x05, 0x55, + 0x7c, 0xf3, 0x2c, 0x70, 0x5d, 0xe6, 0x71, 0x3a, 0x95, 0xd7, 0x94, 0x71, 0x36, 0x25, 0x10, 0xe3, + 0x0c, 0x42, 0x0d, 0x11, 0x99, 0x14, 0x3a, 0x80, 0xf2, 0x19, 0xf1, 0x69, 0x66, 0x96, 0x49, 0x2c, + 0xb6, 0x3f, 0x66, 0x9c, 0xd8, 0xb1, 0x65, 0x44, 0x35, 0x93, 0x31, 0x3f, 0xaa, 0xd0, 0xd8, 0xfe, + 0x18, 0xf1, 0x86, 0x1e, 0xf5, 0xb8, 0xd4, 0xb4, 0x83, 0xe5, 0x19, 0xfd, 0x07, 0xf5, 0x81, 0x63, + 0x71, 0x8b, 0x70, 0xe6, 0x0d, 0x9c, 0x29, 0xbd, 0x89, 0xd6, 0xbf, 0x96, 0x15, 0x38, 0x4c, 0x7d, + 0x97, 0x39, 0x53, 0x1a, 0xe1, 0x42, 0x61, 0x6b, 0x59, 0xd4, 0x80, 0x62, 0x8f, 0xb1, 0xb9, 0x45, + 0xa5, 0x34, 0x0d, 0x47, 0x51, 0xb2, 0xc4, 0x42, 0xba, 0x44, 0xd4, 0x07, 0x94, 0xdc, 0x92, 0xcc, + 0xb1, 0x59, 0x94, 0x8b, 0x69, 0xc6, 0x8b, 0x59, 0x1f, 0x30, 0xde, 0xc2, 0x11, 0x9d, 0x12, 0x1d, + 0x69, 0xa7, 0xd2, 0xef, 0x3a, 0x6d, 0x72, 0xcc, 0x0f, 0x2a, 0xd4, 0xc2, 0xf1, 0xf5, 0x98, 0xc3, + 0x3d, 0x66, 0xa3, 0xff, 0x57, 0x2c, 0xfb, 0xef, 0xaa, 0x61, 0x22, 0xd0, 0x16, 0xd7, 0x1e, 0xc1, + 0x5e, 0x22, 0x54, 0xfe, 0x38, 0xb3, 0xd3, 0xdd, 0x56, 0x12, 0x8c, 0x44, 0x50, 0x86, 0x11, 0xce, + 0x79, 0x5b, 0x09, 0xfd, 0x03, 0x15, 0x19, 0x8d, 0xd9, 0xc0, 0x8d, 0xac, 0x90, 0x26, 0xa4, 0x13, + 0x45, 0xf0, 0xc4, 0x63, 0x0b, 0xf9, 0xa1, 0x10, 0xf5, 0x6c, 0xca, 0xec, 0xff, 0xec, 0xbb, 0xdd, + 0x00, 0xd4, 0xf3, 0x28, 0xe1, 0x54, 0xa2, 0x31, 0xbd, 0x0e, 0xa8, 0xcf, 0x75, 0x05, 0xfd, 0x0d, + 0x7b, 0x2b, 0x79, 0x21, 0xc9, 0xa7, 0xba, 0x7a, 0x76, 0xfc, 0xe9, 0xce, 0x50, 0x6e, 0xef, 0x0c, + 0xe5, 0xfb, 0x9d, 0xa1, 0xbc, 0xbb, 0x37, 0x72, 0xb7, 0xf7, 0x46, 0xee, 0xdb, 0xbd, 0x91, 0x7b, + 0xb5, 0x7f, 0x65, 0xf1, 0x59, 0x70, 0xd9, 0x99, 0xb0, 0xc5, 0xa1, 0x6f, 0x93, 0xc9, 0x7c, 0x76, + 0x7d, 0x18, 0x8e, 0xf0, 0xb2, 0x28, 0xff, 0xbe, 0x8e, 0x7f, 0x04, 0x00, 0x00, 0xff, 0xff, 0xdc, + 0x87, 0xe2, 0x33, 0xce, 0x06, 0x00, 0x00, } func (m *NebulaMeta) Marshal() (dAtA []byte, err error) { @@ -949,6 +1039,59 @@ func (m *NebulaHandshake) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *MultiPortDetails) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *MultiPortDetails) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *MultiPortDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.TotalPorts != 0 { + i = encodeVarintNebula(dAtA, i, uint64(m.TotalPorts)) + i-- + dAtA[i] = 0x20 + } + if m.BasePort != 0 { + i = encodeVarintNebula(dAtA, i, uint64(m.BasePort)) + i-- + dAtA[i] = 0x18 + } + if m.TxSupported { + i-- + if m.TxSupported { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x10 + } + if m.RxSupported { + i-- + if m.RxSupported { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *NebulaHandshakeDetails) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -969,6 +1112,30 @@ func (m *NebulaHandshakeDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) _ = i var l int _ = l + if m.ResponderMultiPort != nil { + { + size, err := m.ResponderMultiPort.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintNebula(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } + if m.InitiatorMultiPort != nil { + { + size, err := m.InitiatorMultiPort.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintNebula(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } if m.Time != 0 { i = encodeVarintNebula(dAtA, i, uint64(m.Time)) i-- @@ -1173,6 +1340,27 @@ func (m *NebulaHandshake) Size() (n int) { return n } +func (m *MultiPortDetails) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.RxSupported { + n += 2 + } + if m.TxSupported { + n += 2 + } + if m.BasePort != 0 { + n += 1 + sovNebula(uint64(m.BasePort)) + } + if m.TotalPorts != 0 { + n += 1 + sovNebula(uint64(m.TotalPorts)) + } + return n +} + func (m *NebulaHandshakeDetails) Size() (n int) { if m == nil { return 0 @@ -1195,6 +1383,14 @@ func (m *NebulaHandshakeDetails) Size() (n int) { if m.Time != 0 { n += 1 + sovNebula(uint64(m.Time)) } + if m.InitiatorMultiPort != nil { + l = m.InitiatorMultiPort.Size() + n += 1 + l + sovNebula(uint64(l)) + } + if m.ResponderMultiPort != nil { + l = m.ResponderMultiPort.Size() + n += 1 + l + sovNebula(uint64(l)) + } return n } @@ -1968,6 +2164,134 @@ func (m *NebulaHandshake) Unmarshal(dAtA []byte) error { } return nil } +func (m *MultiPortDetails) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowNebula + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: MultiPortDetails: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: MultiPortDetails: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RxSupported", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowNebula + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RxSupported = bool(v != 0) + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TxSupported", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowNebula + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.TxSupported = bool(v != 0) + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BasePort", wireType) + } + m.BasePort = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowNebula + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.BasePort |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TotalPorts", wireType) + } + m.TotalPorts = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowNebula + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TotalPorts |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipNebula(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthNebula + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *NebulaHandshakeDetails) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -2107,6 +2431,78 @@ func (m *NebulaHandshakeDetails) Unmarshal(dAtA []byte) error { break } } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InitiatorMultiPort", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowNebula + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthNebula + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthNebula + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.InitiatorMultiPort == nil { + m.InitiatorMultiPort = &MultiPortDetails{} + } + if err := m.InitiatorMultiPort.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ResponderMultiPort", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowNebula + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthNebula + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthNebula + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ResponderMultiPort == nil { + m.ResponderMultiPort = &MultiPortDetails{} + } + if err := m.ResponderMultiPort.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipNebula(dAtA[iNdEx:]) diff --git a/nebula.proto b/nebula.proto index 5e839be..d4ff4e6 100644 --- a/nebula.proto +++ b/nebula.proto @@ -55,14 +55,22 @@ message NebulaHandshake { bytes Hmac = 2; } +message MultiPortDetails { + bool RxSupported = 1; + bool TxSupported = 2; + uint32 BasePort = 3; + uint32 TotalPorts = 4; +} + message NebulaHandshakeDetails { bytes Cert = 1; uint32 InitiatorIndex = 2; uint32 ResponderIndex = 3; uint64 Cookie = 4; uint64 Time = 5; - // reserved for WIP multiport - reserved 6, 7; + + MultiPortDetails InitiatorMultiPort = 6; + MultiPortDetails ResponderMultiPort = 7; } message NebulaControl { diff --git a/outside.go b/outside.go index 1a1ad6f..11e9cdc 100644 --- a/outside.go +++ b/outside.go @@ -257,6 +257,19 @@ func (f *Interface) sendCloseTunnel(h *HostInfo) { func (f *Interface) handleHostRoaming(hostinfo *HostInfo, addr *udp.Addr) { if addr != nil && !hostinfo.remote.Equals(addr) { + if hostinfo.multiportRx { + // If the remote is sending with multiport, we aren't roaming unless + // the IP has changed + if hostinfo.remote.IP.Equal(addr.IP) { + return + } + // Keep the port from the original hostinfo, because the remote is transmitting from multiport ports + addr = &udp.Addr{ + IP: addr.IP, + Port: hostinfo.remote.Port, + } + } + if !f.lightHouse.GetRemoteAllowList().Allow(hostinfo.vpnIp, addr.IP) { hostinfo.logger(f.l).WithField("newAddr", addr).Debug("lighthouse.remote_allow_list denied roaming") return diff --git a/udp/udp_raw.go b/udp/udp_raw.go new file mode 100644 index 0000000..4c5c287 --- /dev/null +++ b/udp/udp_raw.go @@ -0,0 +1,16 @@ +package udp + +import mathrand "math/rand" + +type SendPortGetter interface { + // UDPSendPort returns the port to use + UDPSendPort(maxPort int) uint16 +} + +type randomSendPort struct{} + +func (randomSendPort) UDPSendPort(maxPort int) uint16 { + return uint16(mathrand.Intn(maxPort)) +} + +var RandomSendPort = randomSendPort{} diff --git a/udp/udp_raw_linux.go b/udp/udp_raw_linux.go new file mode 100644 index 0000000..8da6dfb --- /dev/null +++ b/udp/udp_raw_linux.go @@ -0,0 +1,190 @@ +//go:build !e2e_testing +// +build !e2e_testing + +package udp + +import ( + "encoding/binary" + "fmt" + "net" + "syscall" + "unsafe" + + "github.com/rcrowley/go-metrics" + "github.com/sirupsen/logrus" + "github.com/slackhq/nebula/config" + "golang.org/x/net/ipv4" + "golang.org/x/sys/unix" +) + +// RawOverhead is the number of bytes that need to be reserved at the start of +// the raw bytes passed to (*RawConn).WriteTo. This is used by WriteTo to prefix +// the IP and UDP headers. +const RawOverhead = 28 + +type RawConn struct { + sysFd int + basePort uint16 + l *logrus.Logger +} + +func NewRawConn(l *logrus.Logger, ip string, port int, basePort uint16) (*RawConn, error) { + syscall.ForkLock.RLock() + // With IPPROTO_UDP, the linux kernel tries to deliver every UDP packet + // received in the system to our socket. This constantly overflows our + // buffer and marks our socket as having dropped packets. This makes the + // stats on the socket useless. + // + // In contrast, IPPROTO_RAW is not delivered any packets and thus our read + // buffer will not fill up and mark as having dropped packets. The only + // difference is that we have to assemble the IP header as well, but this + // is fairly easy since Linux does the checksum for us. + // + // TODO: How to get this working with Inet6 correctly? I was having issues + // with the source address when testing before, probably need to `bind(2)`? + fd, err := unix.Socket(unix.AF_INET, unix.SOCK_RAW, unix.IPPROTO_RAW) + if err == nil { + unix.CloseOnExec(fd) + } + syscall.ForkLock.RUnlock() + if err != nil { + return nil, err + } + + // We only want to send, not recv. This will hopefully help the kernel avoid + // wasting time on us + if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, 0); err != nil { + return nil, fmt.Errorf("unable to set SO_RCVBUF: %s", err) + } + + var lip [16]byte + copy(lip[:], net.ParseIP(ip)) + + // TODO do we need to `bind(2)` so that we send from the correct address/interface? + if err = unix.Bind(fd, &unix.SockaddrInet6{Addr: lip, Port: port}); err != nil { + return nil, fmt.Errorf("unable to bind to socket: %s", err) + } + + return &RawConn{ + sysFd: fd, + basePort: basePort, + l: l, + }, nil +} + +// WriteTo must be called with raw leaving the first `udp.RawOverhead` bytes empty, +// for the IP/UDP headers. +func (u *RawConn) WriteTo(raw []byte, fromPort uint16, addr *Addr) error { + var rsa unix.RawSockaddrInet4 + rsa.Family = unix.AF_INET + copy(rsa.Addr[:], addr.IP.To4()) + + totalLen := len(raw) + udpLen := totalLen - ipv4.HeaderLen + + // IP header + raw[0] = byte(ipv4.Version<<4 | (ipv4.HeaderLen >> 2 & 0x0f)) + raw[1] = 0 // tos + binary.BigEndian.PutUint16(raw[2:4], uint16(totalLen)) + binary.BigEndian.PutUint16(raw[4:6], 0) // id (linux does it for us) + binary.BigEndian.PutUint16(raw[6:8], 0) // frag options + raw[8] = byte(64) // ttl + raw[9] = byte(17) // protocol + binary.BigEndian.PutUint16(raw[10:12], 0) // checksum (linux does it for us) + binary.BigEndian.PutUint32(raw[12:16], 0) // src (linux does it for us) + copy(raw[16:20], rsa.Addr[:]) // dst + + // UDP header + fromPort = u.basePort + fromPort + binary.BigEndian.PutUint16(raw[20:22], uint16(fromPort)) // src port + binary.BigEndian.PutUint16(raw[22:24], uint16(addr.Port)) // dst port + binary.BigEndian.PutUint16(raw[24:26], uint16(udpLen)) // UDP length + binary.BigEndian.PutUint16(raw[26:28], 0) // checksum (optional) + + for { + _, _, err := unix.Syscall6( + unix.SYS_SENDTO, + uintptr(u.sysFd), + uintptr(unsafe.Pointer(&raw[0])), + uintptr(len(raw)), + uintptr(0), + uintptr(unsafe.Pointer(&rsa)), + uintptr(unix.SizeofSockaddrInet4), + ) + + if err != 0 { + return &net.OpError{Op: "sendto", Err: err} + } + + //TODO: handle incomplete writes + + return nil + } +} + +func (u *RawConn) ReloadConfig(c *config.C) { + b := c.GetInt("listen.write_buffer", 0) + if b <= 0 { + return + } + + if err := u.SetSendBuffer(b); err != nil { + u.l.WithError(err).Error("Failed to set listen.write_buffer") + return + } + + s, err := u.GetSendBuffer() + if err != nil { + u.l.WithError(err).Warn("Failed to get listen.write_buffer") + return + } + + u.l.WithField("size", s).Info("listen.write_buffer was set") +} + +func (u *RawConn) SetSendBuffer(n int) error { + return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n) +} + +func (u *RawConn) GetSendBuffer() (int, error) { + return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF) +} + +func (u *RawConn) getMemInfo(meminfo *_SK_MEMINFO) error { + var vallen uint32 = 4 * _SK_MEMINFO_VARS + _, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0) + if err != 0 { + return err + } + return nil +} + +func NewRawStatsEmitter(rawConn *RawConn) func() { + // Check if our kernel supports SO_MEMINFO before registering the gauges + var gauges [_SK_MEMINFO_VARS]metrics.Gauge + var meminfo _SK_MEMINFO + if err := rawConn.getMemInfo(&meminfo); err == nil { + gauges = [_SK_MEMINFO_VARS]metrics.Gauge{ + metrics.GetOrRegisterGauge("raw.rmem_alloc", nil), + metrics.GetOrRegisterGauge("raw.rcvbuf", nil), + metrics.GetOrRegisterGauge("raw.wmem_alloc", nil), + metrics.GetOrRegisterGauge("raw.sndbuf", nil), + metrics.GetOrRegisterGauge("raw.fwd_alloc", nil), + metrics.GetOrRegisterGauge("raw.wmem_queued", nil), + metrics.GetOrRegisterGauge("raw.optmem", nil), + metrics.GetOrRegisterGauge("raw.backlog", nil), + metrics.GetOrRegisterGauge("raw.drops", nil), + } + } else { + // return no-op because we don't support SO_MEMINFO + return func() {} + } + + return func() { + if err := rawConn.getMemInfo(&meminfo); err == nil { + for j := 0; j < _SK_MEMINFO_VARS; j++ { + gauges[j].Update(int64(meminfo[j])) + } + } + } +} diff --git a/udp/udp_raw_unsupported.go b/udp/udp_raw_unsupported.go new file mode 100644 index 0000000..4e8d83d --- /dev/null +++ b/udp/udp_raw_unsupported.go @@ -0,0 +1,28 @@ +//go:build !linux || e2e_testing +// +build !linux e2e_testing + +package udp + +import ( + "fmt" + "runtime" + + "github.com/sirupsen/logrus" + "github.com/slackhq/nebula/config" +) + +const RawOverhead = 0 + +type RawConn struct{} + +func NewRawConn(l *logrus.Logger, ip string, port int, basePort uint16) (*RawConn, error) { + return nil, fmt.Errorf("multiport tx is not supported on %s", runtime.GOOS) +} + +func (u *RawConn) WriteTo(raw []byte, fromPort uint16, addr *Addr) error { + return fmt.Errorf("multiport tx is not supported on %s", runtime.GOOS) +} + +func (u *RawConn) ReloadConfig(c *config.C) {} + +func NewRawStatsEmitter(rawConn *RawConn) func() { return func() {} }