Compare commits

..
28 changed files with 2094 additions and 2937 deletions

No files matched your search

+6 -1
View File
@@ -449,7 +449,9 @@ firewall:
#default_local_cidr_any: false
conntrack:
# tcp_timeout also applies to sctp and dccp
tcp_timeout: 12m
# udp_timeout also applies to udplite
udp_timeout: 3m
default_timeout: 10m
@@ -457,8 +459,11 @@ firewall:
# Rules are comprised of a protocol, port, and one or more of host, group, or CIDR
# Logical evaluation is roughly: port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND (local cidr)
# - port: Takes `0` or `any` as any, a single number `80`, a range `200-901`, or `fragment` to match second and further fragments of fragmented packets (since there is no port available).
# proto: `any`, `tcp`, `udp`, or `icmp`
# proto: `any`, `tcp`, `udp`, `udplite`, `dccp`, `sctp`, `icmp`, or any other IP protocol number from 1 to 255, ie `47` for gre
# a port specification is ignored if proto is `icmp`
# `icmp`, `1`, and `58` are the same, each matches both ICMP and ICMPv6
# Nebula only reads ports from tcp, udp, udplite, dccp, and sctp, so a proto `any` rule with a specific port or range only matches those protocols.
# Rules for other protocols (gre, esp, etc) need port `any`.
# host: `any` or a literal hostname, ie `test-host`
# group: `any` or a literal group name, ie `default-group`
# groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass
+36 -755
View File
@@ -2,17 +2,10 @@ package nebula
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"hash/fnv"
"log/slog"
"net/netip"
"reflect"
"slices"
"strconv"
"strings"
"sync"
"time"
@@ -24,10 +17,6 @@ import (
"github.com/slackhq/nebula/iputil"
)
type FirewallInterface interface {
AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
}
type conn struct {
Expires time.Time // Time when this conntrack entry will expire
@@ -42,8 +31,7 @@ type conn struct {
type Firewall struct {
Conntrack *FirewallConntrack
InRules *FirewallTable
OutRules *FirewallTable
rules firewall.Rules
InboundSendReject bool
OutboundSendReject bool
@@ -63,10 +51,8 @@ type Firewall struct {
// unsafeNetworks is the list of unsafe networks issued to us in the certificate
unsafeNetworks []netip.Prefix
rules string
rulesVersion uint16
defaultLocalCIDRAny bool
incomingMetrics firewallMetrics
outgoingMetrics firewallMetrics
@@ -86,52 +72,6 @@ type FirewallConntrack struct {
TimerWheel *TimerWheel[firewall.Packet]
}
// FirewallTable is the entry point for a rule, the evaluation order is:
// Proto AND port AND (CA SHA or CA name) AND local CIDR AND (group OR groups OR name OR remote CIDR)
type FirewallTable struct {
TCP firewallPort
UDP firewallPort
ICMP firewallPort
AnyProto firewallPort
}
func newFirewallTable() *FirewallTable {
return &FirewallTable{
TCP: firewallPort{},
UDP: firewallPort{},
ICMP: firewallPort{},
AnyProto: firewallPort{},
}
}
type FirewallCA struct {
Any *FirewallRule
CANames map[string]*FirewallRule
CAShas map[string]*FirewallRule
}
type FirewallRule struct {
// Any makes Hosts, Groups, and CIDR irrelevant
Any *firewallLocalCIDR
Hosts map[string]*firewallLocalCIDR
Groups []*firewallGroups
CIDR *bart.Table[*firewallLocalCIDR]
}
type firewallGroups struct {
Groups []string
LocalCIDR *firewallLocalCIDR
}
// Even though ports are uint16, int32 maps are faster for lookup
// Plus we can use `-1` for fragment rules
type firewallPort map[int32]*FirewallCA
type firewallLocalCIDR struct {
Any bool
LocalCIDR *bart.Lite
}
// NewFirewall creates a new Firewall object. A TimerWheel is created for you from the provided timeouts.
// The certificate provided should be the highest version loaded in memory.
func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c cert.Certificate) *Firewall {
@@ -170,8 +110,8 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
Conns: make(map[firewall.Packet]*conn),
TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax),
},
InRules: newFirewallTable(),
OutRules: newFirewallTable(),
// Nothing is allowed until there are rules
rules: firewall.NewRulesBuilder(l).Build(assignedNetworks, unsafeNetworks),
TCPTimeout: tcpTimeout,
UDPTimeout: UDPTimeout,
DefaultTimeout: defaultTimeout,
@@ -212,8 +152,6 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
//TODO: max_connections
)
fw.defaultLocalCIDRAny = c.GetBool("firewall.default_local_cidr_any", false)
inboundAction := c.GetString("firewall.inbound_action", "drop")
switch inboundAction {
case "reject":
@@ -236,80 +174,23 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
fw.OutboundSendReject = false
}
err := AddFirewallRulesFromConfig(l, false, c, fw)
if err != nil {
return nil, err
}
err = AddFirewallRulesFromConfig(l, true, c, fw)
rules, err := firewall.RulesFromConfig(l, c, fw.assignedNetworks, fw.unsafeNetworks)
if err != nil {
return nil, err
}
fw.rules = rules
return fw, nil
}
// AddRule properly creates the in memory rule structure for a firewall table.
func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
var (
ft *FirewallTable
fp firewallPort
)
if incoming {
ft = f.InRules
} else {
ft = f.OutRules
}
switch proto {
case iputil.IPProtocolTCP:
fp = ft.TCP
case iputil.IPProtocolUDP:
fp = ft.UDP
case iputil.IPProtocolICMP, iputil.IPProtocolICMPv6:
//ICMP traffic doesn't have ports, so we always coerce to "any", even if a value is provided
if startPort != firewall.PortAny {
f.l.Warn("ignoring port specification for ICMP firewall rule", "startPort", startPort)
}
startPort = firewall.PortAny
endPort = firewall.PortAny
fp = ft.ICMP
case firewall.ProtoAny:
fp = ft.AnyProto
default:
return fmt.Errorf("unknown protocol %v", proto)
}
// We need this rule string because we generate a hash. Removing this will break firewall reload.
ruleString := fmt.Sprintf(
"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s",
incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha,
)
f.rules += ruleString + "\n"
direction := "incoming"
if !incoming {
direction = "outgoing"
}
f.l.Info("Firewall rule added",
"firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha},
)
return fp.addRule(f, startPort, endPort, groups, host, cidr, localCidr, caName, caSha)
}
// GetRuleHash returns a hash representation of all inbound and outbound rules
func (f *Firewall) GetRuleHash() string {
sum := sha256.Sum256([]byte(f.rules))
return hex.EncodeToString(sum[:])
return f.rules.Hash
}
// GetRuleHashFNV returns a uint32 FNV-1 hash representation the rules, for use as a metric value
func (f *Firewall) GetRuleHashFNV() uint32 {
h := fnv.New32a()
h.Write([]byte(f.rules))
return h.Sum32()
return f.rules.HashFNV
}
// GetRuleHashes returns both the sha256 and FNV-1 hashes, suitable for logging
@@ -317,104 +198,6 @@ func (f *Firewall) GetRuleHashes() string {
return "SHA:" + f.GetRuleHash() + ",FNV:" + strconv.FormatUint(uint64(f.GetRuleHashFNV()), 10)
}
func AddFirewallRulesFromConfig(l *slog.Logger, inbound bool, c *config.C, fw FirewallInterface) error {
var table string
if inbound {
table = "firewall.inbound"
} else {
table = "firewall.outbound"
}
r := c.Get(table)
if r == nil {
return nil
}
rs, ok := r.([]any)
if !ok {
return fmt.Errorf("%s failed to parse, should be an array of rules", table)
}
for i, t := range rs {
r, err := convertRule(l, t, table, i)
if err != nil {
return fmt.Errorf("%s rule #%v; %s", table, i, err)
}
if r.Code != "" && r.Port != "" {
return fmt.Errorf("%s rule #%v; only one of port or code should be provided", table, i)
}
if r.Host == "" && len(r.Groups) == 0 && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
return fmt.Errorf("%s rule #%v; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided", table, i)
}
var sPort, errPort string
if r.Code != "" {
errPort = "code"
sPort = r.Code
} else {
errPort = "port"
sPort = r.Port
}
var proto uint8
var startPort, endPort int32
switch r.Proto {
case "any":
proto = firewall.ProtoAny
startPort, endPort, err = parsePort(sPort)
case "tcp":
proto = iputil.IPProtocolTCP
startPort, endPort, err = parsePort(sPort)
case "udp":
proto = iputil.IPProtocolUDP
startPort, endPort, err = parsePort(sPort)
case "icmp":
proto = iputil.IPProtocolICMP
startPort = firewall.PortAny
endPort = firewall.PortAny
if sPort != "" {
l.Warn("ignoring port specification for ICMP firewall rule", "port", sPort)
}
default:
return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
}
if err != nil {
return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err)
}
if r.Cidr != "" && r.Cidr != "any" {
_, err = netip.ParsePrefix(r.Cidr)
if err != nil {
return fmt.Errorf("%s rule #%v; cidr did not parse; %s", table, i, err)
}
}
if r.LocalCidr != "" && r.LocalCidr != "any" {
_, err = netip.ParsePrefix(r.LocalCidr)
if err != nil {
return fmt.Errorf("%s rule #%v; local_cidr did not parse; %s", table, i, err)
}
}
if warning := r.sanity(); warning != nil {
l.Warn("firewall rule sanity check",
"table", table,
"rule", i,
"warning", warning,
)
}
err = fw.AddRule(inbound, proto, startPort, endPort, r.Groups, r.Host, r.Cidr, r.LocalCidr, r.CAName, r.CASha)
if err != nil {
return fmt.Errorf("%s rule #%v; `%s`", table, i, err)
}
}
return nil
}
var ErrUnknownNetworkType = errors.New("unknown network type")
var ErrPeerRejected = errors.New("remote address is not within a network that we handle")
var ErrInvalidRemoteIP = errors.New("remote address is not in remote certificate networks")
@@ -458,23 +241,17 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
}
// Check if we spoke to this tuple, if we did then allow this packet
if f.inConns(fp, h, caPool, localCache) {
if f.inConns(&fp, h, caPool, localCache) {
return nil
}
table := f.OutRules
if incoming {
table = f.InRules
}
// We now know which firewall table to check against
if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
if !f.rules.Match(&fp, incoming, h.ConnectionState.peerCert, caPool) {
f.metrics(incoming).droppedNoRule.Inc(1)
return ErrNoMatchingRule
}
// We always want to conntrack since it is a faster operation
f.addConn(fp, incoming)
f.addConn(&fp, incoming)
return nil
}
@@ -503,9 +280,9 @@ func (f *Firewall) EmitStats() {
metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
}
func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
func (f *Firewall) inConns(fp *firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
if localCache != nil {
if _, ok := localCache[fp]; ok {
if _, ok := localCache[*fp]; ok {
return true
}
}
@@ -518,7 +295,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
f.evict(ep)
}
c, ok := conntrack.Conns[fp]
c, ok := conntrack.Conns[*fp]
if !ok {
conntrack.Unlock()
@@ -528,29 +305,23 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
if c.rulesVersion != f.rulesVersion {
// This conntrack entry was for an older rule set, validate
// it still passes with the current rule set
table := f.OutRules
if c.incoming {
table = f.InRules
}
// We now know which firewall table to check against
if !table.match(fp, c.incoming, h.ConnectionState.peerCert, caPool) {
if !f.rules.Match(fp, c.incoming, h.ConnectionState.peerCert, caPool) {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
h.logger(f.l).Debug("dropping old conntrack entry, does not match new ruleset",
"fwPacket", fp,
"fwPacket", *fp,
"incoming", c.incoming,
"rulesVersion", f.rulesVersion,
"oldRulesVersion", c.rulesVersion,
)
}
delete(conntrack.Conns, fp)
delete(conntrack.Conns, *fp)
conntrack.Unlock()
return false
}
if f.l.Enabled(context.Background(), slog.LevelDebug) {
h.logger(f.l).Debug("keeping old conntrack entry, does match new ruleset",
"fwPacket", fp,
"fwPacket", *fp,
"incoming", c.incoming,
"rulesVersion", f.rulesVersion,
"oldRulesVersion", c.rulesVersion,
@@ -560,42 +331,39 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
c.rulesVersion = f.rulesVersion
}
switch fp.Protocol {
case iputil.IPProtocolTCP:
c.Expires = time.Now().Add(f.TCPTimeout)
case iputil.IPProtocolUDP:
c.Expires = time.Now().Add(f.UDPTimeout)
default:
c.Expires = time.Now().Add(f.DefaultTimeout)
}
c.Expires = time.Now().Add(f.conntrackTimeout(fp.Protocol))
conntrack.Unlock()
if localCache != nil {
localCache[fp] = struct{}{}
localCache[*fp] = struct{}{}
}
return true
}
func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
var timeout time.Duration
c := &conn{}
switch fp.Protocol {
case iputil.IPProtocolTCP:
timeout = f.TCPTimeout
case iputil.IPProtocolUDP:
timeout = f.UDPTimeout
// conntrackTimeout returns how long a conntrack entry for proto lives without traffic.
// Connection oriented protocols share the tcp timeout, datagram protocols share the udp timeout.
func (f *Firewall) conntrackTimeout(proto uint8) time.Duration {
switch proto {
case iputil.IPProtocolTCP, iputil.IPProtocolSCTP, iputil.IPProtocolDCCP:
return f.TCPTimeout
case iputil.IPProtocolUDP, iputil.IPProtocolUDPLite:
return f.UDPTimeout
default:
timeout = f.DefaultTimeout
return f.DefaultTimeout
}
}
func (f *Firewall) addConn(fp *firewall.Packet, incoming bool) {
timeout := f.conntrackTimeout(fp.Protocol)
c := &conn{}
conntrack := f.Conntrack
conntrack.Lock()
if _, ok := conntrack.Conns[fp]; !ok {
if _, ok := conntrack.Conns[*fp]; !ok {
conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(fp, timeout)
conntrack.TimerWheel.Add(*fp, timeout)
}
// Record which rulesVersion allowed this connection, so we can retest after
@@ -603,7 +371,7 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
c.incoming = incoming
c.rulesVersion = f.rulesVersion
c.Expires = time.Now().Add(timeout)
conntrack.Conns[fp] = c
conntrack.Conns[*fp] = c
conntrack.Unlock()
}
@@ -629,490 +397,3 @@ func (f *Firewall) evict(p firewall.Packet) {
// This conn is done
delete(conntrack.Conns, p)
}
func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
if ft.AnyProto.match(p, incoming, c, caPool) {
return true
}
switch p.Protocol {
case iputil.IPProtocolTCP:
if ft.TCP.match(p, incoming, c, caPool) {
return true
}
case iputil.IPProtocolUDP:
if ft.UDP.match(p, incoming, c, caPool) {
return true
}
case iputil.IPProtocolICMP, iputil.IPProtocolICMPv6:
if ft.ICMP.match(p, incoming, c, caPool) {
return true
}
}
return false
}
func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
if startPort > endPort {
return fmt.Errorf("start port was lower than end port")
}
for i := startPort; i <= endPort; i++ {
if _, ok := fp[i]; !ok {
fp[i] = &FirewallCA{
CANames: make(map[string]*FirewallRule),
CAShas: make(map[string]*FirewallRule),
}
}
if err := fp[i].addRule(f, groups, host, cidr, localCidr, caName, caSha); err != nil {
return err
}
}
return nil
}
func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
// We don't have any allowed ports, bail
if fp == nil {
return false
}
// this branch is here to catch traffic from FirewallTable.Any.match and FirewallTable.ICMP.match
if p.Protocol == iputil.IPProtocolICMP || p.Protocol == iputil.IPProtocolICMPv6 {
// port numbers are re-used for connection tracking of ICMP,
// but we don't want to actually filter on them.
return fp[firewall.PortAny].match(p, c, caPool)
}
var port int32
if p.Fragment {
port = firewall.PortFragment
} else if incoming {
port = int32(p.LocalPort)
} else {
port = int32(p.RemotePort)
}
if fp[port].match(p, c, caPool) {
return true
}
return fp[firewall.PortAny].match(p, c, caPool)
}
func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, localCidr, caName, caSha string) error {
fr := func() *FirewallRule {
return &FirewallRule{
Hosts: make(map[string]*firewallLocalCIDR),
Groups: make([]*firewallGroups, 0),
CIDR: new(bart.Table[*firewallLocalCIDR]),
}
}
if caSha == "" && caName == "" {
if fc.Any == nil {
fc.Any = fr()
}
return fc.Any.addRule(f, groups, host, cidr, localCidr)
}
if caSha != "" {
if _, ok := fc.CAShas[caSha]; !ok {
fc.CAShas[caSha] = fr()
}
err := fc.CAShas[caSha].addRule(f, groups, host, cidr, localCidr)
if err != nil {
return err
}
}
if caName != "" {
if _, ok := fc.CANames[caName]; !ok {
fc.CANames[caName] = fr()
}
err := fc.CANames[caName].addRule(f, groups, host, cidr, localCidr)
if err != nil {
return err
}
}
return nil
}
func (fc *FirewallCA) match(p firewall.Packet, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
if fc == nil {
return false
}
if fc.Any.match(p, c) {
return true
}
if t, ok := fc.CAShas[c.Certificate.Issuer()]; ok {
if t.match(p, c) {
return true
}
}
s, err := caPool.GetCAForCert(c.Certificate)
if err != nil {
return false
}
return fc.CANames[s.Certificate.Name()].match(p, c)
}
func (fr *FirewallRule) addRule(f *Firewall, groups []string, host, cidr, localCidr string) error {
flc := func() *firewallLocalCIDR {
return &firewallLocalCIDR{
LocalCIDR: new(bart.Lite),
}
}
if fr.isAny(groups, host, cidr) {
if fr.Any == nil {
fr.Any = flc()
}
return fr.Any.addRule(f, localCidr)
}
if len(groups) > 0 {
nlc := flc()
err := nlc.addRule(f, localCidr)
if err != nil {
return err
}
fr.Groups = append(fr.Groups, &firewallGroups{
Groups: groups,
LocalCIDR: nlc,
})
}
if host != "" {
nlc := fr.Hosts[host]
if nlc == nil {
nlc = flc()
}
err := nlc.addRule(f, localCidr)
if err != nil {
return err
}
fr.Hosts[host] = nlc
}
if cidr != "" {
c, err := netip.ParsePrefix(cidr)
if err != nil {
return err
}
nlc, _ := fr.CIDR.Get(c)
if nlc == nil {
nlc = flc()
}
err = nlc.addRule(f, localCidr)
if err != nil {
return err
}
fr.CIDR.Insert(c, nlc)
}
return nil
}
func (fr *FirewallRule) isAny(groups []string, host string, cidr string) bool {
if len(groups) == 0 && host == "" && cidr == "" {
return true
}
if slices.Contains(groups, "any") {
return true
}
if host == "any" {
return true
}
if cidr == "any" {
return true
}
return false
}
func (fr *FirewallRule) match(p firewall.Packet, c *cert.CachedCertificate) bool {
if fr == nil {
return false
}
// Shortcut path for if groups, hosts, or cidr contained an `any`
if fr.Any.match(p, c) {
return true
}
// Need any of group, host, or cidr to match
for _, sg := range fr.Groups {
found := false
for _, g := range sg.Groups {
if _, ok := c.InvertedGroups[g]; !ok {
found = false
break
}
found = true
}
if found && sg.LocalCIDR.match(p, c) {
return true
}
}
if fr.Hosts != nil {
if flc, ok := fr.Hosts[c.Certificate.Name()]; ok {
if flc.match(p, c) {
return true
}
}
}
for _, v := range fr.CIDR.Supernets(netip.PrefixFrom(p.RemoteAddr, p.RemoteAddr.BitLen())) {
if v.match(p, c) {
return true
}
}
return false
}
func (flc *firewallLocalCIDR) addRule(f *Firewall, localCidr string) error {
if localCidr == "any" {
flc.Any = true
return nil
}
if localCidr == "" {
if len(f.unsafeNetworks) == 0 || f.defaultLocalCIDRAny {
flc.Any = true
return nil
}
for _, network := range f.assignedNetworks {
flc.LocalCIDR.Insert(network)
}
return nil
}
c, err := netip.ParsePrefix(localCidr)
if err != nil {
return err
}
flc.LocalCIDR.Insert(c)
return nil
}
func (flc *firewallLocalCIDR) match(p firewall.Packet, c *cert.CachedCertificate) bool {
if flc == nil {
return false
}
if flc.Any {
return true
}
return flc.LocalCIDR.Contains(p.LocalAddr)
}
type rule struct {
Port string
Code string
Proto string
Host string
Groups []string
Cidr string
LocalCidr string
CAName string
CASha string
}
func convertRule(l *slog.Logger, p any, table string, i int) (rule, error) {
r := rule{}
m, ok := p.(map[string]any)
if !ok {
return r, errors.New("could not parse rule")
}
toString := func(k string, m map[string]any) string {
v, ok := m[k]
if !ok {
return ""
}
return fmt.Sprintf("%v", v)
}
r.Port = toString("port", m)
r.Code = toString("code", m)
r.Proto = toString("proto", m)
r.Host = toString("host", m)
r.Cidr = toString("cidr", m)
r.LocalCidr = toString("local_cidr", m)
r.CAName = toString("ca_name", m)
r.CASha = toString("ca_sha", m)
// Make sure group isn't an array
if v, ok := m["group"].([]any); ok {
if len(v) > 1 {
return r, errors.New("group should contain a single value, an array with more than one entry was provided")
}
l.Warn("group was an array with a single value, converting to simple value",
"table", table,
"rule", i,
)
m["group"] = v[0]
}
singleGroup := toString("group", m)
if rg, ok := m["groups"]; ok {
switch reflect.TypeOf(rg).Kind() {
case reflect.Slice:
v := reflect.ValueOf(rg)
r.Groups = make([]string, v.Len())
for i := 0; i < v.Len(); i++ {
r.Groups[i] = v.Index(i).Interface().(string)
}
case reflect.String:
r.Groups = []string{rg.(string)}
default:
r.Groups = []string{fmt.Sprintf("%v", rg)}
}
}
//flatten group vs groups
if singleGroup != "" {
// Check if we have both groups and group provided in the rule config
if len(r.Groups) > 0 {
return r, fmt.Errorf("only one of group or groups should be defined, both provided")
}
r.Groups = []string{singleGroup}
}
return r, nil
}
// sanity returns an error if the rule would be evaluated in a way that would short-circuit a configured check on a wildcard value
// rules are evaluated as "port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND local_cidr"
func (r *rule) sanity() error {
//port, proto, local_cidr are AND, no need to check here
//ca_sha and ca_name don't have a wildcard value, no need to check here
groupsEmpty := len(r.Groups) == 0
hostEmpty := r.Host == ""
cidrEmpty := r.Cidr == ""
if (groupsEmpty && hostEmpty && cidrEmpty) == true {
return nil //no content!
}
groupsHasAny := slices.Contains(r.Groups, "any")
if groupsHasAny && len(r.Groups) > 1 {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the other groups specified", r.Groups)
}
if r.Host == "any" {
if !groupsEmpty {
return fmt.Errorf("groups specified as %s, but host=any will match any host, regardless of groups", r.Groups)
}
if !cidrEmpty {
return fmt.Errorf("cidr specified as %s, but host=any will match any host, regardless of cidr", r.Cidr)
}
}
if groupsHasAny {
if !hostEmpty && r.Host != "any" {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified host %s", r.Groups, r.Host)
}
if !cidrEmpty {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified cidr %s", r.Groups, r.Cidr)
}
}
if r.Code != "" {
return fmt.Errorf("code specified as [%s]. Support for 'code' will be dropped in a future release, as it has never been functional", r.Code)
}
//todo alert on cidr-any
return nil
}
func parsePort(s string) (int32, int32, error) {
const notAPort int32 = -2
if s == "any" {
return firewall.PortAny, firewall.PortAny, nil
}
if s == "fragment" {
return firewall.PortFragment, firewall.PortFragment, nil
}
if !strings.Contains(s, `-`) {
rPort, err := parsePortValue("", s)
if err != nil {
return notAPort, notAPort, err
}
return rPort, rPort, nil
}
sPorts := strings.SplitN(s, `-`, 2)
for i := range sPorts {
sPorts[i] = strings.Trim(sPorts[i], " ")
}
if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
return notAPort, notAPort, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
}
startPort, err := parsePortValue("beginning range ", sPorts[0])
if err != nil {
return notAPort, notAPort, err
}
endPort, err := parsePortValue("ending range ", sPorts[1])
if err != nil {
return notAPort, notAPort, err
}
if startPort == firewall.PortAny {
endPort = firewall.PortAny
}
return startPort, endPort, nil
}
// parsePortValue accepts a base-10 decimal in [0, 65535] and returns it
// widened to int32. Using strconv.ParseUint with bitSize 16 rejects
// negative input, out-of-range input (>65535), and any non-decimal byte
// by construction, so the int32 widening that follows is provably safe
// and cannot collide with firewall.PortAny (0) or firewall.PortFragment
// (-1) via integer truncation.
//
// prefix is prepended to both error messages so callers can disambiguate
// the single-port path (prefix="") from the range bounds (prefix="beginning
// range " / "ending range "), preserving the historical error strings.
func parsePortValue(prefix, s string) (int32, error) {
n, err := strconv.ParseUint(s, 10, 16)
if err == nil {
return int32(n), nil
}
if errors.Is(err, strconv.ErrRange) {
return 0, fmt.Errorf("%sout of range [0,65535]; `%s`", prefix, s)
}
return 0, fmt.Errorf("%swas not a number; `%s`", prefix, s)
}
+319
View File
@@ -0,0 +1,319 @@
package firewall
import (
"errors"
"fmt"
"log/slog"
"net/netip"
"reflect"
"slices"
"strconv"
"strings"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/iputil"
)
// ruleAdder takes firewall rules, it lets tests see what addRulesFromConfig adds
type ruleAdder interface {
AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
}
// RulesFromConfig reads the inbound and outbound firewall rules from c and builds them, see RulesBuilder.Build
func RulesFromConfig(l *slog.Logger, c *config.C, vpnNetworks, unsafeNetworks []netip.Prefix) (Rules, error) {
b := NewRulesBuilder(l)
b.defaultLocalCIDRAny = c.GetBool("firewall.default_local_cidr_any", false)
if err := addRulesFromConfig(l, false, c, b); err != nil {
return Rules{}, err
}
if err := addRulesFromConfig(l, true, c, b); err != nil {
return Rules{}, err
}
return b.Build(vpnNetworks, unsafeNetworks), nil
}
func addRulesFromConfig(l *slog.Logger, inbound bool, c *config.C, ra ruleAdder) error {
var table string
if inbound {
table = "firewall.inbound"
} else {
table = "firewall.outbound"
}
r := c.Get(table)
if r == nil {
return nil
}
rs, ok := r.([]any)
if !ok {
return fmt.Errorf("%s failed to parse, should be an array of rules", table)
}
for i, t := range rs {
r, err := yamlToConfigRule(l, t, table, i)
if err != nil {
return fmt.Errorf("%s rule #%v; %s", table, i, err)
}
if r.Code != "" && r.Port != "" {
return fmt.Errorf("%s rule #%v; only one of port or code should be provided", table, i)
}
if r.Host == "" && len(r.Groups) == 0 && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
return fmt.Errorf("%s rule #%v; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided", table, i)
}
var sPort, errPort string
if r.Code != "" {
errPort = "code"
sPort = r.Code
} else {
errPort = "port"
sPort = r.Port
}
var proto uint8
switch r.Proto {
case "any":
proto = ProtoAny
case "tcp":
proto = iputil.IPProtocolTCP
case "udp":
proto = iputil.IPProtocolUDP
case "udplite":
proto = iputil.IPProtocolUDPLite
case "dccp":
proto = iputil.IPProtocolDCCP
case "sctp":
proto = iputil.IPProtocolSCTP
case "icmp":
proto = iputil.IPProtocolICMP
default:
// Any other protocol by number. 0 is reserved for `any`.
n, perr := strconv.ParseUint(r.Proto, 10, 8)
if perr != nil || n == 0 {
return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
}
proto = uint8(n)
}
var startPort, endPort int32
if proto == iputil.IPProtocolICMP || proto == iputil.IPProtocolICMPv6 {
startPort = PortAny
endPort = PortAny
if sPort != "" {
l.Warn("ignoring port specification for ICMP firewall rule", "port", sPort)
}
} else {
startPort, endPort, err = parsePort(sPort)
}
if err != nil {
return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err)
}
if warning := r.sanity(); warning != nil {
l.Warn("firewall rule sanity check",
"table", table,
"rule", i,
"warning", warning,
)
}
err = ra.AddRule(inbound, proto, startPort, endPort, r.Groups, r.Host, r.Cidr, r.LocalCidr, r.CAName, r.CASha)
if err != nil {
return fmt.Errorf("%s rule #%v; %s", table, i, err)
}
}
return nil
}
type configRule struct {
Port string
Code string
Proto string
Host string
Groups []string
Cidr string
LocalCidr string
CAName string
CASha string
}
func yamlToConfigRule(l *slog.Logger, p any, table string, i int) (configRule, error) {
r := configRule{}
m, ok := p.(map[string]any)
if !ok {
return r, errors.New("could not parse rule")
}
toString := func(k string, m map[string]any) string {
v, ok := m[k]
if !ok {
return ""
}
return fmt.Sprintf("%v", v)
}
r.Port = toString("port", m)
r.Code = toString("code", m)
r.Proto = toString("proto", m)
r.Host = toString("host", m)
r.Cidr = toString("cidr", m)
r.LocalCidr = toString("local_cidr", m)
r.CAName = toString("ca_name", m)
r.CASha = toString("ca_sha", m)
// Make sure group isn't an array
if v, ok := m["group"].([]any); ok {
if len(v) > 1 {
return r, errors.New("group should contain a single value, an array with more than one entry was provided")
}
l.Warn("group was an array with a single value, converting to simple value",
"table", table,
"rule", i,
)
m["group"] = v[0]
}
singleGroup := toString("group", m)
if rg, ok := m["groups"]; ok {
switch reflect.TypeOf(rg).Kind() {
case reflect.Slice:
v := reflect.ValueOf(rg)
r.Groups = make([]string, v.Len())
for i := 0; i < v.Len(); i++ {
r.Groups[i] = v.Index(i).Interface().(string)
}
case reflect.String:
r.Groups = []string{rg.(string)}
default:
r.Groups = []string{fmt.Sprintf("%v", rg)}
}
}
//flatten group vs groups
if singleGroup != "" {
// Check if we have both groups and group provided in the rule config
if len(r.Groups) > 0 {
return r, fmt.Errorf("only one of group or groups should be defined, both provided")
}
r.Groups = []string{singleGroup}
}
return r, nil
}
// sanity returns an error if the rule would be evaluated in a way that would short-circuit a configured check on a wildcard value
// rules are evaluated as "port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND local_cidr"
func (r *configRule) sanity() error {
//port, proto, local_cidr are AND, no need to check here
//ca_sha and ca_name don't have a wildcard value, no need to check here
groupsEmpty := len(r.Groups) == 0
hostEmpty := r.Host == ""
cidrEmpty := r.Cidr == ""
if (groupsEmpty && hostEmpty && cidrEmpty) == true {
return nil //no content!
}
groupsHasAny := slices.Contains(r.Groups, "any")
if groupsHasAny && len(r.Groups) > 1 {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the other groups specified", r.Groups)
}
if r.Host == "any" {
if !groupsEmpty {
return fmt.Errorf("groups specified as %s, but host=any will match any host, regardless of groups", r.Groups)
}
if !cidrEmpty {
return fmt.Errorf("cidr specified as %s, but host=any will match any host, regardless of cidr", r.Cidr)
}
}
if groupsHasAny {
if !hostEmpty && r.Host != "any" {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified host %s", r.Groups, r.Host)
}
if !cidrEmpty {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified cidr %s", r.Groups, r.Cidr)
}
}
if r.Code != "" {
return fmt.Errorf("code specified as [%s]. Support for 'code' will be dropped in a future release, as it has never been functional", r.Code)
}
//todo alert on cidr-any
return nil
}
func parsePort(s string) (int32, int32, error) {
const notAPort int32 = -2
if s == "any" {
return PortAny, PortAny, nil
}
if s == "fragment" {
return PortFragment, PortFragment, nil
}
if !strings.Contains(s, `-`) {
rPort, err := parsePortValue("", s)
if err != nil {
return notAPort, notAPort, err
}
return rPort, rPort, nil
}
sPorts := strings.SplitN(s, `-`, 2)
for i := range sPorts {
sPorts[i] = strings.Trim(sPorts[i], " ")
}
if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
return notAPort, notAPort, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
}
startPort, err := parsePortValue("beginning range ", sPorts[0])
if err != nil {
return notAPort, notAPort, err
}
endPort, err := parsePortValue("ending range ", sPorts[1])
if err != nil {
return notAPort, notAPort, err
}
if startPort == PortAny {
endPort = PortAny
}
return startPort, endPort, nil
}
// parsePortValue accepts a base-10 decimal in [0, 65535] and returns it
// widened to int32. Using strconv.ParseUint with bitSize 16 rejects
// negative input, out-of-range input (>65535), and any non-decimal byte
// by construction, so the int32 widening that follows is provably safe
// and cannot collide with PortAny (0) or PortFragment
// (-1) via integer truncation.
//
// prefix is prepended to both error messages so callers can disambiguate
// the single-port path (prefix="") from the range bounds (prefix="beginning
// range " / "ending range "), preserving the historical error strings.
func parsePortValue(prefix, s string) (int32, error) {
n, err := strconv.ParseUint(s, 10, 16)
if err == nil {
return int32(n), nil
}
if errors.Is(err, strconv.ErrRange) {
return 0, fmt.Errorf("%sout of range [0,65535]; `%s`", prefix, s)
}
return 0, fmt.Errorf("%swas not a number; `%s`", prefix, s)
}
+477
View File
@@ -0,0 +1,477 @@
package firewall
import (
"bytes"
"errors"
"net/netip"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_parsePort(t *testing.T) {
_, _, err := parsePort("")
require.EqualError(t, err, "was not a number; ``")
_, _, err = parsePort(" ")
require.EqualError(t, err, "was not a number; ` `")
_, _, err = parsePort("-")
require.EqualError(t, err, "appears to be a range but could not be parsed; `-`")
_, _, err = parsePort(" - ")
require.EqualError(t, err, "appears to be a range but could not be parsed; ` - `")
_, _, err = parsePort("a-b")
require.EqualError(t, err, "beginning range was not a number; `a`")
_, _, err = parsePort("1-b")
require.EqualError(t, err, "ending range was not a number; `b`")
s, e, err := parsePort(" 1 - 2 ")
assert.Equal(t, int32(1), s)
assert.Equal(t, int32(2), e)
require.NoError(t, err)
s, e, err = parsePort("0-1")
assert.Equal(t, int32(0), s)
assert.Equal(t, int32(0), e)
require.NoError(t, err)
s, e, err = parsePort("9919")
assert.Equal(t, int32(9919), s)
assert.Equal(t, int32(9919), e)
require.NoError(t, err)
s, e, err = parsePort("any")
assert.Equal(t, int32(0), s)
assert.Equal(t, int32(0), e)
require.NoError(t, err)
}
// Test_parsePort_invalid covers inputs that must error. The named bug is
// that int32(strconv.Atoi("4294967296")) truncates to 0 == PortAny,
// silently turning a typo into a match-all-ports rule; the rest are
// representative syntax/range probes.
func Test_parsePort_invalid(t *testing.T) {
tests := []struct {
name string
input string
wantErrContains string
}{
// Numeric overflow (the named bug + boundary).
{"named bug: 2^32 truncates to PortAny", "4294967296", "out of range"},
{"just above max real port", "65536", "out of range"},
// Negatives route through the range branch and hit the empty-half
// guard; included as defense in depth so a future refactor cannot
// accidentally reach the int32 cast.
{"negative", "-1", "could not be parsed"},
// Syntax probes.
{"NUL between digits", "4\x002", "was not a number"},
{"hex notation", "0x10", "was not a number"},
{"scientific notation", "1e3", "was not a number"},
{"leading whitespace", " 42", "was not a number"},
{"fullwidth digits", "42", "was not a number"},
// Range branch.
{"range upper out of range", "1-65536", "ending range out of range"},
{"range lower out of range", "65536-65537", "beginning range out of range"},
{"range with negative upper", "1--1", "ending range was not a number"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, _, err := parsePort(tc.input)
require.Error(t, err, "input %q must error", tc.input)
require.ErrorContains(t, err, tc.wantErrContains)
})
}
}
// Test_parsePort_valid_boundaries locks in success cases at 0, 1, and 65535
// so a future refactor cannot regress the boundaries.
func Test_parsePort_valid_boundaries(t *testing.T) {
tests := []struct {
name string
input string
wantStart int32
wantEnd int32
}{
{"zero is PortAny", "0", 0, 0},
{"min real port", "1", 1, 1},
{"max real port", "65535", 65535, 65535},
{"range zero to max forces end to zero", "0-65535", 0, 0},
{"range max to max", "65535-65535", 65535, 65535},
{"range one to max", "1-65535", 1, 65535},
{"range with whitespace inside", " 1 - 2 ", 1, 2},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
s, e, err := parsePort(tc.input)
require.NoError(t, err)
assert.Equal(t, tc.wantStart, s, "start port")
assert.Equal(t, tc.wantEnd, e, "end port")
})
}
}
func TestAddRulesFromConfig(t *testing.T) {
l := test.NewLogger()
// Test adding tcp rule
conf := config.NewC(test.NewLogger())
mf := &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "tcp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolTCP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding udp rule
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "udp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolUDP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding udplite, dccp, and sctp rules
for name, proto := range map[string]uint8{"udplite": iputil.IPProtocolUDPLite, "dccp": iputil.IPProtocolDCCP, "sctp": iputil.IPProtocolSCTP} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1-2", "proto": name, "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf), name)
assert.Equal(t, addRuleCall{incoming: false, proto: proto, startPort: 1, endPort: 2, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall, name)
}
// Test adding icmp rule
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "icmp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolICMP, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding icmp rule no port
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"proto": "icmp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolICMP, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding rules by protocol number, as yaml would give a string or an int
for _, proto := range []any{"47", 47} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "any", "proto": proto, "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf), proto)
assert.Equal(t, addRuleCall{incoming: false, proto: 47, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall, proto)
}
// Test ICMP and ICMPv6 by number still ignore ports
for _, proto := range []uint8{iputil.IPProtocolICMP, iputil.IPProtocolICMPv6} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": proto, "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf), proto)
assert.Equal(t, addRuleCall{incoming: false, proto: proto, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall, proto)
}
// Test protocols that aren't a known name or a number from 1 to 255
for _, proto := range []string{"0", "256", "-1", "gre"} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "any", "proto": proto, "host": "a"}}}
require.EqualError(t, addRulesFromConfig(l, false, conf, mf), "firewall.outbound rule #0; proto was not understood; `"+proto+"`")
}
// Test adding any rule
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding rule with cidr
cidr := netip.MustParsePrefix("10.0.0.0/8")
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr.String(), localIp: ""}, mf.lastCall)
// Test adding rule with local_cidr
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: cidr.String()}, mf.lastCall)
// Test adding rule with cidr ipv6
cidr6 := netip.MustParsePrefix("fd00::/8")
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr6.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr6.String(), localIp: ""}, mf.lastCall)
// Test adding rule with any cidr
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "any"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "any", localIp: ""}, mf.lastCall)
// Test adding rule with junk cidr, which RulesBuilder rejects
conf = config.NewC(test.NewLogger())
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "junk/junk"}}}
require.EqualError(t, addRulesFromConfig(l, true, conf, NewRulesBuilder(l)), "firewall.inbound rule #0; cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP")
// Test adding rule with local_cidr ipv6
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr6.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: cidr6.String()}, mf.lastCall)
// Test adding rule with any local_cidr
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "any"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, localIp: "any"}, mf.lastCall)
// Test adding rule with junk local_cidr, which RulesBuilder rejects
conf = config.NewC(test.NewLogger())
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "junk/junk"}}}
require.EqualError(t, addRulesFromConfig(l, true, conf, NewRulesBuilder(l)), "firewall.inbound rule #0; local_cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP")
// Test adding rule with ca_sha
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_sha": "12312313123"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caSha: "12312313123"}, mf.lastCall)
// Test adding rule with ca_name
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_name": "root01"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caName: "root01"}, mf.lastCall)
// Test single group
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "group": "a"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall)
// Test single groups
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": "a"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall)
// Test multiple AND groups
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": []string{"a", "b"}}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: []string{"a", "b"}, ip: "", localIp: ""}, mf.lastCall)
// Test Add error
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
mf.nextCallReturn = errors.New("test error")
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "host": "a"}}}
require.EqualError(t, addRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; test error")
}
// TestRulesFromConfig_defaultLocalCIDR ensures rules without a local_cidr allow only the vpn networks when there are
// unsafe networks, unless default_local_cidr_any is set, and that a rule's own local_cidr is kept either way
func TestRulesFromConfig_defaultLocalCIDR(t *testing.T) {
vpnNetworks := []netip.Prefix{netip.MustParsePrefix("10.1.0.5/16")}
unsafeNetworks := []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}
vpnAddr := netip.MustParseAddr("10.1.9.9")
unsafeAddr := netip.MustParseAddr("192.168.0.3")
allows := func(lr *localRules, addr netip.Addr) bool {
return lr.match(&Packet{LocalAddr: addr}, nil)
}
for _, tc := range []struct {
name string
unsafeNetworks []netip.Prefix
defaultLocalCIDRAny bool
wantUnsafe bool
}{
{"no unsafe networks", nil, false, true},
{"unsafe networks", unsafeNetworks, false, false},
{"unsafe networks and default_local_cidr_any", unsafeNetworks, true, true},
} {
t.Run(tc.name, func(t *testing.T) {
conf := config.NewC(test.NewLogger())
conf.Settings["firewall"] = map[string]any{
"default_local_cidr_any": tc.defaultLocalCIDRAny,
"inbound": []any{
map[string]any{"port": "any", "proto": "any", "host": "any"},
map[string]any{"port": "22", "proto": "tcp", "host": "any"},
map[string]any{"port": "80", "proto": "tcp", "host": "any", "local_cidr": "192.168.0.0/24"},
map[string]any{"port": "443", "proto": "tcp", "host": "any", "local_cidr": "any"},
},
}
rules, err := RulesFromConfig(test.NewLogger(), conf, vpnNetworks, tc.unsafeNetworks)
require.NoError(t, err)
// The proto `any` rule, copied into a protocol with rules of its own and shared by one without
for _, proto := range []uint8{iputil.IPProtocolTCP, 47} {
lr := rules.In.protos[proto][PortAny].Any.Any
assert.True(t, allows(lr, vpnAddr), "proto %d", proto)
assert.Equal(t, tc.wantUnsafe, allows(lr, unsafeAddr), "proto %d", proto)
}
tcp := rules.In.protos[iputil.IPProtocolTCP]
assert.True(t, allows(tcp[22].Any.Any, vpnAddr))
assert.Equal(t, tc.wantUnsafe, allows(tcp[22].Any.Any, unsafeAddr))
assert.False(t, allows(tcp[80].Any.Any, vpnAddr))
assert.True(t, allows(tcp[80].Any.Any, unsafeAddr))
assert.True(t, allows(tcp[443].Any.Any, vpnAddr))
assert.True(t, allows(tcp[443].Any.Any, unsafeAddr))
})
}
}
func Test_convertRule(t *testing.T) {
ob := &bytes.Buffer{}
l := test.NewLoggerWithOutput(ob)
// Ensure group array of 1 is converted and a warning is printed
c := map[string]any{
"group": []any{"group1"},
}
r, err := yamlToConfigRule(l, c, "test", 1)
assert.Contains(t, ob.String(), "group was an array with a single value, converting to simple value")
assert.Contains(t, ob.String(), "table=test")
assert.Contains(t, ob.String(), "rule=1")
require.NoError(t, err)
assert.Equal(t, []string{"group1"}, r.Groups)
// Ensure group array of > 1 is errord
ob.Reset()
c = map[string]any{
"group": []any{"group1", "group2"},
}
r, err = yamlToConfigRule(l, c, "test", 1)
assert.Empty(t, ob.String())
require.Error(t, err, "group should contain a single value, an array with more than one entry was provided")
// Make sure a well formed group is alright
ob.Reset()
c = map[string]any{
"group": "group1",
}
r, err = yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
assert.Equal(t, []string{"group1"}, r.Groups)
}
func Test_convertRuleSanity(t *testing.T) {
ob := &bytes.Buffer{}
l := test.NewLoggerWithOutput(ob)
noWarningPlease := []map[string]any{
{"group": "group1"},
{"groups": []any{"group2"}},
{"host": "bob"},
{"cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "host": "bob"},
{"cidr": "1.1.1.1/1", "host": "bob"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
}
for _, c := range noWarningPlease {
r, err := yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
require.NoError(t, r.sanity(), "should not generate a sanity warning, %+v", c)
}
yesWarningPlease := []map[string]any{
{"group": "group1"},
{"groups": []any{"group2"}},
{"cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "host": "bob"},
{"cidr": "1.1.1.1/1", "host": "bob"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
}
for _, c := range yesWarningPlease {
c["host"] = "any"
r, err := yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
err = r.sanity()
require.Error(t, err, "I wanted a warning: %+v", c)
}
//reset the list
yesWarningPlease = []map[string]any{
{"group": "group1"},
{"groups": []any{"group2"}},
{"cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "host": "bob"},
{"cidr": "1.1.1.1/1", "host": "bob"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
}
for _, c := range yesWarningPlease {
r, err := yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
r.Groups = append(r.Groups, "any")
err = r.sanity()
require.Error(t, err, "I wanted a warning: %+v", c)
}
}
type addRuleCall struct {
incoming bool
proto uint8
startPort int32
endPort int32
groups []string
host string
ip string
localIp string
caName string
caSha string
}
type mockFirewall struct {
lastCall addRuleCall
nextCallReturn error
}
func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip, localIp, caName string, caSha string) error {
mf.lastCall = addRuleCall{
incoming: incoming,
proto: proto,
startPort: startPort,
endPort: endPort,
groups: groups,
host: host,
ip: ip,
localIp: localIp,
caName: caName,
caSha: caSha,
}
err := mf.nextCallReturn
mf.nextCallReturn = nil
return err
}
+6
View File
@@ -50,6 +50,12 @@ func (fp Packet) MarshalJSON() ([]byte, error) {
proto = "icmpv6"
case iputil.IPProtocolUDP:
proto = "udp"
case iputil.IPProtocolUDPLite:
proto = "udplite"
case iputil.IPProtocolDCCP:
proto = "dccp"
case iputil.IPProtocolSCTP:
proto = "sctp"
default:
proto = fmt.Sprintf("unknown %v", fp.Protocol)
}
+330
View File
@@ -0,0 +1,330 @@
package firewall
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"hash"
"hash/fnv"
"io"
"log/slog"
"net/netip"
"slices"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/iputil"
)
// RulesBuilder collects the inbound and outbound rules, Build turns them into a Table for each direction
type RulesBuilder struct {
in, out tableRules
// defaultLocalCIDRAny makes rules without a local cidr allow any local address, even with unsafe networks
defaultLocalCIDRAny bool
// ruleSHA and ruleFNV hash every rule as it's added, see Rules
ruleSHA hash.Hash
ruleFNV hash.Hash32
l *slog.Logger
}
// tableRules are one direction's rules, waiting to be built into a Table
type tableRules struct {
anyRules []rule
// protoRules holds the rules for each protocol besides proto `any`, ICMPv6 rules are kept with ICMP
protoRules [256][]rule
}
// rule is a parsed rule, so adding it to a Table can't fail
type rule struct {
startPort, endPort int32
groups []string
host string
caName, caSha string
// cidr is the remote cidr, not valid when the rule doesn't have one
cidr netip.Prefix
anyCIDR bool
// localCIDR is the local addresses the rule allows, anyLocalCIDR allows all of them.
// Neither is set for a rule without a local cidr until the Table is built and it gets the default.
// localCIDR is shared by every localRules the rule is added to, so it's never changed, see localRules.add
localCIDR *bart.Lite
anyLocalCIDR bool
}
func NewRulesBuilder(l *slog.Logger) *RulesBuilder {
return &RulesBuilder{
ruleSHA: sha256.New(),
ruleFNV: fnv.New32a(),
l: l,
}
}
// AddRule adds a rule for incoming or outgoing traffic, on proto or on every protocol when proto is ProtoAny.
// The rule is parsed now, so a bad rule is reported when it's added rather than when the Tables are built.
func (b *RulesBuilder) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
if proto == iputil.IPProtocolICMP || proto == iputil.IPProtocolICMPv6 {
//ICMP traffic doesn't have ports, so we always coerce to "any", even if a value is provided
if startPort != PortAny {
b.l.Warn("ignoring port specification for ICMP firewall rule", "startPort", startPort)
}
startPort = PortAny
endPort = PortAny
}
r, err := parseRule(startPort, endPort, groups, host, cidr, localCidr, caName, caSha)
if err != nil {
return err
}
if incoming {
b.in.add(proto, r)
} else {
b.out.add(proto, r)
}
// The rule hashes are of these lines, changing them changes every hash
fmt.Fprintf(io.MultiWriter(b.ruleSHA, b.ruleFNV),
"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s\n",
incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha,
)
direction := "incoming"
if !incoming {
direction = "outgoing"
}
b.l.Info("Firewall rule added",
"firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha},
)
return nil
}
// Build turns the rules into a Table for each direction, and hashes them. Rules without a local cidr allow
// vpnNetworks when there are unsafe networks, otherwise any local address.
func (b *RulesBuilder) Build(vpnNetworks, unsafeNetworks []netip.Prefix) Rules {
var defaultLocal *bart.Lite
if len(unsafeNetworks) > 0 && !b.defaultLocalCIDRAny {
defaultLocal = new(bart.Lite)
for _, network := range vpnNetworks {
defaultLocal.Insert(network)
}
}
return Rules{
In: b.in.build(defaultLocal),
Out: b.out.build(defaultLocal),
Hash: hex.EncodeToString(b.ruleSHA.Sum(nil)),
HashFNV: b.ruleFNV.Sum32(),
}
}
func parseRule(startPort, endPort int32, groups []string, host, cidr, localCidr, caName, caSha string) (rule, error) {
if startPort > endPort {
return rule{}, fmt.Errorf("start port was lower than end port")
}
r := rule{
startPort: startPort,
endPort: endPort,
groups: groups,
host: host,
caName: caName,
caSha: caSha,
}
var err error
r.cidr, r.anyCIDR, err = parseCIDR(cidr)
if err != nil {
return rule{}, fmt.Errorf("cidr did not parse; %w", err)
}
var localCIDR netip.Prefix
localCIDR, r.anyLocalCIDR, err = parseCIDR(localCidr)
if err != nil {
return rule{}, fmt.Errorf("local_cidr did not parse; %w", err)
}
if localCIDR.IsValid() {
r.localCIDR = new(bart.Lite)
r.localCIDR.Insert(localCIDR)
}
return r, nil
}
// add keeps r for proto, or for every protocol when proto is ProtoAny
func (tr *tableRules) add(proto uint8, r rule) {
switch proto {
case ProtoAny:
tr.anyRules = append(tr.anyRules, r)
case iputil.IPProtocolICMPv6:
tr.protoRules[iputil.IPProtocolICMP] = append(tr.protoRules[iputil.IPProtocolICMP], r)
default:
tr.protoRules[proto] = append(tr.protoRules[proto], r)
}
}
// build turns the rules into a Table. Rules without a local cidr allow defaultLocal,
// or any local address when defaultLocal is nil.
func (tr *tableRules) build(defaultLocal *bart.Lite) *Table {
t := &Table{}
// Every protocol starts out sharing the proto `any` rules, including protos[ProtoAny]
anyProto := newPortRules(defaultLocal, tr.anyRules)
for proto := range t.protos {
t.protos[proto] = anyProto
}
// Protocols with rules of their own get a copy of the proto `any` rules too, so a packet only checks one set
for proto, rules := range tr.protoRules {
if len(rules) > 0 {
t.protos[proto] = newPortRules(defaultLocal, tr.anyRules, rules)
}
}
// ICMP and ICMPv6 share one set of rules, add keeps them together under ICMP
t.protos[iputil.IPProtocolICMPv6] = t.protos[iputil.IPProtocolICMP]
return t
}
// newPortRules builds portRules from each set of rules, or returns nil when there are none
func newPortRules(defaultLocal *bart.Lite, ruleSets ...[]rule) portRules {
var pr portRules
for _, rules := range ruleSets {
for _, r := range rules {
if pr == nil {
pr = portRules{}
}
pr.add(r.withDefaultLocal(defaultLocal))
}
}
return pr
}
// parseCIDR parses a rule's cidr, which is empty, `any`, or a prefix
func parseCIDR(s string) (prefix netip.Prefix, isAny bool, err error) {
switch s {
case "":
return netip.Prefix{}, false, nil
case "any":
return netip.Prefix{}, true, nil
}
prefix, err = netip.ParsePrefix(s)
return prefix, false, err
}
// isAny reports whether r matches any host, making its groups, host, and cidr irrelevant
func (r rule) isAny() bool {
if len(r.groups) == 0 && r.host == "" && !r.cidr.IsValid() {
return true
}
return slices.Contains(r.groups, "any") || r.host == "any" || r.anyCIDR
}
// withDefaultLocal returns r allowing defaultLocal, or any local address when defaultLocal is nil,
// unless r has a local cidr of its own
func (r rule) withDefaultLocal(defaultLocal *bart.Lite) rule {
if !r.anyLocalCIDR && r.localCIDR == nil {
r.anyLocalCIDR = defaultLocal == nil
r.localCIDR = defaultLocal
}
return r
}
func (pr portRules) add(r rule) {
for i := r.startPort; i <= r.endPort; i++ {
cr := pr[i]
if cr == nil {
cr = &caRules{}
pr[i] = cr
}
cr.add(r)
}
}
func (cr *caRules) add(r rule) {
if r.caSha == "" && r.caName == "" {
if cr.Any == nil {
cr.Any = &remoteRules{}
}
cr.Any.add(r)
return
}
if r.caSha != "" {
getOrNew(&cr.CAShas, r.caSha).add(r)
}
if r.caName != "" {
getOrNew(&cr.CANames, r.caName).add(r)
}
}
func (rr *remoteRules) add(r rule) {
if r.isAny() {
if rr.Any == nil {
rr.Any = &localRules{}
}
rr.Any.add(r)
return
}
if len(r.groups) > 0 {
lr := &localRules{}
lr.add(r)
rr.Groups = append(rr.Groups, &groupsRule{
Groups: r.groups,
LocalCIDR: lr,
})
}
if r.host != "" {
getOrNew(&rr.Hosts, r.host).add(r)
}
if r.cidr.IsValid() {
if rr.CIDR == nil {
rr.CIDR = new(bart.Table[*localRules])
}
lr, ok := rr.CIDR.Get(r.cidr)
if !ok {
lr = &localRules{}
rr.CIDR.Insert(r.cidr, lr)
}
lr.add(r)
}
}
func (lr *localRules) add(r rule) {
if r.anyLocalCIDR {
lr.Any = true
return
}
// Share the rule's local cidrs until another rule's need adding, then copy them instead of changing them
if lr.LocalCIDR == nil {
lr.LocalCIDR = r.localCIDR
return
}
for prefix := range r.localCIDR.All() {
lr.LocalCIDR = lr.LocalCIDR.InsertPersist(prefix)
}
}
// getOrNew returns m[k], adding a new value for k first when there isn't one. m is made if it's nil.
func getOrNew[M ~map[K]*V, K comparable, V any](m *M, k K) *V {
if *m == nil {
*m = make(M)
}
v := (*m)[k]
if v == nil {
v = new(V)
(*m)[k] = v
}
return v
}
+239
View File
@@ -0,0 +1,239 @@
package firewall
import (
"math"
"net/netip"
"testing"
"github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRulesBuilder_AddRule(t *testing.T) {
l := test.NewLogger()
ti, err := netip.ParsePrefix("1.2.3.4/32")
require.NoError(t, err)
ti6, err := netip.ParsePrefix("fd12::34/128")
require.NoError(t, err)
rb := NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, []string{}, "", "", "", "", ""))
rules := rb.Build(nil, nil)
// An empty rule is any
assert.True(t, rules.In.protos[iputil.IPProtocolTCP][1].Any.Any.Any)
assert.Empty(t, rules.In.protos[iputil.IPProtocolTCP][1].Any.Groups)
assert.Empty(t, rules.In.protos[iputil.IPProtocolTCP][1].Any.Hosts)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolUDP, 1, 1, []string{"g1"}, "", "", "", "", ""))
rules = rb.Build(nil, nil)
assert.Nil(t, rules.In.protos[iputil.IPProtocolUDP][1].Any.Any)
assert.Contains(t, rules.In.protos[iputil.IPProtocolUDP][1].Any.Groups[0].Groups, "g1")
assert.Empty(t, rules.In.protos[iputil.IPProtocolUDP][1].Any.Hosts)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolICMP, 1, 1, []string{}, "h1", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolICMPv6, 1, 1, []string{}, "h2", "", "", "", ""))
rules = rb.Build(nil, nil)
//no matter what port is given for icmp, it should end up as "any"
assert.Nil(t, rules.In.protos[iputil.IPProtocolICMP][PortAny].Any.Any)
assert.Empty(t, rules.In.protos[iputil.IPProtocolICMP][PortAny].Any.Groups)
assert.Contains(t, rules.In.protos[iputil.IPProtocolICMP][PortAny].Any.Hosts, "h1")
// ICMP and ICMPv6 share one set of rules
assert.Contains(t, rules.In.protos[iputil.IPProtocolICMP][PortAny].Any.Hosts, "h2")
assert.Contains(t, rules.In.protos[iputil.IPProtocolICMPv6][PortAny].Any.Hosts, "h1")
// Any other protocol number gets its own rules
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, 47, 0, 0, []string{"g1"}, "", "", "", "", "")) // GRE
rules = rb.Build(nil, nil)
assert.Contains(t, rules.In.protos[47][PortAny].Any.Groups[0].Groups, "g1")
assert.Nil(t, rules.In.protos[iputil.IPProtocolTCP])
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", ti.String(), "", "", ""))
rules = rb.Build(nil, nil)
assert.Nil(t, rules.Out.protos[ProtoAny][1].Any.Any)
_, ok := rules.Out.protos[ProtoAny][1].Any.CIDR.Get(ti)
assert.True(t, ok)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", ti6.String(), "", "", ""))
rules = rb.Build(nil, nil)
assert.Nil(t, rules.Out.protos[ProtoAny][1].Any.Any)
_, ok = rules.Out.protos[ProtoAny][1].Any.CIDR.Get(ti6)
assert.True(t, ok)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", "", ti.String(), "", ""))
rules = rb.Build(nil, nil)
assert.NotNil(t, rules.Out.protos[ProtoAny][1].Any.Any)
ok = rules.Out.protos[ProtoAny][1].Any.Any.LocalCIDR.Get(ti)
assert.True(t, ok)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", "", ti6.String(), "", ""))
rules = rb.Build(nil, nil)
assert.NotNil(t, rules.Out.protos[ProtoAny][1].Any.Any)
ok = rules.Out.protos[ProtoAny][1].Any.Any.LocalCIDR.Get(ti6)
assert.True(t, ok)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolUDP, 1, 1, []string{"g1"}, "", "", "", "ca-name", ""))
rules = rb.Build(nil, nil)
assert.Contains(t, rules.In.protos[iputil.IPProtocolUDP][1].CANames, "ca-name")
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolUDP, 1, 1, []string{"g1"}, "", "", "", "", "ca-sha"))
rules = rb.Build(nil, nil)
assert.Contains(t, rules.In.protos[iputil.IPProtocolUDP][1].CAShas, "ca-sha")
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "any", "", "", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, rules.Out.protos[ProtoAny][0].Any.Any.Any)
rb = NewRulesBuilder(l)
anyIp, err := netip.ParsePrefix("0.0.0.0/0")
require.NoError(t, err)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", anyIp.String(), "", "", ""))
rules = rb.Build(nil, nil)
assert.Nil(t, rules.Out.protos[ProtoAny][0].Any.Any)
table, ok := rules.Out.protos[ProtoAny][0].Any.CIDR.Lookup(netip.MustParseAddr("1.1.1.1"))
assert.True(t, table.Any)
table, ok = rules.Out.protos[ProtoAny][0].Any.CIDR.Lookup(netip.MustParseAddr("9::9"))
assert.False(t, ok)
rb = NewRulesBuilder(l)
anyIp6, err := netip.ParsePrefix("::/0")
require.NoError(t, err)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", anyIp6.String(), "", "", ""))
rules = rb.Build(nil, nil)
assert.Nil(t, rules.Out.protos[ProtoAny][0].Any.Any)
table, ok = rules.Out.protos[ProtoAny][0].Any.CIDR.Lookup(netip.MustParseAddr("9::9"))
assert.True(t, table.Any)
table, ok = rules.Out.protos[ProtoAny][0].Any.CIDR.Lookup(netip.MustParseAddr("1.1.1.1"))
assert.False(t, ok)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "any", "", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, rules.Out.protos[ProtoAny][0].Any.Any.Any)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "", anyIp.String(), "", ""))
rules = rb.Build(nil, nil)
assert.False(t, rules.Out.protos[ProtoAny][0].Any.Any.Any)
assert.True(t, rules.Out.protos[ProtoAny][0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("1.1.1.1")))
assert.False(t, rules.Out.protos[ProtoAny][0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("9::9")))
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "", anyIp6.String(), "", ""))
rules = rb.Build(nil, nil)
assert.False(t, rules.Out.protos[ProtoAny][0].Any.Any.Any)
assert.True(t, rules.Out.protos[ProtoAny][0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("9::9")))
assert.False(t, rules.Out.protos[ProtoAny][0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("1.1.1.1")))
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "", "any", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, rules.Out.protos[ProtoAny][0].Any.Any.Any)
// Every protocol number is accepted
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, math.MaxUint8, 0, 0, []string{}, "", "", "", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, rules.In.protos[math.MaxUint8][PortAny].Any.Any.Any)
// Test error conditions, a bad rule is reported when it's added
rb = NewRulesBuilder(l)
require.Error(t, rb.AddRule(true, ProtoAny, 10, 0, []string{}, "", "", "", "", ""))
require.Error(t, rb.AddRule(true, ProtoAny, 0, 0, []string{}, "", "junk", "", "", ""))
require.Error(t, rb.AddRule(true, ProtoAny, 0, 0, []string{}, "", "", "junk", "", ""))
// ICMP and ICMPv6 share one copy of each proto `any` rule
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, ProtoAny, PortAny, PortAny, []string{"default-group"}, "", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolICMPv6, PortAny, PortAny, []string{"nope"}, "", "", "", "", ""))
rules = rb.Build(nil, nil)
assert.Len(t, rules.In.protos[iputil.IPProtocolICMP][PortAny].Any.Groups, 2)
assert.Len(t, rules.In.protos[iputil.IPProtocolICMPv6][PortAny].Any.Groups, 2)
// No rules allow nothing
rules = NewRulesBuilder(l).Build(nil, nil)
for proto := range rules.In.protos {
assert.Nil(t, rules.In.protos[proto], "proto %d", proto)
assert.Nil(t, rules.Out.protos[proto], "proto %d", proto)
}
}
// TestRulesBuilder_sharedLocalCIDR ensures local cidrs shared between rules and ports are copied, not changed,
// when another rule's local cidrs are added to them. The prefixes share their first octet so the copy has to reach
// past the root of the shared table.
func TestRulesBuilder_sharedLocalCIDR(t *testing.T) {
vpnNetworks := []netip.Prefix{
netip.MustParsePrefix("10.1.0.5/16"),
netip.MustParsePrefix("10.2.0.5/16"),
netip.MustParsePrefix("fd00:1::5/64"),
}
unsafeNetworks := []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}
vpn1 := netip.MustParseAddr("10.1.9.9")
vpn2 := netip.MustParseAddr("10.2.9.9")
vpn6 := netip.MustParseAddr("fd00:1::9")
h1Extra := netip.MustParseAddr("10.3.0.1")
h1Extra6 := netip.MustParseAddr("fd00:2::1")
h2Own := netip.MustParseAddr("10.4.0.1")
rb := NewRulesBuilder(test.NewLogger())
// h1 allows the vpn networks by default on ports 1 and 2, and 10.3.0.0/16 and fd00:2::/64 as well on port 1
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 2, nil, "h1", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, nil, "h1", "", "10.3.0.0/16", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, nil, "h1", "", "fd00:2::/64", "", ""))
// h2 allows 10.4.0.0/16 on ports 1 and 2, and the vpn networks by default as well on port 1
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 2, nil, "h2", "", "10.4.0.0/16", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, nil, "h2", "", "", "", ""))
// h3 only has the default
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 3, 3, nil, "h3", "", "", "", ""))
rules := rb.Build(vpnNetworks, unsafeNetworks)
for _, tc := range []struct {
port int32
host string
addr netip.Addr
want bool
}{
{1, "h1", vpn1, true},
{1, "h1", vpn2, true},
{1, "h1", vpn6, true},
{1, "h1", h1Extra, true},
{1, "h1", h1Extra6, true},
{1, "h1", h2Own, false},
{2, "h1", vpn1, true},
{2, "h1", h1Extra, false},
{2, "h1", h1Extra6, false},
{1, "h2", h2Own, true},
{1, "h2", vpn1, true},
{1, "h2", vpn2, true},
{1, "h2", vpn6, true},
{1, "h2", h1Extra, false},
{2, "h2", h2Own, true},
{2, "h2", vpn1, false},
{2, "h2", vpn6, false},
{3, "h3", vpn1, true},
{3, "h3", vpn2, true},
{3, "h3", vpn6, true},
{3, "h3", h1Extra, false},
{3, "h3", h1Extra6, false},
{3, "h3", h2Own, false},
} {
lr := rules.In.protos[iputil.IPProtocolTCP][tc.port].Any.Hosts[tc.host]
assert.Equal(t, tc.want, lr.match(&Packet{LocalAddr: tc.addr}, nil), "port %d, %s, %s", tc.port, tc.host, tc.addr)
}
}
+188
View File
@@ -0,0 +1,188 @@
package firewall
import (
"net/netip"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/iputil"
)
// Rules are the Tables built for each direction, with hashes of the rules they were built from
type Rules struct {
In, Out *Table
// Hash is a sha256 of the rules, HashFNV is an FNV-1a of them for use as a metric value.
// Rule sets with the same rules have the same hashes.
Hash string
HashFNV uint32
}
// Table holds the rules for one direction, the evaluation order is:
// Proto AND port AND (CA SHA or CA name) AND local CIDR AND (group OR groups OR name OR remote CIDR)
// A Table doesn't change once built, see RulesBuilder.
type Table struct {
// protos holds the rules for each IP protocol number, so a packet is only checked against one set of rules.
// A protocol with rules of its own also holds a copy of every proto `any` rule, the rest share the proto `any`
// rules, which are at protos[ProtoAny]. nil when a protocol has no rules at all.
// ICMP and ICMPv6 share one set of rules.
protos [256]portRules
}
// Even though ports are uint16, int32 maps are faster for lookup
// Plus we can use `-1` for fragment rules
type portRules map[int32]*caRules
// caRules and the types below it only make their maps and tables once a rule needs them, a range of ports
// gets a copy for every port so empty ones add up
type caRules struct {
Any *remoteRules
CANames map[string]*remoteRules
CAShas map[string]*remoteRules
}
type remoteRules struct {
// Any makes Hosts, Groups, and CIDR irrelevant
Any *localRules
Hosts map[string]*localRules
Groups []*groupsRule
CIDR *bart.Table[*localRules]
}
type groupsRule struct {
Groups []string
LocalCIDR *localRules
}
type localRules struct {
Any bool
// LocalCIDR is always set when Any isn't. It's shared with other localRules, so it's never changed.
LocalCIDR *bart.Lite
}
// Match reports whether a rule for the direction p is going allows it
func (r *Rules) Match(p *Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
t := r.Out
if incoming {
t = r.In
}
return t.Match(p, incoming, c, caPool)
}
// Match reports whether a rule allows p
func (t *Table) Match(p *Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
return t.protos[p.Protocol].match(p, incoming, c, caPool)
}
func (pr portRules) match(p *Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
// We don't have any allowed ports, bail
if pr == nil {
return false
}
// ICMP has no ports, only the port `any` rules apply, including the ones copied in from proto `any` rules
if p.Protocol == iputil.IPProtocolICMP || p.Protocol == iputil.IPProtocolICMPv6 {
// port numbers are re-used for connection tracking of ICMP,
// but we don't want to actually filter on them.
return pr[PortAny].match(p, c, caPool)
}
var port int32
if p.Fragment {
port = PortFragment
} else if incoming {
port = int32(p.LocalPort)
} else {
port = int32(p.RemotePort)
}
// Packets without ports (gre, esp, etc) have port 0, which is PortAny, don't check those rules twice
if port != PortAny && pr[port].match(p, c, caPool) {
return true
}
return pr[PortAny].match(p, c, caPool)
}
func (cr *caRules) match(p *Packet, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
if cr == nil {
return false
}
if cr.Any.match(p, c) {
return true
}
if t, ok := cr.CAShas[c.Certificate.Issuer()]; ok {
if t.match(p, c) {
return true
}
}
s, err := caPool.GetCAForCert(c.Certificate)
if err != nil {
return false
}
return cr.CANames[s.Certificate.Name()].match(p, c)
}
func (rr *remoteRules) match(p *Packet, c *cert.CachedCertificate) bool {
if rr == nil {
return false
}
// Shortcut path for if groups, hosts, or cidr contained an `any`
if rr.Any.match(p, c) {
return true
}
// Need any of group, host, or cidr to match
for _, sg := range rr.Groups {
found := false
for _, g := range sg.Groups {
if _, ok := c.InvertedGroups[g]; !ok {
found = false
break
}
found = true
}
if found && sg.LocalCIDR.match(p, c) {
return true
}
}
if rr.Hosts != nil {
if lr, ok := rr.Hosts[c.Certificate.Name()]; ok {
if lr.match(p, c) {
return true
}
}
}
if rr.CIDR != nil {
for _, v := range rr.CIDR.Supernets(netip.PrefixFrom(p.RemoteAddr, p.RemoteAddr.BitLen())) {
if v.match(p, c) {
return true
}
}
}
return false
}
func (lr *localRules) match(p *Packet, c *cert.CachedCertificate) bool {
if lr == nil {
return false
}
if lr.Any {
return true
}
return lr.LocalCIDR.Contains(p.LocalAddr)
}
+270 -549
View File
File diff suppressed because it is too large. Load diff
-5
View File
@@ -1,5 +0,0 @@
// Package msgx calls xnu's private batched socket calls, sendmsg_x and recvmsg_x, through libSystem.
//
// The calls are not public API, so the package never links them: it looks them up with dlsym on first use.
// The package is darwin-only, and on iOS Available is always false, so App Store builds keep to public API.
package msgx
-16
View File
@@ -1,16 +0,0 @@
package msgx
import "golang.org/x/sys/unix"
// Hdr mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h), which the SDK doesn't ship.
// recvmsg_x reports each message's length in Datalen rather than in its return value.
type Hdr struct {
Name *byte
Namelen uint32
Iov *unix.Iovec
Iovlen int32
Control *byte
Controllen uint32
Flags int32
Datalen uint64
}
-77
View File
@@ -1,77 +0,0 @@
//go:build !ios
package msgx
import (
"sync"
"syscall"
"unsafe"
)
// dlsym is public libSystem API, imported the way x/sys/unix imports its libSystem stubs. The local name is
// distinct from runtime/race's dlsym import.
//
//go:cgo_import_dynamic msgx_dlsym dlsym "/usr/lib/libSystem.B.dylib"
// dlsymTrampolineAddr is the address of msgx_darwin.s's jump to dlsym.
var dlsymTrampolineAddr uintptr
// syscall6X calls the C function at fn on the system stack and returns errno when it returns a 64-bit -1.
// It is how x/sys/unix calls libSystem, and the runtime keeps it linkable for that.
//
//go:linkname syscall6X syscall.syscall6X
func syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno)
// rtldDefault is dlfcn.h's RTLD_DEFAULT, ((void *)-2): search every image loaded in the process.
const rtldDefault = ^uintptr(1)
var (
sendmsgXName = []byte("sendmsg_x\x00")
recvmsgXName = []byte("recvmsg_x\x00")
)
type funcs struct {
sendmsgX uintptr
recvmsgX uintptr
}
var resolve = sync.OnceValue(func() funcs {
return funcs{sendmsgX: lookup(sendmsgXName), recvmsgX: lookup(recvmsgXName)}
})
// lookup returns the address of the NUL-terminated C symbol name, or 0 when no loaded image exports it.
func lookup(name []byte) uintptr {
p, _, _ := syscall6X(dlsymTrampolineAddr, rtldDefault, uintptr(unsafe.Pointer(&name[0])), 0, 0, 0, 0)
return p
}
// Available reports whether libSystem exports both sendmsg_x and recvmsg_x. When it is false, Send and Recv
// return ENOSYS.
func Available() bool {
f := resolve()
return f.sendmsgX != 0 && f.recvmsgX != 0
}
// Send passes hdrs to sendmsg_x on fd and returns how many messages the kernel took.
func Send(fd uintptr, hdrs []Hdr, flags int) (int, syscall.Errno) {
return call(resolve().sendmsgX, fd, hdrs, flags)
}
// Recv passes hdrs to recvmsg_x on fd and returns how many messages it filled.
func Recv(fd uintptr, hdrs []Hdr, flags int) (int, syscall.Errno) {
return call(resolve().recvmsgX, fd, hdrs, flags)
}
func call(fn, fd uintptr, hdrs []Hdr, flags int) (int, syscall.Errno) {
if fn == 0 {
return 0, syscall.ENOSYS
}
if len(hdrs) == 0 {
return 0, syscall.EINVAL
}
r, _, errno := syscall6X(fn, fd, uintptr(unsafe.Pointer(&hdrs[0])), uintptr(len(hdrs)), uintptr(flags), 0, 0)
if errno != 0 {
return 0, errno
}
return int(r), 0
}
-9
View File
@@ -1,9 +0,0 @@
//go:build !ios
#include "textflag.h"
// The same trampoline shape x/sys/unix uses for its libSystem stubs; it assembles for amd64 and arm64 alike.
TEXT msgx_dlsym_trampoline<>(SB),NOSPLIT,$0-0
JMP msgx_dlsym(SB)
GLOBL ·dlsymTrampolineAddr(SB), RODATA, $8
DATA ·dlsymTrampolineAddr(SB)/8, $msgx_dlsym_trampoline<>(SB)
-69
View File
@@ -1,69 +0,0 @@
//go:build !ios
package msgx
import (
"bytes"
"syscall"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
// TestAvailable pins that dlsym finds both calls in this macOS's libSystem, so the batched paths are live.
func TestAvailable(t *testing.T) {
assert.True(t, Available())
}
// TestSendRecvRoundTrip pins the libSystem calling path end to end: a batch goes out through sendmsg_x in one
// call and comes back through recvmsg_x in one call, each message whole, in order, with Datalen set.
func TestSendRecvRoundTrip(t *testing.T) {
fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
require.NoError(t, err)
t.Cleanup(func() {
_ = unix.Close(fds[0])
_ = unix.Close(fds[1])
})
// A dropped datagram fails the receive instead of hanging it.
require.NoError(t, unix.SetsockoptTimeval(fds[1], unix.SOL_SOCKET, unix.SO_RCVTIMEO, &unix.Timeval{Sec: 2}))
const n = 5
var want [n][]byte
var iovs [n]unix.Iovec
hdrs := make([]Hdr, n)
for i := range n {
want[i] = bytes.Repeat([]byte{byte(i + 1)}, 10+i)
iovs[i] = unix.Iovec{Base: &want[i][0]}
iovs[i].SetLen(len(want[i]))
hdrs[i] = Hdr{Iov: &iovs[i], Iovlen: 1}
}
sent, errno := Send(uintptr(fds[0]), hdrs, 0)
require.Zero(t, errno)
require.Equal(t, n, sent)
var bufs [n + 1][64]byte
var riovs [n + 1]unix.Iovec
rhdrs := make([]Hdr, n+1)
for i := range rhdrs {
riovs[i] = unix.Iovec{Base: &bufs[i][0]}
riovs[i].SetLen(len(bufs[i]))
rhdrs[i] = Hdr{Iov: &riovs[i], Iovlen: 1}
}
got, errno := Recv(uintptr(fds[1]), rhdrs, 0)
require.Zero(t, errno)
require.Equal(t, n, got)
for i := range n {
l := int(rhdrs[i].Datalen)
assert.Equal(t, want[i], bufs[i][:l], "message %d", i)
}
}
// TestEmptyBatch pins that an empty batch is refused before it reaches the kernel.
func TestEmptyBatch(t *testing.T) {
_, errno := Send(0, nil, 0)
assert.Equal(t, syscall.EINVAL, errno)
_, errno = Recv(0, nil, 0)
assert.Equal(t, syscall.EINVAL, errno)
}
-12
View File
@@ -1,12 +0,0 @@
package msgx
import "syscall"
// Available is always false on iOS, so App Store builds never look up private symbols.
func Available() bool { return false }
// Send always returns ENOSYS on iOS.
func Send(uintptr, []Hdr, int) (int, syscall.Errno) { return 0, syscall.ENOSYS }
// Recv always returns ENOSYS on iOS.
func Recv(uintptr, []Hdr, int) (int, syscall.Errno) { return 0, syscall.ENOSYS }
+3
View File
@@ -32,6 +32,9 @@ const (
IPProtocolICMPv6 = 58
IPProtocolTCP = 6
IPProtocolUDP = 17
IPProtocolUDPLite = 136
IPProtocolDCCP = 33
IPProtocolSCTP = 132
ICMPv6TypeEchoRequest = 128
ICMPv6TypeEchoReply = 129
)
+15 -3
View File
@@ -385,7 +385,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
fp.RemotePort = 0
}
case iputil.IPProtocolTCP, iputil.IPProtocolUDP:
case iputil.IPProtocolTCP, iputil.IPProtocolUDP, iputil.IPProtocolDCCP, iputil.IPProtocolSCTP, iputil.IPProtocolUDPLite:
if dataLen < offset+4 {
return ErrIPv6PacketTooShort
}
@@ -456,10 +456,16 @@ func parseV4(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
if fp.Fragment {
fp.RemotePort = 0
fp.LocalPort = 0
} else if fp.Protocol == iputil.IPProtocolICMP { //note that orientation doesn't matter on ICMP
return nil
}
switch fp.Protocol {
case iputil.IPProtocolICMP: //note that orientation doesn't matter on ICMP
fp.RemotePort = binary.BigEndian.Uint16(data[ihl+4 : ihl+6]) //identifier
fp.LocalPort = 0 //code would be uint16(data[ihl+1])
} else if incoming {
case iputil.IPProtocolTCP, iputil.IPProtocolUDP, iputil.IPProtocolDCCP, iputil.IPProtocolSCTP, iputil.IPProtocolUDPLite:
if incoming {
fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2]) //src port
fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
} else {
@@ -467,6 +473,12 @@ func parseV4(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
}
default:
// don't set ports for protocols Nebula doesn't inspect, the first 4 bytes of GRE, IPIP, AH, etc. are not ports
fp.RemotePort = 0
fp.LocalPort = 0
}
return nil
}
+84 -19
View File
@@ -77,7 +77,7 @@ func Test_newPacket(t *testing.T) {
// account for variable ip header length - outgoing
h = ipv4.Header{
Version: 1,
Protocol: 2,
Protocol: iputil.IPProtocolUDP,
Len: 100,
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
@@ -89,7 +89,7 @@ func Test_newPacket(t *testing.T) {
err = newPacket(b, false, p)
require.NoError(t, err)
assert.Equal(t, uint8(2), p.Protocol)
assert.Equal(t, uint8(iputil.IPProtocolUDP), p.Protocol)
assert.Equal(t, netip.MustParseAddr("10.0.0.1"), p.LocalAddr)
assert.Equal(t, netip.MustParseAddr("10.0.0.2"), p.RemoteAddr)
assert.Equal(t, uint16(6), p.RemotePort)
@@ -97,6 +97,71 @@ func Test_newPacket(t *testing.T) {
assert.False(t, p.Fragment)
}
// Test_newPacket_v4NoPorts ensures only protocols that lead with src/dst ports report them. The first 4 bytes of other protocols are not ports,
// and are often chosen by the sender: a GRE header's protocol type, an IPIP inner total length, a 6in4 flow label.
// Reading them as ports would let a `proto: any` rule with a port match tunneled traffic.
func Test_newPacket_v4NoPorts(t *testing.T) {
p := &firewall.ParsedPacket{}
for _, proto := range []uint8{4, 41, 47, 50, 51} { // IPIP, 6in4, GRE, ESP, AH
h := ipv4.Header{
Version: 4,
Len: ipv4.HeaderLen,
Protocol: int(proto),
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
}
b, err := h.Marshal()
require.NoError(t, err)
b = append(b, []byte{0x00, 0x00, 0x86, 0xdd}...) // a GRE header carrying IPv6 would read as port 34525
for _, incoming := range []bool{true, false} {
require.NoError(t, newPacket(b, incoming, p))
assert.Equal(t, proto, p.Protocol)
assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment)
}
}
}
// Test_newPacket_portedProtocols ensures UDP-Lite, DCCP, and SCTP report ports from the same leading
// src/dst port words as TCP and UDP, oriented the same way.
func Test_newPacket_portedProtocols(t *testing.T) {
p := &firewall.ParsedPacket{}
ports := []byte{0x13, 0x88, 0x00, 0x50} // src 5000, dst 80
for _, proto := range []uint8{iputil.IPProtocolUDPLite, iputil.IPProtocolDCCP, iputil.IPProtocolSCTP} {
h := ipv4.Header{
Version: 4,
Len: ipv4.HeaderLen,
Protocol: int(proto),
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
}
v4, err := h.Marshal()
require.NoError(t, err)
v4 = append(v4, ports...)
v6 := make([]byte, 40, 44)
v6[0] = 0x60
v6[6] = proto
v6[7] = 64
v6 = append(v6, ports...)
for _, b := range [][]byte{v4, v6} {
require.NoError(t, newPacket(b, true, p))
assert.Equal(t, proto, p.Protocol)
assert.Equal(t, uint16(5000), p.RemotePort)
assert.Equal(t, uint16(80), p.LocalPort)
require.NoError(t, newPacket(b, false, p))
assert.Equal(t, uint16(5000), p.LocalPort)
assert.Equal(t, uint16(80), p.RemotePort)
}
}
}
func Test_newPacket_v6(t *testing.T) {
p := &firewall.ParsedPacket{}
@@ -712,46 +777,46 @@ func Test_newPacket_v6ExtHeaderPastBuffer(t *testing.T) {
// Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized
// Next Header as if it were an ipv6 extension header. A real upper layer protocol Nebula doesn't
// dissect (SCTP here) is not walkable, so applying the (len+1)*8 formula marched into the SCTP
// dissect (GRE here) is not walkable, so applying the (len+1)*8 formula marched into the GRE
// payload and landed on a byte that looked like UDP, forging a protocol/port pair the firewall
// would trust while the host delivered the real SCTP datagram. The fix fails closed: the packet
// would trust while the host delivered the real GRE packet. The fix fails closed: the packet
// is classified as its true protocol with no ports, so it only matches an `any` rule.
func Test_newPacket_v6ExtHeaderConfusion(t *testing.T) {
p := &firewall.ParsedPacket{}
pkt := make([]byte, 52)
pkt[0] = 0x60 // version 6
pkt[6] = byte(layers.IPProtocolSCTP) // NextHeader = SCTP, a real protocol, not an extension header
pkt[6] = byte(layers.IPProtocolGRE) // NextHeader = GRE, a real protocol, not an extension header
pkt[7] = 64 // hop limit
// Real SCTP header at offset 40. Pre-fix parseV6 walked SCTP as an extension header: byte 41 (0x00, the
// low byte of the src port below) was read as the header length, giving next=(0+1)*8=8, which landed the
// walk on byte 40 (0x11), misread as NextHeader=UDP, then bytes 48-51 as ports.
binary.BigEndian.PutUint16(pkt[40:42], 0x1100) // SCTP src port; byte 40=0x11, byte 41=0x00
binary.BigEndian.PutUint16(pkt[42:44], 445) // SCTP dst port, never read by parseV6
binary.BigEndian.PutUint16(pkt[48:50], 53) // SCTP checksum bytes, pre-fix forged RemotePort
// Real GRE header at offset 40. Pre-fix parseV6 walked GRE as an extension header: byte 41 (0x00, the
// low byte of the flags/version word below) was read as the header length, giving next=(0+1)*8=8, which
// landed the walk on byte 40 (0x11), misread as NextHeader=UDP, then bytes 48-51 as ports.
binary.BigEndian.PutUint16(pkt[40:42], 0x1100) // GRE flags/version; byte 40=0x11, byte 41=0x00
binary.BigEndian.PutUint16(pkt[42:44], 0x86dd) // GRE protocol type, never read by parseV6
binary.BigEndian.PutUint16(pkt[48:50], 53) // GRE payload, pre-fix forged RemotePort
binary.BigEndian.PutUint16(pkt[50:52], 53) // pre-fix forged LocalPort
require.NoError(t, newPacket(pkt, true, p))
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must classify as the true protocol, not the forged UDP")
assert.Equal(t, uint8(layers.IPProtocolGRE), p.Protocol, "must classify as the true protocol, not the forged UDP")
assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment)
// Same confusion, but the unknown protocol sits after a real extension header. The HopByHop is walked
// correctly, then SCTP must still fail closed instead of being walked into its own payload. Protocol is
// the only assertion that discriminates the fix here, a regression that walked SCTP would misclassify it.
// correctly, then GRE must still fail closed instead of being walked into its own payload. Protocol is
// the only assertion that discriminates the fix here, a regression that walked GRE would misclassify it.
chained := make([]byte, 60)
chained[0] = 0x60 // version 6
chained[6] = byte(layers.IPProtocolIPv6HopByHop) // NextHeader = HopByHop extension
chained[7] = 64 // hop limit
chained[40] = byte(layers.IPProtocolSCTP) // HopByHop NextHeader = SCTP
chained[41] = 0 // HopByHop length 0 -> 8 bytes, SCTP begins at offset 48
binary.BigEndian.PutUint16(chained[48:50], 0x1100) // SCTP src port, pre-fix forged NextHeader/length bait
binary.BigEndian.PutUint16(chained[50:52], 445) // SCTP dst port, never read by parseV6
chained[40] = byte(layers.IPProtocolGRE) // HopByHop NextHeader = GRE
chained[41] = 0 // HopByHop length 0 -> 8 bytes, GRE begins at offset 48
binary.BigEndian.PutUint16(chained[48:50], 0x1100) // GRE flags/version, pre-fix forged NextHeader/length bait
binary.BigEndian.PutUint16(chained[50:52], 0x86dd) // GRE protocol type, never read by parseV6
require.NoError(t, newPacket(chained, true, p))
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must fail closed on the unknown protocol after the extension header")
assert.Equal(t, uint8(layers.IPProtocolGRE), p.Protocol, "must fail closed on the unknown protocol after the extension header")
assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment)
-62
View File
@@ -435,65 +435,3 @@ func testPP(pkt []byte) *firewall.ParsedPacket {
}
return pp
}
// fakeTunBatchWriter records each WriteBatch call and each plain Write separately.
type fakeTunBatchWriter struct {
writes [][]byte
batches [][][]byte
}
func (w *fakeTunBatchWriter) Write(p []byte) (int, error) {
w.writes = append(w.writes, bytes.Clone(p))
return len(p), nil
}
func (w *fakeTunBatchWriter) WriteBatch(pkts [][]byte) error {
var b [][]byte
for _, p := range pkts {
b = append(b, bytes.Clone(p))
}
w.batches = append(w.batches, b)
return nil
}
// TestPassthroughUsesWriteBatch pins that a writer implementing tio.BatchWriter gets the whole
// passthrough lane in one WriteBatch per Flush, in order, and no plain Writes; an empty Flush
// makes no call.
func TestPassthroughUsesWriteBatch(t *testing.T) {
w := &fakeTunBatchWriter{}
m := newTestMultiCoalescer(t, w)
k := &keySeq{epoch: 1}
var want [][]byte
for i := range 5 {
icmp := make([]byte, 28)
icmp[0] = 0x45
icmp[3] = 28
icmp[9] = 1
icmp[20] = byte(i)
want = append(want, icmp)
if err := m.Commit(icmp, k.next(), testPP(icmp)); err != nil {
t.Fatal(err)
}
}
if err := m.Flush(); err != nil {
t.Fatal(err)
}
if err := m.Flush(); err != nil {
t.Fatal(err)
}
if len(w.writes) != 0 {
t.Fatalf("got %d plain Writes, want 0", len(w.writes))
}
if len(w.batches) != 1 {
t.Fatalf("got %d WriteBatch calls, want 1", len(w.batches))
}
for i, p := range w.batches[0] {
if !bytes.Equal(p, want[i]) {
t.Fatalf("packet %d: got % x, want % x", i, p, want[i])
}
}
if len(w.batches[0]) != len(want) {
t.Fatalf("got %d packets, want %d", len(w.batches[0]), len(want))
}
}
+2 -13
View File
@@ -2,23 +2,18 @@ package batch
import (
"io"
"github.com/slackhq/nebula/overlay/tio"
)
// Passthrough is MultiCoalescer's verbatim lane: no coalescing, packets are written at Flush in the
// order enqueued, in one WriteBatch when out implements tio.BatchWriter.
// Passthrough is MultiCoalescer's verbatim lane: no batching, packets are written at Flush in the
// order enqueued.
type Passthrough struct {
out io.Writer
bw tio.BatchWriter // nil: Flush writes one packet per Write
slots [][]byte
}
func NewPassthrough(w io.Writer) *Passthrough {
bw, _ := w.(tio.BatchWriter)
return &Passthrough{
out: w,
bw: bw,
slots: make([][]byte, 0, 128),
}
}
@@ -31,18 +26,12 @@ func (p *Passthrough) enqueue(pkt []byte) error {
func (p *Passthrough) Flush() error {
var firstErr error
if p.bw != nil {
if len(p.slots) > 0 {
firstErr = p.bw.WriteBatch(p.slots)
}
} else {
for _, s := range p.slots {
_, err := p.out.Write(s)
if err != nil && firstErr == nil {
firstErr = err
}
}
}
clear(p.slots)
p.slots = p.slots[:0]
return firstErr
-11
View File
@@ -145,14 +145,3 @@ func SupportsGSO(w io.Writer, want GSOProto) (GSOWriter, bool) {
return gw, false
}
}
// BatchWriter is implemented by Queues that can emit several already-segmented packets in one call.
// Backends without a batched write do not implement it and callers loop over Write.
//
// pkts are borrowed: WriteBatch must not retain them past its return.
// A returned error doesn't say which packets were delivered, so callers must not retry the batch.
// Safe for concurrent use, like Write.
type BatchWriter interface {
io.Writer
WriteBatch(pkts [][]byte) error
}
-328
View File
@@ -1,328 +0,0 @@
//go:build darwin && !ios
package tio
import (
"fmt"
"log/slog"
"os"
"sync"
"sync/atomic"
"syscall"
_ "unsafe" // for go:linkname
"github.com/slackhq/nebula/internal/msgx"
"golang.org/x/sys/unix"
)
// tunMaxPacket is the largest packet utun can return at any device MTU.
const tunMaxPacket = 65535
// tunReadBatch is the most packets one Utun.Read returns.
const tunReadBatch = 64
// tunReadArena is Utun's receive buffer. Draining stops once less than tunMaxPacket of it is left,
// so every readv has room for the largest packet utun can return.
const tunReadArena = 4 * tunMaxPacket
// tunWriteBatch is how many packets one sendmsg_x call may carry.
const tunWriteBatch = 64
// Utun is a darwin utun control socket as a Queue. Every packet crosses the socket behind a 4 byte
// address family prefix, which Read strips and the writes add.
//
// The fd lives in the runtime poller, so Close wakes a Read parked on it; darwin needs no QueueSet
// shutdown fd.
type Utun struct {
f *os.File
l *slog.Logger
// readBuf and readRet are Read's scratch, owned by the single reader.
readBuf []byte
readRet [tunReadBatch]Packet
// noSendmsgX is set when libSystem lacks sendmsg_x or the kernel refuses it; WriteBatch then writes one packet at a time.
noSendmsgX atomic.Bool
batchMu sync.Mutex
batch tunBatch
}
// tunBatch is WriteBatch's scratch, guarded by Utun.batchMu. The kernel reads every entry in place.
type tunBatch struct {
heads [tunWriteBatch][4]byte
iovs [tunWriteBatch][2]unix.Iovec
hdrs [tunWriteBatch]msgx.Hdr
pkts [tunWriteBatch][]byte
}
// NewUtun wraps a utun control socket fd and makes it non-blocking.
// On failure it does NOT close fd: the caller owns fd until NewUtun succeeds, after which Close closes it.
func NewUtun(fd int, l *slog.Logger) (*Utun, error) {
// Before os.NewFile, which only hands a non-blocking fd to the runtime poller.
if err := unix.SetNonblock(fd, true); err != nil {
return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err)
}
u := &Utun{
f: os.NewFile(uintptr(fd), "utun"),
l: l,
readBuf: make([]byte, tunReadArena),
}
u.noSendmsgX.Store(!msgx.Available())
return u, nil
}
// tunWritev and tunReadv are linkname'd to x/sys/unix's libc-routed writev/readv stubs so the
// calls go through libSystem's pinned trampoline. A raw syscall.Syscall(SYS_WRITEV/SYS_READV, ...)
// on darwin/arm64 emits an SVC #0x80 trap (see $GOROOT/src/syscall/asm_darwin_arm64.s), the path
// Apple keeps warning they will eventually disallow. We pull the low-level stubs instead of calling
// unix.Writev/unix.Readv because those take [][]byte and rebuild the []Iovec every call, which
// heap-allocates the header; linkname'ing the stubs lets us hand them our own stack-allocated
// iovecs. See golang/go#78049.
//go:linkname tunWritev golang.org/x/sys/unix.writev
//go:noescape
func tunWritev(fd int, iovecs []unix.Iovec) (n int, err error)
//go:linkname tunReadv golang.org/x/sys/unix.readv
//go:noescape
func tunReadv(fd int, iovecs []unix.Iovec) (n int, err error)
// Read waits for the utun to become readable, then reads packets until it would block, up to
// tunReadBatch, so the caller encrypts and sends them as one batch rather than one per wakeup. Each
// packet is read with its AF prefix scattered away, so the payload lands directly in readBuf.
// An error after at least one packet is dropped in favor of returning those packets; a persistent
// one recurs on the next Read.
func (u *Utun) Read() ([]Packet, error) {
rc, err := u.f.SyscallConn()
if err != nil {
return nil, err
}
var head [4]byte
n, off := 0, 0
var callErr error
err = rc.Read(func(fd uintptr) bool {
for n < tunReadBatch && len(u.readBuf)-off >= tunMaxPacket {
iovecs := [2]unix.Iovec{
{Base: &head[0], Len: 4},
{Base: &u.readBuf[off], Len: uint64(len(u.readBuf) - off)},
}
l, e := tunReadv(int(fd), iovecs[:])
if e != nil {
if errno, ok := e.(syscall.Errno); ok && errno.Temporary() {
// Park on the poller only while there is nothing to hand back.
return n > 0
}
callErr = e
return true
}
if l < 4 {
// A datagram too short to carry the AF prefix, or end of file; stop rather than spin on it.
return true
}
end := off + l - 4
u.readRet[n] = Packet{Bytes: u.readBuf[off:end:end]}
n++
off = end
}
return true
})
if n > 0 {
return u.readRet[:n], nil
}
if err != nil {
return nil, err
}
return nil, callErr
}
// Write pushes one IP packet onto the utun device. Safe for concurrent use:
// the AF prefix and iovecs are per-call stack state, and the fd write itself
// serializes on the runtime's fd mutex (see the Queue contract in tio.go).
func (u *Utun) Write(from []byte) (int, error) {
if len(from) == 0 {
return 0, syscall.EIO
}
var head [4]byte
af, err := tunAF(from)
if err != nil {
return 0, err
}
head[3] = af
// Grab rc as a local so the compiler can devirtualize the call and keep the closure on the stack.
rc, err := u.f.SyscallConn()
if err != nil {
return 0, err
}
var n int
var callErr error
err = rc.Write(func(fd uintptr) bool {
iovecs := []unix.Iovec{
{Base: &head[0], Len: 4},
{Base: &from[0], Len: uint64(len(from))},
}
n, callErr = tunWritev(int(fd), iovecs)
// Type-assert to syscall.Errno so the EAGAIN/EWOULDBLOCK/EINTR check doesn't box the errno
// constants into error interfaces on every call.
if errno, ok := callErr.(syscall.Errno); ok && errno.Temporary() {
return false
}
return true
})
if err != nil {
return 0, err
}
if callErr != nil {
return 0, callErr
}
return n - 4, nil
}
// tunAF returns the utun address-family prefix byte for an IP packet.
func tunAF(pkt []byte) (byte, error) {
switch pkt[0] >> 4 {
case 4:
return syscall.AF_INET, nil
case 6:
return syscall.AF_INET6, nil
default:
return 0, fmt.Errorf("unable to determine IP version from packet")
}
}
// WriteBatch writes pkts to the utun device with sendmsg_x, xnu's private batched sendmsg, up to
// tunWriteBatch packets per syscall. The kernel still hands each packet to utun on its own, so this
// saves syscalls, not per-packet kernel work. Safe for concurrent use.
//
// An empty packet or one with no IP version is skipped and reported, as Write would. Any other
// error from sendmsg_x means the kernel took an unknown prefix of that call and dropped the rest, so
// it is reported rather than retried, which could deliver packets twice.
//
// Under mbuf exhaustion, sendmsg_x silently drops packets that writev would have delivered: the
// kernel counts each packet as sent once it is copied in, and when a later allocation fails it
// frees that prefix unsent and returns its length as a short count, which is indistinguishable
// from a real one.
func (u *Utun) WriteBatch(pkts [][]byte) error {
if u.noSendmsgX.Load() {
return u.writeEach(pkts)
}
u.batchMu.Lock()
defer u.batchMu.Unlock()
b := &u.batch
var firstErr error
for len(pkts) > 0 {
n := 0
for len(pkts) > 0 && n < tunWriteBatch {
p := pkts[0]
pkts = pkts[1:]
var af byte
err := error(syscall.EIO)
if len(p) > 0 {
af, err = tunAF(p)
}
if err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
b.heads[n] = [4]byte{3: af}
b.iovs[n] = [2]unix.Iovec{
{Base: &b.heads[n][0], Len: 4},
{Base: &p[0], Len: uint64(len(p))},
}
b.hdrs[n] = msgx.Hdr{Iov: &b.iovs[n][0], Iovlen: 2}
b.pkts[n] = p
n++
}
if n == 0 {
continue
}
for off := 0; off < n; {
sent, err := u.sendmsgX(off, n)
switch {
case err == nil && sent > 0:
// The kernel stops early without an error on a packet larger than the socket's send
// buffer, or when sending a later packet fails with ENOBUFS; resend the remainder.
off += sent
case err == nil:
if werr := u.writeEach(b.pkts[off:n]); werr != nil && firstErr == nil {
firstErr = werr
}
off = n
case err == unix.EMSGSIZE:
// Only the first packet being larger than the socket's send buffer fails the whole call,
// before the kernel takes any; Write reports that one and the rest go out in the next call.
if werr := u.writeEach(b.pkts[off : off+1]); werr != nil && firstErr == nil {
firstErr = werr
}
off++
case err == unix.ENOSYS || err == unix.EPERM || err == unix.EOPNOTSUPP:
// sendmsg_x is private API: fall back to writev if a kernel or sandbox refuses it.
if u.noSendmsgX.CompareAndSwap(false, true) {
u.l.Warn("sendmsg_x unavailable on the tun device, writing one packet per syscall", "error", err)
}
if werr := u.writeEach(b.pkts[off:n]); werr != nil && firstErr == nil {
firstErr = werr
}
clear(b.iovs[:n])
clear(b.pkts[:n])
if werr := u.writeEach(pkts); werr != nil && firstErr == nil {
firstErr = werr
}
return firstErr
default:
if firstErr == nil {
firstErr = err
}
off = n
}
}
clear(b.iovs[:n])
clear(b.pkts[:n])
}
return firstErr
}
// sendmsgX hands entries off through n-1 of u.batch to sendmsg_x and returns how many the kernel took.
func (u *Utun) sendmsgX(off, n int) (int, error) {
rc, err := u.f.SyscallConn()
if err != nil {
return 0, err
}
var sent int
var errno syscall.Errno
err = rc.Write(func(fd uintptr) bool {
sent, errno = msgx.Send(fd, u.batch.hdrs[off:n], 0)
// sendmsg_x reports EAGAIN only when it took nothing; a partial batch returns its count.
return !errno.Temporary()
})
if err != nil {
return 0, err
}
if errno != 0 {
return 0, errno
}
return sent, nil
}
// writeEach writes pkts one at a time with Write.
func (u *Utun) writeEach(pkts [][]byte) error {
var firstErr error
for _, p := range pkts {
if _, err := u.Write(p); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
func (u *Utun) Close() error {
return u.f.Close()
}
-198
View File
@@ -1,198 +0,0 @@
//go:build darwin && !ios
package tio
import (
"bytes"
"log/slog"
"syscall"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
// newSocketpairUtun returns a Utun over one end of a connected AF_UNIX datagram socketpair, which
// takes sendmsg_x the way a utun control socket does, and the other end's fd for the test to use.
// sndbuf is the Utun's send buffer, which caps the largest datagram it accepts.
func newSocketpairUtun(t *testing.T, sndbuf int) (*Utun, int) {
t.Helper()
fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
require.NoError(t, err)
require.NoError(t, unix.SetsockoptInt(fds[0], unix.SOL_SOCKET, unix.SO_SNDBUF, sndbuf))
// An AF_UNIX datagram send is bounded by the receiver's buffer, so size both ends' for the tests.
require.NoError(t, unix.SetsockoptInt(fds[0], unix.SOL_SOCKET, unix.SO_RCVBUF, 1<<20))
require.NoError(t, unix.SetsockoptInt(fds[1], unix.SOL_SOCKET, unix.SO_RCVBUF, 1<<20))
// A dropped datagram fails the read-back instead of hanging it.
require.NoError(t, unix.SetsockoptTimeval(fds[1], unix.SOL_SOCKET, unix.SO_RCVTIMEO, &unix.Timeval{Sec: 2}))
u, err := NewUtun(fds[0], slog.New(slog.DiscardHandler))
if err != nil {
_ = unix.Close(fds[0])
_ = unix.Close(fds[1])
t.Fatal(err)
}
t.Cleanup(func() {
_ = u.Close()
_ = unix.Close(fds[1])
})
return u, fds[1]
}
// testTunPkt returns packet i of size bytes, alternating IPv4 and IPv6, and the datagram the utun
// device should receive for it.
func testTunPkt(i, size int) (pkt, wire []byte) {
pkt = make([]byte, size)
pkt[0] = 0x45
af := byte(syscall.AF_INET)
if i%2 == 1 {
pkt[0] = 0x60
af = syscall.AF_INET6
}
pkt[1] = byte(i)
pkt[2] = byte(i >> 8)
return pkt, append([]byte{0, 0, 0, af}, pkt...)
}
// readTunBack asserts r receives exactly want, in order.
func readTunBack(t *testing.T, r int, want [][]byte) {
t.Helper()
buf := make([]byte, 4096)
for i, w := range want {
n, _, err := unix.Recvfrom(r, buf, 0)
require.NoError(t, err, "datagram %d", i)
if !bytes.Equal(w, buf[:n]) {
t.Fatalf("datagram %d: got % x, want % x", i, buf[:n], w)
}
}
_, _, err := unix.Recvfrom(r, buf, unix.MSG_DONTWAIT)
assert.ErrorIs(t, err, unix.EAGAIN, "extra datagrams delivered")
}
// forEachTunWritePath runs fn once over sendmsg_x and once over the per-packet writev fallback.
func forEachTunWritePath(t *testing.T, fn func(t *testing.T, fallback bool)) {
for _, fallback := range []bool{false, true} {
name := "sendmsg_x"
if fallback {
name = "writev"
}
t.Run(name, func(t *testing.T) { fn(t, fallback) })
}
}
// TestUtunWriteBatch pins that WriteBatch delivers every packet whole, in order and behind the right
// utun AF prefix, across several sendmsg_x calls, and returns nil.
func TestUtunWriteBatch(t *testing.T) {
forEachTunWritePath(t, func(t *testing.T, fallback bool) {
u, r := newSocketpairUtun(t, 1<<20)
u.noSendmsgX.Store(fallback)
var pkts, want [][]byte
for i := range 3*tunWriteBatch + 5 {
p, w := testTunPkt(i, 60+i)
pkts = append(pkts, p)
want = append(want, w)
}
require.NoError(t, u.WriteBatch(pkts))
assert.Equal(t, fallback, u.noSendmsgX.Load())
readTunBack(t, r, want)
})
}
// TestUtunWriteBatchSkipsInvalid pins that an empty packet or one with no IP version is skipped
// without shifting the packets after it, and that the first such error is the one reported.
func TestUtunWriteBatchSkipsInvalid(t *testing.T) {
forEachTunWritePath(t, func(t *testing.T, fallback bool) {
u, r := newSocketpairUtun(t, 1<<20)
u.noSendmsgX.Store(fallback)
var pkts, want [][]byte
for i := range 20 {
p, w := testTunPkt(i, 60)
pkts = append(pkts, p)
want = append(want, w)
}
pkts = append(pkts[:10], append([][]byte{{0x10, 1, 2}, {}}, pkts[10:]...)...)
err := u.WriteBatch(pkts)
assert.ErrorContains(t, err, "IP version")
readTunBack(t, r, want)
})
}
// TestUtunWriteBatchOversize pins that a packet larger than the socket's send buffer drops only
// itself and is reported, whether sendmsg_x reaches it mid-call (a short count) or first (EMSGSIZE
// for the whole call). The send buffer is 1024 bytes and the oversize packet 2000.
func TestUtunWriteBatchOversize(t *testing.T) {
for _, tc := range []struct {
name string
at int
}{
{"mid-call", tunWriteBatch + 6},
{"first-in-call", tunWriteBatch},
} {
t.Run(tc.name, func(t *testing.T) {
forEachTunWritePath(t, func(t *testing.T, fallback bool) {
u, r := newSocketpairUtun(t, 1024)
u.noSendmsgX.Store(fallback)
var pkts, want [][]byte
for i := range 2*tunWriteBatch + 10 {
size := 100
if i == tc.at {
size = 2000
}
p, w := testTunPkt(i, size)
pkts = append(pkts, p)
if i != tc.at {
want = append(want, w)
}
}
err := u.WriteBatch(pkts)
assert.ErrorIs(t, err, unix.EMSGSIZE)
assert.Equal(t, fallback, u.noSendmsgX.Load())
readTunBack(t, r, want)
})
})
}
}
// TestUtunReadDrains pins that Read hands back every queued packet, AF prefix stripped and in
// order, capped at tunReadBatch per call, and doesn't wait for more once the queue is empty.
func TestUtunReadDrains(t *testing.T) {
u, w := newSocketpairUtun(t, 1<<20)
total := tunReadBatch + 3
var want [][]byte
for i := range total {
pkt, wire := testTunPkt(i, 60+i)
_, err := unix.Write(w, wire)
require.NoError(t, err)
want = append(want, pkt)
}
var got [][]byte
for _, size := range []int{tunReadBatch, 3} {
pkts, err := u.Read()
require.NoError(t, err)
require.Len(t, pkts, size)
for _, p := range pkts {
got = append(got, p.Clone().Bytes)
}
}
assert.Equal(t, want, got)
}
// TestUtunReadClipsPackets pins that each returned packet's capacity ends at its own bytes, so
// appending to one can't overwrite the next.
func TestUtunReadClipsPackets(t *testing.T) {
u, w := newSocketpairUtun(t, 1<<20)
for i := range 2 {
_, wire := testTunPkt(i, 100)
_, err := unix.Write(w, wire)
require.NoError(t, err)
}
pkts, err := u.Read()
require.NoError(t, err)
require.Len(t, pkts, 2)
assert.Equal(t, len(pkts[0].Bytes), cap(pkts[0].Bytes))
}
+115 -12
View File
@@ -7,7 +7,9 @@ import (
"fmt"
"log/slog"
"net/netip"
"os"
"sync/atomic"
"syscall"
"unsafe"
"github.com/gaissmai/bart"
@@ -20,7 +22,7 @@ import (
)
type tun struct {
queue *tio.Utun
f *os.File
Device string
vpnNetworks []netip.Prefix
DefaultMTU int
@@ -115,13 +117,13 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
return nil, fmt.Errorf("failed to retrieve tun name: %w", err)
}
queue, err := tio.NewUtun(fd, l)
err = unix.SetNonblock(fd, true)
if err != nil {
return nil, err
return nil, fmt.Errorf("SetNonblock: %v", err)
}
t := &tun{
queue: queue,
f: os.NewFile(uintptr(fd), ""),
Device: name,
vpnNetworks: vpnNetworks,
DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
@@ -154,15 +156,15 @@ func (t *tun) deviceBytes() (o [16]byte) {
// network extension is handed its device. Everything about moving packets is shared with newTun,
// only the setup differs: the host owns addressing and routing here.
func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
queue, err := tio.NewUtun(deviceFd, l)
if err != nil {
if err := unix.SetNonblock(deviceFd, true); err != nil {
// We own the fd from the moment it is handed to us
_ = unix.Close(deviceFd)
return nil, err
return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err)
}
file := os.NewFile(uintptr(deviceFd), "/dev/tun")
t := &tun{
queue: queue,
f: file,
Device: utunNameFromFd(deviceFd),
vpnNetworks: vpnNetworks,
DefaultMTU: c.GetInt("tun.mtu", DefaultMTU),
@@ -171,7 +173,7 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
}
if err := t.reload(c, true); err != nil {
_ = queue.Close()
_ = file.Close()
return nil, err
}
@@ -195,8 +197,8 @@ func utunNameFromFd(fd int) string {
}
func (t *tun) Close() error {
if t.queue != nil {
return t.queue.Close()
if t.f != nil {
return t.f.Close()
}
return nil
}
@@ -554,6 +556,107 @@ func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
return nil
}
// tunWritev and tunReadv are linkname'd to x/sys/unix's libc-routed writev/readv stubs so the
// calls go through libSystem's pinned trampoline. A raw syscall.Syscall(SYS_WRITEV/SYS_READV, ...)
// on darwin/arm64 emits an SVC #0x80 trap (see $GOROOT/src/syscall/asm_darwin_arm64.s), the path
// Apple keeps warning they will eventually disallow. We pull the low-level stubs instead of calling
// unix.Writev/unix.Readv because those take [][]byte and rebuild the []Iovec every call, which
// heap-allocates the header; linkname'ing the stubs lets us hand them our own stack-allocated
// iovecs. See golang/go#78049.
//go:linkname tunWritev golang.org/x/sys/unix.writev
//go:noescape
func tunWritev(fd int, iovecs []unix.Iovec) (n int, err error)
//go:linkname tunReadv golang.org/x/sys/unix.readv
//go:noescape
func tunReadv(fd int, iovecs []unix.Iovec) (n int, err error)
// Read pulls one IP packet off the utun device, scattering the 4 byte protocol header away from
// the packet so the payload lands directly in to.
func (t *tun) Read(to []byte) (int, error) {
var head [4]byte
rc, err := t.f.SyscallConn()
if err != nil {
return 0, err
}
var n int
var callErr error
err = rc.Read(func(fd uintptr) bool {
iovecs := []unix.Iovec{
{Base: &head[0], Len: 4},
{Base: &to[0], Len: uint64(len(to))},
}
n, callErr = tunReadv(int(fd), iovecs)
if errno, ok := callErr.(syscall.Errno); ok && errno.Temporary() {
return false
}
return true
})
if err != nil {
return 0, err
}
if callErr != nil {
return 0, callErr
}
if n < 4 {
return 0, nil
}
return n - 4, nil
}
// Write pushes one IP packet onto the utun device. Safe for concurrent use:
// the AF prefix and iovecs are per-call stack state, and the fd write itself
// serializes on the runtime's fd mutex (see the Queue contract in tio.go).
func (t *tun) Write(from []byte) (int, error) {
if len(from) == 0 {
return 0, syscall.EIO
}
ipVer := from[0] >> 4
var head [4]byte
switch ipVer {
case 4:
head[3] = syscall.AF_INET
case 6:
head[3] = syscall.AF_INET6
default:
return 0, fmt.Errorf("unable to determine IP version from packet")
}
// Grab rc as a local so the compiler can devirtualize the call and keep the closure on the stack.
rc, err := t.f.SyscallConn()
if err != nil {
return 0, err
}
var n int
var callErr error
err = rc.Write(func(fd uintptr) bool {
iovecs := []unix.Iovec{
{Base: &head[0], Len: 4},
{Base: &from[0], Len: uint64(len(from))},
}
n, callErr = tunWritev(int(fd), iovecs)
// Type-assert to syscall.Errno so the EAGAIN/EWOULDBLOCK/EINTR check doesn't box the errno
// constants into error interfaces on every call.
if errno, ok := callErr.(syscall.Errno); ok && errno.Temporary() {
return false
}
return true
})
if err != nil {
return 0, err
}
if callErr != nil {
return 0, callErr
}
return n - 4, nil
}
func (t *tun) Networks() []netip.Prefix {
return t.vpnNetworks
}
@@ -563,5 +666,5 @@ func (t *tun) Name() string {
}
func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{t.queue}, nil
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
}
+4 -261
View File
@@ -10,12 +10,10 @@ import (
"log/slog"
"net"
"net/netip"
"sync/atomic"
"syscall"
"unsafe"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/internal/msgx"
"golang.org/x/sys/unix"
)
@@ -24,9 +22,6 @@ type StdConn struct {
isV4 bool
sysFd uintptr
l *slog.Logger
noSendmsgX atomic.Bool
sb *sendScratch
}
var _ Conn = &StdConn{}
@@ -40,7 +35,6 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
if uc, ok := pc.(*net.UDPConn); ok {
c := &StdConn{UDPConn: uc, l: l}
c.noSendmsgX.Store(!msgx.Available())
rc, err := uc.SyscallConn()
if err != nil {
@@ -145,138 +139,11 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
}
}
// sendBatch is how many datagrams one sendmsg_x call may carry.
const sendBatch = 64
// sendScratch is WriteBatch's scratch. The kernel reads every entry in place.
type sendScratch struct {
names [sendBatch]unix.RawSockaddrInet6
iovs [sendBatch]unix.Iovec
hdrs [sendBatch]msgx.Hdr
// idx maps each entry back to its index in WriteBatch's bufs.
idx [sendBatch]int
}
// WriteBatch sends bufs with sendmsg_x, darwin's private batched sendmsg, so one syscall carries up to
// sendBatch datagrams. On this unconnected socket xnu still runs each message through sendit, so the
// saving is the syscall crossings, not the per-send policy checks.
//
// xnu's per-message loop stops at the first message that fails. For the errors in sentNothing it reports
// how many went out; for any other it returns only the errno, with the messages before it already sent.
// Each call therefore carries one run of datagrams to the same destination: a destination the kernel
// refuses fails on its first datagram, so a failed call costs that run and nothing else, and no datagram
// is ever resent after the kernel may have taken it.
//
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
// silently drops the rest of a burst (linux blocks instead). Poll for
// writability on EAGAIN before giving up on the remainder.
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
if u.noSendmsgX.Load() {
return u.writeEach(bufs, addrs), nil
}
if u.sb == nil {
u.sb = new(sendScratch)
}
b := u.sb
written := 0
for i := 0; i < len(bufs); {
n := 0
for ; i < len(bufs) && n < sendBatch; i++ {
namelen, err := u.putSockaddr(&b.names[n], addrs[i])
if err != nil {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
continue
}
b.iovs[n] = unix.Iovec{}
if len(bufs[i]) > 0 {
b.iovs[n].Base = &bufs[i][0]
b.iovs[n].SetLen(len(bufs[i]))
}
b.hdrs[n] = msgx.Hdr{Name: (*byte)(unsafe.Pointer(&b.names[n])), Namelen: namelen, Iov: &b.iovs[n], Iovlen: 1}
b.idx[n] = i
n++
}
for off := 0; off < n; {
dst := addrs[b.idx[off]]
end := off + 1
for end < n && addrs[b.idx[end]] == dst {
end++
}
sent, errno := u.sendmsgX(off, end)
switch {
case errno == 0 && sent > 0:
written += sent
off += sent
continue
case sentNothing(errno):
// The datagram at off failed on its own account. sendto delivers or reports it, as without
// batching, and the rest of the run goes back to sendmsg_x.
j := b.idx[off]
written += u.writeEach(bufs[j:j+1], addrs[j:j+1])
off++
continue
case errno == 0:
case errno == unix.ENOSYS || errno == unix.EOPNOTSUPP:
u.fallBack(errno)
case errno == unix.EPERM || errno == unix.EINVAL:
// Either can refuse one destination, but a datagram that sendto delivers was refused by
// sendmsg_x itself, as a sandbox or an older kernel would.
j := b.idx[off]
if u.writeEach(bufs[j:j+1], addrs[j:j+1]) == 0 {
off = end
continue
}
written++
off++
u.fallBack(errno)
default:
u.l.Debug("failed to write packets in batch", "udpAddr", dst, "packets", end-off, "error", errno)
off = end
continue
}
// The kernel took nothing from this call, so the rest can go out one datagram at a time.
for _, j := range b.idx[off:n] {
written += u.writeEach(bufs[j:j+1], addrs[j:j+1])
}
off = n
if u.noSendmsgX.Load() {
clear(b.iovs[:n])
return written + u.writeEach(bufs[i:], addrs[i:]), nil
}
}
clear(b.iovs[:n])
}
return written, nil
}
// sendmsgX hands entries off through end-1 of u.sb to sendmsg_x and returns how many the kernel took.
// The socket is non-blocking, so a full send buffer returns EAGAIN rather than waiting.
func (u *StdConn) sendmsgX(off, end int) (int, syscall.Errno) {
for {
sent, errno := msgx.Send(u.sysFd, u.sb.hdrs[off:end], 0)
if errno == unix.EINTR {
continue
}
return sent, errno
}
}
// sentNothing reports whether sendmsg_x failing with errno means no datagram went out: once any has, xnu
// turns these errors into a short count instead.
func sentNothing(errno syscall.Errno) bool {
return errno == unix.EWOULDBLOCK || errno == unix.ENOBUFS || errno == unix.EMSGSIZE
}
// fallBack switches WriteBatch to one sendto per datagram for the life of the socket.
func (u *StdConn) fallBack(errno syscall.Errno) {
if u.noSendmsgX.CompareAndSwap(false, true) {
u.l.Warn("sendmsg_x unavailable, sending one datagram per syscall", "error", errno)
}
}
// writeEach sends bufs one at a time with WriteTo and returns how many were written.
func (u *StdConn) writeEach(bufs [][]byte, addrs []netip.AddrPort) int {
written := 0
for i, b := range bufs {
if err := u.WriteTo(b, addrs[i]); err == nil {
@@ -285,23 +152,7 @@ func (u *StdConn) writeEach(bufs [][]byte, addrs []netip.AddrPort) int {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
}
}
return written
}
// putSockaddr writes ap into sa in the socket's address family, as WriteTo would, and returns its length.
func (u *StdConn) putSockaddr(sa *unix.RawSockaddrInet6, ap netip.AddrPort) (uint32, error) {
if u.isV4 {
if ap.Addr().Is6() {
return 0, ErrInvalidIPv6RemoteForSocket
}
sa4 := (*unix.RawSockaddrInet4)(unsafe.Pointer(sa))
*sa4 = unix.RawSockaddrInet4{Family: unix.AF_INET, Addr: ap.Addr().As4()}
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa4.Port))[:], ap.Port())
return unix.SizeofSockaddrInet4, nil
}
*sa = unix.RawSockaddrInet6{Family: unix.AF_INET6, Addr: ap.Addr().As16()}
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], ap.Port())
return unix.SizeofSockaddrInet6, nil
return written, nil
}
func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
@@ -329,106 +180,11 @@ func NewUDPStatsEmitter(udpConns []Conn) func() {
return func() {}
}
// recvBatch is how many datagrams one recvmsg_x call may return.
const recvBatch = 64
// ListenOut drains the socket with recvmsg_x, darwin's private batched recvmsg, so one syscall reads
// up to recvBatch datagrams instead of one.
func (u *StdConn) ListenOut(r EncReader, flush func()) error {
if !msgx.Available() {
return u.listenOutSingle(r, flush)
}
rc, err := u.UDPConn.SyscallConn()
if err != nil {
return err
}
bufs := make([][]byte, recvBatch)
names := make([]unix.RawSockaddrInet6, recvBatch)
iovs := make([]unix.Iovec, recvBatch)
hdrs := make([]msgx.Hdr, recvBatch)
for i := range hdrs {
bufs[i] = make([]byte, MTU)
iovs[i].Base = &bufs[i][0]
iovs[i].SetLen(MTU)
hdrs[i].Iov = &iovs[i]
hdrs[i].Iovlen = 1
hdrs[i].Name = (*byte)(unsafe.Pointer(&names[i]))
}
for {
var n int
var errno syscall.Errno
err := rc.Read(func(fd uintptr) bool {
for i := range hdrs {
hdrs[i].Namelen = unix.SizeofSockaddrInet6
hdrs[i].Flags = 0
hdrs[i].Datalen = 0
}
n, errno = msgx.Recv(fd, hdrs, 0)
return errno != unix.EAGAIN
})
if err != nil {
if errors.Is(err, net.ErrClosed) {
return err
}
u.l.Error("unexpected udp socket receive error", "error", err)
continue
}
if errno == unix.ENOSYS || errno == unix.EPERM || errno == unix.EOPNOTSUPP {
// recvmsg_x is private, so a kernel or sandbox may refuse it.
u.l.Warn("recvmsg_x unavailable, reading one datagram per syscall", "error", errno)
return u.listenOutSingle(r, flush)
}
if errno != 0 {
if errno != unix.EINTR {
u.l.Error("unexpected udp socket receive error", "error", errno)
}
continue
}
if err := checkMsghdrX(hdrs, names, n); err != nil {
// The batch was read through a layout the kernel no longer writes, so none of it is trustworthy.
u.l.Warn("recvmsg_x returned an unexpected header, reading one datagram per syscall", "error", err)
return u.listenOutSingle(r, flush)
}
for i := 0; i < n; i++ {
addr, ok := sockaddrToAddrPort(&names[i])
if !ok {
continue
}
l := int(hdrs[i].Datalen)
r(addr, bufs[i][:l:l])
}
flush()
}
}
// checkMsghdrX rejects a recvmsg_x result that intact msghdr_x entries can't produce, which is how a change
// to xnu's private struct layout would show up.
func checkMsghdrX(hdrs []msgx.Hdr, names []unix.RawSockaddrInet6, n int) error {
if n > len(hdrs) {
return fmt.Errorf("returned %d datagrams for %d headers", n, len(hdrs))
}
for i := range n {
if hdrs[i].Datalen > MTU {
return fmt.Errorf("datagram %d is %d bytes, past the %d byte buffer", i, hdrs[i].Datalen, MTU)
}
switch {
case names[i].Family == unix.AF_INET && hdrs[i].Namelen == unix.SizeofSockaddrInet4:
case names[i].Family == unix.AF_INET6 && hdrs[i].Namelen == unix.SizeofSockaddrInet6:
default:
return fmt.Errorf("datagram %d has address family %d with length %d", i, names[i].Family, hdrs[i].Namelen)
}
}
return nil
}
// listenOutSingle reads one datagram per recvfrom and flushes after each, for when recvmsg_x is refused or misbehaves.
func (u *StdConn) listenOutSingle(r EncReader, flush func()) error {
buffer := make([]byte, MTU)
for {
// Just read one packet at a time
n, rua, err := u.ReadFromUDPAddrPort(buffer)
if err != nil {
if errors.Is(err, net.ErrClosed) {
@@ -443,19 +199,6 @@ func (u *StdConn) listenOutSingle(r EncReader, flush func()) error {
}
}
func sockaddrToAddrPort(sa *unix.RawSockaddrInet6) (netip.AddrPort, bool) {
switch sa.Family {
case unix.AF_INET:
sa4 := (*unix.RawSockaddrInet4)(unsafe.Pointer(sa))
port := binary.BigEndian.Uint16((*[2]byte)(unsafe.Pointer(&sa4.Port))[:])
return netip.AddrPortFrom(netip.AddrFrom4(sa4.Addr), port), true
case unix.AF_INET6:
port := binary.BigEndian.Uint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:])
return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr).Unmap(), port), true
}
return netip.AddrPort{}, false
}
func (u *StdConn) SupportsMultipleReaders() bool {
return false
}
-85
View File
@@ -1,85 +0,0 @@
//go:build !e2e_testing
package udp
import (
"log/slog"
"net"
"net/netip"
"sync"
"sync/atomic"
"testing"
)
// BenchmarkListenOut measures the receive path on loopback: senders flood datagrams sized like a full tunnel
// packet, and each op is one datagram delivered to the reader. There are enough senders that the reader, not the
// senders, sets the pace. flushes/pkt is 1 when every receive is its own syscall and falls as receives batch. The
// senders share the machine, so ns/op is only comparable between runs on the same host.
func BenchmarkListenOut(b *testing.B) {
for _, tc := range []struct{ name, listen string }{
{"v4", "127.0.0.1"},
{"v6", "::1"},
} {
b.Run(tc.name, func(b *testing.B) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.AddrPortFrom(netip.MustParseAddr(tc.listen), 0)})
if err != nil {
b.Fatal(err)
}
la, err := c.LocalAddr()
if err != nil {
b.Fatal(err)
}
const size, senders = 1300, 4
txs := make([]net.Conn, senders)
for i := range txs {
if txs[i], err = net.Dial("udp", la.String()); err != nil {
b.Fatal(err)
}
defer txs[i].Close()
}
target := int64(b.N)
var got, flushes atomic.Int64
reached := make(chan struct{})
done := make(chan struct{})
go func() {
_ = c.ListenOut(func(netip.AddrPort, []byte) {
if got.Add(1) == target {
close(reached)
}
}, func() { flushes.Add(1) })
close(done)
}()
// Writes fail with ENOBUFS while the send queue is full; the flood only has to keep the receiver busy.
stop := make(chan struct{})
var sent sync.WaitGroup
for _, tx := range txs {
sent.Go(func() {
payload := make([]byte, size)
for {
select {
case <-stop:
return
default:
_, _ = tx.Write(payload)
}
}
})
}
b.SetBytes(size)
b.ResetTimer()
<-reached
b.StopTimer()
close(stop)
sent.Wait()
if err := c.Close(); err != nil {
b.Fatal(err)
}
<-done
b.ReportMetric(float64(flushes.Load())/float64(got.Load()), "flushes/pkt")
})
}
}
-302
View File
@@ -1,302 +0,0 @@
//go:build !e2e_testing
package udp
import (
"bytes"
"fmt"
"log/slog"
"net"
"net/netip"
"sync"
"syscall"
"testing"
"time"
"unsafe"
"github.com/slackhq/nebula/internal/msgx"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
// A burst larger than recvBatch arrives whole, with each datagram's own bytes and source address, and
// flush runs once per recvmsg_x batch rather than once per datagram.
func TestListenOutBatchesBurst(t *testing.T) {
for _, tc := range []struct{ name, listen string }{
{"v4", "127.0.0.1"},
{"v6", "::1"},
} {
t.Run(tc.name, func(t *testing.T) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.AddrPortFrom(netip.MustParseAddr(tc.listen), 0)})
require.NoError(t, err)
la, err := c.LocalAddr()
require.NoError(t, err)
tx, err := net.Dial("udp", la.String())
require.NoError(t, err)
defer tx.Close()
txAddr := netip.MustParseAddrPort(tx.LocalAddr().String())
const total = recvBatch*3 + 7
var mu sync.Mutex
var got []string
var froms []netip.AddrPort
flushes := 0
// Queued before the reader starts, so recvmsg_x sees full batches.
for i := 0; i < total; i++ {
_, err := tx.Write([]byte(fmt.Sprintf("pkt-%03d", i)))
require.NoError(t, err)
}
done := make(chan struct{})
go func() {
_ = c.ListenOut(func(addr netip.AddrPort, payload []byte) {
mu.Lock()
got = append(got, string(payload))
froms = append(froms, addr)
mu.Unlock()
}, func() {
mu.Lock()
flushes++
mu.Unlock()
})
close(done)
}()
require.Eventually(t, func() bool {
mu.Lock()
defer mu.Unlock()
return len(got) == total
}, 5*time.Second, 10*time.Millisecond)
require.NoError(t, c.Close())
<-done
for i, p := range got {
assert.Equal(t, fmt.Sprintf("pkt-%03d", i), p)
assert.Equal(t, txAddr, froms[i])
}
assert.Equal(t, (total+recvBatch-1)/recvBatch, flushes)
})
}
}
// msgx.Hdr round-trips through xnu's recvmsg_x on a loopback socket, v4 and v6. The kernel reads Name, Namelen,
// Iov and Iovlen from each entry and writes Namelen, Flags and Datalen back, so a field at the wrong offset or a
// wrong stride between entries shows up as a wrong length, sender, or payload.
// TestListenOutSingleFallback pins the path ListenOut takes when the kernel refuses recvmsg_x.
func TestListenOutSingleFallback(t *testing.T) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0")})
require.NoError(t, err)
la, err := c.LocalAddr()
require.NoError(t, err)
tx, err := net.Dial("udp", la.String())
require.NoError(t, err)
defer tx.Close()
txAddr := netip.MustParseAddrPort(tx.LocalAddr().String())
const total = 5
for i := 0; i < total; i++ {
_, err := tx.Write([]byte(fmt.Sprintf("pkt-%03d", i)))
require.NoError(t, err)
}
var mu sync.Mutex
var got []string
flushes := 0
done := make(chan struct{})
go func() {
_ = c.(*StdConn).listenOutSingle(func(addr netip.AddrPort, payload []byte) {
assert.Equal(t, txAddr, addr)
mu.Lock()
got = append(got, string(payload))
mu.Unlock()
}, func() {
mu.Lock()
flushes++
mu.Unlock()
})
close(done)
}()
require.Eventually(t, func() bool {
mu.Lock()
defer mu.Unlock()
return len(got) == total
}, 5*time.Second, 10*time.Millisecond)
require.NoError(t, c.Close())
<-done
for i, p := range got {
assert.Equal(t, fmt.Sprintf("pkt-%03d", i), p)
}
assert.Equal(t, total, flushes)
}
func TestCheckMsghdrX(t *testing.T) {
valid := func() ([]msgx.Hdr, []unix.RawSockaddrInet6) {
hdrs := make([]msgx.Hdr, 2)
names := make([]unix.RawSockaddrInet6, 2)
hdrs[0].Namelen, hdrs[0].Datalen = unix.SizeofSockaddrInet4, 1
names[0].Family = unix.AF_INET
hdrs[1].Namelen, hdrs[1].Datalen = unix.SizeofSockaddrInet6, MTU
names[1].Family = unix.AF_INET6
return hdrs, names
}
hdrs, names := valid()
require.NoError(t, checkMsghdrX(hdrs, names, 2))
require.NoError(t, checkMsghdrX(hdrs, names, 0))
for _, tc := range []struct {
name string
n int
mutate func([]msgx.Hdr, []unix.RawSockaddrInet6)
}{
{"count past headers", 3, func([]msgx.Hdr, []unix.RawSockaddrInet6) {}},
{"datalen past buffer", 2, func(h []msgx.Hdr, _ []unix.RawSockaddrInet6) { h[1].Datalen = MTU + 1 }},
{"unknown family", 2, func(_ []msgx.Hdr, a []unix.RawSockaddrInet6) { a[0].Family = unix.AF_UNIX }},
{"family and length disagree", 2, func(h []msgx.Hdr, _ []unix.RawSockaddrInet6) { h[0].Namelen = unix.SizeofSockaddrInet6 }},
} {
t.Run(tc.name, func(t *testing.T) {
hdrs, names := valid()
tc.mutate(hdrs, names)
assert.Error(t, checkMsghdrX(hdrs, names, tc.n))
})
}
}
func TestMsghdrXKernelRoundTrip(t *testing.T) {
for _, v6 := range []bool{false, true} {
t.Run(fmt.Sprintf("v6=%v", v6), func(t *testing.T) {
// Loopback delivery goes through the dlil input thread, so the settle gives all three time to
// queue and come back from one call, which is what exercises the stride.
got := recvmsgXRoundTrip(t, v6, []int{1, 300, 1400}, 4, 50*time.Millisecond)
assert.Equal(t, 3, got, "largest batch from one recvmsg_x call")
})
}
}
// Any burst, of any datagram lengths up to the kernel's maxdgram, into any number of headers, comes back in order
// with every entry's fields intact, and headers past the returned count untouched.
func FuzzMsghdrXKernelRoundTrip(f *testing.F) {
f.Add(false, uint8(3), []byte{0, 1, 1, 44, 5, 120})
f.Add(true, uint8(0), []byte{0, 0, 0x23, 0x29}) // one empty datagram, then one at MTU
f.Add(false, uint8(1), []byte{0x23, 0x2a, 0x24, 0x00, 0}) // past MTU: cut to MTU, no MSG_TRUNC
f.Add(true, uint8(recvBatch-1), bytes.Repeat([]byte{0x05, 0xdc}, recvBatch))
f.Fuzz(func(t *testing.T, v6 bool, spare uint8, sizes []byte) {
// Two bytes per datagram, big-endian, folded into [0, maxdgram].
var lens []int
for i := 0; i+1 < len(sizes) && len(lens) < recvBatch; i += 2 {
lens = append(lens, (int(sizes[i])<<8|int(sizes[i+1]))%(maxDgram+1))
}
if len(lens) == 0 {
return
}
recvmsgXRoundTrip(t, v6, lens, 1+int(spare)%recvBatch, time.Millisecond)
})
}
// maxDgram is darwin's default net.inet.udp.maxdgram, the largest datagram a loopback sender may write.
const maxDgram = 9216
// recvmsgXRoundTrip sends one datagram per entry of lens to a fresh loopback socket, drains them with recvmsg_x
// into nh headers, and checks every field the kernel writes. Neighboring datagrams differ in sender, length and
// fill byte, so no field can pass by reading a neighbor's value. It returns the largest batch one call returned.
func recvmsgXRoundTrip(t *testing.T, v6 bool, lens []int, nh int, settle time.Duration) int {
t.Helper()
network, lo := "udp4", netip.MustParseAddr("127.0.0.1")
if v6 {
network, lo = "udp6", netip.IPv6Loopback()
}
rx, err := net.ListenUDP(network, net.UDPAddrFromAddrPort(netip.AddrPortFrom(lo, 0)))
require.NoError(t, err)
defer rx.Close()
txs := make([]*net.UDPConn, 3)
for i := range txs {
txs[i], err = net.DialUDP(network, nil, rx.LocalAddr().(*net.UDPAddr))
require.NoError(t, err)
defer txs[i].Close()
}
senders := make([]netip.AddrPort, len(lens))
for i, l := range lens {
tx := txs[i%len(txs)]
senders[i] = tx.LocalAddr().(*net.UDPAddr).AddrPort()
_, err := tx.Write(bytes.Repeat([]byte{byte(i + 1)}, l))
require.NoError(t, err)
}
time.Sleep(settle)
// Namelen goes in as the v6 size on both families, so the v4 size coming back proves the kernel wrote it.
const sentinel = 0xa5a5a5a5a5a5a5a5
wantNamelen := uint32(unix.SizeofSockaddrInet4)
if v6 {
wantNamelen = unix.SizeofSockaddrInet6
}
bufs := make([][]byte, nh)
names := make([]unix.RawSockaddrInet6, nh)
iovs := make([]unix.Iovec, nh)
hdrs := make([]msgx.Hdr, nh)
for i := range hdrs {
bufs[i] = make([]byte, MTU)
iovs[i].Base = &bufs[i][0]
iovs[i].SetLen(MTU)
}
// A datagram dropped on loopback would otherwise block rc.Read forever.
require.NoError(t, rx.SetReadDeadline(time.Now().Add(5*time.Second)))
rc, err := rx.SyscallConn()
require.NoError(t, err)
largest := 0
for next := 0; next < len(lens); {
for i := range hdrs {
clear(bufs[i])
hdrs[i] = msgx.Hdr{
Name: (*byte)(unsafe.Pointer(&names[i])),
Namelen: unix.SizeofSockaddrInet6,
Iov: &iovs[i],
Iovlen: 1,
Flags: -1,
Datalen: sentinel,
}
}
var n int
var errno syscall.Errno
require.NoError(t, rc.Read(func(fd uintptr) bool {
n, errno = msgx.Recv(fd, hdrs[:nh], 0)
return errno != unix.EAGAIN
}), "received %d of %d datagrams", next, len(lens))
require.Zero(t, errno)
require.Positive(t, n)
require.LessOrEqual(t, n, min(nh, len(lens)-next))
largest = max(largest, n)
require.NoError(t, checkMsghdrX(hdrs[:nh], names[:nh], n), "ListenOut would fall back on this batch")
for i := range n {
d := next + i
h := &hdrs[i]
want := min(lens[d], MTU)
require.EqualValues(t, want, h.Datalen, "datagram %d Datalen", d)
require.Equal(t, wantNamelen, h.Namelen, "datagram %d Namelen", d)
from, ok := sockaddrToAddrPort(&names[i])
require.True(t, ok, "datagram %d family %d", d, names[i].Family)
require.Equal(t, senders[d], from, "datagram %d sender", d)
// Flags goes in as -1 and must come back 0. xnu's list receive (soreceive_m_list) never sets
// MSG_TRUNC, so a datagram past MTU arrives silently cut to MTU, as recvfrom through net.UDPConn does.
require.Zero(t, h.Flags, "datagram %d Flags", d)
require.Equal(t, bytes.Repeat([]byte{byte(d + 1)}, want), bufs[i][:want], "datagram %d payload", d)
if want < MTU {
require.Zero(t, bufs[i][want], "datagram %d wrote past Datalen", d)
}
}
for i := n; i < nh; i++ {
require.EqualValues(t, uint64(sentinel), hdrs[i].Datalen, "spare header %d Datalen", i)
require.EqualValues(t, unix.SizeofSockaddrInet6, hdrs[i].Namelen, "spare header %d Namelen", i)
}
next += n
}
return largest
}
-150
View File
@@ -1,150 +0,0 @@
//go:build !e2e_testing
package udp
import (
"fmt"
"log/slog"
"net"
"net/netip"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// recvAll reads n datagrams from pc, or fails the test after a second without one.
func recvAll(t *testing.T, pc net.PacketConn, n int) []string {
t.Helper()
var got []string
buf := make([]byte, 64)
for len(got) < n {
require.NoError(t, pc.SetReadDeadline(time.Now().Add(time.Second)))
l, _, err := pc.ReadFrom(buf)
require.NoError(t, err, "after %d of %d datagrams", len(got), n)
got = append(got, string(buf[:l]))
}
return got
}
// A batch longer than sendBatch reaches each peer whole and in order through sendmsg_x, without falling
// back to sendto: a run to one peer that spans two calls, then datagrams alternating between peers.
func TestWriteBatchSendsEachToItsPeer(t *testing.T) {
for _, tc := range []struct{ name, listen string }{
{"v4", "127.0.0.1"},
{"v6", "::1"},
} {
t.Run(tc.name, func(t *testing.T) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.AddrPortFrom(netip.MustParseAddr(tc.listen), 0)})
require.NoError(t, err)
defer c.Close()
var peers [2]net.PacketConn
var dsts [2]netip.AddrPort
for i := range peers {
peers[i], err = net.ListenPacket("udp", net.JoinHostPort(tc.listen, "0"))
require.NoError(t, err)
defer peers[i].Close()
dsts[i] = netip.MustParseAddrPort(peers[i].LocalAddr().String())
}
const total = sendBatch*2 + 5
bufs := make([][]byte, total)
addrs := make([]netip.AddrPort, total)
var want [2][]string
for i := range bufs {
bufs[i] = fmt.Appendf(nil, "pkt-%03d", i)
p := i % 2
if i < sendBatch+6 {
p = 0
}
addrs[i] = dsts[p]
want[p] = append(want[p], string(bufs[i]))
}
n, err := c.WriteBatch(bufs, addrs)
require.NoError(t, err)
assert.Equal(t, total, n)
assert.False(t, c.(*StdConn).noSendmsgX.Load(), "fell back to sendto")
for i := range peers {
assert.Equal(t, want[i], recvAll(t, peers[i], len(want[i])))
}
})
}
}
// Destinations that can't be sent to cost only their own packets: an IPv6 peer on an IPv4 socket is
// skipped before the call, and a broadcast address without SO_BROADCAST is refused by the kernel
// mid-batch. Neither stops the packets around it, delivers any twice, or turns sendmsg_x off.
func TestWriteBatchSkipsUnsendableDestinations(t *testing.T) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0")})
require.NoError(t, err)
defer c.Close()
peer, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
defer peer.Close()
good := netip.MustParseAddrPort(peer.LocalAddr().String())
bcast := netip.AddrPortFrom(netip.MustParseAddr("255.255.255.255"), good.Port())
v6 := netip.MustParseAddrPort("[::1]:4242")
addrs := []netip.AddrPort{good, bcast, bcast, good, v6, good, bcast}
bufs := make([][]byte, len(addrs))
for i := range bufs {
bufs[i] = fmt.Appendf(nil, "pkt-%d", i)
}
n, err := c.WriteBatch(bufs, addrs)
require.NoError(t, err)
assert.Equal(t, 3, n)
assert.False(t, c.(*StdConn).noSendmsgX.Load(), "fell back to sendto")
assert.Equal(t, []string{"pkt-0", "pkt-3", "pkt-5"}, recvAll(t, peer, 3))
require.NoError(t, peer.SetReadDeadline(time.Now().Add(100*time.Millisecond)))
_, _, err = peer.ReadFrom(make([]byte, 64))
assert.Error(t, err, "a datagram arrived twice")
}
// A datagram the kernel refuses as its own fault (EMSGSIZE here, as ENOBUFS would be under a full
// interface queue) costs only itself, whether it fails after others in the same call (a short count) or
// first (the errno alone): the rest of its run still arrives, in order, and sendmsg_x stays on.
func TestWriteBatchLosesOnlyTheFailedDatagram(t *testing.T) {
for _, tc := range []struct {
name string
big int
}{
{"mid-call", 2},
{"first", 0},
} {
t.Run(tc.name, func(t *testing.T) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0")})
require.NoError(t, err)
defer c.Close()
peer, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
defer peer.Close()
dst := netip.MustParseAddrPort(peer.LocalAddr().String())
bufs := make([][]byte, 5)
addrs := make([]netip.AddrPort, len(bufs))
var want []string
for i := range bufs {
addrs[i] = dst
if i == tc.big {
// Past the 65535-byte IP datagram limit, so the kernel can only refuse it.
bufs[i] = make([]byte, 70000)
continue
}
bufs[i] = fmt.Appendf(nil, "pkt-%d", i)
want = append(want, string(bufs[i]))
}
n, err := c.WriteBatch(bufs, addrs)
require.NoError(t, err)
assert.Equal(t, len(want), n)
assert.False(t, c.(*StdConn).noSendmsgX.Load(), "fell back to sendto")
assert.Equal(t, want, recvAll(t, peer, len(want)))
})
}
}