Compare commits

..
Author SHA1 Message Date
Caleb JasikandClaude Opus 5.5 3bb5918e7c udp/darwin: send to busy established peers on connected sockets
On darwin an unconnected sendto runs the address and MAC policy checks
and the EndpointSecurity send check for every datagram, and while
xprotectd throttles it each send takes about 185us, which caps a
tunnel's upload and, through its ACKs, its download. A connected socket
runs those checks once, at connect, and it is the only UDP send that
reports the interface queue's flow control, as ENOBUFS.

Only the current direct remote of an established tunnel is eligible.
udp.EstablishedPeerConn is an optional Conn interface, implemented on
darwin; establishedRemotes refcounts remotes over the main hostmap and
tells each writer on the first and last reference: handshake completion
adds one, deleting or retiring a hostinfo removes it, SetRemote (roaming,
SetRemoteIfPreferred, control, ssh) moves it. Pending and relayed
hostinfos hold none, and a rehandshake's two hostinfos share one.

An eligible peer that Nebula sends NEBULA_CONNSOCK_RUN (64) datagrams to
within NEBULA_CONNSOCK_WINDOW (1s) gets a connected socket: SO_REUSEPORT,
bound to the source address the listener's route to the peer picks on
the listener's port, carrying the listener's IP_BOUND_IF. The peer sees
the same address and port as from the listener, and the socket's reader
delivers through the listener's reader. The listener itself never sets
SO_REUSEPORT (listen routines still do), so no other uid can bind the
port; with a specific listen.host and no routines connected sockets stay
off, since sharing an exact address needs SO_REUSEPORT on both.

All routines listening on one address share one set of connected
sockets: at most NEBULA_CONNSOCKS (8), one per peer, with opens from a
bucket that refills one per NEBULA_CONNSOCK_OPEN_EVERY (1s). A socket
closes when its peer stops being eligible, after NEBULA_CONNSOCK_IDLE
(30s) without traffic, on Rebind (of every routine) and Close, and on an
error that means the connected path is gone. Closing never runs under
the hostmap's or the eligibility lock; the peer can't reopen until the
close finishes. A send that gets ENOBUFS or EAGAIN waits within one
10ms budget shared by every writer since the last send that went
through, then drops; nothing waits in the netpoller without a deadline.
When nothing is eligible or open, a send costs one atomic load before
the listener path.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-28 16:28:07 -05:00
26 changed files with 3717 additions and 2272 deletions

No files matched your search

-17
View File
@@ -7,23 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Changed
- Firewall rules are held as a flat list with a bitset of rule ids per port, instead of a tree of maps copied
into every port of a range. A rule like `port: 1024-65535` now costs kilobytes rather than tens of megabytes,
and matching is faster on every path.
- A firewall rule's `proto` can be `udplite`, `dccp`, `sctp`, `icmpv6`, or any IP protocol number. A rule with
a port or port range on a protocol that has no ports, such as `proto: 47`, is rejected rather than silently
never matching. `port: any` and `port: fragment` apply to every protocol.
- An ICMP non-first fragment is matched against `port: fragment` rules like a fragment of any other protocol.
It was only matched against `port: any` rules before.
### Fixed
- A firewall rule with `group: []`, `groups:` with no value, or a `groups` entry that isn't a string, such as
`groups: [1, 2]`, no longer crashes Nebula on startup or reload. An empty `group` array is rejected, a null
`groups` means no groups, and non-string entries are read as strings.
## [1.11.1] - 2026-08-21
See the [v1.11.1](https://github.com/slackhq/nebula/milestone/30?closed=1) milestone for a complete list of changes.
+2 -1
View File
@@ -203,7 +203,8 @@ func (c *Control) RebindUDPServer() {
}
// A failure here means we are likely still pinned to the interface we came up on, so the rest of this is
// unlikely to help. Say so instead of silently carrying on as if we rebound.
// unlikely to help. Say so instead of silently carrying on as if we rebound. Only the first writer is rebound;
// darwin's Rebind clears every listen routine's listener.
if err := c.f.outside.Rebind(); err != nil {
c.l.Error("Failed to rebind udp socket", "error", err)
}
+90
View File
@@ -0,0 +1,90 @@
package nebula
import (
"net/netip"
"sync"
"github.com/slackhq/nebula/udp"
)
// establishedRemotes tells the udp conns that implement udp.EstablishedPeerConn which addresses are the current
// direct remote of at least one established tunnel. A hostinfo holds a reference on its remote while it is in the
// main hostmap, and SetRemote moves the reference; a relayed tunnel has no direct remote and holds none.
type establishedRemotes struct {
mu sync.Mutex
conns []udp.EstablishedPeerConn
refs map[netip.AddrPort]int
}
// newEstablishedRemotes returns nil when no conn wants to know, which every method accepts.
func newEstablishedRemotes(conns []udp.Conn) *establishedRemotes {
var pc []udp.EstablishedPeerConn
for _, c := range conns {
if p, ok := c.(udp.EstablishedPeerConn); ok {
pc = append(pc, p)
}
}
if len(pc) == 0 {
return nil
}
return &establishedRemotes{conns: pc, refs: map[netip.AddrPort]int{}}
}
// add makes hostinfo hold a reference on its remote, as it joins the main hostmap. A hostinfo holds references in
// one establishedRemotes at a time, so an add while it is in another is ignored.
func (e *establishedRemotes) add(hostinfo *HostInfo) {
if e == nil {
return
}
e.mu.Lock()
defer e.mu.Unlock()
if !hostinfo.establishedIn.CompareAndSwap(nil, e) && hostinfo.establishedIn.Load() != e {
return
}
e.syncLocked(hostinfo)
}
// remove drops hostinfo's reference, as it leaves the main hostmap.
func (e *establishedRemotes) remove(hostinfo *HostInfo) {
if e == nil {
return
}
e.mu.Lock()
defer e.mu.Unlock()
if hostinfo.establishedIn.CompareAndSwap(e, nil) {
e.syncLocked(hostinfo)
}
}
// syncLocked moves hostinfo's reference to the remote it should hold now: its current remote while it is
// established, none otherwise. Reading both under mu makes the last call win however add, remove and SetRemote
// interleave. Only the establishedRemotes that hostinfo is in, or has just left, may call it, since
// hostinfo.establishedRemote is a reference in that one's refs.
func (e *establishedRemotes) syncLocked(hostinfo *HostInfo) {
var want netip.AddrPort
if hostinfo.establishedIn.Load() == e {
want = hostinfo.GetRemote()
}
held := hostinfo.establishedRemote
if want == held {
return
}
if held.IsValid() {
e.refs[held]--
if e.refs[held] == 0 {
delete(e.refs, held)
for _, c := range e.conns {
c.SetEstablishedPeer(held, false)
}
}
}
if want.IsValid() {
e.refs[want]++
if e.refs[want] == 1 {
for _, c := range e.conns {
c.SetEstablishedPeer(want, true)
}
}
}
hostinfo.establishedRemote = want
}
+173
View File
@@ -0,0 +1,173 @@
package nebula
import (
"net/netip"
"sync"
"testing"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// establishedPeerConn records which destinations the hostmap has told it are established peers, and fails the test
// on a call that doesn't change anything, which would mean a reference count went wrong.
type establishedPeerConn struct {
udp.NoopConn
t *testing.T
mu sync.Mutex
peers map[netip.AddrPort]bool
}
func (c *establishedPeerConn) SetEstablishedPeer(dst netip.AddrPort, established bool) {
c.mu.Lock()
defer c.mu.Unlock()
assert.NotEqual(c.t, established, c.peers[dst], "SetEstablishedPeer(%v, %v) repeated", dst, established)
if established {
c.peers[dst] = true
} else {
delete(c.peers, dst)
}
}
func (c *establishedPeerConn) established() []netip.AddrPort {
c.mu.Lock()
defer c.mu.Unlock()
var out []netip.AddrPort
for dst := range c.peers {
out = append(out, dst)
}
return out
}
// newEstablishedTestHostMap returns a hostmap whose conns are a recording conn and one that doesn't track peers.
func newEstablishedTestHostMap(t *testing.T) (*HostMap, *establishedPeerConn) {
hm := newHostMap(test.NewLogger())
hm.preferredRanges.Store(&[]netip.Prefix{netip.MustParsePrefix("192.168.0.0/16")})
c := &establishedPeerConn{t: t, peers: map[netip.AddrPort]bool{}}
hm.established = newEstablishedRemotes([]udp.Conn{c, udp.NoopConn{}})
require.NotNil(t, hm.established)
return hm, c
}
func newEstablishedTestHostInfo(vpnAddr netip.Addr, index uint32) *HostInfo {
return &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
localIndexId: index,
remotes: NewRemoteList([]netip.Addr{vpnAddr}, nil),
}
}
func TestNewEstablishedRemotesWithoutTrackingConns(t *testing.T) {
assert.Nil(t, newEstablishedRemotes([]udp.Conn{udp.NoopConn{}}))
}
// A conn hears about a remote while at least one hostinfo in the main hostmap has it as its direct remote: from
// the add, through roaming and SetRemoteIfPreferred, until the last such hostinfo is deleted or retired by
// MaxHostInfosPerVpnIp. Pending and relayed hostinfos never count.
func TestHostMap_EstablishedRemotes(t *testing.T) {
hm, c := newEstablishedTestHostMap(t)
f := &Interface{}
a := netip.MustParseAddr("10.0.0.1")
r1 := netip.MustParseAddrPort("203.0.113.1:4242")
r2 := netip.MustParseAddrPort("203.0.113.2:4242")
r3 := netip.MustParseAddrPort("192.168.1.3:4242")
// A pending hostinfo, as a handshake sets its remote, isn't established.
h1 := newEstablishedTestHostInfo(a, 1)
h1.SetRemote(r1)
assert.Empty(t, c.established())
// Handshake completion adds it to the main hostmap.
hm.unlockedAddHostInfo(h1, f)
assert.ElementsMatch(t, []netip.AddrPort{r1}, c.established())
// A relayed tunnel has no direct remote.
relayed := newEstablishedTestHostInfo(netip.MustParseAddr("10.0.0.2"), 2)
hm.unlockedAddHostInfo(relayed, f)
assert.ElementsMatch(t, []netip.AddrPort{r1}, c.established())
// A rehandshake's new hostinfo shares the remote, which stays established until both are gone.
h3 := newEstablishedTestHostInfo(a, 3)
h3.SetRemote(r1)
hm.unlockedAddHostInfo(h3, f)
hm.DeleteHostInfo(h1)
assert.ElementsMatch(t, []netip.AddrPort{r1}, c.established())
// Roaming moves the reference.
h3.SetRemote(r2)
assert.ElementsMatch(t, []netip.AddrPort{r2}, c.established())
require.True(t, h3.SetRemoteIfPreferred(hm, ViaSender{UdpAddr: r3}))
assert.ElementsMatch(t, []netip.AddrPort{r3}, c.established())
// A relayed tunnel that hears from its peer directly roams onto a direct remote.
relayed.SetRemote(r1)
assert.ElementsMatch(t, []netip.AddrPort{r1, r3}, c.established())
hm.DeleteHostInfo(relayed)
// A deleted hostinfo lets go, and a SetRemote after that changes nothing.
hm.DeleteHostInfo(h3)
assert.Empty(t, c.established())
h3.SetRemote(r2)
assert.Empty(t, c.established())
hm.DeleteHostInfo(h3)
// Adding past MaxHostInfosPerVpnIp retires the oldest hostinfo and its remote.
var hs []*HostInfo
for i := range MaxHostInfosPerVpnIp + 1 {
h := newEstablishedTestHostInfo(a, uint32(10+i))
h.SetRemote(netip.AddrPortFrom(netip.MustParseAddr("198.51.100.1"), uint16(1000+i)))
hm.unlockedAddHostInfo(h, f)
hs = append(hs, h)
}
assert.Len(t, c.established(), MaxHostInfosPerVpnIp)
assert.NotContains(t, c.established(), hs[0].GetRemote())
}
// Roaming on the receive path races tunnel teardown on the connection manager; whatever the order, a deleted
// hostinfo holds no reference.
func TestHostMap_EstablishedRemotesRoamRacesDelete(t *testing.T) {
hm, c := newEstablishedTestHostMap(t)
f := &Interface{}
a := netip.MustParseAddr("10.0.0.1")
for i := range 200 {
h := newEstablishedTestHostInfo(a, uint32(i+1))
h.SetRemote(netip.MustParseAddrPort("203.0.113.1:4242"))
hm.Lock()
hm.unlockedAddHostInfo(h, f)
hm.Unlock()
var wg sync.WaitGroup
wg.Go(func() { h.SetRemote(netip.AddrPortFrom(netip.MustParseAddr("203.0.113.2"), uint16(i+1))) })
wg.Go(func() { hm.DeleteHostInfo(h) })
wg.Wait()
require.Empty(t, c.established(), "round %d", i)
}
}
// A hostinfo holds its reference in the first tracking hostmap it joins; another's add and delete of it change
// nothing, so neither hostmap's counts go wrong.
func TestHostMap_EstablishedRemotesSingleOwner(t *testing.T) {
hm1, c1 := newEstablishedTestHostMap(t)
hm2, c2 := newEstablishedTestHostMap(t)
f := &Interface{}
r := netip.MustParseAddrPort("203.0.113.1:4242")
h := newEstablishedTestHostInfo(netip.MustParseAddr("10.0.0.1"), 1)
h.SetRemote(r)
hm1.unlockedAddHostInfo(h, f)
hm2.unlockedAddHostInfo(h, f)
assert.ElementsMatch(t, []netip.AddrPort{r}, c1.established())
assert.Empty(t, c2.established())
hm2.DeleteHostInfo(h)
assert.Equal(t, map[netip.AddrPort]int{r: 1}, hm1.established.refs)
assert.Empty(t, hm2.established.refs)
assert.ElementsMatch(t, []netip.AddrPort{r}, c1.established())
hm1.DeleteHostInfo(h)
assert.Empty(t, hm1.established.refs)
assert.Empty(t, hm2.established.refs)
assert.Empty(t, c1.established())
assert.Empty(t, c2.established())
}
+1 -6
View File
@@ -449,9 +449,7 @@ firewall:
#default_local_cidr_any: false
conntrack:
# tcp_timeout also applies to sctp and dccp
tcp_timeout: 12m
# udp_timeout also applies to udplite
udp_timeout: 3m
default_timeout: 10m
@@ -459,11 +457,8 @@ firewall:
# Rules are comprised of a protocol, port, and one or more of host, group, or CIDR
# Logical evaluation is roughly: port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND (local cidr)
# - port: Takes `0` or `any` as any, a single number `80`, a range `200-901`, or `fragment` to match second and further fragments of fragmented packets (since there is no port available).
# proto: `any`, `tcp`, `udp`, `udplite`, `dccp`, `sctp`, `icmp`, or any other IP protocol number from 1 to 255, ie `47` for gre
# proto: `any`, `tcp`, `udp`, or `icmp`
# a port specification is ignored if proto is `icmp`
# `icmp`, `1`, and `58` are the same, each matches both ICMP and ICMPv6
# Nebula only reads ports from tcp, udp, udplite, dccp, and sctp, so a proto `any` rule with a specific port or range only matches those protocols.
# Rules for other protocols (gre, esp, etc) need port `any`.
# host: `any` or a literal hostname, ie `test-host`
# group: `any` or a literal group name, ie `default-group`
# groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass
+758 -39
View File
@@ -2,10 +2,17 @@ package nebula
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"hash/fnv"
"log/slog"
"net/netip"
"reflect"
"slices"
"strconv"
"strings"
"sync"
"time"
@@ -17,6 +24,10 @@ import (
"github.com/slackhq/nebula/iputil"
)
type FirewallInterface interface {
AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
}
type conn struct {
Expires time.Time // Time when this conntrack entry will expire
@@ -31,7 +42,8 @@ type conn struct {
type Firewall struct {
Conntrack *FirewallConntrack
rules firewall.Rules
InRules *FirewallTable
OutRules *FirewallTable
InboundSendReject bool
OutboundSendReject bool
@@ -51,10 +63,12 @@ type Firewall struct {
// unsafeNetworks is the list of unsafe networks issued to us in the certificate
unsafeNetworks []netip.Prefix
rules string
rulesVersion uint16
incomingMetrics firewallMetrics
outgoingMetrics firewallMetrics
defaultLocalCIDRAny bool
incomingMetrics firewallMetrics
outgoingMetrics firewallMetrics
l *slog.Logger
}
@@ -72,6 +86,52 @@ type FirewallConntrack struct {
TimerWheel *TimerWheel[firewall.Packet]
}
// FirewallTable is the entry point for a rule, the evaluation order is:
// Proto AND port AND (CA SHA or CA name) AND local CIDR AND (group OR groups OR name OR remote CIDR)
type FirewallTable struct {
TCP firewallPort
UDP firewallPort
ICMP firewallPort
AnyProto firewallPort
}
func newFirewallTable() *FirewallTable {
return &FirewallTable{
TCP: firewallPort{},
UDP: firewallPort{},
ICMP: firewallPort{},
AnyProto: firewallPort{},
}
}
type FirewallCA struct {
Any *FirewallRule
CANames map[string]*FirewallRule
CAShas map[string]*FirewallRule
}
type FirewallRule struct {
// Any makes Hosts, Groups, and CIDR irrelevant
Any *firewallLocalCIDR
Hosts map[string]*firewallLocalCIDR
Groups []*firewallGroups
CIDR *bart.Table[*firewallLocalCIDR]
}
type firewallGroups struct {
Groups []string
LocalCIDR *firewallLocalCIDR
}
// Even though ports are uint16, int32 maps are faster for lookup
// Plus we can use `-1` for fragment rules
type firewallPort map[int32]*FirewallCA
type firewallLocalCIDR struct {
Any bool
LocalCIDR *bart.Lite
}
// NewFirewall creates a new Firewall object. A TimerWheel is created for you from the provided timeouts.
// The certificate provided should be the highest version loaded in memory.
func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c cert.Certificate) *Firewall {
@@ -110,8 +170,8 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
Conns: make(map[firewall.Packet]*conn),
TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax),
},
// Nothing is allowed until there are rules
rules: firewall.NewRulesBuilder(l).Build(assignedNetworks, unsafeNetworks),
InRules: newFirewallTable(),
OutRules: newFirewallTable(),
TCPTimeout: tcpTimeout,
UDPTimeout: UDPTimeout,
DefaultTimeout: defaultTimeout,
@@ -152,6 +212,8 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
//TODO: max_connections
)
fw.defaultLocalCIDRAny = c.GetBool("firewall.default_local_cidr_any", false)
inboundAction := c.GetString("firewall.inbound_action", "drop")
switch inboundAction {
case "reject":
@@ -174,23 +236,80 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
fw.OutboundSendReject = false
}
rules, err := firewall.RulesFromConfig(l, c, fw.assignedNetworks, fw.unsafeNetworks)
err := AddFirewallRulesFromConfig(l, false, c, fw)
if err != nil {
return nil, err
}
err = AddFirewallRulesFromConfig(l, true, c, fw)
if err != nil {
return nil, err
}
fw.rules = rules
return fw, nil
}
// AddRule properly creates the in memory rule structure for a firewall table.
func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
var (
ft *FirewallTable
fp firewallPort
)
if incoming {
ft = f.InRules
} else {
ft = f.OutRules
}
switch proto {
case iputil.IPProtocolTCP:
fp = ft.TCP
case iputil.IPProtocolUDP:
fp = ft.UDP
case iputil.IPProtocolICMP, iputil.IPProtocolICMPv6:
//ICMP traffic doesn't have ports, so we always coerce to "any", even if a value is provided
if startPort != firewall.PortAny {
f.l.Warn("ignoring port specification for ICMP firewall rule", "startPort", startPort)
}
startPort = firewall.PortAny
endPort = firewall.PortAny
fp = ft.ICMP
case firewall.ProtoAny:
fp = ft.AnyProto
default:
return fmt.Errorf("unknown protocol %v", proto)
}
// We need this rule string because we generate a hash. Removing this will break firewall reload.
ruleString := fmt.Sprintf(
"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s",
incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha,
)
f.rules += ruleString + "\n"
direction := "incoming"
if !incoming {
direction = "outgoing"
}
f.l.Info("Firewall rule added",
"firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha},
)
return fp.addRule(f, startPort, endPort, groups, host, cidr, localCidr, caName, caSha)
}
// GetRuleHash returns a hash representation of all inbound and outbound rules
func (f *Firewall) GetRuleHash() string {
return f.rules.Hash
sum := sha256.Sum256([]byte(f.rules))
return hex.EncodeToString(sum[:])
}
// GetRuleHashFNV returns a uint32 FNV-1 hash representation the rules, for use as a metric value
func (f *Firewall) GetRuleHashFNV() uint32 {
return f.rules.HashFNV
h := fnv.New32a()
h.Write([]byte(f.rules))
return h.Sum32()
}
// GetRuleHashes returns both the sha256 and FNV-1 hashes, suitable for logging
@@ -198,6 +317,104 @@ func (f *Firewall) GetRuleHashes() string {
return "SHA:" + f.GetRuleHash() + ",FNV:" + strconv.FormatUint(uint64(f.GetRuleHashFNV()), 10)
}
func AddFirewallRulesFromConfig(l *slog.Logger, inbound bool, c *config.C, fw FirewallInterface) error {
var table string
if inbound {
table = "firewall.inbound"
} else {
table = "firewall.outbound"
}
r := c.Get(table)
if r == nil {
return nil
}
rs, ok := r.([]any)
if !ok {
return fmt.Errorf("%s failed to parse, should be an array of rules", table)
}
for i, t := range rs {
r, err := convertRule(l, t, table, i)
if err != nil {
return fmt.Errorf("%s rule #%v; %s", table, i, err)
}
if r.Code != "" && r.Port != "" {
return fmt.Errorf("%s rule #%v; only one of port or code should be provided", table, i)
}
if r.Host == "" && len(r.Groups) == 0 && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
return fmt.Errorf("%s rule #%v; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided", table, i)
}
var sPort, errPort string
if r.Code != "" {
errPort = "code"
sPort = r.Code
} else {
errPort = "port"
sPort = r.Port
}
var proto uint8
var startPort, endPort int32
switch r.Proto {
case "any":
proto = firewall.ProtoAny
startPort, endPort, err = parsePort(sPort)
case "tcp":
proto = iputil.IPProtocolTCP
startPort, endPort, err = parsePort(sPort)
case "udp":
proto = iputil.IPProtocolUDP
startPort, endPort, err = parsePort(sPort)
case "icmp":
proto = iputil.IPProtocolICMP
startPort = firewall.PortAny
endPort = firewall.PortAny
if sPort != "" {
l.Warn("ignoring port specification for ICMP firewall rule", "port", sPort)
}
default:
return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
}
if err != nil {
return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err)
}
if r.Cidr != "" && r.Cidr != "any" {
_, err = netip.ParsePrefix(r.Cidr)
if err != nil {
return fmt.Errorf("%s rule #%v; cidr did not parse; %s", table, i, err)
}
}
if r.LocalCidr != "" && r.LocalCidr != "any" {
_, err = netip.ParsePrefix(r.LocalCidr)
if err != nil {
return fmt.Errorf("%s rule #%v; local_cidr did not parse; %s", table, i, err)
}
}
if warning := r.sanity(); warning != nil {
l.Warn("firewall rule sanity check",
"table", table,
"rule", i,
"warning", warning,
)
}
err = fw.AddRule(inbound, proto, startPort, endPort, r.Groups, r.Host, r.Cidr, r.LocalCidr, r.CAName, r.CASha)
if err != nil {
return fmt.Errorf("%s rule #%v; `%s`", table, i, err)
}
}
return nil
}
var ErrUnknownNetworkType = errors.New("unknown network type")
var ErrPeerRejected = errors.New("remote address is not within a network that we handle")
var ErrInvalidRemoteIP = errors.New("remote address is not in remote certificate networks")
@@ -241,17 +458,23 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
}
// Check if we spoke to this tuple, if we did then allow this packet
if f.inConns(&fp, h, caPool, localCache) {
if f.inConns(fp, h, caPool, localCache) {
return nil
}
if !f.rules.Match(&fp, incoming, h.ConnectionState.peerCert, caPool) {
table := f.OutRules
if incoming {
table = f.InRules
}
// We now know which firewall table to check against
if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
f.metrics(incoming).droppedNoRule.Inc(1)
return ErrNoMatchingRule
}
// We always want to conntrack since it is a faster operation
f.addConn(&fp, incoming)
f.addConn(fp, incoming)
return nil
}
@@ -280,9 +503,9 @@ func (f *Firewall) EmitStats() {
metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
}
func (f *Firewall) inConns(fp *firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
if localCache != nil {
if _, ok := localCache[*fp]; ok {
if _, ok := localCache[fp]; ok {
return true
}
}
@@ -295,7 +518,7 @@ func (f *Firewall) inConns(fp *firewall.Packet, h *HostInfo, caPool *cert.CAPool
f.evict(ep)
}
c, ok := conntrack.Conns[*fp]
c, ok := conntrack.Conns[fp]
if !ok {
conntrack.Unlock()
@@ -305,23 +528,29 @@ func (f *Firewall) inConns(fp *firewall.Packet, h *HostInfo, caPool *cert.CAPool
if c.rulesVersion != f.rulesVersion {
// This conntrack entry was for an older rule set, validate
// it still passes with the current rule set
if !f.rules.Match(fp, c.incoming, h.ConnectionState.peerCert, caPool) {
table := f.OutRules
if c.incoming {
table = f.InRules
}
// We now know which firewall table to check against
if !table.match(fp, c.incoming, h.ConnectionState.peerCert, caPool) {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
h.logger(f.l).Debug("dropping old conntrack entry, does not match new ruleset",
"fwPacket", *fp,
"fwPacket", fp,
"incoming", c.incoming,
"rulesVersion", f.rulesVersion,
"oldRulesVersion", c.rulesVersion,
)
}
delete(conntrack.Conns, *fp)
delete(conntrack.Conns, fp)
conntrack.Unlock()
return false
}
if f.l.Enabled(context.Background(), slog.LevelDebug) {
h.logger(f.l).Debug("keeping old conntrack entry, does match new ruleset",
"fwPacket", *fp,
"fwPacket", fp,
"incoming", c.incoming,
"rulesVersion", f.rulesVersion,
"oldRulesVersion", c.rulesVersion,
@@ -331,39 +560,42 @@ func (f *Firewall) inConns(fp *firewall.Packet, h *HostInfo, caPool *cert.CAPool
c.rulesVersion = f.rulesVersion
}
c.Expires = time.Now().Add(f.conntrackTimeout(fp.Protocol))
switch fp.Protocol {
case iputil.IPProtocolTCP:
c.Expires = time.Now().Add(f.TCPTimeout)
case iputil.IPProtocolUDP:
c.Expires = time.Now().Add(f.UDPTimeout)
default:
c.Expires = time.Now().Add(f.DefaultTimeout)
}
conntrack.Unlock()
if localCache != nil {
localCache[*fp] = struct{}{}
localCache[fp] = struct{}{}
}
return true
}
// conntrackTimeout returns how long a conntrack entry for proto lives without traffic.
// Connection oriented protocols share the tcp timeout, datagram protocols share the udp timeout.
func (f *Firewall) conntrackTimeout(proto uint8) time.Duration {
switch proto {
case iputil.IPProtocolTCP, iputil.IPProtocolSCTP, iputil.IPProtocolDCCP:
return f.TCPTimeout
case iputil.IPProtocolUDP, iputil.IPProtocolUDPLite:
return f.UDPTimeout
default:
return f.DefaultTimeout
}
}
func (f *Firewall) addConn(fp *firewall.Packet, incoming bool) {
timeout := f.conntrackTimeout(fp.Protocol)
func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
var timeout time.Duration
c := &conn{}
switch fp.Protocol {
case iputil.IPProtocolTCP:
timeout = f.TCPTimeout
case iputil.IPProtocolUDP:
timeout = f.UDPTimeout
default:
timeout = f.DefaultTimeout
}
conntrack := f.Conntrack
conntrack.Lock()
if _, ok := conntrack.Conns[*fp]; !ok {
if _, ok := conntrack.Conns[fp]; !ok {
conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(*fp, timeout)
conntrack.TimerWheel.Add(fp, timeout)
}
// Record which rulesVersion allowed this connection, so we can retest after
@@ -371,7 +603,7 @@ func (f *Firewall) addConn(fp *firewall.Packet, incoming bool) {
c.incoming = incoming
c.rulesVersion = f.rulesVersion
c.Expires = time.Now().Add(timeout)
conntrack.Conns[*fp] = c
conntrack.Conns[fp] = c
conntrack.Unlock()
}
@@ -397,3 +629,490 @@ func (f *Firewall) evict(p firewall.Packet) {
// This conn is done
delete(conntrack.Conns, p)
}
func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
if ft.AnyProto.match(p, incoming, c, caPool) {
return true
}
switch p.Protocol {
case iputil.IPProtocolTCP:
if ft.TCP.match(p, incoming, c, caPool) {
return true
}
case iputil.IPProtocolUDP:
if ft.UDP.match(p, incoming, c, caPool) {
return true
}
case iputil.IPProtocolICMP, iputil.IPProtocolICMPv6:
if ft.ICMP.match(p, incoming, c, caPool) {
return true
}
}
return false
}
func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
if startPort > endPort {
return fmt.Errorf("start port was lower than end port")
}
for i := startPort; i <= endPort; i++ {
if _, ok := fp[i]; !ok {
fp[i] = &FirewallCA{
CANames: make(map[string]*FirewallRule),
CAShas: make(map[string]*FirewallRule),
}
}
if err := fp[i].addRule(f, groups, host, cidr, localCidr, caName, caSha); err != nil {
return err
}
}
return nil
}
func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
// We don't have any allowed ports, bail
if fp == nil {
return false
}
// this branch is here to catch traffic from FirewallTable.Any.match and FirewallTable.ICMP.match
if p.Protocol == iputil.IPProtocolICMP || p.Protocol == iputil.IPProtocolICMPv6 {
// port numbers are re-used for connection tracking of ICMP,
// but we don't want to actually filter on them.
return fp[firewall.PortAny].match(p, c, caPool)
}
var port int32
if p.Fragment {
port = firewall.PortFragment
} else if incoming {
port = int32(p.LocalPort)
} else {
port = int32(p.RemotePort)
}
if fp[port].match(p, c, caPool) {
return true
}
return fp[firewall.PortAny].match(p, c, caPool)
}
func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, localCidr, caName, caSha string) error {
fr := func() *FirewallRule {
return &FirewallRule{
Hosts: make(map[string]*firewallLocalCIDR),
Groups: make([]*firewallGroups, 0),
CIDR: new(bart.Table[*firewallLocalCIDR]),
}
}
if caSha == "" && caName == "" {
if fc.Any == nil {
fc.Any = fr()
}
return fc.Any.addRule(f, groups, host, cidr, localCidr)
}
if caSha != "" {
if _, ok := fc.CAShas[caSha]; !ok {
fc.CAShas[caSha] = fr()
}
err := fc.CAShas[caSha].addRule(f, groups, host, cidr, localCidr)
if err != nil {
return err
}
}
if caName != "" {
if _, ok := fc.CANames[caName]; !ok {
fc.CANames[caName] = fr()
}
err := fc.CANames[caName].addRule(f, groups, host, cidr, localCidr)
if err != nil {
return err
}
}
return nil
}
func (fc *FirewallCA) match(p firewall.Packet, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
if fc == nil {
return false
}
if fc.Any.match(p, c) {
return true
}
if t, ok := fc.CAShas[c.Certificate.Issuer()]; ok {
if t.match(p, c) {
return true
}
}
s, err := caPool.GetCAForCert(c.Certificate)
if err != nil {
return false
}
return fc.CANames[s.Certificate.Name()].match(p, c)
}
func (fr *FirewallRule) addRule(f *Firewall, groups []string, host, cidr, localCidr string) error {
flc := func() *firewallLocalCIDR {
return &firewallLocalCIDR{
LocalCIDR: new(bart.Lite),
}
}
if fr.isAny(groups, host, cidr) {
if fr.Any == nil {
fr.Any = flc()
}
return fr.Any.addRule(f, localCidr)
}
if len(groups) > 0 {
nlc := flc()
err := nlc.addRule(f, localCidr)
if err != nil {
return err
}
fr.Groups = append(fr.Groups, &firewallGroups{
Groups: groups,
LocalCIDR: nlc,
})
}
if host != "" {
nlc := fr.Hosts[host]
if nlc == nil {
nlc = flc()
}
err := nlc.addRule(f, localCidr)
if err != nil {
return err
}
fr.Hosts[host] = nlc
}
if cidr != "" {
c, err := netip.ParsePrefix(cidr)
if err != nil {
return err
}
nlc, _ := fr.CIDR.Get(c)
if nlc == nil {
nlc = flc()
}
err = nlc.addRule(f, localCidr)
if err != nil {
return err
}
fr.CIDR.Insert(c, nlc)
}
return nil
}
func (fr *FirewallRule) isAny(groups []string, host string, cidr string) bool {
if len(groups) == 0 && host == "" && cidr == "" {
return true
}
if slices.Contains(groups, "any") {
return true
}
if host == "any" {
return true
}
if cidr == "any" {
return true
}
return false
}
func (fr *FirewallRule) match(p firewall.Packet, c *cert.CachedCertificate) bool {
if fr == nil {
return false
}
// Shortcut path for if groups, hosts, or cidr contained an `any`
if fr.Any.match(p, c) {
return true
}
// Need any of group, host, or cidr to match
for _, sg := range fr.Groups {
found := false
for _, g := range sg.Groups {
if _, ok := c.InvertedGroups[g]; !ok {
found = false
break
}
found = true
}
if found && sg.LocalCIDR.match(p, c) {
return true
}
}
if fr.Hosts != nil {
if flc, ok := fr.Hosts[c.Certificate.Name()]; ok {
if flc.match(p, c) {
return true
}
}
}
for _, v := range fr.CIDR.Supernets(netip.PrefixFrom(p.RemoteAddr, p.RemoteAddr.BitLen())) {
if v.match(p, c) {
return true
}
}
return false
}
func (flc *firewallLocalCIDR) addRule(f *Firewall, localCidr string) error {
if localCidr == "any" {
flc.Any = true
return nil
}
if localCidr == "" {
if len(f.unsafeNetworks) == 0 || f.defaultLocalCIDRAny {
flc.Any = true
return nil
}
for _, network := range f.assignedNetworks {
flc.LocalCIDR.Insert(network)
}
return nil
}
c, err := netip.ParsePrefix(localCidr)
if err != nil {
return err
}
flc.LocalCIDR.Insert(c)
return nil
}
func (flc *firewallLocalCIDR) match(p firewall.Packet, c *cert.CachedCertificate) bool {
if flc == nil {
return false
}
if flc.Any {
return true
}
return flc.LocalCIDR.Contains(p.LocalAddr)
}
type rule struct {
Port string
Code string
Proto string
Host string
Groups []string
Cidr string
LocalCidr string
CAName string
CASha string
}
func convertRule(l *slog.Logger, p any, table string, i int) (rule, error) {
r := rule{}
m, ok := p.(map[string]any)
if !ok {
return r, errors.New("could not parse rule")
}
toString := func(k string, m map[string]any) string {
v, ok := m[k]
if !ok {
return ""
}
return fmt.Sprintf("%v", v)
}
r.Port = toString("port", m)
r.Code = toString("code", m)
r.Proto = toString("proto", m)
r.Host = toString("host", m)
r.Cidr = toString("cidr", m)
r.LocalCidr = toString("local_cidr", m)
r.CAName = toString("ca_name", m)
r.CASha = toString("ca_sha", m)
// Make sure group isn't an array
if v, ok := m["group"].([]any); ok {
if len(v) > 1 {
return r, errors.New("group should contain a single value, an array with more than one entry was provided")
}
l.Warn("group was an array with a single value, converting to simple value",
"table", table,
"rule", i,
)
m["group"] = v[0]
}
singleGroup := toString("group", m)
if rg, ok := m["groups"]; ok {
switch reflect.TypeOf(rg).Kind() {
case reflect.Slice:
v := reflect.ValueOf(rg)
r.Groups = make([]string, v.Len())
for i := 0; i < v.Len(); i++ {
r.Groups[i] = v.Index(i).Interface().(string)
}
case reflect.String:
r.Groups = []string{rg.(string)}
default:
r.Groups = []string{fmt.Sprintf("%v", rg)}
}
}
//flatten group vs groups
if singleGroup != "" {
// Check if we have both groups and group provided in the rule config
if len(r.Groups) > 0 {
return r, fmt.Errorf("only one of group or groups should be defined, both provided")
}
r.Groups = []string{singleGroup}
}
return r, nil
}
// sanity returns an error if the rule would be evaluated in a way that would short-circuit a configured check on a wildcard value
// rules are evaluated as "port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND local_cidr"
func (r *rule) sanity() error {
//port, proto, local_cidr are AND, no need to check here
//ca_sha and ca_name don't have a wildcard value, no need to check here
groupsEmpty := len(r.Groups) == 0
hostEmpty := r.Host == ""
cidrEmpty := r.Cidr == ""
if (groupsEmpty && hostEmpty && cidrEmpty) == true {
return nil //no content!
}
groupsHasAny := slices.Contains(r.Groups, "any")
if groupsHasAny && len(r.Groups) > 1 {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the other groups specified", r.Groups)
}
if r.Host == "any" {
if !groupsEmpty {
return fmt.Errorf("groups specified as %s, but host=any will match any host, regardless of groups", r.Groups)
}
if !cidrEmpty {
return fmt.Errorf("cidr specified as %s, but host=any will match any host, regardless of cidr", r.Cidr)
}
}
if groupsHasAny {
if !hostEmpty && r.Host != "any" {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified host %s", r.Groups, r.Host)
}
if !cidrEmpty {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified cidr %s", r.Groups, r.Cidr)
}
}
if r.Code != "" {
return fmt.Errorf("code specified as [%s]. Support for 'code' will be dropped in a future release, as it has never been functional", r.Code)
}
//todo alert on cidr-any
return nil
}
func parsePort(s string) (int32, int32, error) {
const notAPort int32 = -2
if s == "any" {
return firewall.PortAny, firewall.PortAny, nil
}
if s == "fragment" {
return firewall.PortFragment, firewall.PortFragment, nil
}
if !strings.Contains(s, `-`) {
rPort, err := parsePortValue("", s)
if err != nil {
return notAPort, notAPort, err
}
return rPort, rPort, nil
}
sPorts := strings.SplitN(s, `-`, 2)
for i := range sPorts {
sPorts[i] = strings.Trim(sPorts[i], " ")
}
if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
return notAPort, notAPort, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
}
startPort, err := parsePortValue("beginning range ", sPorts[0])
if err != nil {
return notAPort, notAPort, err
}
endPort, err := parsePortValue("ending range ", sPorts[1])
if err != nil {
return notAPort, notAPort, err
}
if startPort == firewall.PortAny {
endPort = firewall.PortAny
}
return startPort, endPort, nil
}
// parsePortValue accepts a base-10 decimal in [0, 65535] and returns it
// widened to int32. Using strconv.ParseUint with bitSize 16 rejects
// negative input, out-of-range input (>65535), and any non-decimal byte
// by construction, so the int32 widening that follows is provably safe
// and cannot collide with firewall.PortAny (0) or firewall.PortFragment
// (-1) via integer truncation.
//
// prefix is prepended to both error messages so callers can disambiguate
// the single-port path (prefix="") from the range bounds (prefix="beginning
// range " / "ending range "), preserving the historical error strings.
func parsePortValue(prefix, s string) (int32, error) {
n, err := strconv.ParseUint(s, 10, 16)
if err == nil {
return int32(n), nil
}
if errors.Is(err, strconv.ErrRange) {
return 0, fmt.Errorf("%sout of range [0,65535]; `%s`", prefix, s)
}
return 0, fmt.Errorf("%swas not a number; `%s`", prefix, s)
}
-310
View File
@@ -1,310 +0,0 @@
package firewall
import (
"errors"
"fmt"
"log/slog"
"net/netip"
"reflect"
"slices"
"strconv"
"strings"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/iputil"
)
// ruleAdder takes firewall rules, it lets tests see what addRulesFromConfig adds
type ruleAdder interface {
AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
}
// RulesFromConfig reads the inbound and outbound firewall rules from c and builds them, see RulesBuilder.Build
func RulesFromConfig(l *slog.Logger, c *config.C, vpnNetworks, unsafeNetworks []netip.Prefix) (Rules, error) {
b := NewRulesBuilder(l)
b.defaultLocalCIDRAny = c.GetBool("firewall.default_local_cidr_any", false)
if err := addRulesFromConfig(l, false, c, b); err != nil {
return Rules{}, err
}
if err := addRulesFromConfig(l, true, c, b); err != nil {
return Rules{}, err
}
return b.Build(vpnNetworks, unsafeNetworks), nil
}
func addRulesFromConfig(l *slog.Logger, inbound bool, c *config.C, ra ruleAdder) error {
var table string
if inbound {
table = "firewall.inbound"
} else {
table = "firewall.outbound"
}
r := c.Get(table)
if r == nil {
return nil
}
rs, ok := r.([]any)
if !ok {
return fmt.Errorf("%s failed to parse, should be an array of rules", table)
}
for i, t := range rs {
r, err := yamlToConfigRule(l, t, table, i)
if err != nil {
return fmt.Errorf("%s rule #%v; %s", table, i, err)
}
if r.Code != "" && r.Port != "" {
return fmt.Errorf("%s rule #%v; only one of port or code should be provided", table, i)
}
if r.Host == "" && len(r.Groups) == 0 && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
return fmt.Errorf("%s rule #%v; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided", table, i)
}
var sPort, errPort string
if r.Code != "" {
errPort = "code"
sPort = r.Code
} else {
errPort = "port"
sPort = r.Port
}
proto, ok := protoByName(r.Proto)
if !ok {
// Any other protocol by number. 0 is reserved for `any`.
n, perr := strconv.ParseUint(r.Proto, 10, 8)
if perr != nil || n == 0 {
return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
}
proto = uint8(n)
}
var startPort, endPort int32
if proto == iputil.IPProtocolICMP || proto == iputil.IPProtocolICMPv6 {
// RulesBuilder.AddRule coerces ICMP ports to `any` as well. This copy stays so that an ICMP rule's
// port is warned about rather than parsed, whatever it holds.
startPort = PortAny
endPort = PortAny
if sPort != "" {
l.Warn("ignoring port specification for ICMP firewall rule", "port", sPort)
}
} else {
startPort, endPort, err = parsePort(sPort)
}
if err != nil {
return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err)
}
if warning := r.sanity(); warning != nil {
l.Warn("firewall rule sanity check",
"table", table,
"rule", i,
"warning", warning,
)
}
err = ra.AddRule(inbound, proto, startPort, endPort, r.Groups, r.Host, r.Cidr, r.LocalCidr, r.CAName, r.CASha)
if err != nil {
return fmt.Errorf("%s rule #%v; %s", table, i, err)
}
}
return nil
}
type configRule struct {
Port string
Code string
Proto string
Host string
Groups []string
Cidr string
LocalCidr string
CAName string
CASha string
}
func yamlToConfigRule(l *slog.Logger, p any, table string, i int) (configRule, error) {
r := configRule{}
m, ok := p.(map[string]any)
if !ok {
return r, errors.New("could not parse rule")
}
toString := func(k string, m map[string]any) string {
v, ok := m[k]
if !ok {
return ""
}
return fmt.Sprintf("%v", v)
}
r.Port = toString("port", m)
r.Code = toString("code", m)
r.Proto = toString("proto", m)
r.Host = toString("host", m)
r.Cidr = toString("cidr", m)
r.LocalCidr = toString("local_cidr", m)
r.CAName = toString("ca_name", m)
r.CASha = toString("ca_sha", m)
// Make sure group isn't an array
if v, ok := m["group"].([]any); ok {
if len(v) > 1 {
return r, errors.New("group should contain a single value, an array with more than one entry was provided")
}
if len(v) == 0 {
return r, errors.New("group should contain a single value, an empty array was provided")
}
l.Warn("group was an array with a single value, converting to simple value",
"table", table,
"rule", i,
)
m["group"] = v[0]
}
singleGroup := toString("group", m)
// A null value, `groups:` with nothing after it, is the same as no groups
if rg, ok := m["groups"]; ok && rg != nil {
switch reflect.TypeOf(rg).Kind() {
case reflect.Slice:
// Every entry is read as a string, as the other fields are, so `groups: [1, 2]` is the groups "1" and "2"
v := reflect.ValueOf(rg)
r.Groups = make([]string, v.Len())
for i := 0; i < v.Len(); i++ {
r.Groups[i] = fmt.Sprintf("%v", v.Index(i).Interface())
}
case reflect.String:
r.Groups = []string{rg.(string)}
default:
r.Groups = []string{fmt.Sprintf("%v", rg)}
}
}
//flatten group vs groups
if singleGroup != "" {
// Check if we have both groups and group provided in the rule config
if len(r.Groups) > 0 {
return r, fmt.Errorf("only one of group or groups should be defined, both provided")
}
r.Groups = []string{singleGroup}
}
return r, nil
}
// sanity returns an error if the rule would be evaluated in a way that would short-circuit a configured check on a wildcard value
// rules are evaluated as "port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND local_cidr"
func (r *configRule) sanity() error {
//port, proto, local_cidr are AND, no need to check here
//ca_sha and ca_name don't have a wildcard value, no need to check here
groupsEmpty := len(r.Groups) == 0
hostEmpty := r.Host == ""
cidrEmpty := r.Cidr == ""
if (groupsEmpty && hostEmpty && cidrEmpty) == true {
return nil //no content!
}
groupsHasAny := slices.Contains(r.Groups, "any")
if groupsHasAny && len(r.Groups) > 1 {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the other groups specified", r.Groups)
}
if r.Host == "any" {
if !groupsEmpty {
return fmt.Errorf("groups specified as %s, but host=any will match any host, regardless of groups", r.Groups)
}
if !cidrEmpty {
return fmt.Errorf("cidr specified as %s, but host=any will match any host, regardless of cidr", r.Cidr)
}
}
if groupsHasAny {
if !hostEmpty && r.Host != "any" {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified host %s", r.Groups, r.Host)
}
if !cidrEmpty {
return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified cidr %s", r.Groups, r.Cidr)
}
}
if r.Code != "" {
return fmt.Errorf("code specified as [%s]. Support for 'code' will be dropped in a future release, as it has never been functional", r.Code)
}
//todo alert on cidr-any
return nil
}
func parsePort(s string) (int32, int32, error) {
const notAPort int32 = -2
if s == "any" {
return PortAny, PortAny, nil
}
if s == "fragment" {
return PortFragment, PortFragment, nil
}
if !strings.Contains(s, `-`) {
rPort, err := parsePortValue("", s)
if err != nil {
return notAPort, notAPort, err
}
return rPort, rPort, nil
}
sPorts := strings.SplitN(s, `-`, 2)
for i := range sPorts {
sPorts[i] = strings.Trim(sPorts[i], " ")
}
if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
return notAPort, notAPort, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
}
startPort, err := parsePortValue("beginning range ", sPorts[0])
if err != nil {
return notAPort, notAPort, err
}
endPort, err := parsePortValue("ending range ", sPorts[1])
if err != nil {
return notAPort, notAPort, err
}
if startPort == PortAny {
endPort = PortAny
}
return startPort, endPort, nil
}
// parsePortValue accepts a base-10 decimal in [0, 65535] and returns it
// widened to int32. Using strconv.ParseUint with bitSize 16 rejects
// negative input, out-of-range input (>65535), and any non-decimal byte
// by construction, so the int32 widening that follows is provably safe
// and cannot collide with PortAny (0) or PortFragment
// (-1) via integer truncation.
//
// prefix is prepended to both error messages so callers can disambiguate
// the single-port path (prefix="") from the range bounds (prefix="beginning
// range " / "ending range "), preserving the historical error strings.
func parsePortValue(prefix, s string) (int32, error) {
n, err := strconv.ParseUint(s, 10, 16)
if err == nil {
return int32(n), nil
}
if errors.Is(err, strconv.ErrRange) {
return 0, fmt.Errorf("%sout of range [0,65535]; `%s`", prefix, s)
}
return 0, fmt.Errorf("%swas not a number; `%s`", prefix, s)
}
-516
View File
@@ -1,516 +0,0 @@
package firewall
import (
"bytes"
"errors"
"net/netip"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_parsePort(t *testing.T) {
_, _, err := parsePort("")
require.EqualError(t, err, "was not a number; ``")
_, _, err = parsePort(" ")
require.EqualError(t, err, "was not a number; ` `")
_, _, err = parsePort("-")
require.EqualError(t, err, "appears to be a range but could not be parsed; `-`")
_, _, err = parsePort(" - ")
require.EqualError(t, err, "appears to be a range but could not be parsed; ` - `")
_, _, err = parsePort("a-b")
require.EqualError(t, err, "beginning range was not a number; `a`")
_, _, err = parsePort("1-b")
require.EqualError(t, err, "ending range was not a number; `b`")
s, e, err := parsePort(" 1 - 2 ")
assert.Equal(t, int32(1), s)
assert.Equal(t, int32(2), e)
require.NoError(t, err)
s, e, err = parsePort("0-1")
assert.Equal(t, int32(0), s)
assert.Equal(t, int32(0), e)
require.NoError(t, err)
s, e, err = parsePort("9919")
assert.Equal(t, int32(9919), s)
assert.Equal(t, int32(9919), e)
require.NoError(t, err)
s, e, err = parsePort("any")
assert.Equal(t, int32(0), s)
assert.Equal(t, int32(0), e)
require.NoError(t, err)
}
// Test_parsePort_invalid covers inputs that must error. The named bug is
// that int32(strconv.Atoi("4294967296")) truncates to 0 == PortAny,
// silently turning a typo into a match-all-ports rule; the rest are
// representative syntax/range probes.
func Test_parsePort_invalid(t *testing.T) {
tests := []struct {
name string
input string
wantErrContains string
}{
// Numeric overflow (the named bug + boundary).
{"named bug: 2^32 truncates to PortAny", "4294967296", "out of range"},
{"just above max real port", "65536", "out of range"},
// Negatives route through the range branch and hit the empty-half
// guard; included as defense in depth so a future refactor cannot
// accidentally reach the int32 cast.
{"negative", "-1", "could not be parsed"},
// Syntax probes.
{"NUL between digits", "4\x002", "was not a number"},
{"hex notation", "0x10", "was not a number"},
{"scientific notation", "1e3", "was not a number"},
{"leading whitespace", " 42", "was not a number"},
{"fullwidth digits", "42", "was not a number"},
// Range branch.
{"range upper out of range", "1-65536", "ending range out of range"},
{"range lower out of range", "65536-65537", "beginning range out of range"},
{"range with negative upper", "1--1", "ending range was not a number"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, _, err := parsePort(tc.input)
require.Error(t, err, "input %q must error", tc.input)
require.ErrorContains(t, err, tc.wantErrContains)
})
}
}
// Test_parsePort_valid_boundaries locks in success cases at 0, 1, and 65535
// so a future refactor cannot regress the boundaries.
func Test_parsePort_valid_boundaries(t *testing.T) {
tests := []struct {
name string
input string
wantStart int32
wantEnd int32
}{
{"zero is PortAny", "0", 0, 0},
{"min real port", "1", 1, 1},
{"max real port", "65535", 65535, 65535},
{"range zero to max forces end to zero", "0-65535", 0, 0},
{"range max to max", "65535-65535", 65535, 65535},
{"range one to max", "1-65535", 1, 65535},
{"range with whitespace inside", " 1 - 2 ", 1, 2},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
s, e, err := parsePort(tc.input)
require.NoError(t, err)
assert.Equal(t, tc.wantStart, s, "start port")
assert.Equal(t, tc.wantEnd, e, "end port")
})
}
}
func TestAddRulesFromConfig(t *testing.T) {
l := test.NewLogger()
// Test adding tcp rule
conf := config.NewC(test.NewLogger())
mf := &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "tcp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolTCP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding udp rule
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "udp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolUDP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding udplite, dccp, and sctp rules
for name, proto := range map[string]uint8{"udplite": iputil.IPProtocolUDPLite, "dccp": iputil.IPProtocolDCCP, "sctp": iputil.IPProtocolSCTP} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1-2", "proto": name, "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf), name)
assert.Equal(t, addRuleCall{incoming: false, proto: proto, startPort: 1, endPort: 2, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall, name)
}
// Test adding icmp rule
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "icmp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolICMP, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding icmp rule no port
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"proto": "icmp", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf))
assert.Equal(t, addRuleCall{incoming: false, proto: iputil.IPProtocolICMP, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding rules by protocol number, as yaml would give a string or an int
for _, proto := range []any{"47", 47} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "any", "proto": proto, "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf), proto)
assert.Equal(t, addRuleCall{incoming: false, proto: 47, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall, proto)
}
// Test ICMP and ICMPv6, by name and by number, still ignore ports
for _, tc := range []struct {
given any
want uint8
}{
{"icmp", iputil.IPProtocolICMP},
{"icmpv6", iputil.IPProtocolICMPv6},
{iputil.IPProtocolICMP, iputil.IPProtocolICMP},
{iputil.IPProtocolICMPv6, iputil.IPProtocolICMPv6},
} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": tc.given, "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, false, conf, mf), tc.given)
assert.Equal(t, addRuleCall{incoming: false, proto: tc.want, startPort: PortAny, endPort: PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall, tc.given)
}
// Test protocols that aren't a known name or a number from 1 to 255
for _, proto := range []string{"0", "256", "-1", "gre"} {
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "any", "proto": proto, "host": "a"}}}
require.EqualError(t, addRulesFromConfig(l, false, conf, mf), "firewall.outbound rule #0; proto was not understood; `"+proto+"`")
}
// Test adding any rule
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
// Test adding rule with cidr
cidr := netip.MustParsePrefix("10.0.0.0/8")
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr.String(), localIp: ""}, mf.lastCall)
// Test adding rule with local_cidr
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: cidr.String()}, mf.lastCall)
// Test adding rule with cidr ipv6
cidr6 := netip.MustParsePrefix("fd00::/8")
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr6.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr6.String(), localIp: ""}, mf.lastCall)
// Test adding rule with any cidr
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "any"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "any", localIp: ""}, mf.lastCall)
// Test adding rule with junk cidr, which RulesBuilder rejects
conf = config.NewC(test.NewLogger())
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "junk/junk"}}}
require.EqualError(t, addRulesFromConfig(l, true, conf, NewRulesBuilder(l)), "firewall.inbound rule #0; cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP")
// Test adding rule with local_cidr ipv6
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr6.String()}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: cidr6.String()}, mf.lastCall)
// Test adding rule with any local_cidr
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "any"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, localIp: "any"}, mf.lastCall)
// Test adding rule with junk local_cidr, which RulesBuilder rejects
conf = config.NewC(test.NewLogger())
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "junk/junk"}}}
require.EqualError(t, addRulesFromConfig(l, true, conf, NewRulesBuilder(l)), "firewall.inbound rule #0; local_cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP")
// Test adding rule with ca_sha
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_sha": "12312313123"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caSha: "12312313123"}, mf.lastCall)
// Test adding rule with ca_name
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_name": "root01"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caName: "root01"}, mf.lastCall)
// Test single group
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "group": "a"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall)
// Test single groups
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": "a"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall)
// Test multiple AND groups
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": []string{"a", "b"}}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, addRuleCall{incoming: true, proto: ProtoAny, startPort: 1, endPort: 1, groups: []string{"a", "b"}, ip: "", localIp: ""}, mf.lastCall)
// Test group and groups values that used to crash: an empty group array is rejected, groups entries that
// aren't strings are read as strings, and a null groups is no groups
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "group": []any{}}}}
require.EqualError(t, addRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; group should contain a single value, an empty array was provided")
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": []any{1, 2}}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Equal(t, []string{"1", "2"}, mf.lastCall.groups)
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": nil, "host": "a"}}}
require.NoError(t, addRulesFromConfig(l, true, conf, mf))
assert.Empty(t, mf.lastCall.groups)
assert.Equal(t, "a", mf.lastCall.host)
// Test Add error
conf = config.NewC(test.NewLogger())
mf = &mockFirewall{}
mf.nextCallReturn = errors.New("test error")
conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "host": "a"}}}
require.EqualError(t, addRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; test error")
}
// TestRulesFromConfig_defaultLocalCIDR ensures rules without a local_cidr allow only the vpn networks when there are
// unsafe networks, unless default_local_cidr_any is set, and that a rule's own local_cidr is kept either way
func TestRulesFromConfig_defaultLocalCIDR(t *testing.T) {
vpnNetworks := []netip.Prefix{netip.MustParsePrefix("10.1.0.5/16")}
unsafeNetworks := []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}
vpnAddr := netip.MustParseAddr("10.1.9.9")
unsafeAddr := netip.MustParseAddr("192.168.0.3")
// ruleOnPort returns the rule on proto whose port clause is exactly port, chosen from the rules that a packet
// on port is checked against.
ruleOnPort := func(t *testing.T, table *Table, proto uint8, port int32) *rule {
t.Helper()
for _, r := range table.rulesAt(proto, port) {
if r.startPort == port {
return r
}
}
require.Failf(t, "no rule found", "proto %d port %d", proto, port)
return nil
}
for _, tc := range []struct {
name string
unsafeNetworks []netip.Prefix
defaultLocalCIDRAny bool
wantUnsafe bool
}{
{"no unsafe networks", nil, false, true},
{"unsafe networks", unsafeNetworks, false, false},
{"unsafe networks and default_local_cidr_any", unsafeNetworks, true, true},
} {
t.Run(tc.name, func(t *testing.T) {
conf := config.NewC(test.NewLogger())
conf.Settings["firewall"] = map[string]any{
"default_local_cidr_any": tc.defaultLocalCIDRAny,
"inbound": []any{
map[string]any{"port": "any", "proto": "any", "host": "any"},
map[string]any{"port": "22", "proto": "tcp", "host": "any"},
map[string]any{"port": "80", "proto": "tcp", "host": "any", "local_cidr": "192.168.0.0/24"},
map[string]any{"port": "443", "proto": "tcp", "host": "any", "local_cidr": "any"},
},
}
rules, err := RulesFromConfig(test.NewLogger(), conf, vpnNetworks, tc.unsafeNetworks)
require.NoError(t, err)
// The proto `any` rule, as indexed by a protocol with rules of its own and as shared by one without.
for _, proto := range []uint8{iputil.IPProtocolTCP, 47} {
r := ruleOnPort(t, rules.In, proto, PortAny)
assert.True(t, r.matchLocal(vpnAddr), "proto %d", proto)
assert.Equal(t, tc.wantUnsafe, r.matchLocal(unsafeAddr), "proto %d", proto)
}
r := ruleOnPort(t, rules.In, iputil.IPProtocolTCP, 22)
assert.True(t, r.matchLocal(vpnAddr))
assert.Equal(t, tc.wantUnsafe, r.matchLocal(unsafeAddr))
r = ruleOnPort(t, rules.In, iputil.IPProtocolTCP, 80)
assert.False(t, r.matchLocal(vpnAddr))
assert.True(t, r.matchLocal(unsafeAddr))
r = ruleOnPort(t, rules.In, iputil.IPProtocolTCP, 443)
assert.True(t, r.matchLocal(vpnAddr))
assert.True(t, r.matchLocal(unsafeAddr))
})
}
}
func Test_convertRule(t *testing.T) {
ob := &bytes.Buffer{}
l := test.NewLoggerWithOutput(ob)
// Ensure group array of 1 is converted and a warning is printed
c := map[string]any{
"group": []any{"group1"},
}
r, err := yamlToConfigRule(l, c, "test", 1)
assert.Contains(t, ob.String(), "group was an array with a single value, converting to simple value")
assert.Contains(t, ob.String(), "table=test")
assert.Contains(t, ob.String(), "rule=1")
require.NoError(t, err)
assert.Equal(t, []string{"group1"}, r.Groups)
// Ensure group array of > 1 is errord
ob.Reset()
c = map[string]any{
"group": []any{"group1", "group2"},
}
r, err = yamlToConfigRule(l, c, "test", 1)
assert.Empty(t, ob.String())
require.Error(t, err, "group should contain a single value, an array with more than one entry was provided")
// Make sure a well formed group is alright
ob.Reset()
c = map[string]any{
"group": "group1",
}
r, err = yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
assert.Equal(t, []string{"group1"}, r.Groups)
}
func Test_convertRuleSanity(t *testing.T) {
ob := &bytes.Buffer{}
l := test.NewLoggerWithOutput(ob)
noWarningPlease := []map[string]any{
{"group": "group1"},
{"groups": []any{"group2"}},
{"host": "bob"},
{"cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "host": "bob"},
{"cidr": "1.1.1.1/1", "host": "bob"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
}
for _, c := range noWarningPlease {
r, err := yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
require.NoError(t, r.sanity(), "should not generate a sanity warning, %+v", c)
}
yesWarningPlease := []map[string]any{
{"group": "group1"},
{"groups": []any{"group2"}},
{"cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "host": "bob"},
{"cidr": "1.1.1.1/1", "host": "bob"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
}
for _, c := range yesWarningPlease {
c["host"] = "any"
r, err := yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
err = r.sanity()
require.Error(t, err, "I wanted a warning: %+v", c)
}
//reset the list
yesWarningPlease = []map[string]any{
{"group": "group1"},
{"groups": []any{"group2"}},
{"cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "host": "bob"},
{"cidr": "1.1.1.1/1", "host": "bob"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1"},
{"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"},
}
for _, c := range yesWarningPlease {
r, err := yamlToConfigRule(l, c, "test", 1)
require.NoError(t, err)
r.Groups = append(r.Groups, "any")
err = r.sanity()
require.Error(t, err, "I wanted a warning: %+v", c)
}
}
type addRuleCall struct {
incoming bool
proto uint8
startPort int32
endPort int32
groups []string
host string
ip string
localIp string
caName string
caSha string
}
type mockFirewall struct {
lastCall addRuleCall
nextCallReturn error
}
func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip, localIp, caName string, caSha string) error {
mf.lastCall = addRuleCall{
incoming: incoming,
proto: proto,
startPort: startPort,
endPort: endPort,
groups: groups,
host: host,
ip: ip,
localIp: localIp,
caName: caName,
caSha: caSha,
}
err := mf.nextCallReturn
mf.nextCallReturn = nil
return err
}
+14 -40
View File
@@ -40,55 +40,29 @@ func (fp *Packet) Copy() *Packet {
}
func (fp Packet) MarshalJSON() ([]byte, error) {
var proto string
switch fp.Protocol {
case iputil.IPProtocolTCP:
proto = "tcp"
case iputil.IPProtocolICMP:
proto = "icmp"
case iputil.IPProtocolICMPv6:
proto = "icmpv6"
case iputil.IPProtocolUDP:
proto = "udp"
default:
proto = fmt.Sprintf("unknown %v", fp.Protocol)
}
return json.Marshal(m{
"LocalAddr": fp.LocalAddr.String(),
"RemoteAddr": fp.RemoteAddr.String(),
"LocalPort": fp.LocalPort,
"RemotePort": fp.RemotePort,
"Protocol": protoName(fp.Protocol),
"Protocol": proto,
"Fragment": fp.Fragment,
})
}
// protoNames are the protocol names that rules accept and packets report, and their numbers. Any other protocol
// is given by number.
var protoNames = []struct {
name string
proto uint8
}{
{"any", ProtoAny},
{"tcp", iputil.IPProtocolTCP},
{"udp", iputil.IPProtocolUDP},
{"udplite", iputil.IPProtocolUDPLite},
{"dccp", iputil.IPProtocolDCCP},
{"sctp", iputil.IPProtocolSCTP},
{"icmp", iputil.IPProtocolICMP},
{"icmpv6", iputil.IPProtocolICMPv6},
}
// protoByName returns the number of a protocol named in protoNames.
func protoByName(name string) (uint8, bool) {
for _, p := range protoNames {
if p.name == name {
return p.proto, true
}
}
return 0, false
}
// protoName returns the name in protoNames of a packet's protocol, or "unknown" and its number. `any` is only
// a rule's protocol, so a packet's protocol 0 is unknown.
func protoName(proto uint8) string {
if proto != ProtoAny {
for _, p := range protoNames {
if p.proto == proto {
return p.name
}
}
}
return fmt.Sprintf("unknown %v", proto)
}
// ParsedPacket is a Packet plus the parse byproducts the RX path reuses
type ParsedPacket struct {
Packet
-258
View File
@@ -1,258 +0,0 @@
package firewall
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"hash/fnv"
"log/slog"
"math"
"net/netip"
"slices"
"strings"
"github.com/slackhq/nebula/iputil"
)
// RulesBuilder collects the inbound and outbound rules, Build turns them into a Table for each direction
type RulesBuilder struct {
// in and out are each direction's rules in the order they were added
in, out []rule
// defaultLocalCIDRAny makes rules without a local cidr allow any local address, even with unsafe networks
defaultLocalCIDRAny bool
// ruleText describes every rule as it is added, one line each. Build hashes it; see Rules.
ruleText strings.Builder
l *slog.Logger
}
func NewRulesBuilder(l *slog.Logger) *RulesBuilder {
return &RulesBuilder{l: l}
}
// AddRule adds a rule for incoming or outgoing traffic, on proto or on every protocol when proto is ProtoAny.
// The rule is parsed now, so a bad rule is reported when it's added rather than when the Tables are built.
func (b *RulesBuilder) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
// ICMPv6 rules are ICMP rules, and a packet of either protocol is checked against both
if proto == iputil.IPProtocolICMPv6 {
proto = iputil.IPProtocolICMP
}
if proto == iputil.IPProtocolICMP {
//ICMP traffic doesn't have ports, so we always coerce to "any", even if a value is provided
if startPort != PortAny {
b.l.Warn("ignoring port specification for ICMP firewall rule", "startPort", startPort)
}
startPort = PortAny
endPort = PortAny
}
r, err := parseRule(proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha)
if err != nil {
return err
}
if incoming {
b.in = append(b.in, r)
} else {
b.out = append(b.out, r)
}
// The rule hashes are of these lines, changing them changes every hash
fmt.Fprintf(&b.ruleText,
"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s\n",
incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha,
)
direction := "incoming"
if !incoming {
direction = "outgoing"
}
b.l.Info("Firewall rule added",
"firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha},
)
return nil
}
// Build turns the rules into a Table for each direction, and hashes them. Rules without a local cidr allow
// vpnNetworks when there are unsafe networks, otherwise any local address.
func (b *RulesBuilder) Build(vpnNetworks, unsafeNetworks []netip.Prefix) Rules {
var defaultLocal []netip.Prefix
if len(unsafeNetworks) > 0 && !b.defaultLocalCIDRAny {
// Never nil, even when there are no vpn networks, so that the rules allow nothing rather than everything.
defaultLocal = append([]netip.Prefix{}, vpnNetworks...)
}
text := []byte(b.ruleText.String())
sha := sha256.Sum256(text)
h := fnv.New32a()
h.Write(text)
return Rules{
In: buildTable(b.in, defaultLocal),
Out: buildTable(b.out, defaultLocal),
Hash: hex.EncodeToString(sha[:]),
HashFNV: h.Sum32(),
}
}
func parseRule(proto uint8, startPort, endPort int32, groups []string, host, cidr, localCidr, caName, caSha string) (rule, error) {
if startPort > endPort {
return rule{}, fmt.Errorf("start port was lower than end port")
}
if startPort < PortFragment || endPort > math.MaxUint16 {
return rule{}, fmt.Errorf("port range %d-%d is outside of %d-%d", startPort, endPort, PortFragment, math.MaxUint16)
}
// A range that includes port 0 is port `any`.
if startPort <= PortAny && PortAny <= endPort {
startPort, endPort = PortAny, PortAny
}
// A port rule on a protocol without ports could never match. ICMP rules were coerced to `any` by AddRule,
// and a proto `any` rule with a port applies to the protocols that have them.
if startPort > PortAny && proto != ProtoAny && !iputil.HasPorts(proto) {
return rule{}, fmt.Errorf("protocol %d has no ports, port must be any or fragment", proto)
}
r := rule{
proto: proto,
startPort: startPort,
endPort: endPort,
groups: groups,
host: host,
caName: caName,
caSha: caSha,
}
var anyCIDR bool
var err error
r.cidr, anyCIDR, err = parseCIDR(cidr)
if err != nil {
return rule{}, fmt.Errorf("cidr did not parse; %w", err)
}
// A rule with no group, host, or cidr allows any host, as does a rule with `any` for one of them.
r.remoteAny = anyCIDR || host == "any" || slices.Contains(groups, "any") ||
(len(groups) == 0 && host == "" && !r.cidr.IsValid())
var localCIDR netip.Prefix
localCIDR, r.localAny, err = parseCIDR(localCidr)
if err != nil {
return rule{}, fmt.Errorf("local_cidr did not parse; %w", err)
}
if localCIDR.IsValid() {
r.local = []netip.Prefix{localCIDR}
}
return r, nil
}
// buildTable turns one direction's rules into a Table. Rules without a local cidr allow defaultLocal,
// or any local address when defaultLocal is nil.
func buildTable(rules []rule, defaultLocal []netip.Prefix) *Table {
t := &Table{rules: make([]rule, len(rules))}
var hasOwn [256]bool
for id, r := range rules {
t.rules[id] = r.withDefaultLocal(defaultLocal)
if r.proto != ProtoAny {
hasOwn[r.proto] = true
}
}
// A protocol with rules of its own gets an index of those rules plus the proto `any` rules, so a packet
// checks a single index. The other protocols share an index of the proto `any` rules, one for protocols with
// ports and one for protocols without.
anyPorts := newProtoIndex(true, t.rules, ProtoAny)
anyNoPorts := newProtoIndex(false, t.rules, ProtoAny)
for proto := range t.protos {
switch {
case hasOwn[proto]:
t.protos[proto] = newProtoIndex(iputil.HasPorts(uint8(proto)), t.rules, uint8(proto))
case iputil.HasPorts(uint8(proto)):
t.protos[proto] = anyPorts
default:
t.protos[proto] = anyNoPorts
}
}
// ICMPv6 rules are ICMP rules, see AddRule, so ICMPv6 packets are checked against the ICMP index
t.protos[iputil.IPProtocolICMPv6] = t.protos[iputil.IPProtocolICMP]
return t
}
// newProtoIndex builds an index of the proto `any` rules and proto's own rules, for a protocol with or without
// ports, or returns nil when there are none. With ProtoAny, it indexes only the proto `any` rules.
func newProtoIndex(hasPorts bool, rules []rule, proto uint8) *protoIndex {
var pi *protoIndex
for id := range rules {
r := &rules[id]
if r.proto != ProtoAny && r.proto != proto {
continue
}
if pi == nil {
pi = &protoIndex{hasPorts: hasPorts, fragment: newRuleSet(len(rules))}
if hasPorts {
pi.byPort = newRuleSets(math.MaxUint16+1, len(rules))
} else {
pi.packet = newRuleSet(len(rules))
}
}
pi.add(r, id)
}
return pi
}
// add puts id in every set that r's port clause covers.
func (pi *protoIndex) add(r *rule, id int) {
switch r.startPort {
case PortAny:
pi.fragment.add(id)
if pi.hasPorts {
pi.addPorts(0, math.MaxUint16, id)
} else {
pi.packet.add(id)
}
case PortFragment:
pi.fragment.add(id)
default:
// A proto `any` rule with a port is indexed for the protocols that have ports, parseRule rejects a port on
// a protocol without them.
if pi.hasPorts {
pi.addPorts(int(r.startPort), int(r.endPort), id)
}
}
}
// addPorts puts id in the set of every port from lo through hi.
func (pi *protoIndex) addPorts(lo, hi, id int) {
for port := lo; port <= hi; port++ {
pi.byPort.at(port).add(id)
}
}
// parseCIDR parses a rule's cidr, which is empty, `any`, or a prefix
func parseCIDR(s string) (prefix netip.Prefix, isAny bool, err error) {
switch s {
case "":
return netip.Prefix{}, false, nil
case "any":
return netip.Prefix{}, true, nil
}
prefix, err = netip.ParsePrefix(s)
return prefix, false, err
}
// withDefaultLocal returns r allowing defaultLocal, or any local address when defaultLocal is nil,
// unless r has a local cidr of its own
func (r rule) withDefaultLocal(defaultLocal []netip.Prefix) rule {
if !r.localAny && r.local == nil {
r.localAny = defaultLocal == nil
r.local = defaultLocal
}
return r
}
-367
View File
@@ -1,367 +0,0 @@
package firewall
import (
"fmt"
"math"
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// rulesAt returns the rules that a packet on proto with the given port is checked against, in id order.
// port may also be PortAny or PortFragment.
func (t *Table) rulesAt(proto uint8, port int32) []*rule {
pi := t.protos[proto]
if pi == nil {
return nil
}
p := &Packet{Protocol: proto, Fragment: port == PortFragment}
if port > 0 {
p.LocalPort = uint16(port)
}
var rules []*rule
for id := range pi.whichMatch(p, true).all() {
rules = append(rules, &t.rules[id])
}
return rules
}
// oneRuleAt returns the single rule that rulesAt finds, and fails the test when there is not exactly one.
func oneRuleAt(t *testing.T, table *Table, proto uint8, key int32) *rule {
t.Helper()
rules := table.rulesAt(proto, key)
require.Len(t, rules, 1)
return rules[0]
}
func TestRulesBuilder_AddRule(t *testing.T) {
l := test.NewLogger()
ti := netip.MustParsePrefix("1.2.3.4/32")
ti6 := netip.MustParsePrefix("fd12::34/128")
rb := NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, []string{}, "", "", "", "", ""))
rules := rb.Build(nil, nil)
// An empty rule is any
r := oneRuleAt(t, rules.In, iputil.IPProtocolTCP, 1)
assert.True(t, r.remoteAny)
assert.Empty(t, r.groups)
assert.Empty(t, r.host)
// It applies only to its own port.
assert.Empty(t, rules.In.rulesAt(iputil.IPProtocolTCP, 2))
assert.Empty(t, rules.In.rulesAt(iputil.IPProtocolTCP, PortAny))
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolUDP, 1, 1, []string{"g1"}, "", "", "", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.In, iputil.IPProtocolUDP, 1)
assert.False(t, r.remoteAny)
assert.Equal(t, []string{"g1"}, r.groups)
assert.Empty(t, r.host)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolICMP, 1, 1, []string{}, "h1", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolICMPv6, 1, 1, []string{}, "h2", "", "", "", ""))
rules = rb.Build(nil, nil)
// An ICMP rule is port `any` regardless of the port given, and ICMP and ICMPv6 share one set of rules.
for _, proto := range []uint8{iputil.IPProtocolICMP, iputil.IPProtocolICMPv6} {
rs := rules.In.rulesAt(proto, PortAny)
require.Len(t, rs, 2, "proto %d", proto)
assert.False(t, rs[0].remoteAny)
assert.Equal(t, "h1", rs[0].host)
assert.Equal(t, "h2", rs[1].host)
}
assert.Same(t, rules.In.protos[iputil.IPProtocolICMP], rules.In.protos[iputil.IPProtocolICMPv6])
// Any other protocol number gets its own rules
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, 47, 0, 0, []string{"g1"}, "", "", "", "", "")) // GRE
rules = rb.Build(nil, nil)
assert.Equal(t, []string{"g1"}, oneRuleAt(t, rules.In, 47, PortAny).groups)
assert.Nil(t, rules.In.protos[iputil.IPProtocolTCP])
// A proto `any` rule with a port applies to every protocol with ports, and ProtoAny itself has none.
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", ti.String(), "", "", ""))
rules = rb.Build(nil, nil)
assert.Empty(t, rules.Out.rulesAt(ProtoAny, 1))
r = oneRuleAt(t, rules.Out, iputil.IPProtocolTCP, 1)
assert.False(t, r.remoteAny)
assert.Equal(t, ti, r.cidr)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", ti6.String(), "", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, iputil.IPProtocolTCP, 1)
assert.False(t, r.remoteAny)
assert.Equal(t, ti6, r.cidr)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", "", ti.String(), "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, iputil.IPProtocolTCP, 1)
assert.True(t, r.remoteAny)
assert.False(t, r.localAny)
assert.Equal(t, []netip.Prefix{ti}, r.local)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 1, 1, []string{}, "", "", ti6.String(), "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, iputil.IPProtocolTCP, 1)
assert.True(t, r.remoteAny)
assert.False(t, r.localAny)
assert.Equal(t, []netip.Prefix{ti6}, r.local)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolUDP, 1, 1, []string{"g1"}, "", "", "", "ca-name", ""))
rules = rb.Build(nil, nil)
assert.Equal(t, "ca-name", oneRuleAt(t, rules.In, iputil.IPProtocolUDP, 1).caName)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, iputil.IPProtocolUDP, 1, 1, []string{"g1"}, "", "", "", "", "ca-sha"))
rules = rb.Build(nil, nil)
assert.Equal(t, "ca-sha", oneRuleAt(t, rules.In, iputil.IPProtocolUDP, 1).caSha)
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "any", "", "", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, oneRuleAt(t, rules.Out, ProtoAny, 0).remoteAny)
// 0.0.0.0/0 allows any IPv4 address, but no IPv6 address.
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "0.0.0.0/0", "", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, ProtoAny, 0)
assert.False(t, r.remoteAny)
assert.True(t, r.matchRemote(netip.MustParseAddr("1.1.1.1"), nil))
assert.False(t, r.matchRemote(netip.MustParseAddr("9::9"), nil))
// ::/0 allows any IPv6 address, but no IPv4 address.
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "::/0", "", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, ProtoAny, 0)
assert.False(t, r.remoteAny)
assert.True(t, r.matchRemote(netip.MustParseAddr("9::9"), nil))
assert.False(t, r.matchRemote(netip.MustParseAddr("1.1.1.1"), nil))
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "any", "", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, oneRuleAt(t, rules.Out, ProtoAny, 0).remoteAny)
// A local cidr is likewise limited to its own address family.
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "", "0.0.0.0/0", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, ProtoAny, 0)
assert.False(t, r.localAny)
assert.True(t, r.matchLocal(netip.MustParseAddr("1.1.1.1")))
assert.False(t, r.matchLocal(netip.MustParseAddr("9::9")))
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "", "::/0", "", ""))
rules = rb.Build(nil, nil)
r = oneRuleAt(t, rules.Out, ProtoAny, 0)
assert.False(t, r.localAny)
assert.True(t, r.matchLocal(netip.MustParseAddr("9::9")))
assert.False(t, r.matchLocal(netip.MustParseAddr("1.1.1.1")))
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(false, ProtoAny, 0, 0, []string{}, "", "", "any", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, oneRuleAt(t, rules.Out, ProtoAny, 0).localAny)
// Every protocol number is accepted
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, math.MaxUint8, 0, 0, []string{}, "", "", "", "", ""))
rules = rb.Build(nil, nil)
assert.True(t, oneRuleAt(t, rules.In, math.MaxUint8, PortAny).remoteAny)
// Test error conditions, a bad rule is reported when it's added
rb = NewRulesBuilder(l)
require.Error(t, rb.AddRule(true, ProtoAny, 10, 0, []string{}, "", "", "", "", ""))
require.Error(t, rb.AddRule(true, ProtoAny, PortFragment-1, 0, []string{}, "", "", "", "", ""))
require.Error(t, rb.AddRule(true, ProtoAny, 1, math.MaxUint16+1, []string{}, "", "", "", "", ""))
require.Error(t, rb.AddRule(true, ProtoAny, 0, 0, []string{}, "", "junk", "", "", ""))
require.Error(t, rb.AddRule(true, ProtoAny, 0, 0, []string{}, "", "", "junk", "", ""))
// ICMP and ICMPv6 share one copy of each proto `any` rule
rb = NewRulesBuilder(l)
require.NoError(t, rb.AddRule(true, ProtoAny, PortAny, PortAny, []string{"default-group"}, "", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolICMPv6, PortAny, PortAny, []string{"nope"}, "", "", "", "", ""))
rules = rb.Build(nil, nil)
assert.Len(t, rules.In.rules, 2)
assert.Len(t, rules.In.rulesAt(iputil.IPProtocolICMP, PortAny), 2)
assert.Len(t, rules.In.rulesAt(iputil.IPProtocolICMPv6, PortAny), 2)
// No rules allow nothing
rules = NewRulesBuilder(l).Build(nil, nil)
for proto := range rules.In.protos {
assert.Nil(t, rules.In.protos[proto], "proto %d", proto)
assert.Nil(t, rules.Out.protos[proto], "proto %d", proto)
}
}
// TestRulesBuilder_protoIndex checks which packets each kind of port clause covers, and which protocols
// share an index.
func TestRulesBuilder_protoIndex(t *testing.T) {
rb := NewRulesBuilder(test.NewLogger())
require.NoError(t, rb.AddRule(true, ProtoAny, PortAny, PortAny, nil, "any", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 22, 22, nil, "port", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1024, math.MaxUint16, nil, "range", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, PortFragment, PortFragment, nil, "fragment", "", "", "", ""))
// A range that includes port 0 is port `any`.
require.NoError(t, rb.AddRule(true, iputil.IPProtocolUDP, 0, 100, nil, "zero", "", "", "", ""))
// GRE has no ports, so a port rule on it is rejected rather than never applying, and `any` and `fragment` apply
require.Error(t, rb.AddRule(true, 47, 22, 22, nil, "gre", "", "", "", ""))
require.NoError(t, rb.AddRule(true, 47, PortFragment, PortFragment, nil, "gre", "", "", "", ""))
rules := rb.Build(nil, nil)
hosts := func(proto uint8, key int32) []string {
var hosts []string
for _, r := range rules.In.rulesAt(proto, key) {
hosts = append(hosts, r.host)
}
return hosts
}
assert.Equal(t, []string{"any", "fragment"}, hosts(iputil.IPProtocolTCP, PortFragment))
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolTCP, PortAny))
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolTCP, 21))
assert.Equal(t, []string{"any", "port"}, hosts(iputil.IPProtocolTCP, 22))
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolTCP, 23))
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolTCP, 1023))
assert.Equal(t, []string{"any", "range"}, hosts(iputil.IPProtocolTCP, 1024))
assert.Equal(t, []string{"any", "range"}, hosts(iputil.IPProtocolTCP, math.MaxUint16))
assert.Equal(t, []string{"any", "zero"}, hosts(iputil.IPProtocolUDP, PortFragment))
assert.Equal(t, []string{"any", "zero"}, hosts(iputil.IPProtocolUDP, PortAny))
assert.Equal(t, []string{"any", "zero"}, hosts(iputil.IPProtocolUDP, math.MaxUint16))
// A protocol without ports sees only the port `any` rules, and the port `fragment` rules for a fragment.
assert.False(t, rules.In.protos[47].hasPorts)
assert.Equal(t, []string{"any"}, hosts(47, PortAny))
assert.Equal(t, []string{"any"}, hosts(47, 22))
assert.Equal(t, []string{"any", "gre"}, hosts(47, PortFragment))
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolICMP, PortFragment))
// Protocols without rules of their own share an index of the proto `any` rules: one for protocols with
// ports and one for protocols without.
assert.True(t, rules.In.protos[iputil.IPProtocolSCTP].hasPorts)
assert.Equal(t, []string{"any"}, hosts(iputil.IPProtocolSCTP, 5000))
assert.Same(t, rules.In.protos[iputil.IPProtocolSCTP], rules.In.protos[iputil.IPProtocolDCCP])
assert.NotSame(t, rules.In.protos[iputil.IPProtocolSCTP], rules.In.protos[iputil.IPProtocolTCP])
assert.Equal(t, []string{"any"}, hosts(50, PortAny))
assert.Same(t, rules.In.protos[ProtoAny], rules.In.protos[50])
assert.Same(t, rules.In.protos[ProtoAny], rules.In.protos[iputil.IPProtocolICMP])
assert.NotSame(t, rules.In.protos[ProtoAny], rules.In.protos[iputil.IPProtocolSCTP])
assert.Nil(t, rules.Out.protos[iputil.IPProtocolTCP])
}
// TestRulesBuilder_manyRules checks that rule ids beyond the first byte of a rule set are indexed correctly.
func TestRulesBuilder_manyRules(t *testing.T) {
rb := NewRulesBuilder(test.NewLogger())
const n = 130
for i := range n {
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, int32(i+1), int32(i+1), nil, fmt.Sprintf("h%d", i), "", "", "", ""))
}
require.NoError(t, rb.AddRule(true, ProtoAny, PortAny, PortAny, nil, "any", "", "", "", ""))
rules := rb.Build(nil, nil)
assert.Equal(t, ruleSetLen(n+1), rules.In.protos[iputil.IPProtocolTCP].byPort.setLen)
// Ids are the order the rules were added in, so the proto `any` rule comes last
for i := range n {
rs := rules.In.rulesAt(iputil.IPProtocolTCP, int32(i+1))
require.Len(t, rs, 2, "port %d", i+1)
assert.Equal(t, fmt.Sprintf("h%d", i), rs[0].host)
assert.Equal(t, "any", rs[1].host)
}
assert.Len(t, rules.In.rulesAt(iputil.IPProtocolTCP, n+1), 1)
}
// TestRulesBuilder_defaultLocalCIDR checks which local addresses rules allow when there are unsafe networks.
// A rule without a local cidr allows only the vpn networks, and a rule with one allows only that cidr.
func TestRulesBuilder_defaultLocalCIDR(t *testing.T) {
vpnNetworks := []netip.Prefix{
netip.MustParsePrefix("10.1.0.5/16"),
netip.MustParsePrefix("10.2.0.5/16"),
netip.MustParsePrefix("fd00:1::5/64"),
}
unsafeNetworks := []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")}
vpn1 := netip.MustParseAddr("10.1.9.9")
vpn2 := netip.MustParseAddr("10.2.9.9")
vpn6 := netip.MustParseAddr("fd00:1::9")
h1Extra := netip.MustParseAddr("10.3.0.1")
h1Extra6 := netip.MustParseAddr("fd00:2::1")
h2Own := netip.MustParseAddr("10.4.0.1")
rb := NewRulesBuilder(test.NewLogger())
// h1 allows the vpn networks by default on ports 1 and 2, and 10.3.0.0/16 and fd00:2::/64 as well on port 1
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 2, nil, "h1", "", "", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, nil, "h1", "", "10.3.0.0/16", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, nil, "h1", "", "fd00:2::/64", "", ""))
// h2 allows 10.4.0.0/16 on ports 1 and 2, and the vpn networks by default as well on port 1
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 2, nil, "h2", "", "10.4.0.0/16", "", ""))
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 1, 1, nil, "h2", "", "", "", ""))
// h3 only has the default
require.NoError(t, rb.AddRule(true, iputil.IPProtocolTCP, 3, 3, nil, "h3", "", "", "", ""))
rules := rb.Build(vpnNetworks, unsafeNetworks)
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
caPool := cert.NewCAPool()
require.NoError(t, caPool.AddCA(ca))
certs := map[string]*cert.CachedCertificate{}
for _, name := range []string{"h1", "h2", "h3"} {
crt, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, name, time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.9/16")}, nil, nil)
c, err := caPool.VerifyCertificate(time.Now(), crt)
require.NoError(t, err)
certs[name] = c
}
for _, tc := range []struct {
port uint16
host string
addr netip.Addr
want bool
}{
{1, "h1", vpn1, true},
{1, "h1", vpn2, true},
{1, "h1", vpn6, true},
{1, "h1", h1Extra, true},
{1, "h1", h1Extra6, true},
{1, "h1", h2Own, false},
{2, "h1", vpn1, true},
{2, "h1", h1Extra, false},
{2, "h1", h1Extra6, false},
{1, "h2", h2Own, true},
{1, "h2", vpn1, true},
{1, "h2", vpn2, true},
{1, "h2", vpn6, true},
{1, "h2", h1Extra, false},
{2, "h2", h2Own, true},
{2, "h2", vpn1, false},
{2, "h2", vpn6, false},
{3, "h3", vpn1, true},
{3, "h3", vpn2, true},
{3, "h3", vpn6, true},
{3, "h3", h1Extra, false},
{3, "h3", h1Extra6, false},
{3, "h3", h2Own, false},
// Only h3 is allowed on port 3.
{3, "h1", vpn1, false},
} {
p := &Packet{Protocol: iputil.IPProtocolTCP, LocalPort: tc.port, LocalAddr: tc.addr, RemoteAddr: netip.MustParseAddr("10.1.0.9")}
assert.Equal(t, tc.want, rules.In.Match(p, true, certs[tc.host], caPool), "port %d, %s, %s", tc.port, tc.host, tc.addr)
}
}
-60
View File
@@ -1,60 +0,0 @@
package firewall
import (
"iter"
"math/bits"
)
// ruleSet is a set of rule ids, stored as a bitset. s[i] holds ids i*8 through i*8+7, one per bit.
// Bytes rather than words keep a set to the size the rule count needs, which matters when there is a set for
// every port number.
type ruleSet []uint8
// ruleSetLen returns the length of a ruleSet that can hold ids below n.
func ruleSetLen(n int) int {
return (n + 7) / 8
}
// newRuleSet returns an empty set that can hold ids below idLimit.
func newRuleSet(idLimit int) ruleSet {
return make(ruleSet, ruleSetLen(idLimit))
}
// add puts id in s.
func (s ruleSet) add(id int) {
s[id/8] |= 1 << (id % 8)
}
// ruleSets holds several ruleSet objects of the same length in a single allocation.
type ruleSets struct {
// setLen is the number of bytes needed to represent one set
setLen int
// bits holds the sets in order. Set i starts at i*setLen.
bits []uint8
}
// newRuleSets returns n empty sets, each of which can hold ids below idLimit.
func newRuleSets(n, idLimit int) ruleSets {
setLen := ruleSetLen(idLimit)
return ruleSets{setLen: setLen, bits: make([]uint8, n*setLen)}
}
// at returns set i.
func (s ruleSets) at(i int) ruleSet {
return s.bits[i*s.setLen : (i+1)*s.setLen]
}
// all returns an iterator over the ids in s, in ascending order.
func (s ruleSet) all() iter.Seq[int] {
return func(yield func(int) bool) {
for i, ids := range s {
for ids != 0 {
id := i*8 + bits.TrailingZeros8(ids)
ids &= ids - 1 // Clear the lowest set bit, which is id's.
if !yield(id) {
return
}
}
}
}
}
-189
View File
@@ -1,189 +0,0 @@
package firewall
import (
"net/netip"
"github.com/slackhq/nebula/cert"
)
// Rules are the Tables built for each direction, with hashes of the rules they were built from
type Rules struct {
In, Out *Table
// Hash is a sha256 of the rules, HashFNV is an FNV-1a of them for use as a metric value.
// Rule sets with the same rules have the same hashes.
Hash string
HashFNV uint32
}
// Table holds the rules for one direction. A packet is allowed when every clause of a single rule allows it:
// proto AND port AND (CA SHA or CA name) AND local CIDR AND (group OR groups OR name OR remote CIDR).
// A Table does not change once built; see RulesBuilder.
type Table struct {
// rules holds every rule for the direction, in the order they were added. A rule's index in this slice is
// its id in the ruleSets.
rules []rule
// protos holds the index for each IP protocol number, so a packet is checked against a single index.
// A protocol with rules of its own has an index that also includes every proto `any` rule. The remaining
// protocols share an index of only the proto `any` rules: one for protocols with ports and one for
// protocols without; see hasPorts. An entry is nil when the protocol has no rules at all.
// ICMP and ICMPv6 share one index.
protos [256]*protoIndex
}
// protoIndex holds the rules that can apply to one protocol, in a set for each kind of packet. Rule ids are
// indexes into the direction's rules, so every set is sized by the direction's rule count, not the protocol's:
// byPort costs 64 KB per 8 rules in the direction.
type protoIndex struct {
// hasPorts reports whether the protocol has ports; see iputil.HasPorts.
hasPorts bool
// byPort has a set for every port number of a protocol with ports. Port 0 is covered only by port `any`
// rules, since a range starts at 1.
byPort ruleSets
// packet holds the rules for a packet of a protocol without ports: the port `any` rules.
packet ruleSet
// fragment holds the rules for a fragment of any protocol: the port `any` and port `fragment` rules.
fragment ruleSet
}
// whichMatch returns the set of rules whose proto and port clauses allow p.
func (pi *protoIndex) whichMatch(p *Packet, incoming bool) ruleSet {
switch {
case p.Fragment:
// The ports of a fragmented packet are in its first fragment, and this is a later one.
return pi.fragment
case !pi.hasPorts:
// The port fields are zero, or for ICMP the identifier, which is only for connection tracking.
return pi.packet
case incoming:
return pi.byPort.at(int(p.LocalPort))
default:
return pi.byPort.at(int(p.RemotePort))
}
}
// rule is a parsed rule. A packet is allowed by the rule when its port, certificate, and address clauses all pass.
type rule struct {
// proto is the IP protocol number the rule applies to, or ProtoAny for every protocol. ICMPv6 rules are kept
// as ICMP. The protoIndex checks it, so match does not.
proto uint8
// startPort and endPort are the port clause. A startPort of PortAny covers every port, every packet of a
// protocol without ports, and every fragment. PortFragment covers only fragments. A range covers the ports
// in it, so it never applies to a protocol without ports.
startPort, endPort int32
// caSha and caName pass a certificate issued by either CA. When neither is set, any CA passes.
caSha, caName string
// remoteAny makes host, groups, and cidr irrelevant. Otherwise, the peer passes when any one of them matches.
remoteAny bool
host string
groups []string
cidr netip.Prefix // Invalid when the rule has no cidr.
// localAny allows any local address. Otherwise, local must contain the address. Neither is set for a rule
// without a local cidr until the Table is built and the rule gets the default; see withDefaultLocal.
localAny bool
local []netip.Prefix
}
// Match reports whether a rule for the direction p is going allows it
func (r *Rules) Match(p *Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
t := r.Out
if incoming {
t = r.In
}
return t.Match(p, incoming, c, caPool)
}
// Match reports whether a rule allows p
func (t *Table) Match(p *Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
proto := t.protos[p.Protocol]
if proto == nil {
// The protocol has no rules of its own, and there are no proto `any` rules.
return false
}
// The index yields every rule whose proto and port clauses allow p.
// The packet is allowed when the remaining clauses of any of those rules allow it as well.
for id := range proto.whichMatch(p, incoming).all() {
if t.rules[id].match(p, c, caPool) {
return true
}
}
return false
}
// match reports whether r allows p from the peer with certificate c. The port clause is not checked here; the
// protoIndex has already done so.
func (r *rule) match(p *Packet, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
return r.matchLocal(p.LocalAddr) && r.matchRemote(p.RemoteAddr, c) && r.matchCA(c, caPool)
}
// matchLocal reports whether r allows the local address addr.
func (r *rule) matchLocal(addr netip.Addr) bool {
if r.localAny {
return true
}
for _, prefix := range r.local {
if prefix.Contains(addr) {
return true
}
}
return false
}
// matchRemote reports whether r allows the peer with certificate c, sending from addr. The cidr is checked with
// Prefix.Contains, so 0.0.0.0/0 allows any IPv4 address and ::/0 allows any IPv6 address; neither allows the
// other family.
func (r *rule) matchRemote(addr netip.Addr, c *cert.CachedCertificate) bool {
if r.remoteAny {
return true
}
if r.cidr.IsValid() && r.cidr.Contains(addr) {
return true
}
if r.host != "" && r.host == c.Certificate.Name() {
return true
}
// Every group must be present.
if len(r.groups) == 0 {
return false
}
for _, g := range r.groups {
if _, ok := c.InvertedGroups[g]; !ok {
return false
}
}
return true
}
// matchCA reports whether c was issued by a CA that r allows.
func (r *rule) matchCA(c *cert.CachedCertificate, caPool *cert.CAPool) bool {
if r.caSha == "" && r.caName == "" {
return true
}
if r.caSha != "" && r.caSha == c.Certificate.Issuer() {
return true
}
if r.caName != "" {
s, err := caPool.GetCAForCert(c.Certificate)
if err != nil {
return false
}
return r.caName == s.Certificate.Name()
}
return false
}
+549 -333
View File
File diff suppressed because it is too large. Load diff
+17 -1
View File
@@ -69,7 +69,9 @@ type HostMap struct {
Hosts map[netip.Addr]*HostInfo
moreHosts map[netip.Addr][]*HostInfo
preferredRanges atomic.Pointer[[]netip.Prefix]
l *slog.Logger
// established hears about each hostinfo that joins or leaves the hostmap; nil when no conn wants to know.
established *establishedRemotes
l *slog.Logger
}
// For synchronization, treat the pointed-to Relay struct as immutable. To edit the Relay
@@ -276,6 +278,11 @@ type HostInfo struct {
// This value will be behind against actual tunnel utilization in the hot path.
// This should only be used by the ConnectionManagers ticker routine.
lastUsed time.Time
// establishedIn is set while the hostinfo is in a main hostmap whose conns track established remotes, and
// establishedRemote is the remote it holds a reference on there, guarded by establishedIn's mu.
establishedIn atomic.Pointer[establishedRemotes]
establishedRemote netip.AddrPort
}
type ViaSender struct {
@@ -516,6 +523,7 @@ func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) bool {
if len(hm.Indexes) == 0 {
hm.Indexes = map[uint32]*HostInfo{}
}
hm.established.remove(hostinfo)
if hm.l.Enabled(context.Background(), slog.LevelDebug) {
hm.l.Debug("Hostmap hostInfo deleted",
@@ -662,6 +670,7 @@ func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
hm.Indexes[hostinfo.localIndexId] = hostinfo
hm.RemoteIndexes[hostinfo.remoteIndexId] = hostinfo
hm.established.add(hostinfo)
hostinfo.markOut(f.rebindEpoch.Load())
if f.connectionManager != nil { // f.connectionManager is only nil in some unit tests
@@ -842,6 +851,13 @@ func (i *HostInfo) SetRemote(remote netip.AddrPort) {
if i.GetRemote() != remote {
i.remote.Store(&remote)
i.remotes.LearnRemote(i.vpnAddrs[0], remote)
if e := i.establishedIn.Load(); e != nil {
e.mu.Lock()
if i.establishedIn.Load() == e {
e.syncLocked(i)
}
e.mu.Unlock()
}
}
}
-13
View File
@@ -32,23 +32,10 @@ const (
IPProtocolICMPv6 = 58
IPProtocolTCP = 6
IPProtocolUDP = 17
IPProtocolUDPLite = 136
IPProtocolDCCP = 33
IPProtocolSCTP = 132
ICMPv6TypeEchoRequest = 128
ICMPv6TypeEchoReply = 129
)
// HasPorts reports whether packets of proto carry port numbers that the packet parsers read: TCP, UDP, UDPLite,
// DCCP, and SCTP. Firewall rules with a port or a port range only apply to these protocols.
func HasPorts(proto uint8) bool {
switch proto {
case IPProtocolTCP, IPProtocolUDP, IPProtocolUDPLite, IPProtocolDCCP, IPProtocolSCTP:
return true
}
return false
}
func CreateRejectPacket(packet []byte, out []byte) []byte {
if len(packet) < 1 {
return nil
+1
View File
@@ -213,6 +213,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
}
hostMap := NewHostMapFromConfig(l, c)
hostMap.established = newEstablishedRemotes(udpConns)
punchy := NewPunchyFromConfig(l, c, udpConns[0])
connManager := newConnectionManagerFromConfig(l, c, hostMap, punchy)
lightHouse, err := NewLightHouseFromConfig(ctx, l, c, pki.getCertState(), udpConns[0], punchy)
+25 -25
View File
@@ -361,29 +361,31 @@ func parseV6(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
fp.Fragment = isFragment
fp.FragAny = anyFragment
fp.IPHdrLen = offset
// fp is reused, so the ports are cleared for the packets that have none: fragments, protocols Nebula
// doesn't inspect, and ICMPv6 other than echo. Only the cases below set them.
fp.RemotePort, fp.LocalPort = 0, 0
if isFragment {
// Non-first fragments carry no transport header, so we have no ports to read
fp.RemotePort = 0
fp.LocalPort = 0
return nil
}
switch {
case proto == iputil.IPProtocolICMPv6:
switch proto {
case iputil.IPProtocolICMPv6:
// An ICMPv6 message is at least type, code and checksum, 4 bytes. Only echo carries more than we read.
if dataLen < offset+4 {
return ErrIPv6PacketTooShort
}
fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6
switch data[offset] { //icmp type
case iputil.ICMPv6TypeEchoRequest, iputil.ICMPv6TypeEchoReply:
if dataLen < offset+6 {
return ErrIPv6PacketTooShort
}
fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier, orientation doesn't matter
fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier
default:
fp.RemotePort = 0
}
case iputil.HasPorts(proto):
case iputil.IPProtocolTCP, iputil.IPProtocolUDP:
if dataLen < offset+4 {
return ErrIPv6PacketTooShort
}
@@ -394,6 +396,11 @@ func parseV6(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
}
default:
// don't set ports for protocols Nebula doesn't inspect
fp.RemotePort = 0
fp.LocalPort = 0
}
return nil
@@ -446,25 +453,18 @@ func parseV4(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
fp.RemoteAddr, _ = netip.AddrFromSlice(data[16:20])
}
// fp is reused, so the ports are cleared for the packets that have none: fragments, and protocols Nebula
// doesn't inspect, whose first 4 bytes (GRE, IPIP, AH, etc.) are not ports. Only the cases below set them.
fp.RemotePort, fp.LocalPort = 0, 0
if fp.Fragment {
return nil
}
switch {
case fp.Protocol == iputil.IPProtocolICMP: //note that orientation doesn't matter on ICMP
fp.RemotePort = binary.BigEndian.Uint16(data[ihl+4 : ihl+6]) //identifier, LocalPort stays 0 but could be the code, data[ihl+1]
case iputil.HasPorts(fp.Protocol):
if incoming {
fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2]) //src port
fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
} else {
fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2]) //src port
fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
}
fp.RemotePort = 0
fp.LocalPort = 0
} else if fp.Protocol == iputil.IPProtocolICMP { //note that orientation doesn't matter on ICMP
fp.RemotePort = binary.BigEndian.Uint16(data[ihl+4 : ihl+6]) //identifier
fp.LocalPort = 0 //code would be uint16(data[ihl+1])
} else if incoming {
fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2]) //src port
fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
} else {
fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2]) //src port
fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
}
return nil
+21 -86
View File
@@ -77,7 +77,7 @@ func Test_newPacket(t *testing.T) {
// account for variable ip header length - outgoing
h = ipv4.Header{
Version: 1,
Protocol: iputil.IPProtocolUDP,
Protocol: 2,
Len: 100,
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
@@ -89,7 +89,7 @@ func Test_newPacket(t *testing.T) {
err = newPacket(b, false, p)
require.NoError(t, err)
assert.Equal(t, uint8(iputil.IPProtocolUDP), p.Protocol)
assert.Equal(t, uint8(2), p.Protocol)
assert.Equal(t, netip.MustParseAddr("10.0.0.1"), p.LocalAddr)
assert.Equal(t, netip.MustParseAddr("10.0.0.2"), p.RemoteAddr)
assert.Equal(t, uint16(6), p.RemotePort)
@@ -97,71 +97,6 @@ func Test_newPacket(t *testing.T) {
assert.False(t, p.Fragment)
}
// Test_newPacket_v4NoPorts ensures only protocols that lead with src/dst ports report them. The first 4 bytes of other protocols are not ports,
// and are often chosen by the sender: a GRE header's protocol type, an IPIP inner total length, a 6in4 flow label.
// Reading them as ports would let a `proto: any` rule with a port match tunneled traffic.
func Test_newPacket_v4NoPorts(t *testing.T) {
p := &firewall.ParsedPacket{}
for _, proto := range []uint8{4, 41, 47, 50, 51} { // IPIP, 6in4, GRE, ESP, AH
h := ipv4.Header{
Version: 4,
Len: ipv4.HeaderLen,
Protocol: int(proto),
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
}
b, err := h.Marshal()
require.NoError(t, err)
b = append(b, []byte{0x00, 0x00, 0x86, 0xdd}...) // a GRE header carrying IPv6 would read as port 34525
for _, incoming := range []bool{true, false} {
require.NoError(t, newPacket(b, incoming, p))
assert.Equal(t, proto, p.Protocol)
assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment)
}
}
}
// Test_newPacket_portedProtocols ensures UDP-Lite, DCCP, and SCTP report ports from the same leading
// src/dst port words as TCP and UDP, oriented the same way.
func Test_newPacket_portedProtocols(t *testing.T) {
p := &firewall.ParsedPacket{}
ports := []byte{0x13, 0x88, 0x00, 0x50} // src 5000, dst 80
for _, proto := range []uint8{iputil.IPProtocolUDPLite, iputil.IPProtocolDCCP, iputil.IPProtocolSCTP} {
h := ipv4.Header{
Version: 4,
Len: ipv4.HeaderLen,
Protocol: int(proto),
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
}
v4, err := h.Marshal()
require.NoError(t, err)
v4 = append(v4, ports...)
v6 := make([]byte, 40, 44)
v6[0] = 0x60
v6[6] = proto
v6[7] = 64
v6 = append(v6, ports...)
for _, b := range [][]byte{v4, v6} {
require.NoError(t, newPacket(b, true, p))
assert.Equal(t, proto, p.Protocol)
assert.Equal(t, uint16(5000), p.RemotePort)
assert.Equal(t, uint16(80), p.LocalPort)
require.NoError(t, newPacket(b, false, p))
assert.Equal(t, uint16(5000), p.LocalPort)
assert.Equal(t, uint16(80), p.RemotePort)
}
}
}
func Test_newPacket_v6(t *testing.T) {
p := &firewall.ParsedPacket{}
@@ -777,46 +712,46 @@ func Test_newPacket_v6ExtHeaderPastBuffer(t *testing.T) {
// Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized
// Next Header as if it were an ipv6 extension header. A real upper layer protocol Nebula doesn't
// dissect (GRE here) is not walkable, so applying the (len+1)*8 formula marched into the GRE
// dissect (SCTP here) is not walkable, so applying the (len+1)*8 formula marched into the SCTP
// payload and landed on a byte that looked like UDP, forging a protocol/port pair the firewall
// would trust while the host delivered the real GRE packet. The fix fails closed: the packet
// would trust while the host delivered the real SCTP datagram. The fix fails closed: the packet
// is classified as its true protocol with no ports, so it only matches an `any` rule.
func Test_newPacket_v6ExtHeaderConfusion(t *testing.T) {
p := &firewall.ParsedPacket{}
pkt := make([]byte, 52)
pkt[0] = 0x60 // version 6
pkt[6] = byte(layers.IPProtocolGRE) // NextHeader = GRE, a real protocol, not an extension header
pkt[7] = 64 // hop limit
pkt[0] = 0x60 // version 6
pkt[6] = byte(layers.IPProtocolSCTP) // NextHeader = SCTP, a real protocol, not an extension header
pkt[7] = 64 // hop limit
// Real GRE header at offset 40. Pre-fix parseV6 walked GRE as an extension header: byte 41 (0x00, the
// low byte of the flags/version word below) was read as the header length, giving next=(0+1)*8=8, which
// landed the walk on byte 40 (0x11), misread as NextHeader=UDP, then bytes 48-51 as ports.
binary.BigEndian.PutUint16(pkt[40:42], 0x1100) // GRE flags/version; byte 40=0x11, byte 41=0x00
binary.BigEndian.PutUint16(pkt[42:44], 0x86dd) // GRE protocol type, never read by parseV6
binary.BigEndian.PutUint16(pkt[48:50], 53) // GRE payload, pre-fix forged RemotePort
// Real SCTP header at offset 40. Pre-fix parseV6 walked SCTP as an extension header: byte 41 (0x00, the
// low byte of the src port below) was read as the header length, giving next=(0+1)*8=8, which landed the
// walk on byte 40 (0x11), misread as NextHeader=UDP, then bytes 48-51 as ports.
binary.BigEndian.PutUint16(pkt[40:42], 0x1100) // SCTP src port; byte 40=0x11, byte 41=0x00
binary.BigEndian.PutUint16(pkt[42:44], 445) // SCTP dst port, never read by parseV6
binary.BigEndian.PutUint16(pkt[48:50], 53) // SCTP checksum bytes, pre-fix forged RemotePort
binary.BigEndian.PutUint16(pkt[50:52], 53) // pre-fix forged LocalPort
require.NoError(t, newPacket(pkt, true, p))
assert.Equal(t, uint8(layers.IPProtocolGRE), p.Protocol, "must classify as the true protocol, not the forged UDP")
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must classify as the true protocol, not the forged UDP")
assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment)
// Same confusion, but the unknown protocol sits after a real extension header. The HopByHop is walked
// correctly, then GRE must still fail closed instead of being walked into its own payload. Protocol is
// the only assertion that discriminates the fix here, a regression that walked GRE would misclassify it.
// correctly, then SCTP must still fail closed instead of being walked into its own payload. Protocol is
// the only assertion that discriminates the fix here, a regression that walked SCTP would misclassify it.
chained := make([]byte, 60)
chained[0] = 0x60 // version 6
chained[6] = byte(layers.IPProtocolIPv6HopByHop) // NextHeader = HopByHop extension
chained[7] = 64 // hop limit
chained[40] = byte(layers.IPProtocolGRE) // HopByHop NextHeader = GRE
chained[41] = 0 // HopByHop length 0 -> 8 bytes, GRE begins at offset 48
binary.BigEndian.PutUint16(chained[48:50], 0x1100) // GRE flags/version, pre-fix forged NextHeader/length bait
binary.BigEndian.PutUint16(chained[50:52], 0x86dd) // GRE protocol type, never read by parseV6
chained[40] = byte(layers.IPProtocolSCTP) // HopByHop NextHeader = SCTP
chained[41] = 0 // HopByHop length 0 -> 8 bytes, SCTP begins at offset 48
binary.BigEndian.PutUint16(chained[48:50], 0x1100) // SCTP src port, pre-fix forged NextHeader/length bait
binary.BigEndian.PutUint16(chained[50:52], 445) // SCTP dst port, never read by parseV6
require.NoError(t, newPacket(chained, true, p))
assert.Equal(t, uint8(layers.IPProtocolGRE), p.Protocol, "must fail closed on the unknown protocol after the extension header")
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must fail closed on the unknown protocol after the extension header")
assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment)
+8
View File
@@ -48,6 +48,14 @@ type Conn interface {
Close() error
}
// EstablishedPeerConn is a Conn that treats the remotes of established tunnels differently from other
// destinations, as darwin's gives busy ones connected sockets.
type EstablishedPeerConn interface {
// SetEstablishedPeer reports whether dst is the current remote of at least one established tunnel. Calls for
// one dst must not race each other.
SetEstablishedPeer(dst netip.AddrPort, established bool)
}
type NoopConn struct{}
func (NoopConn) Rebind() error {
+115
View File
@@ -0,0 +1,115 @@
//go:build !e2e_testing
package udp
import (
"log/slog"
"net"
"net/netip"
"testing"
"time"
)
// benchConnSocks opens a wildcard listener with ListenOut running, with the connected socket knobs pinned to
// their defaults and then cfg applied, and a loopback peer that never reads, so its sends cost only the sender.
func benchConnSocks(b *testing.B, s Settings, cfg func(*connSockConfig)) (*StdConn, netip.AddrPort) {
b.Helper()
for k, v := range map[string]string{
"NEBULA_CONNSOCKS": "8",
"NEBULA_CONNSOCK_RUN": "64",
"NEBULA_CONNSOCK_WINDOW": "1s",
"NEBULA_CONNSOCK_IDLE": "30s",
"NEBULA_CONNSOCK_OPEN_EVERY": "1s",
} {
b.Setenv(k, v)
}
c, err := NewListener(slog.New(slog.DiscardHandler), s)
if err != nil {
b.Fatal(err)
}
b.Cleanup(func() { _ = c.Close() })
u := c.(*StdConn)
if cfg != nil {
cfg(&u.socks.cfg)
}
go func() { _ = u.ListenOut(func(netip.AddrPort, []byte) {}, func() {}) }()
for u.reader.Load() == nil {
time.Sleep(time.Millisecond)
}
pc, err := net.ListenPacket("udp4", "127.0.0.1:0")
if err != nil {
b.Fatal(err)
}
b.Cleanup(func() { _ = pc.Close() })
return u, pc.LocalAddr().(*net.UDPAddr).AddrPort()
}
// BenchmarkConnSockWriteTo measures one WriteTo of a full-size datagram down each path: the listener with
// connected sockets off, to a destination that is no tunnel's remote while another is, to an established peer
// that never gets busy, and on a connected socket.
func BenchmarkConnSockWriteTo(b *testing.B) {
pkt := make([]byte, 1400)
cases := []struct {
name string
cfg func(*connSockConfig)
setup func(b *testing.B, u *StdConn, dst netip.AddrPort)
}{
{name: "listener/off", cfg: func(c *connSockConfig) { c.max = 0 }},
{name: "listener/ineligible", setup: func(b *testing.B, u *StdConn, dst netip.AddrPort) {
u.SetEstablishedPeer(netip.AddrPortFrom(dst.Addr(), dst.Port()+1), true)
}},
{name: "listener/eligible-not-busy", cfg: func(c *connSockConfig) { c.run = 1 << 30 },
setup: func(b *testing.B, u *StdConn, dst netip.AddrPort) { u.SetEstablishedPeer(dst, true) }},
{name: "connected", setup: func(b *testing.B, u *StdConn, dst netip.AddrPort) {
u.SetEstablishedPeer(dst, true)
for range u.socks.cfg.run {
_ = u.WriteTo(pkt, dst)
}
if u.connSockFor(dst) == nil {
b.Fatal("no connected socket")
}
}},
}
for _, c := range cases {
b.Run(c.name, func(b *testing.B) {
u, dst := benchConnSocks(b, Settings{Listen: netip.MustParseAddrPort("0.0.0.0:0")}, c.cfg)
if c.setup != nil {
c.setup(b, u, dst)
}
b.ReportAllocs()
b.ResetTimer()
for range b.N {
_ = u.WriteTo(pkt, dst)
}
})
}
}
// BenchmarkConnSockDispatch measures what WriteTo adds before the listener's sendto, without the syscall: a v4
// listener refuses a v6 destination before sending, with connected sockets off or with another peer established.
func BenchmarkConnSockDispatch(b *testing.B) {
pkt := make([]byte, 1400)
dst := netip.MustParseAddrPort("[::1]:4242")
for _, c := range []struct {
name string
cfg func(*connSockConfig)
establish bool
}{
{name: "off", cfg: func(c *connSockConfig) { c.max = 0 }},
{name: "ineligible", establish: true},
} {
b.Run(c.name, func(b *testing.B) {
u, peer := benchConnSocks(b, Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Multi: true}, c.cfg)
if c.establish {
u.SetEstablishedPeer(peer, true)
}
b.ReportAllocs()
b.ResetTimer()
for range b.N {
if u.WriteTo(pkt, dst) == nil {
b.Fatal("sent a v6 datagram on a v4 listener")
}
}
})
}
}
+803
View File
@@ -0,0 +1,803 @@
//go:build !e2e_testing
package udp
import (
"context"
"errors"
"fmt"
"log/slog"
"maps"
"net"
"net/netip"
"os"
"slices"
"strconv"
"sync"
"sync/atomic"
"syscall"
"time"
"golang.org/x/sys/unix"
)
// A connected socket is a UDP socket connected to one busy peer, on the listener's port and bound to the source
// address the listener's route to that peer picks, so the peer sees the same address and port from either. A send
// on a connected socket skips the per-datagram address and MAC policy checks, and the EndpointSecurity send check
// that xprotectd can hold every unconnected sendto on, and it is the only UDP send that reports the interface
// queue's flow control, as ENOBUFS. A connected socket wins xnu's lookup for its peer's datagrams; it never sees
// anyone else's.
//
// A connected socket sets SO_REUSEPORT and binds a specific address beside a wildcard listener, which sets
// SO_REUSEPORT only for listen routines. xnu refuses a wildcard SO_REUSEPORT bind beside a holder without the
// option, and refuses another uid's bind to an address a socket holds specifically, so no other user can join the
// port while connected sockets open and close: uid nobody got EADDRINUSE on every bind against a root listener
// with connected sockets in the 09-25 probes. A connected socket is closed, never disconnected, because xnu widens
// a disconnected socket's local address to the wildcard.
//
// Only the current remote of an established tunnel may get one: Nebula names those with SetEstablishedPeer, so an
// unauthenticated sender can't earn a connected socket by drawing replies such as recv_error or handshake
// responses.
type connSock struct {
dst netip.AddrPort
peer *connSockPeer
conn *net.UDPConn
rc syscall.RawConn
// used is the monotonic time in nanoseconds of the connected socket's last send or receive.
used atomic.Int64
// stallAt is the monotonic time of the first ENOBUFS or EAGAIN since a send last went through, or 0, and
// dropping is set once that stall has dropped a datagram, so each stall logs its drops once.
stallAt atomic.Int64
dropping atomic.Bool
// parkMu serializes the writes that wait for a full send buffer, since the write deadline bounding each wait
// is the descriptor's, not the writer's.
parkMu sync.Mutex
// sys writes one datagram on the descriptor: writeFd, or a stand-in for the interface in tests.
sys func(fd int, b []byte) error
}
func writeFd(fd int, b []byte) error {
_, err := syscall.Write(fd, b)
return err
}
// errStallSpent is a write that found the connected socket's stall budget spent, so its datagram is dropped.
var errStallSpent = errors.New("connected socket stall budget spent")
// connSockWriter carries one write through RawConn.Write. They are pooled with fn bound once, since a closure
// passed through the RawConn interface escapes and would allocate on every send.
type connSockWriter struct {
s *connSock
b []byte
park bool
err error
fn func(fd uintptr) bool
}
var connSockWriters = sync.Pool{New: func() any {
w := &connSockWriter{}
w.fn = w.write
return w
}}
func (w *connSockWriter) write(fd uintptr) bool {
for {
w.err = w.s.sys(int(fd), w.b)
switch {
case w.err == syscall.EINTR:
case w.err == syscall.EAGAIN && w.park:
return false
default:
return true
}
}
}
// try writes b on s once and returns nil, the write's errno, os.ErrDeadlineExceeded, or net.ErrClosed. With park
// set, EAGAIN waits in the netpoller until the socket is writable or the write deadline passes; without it, EAGAIN
// is returned.
func (s *connSock) try(b []byte, park bool) error {
w := connSockWriters.Get().(*connSockWriter)
w.s, w.b, w.park = s, b, park
rerr := s.rc.Write(w.fn)
err := w.err
*w = connSockWriter{fn: w.fn}
connSockWriters.Put(w)
switch {
case rerr == nil:
return err
case errors.Is(rerr, os.ErrDeadlineExceeded):
return os.ErrDeadlineExceeded
default:
return net.ErrClosed
}
}
// connSockConfig holds the knobs. connSockConfigFromEnv reads all but backoff and enobufsMax from the environment,
// so one binary can be tuned without a rebuild.
type connSockConfig struct {
// max is how many connected sockets the listen routines may have open at once; 0 turns connected sockets off.
max int
// run is how many datagrams to one destination within one window earn it a connected socket. darwin's tun
// hands over one packet per read, so a peer's sends arrive one WriteTo or one-datagram batch at a time and
// busyness has to be counted across calls.
run int
window time.Duration
// idle closes a connected socket after it has neither sent nor received for this long.
idle time.Duration
// backoff keeps a destination whose connected socket failed on the listener for this long.
backoff time.Duration
// openEvery is how often the open bucket earns back an open once max are spent, which bounds connected socket
// churn however fast senders earn connected sockets and errors close them.
openEvery time.Duration
// enobufsMax is how long a connected socket's sends may wait out ENOBUFS without one going through before they
// drop instead, so a suspended interface costs one write per datagram rather than stalling the tun reader.
enobufsMax time.Duration
}
func connSockConfigFromEnv() connSockConfig {
c := connSockConfig{
max: 8,
run: 64,
window: time.Second,
idle: 30 * time.Second,
backoff: 30 * time.Second,
openEvery: time.Second,
enobufsMax: 10 * time.Millisecond,
}
if v, err := strconv.Atoi(os.Getenv("NEBULA_CONNSOCKS")); err == nil {
c.max = v
}
if v, err := strconv.Atoi(os.Getenv("NEBULA_CONNSOCK_RUN")); err == nil && v > 0 {
c.run = v
}
if v, err := time.ParseDuration(os.Getenv("NEBULA_CONNSOCK_WINDOW")); err == nil && v > 0 {
c.window = v
}
if v, err := time.ParseDuration(os.Getenv("NEBULA_CONNSOCK_IDLE")); err == nil && v > 0 {
c.idle = v
}
if v, err := time.ParseDuration(os.Getenv("NEBULA_CONNSOCK_OPEN_EVERY")); err == nil && v > 0 {
c.openEvery = v
}
return c
}
// connSockGroups holds the connSocks that the listen routines bound to one address share, so the limits, the
// connected sockets and Rebind cover every routine, and a peer has at most one connected socket.
var connSockGroups = struct {
sync.Mutex
m map[netip.AddrPort]*connSocks
}{m: map[netip.AddrPort]*connSocks{}}
// joinConnSocks returns the connSocks u shares with the other listen routines bound to la, and makes it for the
// first.
func joinConnSocks(u *StdConn, la netip.AddrPort, s Settings) *connSocks {
g := &connSockGroups
g.Lock()
defer g.Unlock()
if cs := g.m[la]; s.Multi && cs != nil {
cs.mu.Lock()
cs.members = append(cs.members, u)
cs.mu.Unlock()
return cs
}
cs := &connSocks{cfg: connSockConfigFromEnv(), key: la, members: []*StdConn{u}, clock: monoNow, sleep: time.Sleep}
if cs.cfg.max > 0 && !u.listenHost.IsUnspecified() && !s.Multi {
// A connected socket on a specific listen.host binds the listener's own address, which xnu allows only when
// both sockets set SO_REUSEPORT, and a listener with it lets another uid bind a dual-stack wildcard on the
// port.
cs.off.Store(true)
u.l.Debug("connected sockets: off for a specific listen.host", "addr", s.Listen)
}
if s.Multi {
g.m[la] = cs
}
return cs
}
type connSocks struct {
cfg connSockConfig
// key is the listeners' bound address, which connSockGroups files this under.
key netip.AddrPort
// off keeps connected sockets from opening, when the listener's address can't be shared with them.
off atomic.Bool
// table maps each destination that is the remote of an established tunnel, and that a connected socket could
// reach, to its peer; it is nil when there are none, so a send with nothing to consider costs one load. Every
// send reads it without a lock, and mu guards copying it. An open that holds mu and still finds its peer here
// can't outlive the tunnel.
table atomic.Pointer[map[netip.AddrPort]*connSockPeer]
// reserved counts the connected sockets that are open, opening or closing, against cfg.max; mu guards changing it.
reserved atomic.Int32
mu sync.Mutex
closed bool
// members are the listen routines' listeners that share these connected sockets.
members []*StdConn
// pending holds each destination whose connected socket is opening or closing, and not on its peer. xnu
// refuses to connect a 4-tuple that is still bound, so a destination in pending can't open another.
pending map[netip.AddrPort]struct{}
// rebinds counts Rebinds, so an open that copied the listener's interface before one is discarded.
rebinds uint64
// closers tracks the goroutines that close detached connected sockets.
closers sync.WaitGroup
// done stops the reaper; it is made when ListenOut starts the connected sockets.
done chan struct{}
// opens bounds connected socket opens to cfg.max at once and one per cfg.openEvery after.
opens gcra
// logs bounds connected socket log lines at info or warn; see connSockLogBurst.
logs gcra
// clock returns monotonic nanoseconds and sleep waits out ENOBUFS: monoNow and time.Sleep, or stand-ins in
// tests.
clock func() int64
sleep func(time.Duration)
// dialHook and closeHook, when set, run just before an open's dials and a close's Close, outside mu; tests
// use them to hold an open or close in flight.
dialHook, closeHook func(dst netip.AddrPort)
}
// connSockPeer is what connSocks keeps about an established tunnel's remote: its connected socket, how busy it is,
// and how long a failed connected socket keeps it on the listener.
type connSockPeer struct {
sock atomic.Pointer[connSock]
// windowMu guards start, the monotonic time the current window began, and n, the datagrams sent in it, so a
// window's reset and its counts can't interleave with another sender's.
windowMu sync.Mutex
start int64
n int
// retryAt is the monotonic time before which a peer whose connected socket failed stays on the listener;
// connSocks.mu guards it.
retryAt int64
}
// busy counts n more datagrams to p and reports whether it has now sent cfg.run within one window. A busy peer's
// count starts over, so a peer that can't get a connected socket retries at most once per cfg.run sends.
func (p *connSockPeer) busy(now int64, n int, cfg *connSockConfig) bool {
p.windowMu.Lock()
defer p.windowMu.Unlock()
if time.Duration(now-p.start) > cfg.window {
p.start, p.n = now, 0
}
p.n += n
if p.n < cfg.run {
return false
}
p.n = 0
return true
}
// connSockLogBurst and connSockLogEvery size the bucket that lets connected socket opens, closes and failures log
// at info or warn; past it they log at debug, so churn can't flood the log.
const (
connSockLogBurst = 32
connSockLogEvery = 10 * time.Second
)
var monoStart = time.Now()
func monoNow() int64 { return int64(time.Since(monoStart)) }
// gcra is a token bucket kept as the monotonic time it is next full (the generic cell rate algorithm), so it needs
// no refill tick.
type gcra struct{ tat int64 }
// take spends a token from a bucket of burst that earns one back per every, and reports whether there was one to
// spend.
func (g *gcra) take(now int64, burst int, every time.Duration) bool {
e := int64(every)
tat := max(g.tat, now)
if tat-now > int64(burst-1)*e {
return false
}
g.tat = tat + e
return true
}
// logLevelLocked returns level while the log bucket has a token, and debug once it is spent.
func (cs *connSocks) logLevelLocked(level slog.Level) slog.Level {
if cs.logs.take(cs.clock(), connSockLogBurst, connSockLogEvery) {
return level
}
return slog.LevelDebug
}
// startConnSocks starts the idle reaper.
func (u *StdConn) startConnSocks() {
cs := u.socks
cs.mu.Lock()
defer cs.mu.Unlock()
if cs.closed || cs.done != nil || cs.cfg.max <= 0 || cs.off.Load() {
return
}
cs.done = make(chan struct{})
go u.reapConnSocks(cs.done)
}
// stopConnSocks closes every connected socket and keeps new ones from opening, once any listen routine's listener
// closes, as they do together at shutdown. It returns once their descriptors are closed.
func (u *StdConn) stopConnSocks() {
cs := u.socks
cs.mu.Lock()
cs.members = slices.DeleteFunc(cs.members, func(m *StdConn) bool { return m == u })
if !cs.closed {
cs.closed = true
if cs.done != nil {
close(cs.done)
}
u.detachAllLocked("close")
}
cs.mu.Unlock()
connSockGroups.Lock()
if connSockGroups.m[cs.key] == cs {
delete(connSockGroups.m, cs.key)
}
connSockGroups.Unlock()
cs.closers.Wait()
}
// siblings returns the other listen routines' listeners.
func (cs *connSocks) siblings(u *StdConn) []*StdConn {
cs.mu.Lock()
defer cs.mu.Unlock()
return slices.DeleteFunc(slices.Clone(cs.members), func(m *StdConn) bool { return m == u })
}
// connSockFor returns dst's connected socket, or nil.
func (u *StdConn) connSockFor(dst netip.AddrPort) *connSock {
if p := u.socks.peer(dst); p != nil {
return p.sock.Load()
}
return nil
}
func (cs *connSocks) peer(dst netip.AddrPort) *connSockPeer {
if t := cs.table.Load(); t != nil {
return (*t)[dst]
}
return nil
}
// SetEstablishedPeer makes dst eligible for a connected socket while it is the remote of an established tunnel,
// and closes the one it has once it no longer is. Nebula calls it for every listen routine under the hostmap's
// write lock, so repeats are no-ops and it makes no syscalls: the descriptor is closed by another goroutine.
func (u *StdConn) SetEstablishedPeer(dst netip.AddrPort, established bool) {
cs := u.socks
if cs.cfg.max <= 0 || cs.off.Load() || !u.connSockable(dst) {
return
}
cs.mu.Lock()
defer cs.mu.Unlock()
t := cs.table.Load()
p := cs.peer(dst)
if cs.closed || (p != nil) == established {
return
}
next := map[netip.AddrPort]*connSockPeer{}
if t != nil {
next = maps.Clone(*t)
}
if established {
p = &connSockPeer{start: cs.clock()}
next[dst] = p
} else {
delete(next, dst)
if s := p.sock.Load(); s != nil {
u.detachLocked(s, "tunnel gone", false)
}
}
cs.storeTableLocked(next)
}
// storeTableLocked publishes t, or nil for an empty one.
func (cs *connSocks) storeTableLocked(t map[netip.AddrPort]*connSockPeer) {
if len(t) == 0 {
cs.table.Store(nil)
return
}
cs.table.Store(&t)
}
// connSockable reports whether a connected socket to dst could send what the listener would. A v4 listener can't
// reach a v6 destination at all, and a connected socket bound to a specific listen.host must share its family.
func (u *StdConn) connSockable(dst netip.AddrPort) bool {
if !dst.IsValid() || dst.Port() == 0 {
return false
}
if u.isV4 {
return dst.Addr().Is4()
}
return u.listenHost.IsUnspecified() || u.listenHost.Unmap().Is4() == dst.Addr().Unmap().Is4()
}
// connSockRun sends bufs, all to p's destination, on p's connected socket, opening one if they make p busy. It
// returns how many went out; when that is short of len(bufs), err is the failure of bufs[sent], or nil to leave
// bufs[sent:] to the listener.
func (u *StdConn) connSockRun(p *connSockPeer, dst netip.AddrPort, bufs [][]byte) (sent int, err error) {
s := p.sock.Load()
if s == nil {
if s = u.maybeOpenConnSock(p, dst, len(bufs)); s == nil {
return 0, nil
}
}
sent, errno, closed := u.connSockWrite(s, bufs)
switch {
case sent == len(bufs):
case connSockDead(errno):
u.closeConnSock(s, errno.Error(), true)
case !closed:
return sent, &net.OpError{Op: "write", Err: errno}
}
return sent, nil
}
// maybeOpenConnSock counts n datagrams to p, and opens a connected socket to it once it is busy and nothing rules
// one out.
func (u *StdConn) maybeOpenConnSock(p *connSockPeer, dst netip.AddrPort, n int) *connSock {
cs := u.socks
if n <= 0 || u.reader.Load() == nil || int(cs.reserved.Load()) >= cs.cfg.max || !p.busy(cs.clock(), n, &cs.cfg) {
return nil
}
cs.mu.Lock()
if cs.closed || cs.peer(dst) != p {
cs.mu.Unlock()
return nil
}
if s := p.sock.Load(); s != nil {
cs.mu.Unlock()
return s
}
now := cs.clock()
if _, ok := cs.pending[dst]; ok || int(cs.reserved.Load()) >= cs.cfg.max || now < p.retryAt ||
!cs.opens.take(now, cs.cfg.max, cs.cfg.openEvery) {
cs.mu.Unlock()
return nil
}
if cs.pending == nil {
cs.pending = map[netip.AddrPort]struct{}{}
}
cs.pending[dst] = struct{}{}
cs.reserved.Add(1)
rebinds := cs.rebinds
cs.mu.Unlock()
if cs.dialHook != nil {
cs.dialHook(dst)
}
s, err := u.dialConnSock(p, dst)
cs.mu.Lock()
if err != nil {
delete(cs.pending, dst)
cs.reserved.Add(-1)
p.retryAt = cs.clock() + int64(cs.cfg.backoff)
level := cs.logLevelLocked(slog.LevelWarn)
cs.mu.Unlock()
u.l.Log(context.Background(), level, "connected socket: open failed, staying on the listener", "udpAddr", dst, "error", err)
return nil
}
if cs.closed || cs.rebinds != rebinds || cs.peer(dst) != p {
// The tunnel went away, or a Rebind or shutdown ran, while the socket was opening; it stays in pending until
// it is closed.
cs.mu.Unlock()
u.finishClose(s)
u.l.Debug("connected socket: discarded, its tunnel or route changed while it opened", "udpAddr", dst)
return nil
}
delete(cs.pending, dst)
p.sock.Store(s)
go u.readConnSock(s)
level := cs.logLevelLocked(slog.LevelInfo)
cs.mu.Unlock()
u.l.Log(context.Background(), level, "connected socket: opened", "udpAddr", dst, "local", s.conn.LocalAddr(), "open", cs.reserved.Load())
return s
}
// dialConnSock opens a socket on the listener's port with SO_REUSEPORT, bound to the configured listen.host or,
// for a wildcard listener, to the source address the route to dst picks, and connected to dst. It carries the
// listener's IP_BOUND_IF, so both route the same way.
func (u *StdConn) dialConnSock(p *connSockPeer, dst netip.AddrPort) (*connSock, error) {
raddr := net.UDPAddrFromAddrPort(netip.AddrPortFrom(dst.Addr().Unmap(), dst.Port()))
network := "udp6"
if raddr.IP.To4() != nil {
network = "udp4"
}
ifindex, err := u.boundIf()
if err != nil {
return nil, err
}
local := &net.UDPAddr{IP: u.listenHost.AsSlice(), Zone: u.listenHost.Zone()}
if u.listenHost.IsUnspecified() {
// A connected socket gets the source address xnu would pick for an unconnected send to dst.
d := net.Dialer{Control: connSockControl(false, ifindex)}
probe, err := d.Dial(network, raddr.String())
if err != nil {
return nil, err
}
local = probe.LocalAddr().(*net.UDPAddr)
_ = probe.Close()
}
local.Port = int(u.port)
d := net.Dialer{LocalAddr: local, Control: connSockControl(true, ifindex)}
c, err := d.Dial(network, raddr.String())
if err != nil {
return nil, err
}
uc := c.(*net.UDPConn)
rc, err := uc.SyscallConn()
if err != nil {
_ = uc.Close()
return nil, err
}
s := &connSock{dst: dst, peer: p, conn: uc, rc: rc, sys: writeFd}
s.used.Store(u.socks.clock())
return s, nil
}
// connSockControl sets SO_REUSEPORT when reuse is set, and IP_BOUND_IF or IPV6_BOUND_IF when ifindex isn't 0,
// before the socket binds.
func connSockControl(reuse bool, ifindex int) func(network, address string, c syscall.RawConn) error {
return func(network, _ string, c syscall.RawConn) error {
var serr error
err := c.Control(func(fd uintptr) {
if reuse {
serr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
}
if serr == nil && ifindex != 0 {
if network == "udp4" {
serr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_BOUND_IF, ifindex)
} else {
serr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_BOUND_IF, ifindex)
}
}
})
if err != nil {
return err
}
return serr
}
}
// boundIf returns the interface index the listener is scoped to, or 0. An error keeps the peer on the listener,
// since a connected socket opened without the listener's scope could route out an interface the listener can't.
func (u *StdConn) boundIf() (int, error) {
level, opt := unix.IPPROTO_IPV6, unix.IPV6_BOUND_IF
if u.isV4 {
level, opt = unix.IPPROTO_IP, unix.IP_BOUND_IF
}
v, err := unix.GetsockoptInt(int(u.sysFd), level, opt)
if err != nil {
return 0, fmt.Errorf("reading the listener's bound interface: %w", err)
}
return v, nil
}
// closeConnSock closes s if it is still open. With backoff set, the peer stays on the listener until the backoff
// runs out.
func (u *StdConn) closeConnSock(s *connSock, why string, backoff bool) {
u.socks.mu.Lock()
u.detachLocked(s, why, backoff)
u.socks.mu.Unlock()
}
// detachLocked takes s off its peer, if it is still there, and closes it on another goroutine, since callers such
// as SetEstablishedPeer hold locks that no socket syscall should run under. s.dst stays in pending until the close
// completes.
func (u *StdConn) detachLocked(s *connSock, why string, backoff bool) {
cs := u.socks
if !s.peer.sock.CompareAndSwap(s, nil) {
return
}
if backoff {
s.peer.retryAt = cs.clock() + int64(cs.cfg.backoff)
}
if cs.pending == nil {
cs.pending = map[netip.AddrPort]struct{}{}
}
cs.pending[s.dst] = struct{}{}
level := cs.logLevelLocked(slog.LevelInfo)
cs.closers.Add(1)
go func() {
defer cs.closers.Done()
open := u.finishClose(s)
u.l.Log(context.Background(), level, "connected socket: closed", "udpAddr", s.dst, "reason", why, "open", open)
}()
}
// finishClose closes s, whose destination is in pending, then lets the destination open again, and returns how
// many connected sockets are left. Closing waits only for a write already in a syscall: a write parked on a full
// buffer is woken, and one waiting out ENOBUFS holds no reference on the descriptor.
func (u *StdConn) finishClose(s *connSock) int32 {
cs := u.socks
if cs.closeHook != nil {
cs.closeHook(s.dst)
}
_ = s.conn.Close()
cs.mu.Lock()
defer cs.mu.Unlock()
delete(cs.pending, s.dst)
return cs.reserved.Add(-1)
}
// detachAllLocked detaches every connected socket, for a network change or shutdown.
func (u *StdConn) detachAllLocked(why string) {
if t := u.socks.table.Load(); t != nil {
for _, p := range *t {
if s := p.sock.Load(); s != nil {
u.detachLocked(s, why, false)
}
}
}
}
// closeConnSocks detaches every connected socket and discards the ones still opening, for a network change.
func (u *StdConn) closeConnSocks(why string) {
u.socks.mu.Lock()
u.socks.rebinds++
u.detachAllLocked(why)
u.socks.mu.Unlock()
}
func (u *StdConn) reapConnSocks(done <-chan struct{}) {
cs := u.socks
idle := cs.cfg.idle
tick := time.NewTicker(max(min(idle/4, 5*time.Second), time.Millisecond))
defer tick.Stop()
for {
select {
case <-done:
return
case <-tick.C:
}
t := cs.table.Load()
if t == nil {
continue
}
now := cs.clock()
for _, p := range *t {
if s := p.sock.Load(); s != nil && time.Duration(now-s.used.Load()) > idle {
u.closeConnSock(s, "idle", false)
}
}
}
}
// connSockDead reports whether a connected socket send or receive error means the connected path to the peer is
// gone, so its traffic belongs back on the listener. A connected UDP socket reports a peer's ICMP unreachable as
// ECONNREFUSED on its next call, which the unconnected listener never sees.
func connSockDead(errno syscall.Errno) bool {
switch errno {
case unix.ECONNREFUSED, unix.EHOSTUNREACH, unix.EHOSTDOWN, unix.ENETUNREACH, unix.ENETDOWN,
unix.EADDRNOTAVAIL, unix.EPIPE, unix.ENOTCONN, unix.EDESTADDRREQ:
return true
}
return false
}
// enobufsWait is how long a connected socket send sleeps when the interface queue is flow controlled. The kernel
// posts no wakeup when the advisory lifts, so this polls; 50us kept a gigabit link full without spinning a core in
// the 09-25 probes.
const enobufsWait = 50 * time.Microsecond
// connSockWrite writes bufs on s, one datagram per write. A full socket buffer parks the caller in the netpoller
// until the socket is writable, and a flow-controlled interface makes it wait out ENOBUFS, so a busy link slows
// the tun reader instead of dropping. Once the connected socket has waited cfg.enobufsMax without a send going
// through, either one drops the datagram instead, and a dropped datagram counts as sent, as the listener's
// unreported interface drops do; it never falls back to the listener, which would reorder it. closed reports that
// the connected socket was closed before bufs[sent] went out; any other error stops the run and is returned with
// how many went out.
func (u *StdConn) connSockWrite(s *connSock, bufs [][]byte) (sent int, errno syscall.Errno, closed bool) {
cs := u.socks
defer func() {
if sent > 0 {
s.used.Store(cs.clock())
}
}()
for sent < len(bufs) {
err := s.try(bufs[sent], false)
if err == syscall.EAGAIN || err == os.ErrDeadlineExceeded {
// os.ErrDeadlineExceeded here is another writer's parked deadline, passed but not yet cleared.
err = u.parkedWrite(s, bufs[sent])
}
switch err {
case nil:
sent++
if s.stallAt.Load() != 0 {
s.stallAt.Store(0)
s.dropping.Store(false)
}
continue
case errStallSpent:
u.dropStalled(s)
sent++
continue
}
errno, ok := err.(syscall.Errno)
if !ok {
return sent, 0, true
}
if errno != syscall.ENOBUFS {
return sent, errno, false
}
// The wait holds no reference on the descriptor, so a close doesn't wait for it.
if s.stalledFor(cs.clock()) < cs.cfg.enobufsMax {
cs.sleep(enobufsWait)
} else {
u.dropStalled(s)
sent++
}
}
return sent, 0, false
}
// parkedWrite writes b after EAGAIN, parked in the netpoller for what is left of s's stall budget, and returns
// errStallSpent once none is. golang/go#73919: a darwin UDP write parked on EAGAIN can wait forever for a
// writability event that never comes, so the wait always has a deadline.
func (u *StdConn) parkedWrite(s *connSock, b []byte) error {
cs := u.socks
s.parkMu.Lock()
defer s.parkMu.Unlock()
left := cs.cfg.enobufsMax - s.stalledFor(cs.clock())
if left <= 0 {
return errStallSpent
}
if err := s.conn.SetWriteDeadline(time.Now().Add(left)); err != nil {
return net.ErrClosed
}
err := s.try(b, true)
_ = s.conn.SetWriteDeadline(time.Time{})
if err == os.ErrDeadlineExceeded {
return errStallSpent
}
return err
}
// dropStalled logs the first datagram a stall drops.
func (u *StdConn) dropStalled(s *connSock) {
if !s.dropping.Swap(true) {
u.l.Debug("connected socket: sends stalled, dropping datagrams until one goes through", "udpAddr", s.dst)
}
}
// stalledFor records an ENOBUFS or EAGAIN on s at now and returns how long its sends have stalled: since the first
// ENOBUFS or EAGAIN after a send last went through. Every writer measures from that one time, so concurrent writers
// spend the budget no faster than one, and a send going through ends the stall for every writer, including one
// asleep in it.
func (s *connSock) stalledFor(now int64) time.Duration {
if s.stallAt.CompareAndSwap(0, now) {
return 0
}
at := s.stallAt.Load()
if at == 0 {
return 0
}
return time.Duration(now - at)
}
// readConnSock delivers a connected socket's datagrams through the listener's reader, serialized with ListenOut,
// until the connected socket is closed. Any receive error closes it: a connected socket reports its peer's
// unreachables here, and retrying an unexpected error could spin.
func (u *StdConn) readConnSock(s *connSock) {
rd := u.reader.Load()
buf := make([]byte, MTU)
for {
n, from, err := s.conn.ReadFromUDPAddrPort(buf)
if err != nil {
if errors.Is(err, net.ErrClosed) {
return
}
why := err.Error()
var errno syscall.Errno
if errors.As(err, &errno) && connSockDead(errno) {
why = errno.Error()
} else {
u.l.Warn("connected socket: unexpected receive error", "udpAddr", s.dst, "error", err)
}
u.closeConnSock(s, why, true)
return
}
s.used.Store(u.socks.clock())
u.readMu.Lock()
rd.r(netip.AddrPortFrom(from.Addr().Unmap(), from.Port()), buf[:n:n])
rd.flush()
u.readMu.Unlock()
}
}
File diff suppressed because it is too large. Load diff
+6
View File
@@ -0,0 +1,6 @@
//go:build !race && !e2e_testing
package udp
// raceEnabled reports a -race build, whose sync.Pool drops a random share of what is put back.
const raceEnabled = false
+6
View File
@@ -0,0 +1,6 @@
//go:build race && !e2e_testing
package udp
// raceEnabled reports a -race build, whose sync.Pool drops a random share of what is put back.
const raceEnabled = true
+108 -11
View File
@@ -10,6 +10,8 @@ import (
"log/slog"
"net"
"net/netip"
"sync"
"sync/atomic"
"syscall"
"unsafe"
@@ -22,9 +24,26 @@ type StdConn struct {
isV4 bool
sysFd uintptr
l *slog.Logger
// listenHost and port are the configured address and the bound port, which a connected socket shares.
listenHost netip.Addr
port uint16
// rc reaches the listener's descriptor for another listen routine's Rebind.
rc syscall.RawConn
socks *connSocks
// reader is ListenOut's reader, which connected socket readers share; readMu serializes them, since an
// EncReader isn't safe for concurrent use.
reader atomic.Pointer[readerPair]
readMu sync.Mutex
}
type readerPair struct {
r EncReader
flush func()
}
var _ Conn = &StdConn{}
var _ EstablishedPeerConn = &StdConn{}
func NewListener(l *slog.Logger, s Settings) (Conn, error) {
lc := NewListenConfig(s.Multi)
@@ -34,12 +53,13 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
}
if uc, ok := pc.(*net.UDPConn); ok {
c := &StdConn{UDPConn: uc, l: l}
c := &StdConn{UDPConn: uc, l: l, listenHost: s.Listen.Addr()}
rc, err := uc.SyscallConn()
if err != nil {
return nil, fmt.Errorf("failed to open udp socket: %w", err)
}
c.rc = rc
err = rc.Control(func(fd uintptr) {
c.sysFd = fd
@@ -53,6 +73,8 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
return nil, err
}
c.isV4 = la.Addr().Is4()
c.port = la.Port()
c.socks = joinConnSocks(c, la, s)
return c, nil
}
@@ -89,6 +111,19 @@ func NewListenConfig(multi bool) net.ListenConfig {
func sendto(s int, buf []byte, flags int, to unsafe.Pointer, addrlen int32) (err error)
func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
if t := u.socks.table.Load(); t != nil {
if p := (*t)[ap]; p != nil {
bufs := [1][]byte{b}
if sent, err := u.connSockRun(p, ap, bufs[:]); sent == 1 || err != nil {
return err
}
}
}
return u.writeTo(b, ap)
}
// writeTo sends b through the listener.
func (u *StdConn) writeTo(b []byte, ap netip.AddrPort) error {
var sa unsafe.Pointer
var addrLen int32
@@ -141,15 +176,47 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch.
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
// silently drops the rest of a burst (linux blocks instead). Poll for
// writability on EAGAIN before giving up on the remainder.
// TODO: writeTo maps EWOULDBLOCK to an error, so a full listener send buffer
// silently drops those packets (linux blocks instead). Poll for
// writability on EAGAIN before giving up on them.
written := 0
for i, b := range bufs {
if err := u.WriteTo(b, addrs[i]); err == nil {
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
t := u.socks.table.Load()
if t == nil {
for i, b := range bufs {
if err := u.writeTo(b, addrs[i]); err == nil {
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
}
}
return written, nil
}
for off := 0; off < len(bufs); {
// A run of datagrams to one peer goes out together, on its connected socket if it has or earns one.
dst := addrs[off]
end := off + 1
for end < len(bufs) && addrs[end] == dst {
end++
}
if p := (*t)[dst]; p != nil {
for off < end {
sent, err := u.connSockRun(p, dst, bufs[off:end])
written += sent
off += sent
if err == nil {
break
}
u.l.Debug("failed to write packet in batch", "udpAddr", dst, "error", err)
off++
}
}
// The listener sends what no connected socket took.
for ; off < end; off++ {
if err := u.writeTo(bufs[off], dst); err == nil {
written++
} else {
u.l.Debug("failed to write packet in batch", "udpAddr", dst, "error", err)
}
}
}
return written, nil
@@ -182,6 +249,8 @@ func NewUDPStatsEmitter(udpConns []Conn) func() {
func (u *StdConn) ListenOut(r EncReader, flush func()) error {
buffer := make([]byte, MTU)
u.reader.Store(&readerPair{r: r, flush: flush})
u.startConnSocks()
for {
// Just read one packet at a time
@@ -194,18 +263,29 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
continue
}
u.readMu.Lock()
r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n:n])
flush()
u.readMu.Unlock()
}
}
// Close closes the connected sockets of every listen routine, then the listener.
func (u *StdConn) Close() error {
u.stopConnSocks()
return u.UDPConn.Close()
}
func (u *StdConn) SupportsMultipleReaders() bool {
return false
}
// Rebind clears the interface the kernel scoped this socket to, so that sends are routed against the current
// routing table instead of the interface we happened to be on when the socket was created. Darwin pins sockets
// this way on its own, which is what strands us after the underlying network changes.
// this way on its own, which is what strands us after the underlying network changes. Control.RebindUDPServer
// rebinds only the first writer, so this clears the other listen routines' listeners too. Connected sockets are
// closed after the listeners are cleared, and later ones copy the cleared scope and pick their source addresses
// afresh.
func (u *StdConn) Rebind() error {
var err error
if u.isV4 {
@@ -213,6 +293,23 @@ func (u *StdConn) Rebind() error {
} else {
err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0)
}
for _, m := range u.socks.siblings(u) {
err = errors.Join(err, m.clearBoundIf())
}
u.closeConnSocks("rebind")
return err
}
// clearBoundIf clears the interface the listener is scoped to through its RawConn, which fails on a listener that
// closed concurrently instead of reaching whatever reused its descriptor number.
func (u *StdConn) clearBoundIf() error {
level, opt := syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF
if u.isV4 {
level, opt = syscall.IPPROTO_IP, syscall.IP_BOUND_IF
}
var serr error
if err := u.rc.Control(func(fd uintptr) { serr = syscall.SetsockoptInt(int(fd), level, opt, 0) }); err != nil {
return err
}
return serr
}