Compare commits

..

2 Commits

Author SHA1 Message Date
Jay Wren 7794e93762 Address PR feedback: remove outbound rate limit, improve config docs
Remove rate limiting from StartHandshake (outbound) since DoS
protection only needs to limit inbound handshakes. This also avoids
returning nil from StartHandshake which historically always returned
non-nil. Update config comment to note openssl speed is single-core
and suggest scaling by routines.

Co-Authored-By: Claude <svc-devxp-claude@slack-corp.com>
2026-04-10 14:36:32 -04:00
Jay Wren 3df60ae195 Add handshakes.max_rate to limit new handshakes per second
Nebula is vulnerable to DoS via handshake flooding since each incoming
  handshake performs expensive DH operations. This adds a token bucket
  rate limiter to the handshake manager that caps both inbound and
  outbound new handshakes per second. When the limit is reached, new
  handshakes are silently dropped and counted via the
  handshake_manager.rate_limited metric.

  Configured via handshakes.max_rate (default 0 = unlimited).

  Co-Authored-By: Claude <svc-devxp-claude@slack-corp.com>
2026-03-30 15:45:38 -04:00
55 changed files with 811 additions and 3707 deletions
+2 -3
View File
@@ -209,11 +209,10 @@ jobs:
id: create_release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_REF_NAME: ${{ github.ref_name }}
run: |
cd artifacts
gh release create \
--verify-tag \
--title "Release ${GITHUB_REF_NAME}" \
"${GITHUB_REF_NAME}" \
--title "Release ${{ github.ref_name }}" \
"${{ github.ref_name }}" \
SHASUM256.txt *-latest/*.zip *-latest/*.tar.gz
+8 -21
View File
@@ -18,8 +18,6 @@ jobs:
if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra')
name: Run extra smoke tests
runs-on: ubuntu-latest
env:
VAGRANT_DEFAULT_PROVIDER: libvirt
steps:
- uses: actions/checkout@v6
@@ -32,13 +30,11 @@ jobs:
- name: add hashicorp source
run: wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg && echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
- name: install vagrant and libvirt
run: |
sudo apt-get update && sudo apt-get install -y vagrant libvirt-daemon-system libvirt-dev
sudo chmod 666 /dev/kvm
sudo usermod -aG libvirt $(whoami)
sudo chmod 666 /var/run/libvirt/libvirt-sock
vagrant plugin install vagrant-libvirt
- name: workaround AMD-V issue # https://github.com/cri-o/packaging/pull/306
run: sudo rmmod kvm_amd
- name: install vagrant
run: sudo apt-get update && sudo apt-get install -y vagrant virtualbox
- name: freebsd-amd64
run: make smoke-vagrant/freebsd-amd64
@@ -49,19 +45,10 @@ jobs:
- name: netbsd-amd64
run: make smoke-vagrant/netbsd-amd64
- name: linux-386
run: make smoke-vagrant/linux-386
- name: linux-amd64-ipv6disable
run: make smoke-vagrant/linux-amd64-ipv6disable
# linux-386 runs last because it requires disabling KVM to use VirtualBox,
# which prevents libvirt (used by the other tests) from working after this point.
- name: install virtualbox for i386 test
run: |
sudo apt-get install -y virtualbox
sudo rmmod kvm_amd kvm_intel kvm 2>/dev/null || true
- name: linux-386
env:
VAGRANT_DEFAULT_PROVIDER: virtualbox
run: make smoke-vagrant/linux-386
timeout-minutes: 30
+3 -5
View File
@@ -16,10 +16,8 @@ relay:
am_relay: true
EOF
# TEST-NET-3 placeholder IPs; smoke-relay.sh seds them to real container IPs.
# Mapping: .2 lighthouse1, .3 host2, .4 host3, .5 host4.
export LIGHTHOUSES="192.168.100.1 203.0.113.2:4242"
export REMOTE_ALLOW_LIST='{"203.0.113.4/32": false, "203.0.113.5/32": false}'
export LIGHTHOUSES="192.168.100.1 172.17.0.2:4242"
export REMOTE_ALLOW_LIST='{"172.17.0.4/32": false, "172.17.0.5/32": false}'
HOST="host2" ../genconfig.sh >host2.yml <<EOF
relay:
@@ -27,7 +25,7 @@ relay:
- 192.168.100.1
EOF
export REMOTE_ALLOW_LIST='{"203.0.113.3/32": false}'
export REMOTE_ALLOW_LIST='{"172.17.0.3/32": false}'
HOST="host3" ../genconfig.sh >host3.yml
+6 -12
View File
@@ -5,15 +5,9 @@ set -e -x
rm -rf ./build
mkdir ./build
# Smoke containers run on a dedicated docker network whose subnet is allocated
# at smoke time, not known at build time. Configs are written with TEST-NET-3
# placeholder IPs (RFC 5737) and smoke.sh / smoke-vagrant.sh / smoke-relay.sh
# sed the real container IPs in before starting nebula.
#
# Placeholder mapping (last octet == fixed container slot):
# 203.0.113.2 -> lighthouse1, 203.0.113.3 -> host2,
# 203.0.113.4 -> host3, 203.0.113.5 -> host4.
LIGHTHOUSE_IP="203.0.113.2"
# TODO: Assumes your docker bridge network is a /24, and the first container that launches will be .1
# - We could make this better by launching the lighthouse first and then fetching what IP it is.
NET="$(docker network inspect bridge -f '{{ range .IPAM.Config }}{{ .Subnet }}{{ end }}' | cut -d. -f1-3)"
(
cd build
@@ -31,16 +25,16 @@ LIGHTHOUSE_IP="203.0.113.2"
../genconfig.sh >lighthouse1.yml
HOST="host2" \
LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
../genconfig.sh >host2.yml
HOST="host3" \
LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
INBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
../genconfig.sh >host3.yml
HOST="host4" \
LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
OUTBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
../genconfig.sh >host4.yml
+8 -47
View File
@@ -6,8 +6,6 @@ set -o pipefail
mkdir -p logs
NETWORK="nebula-smoke-relay"
cleanup() {
echo
echo " *** cleanup"
@@ -18,53 +16,22 @@ cleanup() {
then
docker kill lighthouse1 host2 host3 host4
fi
docker network rm "$NETWORK" >/dev/null 2>&1
}
trap cleanup EXIT
# Create a dedicated smoke network with an explicit subnet (required for --ip
# below). Probe a short list of candidates so a locally-used range doesn't
# fail the whole test — we only need one to be free.
docker network rm "$NETWORK" >/dev/null 2>&1 || true
for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
break
fi
done
if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
echo "failed to create $NETWORK: every candidate subnet is in use" >&2
exit 1
fi
# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
# .3 host2, .4 host3, .5 host4 — matches the placeholders in build-relay.sh.
SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
PREFIX="${SUBNET%/*}"
PREFIX="${PREFIX%.*}"
LIGHTHOUSE_IP="$PREFIX.2"
HOST2_IP="$PREFIX.3"
HOST3_IP="$PREFIX.4"
HOST4_IP="$PREFIX.5"
# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
for f in build/host2.yml build/host3.yml build/host4.yml; do
sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
mv "$f.tmp" "$f"
done
docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test
docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" nebula:smoke-relay -config host2.yml -test
docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" nebula:smoke-relay -config host3.yml -test
docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" nebula:smoke-relay -config host4.yml -test
docker run --name host2 --rm nebula:smoke-relay -config host2.yml -test
docker run --name host3 --rm nebula:smoke-relay -config host3.yml -test
docker run --name host4 --rm nebula:smoke-relay -config host4.yml -test
docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
sleep 1
docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
sleep 1
docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' &
docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' &
sleep 1
docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' &
docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' &
sleep 1
set +x
@@ -109,13 +76,7 @@ docker exec host4 sh -c 'kill 1'
docker exec host3 sh -c 'kill 1'
docker exec host2 sh -c 'kill 1'
docker exec lighthouse1 sh -c 'kill 1'
# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
# fixed sleep.
for _ in $(seq 1 30); do
[ -z "$(jobs -r)" ] && break
sleep 1
done
sleep 5
if [ "$(jobs -r)" ]
then
+4 -43
View File
@@ -8,8 +8,6 @@ export VAGRANT_CWD="$PWD/vagrant-$1"
mkdir -p logs
NETWORK="nebula-smoke"
cleanup() {
echo
echo " *** cleanup"
@@ -21,51 +19,21 @@ cleanup() {
docker kill lighthouse1 host2
fi
vagrant destroy -f
docker network rm "$NETWORK" >/dev/null 2>&1
}
trap cleanup EXIT
# Create a dedicated smoke network with an explicit subnet (required for --ip
# below). Probe a short list of candidates so a locally-used range doesn't
# fail the whole test — we only need one to be free.
docker network rm "$NETWORK" >/dev/null 2>&1 || true
for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
break
fi
done
if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
echo "failed to create $NETWORK: every candidate subnet is in use" >&2
exit 1
fi
# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
# .3 host2 — matches the placeholders in build.sh.
SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
PREFIX="${SUBNET%/*}"
PREFIX="${PREFIX%.*}"
LIGHTHOUSE_IP="$PREFIX.2"
HOST2_IP="$PREFIX.3"
# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
# This must happen before `vagrant up` rsyncs build/ into the VM for host3.
for f in build/host2.yml build/host3.yml; do
sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
mv "$f.tmp" "$f"
done
CONTAINER="nebula:${NAME:-smoke}"
docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
vagrant up
vagrant ssh -c "cd /nebula && /nebula/$1-nebula -config host3.yml -test" -- -T
docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
sleep 1
docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
sleep 1
vagrant ssh -c "cd /nebula && sudo sh -c 'echo \$\$ >/nebula/pid && exec /nebula/$1-nebula -config host3.yml'" 2>&1 -- -T | tee logs/host3 | sed -u 's/^/ [host3] /' &
sleep 15
@@ -128,14 +96,7 @@ vagrant ssh -c "ping -c1 192.168.100.2" -- -T
vagrant ssh -c "sudo xargs kill </nebula/pid" -- -T
docker exec host2 sh -c 'kill 1'
docker exec lighthouse1 sh -c 'kill 1'
# Wait up to 30s for all backgrounded jobs to exit. vagrant ssh in particular
# takes a beat to tear down after nebula exits on the VM, so a fixed sleep is
# racy.
for _ in $(seq 1 30); do
[ -z "$(jobs -r)" ] && break
sleep 1
done
sleep 1
if [ "$(jobs -r)" ]
then
+8 -49
View File
@@ -6,8 +6,6 @@ set -o pipefail
mkdir -p logs
NETWORK="nebula-smoke"
cleanup() {
echo
echo " *** cleanup"
@@ -18,56 +16,24 @@ cleanup() {
then
docker kill lighthouse1 host2 host3 host4
fi
docker network rm "$NETWORK" >/dev/null 2>&1
}
trap cleanup EXIT
# Create a dedicated smoke network with an explicit subnet (required for --ip
# below). Probe a short list of candidates so a locally-used range doesn't
# fail the whole test — we only need one to be free.
docker network rm "$NETWORK" >/dev/null 2>&1 || true
for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
break
fi
done
if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
echo "failed to create $NETWORK: every candidate subnet is in use" >&2
exit 1
fi
# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
# .3 host2, .4 host3, .5 host4 — matches the placeholders in build.sh.
SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
PREFIX="${SUBNET%/*}"
PREFIX="${PREFIX%.*}"
LIGHTHOUSE_IP="$PREFIX.2"
HOST2_IP="$PREFIX.3"
HOST3_IP="$PREFIX.4"
HOST4_IP="$PREFIX.5"
# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
# build/lighthouse1.yml has no IPs to rewrite so it's skipped.
for f in build/host2.yml build/host3.yml build/host4.yml; do
sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
mv "$f.tmp" "$f"
done
CONTAINER="nebula:${NAME:-smoke}"
docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" "$CONTAINER" -config host3.yml -test
docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" "$CONTAINER" -config host4.yml -test
docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
docker run --name host3 --rm "$CONTAINER" -config host3.yml -test
docker run --name host4 --rm "$CONTAINER" -config host4.yml -test
docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
sleep 1
docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
sleep 1
docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' &
docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' &
sleep 1
docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' &
docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' &
sleep 1
# grab tcpdump pcaps for debugging
@@ -158,20 +124,13 @@ set -x
# host2 speaking to host4 on UDP 4000 should allow it to reply, when firewall rules would normally not permit this
docker exec host2 sh -c "/usr/bin/echo host2 | ncat -nuv 192.168.100.4 4000"
docker exec host2 ncat -e '/usr/bin/echo helloagainfromhost2' -nkluv 0.0.0.0 4000 &
sleep 1
docker exec host4 sh -c "/usr/bin/echo host4 | ncat -nuv 192.168.100.2 4000"
docker exec host4 sh -c 'kill 1'
docker exec host3 sh -c 'kill 1'
docker exec host2 sh -c 'kill 1'
docker exec lighthouse1 sh -c 'kill 1'
# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
# fixed sleep.
for _ in $(seq 1 30); do
[ -z "$(jobs -r)" ] && break
sleep 1
done
sleep 5
if [ "$(jobs -r)" ]
then
@@ -1,7 +1,7 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
Vagrant.configure("2") do |config|
config.vm.box = "bento/ubuntu-24.04"
config.vm.box = "ubuntu/jammy64"
config.vm.synced_folder "../build", "/nebula"
@@ -1,7 +1,7 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
Vagrant.configure("2") do |config|
config.vm.box = "DefinedNet/openbsd78"
config.vm.box = "generic/openbsd7"
config.vm.synced_folder "../build", "/nebula", type: "rsync"
end
+1 -3
View File
@@ -57,7 +57,7 @@ Check the [releases](https://github.com/slackhq/nebula/releases/latest) page for
docker pull nebulaoss/nebula
```
#### Mobile ([source code](https://github.com/DefinedNet/mobile_nebula))
#### Mobile
- [iOS](https://apps.apple.com/us/app/mobile-nebula/id1509587936?itsct=apps_box&amp;itscg=30200)
- [Android](https://play.google.com/store/apps/details?id=net.defined.mobile_nebula&pcampaignid=pcampaignidMKT-Other-global-all-co-prtnr-py-PartBadge-Mar2515-1)
@@ -76,8 +76,6 @@ Nebula was created to provide a mechanism for groups of hosts to communicate sec
## Getting started (quickly)
**Don't want to manage your own PKI and lighthouses?** [Managed Nebula](https://www.defined.net/) from Defined Networking handles all of this for you.
To set up a Nebula network, you'll need:
#### 1. The [Nebula binaries](https://github.com/slackhq/nebula/releases) or [Distribution Packages](https://github.com/slackhq/nebula#distribution-packages) for your specific platform. Specifically you'll need `nebula-cert` and the specific nebula binary for each platform you use.
+9 -36
View File
@@ -1,14 +1,11 @@
package cert
import (
"bufio"
"bytes"
"encoding/pem"
"errors"
"fmt"
"io"
"net/netip"
"slices"
"strings"
"time"
)
@@ -32,46 +29,22 @@ func NewCAPool() *CAPool {
// If the pool contains any expired certificates, an ErrExpired will be
// returned along with the pool. The caller must handle any such errors.
func NewCAPoolFromPEM(caPEMs []byte) (*CAPool, error) {
return NewCAPoolFromPEMReader(bytes.NewReader(caPEMs))
}
// NewCAPoolFromPEMReader will create a new CA pool from the provided reader.
// The reader must contain a PEM-encoded set of nebula certificates.
func NewCAPoolFromPEMReader(r io.Reader) (*CAPool, error) {
pool := NewCAPool()
var err error
var expired bool
scanner := bufio.NewScanner(r)
scanner.Split(SplitPEM)
for scanner.Scan() {
pemBytes := scanner.Bytes()
block, rest := pem.Decode(pemBytes)
if len(bytes.TrimSpace(rest)) > 0 {
return nil, ErrInvalidPEMBlock
for {
caPEMs, err = pool.AddCAFromPEM(caPEMs)
if errors.Is(err, ErrExpired) {
expired = true
err = nil
}
if block == nil {
return nil, ErrInvalidPEMBlock
}
c, err := unmarshalCertificateBlock(block)
if err != nil {
return nil, err
}
err = pool.AddCA(c)
if errors.Is(err, ErrExpired) {
expired = true
continue
} else if err != nil {
return nil, err
if len(caPEMs) == 0 || strings.TrimSpace(string(caPEMs)) == "" {
break
}
}
if err := scanner.Err(); err != nil {
return nil, ErrInvalidPEMBlock
}
if expired {
return pool, ErrExpired
-57
View File
@@ -1,10 +1,7 @@
package cert
import (
"bytes"
"io"
"net/netip"
"strings"
"testing"
"time"
@@ -115,60 +112,6 @@ k+coOv04r+zh33ISyhbsafnYduN17p2eD7CmHvHuerguXD9f32gcxo/KsFCKEjMe
assert.Len(t, ppppp.CAs, 1)
}
// oneByteReader wraps a reader to return at most 1 byte per Read call,
// exercising the streaming accumulation logic in NewCAPoolFromPEMReader.
type oneByteReader struct {
r io.Reader
}
func (o *oneByteReader) Read(p []byte) (int, error) {
if len(p) == 0 {
return 0, nil
}
return o.r.Read(p[:1])
}
func TestNewCAPoolFromPEMReader_EmptyReader(t *testing.T) {
pool, err := NewCAPoolFromPEMReader(bytes.NewReader(nil))
require.NoError(t, err)
assert.Empty(t, pool.CAs)
pool, err = NewCAPoolFromPEMReader(strings.NewReader(" \n\t\n "))
require.NoError(t, err)
assert.Empty(t, pool.CAs)
}
func TestNewCAPoolFromPEMReader_OneByteReads(t *testing.T) {
ca1, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
ca2, _, _, pem2 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
bundle := append(pem1, pem2...)
pool, err := NewCAPoolFromPEMReader(&oneByteReader{r: bytes.NewReader(bundle)})
require.NoError(t, err)
assert.Len(t, pool.CAs, 2)
fp1, err := ca1.Fingerprint()
require.NoError(t, err)
fp2, err := ca2.Fingerprint()
require.NoError(t, err)
assert.Contains(t, pool.CAs, fp1)
assert.Contains(t, pool.CAs, fp2)
}
func TestNewCAPoolFromPEMReader_TruncatedPEM(t *testing.T) {
_, err := NewCAPoolFromPEMReader(strings.NewReader("-----BEGIN NEBULA CERTIFICATE-----\npartialdata"))
assert.ErrorIs(t, err, ErrInvalidPEMBlock)
}
func TestNewCAPoolFromPEMReader_TrailingGarbage(t *testing.T) {
_, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
bundle := append(pem1, []byte("some trailing garbage")...)
_, err := NewCAPoolFromPEMReader(bytes.NewReader(bundle))
assert.ErrorIs(t, err, ErrInvalidPEMBlock)
}
func TestCertificateV1_Verify(t *testing.T) {
ca, _, caKey, _ := NewTestCaCert(Version1, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil)
c, _, _, _ := NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test cert", time.Now(), time.Now().Add(5*time.Minute), nil, nil, nil)
+1 -6
View File
@@ -44,12 +44,7 @@ func swap(r, s []byte) ([]byte, []byte, error) {
}
sNormalized := nMod.Nat().Sub(bigS, nMod)
result := sNormalized.Bytes(nMod)
for len(result) > 1 && result[0] == 0 {
result = result[1:]
}
return r, result, nil
return r, sNormalized.Bytes(nMod), nil
}
func Normalize(sig []byte) ([]byte, error) {
+13 -69
View File
@@ -1,66 +1,12 @@
package cert
import (
"bytes"
"encoding/pem"
"errors"
"fmt"
"golang.org/x/crypto/ed25519"
)
var ErrTruncatedPEMBlock = errors.New("truncated PEM block")
// SplitPEM is a split function for bufio.Scanner that returns each PEM block.
func SplitPEM(data []byte, atEOF bool) (advance int, token []byte, err error) {
// Look for the start of a PEM block
start := bytes.Index(data, []byte("-----BEGIN "))
if start == -1 {
if atEOF && len(bytes.TrimSpace(data)) > 0 {
// Non-whitespace content with no PEM block
return 0, nil, ErrTruncatedPEMBlock
}
if atEOF {
return len(data), nil, nil
}
// Request more data
return 0, nil, nil
}
// Look for the end marker
endMarkerStart := bytes.Index(data[start:], []byte("-----END "))
if endMarkerStart == -1 {
if atEOF {
// Incomplete PEM block at EOF
return 0, nil, ErrTruncatedPEMBlock
}
// Need more data to find the end
return 0, nil, nil
}
// Find the actual end of the END line (after the newline)
endMarkerStart += start
endLineEnd := bytes.IndexByte(data[endMarkerStart:], '\n')
var end int
if endLineEnd == -1 {
if atEOF {
// END marker without newline at EOF - take it anyway
end = len(data)
} else {
// Need more data
return 0, nil, nil
}
} else {
end = endMarkerStart + endLineEnd + 1
}
// Extract the PEM block
pemBlock := data[start:end]
// Return the valid PEM block
return end, pemBlock, nil
}
const ( //cert banners
CertificateBanner = "NEBULA CERTIFICATE"
CertificateV2Banner = "NEBULA CERTIFICATE V2"
@@ -91,7 +37,19 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
return nil, r, ErrInvalidPEMBlock
}
c, err := unmarshalCertificateBlock(p)
var c Certificate
var err error
switch p.Type {
// Implementations must validate the resulting certificate contains valid information
case CertificateBanner:
c, err = unmarshalCertificateV1(p.Bytes, nil)
case CertificateV2Banner:
c, err = unmarshalCertificateV2(p.Bytes, nil, Curve_CURVE25519)
default:
return nil, r, ErrInvalidPEMCertificateBanner
}
if err != nil {
return nil, r, err
}
@@ -100,20 +58,6 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
}
// unmarshalCertificateBlock decodes a single PEM block into a certificate.
// It expects a Nebula certificate banner and returns ErrInvalidPEMCertificateBanner otherwise.
func unmarshalCertificateBlock(block *pem.Block) (Certificate, error) {
switch block.Type {
// Implementations must validate the resulting certificate contains valid information
case CertificateBanner:
return unmarshalCertificateV1(block.Bytes, nil)
case CertificateV2Banner:
return unmarshalCertificateV2(block.Bytes, nil, Curve_CURVE25519)
default:
return nil, ErrInvalidPEMCertificateBanner
}
}
func marshalCertPublicKeyToPEM(c Certificate) []byte {
if c.IsCA() {
return MarshalSigningPublicKeyToPEM(c.Curve(), c.PublicKey())
-76
View File
@@ -1,88 +1,12 @@
package cert
import (
"bufio"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func scanAll(t *testing.T, input string) ([]string, error) {
t.Helper()
scanner := bufio.NewScanner(strings.NewReader(input))
scanner.Split(SplitPEM)
var blocks []string
for scanner.Scan() {
blocks = append(blocks, scanner.Text())
}
return blocks, scanner.Err()
}
func TestSplitPEM_Single(t *testing.T) {
input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\n"
blocks, err := scanAll(t, input)
require.NoError(t, err)
require.Len(t, blocks, 1)
require.Equal(t, input, blocks[0])
}
func TestSplitPEM_Multiple(t *testing.T) {
block1 := "-----BEGIN TEST-----\naaa\n-----END TEST-----\n"
block2 := "-----BEGIN TEST-----\nbbb\n-----END TEST-----\n"
blocks, err := scanAll(t, block1+block2)
require.NoError(t, err)
require.Len(t, blocks, 2)
require.Equal(t, block1, blocks[0])
require.Equal(t, block2, blocks[1])
}
func TestSplitPEM_CommentsAndWhitespaceBetweenBlocks(t *testing.T) {
input := "# comment\n\n-----BEGIN TEST-----\naaa\n-----END TEST-----\n\n# another comment\n\n-----BEGIN TEST-----\nbbb\n-----END TEST-----\n"
blocks, err := scanAll(t, input)
require.NoError(t, err)
require.Len(t, blocks, 2)
}
func TestSplitPEM_Empty(t *testing.T) {
blocks, err := scanAll(t, "")
require.NoError(t, err)
require.Empty(t, blocks)
}
func TestSplitPEM_WhitespaceOnly(t *testing.T) {
blocks, err := scanAll(t, " \n\t\n ")
require.NoError(t, err)
require.Empty(t, blocks)
}
func TestSplitPEM_TrailingGarbage(t *testing.T) {
input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\ngarbage"
blocks, err := scanAll(t, input)
require.ErrorIs(t, err, ErrTruncatedPEMBlock)
require.Len(t, blocks, 1)
}
func TestSplitPEM_TruncatedBlock(t *testing.T) {
input := "-----BEGIN TEST-----\npartial data with no end"
_, err := scanAll(t, input)
require.ErrorIs(t, err, ErrTruncatedPEMBlock)
}
func TestSplitPEM_NoEndNewline(t *testing.T) {
input := "-----BEGIN TEST-----\ndata\n-----END TEST-----"
blocks, err := scanAll(t, input)
require.NoError(t, err)
require.Len(t, blocks, 1)
require.Equal(t, input, blocks[0])
}
func TestSplitPEM_GarbageOnly(t *testing.T) {
_, err := scanAll(t, "this is not PEM data")
require.ErrorIs(t, err, ErrTruncatedPEMBlock)
}
func TestUnmarshalCertificateFromPEM(t *testing.T) {
goodCert := []byte(`
# A good cert
+12 -5
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"os"
"strings"
"time"
"github.com/slackhq/nebula/cert"
@@ -39,15 +40,21 @@ func verify(args []string, out io.Writer, errOut io.Writer) error {
return err
}
caFile, err := os.Open(*vf.caPath)
rawCACert, err := os.ReadFile(*vf.caPath)
if err != nil {
return fmt.Errorf("error while reading ca: %w", err)
}
defer caFile.Close()
caPool, err := cert.NewCAPoolFromPEMReader(caFile)
if err != nil && !errors.Is(err, cert.ErrExpired) {
return fmt.Errorf("error while adding ca cert to pool: %w", err)
caPool := cert.NewCAPool()
for {
rawCACert, err = caPool.AddCAFromPEM(rawCACert)
if err != nil {
return fmt.Errorf("error while adding ca cert to pool: %w", err)
}
if rawCACert == nil || len(rawCACert) == 0 || strings.TrimSpace(string(rawCACert)) == "" {
break
}
}
rawCert, err := os.ReadFile(*vf.certPath)
+1 -1
View File
@@ -64,7 +64,7 @@ func Test_verify(t *testing.T) {
err = verify([]string{"-ca", caFile.Name(), "-crt", "does_not_exist"}, ob, eb)
assert.Empty(t, ob.String())
assert.Empty(t, eb.String())
require.ErrorIs(t, err, cert.ErrInvalidPEMBlock)
require.EqualError(t, err, "error while adding ca cert to pool: input did not contain a valid PEM encoded block")
// make a ca for later
caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader)
+7 -22
View File
@@ -50,15 +50,9 @@ func main() {
os.Exit(0)
}
l := logrus.New()
l.Out = os.Stdout
if *serviceFlag != "" {
if err := doService(configPath, configTest, Build, serviceFlag); err != nil {
l.WithError(err).Error("Service command failed")
os.Exit(1)
}
return
doService(configPath, configTest, Build, serviceFlag)
os.Exit(1)
}
if *configPath == "" {
@@ -67,6 +61,9 @@ func main() {
os.Exit(1)
}
l := logrus.New()
l.Out = os.Stdout
c := config.NewC(l)
err := c.Load(*configPath)
if err != nil {
@@ -81,20 +78,8 @@ func main() {
}
if !*configTest {
wait, err := ctrl.Start()
if err != nil {
util.LogWithContextIfNeeded("Error while running", err, l)
os.Exit(1)
}
go ctrl.ShutdownBlock()
if err := wait(); err != nil {
l.WithError(err).Error("Nebula stopped due to fatal error")
os.Exit(2)
}
l.Info("Goodbye")
ctrl.Start()
ctrl.ShutdownBlock()
}
os.Exit(0)
+10 -8
View File
@@ -57,11 +57,11 @@ func fileExists(filename string) bool {
return true
}
func doService(configPath *string, configTest *bool, build string, serviceFlag *string) error {
func doService(configPath *string, configTest *bool, build string, serviceFlag *string) {
if *configPath == "" {
ex, err := os.Executable()
if err != nil {
return err
panic(err)
}
*configPath = filepath.Dir(ex) + "/config.yaml"
if !fileExists(*configPath) {
@@ -88,13 +88,13 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
// - above, in `Run` we create a `logrus.Logger` which is what nebula expects to use
s, err := service.New(prg, svcConfig)
if err != nil {
return err
log.Fatal(err)
}
errs := make(chan error, 5)
logger, err = s.Logger(errs)
if err != nil {
return err
log.Fatal(err)
}
go func() {
@@ -109,16 +109,18 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
switch *serviceFlag {
case "run":
if err := s.Run(); err != nil {
err = s.Run()
if err != nil {
// Route any errors to the system logger
logger.Error(err)
}
default:
if err := service.Control(s, *serviceFlag); err != nil {
err := service.Control(s, *serviceFlag)
if err != nil {
log.Printf("Valid actions: %q\n", service.ControlAction)
return err
log.Fatal(err)
}
return
}
return nil
}
+2 -14
View File
@@ -72,21 +72,9 @@ func main() {
}
if !*configTest {
wait, err := ctrl.Start()
if err != nil {
util.LogWithContextIfNeeded("Error while running", err, l)
os.Exit(1)
}
go ctrl.ShutdownBlock()
ctrl.Start()
notifyReady(l)
if err := wait(); err != nil {
l.WithError(err).Error("Nebula stopped due to fatal error")
os.Exit(2)
}
l.Info("Goodbye")
ctrl.ShutdownBlock()
}
os.Exit(0)
+6 -103
View File
@@ -2,34 +2,17 @@ package nebula
import (
"context"
"errors"
"net/netip"
"os"
"os/signal"
"sync"
"syscall"
"github.com/sirupsen/logrus"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/firewall/events"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay"
)
type RunState int
const (
StateUnknown RunState = iota
StateReady
StateStarted
StateStopping
StateStopped
)
var ErrAlreadyStarted = errors.New("nebula is already started")
var ErrAlreadyStopped = errors.New("nebula cannot be restarted")
var ErrUnknownState = errors.New("nebula state is invalid")
// Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching
// core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc
@@ -43,9 +26,6 @@ type controlHostLister interface {
}
type Control struct {
stateLock sync.Mutex
state RunState
f *Interface
l *logrus.Logger
ctx context.Context
@@ -69,31 +49,10 @@ type ControlHostInfo struct {
CurrentRelaysThroughMe []netip.Addr `json:"currentRelaysThroughMe"`
}
// Start actually runs nebula, this is a nonblocking call.
// The returned function blocks until nebula has fully stopped and returns the
// first fatal reader error (if any). A nil error means nebula shut down
// gracefully; a non-nil error means a reader hit an unexpected failure that
// triggered the shutdown.
func (c *Control) Start() (func() error, error) {
c.stateLock.Lock()
defer c.stateLock.Unlock()
switch c.state {
case StateReady:
//yay!
case StateStopped, StateStopping:
return nil, ErrAlreadyStopped
case StateStarted:
return nil, ErrAlreadyStarted
default:
return nil, ErrUnknownState
}
// Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock()
func (c *Control) Start() {
// Activate the interface
err := c.f.activate()
if err != nil {
c.state = StateStopped
return nil, err
}
c.f.activate()
// Call all the delayed funcs that waited patiently for the interface to be created.
if c.sshStart != nil {
@@ -112,40 +71,16 @@ func (c *Control) Start() (func() error, error) {
c.lighthouseStart()
}
c.f.triggerShutdown = c.Stop
// Start reading packets.
out, err := c.f.run()
if err != nil {
c.state = StateStopped
return nil, err
}
c.state = StateStarted
return out, nil
}
func (c *Control) State() RunState {
c.stateLock.Lock()
defer c.stateLock.Unlock()
return c.state
c.f.run()
}
func (c *Control) Context() context.Context {
return c.ctx
}
// Stop is a non-blocking call that signals nebula to close all tunnels and shut down
// Stop signals nebula to shutdown and close all tunnels, returns after the shutdown is complete
func (c *Control) Stop() {
c.stateLock.Lock()
if c.state != StateStarted {
c.stateLock.Unlock()
// We are stopping or stopped already
return
}
c.state = StateStopping
c.stateLock.Unlock()
// Stop the handshakeManager (and other services), to prevent new tunnels from
// being created while we're shutting them all down.
c.cancel()
@@ -154,9 +89,7 @@ func (c *Control) Stop() {
if err := c.f.Close(); err != nil {
c.l.WithError(err).Error("Close interface failed")
}
c.stateLock.Lock()
c.state = StateStopped
c.stateLock.Unlock()
c.l.Info("Goodbye")
}
// ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled
@@ -341,36 +274,6 @@ func (c *Control) Device() overlay.Device {
return c.f.inside
}
// SetFirewallEventReporter installs an event reporter on the current firewall.
// Passing nil clears any installed reporter. The reporter is carried across
// firewall rule reloads. Report* methods are invoked while nebula holds
// internal locks and must be non-blocking; in particular they must not call
// back into *Control methods that touch the firewall, or deadlock will
// result.
//
// Installation is performed by shallow-copying the current *Firewall,
// setting the reporter field on the copy, and swapping the pointer under
// the conntrack lock. Every Firewall the data path sees therefore has an
// immutable reporter slot, and emit sites can read it without any
// synchronization of their own.
func (c *Control) SetFirewallEventReporter(r events.Reporter) {
old := c.f.firewall
if old == nil {
return
}
old.Conntrack.Lock()
defer old.Conntrack.Unlock()
// Re-read under the lock in case a concurrent reload swapped in a new
// Firewall between the unlocked load above and here. Both Firewalls share
// the same Conntrack pointer in the normal (non-overflow) reload path,
// so the lock we hold is the right one for whichever we see now.
current := c.f.firewall
fw := *current
fw.reporter = r
c.f.firewall = &fw
}
func copyHostInfo(h *HostInfo, preferredRanges []netip.Prefix) ControlHostInfo {
chi := ControlHostInfo{
VpnAddrs: make([]netip.Addr, len(h.vpnAddrs)),
-1
View File
@@ -79,7 +79,6 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
}, &Interface{})
c := Control{
state: StateReady,
f: &Interface{
hostMap: hm,
},
+52 -203
View File
@@ -1,14 +1,12 @@
package nebula
import (
"context"
"fmt"
"net"
"net/netip"
"strconv"
"strings"
"sync"
"sync/atomic"
"github.com/gaissmai/bart"
"github.com/miekg/dns"
@@ -16,207 +14,32 @@ import (
"github.com/slackhq/nebula/config"
)
type dnsServer struct {
// This whole thing should be rewritten to use context
var dnsR *dnsRecords
var dnsServer *dns.Server
var dnsAddr string
type dnsRecords struct {
sync.RWMutex
l *logrus.Logger
ctx context.Context
dnsMap4 map[string]netip.Addr
dnsMap6 map[string]netip.Addr
hostMap *HostMap
myVpnAddrsTable *bart.Lite
mux *dns.ServeMux
// enabled mirrors `lighthouse.serve_dns && lighthouse.am_lighthouse`.
// Start, Add, and reload consult it so callers don't need to know the
// gating rules. When it toggles off via reload, accumulated records are
// cleared so a later re-enable starts with a fresh map populated from
// new handshakes.
enabled atomic.Bool
serverMu sync.Mutex
server *dns.Server
// started is closed once `server` has finished binding (or after
// ListenAndServe returns on a bind failure). Stop waits on it before
// calling Shutdown to avoid the miekg/dns "server not started" race
// where a Shutdown that arrives before bind completes is silently
// ignored, leaving the listener running forever.
started chan struct{}
addr string
}
// newDnsServerFromConfig builds a dnsServer, applies the initial config, and
// registers a reload callback. The reload callback is registered before the
// initial config is applied, so a SIGHUP can later enable, fix, or disable
// DNS even if the initial application failed.
//
// The dnsServer internally gates on `lighthouse.serve_dns &&
// lighthouse.am_lighthouse`. Start and Add are safe to call unconditionally,
// they no-op when DNS isn't enabled. Each Start invocation owns a ctx-cancel
// watcher that tears the listener down on nebula shutdown. The returned
// pointer is always non-nil, even on error.
func newDnsServerFromConfig(ctx context.Context, l *logrus.Logger, cs *CertState, hostMap *HostMap, c *config.C) (*dnsServer, error) {
ds := &dnsServer{
func newDnsRecords(l *logrus.Logger, cs *CertState, hostMap *HostMap) *dnsRecords {
return &dnsRecords{
l: l,
ctx: ctx,
dnsMap4: make(map[string]netip.Addr),
dnsMap6: make(map[string]netip.Addr),
hostMap: hostMap,
myVpnAddrsTable: cs.myVpnAddrsTable,
}
ds.mux = dns.NewServeMux()
ds.mux.HandleFunc(".", ds.handleDnsRequest)
c.RegisterReloadCallback(func(c *config.C) {
if err := ds.reload(c, false); err != nil {
l.WithError(err).Error("Failed to reload DNS responder from config")
}
})
if err := ds.reload(c, true); err != nil {
return ds, err
}
return ds, nil
}
// reload applies the latest config and reconciles the running state with it:
// - enabled toggled on -> spawn a runner
// - enabled toggled off -> stop the runner
// - listen address changed (while running) -> restart on the new address
// - everything else -> no-op
//
// On the initial call it only records configuration; Control.Start is what
// launches the first runner via dnsStart.
func (d *dnsServer) reload(c *config.C, initial bool) error {
wantsDns := c.GetBool("lighthouse.serve_dns", false)
amLighthouse := c.GetBool("lighthouse.am_lighthouse", false)
enabled := wantsDns && amLighthouse
newAddr := getDnsServerAddr(c)
d.serverMu.Lock()
running := d.server
runningStarted := d.started
sameAddr := d.addr == newAddr
d.addr = newAddr
d.enabled.Store(enabled)
d.serverMu.Unlock()
if initial {
if wantsDns && !amLighthouse {
d.l.Warn("DNS server refusing to run because this host is not a lighthouse.")
}
return nil
}
if !enabled {
if running != nil {
d.Stop()
}
// Drop any records that accumulated while enabled; a later re-enable
// will repopulate from fresh handshakes.
d.clearRecords()
return nil
}
if running == nil {
// Was disabled (or never started); bring it up now.
go d.Start()
return nil
}
if sameAddr {
return nil
}
d.shutdownServer(running, runningStarted, "reload")
// Old Start goroutine has now exited; bring up a fresh listener on the
// new address.
go d.Start()
return nil
}
// shutdownServer waits for the server to finish binding (so Shutdown actually
// stops it rather than no-oping) and then shuts it down.
func (d *dnsServer) shutdownServer(srv *dns.Server, started chan struct{}, reason string) {
if srv == nil {
return
}
if started != nil {
<-started
}
if err := srv.Shutdown(); err != nil {
d.l.WithError(err).WithField("reason", reason).Warn("Failed to shut down the DNS responder")
}
}
// Start binds and serves the DNS responder. Blocks until Stop is called or
// the listener errors. Safe to call when DNS is disabled (returns
// immediately). This is what Control.dnsStart points at.
//
// Must be invoked after the tun device is active so that lighthouse.dns.host
// may bind to a nebula IP.
func (d *dnsServer) Start() {
if !d.enabled.Load() {
return
}
started := make(chan struct{})
d.serverMu.Lock()
if d.ctx.Err() != nil {
d.serverMu.Unlock()
return
}
addr := d.addr
server := &dns.Server{
Addr: addr,
Net: "udp",
Handler: d.mux,
NotifyStartedFunc: func() { close(started) },
}
d.server = server
d.started = started
d.serverMu.Unlock()
// Per-invocation ctx watcher. Exits when Start does, so we don't leak a
// watcher per reload-driven restart.
done := make(chan struct{})
go func() {
select {
case <-d.ctx.Done():
d.shutdownServer(server, started, "shutdown")
case <-done:
}
}()
d.l.WithField("dnsListener", addr).Info("Starting DNS responder")
err := server.ListenAndServe()
close(done)
// If the listener never bound (bind error) NotifyStartedFunc never fires,
// so close started here to release any Stop caller waiting on it.
select {
case <-started:
default:
close(started)
}
if err != nil {
d.l.WithError(err).Warn("Failed to run the DNS responder")
}
}
// Stop shuts down the active server, if any. Idempotent.
func (d *dnsServer) Stop() {
d.serverMu.Lock()
srv := d.server
started := d.started
d.server = nil
d.started = nil
d.serverMu.Unlock()
d.shutdownServer(srv, started, "stop")
}
func (d *dnsServer) Query(q uint16, data string) netip.Addr {
func (d *dnsRecords) Query(q uint16, data string) netip.Addr {
data = strings.ToLower(data)
d.RLock()
defer d.RUnlock()
@@ -234,7 +57,7 @@ func (d *dnsServer) Query(q uint16, data string) netip.Addr {
return netip.Addr{}
}
func (d *dnsServer) QueryCert(data string) string {
func (d *dnsRecords) QueryCert(data string) string {
ip, err := netip.ParseAddr(data[:len(data)-1])
if err != nil {
return ""
@@ -257,19 +80,8 @@ func (d *dnsServer) QueryCert(data string) string {
return string(b)
}
// clearRecords drops all DNS records.
func (d *dnsServer) clearRecords() {
d.Lock()
defer d.Unlock()
clear(d.dnsMap4)
clear(d.dnsMap6)
}
// Add adds the first IPv4 and IPv6 address that appears in `addresses` as the record for `host`
func (d *dnsServer) Add(host string, addresses []netip.Addr) {
if !d.enabled.Load() {
return
}
func (d *dnsRecords) Add(host string, addresses []netip.Addr) {
host = strings.ToLower(host)
d.Lock()
defer d.Unlock()
@@ -289,7 +101,7 @@ func (d *dnsServer) Add(host string, addresses []netip.Addr) {
}
}
func (d *dnsServer) isSelfNebulaOrLocalhost(addr string) bool {
func (d *dnsRecords) isSelfNebulaOrLocalhost(addr string) bool {
a, _, _ := net.SplitHostPort(addr)
b, err := netip.ParseAddr(a)
if err != nil {
@@ -304,7 +116,7 @@ func (d *dnsServer) isSelfNebulaOrLocalhost(addr string) bool {
return d.myVpnAddrsTable.Contains(b)
}
func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
func (d *dnsRecords) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
for _, q := range m.Question {
switch q.Qtype {
case dns.TypeA, dns.TypeAAAA:
@@ -338,7 +150,7 @@ func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
}
}
func (d *dnsServer) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
func (d *dnsRecords) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
m := new(dns.Msg)
m.SetReply(r)
m.Compress = false
@@ -351,6 +163,21 @@ func (d *dnsServer) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
w.WriteMsg(m)
}
func dnsMain(l *logrus.Logger, cs *CertState, hostMap *HostMap, c *config.C) func() {
dnsR = newDnsRecords(l, cs, hostMap)
// attach request handler func
dns.HandleFunc(".", dnsR.handleDnsRequest)
c.RegisterReloadCallback(func(c *config.C) {
reloadDns(l, c)
})
return func() {
startDns(l, c)
}
}
func getDnsServerAddr(c *config.C) string {
dnsHost := strings.TrimSpace(c.GetString("lighthouse.dns.host", ""))
// Old guidance was to provide the literal `[::]` in `lighthouse.dns.host` but that won't resolve.
@@ -359,3 +186,25 @@ func getDnsServerAddr(c *config.C) string {
}
return net.JoinHostPort(dnsHost, strconv.Itoa(c.GetInt("lighthouse.dns.port", 53)))
}
func startDns(l *logrus.Logger, c *config.C) {
dnsAddr = getDnsServerAddr(c)
dnsServer = &dns.Server{Addr: dnsAddr, Net: "udp"}
l.WithField("dnsListener", dnsAddr).Info("Starting DNS responder")
err := dnsServer.ListenAndServe()
defer dnsServer.Shutdown()
if err != nil {
l.Errorf("Failed to start server: %s\n ", err.Error())
}
}
func reloadDns(l *logrus.Logger, c *config.C) {
if dnsAddr == getDnsServerAddr(c) {
l.Debug("No DNS server config change detected")
return
}
l.Debug("Restarting DNS server")
dnsServer.Shutdown()
go startDns(l, c)
}
+1 -219
View File
@@ -1,31 +1,19 @@
package nebula
import (
"context"
"io"
"net"
"net/netip"
"strconv"
"testing"
"time"
"github.com/miekg/dns"
"github.com/sirupsen/logrus"
"github.com/slackhq/nebula/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParsequery(t *testing.T) {
l := logrus.New()
hostMap := &HostMap{}
ds := &dnsServer{
l: l,
dnsMap4: make(map[string]netip.Addr),
dnsMap6: make(map[string]netip.Addr),
hostMap: hostMap,
}
ds.enabled.Store(true)
ds := newDnsRecords(l, &CertState{}, hostMap)
addrs := []netip.Addr{
netip.MustParseAddr("1.2.3.4"),
netip.MustParseAddr("1.2.3.5"),
@@ -83,209 +71,3 @@ func Test_getDnsServerAddr(t *testing.T) {
}
assert.Equal(t, "[::]:1", getDnsServerAddr(c))
}
func newTestDnsServer(t *testing.T) (*dnsServer, *config.C) {
t.Helper()
l := logrus.New()
l.Out = io.Discard
ds := &dnsServer{
l: l,
ctx: context.Background(),
dnsMap4: make(map[string]netip.Addr),
dnsMap6: make(map[string]netip.Addr),
hostMap: &HostMap{},
}
ds.mux = dns.NewServeMux()
ds.mux.HandleFunc(".", ds.handleDnsRequest)
return ds, config.NewC(l)
}
func setDnsConfig(c *config.C, host string, port string, amLighthouse, serveDns bool) {
c.Settings["lighthouse"] = map[string]any{
"am_lighthouse": amLighthouse,
"serve_dns": serveDns,
"dns": map[string]any{
"host": host,
"port": port,
},
}
}
func TestDnsServer_reload_initial_disabled(t *testing.T) {
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", "0", true, false)
require.NoError(t, ds.reload(c, true))
assert.False(t, ds.enabled.Load())
assert.Equal(t, "127.0.0.1:0", ds.addr)
assert.Nil(t, ds.server)
}
func TestDnsServer_reload_initial_enabled(t *testing.T) {
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", "0", true, true)
require.NoError(t, ds.reload(c, true))
assert.True(t, ds.enabled.Load())
assert.Equal(t, "127.0.0.1:0", ds.addr)
// initial never starts a runner; that's Control.Start's job
assert.Nil(t, ds.server)
}
func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) {
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", "0", false, true)
require.NoError(t, ds.reload(c, true))
// Wants DNS but isn't a lighthouse: gated off, no runner.
assert.False(t, ds.enabled.Load())
}
func TestDnsServer_reload_sameAddr_noOp(t *testing.T) {
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", "0", true, true)
require.NoError(t, ds.reload(c, true))
// No server running yet, no addr change. Reload should not spawn anything.
require.NoError(t, ds.reload(c, false))
assert.True(t, ds.enabled.Load())
assert.Nil(t, ds.server)
}
func TestDnsServer_StartStop_lifecycle(t *testing.T) {
// Bind to a real (random) UDP port so we exercise the actual
// ListenAndServe + Shutdown plumbing including the started-chan race fix.
port := freeUDPPort(t)
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", port, true, true)
require.NoError(t, ds.reload(c, true))
done := make(chan struct{})
go func() {
ds.Start()
close(done)
}()
waitFor(t, func() bool {
ds.serverMu.Lock()
started := ds.started
ds.serverMu.Unlock()
if started == nil {
return false
}
select {
case <-started:
return true
default:
return false
}
})
ds.Stop()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("Start did not return after Stop")
}
}
func TestDnsServer_Stop_beforeBind_doesNotHang(t *testing.T) {
// Stop called immediately after Start should not deadlock even if bind
// hasn't completed yet. This exercises the started-chan close-on-bind-fail
// path: by binding to an obviously bad port (privileged) we get a fast
// bind error before NotifyStartedFunc fires.
ds, c := newTestDnsServer(t)
// Use a port that should fail to bind (negative would be invalid, use a
// host that won't resolve to ensure listenUDP fails quickly).
setDnsConfig(c, "256.256.256.256", "53", true, true)
require.NoError(t, ds.reload(c, true))
done := make(chan struct{})
go func() {
ds.Start()
close(done)
}()
// Give Start a moment to attempt the bind and fail.
select {
case <-done:
// Bind failed and Start returned; Stop should be a no-op.
case <-time.After(time.Second):
t.Fatal("Start did not return after a bad bind")
}
stopped := make(chan struct{})
go func() {
ds.Stop()
close(stopped)
}()
select {
case <-stopped:
case <-time.After(time.Second):
t.Fatal("Stop hung after a failed bind")
}
}
func TestDnsServer_reload_disable_stopsRunningServer(t *testing.T) {
port := freeUDPPort(t)
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", port, true, true)
require.NoError(t, ds.reload(c, true))
startReturned := make(chan struct{})
go func() {
ds.Start()
close(startReturned)
}()
waitForBind(t, ds)
// Toggle serve_dns off; reload should shut the running server down.
setDnsConfig(c, "127.0.0.1", port, true, false)
require.NoError(t, ds.reload(c, false))
select {
case <-startReturned:
case <-time.After(5 * time.Second):
t.Fatal("Start did not return after reload disabled DNS")
}
assert.False(t, ds.enabled.Load())
}
func freeUDPPort(t *testing.T) string {
t.Helper()
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
port := conn.LocalAddr().(*net.UDPAddr).Port
require.NoError(t, conn.Close())
return strconv.Itoa(port)
}
func waitForBind(t *testing.T, ds *dnsServer) {
t.Helper()
waitFor(t, func() bool {
ds.serverMu.Lock()
started := ds.started
ds.serverMu.Unlock()
if started == nil {
return false
}
select {
case <-started:
return true
default:
return false
}
})
}
func waitFor(t *testing.T, cond func() bool) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(5 * time.Millisecond)
}
t.Fatal("timed out waiting for condition")
}
-565
View File
@@ -1,565 +0,0 @@
//go:build e2e_testing
// +build e2e_testing
package e2e
import (
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/e2e/router"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
)
// makeHandshakePacket creates a handshake packet with the given parameters.
func makeHandshakePacket(from, to netip.AddrPort, subtype header.MessageSubType, remoteIndex uint32, counter uint64) *udp.Packet {
data := make([]byte, 200)
header.Encode(data, header.Version, header.Handshake, subtype, remoteIndex, counter)
for i := header.Len; i < len(data); i++ {
data[i] = byte(i)
}
return &udp.Packet{To: to, From: from, Data: data}
}
func TestHandshakeRetransmitDuplicate(t *testing.T) {
// Verify the responder correctly handles receiving the same msg1 multiple times
// (retransmission). The duplicate goes through CheckAndComplete -> ErrAlreadySeen
// and the cached response is resent.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
myControl.Start()
theirControl.Start()
r := router.NewR(t, myControl, theirControl)
defer r.RenderFlow()
t.Log("Trigger handshake from me to them")
myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
t.Log("Grab my msg1")
msg1 := myControl.GetFromUDP(true)
t.Log("Inject msg1 into them, first time")
theirControl.InjectUDPPacket(msg1)
_ = theirControl.GetFromUDP(true)
t.Log("Inject the SAME msg1 again, tests ErrAlreadySeen path")
theirControl.InjectUDPPacket(msg1)
resp2 := theirControl.GetFromUDP(true)
assert.NotNil(t, resp2, "should get cached response on duplicate msg1")
t.Log("Complete handshake with cached response")
myControl.InjectUDPPacket(resp2)
myControl.WaitForType(1, 0, theirControl)
t.Log("Drain cached packet and verify tunnel works")
cachedPacket := theirControl.GetFromTun(true)
assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
t.Log("Verify only one tunnel exists on each side")
assert.Len(t, myControl.ListHostmapHosts(false), 1)
assert.Len(t, theirControl.ListHostmapHosts(false), 1)
myControl.Stop()
theirControl.Stop()
}
func TestHandshakeTruncatedPacketRecovery(t *testing.T) {
// Verify that a truncated handshake packet is ignored and the real
// packet can still complete the handshake.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
myControl.Start()
theirControl.Start()
r := router.NewR(t, myControl, theirControl)
defer r.RenderFlow()
t.Log("Trigger handshake")
myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
t.Log("Get msg1 and deliver to responder")
msg1 := myControl.GetFromUDP(true)
theirControl.InjectUDPPacket(msg1)
t.Log("Get the real response")
realResp := theirControl.GetFromUDP(true)
t.Log("Truncate the response and inject, should be ignored")
truncResp := realResp.Copy()
truncResp.Data = truncResp.Data[:header.Len]
myControl.InjectUDPPacket(truncResp)
t.Log("Verify pending handshake survived the truncated packet")
assert.NotEmpty(t, myControl.ListHostmapHosts(true), "pending handshake should still exist")
t.Log("Inject real response, should complete handshake")
myControl.InjectUDPPacket(realResp)
myControl.WaitForType(1, 0, theirControl)
t.Log("Drain and verify tunnel")
cachedPacket := theirControl.GetFromTun(true)
assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
myControl.Stop()
theirControl.Stop()
}
func TestHandshakeOrphanedMsg2Dropped(t *testing.T) {
// A msg2 arriving with no matching pending index should be silently dropped
// with no response sent and no state changes.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
myControl.Start()
theirControl.Start()
r := router.NewR(t, myControl, theirControl)
defer r.RenderFlow()
t.Log("Complete a normal handshake")
myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
r.RouteForAllUntilTxTun(theirControl)
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
t.Log("Record hostmap state")
myIndexes := len(myControl.ListHostmapIndexes(false))
t.Log("Inject a fake msg2 with unknown RemoteIndex")
myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0xDEADBEEF, 2))
t.Log("Verify no new indexes created")
assert.Equal(t, myIndexes, len(myControl.ListHostmapIndexes(false)))
t.Log("Verify no UDP response was sent")
time.Sleep(100 * time.Millisecond)
assert.Nil(t, myControl.GetFromUDP(false), "should not send a response to orphaned msg2")
t.Log("Verify existing tunnel still works")
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
myControl.Stop()
theirControl.Stop()
}
func TestHandshakeUnknownMessageCounter(t *testing.T) {
// A handshake packet with an unexpected message counter should be silently
// dropped with no side effects and no UDP response.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
myControl.Start()
theirControl.Start()
t.Log("Inject handshake with MessageCounter=3")
myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 3))
t.Log("Inject handshake with MessageCounter=99")
myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 99))
t.Log("Verify no tunnels or pending handshakes")
assert.Empty(t, myControl.ListHostmapHosts(false))
assert.Empty(t, myControl.ListHostmapHosts(true))
t.Log("Verify no UDP response was sent")
time.Sleep(100 * time.Millisecond)
assert.Nil(t, myControl.GetFromUDP(false))
myControl.Stop()
theirControl.Stop()
}
func TestHandshakeUnknownSubtype(t *testing.T) {
// A handshake packet with an unknown subtype should be silently dropped.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
theirControl, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
myControl.Start()
theirControl.Start()
t.Log("Inject handshake with unknown subtype 99")
myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.MessageSubType(99), 0, 1))
t.Log("Verify no tunnels or pending handshakes")
assert.Empty(t, myControl.ListHostmapHosts(false))
assert.Empty(t, myControl.ListHostmapHosts(true))
t.Log("Verify no UDP response was sent")
time.Sleep(100 * time.Millisecond)
assert.Nil(t, myControl.GetFromUDP(false))
myControl.Stop()
theirControl.Stop()
}
func TestHandshakeLateResponse(t *testing.T) {
// After a handshake times out, a late response should be silently ignored
// with no new tunnels created.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{
"handshakes": m{
"try_interval": "200ms",
"retries": 2,
},
})
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
myControl.Start()
theirControl.Start()
t.Log("Trigger handshake from me")
myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
t.Log("Grab msg1 but don't deliver")
msg1 := myControl.GetFromUDP(true)
t.Log("Wait for handshake to time out")
for i := 0; i < 5; i++ {
time.Sleep(300 * time.Millisecond)
myControl.GetFromUDP(false)
}
t.Log("Confirm no pending handshakes remain")
assert.Empty(t, myControl.ListHostmapHosts(true))
t.Log("Deliver old msg1 to them, they create a tunnel")
theirControl.InjectUDPPacket(msg1)
resp := theirControl.GetFromUDP(true)
assert.NotNil(t, resp)
t.Log("Inject late response into me, should be ignored")
myControl.InjectUDPPacket(resp)
t.Log("No tunnel should exist on my side")
assert.Empty(t, myControl.ListHostmapHosts(false))
assert.Empty(t, myControl.ListHostmapHosts(true))
myControl.Stop()
theirControl.Stop()
}
func TestHandshakeSelfConnectionRejected(t *testing.T) {
// Verify that a node rejects a handshake containing its own VPN IP in the
// peer cert. We do this by sending the initiator's own msg1 back to itself.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
// Need a lighthouse entry to trigger a handshake
myControl.InjectLightHouseAddr(netip.MustParseAddr("10.128.0.2"), netip.MustParseAddrPort("10.0.0.2:4242"))
myControl.Start()
t.Log("Trigger handshake from me")
myControl.InjectTunUDPPacket(netip.MustParseAddr("10.128.0.2"), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
msg1 := myControl.GetFromUDP(true)
t.Log("Drain any handshake retransmits before injecting")
time.Sleep(100 * time.Millisecond)
for myControl.GetFromUDP(false) != nil {
}
t.Log("Feed my own msg1 back to me as if it came from someone else")
selfMsg := msg1.Copy()
selfMsg.From = netip.MustParseAddrPort("10.0.0.99:4242")
selfMsg.To = myUdpAddr
myControl.InjectUDPPacket(selfMsg)
t.Log("Verify no response was sent (self-connection rejected)")
time.Sleep(100 * time.Millisecond)
// Drain any further retransmits from the original handshake, then check
// that none of them are a handshake response (MessageCounter=2)
h := &header.H{}
for {
p := myControl.GetFromUDP(false)
if p == nil {
break
}
_ = h.Parse(p.Data)
assert.NotEqual(t, uint64(2), h.MessageCounter,
"should not send a stage 2 response to self-connection")
}
t.Log("Verify no tunnel to myself was created")
assert.Nil(t, myControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false))
myControl.Stop()
}
func TestHandshakeMessageCounter0Dropped(t *testing.T) {
// MessageCounter=0 is not a valid handshake message and should be dropped.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
_, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
myControl.Start()
t.Log("Inject handshake with MessageCounter=0")
myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 0))
time.Sleep(100 * time.Millisecond)
assert.Empty(t, myControl.ListHostmapHosts(false))
assert.Empty(t, myControl.ListHostmapHosts(true))
assert.Nil(t, myControl.GetFromUDP(false))
myControl.Stop()
}
func TestHandshakeRemoteAllowList(t *testing.T) {
// Verify that a handshake from a blocked underlay IP is dropped with no
// response and no state changes. Then verify the same packet from an
// allowed IP succeeds.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{
"lighthouse": m{
"remote_allow_list": m{
"10.0.0.0/8": true,
"0.0.0.0/0": false,
},
},
})
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
myControl.Start()
theirControl.Start()
r := router.NewR(t, myControl, theirControl)
defer r.RenderFlow()
t.Log("Trigger handshake from them")
theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi"))
msg1 := theirControl.GetFromUDP(true)
t.Log("Rewrite the source to a blocked IP and inject")
blockedMsg := msg1.Copy()
blockedMsg.From = netip.MustParseAddrPort("192.168.1.1:4242")
myControl.InjectUDPPacket(blockedMsg)
t.Log("Verify no tunnel, no pending, no response from blocked source")
time.Sleep(100 * time.Millisecond)
assert.Empty(t, myControl.ListHostmapHosts(false))
assert.Empty(t, myControl.ListHostmapHosts(true))
assert.Nil(t, myControl.GetFromUDP(false), "should not respond to blocked source")
t.Log("Now inject the real packet from the allowed source")
myControl.InjectUDPPacket(msg1)
t.Log("Verify handshake completes from allowed source")
resp := myControl.GetFromUDP(true)
assert.NotNil(t, resp)
theirControl.InjectUDPPacket(resp)
theirControl.WaitForType(1, 0, myControl)
t.Log("Drain cached packet and verify tunnel works")
cachedPacket := myControl.GetFromTun(true)
assertUdpPacket(t, []byte("Hi"), cachedPacket, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
myControl.Stop()
theirControl.Stop()
}
func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) {
// When a duplicate msg1 arrives via ErrAlreadySeen, verify the tunnel
// remains functional and hostmap index count is stable.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
myControl.Start()
theirControl.Start()
r := router.NewR(t, myControl, theirControl)
defer r.RenderFlow()
t.Log("Complete a normal handshake via the router")
myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
r.RouteForAllUntilTxTun(theirControl)
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
t.Log("Record hostmap state")
theirIndexes := len(theirControl.ListHostmapIndexes(false))
hi := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
assert.NotNil(t, hi)
originalRemote := hi.CurrentRemote
t.Log("Re-trigger traffic to cause a new handshake attempt (ErrAlreadySeen)")
myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("roam"))
r.RouteForAllUntilTxTun(theirControl)
t.Log("Verify tunnel still works")
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
t.Log("Verify remote is still valid and index count is stable")
hi2 := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
assert.NotNil(t, hi2)
assert.Equal(t, originalRemote, hi2.CurrentRemote)
assert.Equal(t, theirIndexes, len(theirControl.ListHostmapIndexes(false)),
"no extra indexes should be created from ErrAlreadySeen")
myControl.Stop()
theirControl.Stop()
}
func TestHandshakeWrongResponderPacketStore(t *testing.T) {
// Verify that when the wrong host responds, the cached packets are
// transferred to the new handshake, the evil tunnel is closed, evil's
// address is blocked, and the correct tunnel is eventually established.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.100/24", nil)
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.99/24", nil)
evilControl, evilVpnIpNet, evilUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "evil", "10.128.0.2/24", nil)
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), evilUdpAddr)
r := router.NewR(t, myControl, theirControl, evilControl)
defer r.RenderFlow()
myControl.Start()
theirControl.Start()
evilControl.Start()
t.Log("Send multiple packets to them (cached during handshake)")
myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet1"))
myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet2"))
t.Log("Route until evil tunnel is closed")
h := &header.H{}
r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
if err := h.Parse(p.Data); err != nil {
panic(err)
}
if h.Type == header.CloseTunnel && p.To == evilUdpAddr {
return router.RouteAndExit
}
return router.KeepRouting
})
t.Log("Verify evil's address is blocked in the new pending handshake")
pendingHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), true)
if pendingHI != nil {
assert.NotContains(t, pendingHI.RemoteAddrs, evilUdpAddr,
"evil's address should be blocked")
}
t.Log("Inject correct lighthouse addr for them")
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
t.Log("Route until cached packets arrive at the real them")
p := r.RouteForAllUntilTxTun(theirControl)
assert.NotNil(t, p, "a cached packet should be delivered to the correct host")
t.Log("Verify the correct host has a tunnel")
assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl)
t.Log("Verify no hostinfo artifacts from evil remain")
assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), true),
"no pending hostinfo for evil")
assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), false),
"no main hostinfo for evil")
myControl.Stop()
theirControl.Stop()
evilControl.Stop()
}
func TestHandshakeRelayComplete(t *testing.T) {
// Verify that a relay handshake completes correctly and relay state is
// properly maintained on all three nodes.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
r := router.NewR(t, myControl, relayControl, theirControl)
defer r.RenderFlow()
myControl.Start()
relayControl.Start()
theirControl.Start()
t.Log("Trigger handshake via relay")
myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi via relay"))
p := r.RouteForAllUntilTxTun(theirControl)
assertUdpPacket(t, []byte("Hi via relay"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
t.Log("Verify bidirectional tunnel via relay")
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
t.Log("Verify relay state on my side shows relay-to-me")
myHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
assert.NotNil(t, myHI)
assert.NotEmpty(t, myHI.CurrentRelaysToMe, "should have relay-to-me for them")
t.Log("Verify relay state on their side shows relay-to-me")
theirHI := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
assert.NotNil(t, theirHI)
assert.NotEmpty(t, theirHI.CurrentRelaysToMe, "should have relay-to-me for me")
t.Log("Verify relay node shows through-me relays")
relayHI := relayControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
assert.NotNil(t, relayHI)
myControl.Stop()
relayControl.Stop()
theirControl.Stop()
}
// NOTE: Relay V1 cert + IPv6 rejection is not tested here because
// InjectTunUDPPacket from a V4 node to a V6 address panics in the test
// framework. The check is in handshake_manager.go handleOutbound relay
// logic (lines ~304-313): if the relay host has a V1 cert and either
// address is IPv6, the relay is skipped.
// NOTE: Relay reestablishment (Disestablished state transition) is covered
// by the existing TestReestablishRelays in handshakes_test.go.
+8 -6
View File
@@ -204,12 +204,6 @@ punchy:
# Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
#trusted_cas:
#- "ssh public key string"
# sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
# save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
# and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
# The directory is NOT automatically created.
# Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
#sandbox_dir: /var/tmp/nebula-debug
# EXPERIMENTAL: relay support for networks that can't establish direct connections.
relay:
@@ -348,6 +342,14 @@ logging:
# after receiving the response for lighthouse queries
#trigger_buffer: 64
# max_rate limits the number of new inbound handshakes per second. Once the limit is reached,
# new handshakes are dropped until the next second. A value of 0 means unlimited (default).
# This is useful for preventing DoS attacks that attempt to exhaust CPU with handshake crypto.
# Running `openssl speed ecdhp256` on your hardware can be a good rule of thumb for choosing
# a max, as each handshake performs similar DH operations. Note that this benchmarks a single
# core, so you may wish to scale the value by the number of `routines` configured.
#max_rate: 0
# Tunnel manager settings
#tunnels:
# drop_inactive controls whether inactive tunnels are maintained or dropped after the inactive_timeout period has
+5 -88
View File
@@ -20,7 +20,6 @@ import (
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/firewall/events"
)
type FirewallInterface interface {
@@ -68,14 +67,6 @@ type Firewall struct {
incomingMetrics firewallMetrics
outgoingMetrics firewallMetrics
// reporter is the optional embedder-supplied event sink. Immutable for
// the lifetime of this Firewall; Control.SetFirewallEventReporter
// installs it by shallow-copying the Firewall under the conntrack lock
// and swapping the pointer, and reloadFirewall carries it forward.
// Read unsynchronized on the data path: the preceding Firewall-pointer
// read pins the field's value for the duration of that call.
reporter events.Reporter
l *logrus.Logger
}
@@ -425,27 +416,23 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
// Drop returns an error if the packet should be dropped, explaining why. It
// returns nil if the packet should not be dropped.
func (f *Firewall) Drop(fp firewall.Packet, ctx firewall.PacketContext, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
// Check if we spoke to this tuple, if we did then allow this packet
if f.inConns(fp, h, caPool, localCache) {
return nil
}
peerCert := h.ConnectionState.peerCert
// Make sure remote address matches nebula certificate, and determine how to treat it
if h.networks == nil {
// Simple case: Certificate has one address and no unsafe networks
if h.vpnAddrs[0] != fp.RemoteAddr {
f.metrics(incoming).droppedRemoteAddr.Inc(1)
f.reportDrop(incoming, events.DropInvalidRemoteIP, fp, ctx, peerCert)
return ErrInvalidRemoteIP
}
} else {
nwType, ok := h.networks.Lookup(fp.RemoteAddr)
if !ok {
f.metrics(incoming).droppedRemoteAddr.Inc(1)
f.reportDrop(incoming, events.DropInvalidRemoteIP, fp, ctx, peerCert)
return ErrInvalidRemoteIP
}
switch nwType {
@@ -453,13 +440,11 @@ func (f *Firewall) Drop(fp firewall.Packet, ctx firewall.PacketContext, incoming
break // nothing special
case NetworkTypeVPNPeer:
f.metrics(incoming).droppedRemoteAddr.Inc(1)
f.reportDrop(incoming, events.DropPeerRejected, fp, ctx, peerCert)
return ErrPeerRejected // reject for now, one day this may have different FW rules
case NetworkTypeUnsafe:
break // nothing special, one day this may have different FW rules
default:
f.metrics(incoming).droppedRemoteAddr.Inc(1)
f.reportDrop(incoming, events.DropUnknownNetwork, fp, ctx, peerCert)
return ErrUnknownNetworkType //should never happen
}
}
@@ -467,7 +452,6 @@ func (f *Firewall) Drop(fp firewall.Packet, ctx firewall.PacketContext, incoming
// Make sure we are supposed to be handling this local ip address
if !f.routableNetworks.Contains(fp.LocalAddr) {
f.metrics(incoming).droppedLocalAddr.Inc(1)
f.reportDrop(incoming, events.DropInvalidLocalIP, fp, ctx, peerCert)
return ErrInvalidLocalIP
}
@@ -477,14 +461,13 @@ func (f *Firewall) Drop(fp firewall.Packet, ctx firewall.PacketContext, incoming
}
// We now know which firewall table to check against
if !table.match(fp, incoming, peerCert, caPool) {
if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
f.metrics(incoming).droppedNoRule.Inc(1)
f.reportDrop(incoming, events.DropNoMatchingRule, fp, ctx, peerCert)
return ErrNoMatchingRule
}
// We always want to conntrack since it is a faster operation
f.addConn(fp, ctx, incoming, peerCert)
f.addConn(fp, incoming)
return nil
}
@@ -503,59 +486,6 @@ func (f *Firewall) Destroy() {
//TODO: clean references if/when needed
}
func (f *Firewall) reportDrop(incoming bool, reason events.DropReason, fp firewall.Packet, ctx firewall.PacketContext, peerCert *cert.CachedCertificate) {
r := f.reporter
if r == nil {
return
}
r.ReportDrop(events.DropEvent{
Incoming: incoming,
Reason: reason,
Packet: fp,
Context: ctx,
PeerCert: peerCert,
RulesVersion: f.rulesVersion,
})
}
func (f *Firewall) reportFlowCreate(incoming bool, fp firewall.Packet, ctx firewall.PacketContext, peerCert *cert.CachedCertificate) {
r := f.reporter
if r == nil {
return
}
r.ReportFlowCreate(events.FlowCreateEvent{
Incoming: incoming,
Packet: fp,
Context: ctx,
PeerCert: peerCert,
RulesVersion: f.rulesVersion,
})
}
func (f *Firewall) reportFlowEvict(incoming bool, fp firewall.Packet, rulesVersion uint16, expired bool) {
r := f.reporter
if r == nil {
return
}
r.ReportFlowEvict(events.FlowEvictEvent{
Incoming: incoming,
Packet: fp,
RulesVersion: rulesVersion,
Expired: expired,
})
}
func (f *Firewall) reportRulesReload(oldVersion, newVersion uint16) {
r := f.reporter
if r == nil {
return
}
r.ReportRulesReload(events.RulesReloadEvent{
OldVersion: oldVersion,
NewVersion: newVersion,
})
}
func (f *Firewall) EmitStats() {
conntrack := f.Conntrack
conntrack.Lock()
@@ -606,9 +536,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
WithField("oldRulesVersion", c.rulesVersion).
Debugln("dropping old conntrack entry, does not match new ruleset")
}
oldRulesVersion := c.rulesVersion
delete(conntrack.Conns, fp)
f.reportFlowEvict(c.incoming, fp, oldRulesVersion, false)
conntrack.Unlock()
return false
}
@@ -643,7 +571,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
return true
}
func (f *Firewall) addConn(fp firewall.Packet, ctx firewall.PacketContext, incoming bool, peerCert *cert.CachedCertificate) {
func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
var timeout time.Duration
c := &conn{}
@@ -658,8 +586,7 @@ func (f *Firewall) addConn(fp firewall.Packet, ctx firewall.PacketContext, incom
conntrack := f.Conntrack
conntrack.Lock()
_, existing := conntrack.Conns[fp]
if !existing {
if _, ok := conntrack.Conns[fp]; !ok {
conntrack.TimerWheel.Advance(time.Now())
conntrack.TimerWheel.Add(fp, timeout)
}
@@ -670,13 +597,6 @@ func (f *Firewall) addConn(fp firewall.Packet, ctx firewall.PacketContext, incom
c.rulesVersion = f.rulesVersion
c.Expires = time.Now().Add(timeout)
conntrack.Conns[fp] = c
// Report only when this represents a genuinely new flow. Fires under the
// conntrack lock so FlowCreate/FlowEvict events stay ordered relative to
// RulesReloadEvent, which also fires under this lock.
if !existing {
f.reportFlowCreate(incoming, fp, ctx, peerCert)
}
conntrack.Unlock()
}
@@ -700,10 +620,7 @@ func (f *Firewall) evict(p firewall.Packet) {
}
// This conn is done
rulesVersion := t.rulesVersion
incoming := t.incoming
delete(conntrack.Conns, p)
f.reportFlowEvict(incoming, p, rulesVersion, true)
}
func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
+5 -12
View File
@@ -1,7 +1,6 @@
package firewall
import (
"context"
"sync/atomic"
"time"
@@ -19,7 +18,7 @@ type ConntrackCacheTicker struct {
cache ConntrackCache
}
func NewConntrackCacheTicker(ctx context.Context, d time.Duration) *ConntrackCacheTicker {
func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
if d == 0 {
return nil
}
@@ -28,21 +27,15 @@ func NewConntrackCacheTicker(ctx context.Context, d time.Duration) *ConntrackCac
cache: ConntrackCache{},
}
go c.tick(ctx, d)
go c.tick(d)
return c
}
func (c *ConntrackCacheTicker) tick(ctx context.Context, d time.Duration) {
t := time.NewTicker(d)
defer t.Stop()
func (c *ConntrackCacheTicker) tick(d time.Duration) {
for {
select {
case <-ctx.Done():
return
case <-t.C:
c.cacheTick.Add(1)
}
time.Sleep(d)
c.cacheTick.Add(1)
}
}
-102
View File
@@ -1,102 +0,0 @@
// Package events defines the opt-in firewall event reporting interface.
//
// Nebula emits raw packet-level events (drops, flow creations, flow evictions,
// rule reloads) and does no aggregation, counting, batching, rule-description,
// transport, or timestamping. Embedders correlate events back to yaml rules
// out of band and capture whatever clock they need themselves. All Report*
// methods are invoked while nebula holds internal locks and must be
// non-blocking.
//
// Events are passed to Report* methods by value. Implementations must not
// take the address of a received event: doing so forces Go's escape
// analysis to move the event to the heap and costs one allocation per call.
// To forward an event, either copy its fields into the reporter's own
// pooled record or send it through a value-typed channel (chan DropEvent,
// not chan *DropEvent).
package events
import (
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/firewall"
)
type DropReason uint8
const (
DropInvalidLocalIP DropReason = iota
DropInvalidRemoteIP
DropPeerRejected
DropUnknownNetwork
DropNoMatchingRule
)
func (r DropReason) String() string {
switch r {
case DropInvalidLocalIP:
return "invalid_local_ip"
case DropInvalidRemoteIP:
return "invalid_remote_ip"
case DropPeerRejected:
return "peer_rejected"
case DropUnknownNetwork:
return "unknown_network"
case DropNoMatchingRule:
return "no_matching_rule"
default:
return "unknown"
}
}
// DropEvent is emitted for every packet that fails the firewall check. Drops
// are not aggregated; every drop produces one event.
type DropEvent struct {
Incoming bool
Reason DropReason
Packet firewall.Packet
Context firewall.PacketContext
PeerCert *cert.CachedCertificate
RulesVersion uint16
}
// FlowCreateEvent is emitted when a packet is allowed and a new conntrack
// entry is created. Subsequent packets in the same flow do not re-emit.
type FlowCreateEvent struct {
Incoming bool
Packet firewall.Packet
Context firewall.PacketContext
PeerCert *cert.CachedCertificate
RulesVersion uint16
}
// FlowEvictEvent is emitted when a conntrack entry is removed. Context is
// not carried: timer-wheel eviction has no packet in hand, and reload
// revalidation evicts the OLD flow rather than the triggering packet.
// RulesVersion is the version under which the flow was originally allowed,
// which may differ from the current firewall version.
type FlowEvictEvent struct {
Incoming bool
Packet firewall.Packet
RulesVersion uint16
// Expired is true when eviction was due to conntrack timeout; false when
// the entry was removed because it failed re-validation after a reload.
Expired bool
}
// RulesReloadEvent is emitted once after each successful firewall reload.
// Reporters that bucket state by RulesVersion should close the old bucket
// and open a new one on receipt.
type RulesReloadEvent struct {
OldVersion uint16
NewVersion uint16
}
// Reporter is the embedder-supplied sink for firewall events. Implementations
// that want a timestamp should call time.Now() themselves at the top of the
// method; nebula does not provide one. See the package doc for the
// do-not-take-address rule.
type Reporter interface {
ReportDrop(DropEvent)
ReportFlowCreate(FlowCreateEvent)
ReportFlowEvict(FlowEvictEvent)
ReportRulesReload(RulesReloadEvent)
}
-21
View File
@@ -31,27 +31,6 @@ type Packet struct {
Fragment bool
}
// PacketContext carries additional parsed details about a packet that are
// useful for event reporting but deliberately kept out of Packet so Packet
// can keep being used as a conntrack map key. Populated alongside Packet by
// newPacket.
//
// Fields are interpreted based on Packet.Protocol:
// - ProtoTCP: TCPFlags is meaningful; ICMPType / ICMPCode are zero
// - ProtoICMP, ProtoICMPv6: ICMPType / ICMPCode are meaningful; TCPFlags is zero
// - ProtoUDP and others: only Length is meaningful
type PacketContext struct {
// Length is the total IP packet length in bytes, including headers.
Length uint16
// TCPFlags is the flag byte from the TCP header (bits for FIN, SYN, RST,
// PSH, ACK, URG, ECE, CWR).
TCPFlags uint8
// ICMPType is the type field of the ICMP / ICMPv6 header.
ICMPType uint8
// ICMPCode is the code field of the ICMP / ICMPv6 header.
ICMPCode uint8
}
func (fp *Packet) Copy() *Packet {
return &Packet{
LocalAddr: fp.LocalAddr,
-731
View File
@@ -1,731 +0,0 @@
package nebula
import (
"net"
"net/netip"
"sync"
"testing"
"time"
"github.com/gaissmai/bart"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/firewall/events"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// recordingReporter captures every event fired against it. Its methods take
// the conntrack lock implicitly (via the firewall code path that invokes
// them), so we synchronize accumulator mutations with a small mutex to keep
// the race detector happy across goroutines in case a test introduces any.
type recordingReporter struct {
mu sync.Mutex
drops []recordedDrop
creates []recordedCreate
evicts []recordedEvict
reloads []recordedReload
}
type recordedDrop struct {
incoming bool
reason events.DropReason
remote netip.Addr
local netip.Addr
peerName string
rulesVersion uint16
ctx firewall.PacketContext
}
type recordedCreate struct {
incoming bool
remote netip.Addr
local netip.Addr
peerName string
rulesVersion uint16
ctx firewall.PacketContext
}
type recordedEvict struct {
incoming bool
remote netip.Addr
local netip.Addr
rulesVersion uint16
expired bool
}
type recordedReload struct {
oldVersion uint16
newVersion uint16
}
func (r *recordingReporter) ReportDrop(e events.DropEvent) {
r.mu.Lock()
defer r.mu.Unlock()
name := ""
if e.PeerCert != nil && e.PeerCert.Certificate != nil {
name = e.PeerCert.Certificate.Name()
}
r.drops = append(r.drops, recordedDrop{
incoming: e.Incoming,
reason: e.Reason,
remote: e.Packet.RemoteAddr,
local: e.Packet.LocalAddr,
peerName: name,
rulesVersion: e.RulesVersion,
ctx: e.Context,
})
}
func (r *recordingReporter) ReportFlowCreate(e events.FlowCreateEvent) {
r.mu.Lock()
defer r.mu.Unlock()
name := ""
if e.PeerCert != nil && e.PeerCert.Certificate != nil {
name = e.PeerCert.Certificate.Name()
}
r.creates = append(r.creates, recordedCreate{
incoming: e.Incoming,
remote: e.Packet.RemoteAddr,
local: e.Packet.LocalAddr,
peerName: name,
rulesVersion: e.RulesVersion,
ctx: e.Context,
})
}
func (r *recordingReporter) ReportFlowEvict(e events.FlowEvictEvent) {
r.mu.Lock()
defer r.mu.Unlock()
r.evicts = append(r.evicts, recordedEvict{
incoming: e.Incoming,
remote: e.Packet.RemoteAddr,
local: e.Packet.LocalAddr,
rulesVersion: e.RulesVersion,
expired: e.Expired,
})
}
func (r *recordingReporter) ReportRulesReload(e events.RulesReloadEvent) {
r.mu.Lock()
defer r.mu.Unlock()
r.reloads = append(r.reloads, recordedReload{
oldVersion: e.OldVersion,
newVersion: e.NewVersion,
})
}
// eventFixture builds a Firewall wired to a Control plus a packet/hostinfo
// pair that a test can reuse. By default the ruleset allows the packet;
// callers mutate fw / p / h as needed before invoking Drop.
type eventFixture struct {
ctl *Control
fw *Firewall
p firewall.Packet
h *HostInfo
cp *cert.CAPool
}
func newEventFixture(t *testing.T) *eventFixture {
t.Helper()
l := test.NewLogger()
// myVpnNetworksTable covers our single peer address so buildNetworks takes
// the "simple case" path (h.networks stays nil); tests that want a populated
// BART table overwrite h.networks directly.
vpnNetworks := new(bart.Lite)
vpnNetworks.Insert(netip.MustParsePrefix("1.2.3.0/24"))
// Use the same cert for "peer" and "local" endpoints, matching the
// TestFirewall_Drop fixture style: LocalAddr == RemoteAddr == peer vpn addr.
c := &dummyCert{
name: "host1",
networks: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/24")},
groups: []string{"default-group"},
issuer: "signer-shasum",
}
h := &HostInfo{
ConnectionState: &ConnectionState{
peerCert: &cert.CachedCertificate{
Certificate: c,
InvertedGroups: map[string]struct{}{"default-group": {}},
},
},
vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
}
h.buildNetworks(vpnNetworks, c)
fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
ctl := &Control{
f: &Interface{firewall: fw},
l: l,
}
return &eventFixture{
ctl: ctl,
fw: fw,
p: firewall.Packet{
LocalAddr: netip.MustParseAddr("1.2.3.4"),
RemoteAddr: netip.MustParseAddr("1.2.3.4"),
LocalPort: 10,
RemotePort: 90,
Protocol: firewall.ProtoUDP,
},
h: h,
cp: cert.NewCAPool(),
}
}
// firewall() returns the currently-installed firewall. Needed because
// SetFirewallEventReporter replaces it via shallow-copy swap.
func (f *eventFixture) firewall() *Firewall {
return f.ctl.f.firewall
}
func TestEvents_ReportDrop_InvalidRemoteIP(t *testing.T) {
f := newEventFixture(t)
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
// Packet to an address not in the cert's networks.
f.p.RemoteAddr = netip.MustParseAddr("9.9.9.9")
assert.Equal(t, ErrInvalidRemoteIP, f.firewall().Drop(f.p, firewall.PacketContext{}, false, f.h, f.cp, nil))
require.Len(t, r.drops, 1)
assert.Equal(t, events.DropInvalidRemoteIP, r.drops[0].reason)
assert.False(t, r.drops[0].incoming)
assert.Equal(t, "host1", r.drops[0].peerName)
assert.Empty(t, r.creates)
assert.Empty(t, r.evicts)
}
func TestEvents_ReportDrop_InvalidLocalIP(t *testing.T) {
f := newEventFixture(t)
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
// LocalAddr outside our routable networks.
f.p.LocalAddr = netip.MustParseAddr("9.9.9.9")
assert.Equal(t, ErrInvalidLocalIP, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
require.Len(t, r.drops, 1)
assert.Equal(t, events.DropInvalidLocalIP, r.drops[0].reason)
assert.True(t, r.drops[0].incoming)
}
func TestEvents_ReportDrop_NoMatchingRule(t *testing.T) {
f := newEventFixture(t)
// Reset to a firewall with no matching rule.
l := test.NewLogger()
fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, f.h.ConnectionState.peerCert.Certificate)
// Rule that won't match (group not in peer's groups).
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", ""))
require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", ""))
f.ctl.f.firewall = fw
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
assert.Equal(t, ErrNoMatchingRule, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
require.Len(t, r.drops, 1)
assert.Equal(t, events.DropNoMatchingRule, r.drops[0].reason)
}
func TestEvents_ReportDrop_PeerRejected(t *testing.T) {
f := newEventFixture(t)
// Re-classify the remote as VPNPeer so it triggers DropPeerRejected.
f.h.networks = new(bart.Table[NetworkType])
f.h.networks.Insert(netip.MustParsePrefix("1.2.3.0/24"), NetworkTypeVPNPeer)
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
assert.Equal(t, ErrPeerRejected, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
require.Len(t, r.drops, 1)
assert.Equal(t, events.DropPeerRejected, r.drops[0].reason)
}
func TestEvents_ReportDrop_UnknownNetwork(t *testing.T) {
f := newEventFixture(t)
// Insert an unrecognized NetworkType value to hit the default branch.
f.h.networks = new(bart.Table[NetworkType])
f.h.networks.Insert(netip.MustParsePrefix("1.2.3.0/24"), NetworkTypeUnknown)
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
assert.Equal(t, ErrUnknownNetworkType, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
require.Len(t, r.drops, 1)
assert.Equal(t, events.DropUnknownNetwork, r.drops[0].reason)
}
func TestEvents_ReportFlowCreate_OnceOnly(t *testing.T) {
f := newEventFixture(t)
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
// First allowed packet creates the conntrack entry.
require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
// Second matching packet on the same tuple is short-circuited by conntrack
// and must not fire another FlowCreate.
require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
require.Len(t, r.creates, 1)
assert.True(t, r.creates[0].incoming)
assert.Equal(t, f.p.RemoteAddr, r.creates[0].remote)
assert.Empty(t, r.drops)
}
func TestEvents_ReportFlowEvict_OnReloadPurge(t *testing.T) {
f := newEventFixture(t)
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
// Create a flow under the current rules.
require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
require.Len(t, r.creates, 1)
// Simulate a reload that produces rules the existing flow no longer
// matches. Bump rulesVersion and replace InRules with an empty table so
// revalidation fails.
fw := f.firewall()
fw.Conntrack.Lock()
fw.rulesVersion++
fw.InRules = newFirewallTable()
fw.Conntrack.Unlock()
// Next packet triggers re-validation, which fails and evicts the entry.
err := fw.Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
assert.Equal(t, ErrNoMatchingRule, err)
require.Len(t, r.evicts, 1)
assert.False(t, r.evicts[0].expired, "evict from reload purge is not expiration")
assert.True(t, r.evicts[0].incoming)
}
func TestEvents_ReportFlowEvict_OnTimeout(t *testing.T) {
f := newEventFixture(t)
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
require.Len(t, r.creates, 1)
// Force expiration by rewinding the entry's deadline.
fw := f.firewall()
fw.Conntrack.Lock()
c := fw.Conntrack.Conns[f.p]
require.NotNil(t, c)
c.Expires = time.Now().Add(-time.Hour)
fw.evict(f.p)
fw.Conntrack.Unlock()
require.Len(t, r.evicts, 1)
assert.True(t, r.evicts[0].expired)
}
func TestEvents_SetNil_Clears(t *testing.T) {
f := newEventFixture(t)
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
require.Len(t, r.creates, 1)
f.ctl.SetFirewallEventReporter(nil)
resetConntrack(f.firewall())
require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
// No second create should be recorded.
assert.Len(t, r.creates, 1)
}
func TestEvents_ReporterSurvivesSwap(t *testing.T) {
f := newEventFixture(t)
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
// Simulate a reload by swapping in a fresh Firewall that carries the
// reporter forward. Mirrors what reloadFirewall does with the shared
// conntrack pointer.
l := test.NewLogger()
oldFw := f.firewall()
newFw := NewFirewall(l, time.Minute, time.Minute, time.Minute, f.h.ConnectionState.peerCert.Certificate)
require.NoError(t, newFw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
require.NoError(t, newFw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
newFw.Conntrack = oldFw.Conntrack
newFw.rulesVersion = oldFw.rulesVersion + 1
newFw.reporter = oldFw.reporter
f.ctl.f.firewall = newFw
newFw.reportRulesReload(oldFw.rulesVersion, newFw.rulesVersion)
require.Len(t, r.reloads, 1)
assert.Equal(t, oldFw.rulesVersion, r.reloads[0].oldVersion)
assert.Equal(t, newFw.rulesVersion, r.reloads[0].newVersion)
// Events on the new firewall should still reach the same reporter.
require.NoError(t, newFw.Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
require.Len(t, r.creates, 1)
assert.Equal(t, newFw.rulesVersion, r.creates[0].rulesVersion)
}
func TestEvents_InstallDoesNotMutateOldFirewall(t *testing.T) {
f := newEventFixture(t)
before := f.firewall()
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
after := f.firewall()
assert.NotSame(t, before, after, "SetFirewallEventReporter must replace the Firewall pointer")
assert.Nil(t, before.reporter, "the pre-install Firewall must remain untouched")
assert.NotNil(t, after.reporter)
}
// --- PacketContext parse tests --------------------------------------------
func mustSerialize(t *testing.T, lrs ...gopacket.SerializableLayer) []byte {
t.Helper()
buf := gopacket.NewSerializeBuffer()
opt := gopacket.SerializeOptions{ComputeChecksums: false, FixLengths: true}
require.NoError(t, gopacket.SerializeLayers(buf, opt, lrs...))
return buf.Bytes()
}
func TestPacketContext_IPv4_TCPFlags(t *testing.T) {
ip := &layers.IPv4{
Version: 4, TTL: 64, Protocol: layers.IPProtocolTCP,
SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
}
tcp := &layers.TCP{SrcPort: 1234, DstPort: 80, SYN: true, ACK: true}
require.NoError(t, tcp.SetNetworkLayerForChecksum(ip))
data := mustSerialize(t, ip, tcp, gopacket.Payload([]byte("hello")))
var fp firewall.Packet
var ctx firewall.PacketContext
require.NoError(t, newPacket(data, true, &fp, &ctx))
assert.Equal(t, uint8(firewall.ProtoTCP), fp.Protocol)
// SYN (0x02) + ACK (0x10) = 0x12
assert.Equal(t, uint8(0x12), ctx.TCPFlags)
assert.Equal(t, uint16(len(data)), ctx.Length)
assert.Equal(t, uint8(0), ctx.ICMPType)
assert.Equal(t, uint8(0), ctx.ICMPCode)
}
func TestPacketContext_IPv4_ICMPTypeCode(t *testing.T) {
ip := &layers.IPv4{
Version: 4, TTL: 64, Protocol: layers.IPProtocolICMPv4,
SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
}
// Destination Unreachable, code 3 (port unreachable)
icmp := &layers.ICMPv4{
TypeCode: layers.CreateICMPv4TypeCode(layers.ICMPv4TypeDestinationUnreachable, layers.ICMPv4CodePort),
}
data := mustSerialize(t, ip, icmp, gopacket.Payload([]byte{0, 0, 0, 0}))
var fp firewall.Packet
var ctx firewall.PacketContext
require.NoError(t, newPacket(data, true, &fp, &ctx))
assert.Equal(t, uint8(firewall.ProtoICMP), fp.Protocol)
assert.Equal(t, uint8(layers.ICMPv4TypeDestinationUnreachable), ctx.ICMPType)
assert.Equal(t, uint8(layers.ICMPv4CodePort), ctx.ICMPCode)
assert.Equal(t, uint16(len(data)), ctx.Length)
assert.Equal(t, uint8(0), ctx.TCPFlags)
}
func TestPacketContext_IPv4_UDPLengthOnly(t *testing.T) {
ip := &layers.IPv4{
Version: 4, TTL: 64, Protocol: layers.IPProtocolUDP,
SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
}
udp := &layers.UDP{SrcPort: 1234, DstPort: 53}
require.NoError(t, udp.SetNetworkLayerForChecksum(ip))
data := mustSerialize(t, ip, udp, gopacket.Payload([]byte("query")))
var fp firewall.Packet
var ctx firewall.PacketContext
require.NoError(t, newPacket(data, true, &fp, &ctx))
assert.Equal(t, uint8(firewall.ProtoUDP), fp.Protocol)
assert.Equal(t, uint16(len(data)), ctx.Length)
assert.Zero(t, ctx.TCPFlags)
assert.Zero(t, ctx.ICMPType)
assert.Zero(t, ctx.ICMPCode)
}
func TestPacketContext_IPv6_TCPFlags(t *testing.T) {
ip := &layers.IPv6{
Version: 6, HopLimit: 64, NextHeader: layers.IPProtocolTCP,
SrcIP: net.ParseIP("fd00::1"), DstIP: net.ParseIP("fd00::2"),
}
tcp := &layers.TCP{SrcPort: 1234, DstPort: 443, FIN: true, ACK: true}
require.NoError(t, tcp.SetNetworkLayerForChecksum(ip))
data := mustSerialize(t, ip, tcp, gopacket.Payload([]byte("bye")))
var fp firewall.Packet
var ctx firewall.PacketContext
require.NoError(t, newPacket(data, true, &fp, &ctx))
assert.Equal(t, uint8(firewall.ProtoTCP), fp.Protocol)
// FIN (0x01) + ACK (0x10) = 0x11
assert.Equal(t, uint8(0x11), ctx.TCPFlags)
assert.Equal(t, uint16(len(data)), ctx.Length)
}
func TestPacketContext_IPv6_ICMPv6TypeCode(t *testing.T) {
ip := &layers.IPv6{
Version: 6, HopLimit: 64, NextHeader: layers.IPProtocolICMPv6,
SrcIP: net.ParseIP("fd00::1"), DstIP: net.ParseIP("fd00::2"),
}
icmp := &layers.ICMPv6{
TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeDestinationUnreachable, layers.ICMPv6CodePortUnreachable),
}
require.NoError(t, icmp.SetNetworkLayerForChecksum(ip))
data := mustSerialize(t, ip, icmp, gopacket.Payload([]byte{0, 0, 0, 0, 0, 0, 0, 0}))
var fp firewall.Packet
var ctx firewall.PacketContext
require.NoError(t, newPacket(data, true, &fp, &ctx))
assert.Equal(t, uint8(firewall.ProtoICMPv6), fp.Protocol)
assert.Equal(t, uint8(layers.ICMPv6TypeDestinationUnreachable), ctx.ICMPType)
assert.Equal(t, uint8(layers.ICMPv6CodePortUnreachable), ctx.ICMPCode)
assert.Equal(t, uint16(len(data)), ctx.Length)
}
// TestPacketContext_NilOK confirms a nil context pointer is accepted by
// newPacket (the hot path may elect not to pass one).
func TestPacketContext_NilOK(t *testing.T) {
ip := &layers.IPv4{
Version: 4, TTL: 64, Protocol: layers.IPProtocolUDP,
SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
}
udp := &layers.UDP{SrcPort: 1, DstPort: 2}
require.NoError(t, udp.SetNetworkLayerForChecksum(ip))
data := mustSerialize(t, ip, udp)
var fp firewall.Packet
require.NoError(t, newPacket(data, true, &fp, nil))
}
// TestPacketContext_FlowCreateCarriesContext exercises the full Drop -> addConn
// -> ReportFlowCreate path with a realistic TCP packet and confirms the
// context makes it into the reporter.
func TestPacketContext_FlowCreateCarriesContext(t *testing.T) {
f := newEventFixture(t)
r := &recordingReporter{}
f.ctl.SetFirewallEventReporter(r)
// Hand-construct a matching TCP packet.
ctx := firewall.PacketContext{Length: 1500, TCPFlags: 0x12}
p := f.p
p.Protocol = firewall.ProtoTCP
require.NoError(t, f.firewall().Drop(p, ctx, true, f.h, f.cp, nil))
require.Len(t, r.creates, 1)
assert.Equal(t, uint16(1500), r.creates[0].ctx.Length)
assert.Equal(t, uint8(0x12), r.creates[0].ctx.TCPFlags)
}
// --- benchmarks ------------------------------------------------------------
// noopReporter is the cheapest possible reporter. Methods discard the event.
type noopReporter struct{}
func (noopReporter) ReportDrop(events.DropEvent) {}
func (noopReporter) ReportFlowCreate(events.FlowCreateEvent) {}
func (noopReporter) ReportFlowEvict(events.FlowEvictEvent) {}
func (noopReporter) ReportRulesReload(events.RulesReloadEvent) {
}
// bufferedReporter demonstrates a realistic zero-alloc reporter: each event
// is forwarded to a value-typed channel. The channel send is a memcpy into
// the channel's pre-allocated ring buffer -- no heap traffic. A background
// goroutine would drain these; the bench skips draining to keep the report
// path pure.
type bufferedReporter struct {
drops chan events.DropEvent
flows chan events.FlowCreateEvent
evicts chan events.FlowEvictEvent
}
func newBufferedReporter(cap int) *bufferedReporter {
return &bufferedReporter{
drops: make(chan events.DropEvent, cap),
flows: make(chan events.FlowCreateEvent, cap),
evicts: make(chan events.FlowEvictEvent, cap),
}
}
func (r *bufferedReporter) ReportDrop(e events.DropEvent) {
select {
case r.drops <- e:
default:
}
}
func (r *bufferedReporter) ReportFlowCreate(e events.FlowCreateEvent) {
select {
case r.flows <- e:
default:
}
}
func (r *bufferedReporter) ReportFlowEvict(e events.FlowEvictEvent) {
select {
case r.evicts <- e:
default:
}
}
func (r *bufferedReporter) ReportRulesReload(events.RulesReloadEvent) {}
// pointerReporter is the anti-pattern: it takes the address of the incoming
// event struct, which forces the callee-side copy onto the heap. Kept for
// comparison so we can see the alloc cost an unwary reporter would incur.
type pointerReporter struct {
last *events.DropEvent
}
func (r *pointerReporter) ReportDrop(e events.DropEvent) {
r.last = &e
}
func (r *pointerReporter) ReportFlowCreate(events.FlowCreateEvent) {}
func (r *pointerReporter) ReportFlowEvict(events.FlowEvictEvent) {}
func (r *pointerReporter) ReportRulesReload(events.RulesReloadEvent) {
}
func newBenchFixture(b *testing.B) *eventFixture {
b.Helper()
l := test.NewLogger()
vpnNetworks := new(bart.Lite)
vpnNetworks.Insert(netip.MustParsePrefix("1.2.3.0/24"))
c := &dummyCert{
name: "host1",
networks: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/24")},
groups: []string{"default-group"},
issuer: "signer-shasum",
}
h := &HostInfo{
ConnectionState: &ConnectionState{
peerCert: &cert.CachedCertificate{
Certificate: c,
InvertedGroups: map[string]struct{}{"default-group": {}},
},
},
vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
}
h.buildNetworks(vpnNetworks, c)
fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, c)
// Inbound rule that matches our packet; outbound has no match so we can
// also benchmark the no-rule drop path.
if err := fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""); err != nil {
b.Fatal(err)
}
ctl := &Control{f: &Interface{firewall: fw}, l: l}
return &eventFixture{
ctl: ctl,
fw: fw,
p: firewall.Packet{
LocalAddr: netip.MustParseAddr("1.2.3.4"),
RemoteAddr: netip.MustParseAddr("1.2.3.4"),
LocalPort: 10,
RemotePort: 90,
Protocol: firewall.ProtoUDP,
},
h: h,
cp: cert.NewCAPool(),
}
}
// BenchmarkFirewallDropPath measures the cost of Firewall.Drop on a packet
// that reaches the no-matching-rule branch (the longest drop path). Compare
// reporter shapes:
//
// nilReporter -- no reporter installed (feature cost when off)
// noopReporter -- reporter installed, methods discard args (minimum on-cost)
// bufferedReporter -- realistic zero-alloc reporter: value-typed channels
// pointerReporter -- anti-pattern that takes &composite-literal (allocates)
func BenchmarkFirewallDropPath(b *testing.B) {
run := func(b *testing.B, install func(*Control)) {
f := newBenchFixture(b)
install(f.ctl)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = f.firewall().Drop(f.p, firewall.PacketContext{}, false, f.h, f.cp, nil)
}
}
b.Run("nilReporter", func(b *testing.B) { run(b, func(*Control) {}) })
b.Run("noopReporter", func(b *testing.B) {
run(b, func(c *Control) { c.SetFirewallEventReporter(noopReporter{}) })
})
b.Run("bufferedReporter", func(b *testing.B) {
run(b, func(c *Control) { c.SetFirewallEventReporter(newBufferedReporter(1024)) })
})
b.Run("pointerReporter", func(b *testing.B) {
run(b, func(c *Control) { c.SetFirewallEventReporter(&pointerReporter{}) })
})
}
// BenchmarkConntrackCreate measures Firewall.Drop for an allowed inbound
// packet on a fresh conntrack (so addConn fires each iteration).
func BenchmarkConntrackCreate(b *testing.B) {
run := func(b *testing.B, install func(*Control)) {
f := newBenchFixture(b)
install(f.ctl)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
resetConntrack(f.firewall())
_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
}
}
b.Run("nilReporter", func(b *testing.B) { run(b, func(*Control) {}) })
b.Run("noopReporter", func(b *testing.B) {
run(b, func(c *Control) { c.SetFirewallEventReporter(noopReporter{}) })
})
b.Run("bufferedReporter", func(b *testing.B) {
run(b, func(c *Control) { c.SetFirewallEventReporter(newBufferedReporter(1024)) })
})
}
// BenchmarkConntrackHit measures the hot path where a flow is already in
// conntrack and short-circuits rule evaluation. The reporter slot is checked
// only on create/evict, so this bench should show the reporter having zero
// impact regardless of install state.
func BenchmarkConntrackHit(b *testing.B) {
b.Run("nilReporter", func(b *testing.B) {
f := newBenchFixture(b)
// Prime conntrack.
require.NoError(b, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
}
})
b.Run("noopReporter", func(b *testing.B) {
f := newBenchFixture(b)
f.ctl.SetFirewallEventReporter(noopReporter{})
require.NoError(b, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
}
})
}
+52 -52
View File
@@ -213,44 +213,44 @@ func TestFirewall_Drop(t *testing.T) {
cp := cert.NewCAPool()
// Drop outbound
assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil))
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
// Allow outbound because conntrack
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
require.NoError(t, fw.Drop(p, false, &h, cp, nil))
// test remote mismatch
oldRemote := p.RemoteAddr
p.RemoteAddr = netip.MustParseAddr("1.2.3.10")
assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrInvalidRemoteIP)
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP)
p.RemoteAddr = oldRemote
// ensure signer doesn't get in the way of group checks
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
// test caSha doesn't drop on match
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
// ensure ca name doesn't get in the way of group checks
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
// test caName doesn't drop on match
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
}
func TestFirewall_DropV6(t *testing.T) {
@@ -292,44 +292,44 @@ func TestFirewall_DropV6(t *testing.T) {
cp := cert.NewCAPool()
// Drop outbound
assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil))
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
// Allow outbound because conntrack
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
require.NoError(t, fw.Drop(p, false, &h, cp, nil))
// test remote mismatch
oldRemote := p.RemoteAddr
p.RemoteAddr = netip.MustParseAddr("fd12::56")
assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrInvalidRemoteIP)
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP)
p.RemoteAddr = oldRemote
// ensure signer doesn't get in the way of group checks
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
// test caSha doesn't drop on match
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
// ensure ca name doesn't get in the way of group checks
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
// test caName doesn't drop on match
cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
}
func BenchmarkFirewallTable_match(b *testing.B) {
@@ -537,10 +537,10 @@ func TestFirewall_Drop2(t *testing.T) {
cp := cert.NewCAPool()
// h1/c1 lacks the proper groups
require.ErrorIs(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil), ErrNoMatchingRule)
require.ErrorIs(t, fw.Drop(p, true, &h1, cp, nil), ErrNoMatchingRule)
// c has the proper groups
resetConntrack(fw)
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
}
func TestFirewall_Drop3(t *testing.T) {
@@ -618,18 +618,18 @@ func TestFirewall_Drop3(t *testing.T) {
cp := cert.NewCAPool()
// c1 should pass because host match
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil))
require.NoError(t, fw.Drop(p, true, &h1, cp, nil))
// c2 should pass because ca sha match
resetConntrack(fw)
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h2, cp, nil))
require.NoError(t, fw.Drop(p, true, &h2, cp, nil))
// c3 should fail because no match
resetConntrack(fw)
assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h3, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, true, &h3, cp, nil), ErrNoMatchingRule)
// Test a remote address match
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "1.2.3.4/24", "", "", ""))
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil))
require.NoError(t, fw.Drop(p, true, &h1, cp, nil))
}
func TestFirewall_Drop3V6(t *testing.T) {
@@ -667,7 +667,7 @@ func TestFirewall_Drop3V6(t *testing.T) {
fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
cp := cert.NewCAPool()
require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "fd12::34/120", "", "", ""))
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
}
func TestFirewall_DropConntrackReload(t *testing.T) {
@@ -709,12 +709,12 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
cp := cert.NewCAPool()
// Drop outbound
assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(p, true, &h, cp, nil))
// Allow outbound because conntrack
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
require.NoError(t, fw.Drop(p, false, &h, cp, nil))
oldFw := fw
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -723,7 +723,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
fw.rulesVersion = oldFw.rulesVersion + 1
// Allow outbound because conntrack and new rules allow port 10
require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
require.NoError(t, fw.Drop(p, false, &h, cp, nil))
oldFw = fw
fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -732,7 +732,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
fw.rulesVersion = oldFw.rulesVersion + 1
// Drop outbound because conntrack doesn't match new ruleset
assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
}
func TestFirewall_ICMPPortBehavior(t *testing.T) {
@@ -778,12 +778,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0
p.RemotePort = 0
// Drop outbound
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
//now also allow outbound
require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
})
t.Run("nonzero ports", func(t *testing.T) {
@@ -791,12 +791,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0xabcd
p.RemotePort = 0x1234
// Drop outbound
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
//now also allow outbound
require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
})
})
@@ -808,12 +808,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0
p.RemotePort = 0
// Drop outbound
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
//now also allow outbound
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
})
t.Run("nonzero ports, still blocked", func(t *testing.T) {
@@ -821,12 +821,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0xabcd
p.RemotePort = 0x1234
// Drop outbound
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
//now also allow outbound
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
})
t.Run("nonzero, matching ports, still blocked", func(t *testing.T) {
@@ -834,12 +834,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 80
p.RemotePort = 80
// Drop outbound
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
//now also allow outbound
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
})
})
t.Run("Any proto, any port", func(t *testing.T) {
@@ -851,12 +851,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0
p.RemotePort = 0
// Drop outbound
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
//now also allow outbound
require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
})
t.Run("nonzero ports, allowed", func(t *testing.T) {
@@ -865,15 +865,15 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
p.LocalPort = 0xabcd
p.RemotePort = 0x1234
// Drop outbound
assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
// Allow inbound
resetConntrack(fw)
require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
//now also allow outbound
require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
//different ID is blocked
p.RemotePort++
require.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
require.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
})
})
@@ -922,7 +922,7 @@ func TestFirewall_DropIPSpoofing(t *testing.T) {
Protocol: firewall.ProtoUDP,
Fragment: false,
}
assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil), ErrInvalidRemoteIP)
assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP)
}
func BenchmarkLookup(b *testing.B) {
@@ -1336,7 +1336,7 @@ func (c *testcase) Test(t *testing.T, fw *Firewall) {
t.Helper()
cp := cert.NewCAPool()
resetConntrack(fw)
err := fw.Drop(c.p, firewall.PacketContext{}, true, c.h, cp, nil)
err := fw.Drop(c.p, true, c.h, cp, nil)
if c.err == nil {
require.NoError(t, err, "failed to not drop remote address %s", c.p.RemoteAddr)
} else {
+11 -11
View File
@@ -1,6 +1,6 @@
module github.com/slackhq/nebula
go 1.25.0
go 1.25
require (
dario.cat/mergo v1.0.2
@@ -13,8 +13,8 @@ require (
github.com/gogo/protobuf v1.3.2
github.com/google/gopacket v1.1.19
github.com/kardianos/service v1.2.4
github.com/miekg/dns v1.1.72
github.com/miekg/pkcs11 v1.1.2
github.com/miekg/dns v1.1.70
github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b
github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
github.com/prometheus/client_golang v1.23.2
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
@@ -24,15 +24,15 @@ require (
github.com/stretchr/testify v1.11.1
github.com/vishvananda/netlink v1.3.1
go.yaml.in/yaml/v3 v3.0.4
golang.org/x/crypto v0.50.0
golang.org/x/crypto v0.47.0
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
golang.org/x/net v0.52.0
golang.org/x/sync v0.20.0
golang.org/x/sys v0.43.0
golang.org/x/term v0.42.0
golang.org/x/net v0.49.0
golang.org/x/sync v0.19.0
golang.org/x/sys v0.40.0
golang.org/x/term v0.39.0
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
golang.zx2c4.com/wireguard/windows v0.6.1
golang.zx2c4.com/wireguard/windows v0.5.3
google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
@@ -50,7 +50,7 @@ require (
github.com/prometheus/procfs v0.16.1 // indirect
github.com/vishvananda/netns v0.0.5 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
golang.org/x/mod v0.34.0 // indirect
golang.org/x/mod v0.31.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.43.0 // indirect
golang.org/x/tools v0.40.0 // indirect
)
+20 -20
View File
@@ -85,10 +85,10 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
github.com/miekg/pkcs11 v1.1.2 h1:/VxmeAX5qU6Q3EwafypogwWbYryHFmF2RpkJmw3m4MQ=
github.com/miekg/pkcs11 v1.1.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
github.com/miekg/dns v1.1.70 h1:DZ4u2AV35VJxdD9Fo9fIWm119BsQL5cZU1cQ9s0LkqA=
github.com/miekg/dns v1.1.70/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b h1:J/AzCvg5z0Hn1rqZUJjpbzALUmkKX0Zwbc/i4fw7Sfk=
github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -164,16 +164,16 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -184,8 +184,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -193,8 +193,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -210,11 +210,11 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -225,8 +225,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -235,8 +235,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
golang.zx2c4.com/wireguard/windows v0.6.1 h1:XMaKojH1Hs/raMrmnir4n35nTvzvWj7NmSYzHn2F4qU=
golang.zx2c4.com/wireguard/windows v0.6.1/go.mod h1:04aqInu5GYuTFvMuDw/rKBAF7mHrltW/3rekpfbbZDM=
golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+57 -8
View File
@@ -23,22 +23,25 @@ const (
DefaultHandshakeRetries = 10
DefaultHandshakeTriggerBuffer = 64
DefaultUseRelays = true
DefaultMaxHandshakeRate = 0 // 0 means unlimited
)
var (
defaultHandshakeConfig = HandshakeConfig{
tryInterval: DefaultHandshakeTryInterval,
retries: DefaultHandshakeRetries,
triggerBuffer: DefaultHandshakeTriggerBuffer,
useRelays: DefaultUseRelays,
tryInterval: DefaultHandshakeTryInterval,
retries: DefaultHandshakeRetries,
triggerBuffer: DefaultHandshakeTriggerBuffer,
useRelays: DefaultUseRelays,
maxHandshakeRate: DefaultMaxHandshakeRate,
}
)
type HandshakeConfig struct {
tryInterval time.Duration
retries int64
triggerBuffer int
useRelays bool
tryInterval time.Duration
retries int64
triggerBuffer int
useRelays bool
maxHandshakeRate int
messageMetrics *MessageMetrics
}
@@ -58,9 +61,15 @@ type HandshakeManager struct {
messageMetrics *MessageMetrics
metricInitiated metrics.Counter
metricTimedOut metrics.Counter
metricRateLimited metrics.Counter
f *Interface
l *logrus.Logger
// Rate limiting for new handshakes (token bucket)
rateBucket int // tokens currently available
rateMax int // max tokens (== max handshakes per second), 0 means unlimited
rateLastTick time.Time
// can be used to trigger outbound handshake for the given vpnIp
trigger chan netip.Addr
}
@@ -116,10 +125,41 @@ func NewHandshakeManager(l *logrus.Logger, mainHostMap *HostMap, lightHouse *Lig
messageMetrics: config.messageMetrics,
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
metricRateLimited: metrics.GetOrRegisterCounter("handshake_manager.rate_limited", nil),
rateBucket: config.maxHandshakeRate,
rateMax: config.maxHandshakeRate,
rateLastTick: time.Now(),
l: l,
}
}
// handshakeRateAllow checks the token bucket rate limiter and returns true if a
// new handshake is allowed. Must be called with hm.Lock held.
func (hm *HandshakeManager) handshakeRateAllow(now time.Time) bool {
if hm.rateMax == 0 {
return true
}
// Refill tokens based on elapsed time
elapsed := now.Sub(hm.rateLastTick)
if elapsed >= time.Second {
// Add tokens for full seconds elapsed
tokens := int(elapsed/time.Second) * hm.rateMax
hm.rateBucket += tokens
if hm.rateBucket > hm.rateMax {
hm.rateBucket = hm.rateMax
}
hm.rateLastTick = now
}
if hm.rateBucket > 0 {
hm.rateBucket--
return true
}
return false
}
func (hm *HandshakeManager) Run(ctx context.Context) {
clockSource := time.NewTicker(hm.config.tryInterval)
defer clockSource.Stop()
@@ -149,6 +189,15 @@ func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *head
case header.HandshakeIXPSK0:
switch h.MessageCounter {
case 1:
// Check rate limit for new incoming handshakes
hm.Lock()
allowed := hm.handshakeRateAllow(time.Now())
hm.Unlock()
if !allowed {
hm.metricRateLimited.Inc(1)
hm.l.WithField("from", via).Debug("Handshake rate limit reached, dropping incoming handshake")
return
}
ixHandshakeStage1(hm.f, via, packet, h)
case 2:
+62
View File
@@ -65,6 +65,68 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
assert.NotContains(t, blah.vpnIps, ip)
}
func Test_HandshakeManagerRateLimit(t *testing.T) {
l := test.NewLogger()
localrange := netip.MustParsePrefix("10.1.1.1/24")
preferredRanges := []netip.Prefix{localrange}
mainHM := newHostMap(l)
mainHM.preferredRanges.Store(&preferredRanges)
lh := newTestLighthouse()
config := defaultHandshakeConfig
config.maxHandshakeRate = 2
hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, config)
hm.f = &Interface{handshakeManager: hm, pki: &PKI{}, l: l}
now := time.Now()
// Should allow up to maxHandshakeRate handshakes
hm.Lock()
assert.True(t, hm.handshakeRateAllow(now), "first handshake should be allowed")
assert.True(t, hm.handshakeRateAllow(now), "second handshake should be allowed")
assert.False(t, hm.handshakeRateAllow(now), "third handshake should be rate limited")
hm.Unlock()
// After advancing time by 1 second, tokens should refill
hm.Lock()
assert.True(t, hm.handshakeRateAllow(now.Add(time.Second)), "handshake should be allowed after token refill")
hm.Unlock()
}
func Test_HandshakeManagerRateLimitUnlimited(t *testing.T) {
l := test.NewLogger()
localrange := netip.MustParsePrefix("10.1.1.1/24")
preferredRanges := []netip.Prefix{localrange}
mainHM := newHostMap(l)
mainHM.preferredRanges.Store(&preferredRanges)
lh := newTestLighthouse()
cs := &CertState{
initiatingVersion: cert.Version1,
privateKey: []byte{},
v1Cert: &dummyCert{version: cert.Version1},
v1HandshakeBytes: []byte{},
}
// Default config has maxHandshakeRate=0 (unlimited)
hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig)
hm.f = &Interface{handshakeManager: hm, pki: &PKI{}, l: l}
hm.f.pki.cs.Store(cs)
// Should allow many handshakes with no limit
// Limited to 10 due to test lighthouse query channel buffer
for i := 0; i < 10; i++ {
ip := netip.MustParseAddr("172.1.1.1").As16()
ip[15] = byte(i + 1)
addr := netip.AddrFrom16(ip)
h := hm.StartHandshake(addr, nil)
assert.NotNil(t, h, "handshake %d should be allowed with unlimited rate", i)
}
}
func testCountTimerWheelEntries(tw *LockingTimerWheel[netip.Addr]) (c int) {
for _, i := range tw.t.wheel {
n := i.Head
+2 -2
View File
@@ -604,9 +604,9 @@ func (hm *HostMap) queryVpnAddr(vpnIp netip.Addr, promoteIfce *Interface) *HostI
// unlockedAddHostInfo assumes you have a write-lock and will add a hostinfo object to the hostmap Indexes and RemoteIndexes maps.
// If an entry exists for the Hosts table (vpnIp -> hostinfo) then the provided hostinfo will be made primary
func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
if f.dnsServer != nil {
if f.serveDns {
remoteCert := hostinfo.ConnectionState.peerCert
f.dnsServer.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
dnsR.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
}
for _, addr := range hostinfo.vpnAddrs {
hm.unlockedInnerAddHostInfo(addr, hostinfo, f)
+5 -6
View File
@@ -11,8 +11,8 @@ import (
"github.com/slackhq/nebula/routing"
)
func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, nb, out []byte, q int, localCache firewall.ConntrackCache) {
err := newPacket(packet, false, fwPacket, fwCtx)
func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
err := newPacket(packet, false, fwPacket)
if err != nil {
if f.l.Level >= logrus.DebugLevel {
f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
@@ -66,7 +66,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
return
}
dropReason := f.firewall.Drop(*fwPacket, *fwCtx, false, hostinfo, f.pki.GetCAPool(), localCache)
dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
if dropReason == nil {
f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
@@ -211,15 +211,14 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac
func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
fp := &firewall.Packet{}
ctx := &firewall.PacketContext{}
err := newPacket(p, false, fp, ctx)
err := newPacket(p, false, fp)
if err != nil {
f.l.Warnf("error while parsing outgoing packet for firewall check; %v", err)
return
}
// check if packet is in outbound fw rules
dropReason := f.firewall.Drop(*fp, *ctx, false, hostinfo, f.pki.GetCAPool(), nil)
dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil)
if dropReason != nil {
if f.l.Level >= logrus.DebugLevel {
f.l.WithField("fwPacket", fp).
+39 -86
View File
@@ -6,7 +6,8 @@ import (
"fmt"
"io"
"net/netip"
"sync"
"os"
"runtime"
"sync/atomic"
"time"
@@ -29,7 +30,7 @@ type InterfaceConfig struct {
pki *PKI
Cipher string
Firewall *Firewall
DnsServer *dnsServer
ServeDns bool
HandshakeManager *HandshakeManager
lightHouse *LightHouse
connectionManager *connectionManager
@@ -57,7 +58,7 @@ type Interface struct {
firewall *Firewall
connectionManager *connectionManager
handshakeManager *HandshakeManager
dnsServer *dnsServer
serveDns bool
createTime time.Time
lightHouse *LightHouse
myBroadcastAddrsTable *bart.Lite
@@ -85,16 +86,8 @@ type Interface struct {
conntrackCacheTimeout time.Duration
ctx context.Context
writers []udp.Conn
readers []io.ReadWriteCloser
wg sync.WaitGroup
// fatalErr holds the first unexpected reader error that caused shutdown.
// nil means "no fatal error" (yet)
fatalErr atomic.Pointer[error]
// triggerShutdown is a function that will be run exactly once, when onFatal swaps something non-nil into fatalErr
triggerShutdown func()
metricHandshakes metrics.Histogram
messageMetrics *MessageMetrics
@@ -171,13 +164,12 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
cs := c.pki.getCertState()
ifce := &Interface{
ctx: ctx,
pki: c.pki,
hostMap: c.HostMap,
outside: c.Outside,
inside: c.Inside,
firewall: c.Firewall,
dnsServer: c.DnsServer,
serveDns: c.ServeDns,
handshakeManager: c.HandshakeManager,
createTime: time.Now(),
lightHouse: c.lightHouse,
@@ -218,7 +210,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
// activate creates the interface on the host. After the interface is created, any
// other services that want to bind listeners to its IP may do so successfully. However,
// the interface isn't going to process anything until run() is called.
func (f *Interface) activate() error {
func (f *Interface) activate() {
// actually turn on tun dev
addr, err := f.outside.LocalAddr()
@@ -246,58 +238,33 @@ func (f *Interface) activate() error {
if i > 0 {
reader, err = f.inside.NewMultiQueueReader()
if err != nil {
return err
f.l.Fatal(err)
}
}
f.readers[i] = reader
}
f.wg.Add(1) // for us to wait on Close() to return
if err = f.inside.Activate(); err != nil {
f.wg.Done()
if err := f.inside.Activate(); err != nil {
f.inside.Close()
return err
f.l.Fatal(err)
}
return nil
}
func (f *Interface) run() (func() error, error) {
func (f *Interface) run() {
// Launch n queues to read packets from udp
for i := 0; i < f.routines; i++ {
f.wg.Go(func() {
f.listenOut(i)
})
go f.listenOut(i)
}
// Launch n queues to read packets from tun dev
for i := 0; i < f.routines; i++ {
f.wg.Go(func() {
f.listenIn(f.readers[i], i)
})
}
return func() error {
f.wg.Wait()
if e := f.fatalErr.Load(); e != nil {
return *e
}
return nil
}, nil
}
// onFatal stores the first fatal reader error, and calls triggerShutdown if it was the first one
func (f *Interface) onFatal(err error) {
swapped := f.fatalErr.CompareAndSwap(nil, &err)
if !swapped {
return
}
if f.triggerShutdown != nil {
f.triggerShutdown()
go f.listenIn(f.readers[i], i)
}
}
func (f *Interface) listenOut(i int) {
runtime.LockOSThread()
var li udp.Conn
if i > 0 {
li = f.writers[i]
@@ -305,49 +272,42 @@ func (f *Interface) listenOut(i int) {
li = f.outside
}
ctCache := firewall.NewConntrackCacheTicker(f.ctx, f.conntrackCacheTimeout)
ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
lhh := f.lightHouse.NewRequestHandler()
plaintext := make([]byte, udp.MTU)
h := &header.H{}
fwPacket := &firewall.Packet{}
fwCtx := &firewall.PacketContext{}
nb := make([]byte, 12, 12)
err := li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, fwCtx, lhh, nb, i, ctCache.Get(f.l))
li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get(f.l))
})
if err != nil && !f.closed.Load() {
f.l.WithError(err).Error("Error while reading inbound packet, closing")
f.onFatal(err)
}
f.l.Debugf("underlay reader %v is done", i)
}
func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
runtime.LockOSThread()
packet := make([]byte, mtu)
out := make([]byte, mtu)
fwPacket := &firewall.Packet{}
fwCtx := &firewall.PacketContext{}
nb := make([]byte, 12, 12)
conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.conntrackCacheTimeout)
conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
for {
n, err := reader.Read(packet)
if err != nil {
if !f.closed.Load() {
f.l.WithError(err).WithField("reader", i).Error("Error while reading outbound packet, closing")
f.onFatal(err)
if errors.Is(err, os.ErrClosed) && f.closed.Load() {
return
}
break
f.l.WithError(err).Error("Error while reading outbound packet")
// This only seems to happen when something fatal happens to the fd, so exit.
os.Exit(2)
}
f.consumeInsidePacket(packet[:n], fwPacket, fwCtx, nb, out, i, conntrackCache.Get(f.l))
f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get(f.l))
}
f.l.Debugf("overlay reader %v is done", i)
}
func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
@@ -402,15 +362,8 @@ func (f *Interface) reloadFirewall(c *config.C) {
fw.Conntrack = conntrack
}
fw.reporter = oldFw.reporter
f.firewall = fw
// Fire ReportRulesReload under the conntrack lock so the reporter cannot
// observe a FlowCreate/FlowEvict for the new rulesVersion before it
// observes the reload marker. Report* must be non-blocking.
fw.reportRulesReload(oldFw.rulesVersion, fw.rulesVersion)
oldFw.Destroy()
f.l.WithField("firewallHashes", fw.GetRuleHashes()).
WithField("oldFirewallHashes", oldFw.GetRuleHashes()).
@@ -529,23 +482,23 @@ func (f *Interface) GetCertState() *CertState {
}
func (f *Interface) Close() error {
var errs []error
f.closed.Store(true)
// Release the udp readers
for i, u := range f.writers {
for _, u := range f.writers {
err := u.Close()
if err != nil {
f.l.WithError(err).WithField("writer", i).Error("Error while closing udp socket")
errs = append(errs, err)
f.l.WithError(err).Error("Error while closing udp socket")
}
}
for i, r := range f.readers {
if i == 0 {
continue // f.readers[0] is f.inside, which we want to save for last
}
if err := r.Close(); err != nil {
f.l.WithError(err).Error("Error while closing tun reader")
}
}
// Release the tun device (closing the tun also closes all readers)
closeErr := f.inside.Close()
if closeErr != nil {
errs = append(errs, closeErr)
}
f.wg.Done()
return errors.Join(errs...)
// Release the tun device
return f.inside.Close()
}
+29 -18
View File
@@ -204,10 +204,11 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
useRelays := c.GetBool("relay.use_relays", DefaultUseRelays) && !c.GetBool("relay.am_relay", false)
handshakeConfig := HandshakeConfig{
tryInterval: c.GetDuration("handshakes.try_interval", DefaultHandshakeTryInterval),
retries: int64(c.GetInt("handshakes.retries", DefaultHandshakeRetries)),
triggerBuffer: c.GetInt("handshakes.trigger_buffer", DefaultHandshakeTriggerBuffer),
useRelays: useRelays,
tryInterval: c.GetDuration("handshakes.try_interval", DefaultHandshakeTryInterval),
retries: int64(c.GetInt("handshakes.retries", DefaultHandshakeRetries)),
triggerBuffer: c.GetInt("handshakes.trigger_buffer", DefaultHandshakeTriggerBuffer),
useRelays: useRelays,
maxHandshakeRate: c.GetInt("handshakes.max_rate", DefaultMaxHandshakeRate),
messageMetrics: messageMetrics,
}
@@ -215,9 +216,13 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
handshakeManager := NewHandshakeManager(l, hostMap, lightHouse, udpConns[0], handshakeConfig)
lightHouse.handshakeTrigger = handshakeManager.trigger
ds, err := newDnsServerFromConfig(ctx, l, pki.getCertState(), hostMap, c)
if err != nil {
l.WithError(err).Warn("Failed to start DNS responder")
serveDns := false
if c.GetBool("lighthouse.serve_dns", false) {
if c.GetBool("lighthouse.am_lighthouse", false) {
serveDns = true
} else {
l.Warn("DNS server refusing to run because this host is not a lighthouse.")
}
}
ifConfig := &InterfaceConfig{
@@ -226,7 +231,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
Outside: udpConns[0],
pki: pki,
Firewall: fw,
DnsServer: ds,
ServeDns: serveDns,
HandshakeManager: handshakeManager,
connectionManager: connManager,
lightHouse: lightHouse,
@@ -276,17 +281,23 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
attachCommands(l, c, ssh, ifce)
// Start DNS server last to allow using the nebula IP as lighthouse.dns.host
var dnsStart func()
if lightHouse.amLighthouse && serveDns {
l.Debugln("Starting dns server")
dnsStart = dnsMain(l, pki.getCertState(), hostMap, c)
}
return &Control{
state: StateReady,
f: ifce,
l: l,
ctx: ctx,
cancel: cancel,
sshStart: sshStart,
statsStart: statsStart,
dnsStart: ds.Start,
lighthouseStart: lightHouse.StartUpdateWorker,
connectionManagerStart: connManager.Start,
ifce,
l,
ctx,
cancel,
sshStart,
statsStart,
dnsStart,
lightHouse.StartUpdateWorker,
connManager.Start,
}, nil
}
+11 -40
View File
@@ -19,7 +19,7 @@ const (
minFwPacketLen = 4
)
func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
err := h.Parse(packet)
if err != nil {
// Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
@@ -60,7 +60,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
switch h.Subtype {
case header.MessageNone:
if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, fwCtx, nb, q, localCache) {
if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, nb, q, localCache) {
return
}
case header.MessageRelay:
@@ -102,7 +102,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
relay: relay,
IsRelayed: true,
}
f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, fwCtx, lhf, nb, q, localCache)
f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
return
case ForwardingType:
// Find the target HostInfo relay object
@@ -295,10 +295,7 @@ var (
)
// newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers
func newPacket(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
if ctx != nil {
*ctx = firewall.PacketContext{}
}
func newPacket(data []byte, incoming bool, fp *firewall.Packet) error {
if len(data) < 1 {
return ErrPacketTooShort
}
@@ -306,14 +303,14 @@ func newPacket(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.Pa
version := int((data[0] >> 4) & 0x0f)
switch version {
case ipv4.Version:
return parseV4(data, incoming, fp, ctx)
return parseV4(data, incoming, fp)
case ipv6.Version:
return parseV6(data, incoming, fp, ctx)
return parseV6(data, incoming, fp)
}
return ErrUnknownIPVersion
}
func parseV6(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
dataLen := len(data)
if dataLen < ipv6.HeaderLen {
return ErrIPv6PacketTooShort
@@ -358,11 +355,6 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.Pack
fp.RemotePort = 0
}
fp.Fragment = false
if ctx != nil {
ctx.Length = binary.BigEndian.Uint16(data[4:6]) + uint16(ipv6.HeaderLen)
ctx.ICMPType = data[offset]
ctx.ICMPCode = data[offset+1]
}
return nil
case layers.IPProtocolTCP, layers.IPProtocolUDP:
@@ -380,12 +372,6 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.Pack
}
fp.Fragment = false
if ctx != nil {
ctx.Length = binary.BigEndian.Uint16(data[4:6]) + uint16(ipv6.HeaderLen)
if proto == layers.IPProtocolTCP && dataLen >= offset+14 {
ctx.TCPFlags = data[offset+13]
}
}
return nil
case layers.IPProtocolIPv6Fragment:
@@ -437,7 +423,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.Pack
return ErrIPv6CouldNotFindPayload
}
func parseV4(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
// Do we at least have an ipv4 header worth of data?
if len(data) < ipv4.HeaderLen {
return ErrIPv4PacketTooShort
@@ -494,21 +480,6 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.Pack
fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
}
if ctx != nil {
ctx.Length = binary.BigEndian.Uint16(data[2:4])
if !fp.Fragment {
switch fp.Protocol {
case firewall.ProtoICMP:
ctx.ICMPType = data[ihl]
ctx.ICMPCode = data[ihl+1]
case firewall.ProtoTCP:
if len(data) >= ihl+14 {
ctx.TCPFlags = data[ihl+13]
}
}
}
}
return nil
}
@@ -528,7 +499,7 @@ func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []
return out, nil
}
func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, nb []byte, q int, localCache firewall.ConntrackCache) bool {
func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) bool {
var err error
out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
@@ -537,7 +508,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
return false
}
err = newPacket(out, true, fwPacket, fwCtx)
err = newPacket(out, true, fwPacket)
if err != nil {
hostinfo.logger(f.l).WithError(err).WithField("packet", out).
Warnf("Error while validating inbound packet")
@@ -550,7 +521,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
return false
}
dropReason := f.firewall.Drop(*fwPacket, *fwCtx, true, hostinfo, f.pki.GetCAPool(), localCache)
dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
if dropReason != nil {
// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
// This gives us a buffer to build the reject packet in
+34 -34
View File
@@ -20,13 +20,13 @@ func Test_newPacket(t *testing.T) {
p := &firewall.Packet{}
// length fails
err := newPacket([]byte{}, true, p, nil)
err := newPacket([]byte{}, true, p)
require.ErrorIs(t, err, ErrPacketTooShort)
err = newPacket([]byte{0x40}, true, p, nil)
err = newPacket([]byte{0x40}, true, p)
require.ErrorIs(t, err, ErrIPv4PacketTooShort)
err = newPacket([]byte{0x60}, true, p, nil)
err = newPacket([]byte{0x60}, true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
// length fail with ip options
@@ -39,15 +39,15 @@ func Test_newPacket(t *testing.T) {
}
b, _ := h.Marshal()
err = newPacket(b, true, p, nil)
err = newPacket(b, true, p)
require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
// not an ipv4 packet
err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p, nil)
err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
require.ErrorIs(t, err, ErrUnknownIPVersion)
// invalid ihl
err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p, nil)
err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
// account for variable ip header length - incoming
@@ -62,7 +62,7 @@ func Test_newPacket(t *testing.T) {
b, _ = h.Marshal()
b = append(b, []byte{0, 3, 0, 4}...)
err = newPacket(b, true, p, nil)
err = newPacket(b, true, p)
require.NoError(t, err)
assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
@@ -84,7 +84,7 @@ func Test_newPacket(t *testing.T) {
b, _ = h.Marshal()
b = append(b, []byte{0, 5, 0, 6}...)
err = newPacket(b, false, p, nil)
err = newPacket(b, false, p)
require.NoError(t, err)
assert.Equal(t, uint8(2), p.Protocol)
@@ -114,7 +114,7 @@ func Test_newPacket_v6(t *testing.T) {
err := gopacket.SerializeLayers(buffer, opt, &ip)
require.NoError(t, err)
err = newPacket(buffer.Bytes(), true, p, nil)
err = newPacket(buffer.Bytes(), true, p)
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
// A v6 packet with a hop-by-hop extension
@@ -148,12 +148,12 @@ func Test_newPacket_v6(t *testing.T) {
// A full IPv6 header and 1 byte in the first extension, but missing
// the length byte.
err = newPacket(buffer.Bytes()[:41], true, p, nil)
err = newPacket(buffer.Bytes()[:41], true, p)
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
// A full IPv6 header plus 1 full extension, but only 1 byte of the
// next layer, missing length byte
err = newPacket(buffer.Bytes()[:49], true, p, nil)
err = newPacket(buffer.Bytes()[:49], true, p)
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
err = nil
@@ -173,7 +173,7 @@ func Test_newPacket_v6(t *testing.T) {
buffer.Clear()
require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp))
require.Error(t, newPacket(buffer.Bytes(), true, p, nil))
require.Error(t, newPacket(buffer.Bytes(), true, p))
buffer.Clear()
echo := layers.ICMPv6Echo{
@@ -181,7 +181,7 @@ func Test_newPacket_v6(t *testing.T) {
SeqNumber: 1234,
}
require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp, &echo))
require.NoError(t, newPacket(buffer.Bytes(), true, p, nil))
require.NoError(t, newPacket(buffer.Bytes(), true, p))
assert.Equal(t, uint8(layers.IPProtocolICMPv6), p.Protocol)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -192,7 +192,7 @@ func Test_newPacket_v6(t *testing.T) {
// A good ESP packet
b := buffer.Bytes()
b[6] = byte(layers.IPProtocolESP)
err = newPacket(b, true, p, nil)
err = newPacket(b, true, p)
require.NoError(t, err)
assert.Equal(t, uint8(layers.IPProtocolESP), p.Protocol)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -204,7 +204,7 @@ func Test_newPacket_v6(t *testing.T) {
// A good None packet
b = buffer.Bytes()
b[6] = byte(layers.IPProtocolNoNextHeader)
err = newPacket(b, true, p, nil)
err = newPacket(b, true, p)
require.NoError(t, err)
assert.Equal(t, uint8(layers.IPProtocolNoNextHeader), p.Protocol)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -216,7 +216,7 @@ func Test_newPacket_v6(t *testing.T) {
// An unknown protocol packet
b = buffer.Bytes()
b[6] = 255 // 255 is a reserved protocol number
err = newPacket(b, true, p, nil)
err = newPacket(b, true, p)
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
// A good UDP packet
@@ -243,7 +243,7 @@ func Test_newPacket_v6(t *testing.T) {
b = buffer.Bytes()
// incoming
err = newPacket(b, true, p, nil)
err = newPacket(b, true, p)
require.NoError(t, err)
assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -253,7 +253,7 @@ func Test_newPacket_v6(t *testing.T) {
assert.False(t, p.Fragment)
// outgoing
err = newPacket(b, false, p, nil)
err = newPacket(b, false, p)
require.NoError(t, err)
assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
@@ -263,14 +263,14 @@ func Test_newPacket_v6(t *testing.T) {
assert.False(t, p.Fragment)
// Too short UDP packet
err = newPacket(b[:len(b)-10], false, p, nil) // pull off the last 10 bytes
err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
// A good TCP packet
b[6] = byte(layers.IPProtocolTCP)
// incoming
err = newPacket(b, true, p, nil)
err = newPacket(b, true, p)
require.NoError(t, err)
assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -280,7 +280,7 @@ func Test_newPacket_v6(t *testing.T) {
assert.False(t, p.Fragment)
// outgoing
err = newPacket(b, false, p, nil)
err = newPacket(b, false, p)
require.NoError(t, err)
assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
@@ -290,7 +290,7 @@ func Test_newPacket_v6(t *testing.T) {
assert.False(t, p.Fragment)
// Too short TCP packet
err = newPacket(b[:len(b)-10], false, p, nil) // pull off the last 10 bytes
err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
// A good UDP packet with an AH header
@@ -325,7 +325,7 @@ func Test_newPacket_v6(t *testing.T) {
b = append(b, ahb...)
b = append(b, udpHeader...)
err = newPacket(b, true, p, nil)
err = newPacket(b, true, p)
require.NoError(t, err)
assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -335,12 +335,12 @@ func Test_newPacket_v6(t *testing.T) {
assert.False(t, p.Fragment)
// Ensure buffer bounds checking during processing
err = newPacket(b[:41], true, p, nil)
err = newPacket(b[:41], true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
// Invalid AH header
b = buffer.Bytes()
err = newPacket(b, true, p, nil)
err = newPacket(b, true, p)
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
}
@@ -388,7 +388,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
firstFrag = append(firstFrag, []byte{0xde, 0xad, 0xbe, 0xef}...)
// Test first fragment incoming
err = newPacket(firstFrag, true, p, nil)
err = newPacket(firstFrag, true, p)
require.NoError(t, err)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -398,7 +398,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
assert.False(t, p.Fragment)
// Test first fragment outgoing
err = newPacket(firstFrag, false, p, nil)
err = newPacket(firstFrag, false, p)
require.NoError(t, err)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr)
@@ -427,7 +427,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
secondFrag = append(secondFrag, []byte{0xde, 0xad, 0xbe, 0xef}...)
// Test second fragment incoming
err = newPacket(secondFrag, true, p, nil)
err = newPacket(secondFrag, true, p)
require.NoError(t, err)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -437,7 +437,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
assert.True(t, p.Fragment)
// Test second fragment outgoing
err = newPacket(secondFrag, false, p, nil)
err = newPacket(secondFrag, false, p)
require.NoError(t, err)
assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr)
@@ -447,7 +447,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
assert.True(t, p.Fragment)
// Too short of a fragment packet
err = newPacket(secondFrag[:len(secondFrag)-10], false, p, nil)
err = newPacket(secondFrag[:len(secondFrag)-10], false, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
}
@@ -529,7 +529,7 @@ func BenchmarkParseV6(b *testing.B) {
b.Run("Normal", func(b *testing.B) {
for i := 0; i < b.N; i++ {
if err = parseV6(normalPacket, true, fp, nil); err != nil {
if err = parseV6(normalPacket, true, fp); err != nil {
b.Fatal(err)
}
}
@@ -537,7 +537,7 @@ func BenchmarkParseV6(b *testing.B) {
b.Run("FirstFragment", func(b *testing.B) {
for i := 0; i < b.N; i++ {
if err = parseV6(firstFrag, true, fp, nil); err != nil {
if err = parseV6(firstFrag, true, fp); err != nil {
b.Fatal(err)
}
}
@@ -545,7 +545,7 @@ func BenchmarkParseV6(b *testing.B) {
b.Run("SecondFragment", func(b *testing.B) {
for i := 0; i < b.N; i++ {
if err = parseV6(secondFrag, true, fp, nil); err != nil {
if err = parseV6(secondFrag, true, fp); err != nil {
b.Fatal(err)
}
}
@@ -590,7 +590,7 @@ func BenchmarkParseV6(b *testing.B) {
b.Run("200 HopByHop headers", func(b *testing.B) {
for i := 0; i < b.N; i++ {
if err = parseV6(evilBytes, false, fp, nil); err != nil {
if err = parseV6(evilBytes, false, fp); err != nil {
b.Fatal(err)
}
}
-120
View File
@@ -1,120 +0,0 @@
//go:build linux && !android && !e2e_testing
// +build linux,!android,!e2e_testing
package overlay
import (
"errors"
"os"
"sync"
"testing"
"time"
"golang.org/x/sys/unix"
)
// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
// The caller takes ownership of the read fd (pass it to newTunFd / newFriend).
func newReadPipe(t *testing.T) int {
t.Helper()
var fds [2]int
if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
t.Fatalf("pipe2: %v", err)
}
t.Cleanup(func() { _ = unix.Close(fds[1]) })
return fds[0]
}
func TestTunFile_WakeForShutdown_UnblocksRead(t *testing.T) {
tf, err := newTunFd(newReadPipe(t))
if err != nil {
t.Fatalf("newTunFd: %v", err)
}
t.Cleanup(func() { _ = tf.Close() })
done := make(chan error, 1)
go func() {
_, err := tf.Read(make([]byte, 64))
done <- err
}()
// Verify Read is actually blocked in poll.
select {
case err := <-done:
t.Fatalf("Read returned before shutdown signal: %v", err)
case <-time.After(50 * time.Millisecond):
}
if err := tf.wakeForShutdown(); err != nil {
t.Fatalf("wakeForShutdown: %v", err)
}
select {
case err := <-done:
if !errors.Is(err, os.ErrClosed) {
t.Fatalf("expected os.ErrClosed, got %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("Read did not wake on shutdown")
}
}
func TestTunFile_WakeForShutdown_WakesFriends(t *testing.T) {
parent, err := newTunFd(newReadPipe(t))
if err != nil {
t.Fatalf("newTunFd: %v", err)
}
friend, err := parent.newFriend(newReadPipe(t))
if err != nil {
_ = parent.Close()
t.Fatalf("newFriend: %v", err)
}
t.Cleanup(func() {
_ = friend.Close()
_ = parent.Close()
})
readers := []*tunFile{parent, friend}
errs := make([]error, len(readers))
var wg sync.WaitGroup
for i, r := range readers {
wg.Add(1)
go func(i int, r *tunFile) {
defer wg.Done()
_, errs[i] = r.Read(make([]byte, 64))
}(i, r)
}
time.Sleep(50 * time.Millisecond)
if err := parent.wakeForShutdown(); err != nil {
t.Fatalf("wakeForShutdown: %v", err)
}
done := make(chan struct{})
go func() { wg.Wait(); close(done) }()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("readers did not wake")
}
for i, err := range errs {
if !errors.Is(err, os.ErrClosed) {
t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
}
}
}
func TestTunFile_Close_Idempotent(t *testing.T) {
tf, err := newTunFd(newReadPipe(t))
if err != nil {
t.Fatalf("newTunFd: %v", err)
}
if err := tf.Close(); err != nil {
t.Fatalf("first Close: %v", err)
}
if err := tf.Close(); err != nil {
t.Fatalf("second Close should be a no-op, got %v", err)
}
}
+77 -186
View File
@@ -10,7 +10,6 @@ import (
"io"
"io/fs"
"net/netip"
"os"
"sync/atomic"
"syscall"
"time"
@@ -94,184 +93,107 @@ type tun struct {
routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr *netroute.LinkAddr
l *logrus.Logger
fd int
shutdownR int // read end of the shutdown pipe; closing the write end wakes blocked polls
shutdownW int // write end of the shutdown pipe; closing this signals shutdown to any blocked reader/writer
readPoll [2]unix.PollFd
writePoll [2]unix.PollFd
closed atomic.Bool
}
// blockOnRead waits until the tun fd is readable or shutdown has been signaled.
// Returns os.ErrClosed if Close was called.
func (t *tun) blockOnRead() error {
const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
var err error
for {
_, err = unix.Poll(t.readPoll[:], -1)
if err != unix.EINTR {
break
}
}
tunEvents := t.readPoll[0].Revents
shutdownEvents := t.readPoll[1].Revents
t.readPoll[0].Revents = 0
t.readPoll[1].Revents = 0
if err != nil {
return err
}
if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
return os.ErrClosed
}
if tunEvents&problemFlags != 0 {
return os.ErrClosed
}
return nil
}
func (t *tun) blockOnWrite() error {
const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
var err error
for {
_, err = unix.Poll(t.writePoll[:], -1)
if err != unix.EINTR {
break
}
}
tunEvents := t.writePoll[0].Revents
shutdownEvents := t.writePoll[1].Revents
t.writePoll[0].Revents = 0
t.writePoll[1].Revents = 0
if err != nil {
return err
}
if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
return os.ErrClosed
}
if tunEvents&problemFlags != 0 {
return os.ErrClosed
}
return nil
devFd int
}
func (t *tun) Read(to []byte) (int, error) {
// use readv() to read from the tunnel device, to eliminate the need for copying the buffer
if t.devFd < 0 {
return -1, syscall.EINVAL
}
// first 4 bytes is protocol family, in network byte order
var head [4]byte
iovecs := [2]syscall.Iovec{
head := make([]byte, 4)
iovecs := []syscall.Iovec{
{&head[0], 4},
{&to[0], uint64(len(to))},
}
for {
n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
if errno == 0 {
bytesRead := int(n)
if bytesRead < 4 {
return 0, nil
}
return bytesRead - 4, nil
}
switch errno {
case unix.EAGAIN:
if err := t.blockOnRead(); err != nil {
return 0, err
}
case unix.EINTR:
// retry
case unix.EBADF:
return 0, os.ErrClosed
default:
return 0, errno
}
n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
var err error
if errno != 0 {
err = syscall.Errno(errno)
} else {
err = nil
}
// fix bytes read number to exclude header
bytesRead := int(n)
if bytesRead < 0 {
return bytesRead, err
} else if bytesRead < 4 {
return 0, err
} else {
return bytesRead - 4, err
}
}
// Write is only valid for single threaded use
func (t *tun) Write(from []byte) (int, error) {
// use writev() to write to the tunnel device, to eliminate the need for copying the buffer
if t.devFd < 0 {
return -1, syscall.EINVAL
}
if len(from) <= 1 {
return 0, syscall.EIO
}
ipVer := from[0] >> 4
var head [4]byte
var head []byte
// first 4 bytes is protocol family, in network byte order
switch ipVer {
case 4:
head[3] = syscall.AF_INET
case 6:
head[3] = syscall.AF_INET6
default:
if ipVer == 4 {
head = []byte{0, 0, 0, syscall.AF_INET}
} else if ipVer == 6 {
head = []byte{0, 0, 0, syscall.AF_INET6}
} else {
return 0, fmt.Errorf("unable to determine IP version from packet")
}
iovecs := [2]syscall.Iovec{
iovecs := []syscall.Iovec{
{&head[0], 4},
{&from[0], uint64(len(from))},
}
for {
n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
if errno == 0 {
return int(n) - 4, nil
}
switch errno {
case unix.EAGAIN:
if err := t.blockOnWrite(); err != nil {
return 0, err
}
case unix.EINTR:
// retry
case unix.EBADF:
return 0, os.ErrClosed
default:
return 0, errno
}
n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
var err error
if errno != 0 {
err = syscall.Errno(errno)
} else {
err = nil
}
return int(n) - 4, err
}
func (t *tun) Close() error {
if t.closed.Swap(true) {
return nil
}
// Closing the write end of the shutdown pipe causes any blocked Poll to
// return with POLLHUP on the shutdown fd, so readers/writers wake up and
// exit with os.ErrClosed.
if t.shutdownW >= 0 {
_ = unix.Close(t.shutdownW)
t.shutdownW = -1
}
if t.fd >= 0 {
if err := unix.Close(t.fd); err != nil {
if t.devFd >= 0 {
err := syscall.Close(t.devFd)
if err != nil {
t.l.WithError(err).Error("Error closing device")
}
t.fd = -1
}
t.devFd = -1
if t.shutdownR >= 0 {
_ = unix.Close(t.shutdownR)
t.shutdownR = -1
}
c := make(chan struct{})
go func() {
// destroying the interface can block if a read() is still pending. Do this asynchronously.
defer close(c)
s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
if err == nil {
defer syscall.Close(s)
ifreq := ifreqDestroy{Name: t.deviceBytes()}
err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
}
if err != nil {
t.l.WithError(err).Error("Error destroying tunnel")
}
}()
c := make(chan struct{})
go func() {
// destroying the interface can block if a read() is still pending. Do this asynchronously.
defer close(c)
s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
if err == nil {
defer syscall.Close(s)
ifreq := ifreqDestroy{Name: t.deviceBytes()}
err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
// wait up to 1 second so we start blocking at the ioctl
select {
case <-c:
case <-time.After(1 * time.Second):
}
if err != nil {
t.l.WithError(err).Error("Error destroying tunnel")
}
}()
// wait up to 1 second so we start blocking at the ioctl
select {
case <-c:
case <-time.After(1 * time.Second):
}
return nil
@@ -287,38 +209,16 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
var err error
deviceName := c.GetString("tun.dev", "")
if deviceName != "" {
fd, err = unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
fd, err = syscall.Open("/dev/"+deviceName, syscall.O_RDWR, 0)
}
if errors.Is(err, fs.ErrNotExist) || deviceName == "" {
// If the device doesn't already exist, request a new one and rename it
fd, err = unix.Open("/dev/tun", os.O_RDWR, 0)
fd, err = syscall.Open("/dev/tun", syscall.O_RDWR, 0)
}
if err != nil {
return nil, err
}
if err = unix.SetNonblock(fd, true); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("failed to set tun device as nonblocking: %w", err)
}
// Shutdown pipe lets Close wake any reader/writer blocked in Poll.
var pipeFds [2]int
if err = unix.Pipe2(pipeFds[:], unix.O_CLOEXEC|unix.O_NONBLOCK); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("failed to create shutdown pipe: %w", err)
}
shutdownR, shutdownW := pipeFds[0], pipeFds[1]
closeOnErr := true
defer func() {
if closeOnErr {
_ = unix.Close(fd)
_ = unix.Close(shutdownR)
_ = unix.Close(shutdownW)
}
}()
// Read the name of the interface
var name [16]byte
arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)}
@@ -337,7 +237,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
}
if ctrlErr != nil {
return nil, ctrlErr
return nil, err
}
ifName := string(bytes.TrimRight(name[:], "\x00"))
@@ -353,6 +253,8 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
}
defer syscall.Close(s)
fd := uintptr(s)
var fromName [16]byte
var toName [16]byte
copy(fromName[:], ifName)
@@ -364,7 +266,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
}
// Set the device name
_ = ioctl(uintptr(s), syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
_ = ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
}
t := &tun{
@@ -372,24 +274,13 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
vpnNetworks: vpnNetworks,
MTU: c.GetInt("tun.mtu", DefaultMTU),
l: l,
fd: fd,
shutdownR: shutdownR,
shutdownW: shutdownW,
readPoll: [2]unix.PollFd{
{Fd: int32(fd), Events: unix.POLLIN},
{Fd: int32(shutdownR), Events: unix.POLLIN},
},
writePoll: [2]unix.PollFd{
{Fd: int32(fd), Events: unix.POLLOUT},
{Fd: int32(shutdownR), Events: unix.POLLIN},
},
devFd: fd,
}
err = t.reload(c, true)
if err != nil {
return nil, err
}
closeOnErr = false
c.RegisterReloadCallback(func(c *config.C) {
err := t.reload(c, false)
+42 -223
View File
@@ -4,7 +4,6 @@
package overlay
import (
"encoding/binary"
"fmt"
"io"
"net"
@@ -25,175 +24,9 @@ import (
"golang.org/x/sys/unix"
)
// tunFile wraps a TUN file descriptor with poll-based reads. The FD provided will be changed to non-blocking.
// A shared eventfd allows Close to wake all readers blocked in poll.
type tunFile struct {
fd int
shutdownFd int
lastOne bool
readPoll [2]unix.PollFd
writePoll [2]unix.PollFd
closed bool
}
// newFriend makes a tunFile for a MultiQueueReader that copies the shutdown eventfd from the parent tun
func (r *tunFile) newFriend(fd int) (*tunFile, error) {
if err := unix.SetNonblock(fd, true); err != nil {
return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
}
return &tunFile{
fd: fd,
shutdownFd: r.shutdownFd,
readPoll: [2]unix.PollFd{
{Fd: int32(fd), Events: unix.POLLIN},
{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
},
writePoll: [2]unix.PollFd{
{Fd: int32(fd), Events: unix.POLLOUT},
{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
},
}, nil
}
func newTunFd(fd int) (*tunFile, error) {
if err := unix.SetNonblock(fd, true); err != nil {
return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
}
shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
if err != nil {
return nil, fmt.Errorf("failed to create eventfd: %w", err)
}
out := &tunFile{
fd: fd,
shutdownFd: shutdownFd,
lastOne: true,
readPoll: [2]unix.PollFd{
{Fd: int32(fd), Events: unix.POLLIN},
{Fd: int32(shutdownFd), Events: unix.POLLIN},
},
writePoll: [2]unix.PollFd{
{Fd: int32(fd), Events: unix.POLLOUT},
{Fd: int32(shutdownFd), Events: unix.POLLIN},
},
}
return out, nil
}
func (r *tunFile) blockOnRead() error {
const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
var err error
for {
_, err = unix.Poll(r.readPoll[:], -1)
if err != unix.EINTR {
break
}
}
//always reset these!
tunEvents := r.readPoll[0].Revents
shutdownEvents := r.readPoll[1].Revents
r.readPoll[0].Revents = 0
r.readPoll[1].Revents = 0
//do the err check before trusting the potentially bogus bits we just got
if err != nil {
return err
}
if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
return os.ErrClosed
} else if tunEvents&problemFlags != 0 {
return os.ErrClosed
}
return nil
}
func (r *tunFile) blockOnWrite() error {
const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
var err error
for {
_, err = unix.Poll(r.writePoll[:], -1)
if err != unix.EINTR {
break
}
}
//always reset these!
tunEvents := r.writePoll[0].Revents
shutdownEvents := r.writePoll[1].Revents
r.writePoll[0].Revents = 0
r.writePoll[1].Revents = 0
//do the err check before trusting the potentially bogus bits we just got
if err != nil {
return err
}
if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
return os.ErrClosed
} else if tunEvents&problemFlags != 0 {
return os.ErrClosed
}
return nil
}
func (r *tunFile) Read(buf []byte) (int, error) {
for {
if n, err := unix.Read(r.fd, buf); err == nil {
return n, nil
} else if err == unix.EAGAIN {
if err = r.blockOnRead(); err != nil {
return 0, err
}
continue
} else if err == unix.EINTR {
continue
} else if err == unix.EBADF {
return 0, os.ErrClosed
} else {
return 0, err
}
}
}
func (r *tunFile) Write(buf []byte) (int, error) {
for {
if n, err := unix.Write(r.fd, buf); err == nil {
return n, nil
} else if err == unix.EAGAIN {
if err = r.blockOnWrite(); err != nil {
return 0, err
}
continue
} else if err == unix.EINTR {
continue
} else if err == unix.EBADF {
return 0, os.ErrClosed
} else {
return 0, err
}
}
}
func (r *tunFile) wakeForShutdown() error {
var buf [8]byte
binary.NativeEndian.PutUint64(buf[:], 1)
_, err := unix.Write(int(r.readPoll[1].Fd), buf[:])
return err
}
func (r *tunFile) Close() error {
if r.closed { // avoid closing more than once. Technically a fd could get re-used, which would be a problem
return nil
}
r.closed = true
if r.lastOne {
_ = unix.Close(r.shutdownFd)
}
return unix.Close(r.fd)
}
type tun struct {
*tunFile
readers []*tunFile
closeLock sync.Mutex
io.ReadWriteCloser
fd int
Device string
vpnNetworks []netip.Prefix
MaxMTU int
@@ -239,7 +72,9 @@ type ifreqQLEN struct {
}
func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
t, err := newTunGeneric(c, l, deviceFd, vpnNetworks)
file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
t, err := newTunGeneric(c, l, file, vpnNetworks)
if err != nil {
return nil, err
}
@@ -280,7 +115,6 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueu
nameStr := c.GetString("tun.dev", "")
copy(req.Name[:], nameStr)
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
_ = unix.Close(fd)
return nil, &NameError{
Name: nameStr,
Underlying: err,
@@ -288,7 +122,8 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueu
}
name := strings.Trim(string(req.Name[:]), "\x00")
t, err := newTunGeneric(c, l, fd, vpnNetworks)
file := os.NewFile(uintptr(fd), "/dev/net/tun")
t, err := newTunGeneric(c, l, file, vpnNetworks)
if err != nil {
return nil, err
}
@@ -298,17 +133,10 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueu
return t, nil
}
// newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error.
func newTunGeneric(c *config.C, l *logrus.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) {
tfd, err := newTunFd(fd)
if err != nil {
_ = unix.Close(fd)
return nil, err
}
func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []netip.Prefix) (*tun, error) {
t := &tun{
tunFile: tfd,
readers: []*tunFile{tfd},
closeLock: sync.Mutex{},
ReadWriteCloser: file,
fd: int(file.Fd()),
vpnNetworks: vpnNetworks,
TXQueueLen: c.GetInt("tun.tx_queue", 500),
useSystemRoutes: c.GetBool("tun.use_system_route_table", false),
@@ -317,8 +145,8 @@ func newTunGeneric(c *config.C, l *logrus.Logger, fd int, vpnNetworks []netip.Pr
l: l,
}
if err = t.reload(c, true); err != nil {
_ = t.Close()
err := t.reload(c, true)
if err != nil {
return nil, err
}
@@ -411,9 +239,6 @@ func (t *tun) SupportsMultiqueue() bool {
}
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
t.closeLock.Lock()
defer t.closeLock.Unlock()
fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
if err != nil {
return nil, err
@@ -423,19 +248,12 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
copy(req.Name[:], t.Device)
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
_ = unix.Close(fd)
return nil, err
}
out, err := t.tunFile.newFriend(fd)
if err != nil {
_ = unix.Close(fd)
return nil, err
}
file := os.NewFile(uintptr(fd), "/dev/net/tun")
t.readers = append(t.readers, out)
return out, nil
return file, nil
}
func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
@@ -443,6 +261,29 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
return r
}
func (t *tun) Write(b []byte) (int, error) {
var nn int
maximum := len(b)
for {
n, err := unix.Write(t.fd, b[nn:maximum])
if n > 0 {
nn += n
}
if nn == len(b) {
return nn, err
}
if err != nil {
return nn, err
}
if n == 0 {
return nn, io.ErrUnexpectedEOF
}
}
}
func (t *tun) deviceBytes() (o [16]byte) {
for i, c := range t.Device {
o[i] = byte(c)
@@ -866,40 +707,18 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
}
func (t *tun) Close() error {
t.closeLock.Lock()
defer t.closeLock.Unlock()
if t.routeChan != nil {
close(t.routeChan)
t.routeChan = nil
}
// Signal all readers blocked in poll to wake up and exit
_ = t.tunFile.wakeForShutdown()
if t.ReadWriteCloser != nil {
_ = t.ReadWriteCloser.Close()
}
if t.ioctlFd > 0 {
_ = unix.Close(int(t.ioctlFd))
_ = os.NewFile(t.ioctlFd, "ioctlFd").Close()
t.ioctlFd = 0
}
for i := range t.readers {
if i == 0 {
continue //we want to close the zeroth reader last
}
err := t.readers[i].Close()
if err != nil {
t.l.WithField("reader", i).WithError(err).Error("error closing tun reader")
} else {
t.l.WithField("reader", i).Info("closed tun reader")
}
}
//this is t.readers[0] too
err := t.tunFile.Close()
if err != nil {
t.l.WithField("reader", 0).WithError(err).Error("error closing tun reader")
} else {
t.l.WithField("reader", 0).Info("closed tun reader")
}
return err
return nil
}
+7 -8
View File
@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/netip"
"os"
@@ -488,25 +487,25 @@ func loadCertificate(b []byte) (cert.Certificate, []byte, error) {
}
func loadCAPoolFromConfig(l *logrus.Logger, c *config.C) (*cert.CAPool, error) {
var rawCA []byte
var err error
caPathOrPEM := c.GetString("pki.ca", "")
if caPathOrPEM == "" {
return nil, errors.New("no pki.ca path or PEM data provided")
}
var caReader io.ReadCloser
var err error
if strings.Contains(caPathOrPEM, "-----BEGIN") {
caReader = io.NopCloser(strings.NewReader(caPathOrPEM))
rawCA = []byte(caPathOrPEM)
} else {
caReader, err = os.Open(caPathOrPEM)
rawCA, err = os.ReadFile(caPathOrPEM)
if err != nil {
return nil, fmt.Errorf("unable to read pki.ca file %s: %s", caPathOrPEM, err)
}
}
defer caReader.Close()
caPool, err := cert.NewCAPoolFromPEMReader(caReader)
caPool, err := cert.NewCAPoolFromPEM(rawCA)
if errors.Is(err, cert.ErrExpired) {
var expired int
for _, crt := range caPool.CAs {
-121
View File
@@ -1,121 +0,0 @@
package nebula
import (
"bytes"
"fmt"
"net/netip"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/slackhq/nebula/cert"
cert_test "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/require"
)
func BenchmarkReloadConfigWithCAs(b *testing.B) {
prevProcs := runtime.GOMAXPROCS(1)
b.Cleanup(func() { runtime.GOMAXPROCS(prevProcs) })
for _, size := range []int{100, 250, 500, 1000, 5000} {
b.Run(fmt.Sprintf("%dCAs", size), func(b *testing.B) {
l := test.NewLogger()
dir := b.TempDir()
ca, caKey, caBundle := buildCABundle(b, size)
caPath, certPath, keyPath := writePKIFiles(b, dir, ca, caKey, caBundle)
configBody := fmt.Sprintf(`pki:
ca: %s
cert: %s
key: %s
`, caPath, certPath, keyPath)
configPath := filepath.Join(dir, "config.yml")
require.NoError(b, os.WriteFile(configPath, []byte(configBody), 0o600))
c := config.NewC(l)
require.NoError(b, c.Load(dir))
_, err := NewPKIFromConfig(l, c)
require.NoError(b, err)
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
c.ReloadConfig()
}
})
}
}
func buildCABundle(b *testing.B, count int) (cert.Certificate, []byte, []byte) {
b.Helper()
require.GreaterOrEqual(b, count, 1)
before := time.Now().Add(-24 * time.Hour)
after := time.Now().Add(24 * time.Hour)
ca, _, caKey, pem := cert_test.NewTestCaCert(
cert.Version2,
cert.Curve_CURVE25519,
before,
after,
nil,
nil,
nil,
)
buf := bytes.NewBuffer(pem)
buf.Write([]byte("\n# a comment!\n"))
for i := 1; i < count; i++ {
_, _, _, extraPEM := cert_test.NewTestCaCert(
cert.Version2,
cert.Curve_CURVE25519,
time.Now(),
time.Now().Add(time.Hour),
nil,
nil,
nil,
)
buf.Write([]byte("\n# a comment!\n"))
buf.Write(extraPEM)
}
return ca, caKey, buf.Bytes()
}
func writePKIFiles(b *testing.B, dir string, ca cert.Certificate, caKey []byte, caBundle []byte) (string, string, string) {
b.Helper()
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
_, _, keyPEM, certPEM := cert_test.NewTestCert(
cert.Version2,
cert.Curve_CURVE25519,
ca,
caKey,
"reload-benchmark",
time.Now(),
time.Now().Add(time.Hour),
networks,
nil,
nil,
)
caPath := filepath.Join(dir, "ca.pem")
certPath := filepath.Join(dir, "cert.pem")
keyPath := filepath.Join(dir, "key.pem")
require.NoError(b, os.WriteFile(caPath, caBundle, 0o600))
require.NoError(b, os.WriteFile(certPath, certPEM, 0o600))
require.NoError(b, os.WriteFile(keyPath, keyPEM, 0o600))
return caPath, certPath, keyPath
}
+1 -10
View File
@@ -44,10 +44,7 @@ type Service struct {
}
func New(control *nebula.Control) (*Service, error) {
wait, err := control.Start()
if err != nil {
return nil, err
}
control.Start()
ctx := control.Context()
eg, ctx := errgroup.WithContext(ctx)
@@ -144,12 +141,6 @@ func New(control *nebula.Control) (*Service, error) {
}
})
// Add the nebula wait function to the group so a fatal reader error
// propagates out through errgroup.Wait().
eg.Go(func() error {
return wait()
})
return &s, nil
}
+12 -62
View File
@@ -10,7 +10,6 @@ import (
"net"
"net/netip"
"os"
"path/filepath"
"reflect"
"runtime"
"runtime/pprof"
@@ -189,12 +188,6 @@ func configSSH(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) (func(), erro
}
func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
// sandboxDir defaults to a dir in temp. The intention is that end user will
// create this dir as needed. Overriding this config value to "" allows
// writing to anywhere in the system.
defaultDir := filepath.Join(os.TempDir(), "nebula-debug")
sandboxDir := c.GetString("sshd.sandbox_dir", defaultDir)
ssh.RegisterCommand(&sshd.Command{
Name: "list-hostmap",
ShortDescription: "List all known previously connected hosts",
@@ -253,9 +246,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter
ssh.RegisterCommand(&sshd.Command{
Name: "start-cpu-profile",
ShortDescription: "Starts a cpu profile and write output to the provided file, ex: `cpu-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshStartCpuProfile(sandboxDir, fs, a, w)
},
Callback: sshStartCpuProfile,
})
ssh.RegisterCommand(&sshd.Command{
@@ -270,9 +261,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter
ssh.RegisterCommand(&sshd.Command{
Name: "save-heap-profile",
ShortDescription: "Saves a heap profile to the provided path, ex: `heap-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetHeapProfile(sandboxDir, fs, a, w)
},
Callback: sshGetHeapProfile,
})
ssh.RegisterCommand(&sshd.Command{
@@ -284,9 +273,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter
ssh.RegisterCommand(&sshd.Command{
Name: "save-mutex-profile",
ShortDescription: "Saves a mutex profile to the provided path, ex: `mutex-profile.pb.gz`",
Callback: func(fs any, a []string, w sshd.StringWriter) error {
return sshGetMutexProfile(sandboxDir, fs, a, w)
},
Callback: sshGetMutexProfile,
})
ssh.RegisterCommand(&sshd.Command{
@@ -519,43 +506,13 @@ func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) er
return nil
}
// sshSanitizeFilePath validates that the given file path is within the sandbox directory.
// If sandboxDir is empty, the path is returned as-is for backwards compatibility.
func sshSanitizeFilePath(sandboxDir, filePath string) (string, error) {
if sandboxDir == "" {
return filePath, nil
}
// Clean and resolve the path relative to the sandbox directory
if !filepath.IsAbs(filePath) {
filePath = filepath.Join(sandboxDir, filePath)
}
cleaned := filepath.Clean(filePath)
// Ensure the resolved path is within the sandbox directory
cleanedSandbox := filepath.Clean(sandboxDir)
if cleaned == cleanedSandbox {
return "", fmt.Errorf("path %q resolves to the sandbox directory itself %q", filePath, sandboxDir)
}
if !strings.HasPrefix(cleaned, cleanedSandbox+string(filepath.Separator)) {
return "", fmt.Errorf("path %q is outside the sandbox directory %q", filePath, sandboxDir)
}
return cleaned, nil
}
func sshStartCpuProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
func sshStartCpuProfile(fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
err := w.WriteLine("No path to write profile provided")
return err
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
file, err := os.Create(a[0])
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
@@ -675,6 +632,9 @@ func sshCreateTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) e
}
hostInfo = ifce.handshakeManager.StartHandshake(vpnAddr, nil)
if hostInfo == nil {
return w.WriteLine("Handshake rate limit reached")
}
if addr.IsValid() {
hostInfo.SetRemote(addr)
}
@@ -719,17 +679,12 @@ func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) e
return w.WriteLine("Changed")
}
func sshGetHeapProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
func sshGetHeapProfile(fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
file, err := os.Create(a[0])
if err != nil {
err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
return err
@@ -760,17 +715,12 @@ func sshMutexProfileFraction(fs any, a []string, w sshd.StringWriter) error {
return w.WriteLine(fmt.Sprintf("New value: %d. Old value: %d", newRate, oldRate))
}
func sshGetMutexProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
func sshGetMutexProfile(fs any, a []string, w sshd.StringWriter) error {
if len(a) == 0 {
return w.WriteLine("No path to write profile provided")
}
filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
if err != nil {
return w.WriteLine(err.Error())
}
file, err := os.Create(filePath)
file, err := os.Create(a[0])
if err != nil {
return w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
}
+3 -3
View File
@@ -16,7 +16,7 @@ type EncReader func(
type Conn interface {
Rebind() error
LocalAddr() (netip.AddrPort, error)
ListenOut(r EncReader) error
ListenOut(r EncReader)
WriteTo(b []byte, addr netip.AddrPort) error
ReloadConfig(c *config.C)
SupportsMultipleReaders() bool
@@ -31,8 +31,8 @@ func (NoopConn) Rebind() error {
func (NoopConn) LocalAddr() (netip.AddrPort, error) {
return netip.AddrPort{}, nil
}
func (NoopConn) ListenOut(_ EncReader) error {
return nil
func (NoopConn) ListenOut(_ EncReader) {
return
}
func (NoopConn) SupportsMultipleReaders() bool {
return false
+3 -2
View File
@@ -165,7 +165,7 @@ func NewUDPStatsEmitter(udpConns []Conn) func() {
return func() {}
}
func (u *StdConn) ListenOut(r EncReader) error {
func (u *StdConn) ListenOut(r EncReader) {
buffer := make([]byte, MTU)
for {
@@ -173,7 +173,8 @@ func (u *StdConn) ListenOut(r EncReader) error {
n, rua, err := u.ReadFromUDPAddrPort(buffer)
if err != nil {
if errors.Is(err, net.ErrClosed) {
return err
u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
return
}
u.l.WithError(err).Error("unexpected udp socket receive error")
+3 -2
View File
@@ -73,7 +73,7 @@ type rawMessage struct {
Len uint32
}
func (u *GenericConn) ListenOut(r EncReader) error {
func (u *GenericConn) ListenOut(r EncReader) {
buffer := make([]byte, MTU)
var lastRecvErr time.Time
@@ -83,7 +83,8 @@ func (u *GenericConn) ListenOut(r EncReader) error {
n, rua, err := u.ReadFromUDPAddrPort(buffer)
if err != nil {
if errors.Is(err, net.ErrClosed) {
return err
u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
return
}
// Dampen unexpected message warns to once per minute
if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute {
+162 -158
View File
@@ -4,7 +4,6 @@
package udp
import (
"context"
"encoding/binary"
"fmt"
"net"
@@ -19,58 +18,58 @@ import (
)
type StdConn struct {
udpConn *net.UDPConn
rawConn syscall.RawConn
isV4 bool
l *logrus.Logger
batch int
sysFd int
isV4 bool
l *logrus.Logger
batch int
}
func setReusePort(network, address string, c syscall.RawConn) error {
var opErr error
err := c.Control(func(fd uintptr) {
opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
//CloseOnExec already set by the runtime
})
if err != nil {
return err
func maybeIPV4(ip net.IP) (net.IP, bool) {
ip4 := ip.To4()
if ip4 != nil {
return ip4, true
}
return opErr
return ip, false
}
func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
listen := netip.AddrPortFrom(ip, uint16(port))
lc := net.ListenConfig{}
if multi {
lc.Control = setReusePort
af := unix.AF_INET6
if ip.Is4() {
af = unix.AF_INET
}
//this context is only used during the bind operation, you can't cancel it to kill the socket
pc, err := lc.ListenPacket(context.Background(), "udp", listen.String())
syscall.ForkLock.RLock()
fd, err := unix.Socket(af, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
if err == nil {
unix.CloseOnExec(fd)
}
syscall.ForkLock.RUnlock()
if err != nil {
unix.Close(fd)
return nil, fmt.Errorf("unable to open socket: %s", err)
}
udpConn := pc.(*net.UDPConn)
rawConn, err := udpConn.SyscallConn()
if err != nil {
_ = udpConn.Close()
return nil, err
}
//gotta find out if we got an AF_INET6 socket or not:
out := &StdConn{
udpConn: udpConn,
rawConn: rawConn,
l: l,
batch: batch,
if multi {
if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
return nil, fmt.Errorf("unable to set SO_REUSEPORT: %s", err)
}
}
af, err := out.getSockOptInt(unix.SO_DOMAIN)
if err != nil {
_ = out.Close()
return nil, err
var sa unix.Sockaddr
if ip.Is4() {
sa4 := &unix.SockaddrInet4{Port: port}
sa4.Addr = ip.As4()
sa = sa4
} else {
sa6 := &unix.SockaddrInet6{Port: port}
sa6.Addr = ip.As16()
sa = sa6
}
if err = unix.Bind(fd, sa); err != nil {
return nil, fmt.Errorf("unable to bind to socket: %s", err)
}
out.isV4 = af == unix.AF_INET
return out, nil
return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, err
}
func (u *StdConn) SupportsMultipleReaders() bool {
@@ -81,133 +80,62 @@ func (u *StdConn) Rebind() error {
return nil
}
func (u *StdConn) getSockOptInt(opt int) (int, error) {
if u.rawConn == nil {
return 0, fmt.Errorf("no UDP connection")
}
var out int
var opErr error
err := u.rawConn.Control(func(fd uintptr) {
out, opErr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, opt)
})
if err != nil {
return 0, err
}
return out, opErr
}
func (u *StdConn) setSockOptInt(opt int, n int) error {
if u.rawConn == nil {
return fmt.Errorf("no UDP connection")
}
var opErr error
err := u.rawConn.Control(func(fd uintptr) {
opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, opt, n)
})
if err != nil {
return err
}
return opErr
}
func (u *StdConn) SetRecvBuffer(n int) error {
return u.setSockOptInt(unix.SO_RCVBUFFORCE, n)
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n)
}
func (u *StdConn) SetSendBuffer(n int) error {
return u.setSockOptInt(unix.SO_SNDBUFFORCE, n)
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
}
func (u *StdConn) SetSoMark(mark int) error {
return u.setSockOptInt(unix.SO_MARK, mark)
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK, mark)
}
func (u *StdConn) GetRecvBuffer() (int, error) {
return u.getSockOptInt(unix.SO_RCVBUF)
return unix.GetsockoptInt(int(u.sysFd), unix.SOL_SOCKET, unix.SO_RCVBUF)
}
func (u *StdConn) GetSendBuffer() (int, error) {
return u.getSockOptInt(unix.SO_SNDBUF)
return unix.GetsockoptInt(int(u.sysFd), unix.SOL_SOCKET, unix.SO_SNDBUF)
}
func (u *StdConn) GetSoMark() (int, error) {
return u.getSockOptInt(unix.SO_MARK)
return unix.GetsockoptInt(int(u.sysFd), unix.SOL_SOCKET, unix.SO_MARK)
}
func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
a := u.udpConn.LocalAddr()
sa, err := unix.Getsockname(u.sysFd)
if err != nil {
return netip.AddrPort{}, err
}
switch v := a.(type) {
case *net.UDPAddr:
addr, ok := netip.AddrFromSlice(v.IP)
if !ok {
return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP)
}
return netip.AddrPortFrom(addr, uint16(v.Port)), nil
switch sa := sa.(type) {
case *unix.SockaddrInet4:
return netip.AddrPortFrom(netip.AddrFrom4(sa.Addr), uint16(sa.Port)), nil
case *unix.SockaddrInet6:
return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr), uint16(sa.Port)), nil
default:
return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a)
return netip.AddrPort{}, fmt.Errorf("unsupported sock type: %T", sa)
}
}
func recvmmsg(fd uintptr, msgs []rawMessage) (int, bool, error) {
var errno syscall.Errno
n, _, errno := unix.Syscall6(
unix.SYS_RECVMMSG,
fd,
uintptr(unsafe.Pointer(&msgs[0])),
uintptr(len(msgs)),
unix.MSG_WAITFORONE,
0,
0,
)
if errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK {
// No data available, block for I/O and try again.
return int(n), false, nil
}
if errno != 0 {
return int(n), true, &net.OpError{Op: "recvmmsg", Err: errno}
}
return int(n), true, nil
}
func (u *StdConn) listenOutSingle(r EncReader) error {
var err error
var n int
var from netip.AddrPort
buffer := make([]byte, MTU)
for {
n, from, err = u.udpConn.ReadFromUDPAddrPort(buffer)
if err != nil {
return err
}
from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port())
r(from, buffer[:n])
}
}
func (u *StdConn) listenOutBatch(r EncReader) error {
func (u *StdConn) ListenOut(r EncReader) {
var ip netip.Addr
var n int
var operr error
msgs, buffers, names := u.PrepareRawMessages(u.batch)
//reader needs to capture variables from this function, since it's used as a lambda with rawConn.Read
//defining it outside the loop so it gets re-used
reader := func(fd uintptr) (done bool) {
n, done, operr = recvmmsg(fd, msgs)
return done
read := u.ReadMulti
if u.batch == 1 {
read = u.ReadSingle
}
for {
err := u.rawConn.Read(reader)
n, err := read(msgs)
if err != nil {
return err
}
if operr != nil {
return operr
u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
return
}
for i := 0; i < n; i++ {
@@ -222,17 +150,106 @@ func (u *StdConn) listenOutBatch(r EncReader) error {
}
}
func (u *StdConn) ListenOut(r EncReader) error {
if u.batch == 1 {
return u.listenOutSingle(r)
} else {
return u.listenOutBatch(r)
func (u *StdConn) ReadSingle(msgs []rawMessage) (int, error) {
for {
n, _, err := unix.Syscall6(
unix.SYS_RECVMSG,
uintptr(u.sysFd),
uintptr(unsafe.Pointer(&(msgs[0].Hdr))),
0,
0,
0,
0,
)
if err != 0 {
return 0, &net.OpError{Op: "recvmsg", Err: err}
}
msgs[0].Len = uint32(n)
return 1, nil
}
}
func (u *StdConn) ReadMulti(msgs []rawMessage) (int, error) {
for {
n, _, err := unix.Syscall6(
unix.SYS_RECVMMSG,
uintptr(u.sysFd),
uintptr(unsafe.Pointer(&msgs[0])),
uintptr(len(msgs)),
unix.MSG_WAITFORONE,
0,
0,
)
if err != 0 {
return 0, &net.OpError{Op: "recvmmsg", Err: err}
}
return int(n), nil
}
}
func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
_, err := u.udpConn.WriteToUDPAddrPort(b, ip)
return err
if u.isV4 {
return u.writeTo4(b, ip)
}
return u.writeTo6(b, ip)
}
func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
var rsa unix.RawSockaddrInet6
rsa.Family = unix.AF_INET6
rsa.Addr = ip.Addr().As16()
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
for {
_, _, err := unix.Syscall6(
unix.SYS_SENDTO,
uintptr(u.sysFd),
uintptr(unsafe.Pointer(&b[0])),
uintptr(len(b)),
uintptr(0),
uintptr(unsafe.Pointer(&rsa)),
uintptr(unix.SizeofSockaddrInet6),
)
if err != 0 {
return &net.OpError{Op: "sendto", Err: err}
}
return nil
}
}
func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
if !ip.Addr().Is4() {
return ErrInvalidIPv6RemoteForSocket
}
var rsa unix.RawSockaddrInet4
rsa.Family = unix.AF_INET
rsa.Addr = ip.Addr().As4()
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
for {
_, _, err := unix.Syscall6(
unix.SYS_SENDTO,
uintptr(u.sysFd),
uintptr(unsafe.Pointer(&b[0])),
uintptr(len(b)),
uintptr(0),
uintptr(unsafe.Pointer(&rsa)),
uintptr(unix.SizeofSockaddrInet4),
)
if err != 0 {
return &net.OpError{Op: "sendto", Err: err}
}
return nil
}
}
func (u *StdConn) ReloadConfig(c *config.C) {
@@ -285,28 +302,15 @@ func (u *StdConn) ReloadConfig(c *config.C) {
func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
if u.rawConn == nil {
return fmt.Errorf("no UDP connection")
}
var opErr error
err := u.rawConn.Control(func(fd uintptr) {
_, _, syserr := unix.Syscall6(unix.SYS_GETSOCKOPT, fd, uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
if syserr != 0 {
opErr = syserr
}
})
if err != nil {
_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
if err != 0 {
return err
}
return opErr
return nil
}
func (u *StdConn) Close() error {
if u.udpConn != nil {
return u.udpConn.Close()
}
return nil
return syscall.Close(u.sysFd)
}
func NewUDPStatsEmitter(udpConns []Conn) func() {
+3 -2
View File
@@ -140,7 +140,7 @@ func (u *RIOConn) bind(l *logrus.Logger, sa windows.Sockaddr) error {
return nil
}
func (u *RIOConn) ListenOut(r EncReader) error {
func (u *RIOConn) ListenOut(r EncReader) {
buffer := make([]byte, MTU)
var lastRecvErr time.Time
@@ -151,7 +151,8 @@ func (u *RIOConn) ListenOut(r EncReader) error {
if err != nil {
if errors.Is(err, net.ErrClosed) {
return err
u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
return
}
// Dampen unexpected message warns to once per minute
if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute {
+2 -3
View File
@@ -6,7 +6,6 @@ package udp
import (
"io"
"net/netip"
"os"
"sync/atomic"
"github.com/sirupsen/logrus"
@@ -107,11 +106,11 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
return nil
}
func (u *TesterConn) ListenOut(r EncReader) error {
func (u *TesterConn) ListenOut(r EncReader) {
for {
p, ok := <-u.RxPackets
if !ok {
return os.ErrClosed
return
}
r(p.From, p.Data)
}