mirror of
https://github.com/slackhq/nebula.git
synced 2026-09-30 13:56:37 +02:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 JackDoanandClaude Fable 5
|
10e9514e44 | ||
|
JackDoanandClaude Fable 5
|
913a37cfee |
@@ -43,15 +43,8 @@ runs:
|
||||
with:
|
||||
role-to-assume: ${{ inputs.role }}
|
||||
aws-region: ${{ inputs.region }}
|
||||
# An STS secret key with special characters does not survive the
|
||||
# pwsh -> make -> MSYS sh -> aws.exe chain, and SigV4 then signs with a
|
||||
# key that no longer matches, so the first S3 upload fails with
|
||||
# SignatureDoesNotMatch. Retries the assume until it comes back clean.
|
||||
# Same fix as DefinedNet/dnclient#867.
|
||||
special-characters-workaround: true
|
||||
# Overridden by the workaround above and kept for whenever that goes:
|
||||
# the default 12 rides out IAM trust-policy propagation, and once the
|
||||
# role is stable a real misconfiguration should fail fast.
|
||||
# Default is 12 retries to ride out IAM trust-policy propagation; once
|
||||
# the role is stable we want a real misconfiguration to fail fast.
|
||||
retry-max-attempts: 5
|
||||
|
||||
- name: Sign .exe files
|
||||
|
||||
@@ -12,9 +12,9 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.26'
|
||||
go-version: '1.25'
|
||||
check-latest: true
|
||||
|
||||
- name: Build
|
||||
@@ -38,9 +38,9 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.26'
|
||||
go-version: '1.25'
|
||||
check-latest: true
|
||||
|
||||
- name: Build
|
||||
@@ -73,81 +73,27 @@ jobs:
|
||||
build-darwin:
|
||||
name: Build Universal Darwin
|
||||
env:
|
||||
HAS_SIGNING_CREDS: ${{ secrets.APPLE_SIGNING_ROLE_ARN != '' }}
|
||||
HAS_SIGNING_CREDS: ${{ secrets.AC_USERNAME != '' }}
|
||||
runs-on: macos-latest
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.26'
|
||||
go-version: '1.25'
|
||||
check-latest: true
|
||||
|
||||
# GitHub holds ARNs, not credentials, and ARNs outlive a rotation
|
||||
- name: Configure AWS credentials
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
uses: aws-actions/configure-aws-credentials@v6
|
||||
with:
|
||||
role-to-assume: ${{ secrets.APPLE_SIGNING_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
# parse-json-secrets unpacks into SIGNING_* and ASC_*, masked on the way in
|
||||
- name: Fetch signing credentials
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@v3
|
||||
with:
|
||||
parse-json-secrets: true
|
||||
secret-ids: |
|
||||
SIGNING,${{ secrets.APPLE_SIGNING_DEVELOPER_ID_ARN }}
|
||||
ASC,${{ secrets.APPLE_NOTARY_KEY_ARN }}
|
||||
|
||||
- name: Import certificates
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
uses: Apple-Actions/import-codesign-certs@v7
|
||||
with:
|
||||
p12-file-base64: ${{ env.SIGNING_P12_BASE64 }}
|
||||
p12-password: ${{ env.SIGNING_PASSWORD }}
|
||||
|
||||
# The action imports but does not check the chain validates, which is how a p12
|
||||
# missing its intermediate reaches a failing codesign
|
||||
- name: Check the identity is usable
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
run: |
|
||||
: "${SIGNING_IDENTITY_SHA1:?empty, so the secret has no identity_sha1}"
|
||||
identities=$(security find-identity -v -p codesigning signing_temp.keychain)
|
||||
case "$identities" in
|
||||
*"$SIGNING_IDENTITY_SHA1"*) ;;
|
||||
*) printf '%s\n' "$identities" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
# notarytool wants the key as a file
|
||||
- name: Write the App Store Connect key
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
run: |
|
||||
mkdir -p ~/private_keys
|
||||
chmod 700 ~/private_keys
|
||||
key_path="$HOME/private_keys/AuthKey_${ASC_KEY_ID}.p8"
|
||||
(umask 077; printf '%s\n' "$ASC_PRIVATE_KEY" > "$key_path")
|
||||
echo "ASC_P8=$key_path" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Drop the credentials from the environment
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
run: |
|
||||
# The action's own inventory, so a new field in a secret is covered
|
||||
python3 -c '
|
||||
import json, os
|
||||
raw = os.environ.get("SECRETS_LIST_CLEAN_UP")
|
||||
if raw is None and os.environ.get("SIGNING_P12_BASE64"):
|
||||
raise SystemExit("SECRETS_LIST_CLEAN_UP is gone, fetched secrets are not being scrubbed")
|
||||
keep = {"SIGNING_IDENTITY_SHA1", "ASC_KEY_ID", "ASC_ISSUER_ID"}
|
||||
names = [n for n in json.loads(raw or "[]") if n not in keep]
|
||||
print("\n".join(f"{n}=" for n in dict.fromkeys(names)))
|
||||
' >> "$GITHUB_ENV"
|
||||
p12-file-base64: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }}
|
||||
p12-password: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }}
|
||||
|
||||
- name: Build, sign, and notarize
|
||||
env:
|
||||
AC_USERNAME: ${{ secrets.AC_USERNAME }}
|
||||
AC_PASSWORD: ${{ secrets.AC_PASSWORD }}
|
||||
run: |
|
||||
rm -rf release
|
||||
mkdir release
|
||||
@@ -156,34 +102,17 @@ jobs:
|
||||
lipo -create -output ./release/nebula ./build/darwin-amd64/nebula ./build/darwin-arm64/nebula
|
||||
lipo -create -output ./release/nebula-cert ./build/darwin-amd64/nebula-cert ./build/darwin-arm64/nebula-cert
|
||||
|
||||
# Unset in a fork, which has no credentials to sign with
|
||||
if [ -n "$SIGNING_IDENTITY_SHA1" ]; then
|
||||
codesign -s "$SIGNING_IDENTITY_SHA1" -f -v --timestamp --options=runtime -i "net.defined.nebula" ./release/nebula
|
||||
codesign -s "$SIGNING_IDENTITY_SHA1" -f -v --timestamp --options=runtime -i "net.defined.nebula-cert" ./release/nebula-cert
|
||||
if [ -n "$AC_USERNAME" ]; then
|
||||
codesign -s "10BC1FDDEB6CE753550156C0669109FAC49E4D1E" -f -v --timestamp --options=runtime -i "net.defined.nebula" ./release/nebula
|
||||
codesign -s "10BC1FDDEB6CE753550156C0669109FAC49E4D1E" -f -v --timestamp --options=runtime -i "net.defined.nebula-cert" ./release/nebula-cert
|
||||
fi
|
||||
|
||||
zip -j release/nebula-darwin.zip release/nebula-cert release/nebula
|
||||
|
||||
if [ -n "$ASC_P8" ]; then
|
||||
xcrun notarytool submit ./release/nebula-darwin.zip --key "$ASC_P8" --key-id "$ASC_KEY_ID" --issuer "$ASC_ISSUER_ID" --wait
|
||||
if [ -n "$AC_USERNAME" ]; then
|
||||
xcrun notarytool submit ./release/nebula-darwin.zip --team-id "576H3XS7FP" --apple-id "$AC_USERNAME" --password "$AC_PASSWORD" --wait
|
||||
fi
|
||||
|
||||
- name: Drop the signing key
|
||||
if: always() && env.HAS_SIGNING_CREDS == 'true'
|
||||
run: |
|
||||
# Locked, not deleted: import-codesign-certs deletes it in its own post
|
||||
# step and fails the job if it is already gone. Locked is unusable.
|
||||
security lock-keychain signing_temp.keychain || true
|
||||
rm -f "$ASC_P8"
|
||||
# Nothing later in this job needs AWS
|
||||
python3 -c '
|
||||
import json, os
|
||||
names = json.loads(os.environ.get("SECRETS_LIST_CLEAN_UP") or "[]")
|
||||
names += ["ASC_P8", "SIGNING_IDENTITY_SHA1", "ASC_KEY_ID", "ASC_ISSUER_ID",
|
||||
"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
|
||||
print("\n".join(f"{n}=" for n in dict.fromkeys(names)))
|
||||
' >> "$GITHUB_ENV"
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
|
||||
@@ -32,9 +32,9 @@ jobs:
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.26'
|
||||
go-version: '1.25'
|
||||
check-latest: true
|
||||
|
||||
- name: add hashicorp source
|
||||
@@ -64,9 +64,9 @@ jobs:
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.26'
|
||||
go-version: '1.25'
|
||||
check-latest: true
|
||||
|
||||
- name: add hashicorp source
|
||||
@@ -90,9 +90,9 @@ jobs:
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.26'
|
||||
go-version: '1.25'
|
||||
check-latest: true
|
||||
|
||||
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own
|
||||
|
||||
@@ -20,9 +20,9 @@ jobs:
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.26'
|
||||
go-version: '1.25'
|
||||
check-latest: true
|
||||
|
||||
- name: build
|
||||
|
||||
@@ -20,9 +20,9 @@ jobs:
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.26'
|
||||
go-version: '1.25'
|
||||
check-latest: true
|
||||
|
||||
- name: Install goimports
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: v2.12
|
||||
version: v2.5
|
||||
|
||||
test:
|
||||
name: Test ${{ matrix.name }}
|
||||
@@ -80,9 +80,9 @@ jobs:
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.26'
|
||||
go-version: '1.25'
|
||||
check-latest: true
|
||||
|
||||
- name: Build
|
||||
@@ -125,9 +125,9 @@ jobs:
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.26'
|
||||
go-version: '1.25'
|
||||
check-latest: true
|
||||
|
||||
- name: Build ${{ matrix.name }}
|
||||
|
||||
+1
-110
@@ -7,113 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [1.11.1] - 2026-08-21
|
||||
|
||||
See the [v1.11.1](https://github.com/slackhq/nebula/milestone/30?closed=1) milestone for a complete list of changes.
|
||||
|
||||
### Changed
|
||||
|
||||
- IPv6 packets whose next header is a protocol Nebula does not parse (SCTP, GRE, IP-in-IP, etc.) are now
|
||||
classified as that protocol with no ports, closing a firewall bypass where a crafted payload could steer
|
||||
the classifier into reading one as TCP/UDP and matching a TCP/UDP rule. These packets are now matched as
|
||||
their true protocol, so only a `proto: any` rule allows them. If you carry one of these protocols over the
|
||||
overlay, confirm a `proto: any` rule covers it before upgrading, it may have been passing only through this
|
||||
bypass. (#1840)
|
||||
- Drop the dependency on `github.com/cyberdelia/go-metrics-graphite`, which has been unmaintained for over ten
|
||||
years, by inlining the small amount of code Nebula used. (#1832)
|
||||
|
||||
### Fixed
|
||||
|
||||
- The ICMPv6 type was read from the wrong byte when classifying IPv6 packets, so the echo identifier used
|
||||
for conntrack was never picked up. (#1840)
|
||||
- Enforce outbound message counter limits so a tunnel is rehandshaked before the counter can wrap, preventing
|
||||
nonce reuse. This is unreachable in practice, but is enforced as a defense-in-depth measure. (#1841)
|
||||
- Prevent `nebula-cert ca` from running out of memory on 32bit systems when generating encrypted private keys. (#1834)
|
||||
- Tolerate `ErrDumpInterrupted` when listing tun addresses on Linux, so a transient interrupted netlink dump
|
||||
no longer aborts startup. (#1835)
|
||||
|
||||
## [1.11.0] - 2026-07-23
|
||||
|
||||
See the [v1.11.0](https://github.com/slackhq/nebula/milestone/25?closed=1) milestone for a complete list of changes.
|
||||
|
||||
### Breaking
|
||||
|
||||
- Logging has switched from logrus to Go's structured `slog`. Log output changes: levels are upper case
|
||||
(`level=INFO`), trace prints as `level=DEBUG-4`, timestamps are always RFC3339Nano and `logging.timestamp_format`
|
||||
is ignored, and some messages were reworded. Review any log parsing before upgrading. This is also an API break
|
||||
for embedders, as constructors now take a `*slog.Logger`. (#1672, #1734, #1621)
|
||||
- `firewall.inbound_action` and `firewall.outbound_action` (used to set reject vs. drop policy) were each being
|
||||
applied to the opposite direction, that is now corrected. This only affects how blocked packets are answered, not
|
||||
which packets the firewall allows or denies. If you set either of these you are getting the behavior of the other
|
||||
one today and likely want to swap them before upgrading. (#1798)
|
||||
- On Windows, Nebula now installs WFP PERMIT filters for the nebula adapter and the listener port by default. WFP
|
||||
sits below Windows Defender Firewall, so any WDF inbound rules you rely on for either will no longer apply. Set
|
||||
`tun.windows_bypass_wdf` and `listen.windows_bypass_wdf` to false to leave WDF in charge. (#1710)
|
||||
- On Windows, the nebula device is now set to the `private` network category instead of whatever Windows decided,
|
||||
which is usually `Public`. This makes the host firewall less restrictive on the overlay. Set
|
||||
`tun.network_category` to `unset` to keep the old behavior. (#1710)
|
||||
- Reject packets for non-TCP now use ICMP code 13, communication administratively prohibited, instead of code 3,
|
||||
port unreachable. Anything keying off the old code needs updating. (#1766, #1768)
|
||||
- The SSH debug server's profiling commands are now confined to `sshd.sandbox_dir`, which defaults to
|
||||
`$TMP/nebula-debug`. Relative paths resolve inside it and absolute paths outside it are rejected, so anything
|
||||
scripting `start-cpu-profile`, `save-heap-profile`, or `save-mutex-profile` with a path elsewhere needs the
|
||||
directory set. The directory is not created for you. (#1622)
|
||||
|
||||
### Added
|
||||
|
||||
- Sign the Windows release binaries. (#1718)
|
||||
- Generate IPv6 reject packets, matching the existing IPv4 behavior. (#1766, #1767, #1768)
|
||||
- Accept `-` in `nebula-cert` to read from stdin or write to stdout. (#1714)
|
||||
- Search for both `config.yml` and `config.yaml` in service and command line modes. (#1717)
|
||||
- Add version labels to the Docker/OCI images. (#1772)
|
||||
- Rebind the listener and re-query lighthouses on macOS when the underlay network changes, so devices moving
|
||||
between wifi and wired or between networks recover without waiting for dead tunnel detection. Controlled by
|
||||
`listen.rebind_on_network_change` (default `true`, not reloadable). (#1816)
|
||||
|
||||
### Changed
|
||||
|
||||
- Reload the firewall when the unsafe networks in the certificate change. (#1719)
|
||||
- Reconfigure, start, and stop the stats listener on a config reload instead of requiring a restart. (#1670)
|
||||
- Update a static host's addresses when they change on reload. (#1713)
|
||||
- Don't require a port on ICMP firewall rules. (#1609)
|
||||
- Connection track ICMP traffic. (#1602)
|
||||
- Return `NODATA` instead of `NXDOMAIN` from the DNS server for a name that exists but has no record of the
|
||||
requested type, so clients that query `AAAA` first (busybox/Alpine) fall through to `A`. (#1668)
|
||||
- Record the local host's details in the DNS server. (#1716)
|
||||
- Install Windows unsafe routes as link routes. (#1709)
|
||||
- Reduce relay handshake log spam, and only log a handshake send error at error level when the remote list
|
||||
changes. (#1733, #1765, #1810)
|
||||
- Start, stop, and reload subsystems (DNS, stats, conntrack, ssh, punchy) cleanly without leaking goroutines. (#1640, #1654, #1661, #1667, #1669, #1708, #1806, #1815)
|
||||
- `Control` is now safe to stop and wait on from any lifecycle state, and a new `Control.Wait` blocks until nebula
|
||||
has fully stopped and returns the first fatal reader error. Failed starts release the udp sockets and tun fd
|
||||
instead of leaking them. (#1794)
|
||||
- Trigger an immediate lighthouse update when reconnecting to or adding a lighthouse instead of waiting for the next update tick. (#1645)
|
||||
- Bring the Darwin and OpenBSD tun implementations in line with the other BSDs. (#1703)
|
||||
- Update to build against go v1.26. (#1818)
|
||||
- Various dependency updates. (#1586, #1587, #1604, #1617, #1618, #1627, #1628, #1629, #1652, #1664, #1665, #1697, #1721, #1732, #1742, #1743, #1750, #1763, #1771, #1782, #1800, #1807)
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix a data race on a host's remote address that could send packets to the wrong address during a roam. (#1773)
|
||||
- Fix tunnels that could permanently escape connection manager monitoring. (#1752)
|
||||
- Fix a crash when reloading the SSH server's trusted keys. (#1787)
|
||||
- Fix hostmap corruption when a host has multiple overlay addresses. Each address now gets its own list instead of
|
||||
a single shared chain, which also fixes two latent bugs on the add and makePrimary paths. (#1788, #1790)
|
||||
- Apply `remote_allow_list` IPv4 rules to 4-in-6 mapped addresses. (#1786)
|
||||
- Don't panic in the DNS server on a short or empty query name. (#1635)
|
||||
- Advance the replay window on relayed packets so a relay drops replayed frames instead of re-forwarding them. (#1751)
|
||||
- Fix a race in relay state handling. (#1753)
|
||||
- Lock replay window updates so concurrent readers can't corrupt it. (#1802)
|
||||
- Reject malformed handshakes more reliably, including invalid ed25519 key lengths. (#1601, #1756)
|
||||
- Properly handle `closetunnel` packets. (#1638)
|
||||
- Fix an IPv6 extension-header length overflow that could make the firewall parse the wrong protocol and ports. (#1789)
|
||||
- Fix relay re-establishment when a handshake arrives over a relay entry that a one-sided teardown left
|
||||
`Disestablished`, which silently dropped every send until dead tunnel detection forced a re-handshake. (#1805)
|
||||
- Don't build new relay state on a tunnel that was just discarded. (#1796)
|
||||
- Don't delete the wrong pending hostinfo in the handshake manager. (#1811)
|
||||
- Don't call the packet reader after a UDP error on Darwin. (#1755)
|
||||
- Open the FreeBSD tun device non blocking. (#1666)
|
||||
|
||||
## [1.10.3] - 2026-02-06
|
||||
|
||||
### Security
|
||||
@@ -895,9 +788,7 @@ created.)
|
||||
|
||||
- Initial public release.
|
||||
|
||||
[Unreleased]: https://github.com/slackhq/nebula/compare/v1.11.1...HEAD
|
||||
[1.11.1]: https://github.com/slackhq/nebula/releases/tag/v1.11.1
|
||||
[1.11.0]: https://github.com/slackhq/nebula/releases/tag/v1.11.0
|
||||
[Unreleased]: https://github.com/slackhq/nebula/compare/v1.10.3...HEAD
|
||||
[1.10.3]: https://github.com/slackhq/nebula/releases/tag/v1.10.3
|
||||
[1.10.2]: https://github.com/slackhq/nebula/releases/tag/v1.10.2
|
||||
[1.10.1]: https://github.com/slackhq/nebula/releases/tag/v1.10.1
|
||||
|
||||
+2
-17
@@ -8,7 +8,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"math/bits"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -45,20 +44,6 @@ type caFlags struct {
|
||||
}
|
||||
|
||||
func newCaFlags() *caFlags {
|
||||
// prevent running out of memory on 32-bit systems by defaulting to
|
||||
// RFC9106's recommendation for memory-constrained environments
|
||||
var (
|
||||
defaultArgonMemory uint
|
||||
defaultArgonIterations uint
|
||||
)
|
||||
if bits.UintSize == 32 {
|
||||
defaultArgonMemory = 64 * 1024
|
||||
defaultArgonIterations = 3
|
||||
} else {
|
||||
defaultArgonMemory = 2 * 1024 * 1024
|
||||
defaultArgonIterations = 1
|
||||
}
|
||||
|
||||
cf := caFlags{set: flag.NewFlagSet("ca", flag.ContinueOnError)}
|
||||
cf.set.Usage = func() {}
|
||||
cf.name = cf.set.String("name", "", "Required: name of the certificate authority")
|
||||
@@ -70,9 +55,9 @@ func newCaFlags() *caFlags {
|
||||
cf.groups = cf.set.String("groups", "", "Optional: comma separated list of groups. This will limit which groups subordinate certs can use")
|
||||
cf.networks = cf.set.String("networks", "", "Optional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in networks")
|
||||
cf.unsafeNetworks = cf.set.String("unsafe-networks", "", "Optional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in unsafe networks")
|
||||
cf.argonMemory = cf.set.Uint("argon-memory", defaultArgonMemory, "Optional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase")
|
||||
cf.argonMemory = cf.set.Uint("argon-memory", 2*1024*1024, "Optional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase")
|
||||
cf.argonParallelism = cf.set.Uint("argon-parallelism", 4, "Optional: Argon2 parallelism parameter used for encrypted private key passphrase")
|
||||
cf.argonIterations = cf.set.Uint("argon-iterations", defaultArgonIterations, "Optional: Argon2 iterations parameter used for encrypted private key passphrase")
|
||||
cf.argonIterations = cf.set.Uint("argon-iterations", 1, "Optional: Argon2 iterations parameter used for encrypted private key passphrase")
|
||||
cf.encryption = cf.set.Bool("encrypt", false, "Optional: prompt for passphrase and write out-key in an encrypted format")
|
||||
cf.curve = cf.set.String("curve", "25519", "EdDSA/ECDSA Curve (25519, P256)")
|
||||
cf.p11url = p11Flag(cf.set)
|
||||
|
||||
@@ -7,9 +7,7 @@ import (
|
||||
"bytes"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"math/bits"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -24,18 +22,6 @@ func Test_caSummary(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_caHelp(t *testing.T) {
|
||||
var (
|
||||
defaultArgonMemory string
|
||||
defaultArgonIterations string
|
||||
)
|
||||
if bits.UintSize == 32 {
|
||||
defaultArgonMemory = strconv.Itoa(64 * 1024)
|
||||
defaultArgonIterations = strconv.Itoa(3)
|
||||
} else {
|
||||
defaultArgonMemory = strconv.Itoa(2 * 1024 * 1024)
|
||||
defaultArgonIterations = strconv.Itoa(1)
|
||||
}
|
||||
|
||||
ob := &bytes.Buffer{}
|
||||
caHelp(ob)
|
||||
assert.Equal(
|
||||
@@ -43,9 +29,9 @@ func Test_caHelp(t *testing.T) {
|
||||
"Usage of "+os.Args[0]+" ca <flags>: create a self signed certificate authority\n"+
|
||||
" Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+
|
||||
" -argon-iterations uint\n"+
|
||||
" \tOptional: Argon2 iterations parameter used for encrypted private key passphrase (default "+defaultArgonIterations+")\n"+
|
||||
" \tOptional: Argon2 iterations parameter used for encrypted private key passphrase (default 1)\n"+
|
||||
" -argon-memory uint\n"+
|
||||
" \tOptional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase (default "+defaultArgonMemory+")\n"+
|
||||
" \tOptional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase (default 2097152)\n"+
|
||||
" -argon-parallelism uint\n"+
|
||||
" \tOptional: Argon2 parallelism parameter used for encrypted private key passphrase (default 4)\n"+
|
||||
" -curve string\n"+
|
||||
@@ -202,16 +188,10 @@ func Test_ca(t *testing.T) {
|
||||
k, _ := pem.Decode(rb)
|
||||
ned, err := cert.UnmarshalNebulaEncryptedData(k.Bytes)
|
||||
require.NoError(t, err)
|
||||
|
||||
if bits.UintSize == 32 {
|
||||
assert.Equal(t, uint32(64*1024), ned.EncryptionMetadata.Argon2Parameters.Memory)
|
||||
assert.Equal(t, uint32(3), ned.EncryptionMetadata.Argon2Parameters.Iterations)
|
||||
} else {
|
||||
assert.Equal(t, uint32(2*1024*1024), ned.EncryptionMetadata.Argon2Parameters.Memory)
|
||||
assert.Equal(t, uint32(1), ned.EncryptionMetadata.Argon2Parameters.Iterations)
|
||||
}
|
||||
|
||||
// we won't know salt in advance, so just check start of string
|
||||
assert.Equal(t, uint32(2*1024*1024), ned.EncryptionMetadata.Argon2Parameters.Memory)
|
||||
assert.Equal(t, uint8(4), ned.EncryptionMetadata.Argon2Parameters.Parallelism)
|
||||
assert.Equal(t, uint32(1), ned.EncryptionMetadata.Argon2Parameters.Iterations)
|
||||
|
||||
// verify the key is valid and decrypt-able
|
||||
var curve cert.Curve
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
//go:build linux && !android && !e2e_testing
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/slackhq/nebula"
|
||||
"github.com/slackhq/nebula/cert"
|
||||
cert_test "github.com/slackhq/nebula/cert_test"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestControlStopClosesOnTimer reproduces the dnclient lifecycle: nebula runs as
|
||||
// a library, and on a config update dnclient calls Stop() in-process to tear the
|
||||
// old instance down before starting a new one. This boots a real nebula (real
|
||||
// blocking UDP sockets, tun disabled), lets it run, then Stop()s it on a timer
|
||||
// and asserts it actually closes. If the reader goroutines parked in recvmmsg
|
||||
// don't wake on Close(), Wait() blocks forever and this fails with a goroutine
|
||||
// dump instead of relying on a process signal to unstick them.
|
||||
func TestControlStopClosesOnTimer(t *testing.T) {
|
||||
l := test.NewLogger()
|
||||
dir := t.TempDir()
|
||||
|
||||
before := time.Now().Add(-time.Hour)
|
||||
after := time.Now().Add(time.Hour)
|
||||
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
|
||||
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
|
||||
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "close-on-timer", before, after, networks, nil, nil)
|
||||
|
||||
caPath := filepath.Join(dir, "ca.pem")
|
||||
certPath := filepath.Join(dir, "cert.pem")
|
||||
keyPath := filepath.Join(dir, "key.pem")
|
||||
require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
|
||||
require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
|
||||
require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
|
||||
|
||||
// tun disabled so no device/root is needed; routines: 2 so we exercise the
|
||||
// multi-socket (SO_REUSEPORT) teardown, which is where dnclient runs.
|
||||
configBody := fmt.Sprintf(`
|
||||
pki:
|
||||
ca: %s
|
||||
cert: %s
|
||||
key: %s
|
||||
listen:
|
||||
host: 127.0.0.1
|
||||
port: 0
|
||||
tun:
|
||||
disabled: true
|
||||
firewall:
|
||||
outbound:
|
||||
- port: any
|
||||
proto: any
|
||||
host: any
|
||||
inbound:
|
||||
- port: any
|
||||
proto: any
|
||||
host: any
|
||||
routines: 2
|
||||
`, caPath, certPath, keyPath)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
|
||||
|
||||
c := config.NewC(l)
|
||||
require.NoError(t, c.Load(dir))
|
||||
|
||||
ctrl, err := nebula.Main(c, false, "close-on-timer", l, nil)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, ctrl.Start())
|
||||
|
||||
// Run like a live nebula, then close on a timer, exactly as dnclient does.
|
||||
<-time.NewTimer(5 * time.Second).C
|
||||
|
||||
stopped := make(chan struct{})
|
||||
go func() {
|
||||
ctrl.Stop() // closes the udp sockets (shutdown(2)) and the tun
|
||||
ctrl.Wait() // blocks until every reader goroutine has returned
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-stopped:
|
||||
t.Log("nebula closed cleanly on timer")
|
||||
case <-time.After(10 * time.Second):
|
||||
buf := make([]byte, 1<<20)
|
||||
n := runtime.Stack(buf, true)
|
||||
t.Fatalf("nebula did NOT close within 10s of Stop(): a blocking reader never woke\n%s", buf[:n])
|
||||
}
|
||||
}
|
||||
@@ -323,12 +323,6 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
||||
return closeTunnel, hostinfo, nil
|
||||
}
|
||||
|
||||
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
|
||||
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
|
||||
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
|
||||
return deleteTunnel, hostinfo, nil
|
||||
}
|
||||
|
||||
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
|
||||
mainHostInfo := true
|
||||
if primary != nil && primary != hostinfo {
|
||||
@@ -454,11 +448,6 @@ func (cm *connectionManager) shouldSwapPrimary(current *HostInfo) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
if current.ConnectionState.messageCounter.Load() >= RehandshakeAfterMessages {
|
||||
// This tunnel is being rolled for counter exhaustion, never swap back onto its spent key.
|
||||
return false
|
||||
}
|
||||
|
||||
crt := cm.intf.pki.getCertState().getCertificate(current.ConnectionState.myCert.Version())
|
||||
if crt == nil {
|
||||
//my cert was reloaded away. We should definitely swap from this tunnel
|
||||
@@ -555,15 +544,6 @@ func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
|
||||
"reason", "current cert version < pki.initiatingVersion",
|
||||
)
|
||||
|
||||
cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
|
||||
return
|
||||
}
|
||||
if hostinfo.ConnectionState.messageCounter.Load() >= RehandshakeAfterMessages {
|
||||
cm.l.Info("Re-handshaking with remote",
|
||||
"vpnAddrs", hostinfo.vpnAddrs,
|
||||
"reason", "message counter rehandshake threshold reached",
|
||||
)
|
||||
|
||||
cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -25,7 +25,6 @@ func newTestLighthouse() *LightHouse {
|
||||
lighthouses := []netip.Addr{}
|
||||
staticList := map[netip.Addr]struct{}{}
|
||||
|
||||
lh.localAddrsFn = func(*LocalAllowList) []netip.Addr { return nil }
|
||||
lh.lighthouses.Store(&lighthouses)
|
||||
lh.staticList.Store(&staticList)
|
||||
|
||||
@@ -199,79 +198,6 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
|
||||
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
|
||||
}
|
||||
|
||||
func Test_NewConnectionManager_CounterLimits(t *testing.T) {
|
||||
l := test.NewLogger()
|
||||
localrange := netip.MustParsePrefix("10.1.1.1/24")
|
||||
vpnIp := netip.MustParseAddr("172.1.1.2")
|
||||
preferredRanges := []netip.Prefix{localrange}
|
||||
|
||||
// Very incomplete mock objects
|
||||
hostMap := newHostMap(l)
|
||||
hostMap.preferredRanges.Store(&preferredRanges)
|
||||
|
||||
cs := &CertState{
|
||||
initiatingVersion: cert.Version1,
|
||||
privateKey: []byte{},
|
||||
v1Cert: &dummyCert{version: cert.Version1},
|
||||
v1Credential: nil,
|
||||
}
|
||||
|
||||
lh := newTestLighthouse()
|
||||
ifce := &Interface{
|
||||
hostMap: hostMap,
|
||||
inside: &overlaytest.NoopTun{},
|
||||
outside: &udp.NoopConn{},
|
||||
firewall: &Firewall{},
|
||||
lightHouse: lh,
|
||||
pki: &PKI{},
|
||||
myVpnAddrs: []netip.Addr{netip.MustParseAddr("172.1.1.1")}, // sorts below vpnIp so shouldSwapPrimary can proceed
|
||||
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
|
||||
l: l,
|
||||
}
|
||||
ifce.pki.cs.Store(cs)
|
||||
|
||||
conf := config.NewC(test.NewLogger())
|
||||
punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil)
|
||||
nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
|
||||
nc.intf = ifce
|
||||
|
||||
hostinfo := &HostInfo{
|
||||
vpnAddrs: []netip.Addr{vpnIp},
|
||||
localIndexId: 1099,
|
||||
remoteIndexId: 9901,
|
||||
}
|
||||
hostinfo.ConnectionState = &ConnectionState{
|
||||
myCert: &dummyCert{version: cert.Version1},
|
||||
}
|
||||
nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
|
||||
|
||||
// Below the rehandshake threshold, no handshake is started
|
||||
hostinfo.ConnectionState.messageCounter.Store(RehandshakeAfterMessages - 1)
|
||||
nc.tryRehandshake(hostinfo)
|
||||
assert.Nil(t, ifce.handshakeManager.QueryVpnAddr(vpnIp))
|
||||
|
||||
// A tunnel on its current cert would normally swap to primary
|
||||
assert.True(t, nc.shouldSwapPrimary(hostinfo))
|
||||
|
||||
// At the rehandshake threshold, a new handshake is started
|
||||
hostinfo.ConnectionState.messageCounter.Store(RehandshakeAfterMessages)
|
||||
nc.tryRehandshake(hostinfo)
|
||||
assert.NotNil(t, ifce.handshakeManager.QueryVpnAddr(vpnIp))
|
||||
|
||||
// An exhausted tunnel being rolled must never swap back to primary onto its spent key
|
||||
assert.False(t, nc.shouldSwapPrimary(hostinfo))
|
||||
|
||||
// Still below the reject limit, the tunnel stays up
|
||||
nc.In(hostinfo)
|
||||
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
||||
assert.Equal(t, tryRehandshake, decision)
|
||||
|
||||
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
|
||||
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
|
||||
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
||||
assert.Equal(t, deleteTunnel, decision)
|
||||
}
|
||||
|
||||
func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
|
||||
l := test.NewLogger()
|
||||
localrange := netip.MustParsePrefix("10.1.1.1/24")
|
||||
|
||||
+3
-82
@@ -2,29 +2,15 @@ package nebula
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/slackhq/nebula/cert"
|
||||
"github.com/slackhq/nebula/handshake"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/noiseutil"
|
||||
)
|
||||
|
||||
const (
|
||||
ReplayWindow = 1024
|
||||
|
||||
// RehandshakeAfterMessages rolls keys inside the AES-GCM data-volume margin (~2^-36 advantage at 64KB frames).
|
||||
RehandshakeAfterMessages = uint64(1) << 34
|
||||
|
||||
// RejectAfterMessages is the nonce ceiling enforced by noiseutil; a tunnel here is deleted locally, not notified.
|
||||
RejectAfterMessages = noiseutil.RejectAfterMessages
|
||||
)
|
||||
|
||||
// RehandshakeAfterMessages must stay below RejectAfterMessages so tunnels roll before the hard send stop.
|
||||
const _ = RejectAfterMessages - RehandshakeAfterMessages
|
||||
const ReplayWindow = 1024
|
||||
|
||||
type ConnectionState struct {
|
||||
eKey noiseutil.CipherState
|
||||
@@ -34,7 +20,6 @@ type ConnectionState struct {
|
||||
initiator bool
|
||||
messageCounter atomic.Uint64
|
||||
window *Bits
|
||||
decryptLock sync.Mutex
|
||||
writeLock sync.Mutex
|
||||
}
|
||||
|
||||
@@ -42,12 +27,7 @@ type ConnectionState struct {
|
||||
// completed handshake.Result. It seeds messageCounter and the replay window so
|
||||
// that the post-handshake message indices already used on the wire don't count
|
||||
// as missed traffic in the data plane.
|
||||
func newConnectionStateFromResult(r *handshake.Result) (*ConnectionState, error) {
|
||||
// Refuse a MessageIndex too big for the replay window: it can only be a bug, and would spin the seed loop below.
|
||||
if r.MessageIndex >= ReplayWindow {
|
||||
return nil, fmt.Errorf("handshake message index %d exceeds replay window", r.MessageIndex)
|
||||
}
|
||||
|
||||
func newConnectionStateFromResult(r *handshake.Result) *ConnectionState {
|
||||
ci := &ConnectionState{
|
||||
myCert: r.MyCert,
|
||||
initiator: r.Initiator,
|
||||
@@ -60,7 +40,7 @@ func newConnectionStateFromResult(r *handshake.Result) (*ConnectionState, error)
|
||||
for i := uint64(1); i <= r.MessageIndex; i++ {
|
||||
ci.window.Update(nil, i)
|
||||
}
|
||||
return ci, nil
|
||||
return ci
|
||||
}
|
||||
|
||||
func (cs *ConnectionState) MarshalJSON() ([]byte, error) {
|
||||
@@ -71,65 +51,6 @@ func (cs *ConnectionState) MarshalJSON() ([]byte, error) {
|
||||
})
|
||||
}
|
||||
|
||||
// NextMessageCounter reserves the next 1-based counter; RejectAfterMessages is the first we refuse, pinned to not wrap.
|
||||
func (cs *ConnectionState) NextMessageCounter() (uint64, bool) {
|
||||
c := cs.messageCounter.Add(1)
|
||||
if c >= RejectAfterMessages {
|
||||
cs.messageCounter.Store(RejectAfterMessages)
|
||||
return c, false
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
|
||||
func (cs *ConnectionState) Curve() cert.Curve {
|
||||
return cs.myCert.Curve()
|
||||
}
|
||||
|
||||
func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, out []byte, packet []byte, nb []byte) ([]byte, error) {
|
||||
var err error
|
||||
cs.decryptLock.Lock()
|
||||
result := cs.window.Check(l, messageCounter)
|
||||
cs.decryptLock.Unlock()
|
||||
if !result {
|
||||
return nil, ErrAlreadySeen
|
||||
}
|
||||
|
||||
out, err = cs.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cs.decryptLock.Lock()
|
||||
result = cs.window.Update(l, messageCounter)
|
||||
cs.decryptLock.Unlock()
|
||||
if !result {
|
||||
return nil, ErrAlreadySeen
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// VerifyRelay verifies AEAD protected (but not encrypted) relay frames. packet must be length-checked by the caller.
|
||||
func (cs *ConnectionState) VerifyRelay(l *slog.Logger, messageCounter uint64, packet []byte, nb []byte) error {
|
||||
cs.decryptLock.Lock()
|
||||
result := cs.window.Check(l, messageCounter)
|
||||
cs.decryptLock.Unlock()
|
||||
if !result {
|
||||
return ErrAlreadySeen
|
||||
}
|
||||
|
||||
signedPayload := packet[:len(packet)-cs.dKey.Overhead()]
|
||||
signatureValue := packet[len(packet)-cs.dKey.Overhead():]
|
||||
_, err := cs.dKey.DecryptDanger(nil, signedPayload, signatureValue, messageCounter, nb)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cs.decryptLock.Lock()
|
||||
result = cs.window.Update(l, messageCounter)
|
||||
cs.decryptLock.Unlock()
|
||||
if !result {
|
||||
return ErrAlreadySeen
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -6,12 +6,10 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/flynn/noise"
|
||||
"github.com/rcrowley/go-metrics"
|
||||
"github.com/slackhq/nebula/cert"
|
||||
ct "github.com/slackhq/nebula/cert_test"
|
||||
"github.com/slackhq/nebula/handshake"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -81,51 +79,11 @@ func runTestHandshake(t *testing.T) (initR, respR *handshake.Result) {
|
||||
return initR, respR
|
||||
}
|
||||
|
||||
func TestConnectionState_NextMessageCounter(t *testing.T) {
|
||||
cs := &ConnectionState{}
|
||||
cs.messageCounter.Store(RejectAfterMessages - 2)
|
||||
|
||||
c, ok := cs.NextMessageCounter()
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, RejectAfterMessages-1, c)
|
||||
|
||||
// Hitting the limit refuses and pins the counter there
|
||||
c, ok = cs.NextMessageCounter()
|
||||
assert.False(t, ok)
|
||||
assert.Equal(t, RejectAfterMessages, c)
|
||||
assert.Equal(t, RejectAfterMessages, cs.messageCounter.Load())
|
||||
|
||||
// Continued send attempts stay refused and the counter never wraps
|
||||
for i := 0; i < 10; i++ {
|
||||
_, ok = cs.NextMessageCounter()
|
||||
assert.False(t, ok)
|
||||
}
|
||||
assert.Equal(t, RejectAfterMessages, cs.messageCounter.Load())
|
||||
}
|
||||
|
||||
// TestSendNoMetricsDropsExhausted drives the send path to the exhausted drop; metric and out flag prove it.
|
||||
func TestSendNoMetricsDropsExhausted(t *testing.T) {
|
||||
initR, _ := runTestHandshake(t)
|
||||
ci, err := newConnectionStateFromResult(initR)
|
||||
require.NoError(t, err)
|
||||
ci.messageCounter.Store(RejectAfterMessages - 1)
|
||||
|
||||
f := &Interface{l: test.NewLogger(), messageMetrics: &MessageMetrics{txExhausted: metrics.NewCounter()}}
|
||||
hostinfo := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.1")}, ConnectionState: ci}
|
||||
|
||||
f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, []byte{}, make([]byte, 12), make([]byte, mtu), 0)
|
||||
|
||||
// The crossing send is refused: it records an exhaustion drop and never reaches connectionManager.Out.
|
||||
assert.Equal(t, int64(1), f.messageMetrics.txExhausted.Count())
|
||||
assert.False(t, hostinfo.out.Load())
|
||||
}
|
||||
|
||||
func TestNewConnectionStateFromResult(t *testing.T) {
|
||||
initR, respR := runTestHandshake(t)
|
||||
|
||||
t.Run("initiator", func(t *testing.T) {
|
||||
ci, err := newConnectionStateFromResult(initR)
|
||||
require.NoError(t, err)
|
||||
ci := newConnectionStateFromResult(initR)
|
||||
assert.True(t, ci.initiator)
|
||||
assert.Equal(t, initR.MyCert, ci.myCert)
|
||||
assert.Equal(t, initR.RemoteCert, ci.peerCert)
|
||||
@@ -144,17 +102,8 @@ func TestNewConnectionStateFromResult(t *testing.T) {
|
||||
assert.True(t, ci.window.Check(nil, 3), "counter 3 must not be pre-seeded")
|
||||
})
|
||||
|
||||
t.Run("message index too large is refused", func(t *testing.T) {
|
||||
bad := *initR
|
||||
bad.MessageIndex = ReplayWindow
|
||||
ci, err := newConnectionStateFromResult(&bad)
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, ci)
|
||||
})
|
||||
|
||||
t.Run("responder", func(t *testing.T) {
|
||||
ci, err := newConnectionStateFromResult(respR)
|
||||
require.NoError(t, err)
|
||||
ci := newConnectionStateFromResult(respR)
|
||||
assert.False(t, ci.initiator)
|
||||
assert.Equal(t, respR.MyCert, ci.myCert)
|
||||
assert.Equal(t, respR.RemoteCert, ci.peerCert)
|
||||
|
||||
+1
-9
@@ -53,7 +53,6 @@ type Control struct {
|
||||
statsStart func()
|
||||
dnsStart func()
|
||||
lighthouseStart func()
|
||||
networkChangeStart func(rebind func())
|
||||
connectionManagerStart func(context.Context)
|
||||
}
|
||||
|
||||
@@ -105,9 +104,6 @@ func (c *Control) Start() error {
|
||||
if c.dnsStart != nil {
|
||||
go c.dnsStart()
|
||||
}
|
||||
if c.networkChangeStart != nil {
|
||||
go c.networkChangeStart(c.RebindUDPServer)
|
||||
}
|
||||
if c.connectionManagerStart != nil {
|
||||
go c.connectionManagerStart(c.ctx)
|
||||
}
|
||||
@@ -202,11 +198,7 @@ func (c *Control) RebindUDPServer() {
|
||||
return
|
||||
}
|
||||
|
||||
// A failure here means we are likely still pinned to the interface we came up on, so the rest of this is
|
||||
// unlikely to help. Say so instead of silently carrying on as if we rebound.
|
||||
if err := c.f.outside.Rebind(); err != nil {
|
||||
c.l.Error("Failed to rebind udp socket", "error", err)
|
||||
}
|
||||
_ = c.f.outside.Rebind()
|
||||
|
||||
// Trigger a lighthouse update, useful for mobile clients that should have an update interval of 0
|
||||
c.f.lightHouse.SendUpdate()
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/slackhq/nebula/udp"
|
||||
@@ -30,9 +31,9 @@ func newFakeDevice() *fakeDevice {
|
||||
|
||||
// Read blocks until Close like a real tun with no traffic, then reports EOF
|
||||
// the same way a closed device does
|
||||
func (d *fakeDevice) Read(p []byte) (int, error) {
|
||||
func (d *fakeDevice) Read() ([]tio.Packet, error) {
|
||||
<-d.closedCh
|
||||
return 0, io.EOF
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
func (d *fakeDevice) Write(p []byte) (int, error) { return len(p), nil }
|
||||
@@ -49,10 +50,8 @@ func (d *fakeDevice) Activate() error { return nil }
|
||||
func (d *fakeDevice) Networks() []netip.Prefix { return nil }
|
||||
func (d *fakeDevice) Name() string { return "fake" }
|
||||
func (d *fakeDevice) RoutesFor(netip.Addr) routing.Gateways { return nil }
|
||||
func (d *fakeDevice) SupportsMultiqueue() bool { return false }
|
||||
func (d *fakeDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return nil, errors.New("unsupported")
|
||||
}
|
||||
|
||||
func (d *fakeDevice) Queues(int) ([]tio.Queue, error) { return []tio.Queue{d}, nil }
|
||||
|
||||
// newReadyControl hand-builds the minimum Control that Main would have
|
||||
// produced right before Start, including the construction token NewInterface
|
||||
@@ -78,7 +77,6 @@ func newReadyControl(t *testing.T) (*Control, *fakeDevice, *fakeConn) {
|
||||
inside: dev,
|
||||
outside: conn,
|
||||
writers: []udp.Conn{conn},
|
||||
readers: make([]io.ReadWriteCloser, 1),
|
||||
routines: 1,
|
||||
hostMap: newHostMap(l),
|
||||
lightHouse: lh,
|
||||
@@ -155,7 +153,14 @@ type multiqueueDevice struct {
|
||||
*fakeDevice
|
||||
}
|
||||
|
||||
func (d *multiqueueDevice) SupportsMultiqueue() bool { return true }
|
||||
// Queues claims multiqueue support but fails to open the second queue,
|
||||
// exercising the activation error path.
|
||||
func (d *multiqueueDevice) Queues(n int) ([]tio.Queue, error) {
|
||||
if n > 1 {
|
||||
return nil, errors.New("second queue failed to open")
|
||||
}
|
||||
return d.fakeDevice.Queues(n)
|
||||
}
|
||||
|
||||
func TestControl_StartMultiqueueFailureReleases(t *testing.T) {
|
||||
dev := &multiqueueDevice{fakeDevice: newFakeDevice()}
|
||||
@@ -166,7 +171,6 @@ func TestControl_StartMultiqueueFailureReleases(t *testing.T) {
|
||||
inside: dev,
|
||||
outside: conn,
|
||||
writers: []udp.Conn{conn},
|
||||
readers: make([]io.ReadWriteCloser, 2),
|
||||
routines: 2,
|
||||
l: test.NewLogger(),
|
||||
}
|
||||
|
||||
+1
-13
@@ -108,19 +108,7 @@ func (c *Control) GetVpnAddrs() []netip.Addr {
|
||||
}
|
||||
|
||||
func (c *Control) GetUDPAddr() netip.AddrPort {
|
||||
return c.f.outside.(*udp.TesterConn).GetAddr()
|
||||
}
|
||||
|
||||
// SetUDPAddr moves this node to a new underlay address, standing in for a laptop waking up on a different
|
||||
// network. Register the new address with the router as well or nothing will route back.
|
||||
func (c *Control) SetUDPAddr(addr netip.AddrPort) {
|
||||
c.f.outside.(*udp.TesterConn).SetAddr(addr)
|
||||
}
|
||||
|
||||
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
|
||||
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
|
||||
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
|
||||
c.f.lightHouse.localAddrsFn = fn
|
||||
return c.f.outside.(*udp.TesterConn).Addr
|
||||
}
|
||||
|
||||
func (c *Control) KillPendingTunnel(vpnIp netip.Addr) bool {
|
||||
|
||||
+7
-16
@@ -97,7 +97,8 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
|
||||
newAddr := getDnsServerAddr(c)
|
||||
|
||||
d.serverMu.Lock()
|
||||
running := d.server != nil
|
||||
running := d.server
|
||||
runningStarted := d.started
|
||||
sameAddr := d.addr == newAddr
|
||||
d.addr = newAddr
|
||||
d.enabled.Store(enabled)
|
||||
@@ -111,7 +112,7 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
|
||||
}
|
||||
|
||||
if !enabled {
|
||||
if running {
|
||||
if running != nil {
|
||||
d.Stop()
|
||||
}
|
||||
// Drop any records that accumulated while enabled; a later re-enable
|
||||
@@ -120,12 +121,12 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !running {
|
||||
if running == nil {
|
||||
// Was disabled (or never started); bring it up now.
|
||||
go d.Start()
|
||||
} else if !sameAddr {
|
||||
// Stop clears the slot before shutting down, otherwise the Start below can find the dying server and refuse
|
||||
d.Stop()
|
||||
d.shutdownServer(running, runningStarted, "reload")
|
||||
// Old Start goroutine has now exited; bring up a fresh listener on the new address.
|
||||
go d.Start()
|
||||
}
|
||||
|
||||
@@ -161,9 +162,7 @@ func (d *dnsServer) Start() {
|
||||
|
||||
started := make(chan struct{})
|
||||
d.serverMu.Lock()
|
||||
// Re-check enabled under the lock, a disable that raced our check above snapshots the slot under it too.
|
||||
// Two reloads in quick succession can both spawn a Start, the loser would orphan the live listener past Stop
|
||||
if d.ctx.Err() != nil || d.server != nil || !d.enabled.Load() {
|
||||
if d.ctx.Err() != nil {
|
||||
d.serverMu.Unlock()
|
||||
return
|
||||
}
|
||||
@@ -201,14 +200,6 @@ func (d *dnsServer) Start() {
|
||||
close(started)
|
||||
}
|
||||
|
||||
// Release our slot, unless a reload already replaced us, so a dead listener can't block a future Start
|
||||
d.serverMu.Lock()
|
||||
if d.server == server {
|
||||
d.server = nil
|
||||
d.started = nil
|
||||
}
|
||||
d.serverMu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
d.l.Warn("Failed to run the DNS responder", "error", err)
|
||||
}
|
||||
|
||||
+4
-206
@@ -194,51 +194,14 @@ func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDnsServer_reload_sameAddr_noOp(t *testing.T) {
|
||||
port := freeUDPPort(t)
|
||||
ds, c := newTestDnsServer(t)
|
||||
setDnsConfig(c, "127.0.0.1", port, true, true)
|
||||
setDnsConfig(c, "127.0.0.1", "0", true, true)
|
||||
|
||||
require.NoError(t, ds.reload(c, true))
|
||||
|
||||
go ds.Start()
|
||||
waitForBind(t, ds)
|
||||
|
||||
ds.serverMu.Lock()
|
||||
before := ds.server
|
||||
ds.serverMu.Unlock()
|
||||
require.NotNil(t, before)
|
||||
|
||||
// Same address, so the running listener must be left alone rather than rebuilt under live queries
|
||||
// No server running yet, no addr change. Reload should not spawn anything.
|
||||
require.NoError(t, ds.reload(c, false))
|
||||
assert.True(t, ds.enabled.Load())
|
||||
|
||||
ds.serverMu.Lock()
|
||||
after := ds.server
|
||||
ds.serverMu.Unlock()
|
||||
assert.Same(t, before, after, "a same-address reload must not restart the listener")
|
||||
|
||||
ds.Stop()
|
||||
}
|
||||
|
||||
// The branch the old sameAddr test was accidentally hitting: enabled with nothing running means reload starts it.
|
||||
func TestDnsServer_reload_whenNotRunning_starts(t *testing.T) {
|
||||
port := freeUDPPort(t)
|
||||
ds, c := newTestDnsServer(t)
|
||||
setDnsConfig(c, "127.0.0.1", port, true, true)
|
||||
|
||||
// initial only records config, it never starts anything
|
||||
require.NoError(t, ds.reload(c, true))
|
||||
ds.serverMu.Lock()
|
||||
assert.Nil(t, ds.server, "the initial reload must not start a listener")
|
||||
ds.serverMu.Unlock()
|
||||
|
||||
require.NoError(t, ds.reload(c, false))
|
||||
waitForBind(t, ds)
|
||||
|
||||
ds.serverMu.Lock()
|
||||
assert.NotNil(t, ds.server, "a reload with nothing running should bring DNS up")
|
||||
ds.serverMu.Unlock()
|
||||
|
||||
ds.Stop()
|
||||
assert.Nil(t, ds.server)
|
||||
}
|
||||
|
||||
func TestDnsServer_StartStop_lifecycle(t *testing.T) {
|
||||
@@ -464,168 +427,3 @@ func waitFor(t *testing.T, cond func() bool) {
|
||||
}
|
||||
t.Fatal("timed out waiting for condition")
|
||||
}
|
||||
|
||||
// Two reloads in quick succession, or a HUP before Control.Start, can race two Starts at the same listener.
|
||||
func TestDnsServer_Start_isIdempotent(t *testing.T) {
|
||||
port := freeUDPPort(t)
|
||||
ds, c := newTestDnsServer(t)
|
||||
setDnsConfig(c, "127.0.0.1", port, true, true)
|
||||
require.NoError(t, ds.reload(c, true))
|
||||
|
||||
go ds.Start()
|
||||
waitForBind(t, ds)
|
||||
|
||||
ds.serverMu.Lock()
|
||||
first := ds.server
|
||||
ds.serverMu.Unlock()
|
||||
require.NotNil(t, first)
|
||||
|
||||
// If the second Start replaces the tracked server, Stop kills the wrong one and the port leaks
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
ds.Start()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("second Start never returned")
|
||||
}
|
||||
|
||||
ds.serverMu.Lock()
|
||||
second := ds.server
|
||||
ds.serverMu.Unlock()
|
||||
assert.Same(t, first, second, "a second Start must not replace the running server")
|
||||
|
||||
// The real proof, after Stop the port must actually be free
|
||||
ds.Stop()
|
||||
waitFor(t, func() bool {
|
||||
pc, err := net.ListenPacket("udp", "127.0.0.1:"+port)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = pc.Close()
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
// An address change must actually end up listening on the new port. Start's guard refuses when a server is already
|
||||
// installed, so reload has to clear the slot before shutting the old one down.
|
||||
func TestDnsServer_reload_addrChange_restarts(t *testing.T) {
|
||||
first := freeUDPPort(t)
|
||||
second := freeUDPPort(t)
|
||||
|
||||
ds, c := newTestDnsServer(t)
|
||||
setDnsConfig(c, "127.0.0.1", first, true, true)
|
||||
require.NoError(t, ds.reload(c, true))
|
||||
|
||||
go ds.Start()
|
||||
waitForBind(t, ds)
|
||||
|
||||
// Cycle a few times, the failure this guards against depends on which goroutine wins serverMu
|
||||
for i := range 8 {
|
||||
want := second
|
||||
if i%2 == 1 {
|
||||
want = first
|
||||
}
|
||||
setDnsConfig(c, "127.0.0.1", want, true, true)
|
||||
require.NoError(t, ds.reload(c, false))
|
||||
waitForBind(t, ds)
|
||||
|
||||
ds.serverMu.Lock()
|
||||
srv := ds.server
|
||||
ds.serverMu.Unlock()
|
||||
require.NotNil(t, srv, "reload left DNS down instead of restarting it")
|
||||
require.Equal(t, "127.0.0.1:"+want, srv.Addr, "reload should be serving the new address")
|
||||
}
|
||||
|
||||
// Land back on second so the port assertions below are meaningful
|
||||
setDnsConfig(c, "127.0.0.1", second, true, true)
|
||||
require.NoError(t, ds.reload(c, false))
|
||||
waitForBind(t, ds)
|
||||
|
||||
// The old port must be released and the new one actually held
|
||||
waitFor(t, func() bool {
|
||||
pc, err := net.ListenPacket("udp", "127.0.0.1:"+first)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = pc.Close()
|
||||
return true
|
||||
})
|
||||
_, err := net.ListenPacket("udp", "127.0.0.1:"+second)
|
||||
require.Error(t, err, "the new address should be bound by the DNS responder")
|
||||
|
||||
ds.Stop()
|
||||
}
|
||||
|
||||
// A listener that dies on its own must release the slot, or a later same-addr reload sees it as running and no-ops.
|
||||
func TestDnsServer_Start_bindFailure_releasesSlot(t *testing.T) {
|
||||
port := freeUDPPort(t)
|
||||
blocker, err := net.ListenPacket("udp", "127.0.0.1:"+port)
|
||||
require.NoError(t, err)
|
||||
|
||||
ds, c := newTestDnsServer(t)
|
||||
setDnsConfig(c, "127.0.0.1", port, true, true)
|
||||
require.NoError(t, ds.reload(c, true))
|
||||
|
||||
ds.Start() // returns once the bind fails
|
||||
|
||||
ds.serverMu.Lock()
|
||||
assert.Nil(t, ds.server, "a listener that failed to bind must not stay parked in the slot")
|
||||
ds.serverMu.Unlock()
|
||||
|
||||
// With the slot released, a reload can retry once the port frees up
|
||||
require.NoError(t, blocker.Close())
|
||||
require.NoError(t, ds.reload(c, false))
|
||||
waitForBind(t, ds)
|
||||
|
||||
ds.serverMu.Lock()
|
||||
assert.NotNil(t, ds.server, "a same-addr reload should retry after a failed bind")
|
||||
ds.serverMu.Unlock()
|
||||
|
||||
ds.Stop()
|
||||
}
|
||||
|
||||
// A disable that lands while Start is between its unlocked check and the guard must not leave a listener behind.
|
||||
func TestDnsServer_Start_refusesWhenDisabledUnderLock(t *testing.T) {
|
||||
port := freeUDPPort(t)
|
||||
ds, c := newTestDnsServer(t)
|
||||
setDnsConfig(c, "127.0.0.1", port, true, true)
|
||||
require.NoError(t, ds.reload(c, true))
|
||||
require.True(t, ds.enabled.Load())
|
||||
|
||||
// Holding serverMu parks Start on the lock, the only way to land the disable in that window on purpose
|
||||
ds.serverMu.Lock()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
ds.Start()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
ds.serverMu.Unlock()
|
||||
t.Fatal("Start returned early, the test never exercised the window")
|
||||
case <-time.After(time.Millisecond * 100):
|
||||
}
|
||||
|
||||
// The disable reload's critical section. It sees nothing running, so it never calls Stop.
|
||||
ds.enabled.Store(false)
|
||||
ds.serverMu.Unlock()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("Start never returned")
|
||||
}
|
||||
|
||||
ds.serverMu.Lock()
|
||||
assert.Nil(t, ds.server, "Start must not install a listener a disable already cancelled")
|
||||
ds.serverMu.Unlock()
|
||||
|
||||
pc, err := net.ListenPacket("udp", "127.0.0.1:"+port)
|
||||
require.NoError(t, err, "an orphaned listener is still holding the port")
|
||||
_ = pc.Close()
|
||||
}
|
||||
|
||||
@@ -725,70 +725,6 @@ func TestReestablishRelays(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestRelayHandshakeOverDisestablishedEntry(t *testing.T) {
|
||||
t.Parallel()
|
||||
// If them tears down the tunnel while me keeps Established relay state, me's next
|
||||
// handshake flows through the relay with no fresh CreateRelayRequest and lands on
|
||||
// them's Disestablished terminal relay entry. them must re-establish that entry, or
|
||||
// its first transmit deletes its only relay and the tunnel is born transmit-dead:
|
||||
// them can receive but every send is silently dropped.
|
||||
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||
myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
|
||||
relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
|
||||
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
|
||||
|
||||
// Teach my how to get to the relay and that their can be reached via the relay
|
||||
myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
|
||||
myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
|
||||
relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
|
||||
|
||||
// Build a router so we don't have to reason who gets which packet
|
||||
r := router.NewR(t, myControl, relayControl, theirControl)
|
||||
defer r.RenderFlow()
|
||||
|
||||
// Start the servers
|
||||
myControl.Start()
|
||||
relayControl.Start()
|
||||
theirControl.Start()
|
||||
|
||||
t.Log("Trigger a handshake from me to them via the relay")
|
||||
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
|
||||
|
||||
p := r.RouteForAllUntilTxTun(theirControl)
|
||||
assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
|
||||
oldIdx := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false).LocalIndex
|
||||
|
||||
t.Log("Close the tunnel on them only, marking their relay entry Disestablished")
|
||||
theirControl.CloseTunnel(myVpnIpNet[0].Addr(), true)
|
||||
|
||||
t.Log("Re-handshake from me, riding the still-Established relay state")
|
||||
myControl.ReHandshake(theirVpnIpNet[0].Addr())
|
||||
for {
|
||||
h := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
|
||||
if h != nil && h.LocalIndex != oldIdx && h.RemoteIndex != 0 {
|
||||
break
|
||||
}
|
||||
r.RouteForAllExitFunc(func(*udp.Packet, *nebula.Control) router.ExitType {
|
||||
return router.RouteAndExit
|
||||
})
|
||||
}
|
||||
|
||||
hAtThem := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
|
||||
require.NotNil(t, hAtThem, "them should have completed the relayed handshake")
|
||||
require.Equal(t, []netip.Addr{relayVpnIpNet[0].Addr()}, hAtThem.CurrentRelaysToMe, "them should know a relay for the new tunnel")
|
||||
|
||||
t.Log("Send from them to me; their only relay entry must survive the transmit")
|
||||
theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
|
||||
require.Never(t, func() bool {
|
||||
h := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
|
||||
return h == nil || len(h.CurrentRelaysToMe) == 0
|
||||
}, time.Second, 10*time.Millisecond, "them deleted its only relay entry; the tunnel is permanently transmit-dead")
|
||||
|
||||
p = r.RouteForAllUntilTxTun(myControl)
|
||||
assertUdpPacket(t, []byte("Hi from them"), p, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
|
||||
r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
|
||||
}
|
||||
|
||||
func TestStage1RaceRelays(t *testing.T) {
|
||||
t.Parallel()
|
||||
//NOTE: this is a race between me and relay resulting in a full tunnel from me to them via relay
|
||||
|
||||
@@ -1,225 +0,0 @@
|
||||
//go:build e2e_testing
|
||||
// +build e2e_testing
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/slackhq/nebula"
|
||||
"github.com/slackhq/nebula/cert"
|
||||
"github.com/slackhq/nebula/cert_test"
|
||||
"github.com/slackhq/nebula/e2e/router"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/udp"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// reportedAddrs is what the lighthouse would hand a peer asking where vpnAddr is.
|
||||
func reportedAddrs(t *testing.T, lh *nebula.Control, vpnAddr netip.Addr) []netip.AddrPort {
|
||||
t.Helper()
|
||||
cm := lh.QueryLighthouse(vpnAddr)
|
||||
if cm == nil {
|
||||
return nil
|
||||
}
|
||||
var out []netip.AddrPort
|
||||
for _, c := range *cm {
|
||||
out = append(out, c.Reported...)
|
||||
out = append(out, c.Learned...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// waitForLighthouseMsg routes until a lighthouse message lands on lh, or gives up. Reports whether one arrived.
|
||||
func waitForLighthouseMsg(t *testing.T, r *router.R, lh *nebula.Control, wait time.Duration) bool {
|
||||
t.Helper()
|
||||
h := &header.H{}
|
||||
return r.RouteForAllExitFuncOrTimeout(wait, func(p *udp.Packet, c *nebula.Control) router.ExitType {
|
||||
if c != lh {
|
||||
return router.KeepRouting
|
||||
}
|
||||
// Punches are a single byte and never parse, they are just not what we are after
|
||||
if err := h.Parse(p.Data); err != nil {
|
||||
return router.KeepRouting
|
||||
}
|
||||
if h.Type == header.LightHouse {
|
||||
return router.RouteAndExit
|
||||
}
|
||||
return router.KeepRouting
|
||||
})
|
||||
}
|
||||
|
||||
// A laptop that changes networks has to tell the lighthouse promptly, otherwise the lighthouse keeps handing peers
|
||||
// the old address and their punches land nowhere. On a long lighthouse interval the only thing that closes that
|
||||
// window is the rebind, which on darwin the network change monitor drives. The e2e build compiles the monitor out,
|
||||
// so we call RebindUDPServer directly, which is the same thing the monitor does.
|
||||
func TestRebindSendsLighthouseUpdate(t *testing.T) {
|
||||
t.Parallel()
|
||||
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||
|
||||
lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
|
||||
"lighthouse": m{"am_lighthouse": true},
|
||||
})
|
||||
|
||||
// 600s interval, so nothing scheduled can send an update during this test. A rebind is the only thing that can.
|
||||
myControl, _, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
|
||||
"lighthouse": m{
|
||||
"hosts": []any{lhVpnIpNet[0].Addr().String()},
|
||||
"interval": 600,
|
||||
},
|
||||
"static_host_map": m{
|
||||
lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
|
||||
},
|
||||
})
|
||||
|
||||
r := router.NewR(t, lhControl, myControl)
|
||||
defer r.RenderFlow()
|
||||
|
||||
lhControl.Start()
|
||||
myControl.Start()
|
||||
|
||||
// Let the startup registration finish, then clear everything it left behind
|
||||
require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
|
||||
r.RouteFor(time.Millisecond * 400)
|
||||
|
||||
// Nothing should be talking to the lighthouse on its own now
|
||||
require.False(t, waitForLighthouseMsg(t, r, lhControl, time.Millisecond*200),
|
||||
"nothing should reach the lighthouse before the rebind")
|
||||
|
||||
myControl.RebindUDPServer()
|
||||
|
||||
assert.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5),
|
||||
"a rebind should push an update to the lighthouse rather than waiting out the interval")
|
||||
|
||||
lhControl.Stop()
|
||||
myControl.Stop()
|
||||
}
|
||||
|
||||
// The other half of a rebind: every live tunnel requeries the lighthouse on its next send. That query is what makes
|
||||
// the lighthouse tell the peer to punch toward our new address, which is the part that actually revives a tunnel
|
||||
// whose remote NAT state died while we were on a different network.
|
||||
func TestRebindRequeriesPeersOnNextSend(t *testing.T) {
|
||||
t.Parallel()
|
||||
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||
|
||||
lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
|
||||
"lighthouse": m{"am_lighthouse": true},
|
||||
})
|
||||
|
||||
lhCfg := m{
|
||||
"lighthouse": m{
|
||||
"hosts": []any{lhVpnIpNet[0].Addr().String()},
|
||||
"interval": 600,
|
||||
// Without this the peers advertise this machine's real addresses and then try to punch at them,
|
||||
// which the router has no route for.
|
||||
"local_allow_list": m{
|
||||
"10.0.0.0/24": true,
|
||||
"::/0": false,
|
||||
},
|
||||
},
|
||||
"static_host_map": m{
|
||||
lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
|
||||
},
|
||||
}
|
||||
|
||||
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", lhCfg)
|
||||
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", lhCfg)
|
||||
|
||||
r := router.NewR(t, lhControl, myControl, theirControl)
|
||||
defer r.RenderFlow()
|
||||
|
||||
lhControl.Start()
|
||||
myControl.Start()
|
||||
theirControl.Start()
|
||||
r.RouteFor(time.Millisecond * 500)
|
||||
|
||||
// Point the peers at each other directly, this test is about the rebind and not about lighthouse discovery
|
||||
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
|
||||
theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
|
||||
|
||||
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("initial")))
|
||||
r.RouteFor(time.Second)
|
||||
require.NotNil(t, myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false), "expected a tunnel to them")
|
||||
r.RouteFor(time.Millisecond * 300)
|
||||
|
||||
// Assert on what the peer sees rather than on lighthouse traffic. A query for them makes the lighthouse send
|
||||
// them a punch notification, which is the whole point. Our own update to the lighthouse sends them nothing,
|
||||
// so this cannot be satisfied by the update the rebind itself pushes.
|
||||
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("quiet")))
|
||||
require.False(t, waitForLighthouseMsg(t, r, theirControl, time.Millisecond*300),
|
||||
"an ordinary send should not requery the lighthouse")
|
||||
|
||||
myControl.RebindUDPServer()
|
||||
r.RouteFor(time.Millisecond * 300) // let the update the rebind itself sends pass by
|
||||
|
||||
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("after rebind")))
|
||||
assert.True(t, waitForLighthouseMsg(t, r, theirControl, time.Second*5),
|
||||
"the first send after a rebind should requery the lighthouse, which then tells the peer to punch at us")
|
||||
|
||||
lhControl.Stop()
|
||||
myControl.Stop()
|
||||
theirControl.Stop()
|
||||
}
|
||||
|
||||
// The scenario this whole thing exists for: a laptop sleeps at the office and wakes up at home on a new address.
|
||||
// Until it tells the lighthouse, the lighthouse keeps handing peers the office address, so their punches land
|
||||
// nowhere and the tunnel stays dead. On a long interval the rebind is the only thing that closes that window.
|
||||
func TestRebindAdvertisesNewAddressAfterMove(t *testing.T) {
|
||||
t.Parallel()
|
||||
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||
|
||||
lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
|
||||
"lighthouse": m{"am_lighthouse": true},
|
||||
})
|
||||
|
||||
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
|
||||
"lighthouse": m{
|
||||
"hosts": []any{lhVpnIpNet[0].Addr().String()},
|
||||
"interval": 600,
|
||||
},
|
||||
"static_host_map": m{
|
||||
lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
|
||||
},
|
||||
})
|
||||
|
||||
// Advertise wherever we currently are rather than this machine's real NICs, read fresh each time so a move
|
||||
// is picked up.
|
||||
myControl.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
|
||||
return []netip.Addr{myControl.GetUDPAddr().Addr()}
|
||||
})
|
||||
|
||||
r := router.NewR(t, lhControl, myControl)
|
||||
defer r.RenderFlow()
|
||||
|
||||
lhControl.Start()
|
||||
myControl.Start()
|
||||
|
||||
require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
|
||||
r.RouteFor(time.Millisecond * 400)
|
||||
|
||||
require.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), myUdpAddr,
|
||||
"the lighthouse should know the address we started on")
|
||||
|
||||
// Wake up somewhere else
|
||||
newAddr := netip.MustParseAddrPort("10.0.0.99:4242")
|
||||
myControl.SetUDPAddr(newAddr)
|
||||
r.AddRoute(newAddr.Addr(), newAddr.Port(), myControl)
|
||||
|
||||
// Nothing has told the lighthouse, and with interval 600 nothing scheduled will
|
||||
r.RouteFor(time.Millisecond * 400)
|
||||
require.NotContains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
|
||||
"the lighthouse should still be handing out the old address before the rebind")
|
||||
|
||||
myControl.RebindUDPServer()
|
||||
require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an update after the rebind")
|
||||
r.RouteFor(time.Millisecond * 400)
|
||||
|
||||
assert.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
|
||||
"after the rebind the lighthouse should hand peers our new address")
|
||||
|
||||
lhControl.Stop()
|
||||
myControl.Stop()
|
||||
}
|
||||
+28
-130
@@ -114,28 +114,6 @@ type packet struct {
|
||||
packet *udp.Packet
|
||||
tun bool // a packet pulled off a tun device
|
||||
rx bool // the packet was received by a udp device
|
||||
|
||||
// h is the nebula header, parsed once when the packet is recorded. parseErr says why there isn't one, which
|
||||
// the flow log reports rather than hiding. Punchy sends a single byte, so an unparseable packet is normal.
|
||||
h header.H
|
||||
parseErr error
|
||||
}
|
||||
|
||||
// fromAddr and toAddr are the addresses this packet actually travelled between. Reading them off the control
|
||||
// instead would misreport the whole history once a test moves a node. Tun packets are synthesized without
|
||||
// addresses, so they fall back to the control.
|
||||
func (p *packet) fromAddr() netip.AddrPort {
|
||||
if p.tun || !p.packet.From.IsValid() {
|
||||
return p.from.GetUDPAddr()
|
||||
}
|
||||
return p.packet.From
|
||||
}
|
||||
|
||||
func (p *packet) toAddr() netip.AddrPort {
|
||||
if p.tun || !p.packet.To.IsValid() {
|
||||
return p.to.GetUDPAddr()
|
||||
}
|
||||
return p.packet.To
|
||||
}
|
||||
|
||||
func (p *packet) WasReceived() {
|
||||
@@ -271,7 +249,7 @@ func (r *R) renderFlow() {
|
||||
continue
|
||||
}
|
||||
|
||||
addr := e.packet.fromAddr()
|
||||
addr := e.packet.from.GetUDPAddr()
|
||||
if _, ok := participants[addr]; ok {
|
||||
continue
|
||||
}
|
||||
@@ -290,6 +268,7 @@ func (r *R) renderFlow() {
|
||||
}
|
||||
|
||||
// Print packets
|
||||
h := &header.H{}
|
||||
for _, e := range r.flow {
|
||||
if e.packet == nil {
|
||||
//fmt.Fprintf(f, " note over %s: %s\n", strings.Join(participantsVals, ", "), e.note)
|
||||
@@ -301,22 +280,21 @@ func (r *R) renderFlow() {
|
||||
fmt.Fprintln(f, r.formatUdpPacket(p))
|
||||
|
||||
} else {
|
||||
if err := h.Parse(p.packet.Data); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
line := "--x"
|
||||
if p.rx {
|
||||
line = "->>"
|
||||
}
|
||||
|
||||
detail := fmt.Sprintf("%s(%s), index %v, counter: %v",
|
||||
p.h.TypeName(), p.h.SubTypeName(), p.h.RemoteIndex, p.h.MessageCounter)
|
||||
if p.parseErr != nil {
|
||||
detail = fmt.Sprintf("unparsed, %v (%d bytes)", p.parseErr, len(p.packet.Data))
|
||||
}
|
||||
|
||||
fmt.Fprintf(f, " %s%s%s: %s\n",
|
||||
normalizeName(p.fromAddr().String()),
|
||||
fmt.Fprintf(f,
|
||||
" %s%s%s: %s(%s), index %v, counter: %v\n",
|
||||
normalizeName(p.from.GetUDPAddr().String()),
|
||||
line,
|
||||
normalizeName(p.toAddr().String()),
|
||||
detail,
|
||||
normalizeName(p.to.GetUDPAddr().String()),
|
||||
h.TypeName(), h.SubTypeName(), h.RemoteIndex, h.MessageCounter,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -430,34 +408,29 @@ func (r *R) unlockedInjectFlow(from, to *nebula.Control, p *udp.Packet, tun bool
|
||||
|
||||
r.renderHostmaps(fmt.Sprintf("Packet %v", len(r.flow)))
|
||||
|
||||
var h header.H
|
||||
var parseErr error
|
||||
if !tun {
|
||||
parseErr = h.Parse(p.Data)
|
||||
}
|
||||
|
||||
// Decide before copying, the copy comes from a freelist and an ignored packet would never be released
|
||||
for _, i := range r.ignoreFlows {
|
||||
if tun {
|
||||
if i.tun.HasValue && i.tun.IsTrue {
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
if len(r.ignoreFlows) > 0 {
|
||||
var h header.H
|
||||
err := h.Parse(p.Data)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// A packet we could not parse has no type to match against, so no rule can ignore it
|
||||
if parseErr == nil && i.messageType == h.Type && i.subType == h.Subtype {
|
||||
return nil
|
||||
for _, i := range r.ignoreFlows {
|
||||
if !tun {
|
||||
if i.messageType == h.Type && i.subType == h.Subtype {
|
||||
return nil
|
||||
}
|
||||
} else if i.tun.HasValue && i.tun.IsTrue {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fp := &packet{
|
||||
from: from,
|
||||
to: to,
|
||||
packet: p.Copy(),
|
||||
tun: tun,
|
||||
h: h,
|
||||
parseErr: parseErr,
|
||||
from: from,
|
||||
to: to,
|
||||
packet: p.Copy(),
|
||||
tun: tun,
|
||||
}
|
||||
|
||||
r.flow = append(r.flow, flowEntry{packet: fp})
|
||||
@@ -717,81 +690,6 @@ func (r *R) RouteUntilAfterMsgType(sender *nebula.Control, msgType header.Messag
|
||||
})
|
||||
}
|
||||
|
||||
// RouteFor routes everything that shows up for the given duration and then returns. Use it to let a test settle
|
||||
// deterministically rather than sleeping and hoping: a single FlushAll races a completing handshake, which queues
|
||||
// more packets right behind it.
|
||||
func (r *R) RouteFor(d time.Duration) {
|
||||
r.RouteForAllExitFuncOrTimeout(d, func(*udp.Packet, *nebula.Control) ExitType {
|
||||
return KeepRouting
|
||||
})
|
||||
}
|
||||
|
||||
// RouteForAllExitFuncOrTimeout is RouteForAllExitFunc with a deadline, reporting whether whatDo asked to exit
|
||||
// before time ran out. The unbounded version blocks forever on a quiet network, so this is what a test needs to
|
||||
// assert that something does NOT happen, or to route for a fixed settling period.
|
||||
func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc) bool {
|
||||
sc := make([]reflect.SelectCase, 0, len(r.controls)+1)
|
||||
cm := make([]*nebula.Control, 0, len(r.controls))
|
||||
|
||||
for _, c := range r.controls {
|
||||
sc = append(sc, reflect.SelectCase{
|
||||
Dir: reflect.SelectRecv,
|
||||
Chan: reflect.ValueOf(c.GetUDPTxChan()),
|
||||
Send: reflect.Value{},
|
||||
})
|
||||
cm = append(cm, c)
|
||||
}
|
||||
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
sc = append(sc, reflect.SelectCase{
|
||||
Dir: reflect.SelectRecv,
|
||||
Chan: reflect.ValueOf(timer.C),
|
||||
Send: reflect.Value{},
|
||||
})
|
||||
|
||||
for {
|
||||
x, rx, _ := reflect.Select(sc)
|
||||
if x == len(cm) {
|
||||
return false
|
||||
}
|
||||
|
||||
r.Lock()
|
||||
p := rx.Interface().(*udp.Packet)
|
||||
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
|
||||
if receiver == nil {
|
||||
r.Unlock()
|
||||
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
|
||||
}
|
||||
|
||||
e := whatDo(p, receiver)
|
||||
switch e {
|
||||
case ExitNow:
|
||||
r.Unlock()
|
||||
p.Release()
|
||||
return true
|
||||
|
||||
case RouteAndExit:
|
||||
fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
|
||||
receiver.InjectUDPPacket(p)
|
||||
fp.WasReceived()
|
||||
r.Unlock()
|
||||
p.Release()
|
||||
return true
|
||||
|
||||
case KeepRouting:
|
||||
fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
|
||||
receiver.InjectUDPPacket(p)
|
||||
fp.WasReceived()
|
||||
|
||||
default:
|
||||
panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
|
||||
}
|
||||
r.Unlock()
|
||||
p.Release()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *R) RouteForAllUntilAfterMsgTypeTo(receiver *nebula.Control, msgType header.MessageType, subType header.MessageSubType) {
|
||||
h := &header.H{}
|
||||
r.RouteForAllExitFunc(func(p *udp.Packet, r *nebula.Control) ExitType {
|
||||
|
||||
+14
-8
@@ -146,14 +146,6 @@ listen:
|
||||
# Default true; set to false to leave WDF in charge of inbound decisions on the listener port. Not reloadable.
|
||||
#windows_bypass_wdf: true
|
||||
|
||||
# On macOS only
|
||||
# macOS scopes the udp socket to the interface it was created on, so moving between networks (wifi to wired,
|
||||
# office to home) leaves Nebula sending out an interface that no longer has a route. When true, Nebula watches
|
||||
# the routing socket and rebinds the listener once the change settles.
|
||||
# iOS does not use this, the host app drives the same rebind itself.
|
||||
# Default true. Not reloadable.
|
||||
#rebind_on_network_change: true
|
||||
|
||||
# By default, Nebula replies to packets it has no tunnel for with a "recv_error" packet. This packet helps speed up reconnection
|
||||
# in the case that Nebula on either side did not shut down cleanly. This response can be abused as a way to discover if Nebula is running
|
||||
# on a host though. This option lets you configure if you want to send "recv_error" packets always, never, or only to private network remotes.
|
||||
@@ -262,6 +254,20 @@ tun:
|
||||
# Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
|
||||
mtu: 1300
|
||||
|
||||
# Linux only. pin_threads pins each tun reader/encrypt OS thread to a single CPU. This keeps every goroutine's
|
||||
# sends flowing through one XPS-selected NIC TX ring, so packets within a flow stay ordered on the wire
|
||||
# instead of being sprayed across multiple TX rings and reordered. Not reloadable.
|
||||
#pin_threads: true
|
||||
|
||||
# Linux only. cpu_affinity overrides which CPUs the tun reader threads pin to: a list of CPU IDs, one per routine
|
||||
# (see the top-level `routines` setting). Lists shorter than `routines` are modulo-cycled across the queues; extra
|
||||
# entries are ignored. IDs must be within the process's allowed CPU set, so this respects taskset / cgroup cpusets;
|
||||
# a non-integer or not-allowed entry disables the override and falls back to spreading queues across the allowed
|
||||
# CPUs. Only meaningful while pin_threads is true. Not reloadable.
|
||||
#cpu_affinity:
|
||||
# - 2
|
||||
# - 4
|
||||
|
||||
# Route based MTU overrides, you have known vpn ip paths that can support larger MTUs you can increase/decrease them here
|
||||
routes:
|
||||
#- mtu: 8800
|
||||
|
||||
@@ -8,15 +8,6 @@ Before=sshd.service
|
||||
Type=notify
|
||||
NotifyAccess=main
|
||||
SyslogIdentifier=nebula
|
||||
|
||||
# Uncomment to run as an unprivileged user with only CAP_NET_ADMIN. Requires a
|
||||
# nebula user that owns the config directory. Add CAP_NET_BIND_SERVICE to both
|
||||
# lines if any listener (lighthouse DNS, listen.port, stats, sshd) binds <1024.
|
||||
#User=nebula
|
||||
#Group=nebula
|
||||
#CapabilityBoundingSet=CAP_NET_ADMIN
|
||||
#AmbientCapabilities=CAP_NET_ADMIN
|
||||
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml
|
||||
Restart=always
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
module github.com/slackhq/nebula
|
||||
|
||||
go 1.26.0
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.2
|
||||
filippo.io/bigmod v0.1.0
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
|
||||
github.com/armon/go-radix v1.0.0
|
||||
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
|
||||
github.com/flynn/noise v1.1.0
|
||||
github.com/gaissmai/bart v0.28.0
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
@@ -23,12 +24,12 @@ require (
|
||||
github.com/vishvananda/netlink v1.3.1
|
||||
go.uber.org/goleak v1.3.0
|
||||
go.yaml.in/yaml/v3 v3.0.4
|
||||
golang.org/x/crypto v0.54.0
|
||||
golang.org/x/crypto v0.53.0
|
||||
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
|
||||
golang.org/x/net v0.57.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/sys v0.47.0
|
||||
golang.org/x/term v0.45.0
|
||||
golang.org/x/net v0.56.0
|
||||
golang.org/x/sync v0.21.0
|
||||
golang.org/x/sys v0.46.0
|
||||
golang.org/x/term v0.44.0
|
||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
|
||||
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
|
||||
golang.zx2c4.com/wireguard/windows v1.0.1
|
||||
|
||||
@@ -19,6 +19,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 h1:M5QgkYacWj0Xs8MhpIK/5uwU02icXpEoSo9sM2aRCps=
|
||||
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432/go.mod h1:xwIwAxMvYnVrGJPe2FKx5prTrnAjGOD8zvDOnxnrrkM=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -160,8 +162,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
|
||||
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
|
||||
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
|
||||
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
@@ -180,8 +182,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -189,8 +191,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -206,11 +208,11 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
|
||||
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
|
||||
-117
@@ -1,117 +0,0 @@
|
||||
package nebula
|
||||
|
||||
// This file is a trimmed, inlined copy of the graphite exporter from
|
||||
// github.com/cyberdelia/go-metrics-graphite, retaining only the Config type and
|
||||
// the Once entrypoint that Nebula uses. The upstream package has been
|
||||
// unmaintained for 10+ years, so it was vendored here to drop the dependency.
|
||||
// See https://github.com/slackhq/nebula/issues/1831.
|
||||
//
|
||||
// Copyright 2015 Timothée Peignier. All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rcrowley/go-metrics"
|
||||
)
|
||||
|
||||
// graphiteConfigExport provides a container with configuration parameters for
|
||||
// the Graphite exporter.
|
||||
type graphiteConfigExport struct {
|
||||
Addr *net.TCPAddr // Network address to connect to
|
||||
Registry metrics.Registry // Registry to be exported
|
||||
FlushInterval time.Duration // Flush interval
|
||||
DurationUnit time.Duration // Time conversion unit for durations
|
||||
Prefix string // Prefix to be prepended to metric names
|
||||
Percentiles []float64 // Percentiles to export from timers and histograms
|
||||
}
|
||||
|
||||
// graphiteOnce performs a single submission to Graphite, returning a non-nil
|
||||
// error on failed connections.
|
||||
func graphiteOnce(c graphiteConfigExport) error {
|
||||
now := time.Now().Unix()
|
||||
du := float64(c.DurationUnit)
|
||||
flushSeconds := float64(c.FlushInterval) / float64(time.Second)
|
||||
conn, err := net.DialTCP("tcp", nil, c.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
w := bufio.NewWriter(conn)
|
||||
c.Registry.Each(func(name string, i any) {
|
||||
switch metric := i.(type) {
|
||||
case metrics.Counter:
|
||||
count := metric.Count()
|
||||
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, count, now)
|
||||
fmt.Fprintf(w, "%s.%s.count_ps %.2f %d\n", c.Prefix, name, float64(count)/flushSeconds, now)
|
||||
case metrics.Gauge:
|
||||
fmt.Fprintf(w, "%s.%s.value %d %d\n", c.Prefix, name, metric.Value(), now)
|
||||
case metrics.GaugeFloat64:
|
||||
fmt.Fprintf(w, "%s.%s.value %f %d\n", c.Prefix, name, metric.Value(), now)
|
||||
case metrics.Histogram:
|
||||
h := metric.Snapshot()
|
||||
ps := h.Percentiles(c.Percentiles)
|
||||
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, h.Count(), now)
|
||||
fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, h.Min(), now)
|
||||
fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, h.Max(), now)
|
||||
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, h.Mean(), now)
|
||||
fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, h.StdDev(), now)
|
||||
for psIdx, psKey := range c.Percentiles {
|
||||
key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1)
|
||||
fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx], now)
|
||||
}
|
||||
case metrics.Meter:
|
||||
m := metric.Snapshot()
|
||||
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, m.Count(), now)
|
||||
fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, m.Rate1(), now)
|
||||
fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, m.Rate5(), now)
|
||||
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, m.Rate15(), now)
|
||||
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, m.RateMean(), now)
|
||||
case metrics.Timer:
|
||||
t := metric.Snapshot()
|
||||
ps := t.Percentiles(c.Percentiles)
|
||||
count := t.Count()
|
||||
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, count, now)
|
||||
fmt.Fprintf(w, "%s.%s.count_ps %.2f %d\n", c.Prefix, name, float64(count)/flushSeconds, now)
|
||||
fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, t.Min()/int64(du), now)
|
||||
fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, t.Max()/int64(du), now)
|
||||
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, t.Mean()/du, now)
|
||||
fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, t.StdDev()/du, now)
|
||||
for psIdx, psKey := range c.Percentiles {
|
||||
key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1)
|
||||
fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx]/du, now)
|
||||
}
|
||||
fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, t.Rate1(), now)
|
||||
fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, t.Rate5(), now)
|
||||
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now)
|
||||
fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now)
|
||||
}
|
||||
w.Flush()
|
||||
})
|
||||
return nil
|
||||
}
|
||||
+5
-25
@@ -295,13 +295,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
|
||||
hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1)
|
||||
err := hm.outside.WriteTo(stage0, addr)
|
||||
if err != nil {
|
||||
// These repeat every attempt, so match the success log below and only shout when the remotes changed
|
||||
level := slog.LevelDebug
|
||||
if remotesHaveChanged {
|
||||
level = slog.LevelError
|
||||
}
|
||||
|
||||
hostinfo.logger(hm.l).Log(context.Background(), level, "Failed to send handshake message",
|
||||
hostinfo.logger(hm.l).Error("Failed to send handshake message",
|
||||
"udpAddr", addr,
|
||||
"initiatorIndex", hostinfo.localIndexId,
|
||||
"handshake", hsFields,
|
||||
@@ -535,9 +529,7 @@ func (hm *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) {
|
||||
|
||||
func (hm *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) {
|
||||
for _, addr := range hostinfo.vpnAddrs {
|
||||
if cur, ok := hm.vpnIps[addr]; ok && cur.hostinfo == hostinfo {
|
||||
delete(hm.vpnIps, addr)
|
||||
}
|
||||
delete(hm.vpnIps, addr)
|
||||
}
|
||||
|
||||
if len(hm.vpnIps) == 0 {
|
||||
@@ -749,14 +741,8 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
|
||||
return
|
||||
}
|
||||
|
||||
connState, err := newConnectionStateFromResult(result)
|
||||
if err != nil {
|
||||
f.l.Error("Discarding handshake with an invalid message index", "error", err, "vpnAddrs", vpnAddrs)
|
||||
return
|
||||
}
|
||||
|
||||
hostinfo := &HostInfo{
|
||||
ConnectionState: connState,
|
||||
ConnectionState: newConnectionStateFromResult(result),
|
||||
localIndexId: result.LocalIndex,
|
||||
remoteIndexId: result.RemoteIndex,
|
||||
vpnAddrs: vpnAddrs,
|
||||
@@ -874,13 +860,7 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
|
||||
}
|
||||
|
||||
// Handshake complete; build the ConnectionState now that we have keys and a verified peer cert.
|
||||
cs, err := newConnectionStateFromResult(result)
|
||||
if err != nil {
|
||||
f.l.Error("Discarding handshake with an invalid message index", "error", err, "vpnAddrs", hostinfo.vpnAddrs)
|
||||
hm.DeleteHostInfo(hostinfo)
|
||||
return
|
||||
}
|
||||
hostinfo.ConnectionState = cs
|
||||
hostinfo.ConnectionState = newConnectionStateFromResult(result)
|
||||
|
||||
remoteCert := result.RemoteCert
|
||||
if remoteCert == nil {
|
||||
@@ -1097,7 +1077,7 @@ func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hos
|
||||
hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
|
||||
// We received a valid handshake on this relay, so make sure the relay
|
||||
// state reflects that, in case it had been marked Disestablished.
|
||||
via.relayHI.relayState.UpdateRelayForByIdxState(via.relay.LocalIndex, Established)
|
||||
via.relayHI.relayState.UpdateRelayForByIdxState(via.remoteIdx, Established)
|
||||
f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
|
||||
f.l.Info("Handshake message sent", append(logFields, "relay", via.relayHI.vpnAddrs[0])...)
|
||||
}
|
||||
|
||||
@@ -287,6 +287,7 @@ type HostInfo struct {
|
||||
type ViaSender struct {
|
||||
UdpAddr netip.AddrPort
|
||||
relayHI *HostInfo // relayHI is the host info object of the relay
|
||||
remoteIdx uint32 // remoteIdx is the index included in the header of the received packet
|
||||
relay *Relay // relay contains the rest of the relay information, including the PeerIP of the host trying to communicate with us.
|
||||
IsRelayed bool // IsRelayed is true if the packet was sent through a relay
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
|
||||
// routes packets from the Nebula addr to the Nebula addr through the Nebula
|
||||
// TUN device.
|
||||
if immediatelyForwardToSelf {
|
||||
_, err := f.readers[q].Write(packet)
|
||||
_, err := f.queues[q].Write(packet)
|
||||
if err != nil {
|
||||
f.l.Error("Failed to forward to tun", "error", err)
|
||||
}
|
||||
@@ -96,7 +96,7 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
|
||||
return
|
||||
}
|
||||
|
||||
_, err := f.readers[q].Write(out)
|
||||
_, err := f.queues[q].Write(out)
|
||||
if err != nil {
|
||||
f.l.Error("Failed to write to tun", "error", err)
|
||||
}
|
||||
@@ -275,14 +275,6 @@ func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *C
|
||||
f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0)
|
||||
}
|
||||
|
||||
// dropExhausted records an exhaustion drop and logs once, on the crossing send, for a spent tunnel.
|
||||
func (f *Interface) dropExhausted(hostinfo *HostInfo, c uint64, msg string) {
|
||||
f.messageMetrics.TxExhausted(1)
|
||||
if c == RejectAfterMessages {
|
||||
hostinfo.logger(f.l).Error(msg)
|
||||
}
|
||||
}
|
||||
|
||||
// SendVia sends a payload through a Relay tunnel. No authentication or encryption is done
|
||||
// to the payload for the ultimate target host, making this a useful method for sending
|
||||
// handshake messages to peers through relay tunnels.
|
||||
@@ -302,14 +294,7 @@ func (f *Interface) SendVia(via *HostInfo,
|
||||
// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
|
||||
via.ConnectionState.writeLock.Lock()
|
||||
}
|
||||
c, ok := via.ConnectionState.NextMessageCounter()
|
||||
if !ok {
|
||||
if noiseutil.EncryptLockNeeded {
|
||||
via.ConnectionState.writeLock.Unlock()
|
||||
}
|
||||
f.dropExhausted(via, c, "Dropping outbound relay packets, tunnel message counter is exhausted")
|
||||
return
|
||||
}
|
||||
c := via.ConnectionState.messageCounter.Add(1)
|
||||
|
||||
out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
|
||||
f.connectionManager.Out(via)
|
||||
@@ -376,14 +361,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
|
||||
// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
|
||||
ci.writeLock.Lock()
|
||||
}
|
||||
c, ok := ci.NextMessageCounter()
|
||||
if !ok {
|
||||
if noiseutil.EncryptLockNeeded {
|
||||
ci.writeLock.Unlock()
|
||||
}
|
||||
f.dropExhausted(hostinfo, c, "Dropping outbound packets, tunnel message counter is exhausted")
|
||||
return
|
||||
}
|
||||
c := ci.messageCounter.Add(1)
|
||||
|
||||
//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
|
||||
out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
|
||||
@@ -430,7 +408,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
|
||||
if err != nil {
|
||||
hostinfo.logger(f.l).Error("Failed to write outgoing packet",
|
||||
"error", err,
|
||||
"udpAddr", hr,
|
||||
"udpAddr", remote,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
|
||||
+76
-27
@@ -4,9 +4,9 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"runtime"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -20,7 +20,9 @@ import (
|
||||
"github.com/slackhq/nebula/firewall"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/overlay"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/udp"
|
||||
"github.com/slackhq/nebula/util"
|
||||
)
|
||||
|
||||
const mtu = 9001
|
||||
@@ -49,7 +51,19 @@ type InterfaceConfig struct {
|
||||
reQueryWait time.Duration
|
||||
|
||||
ConntrackCacheTimeout time.Duration
|
||||
l *slog.Logger
|
||||
|
||||
// CpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
|
||||
// should pin to. Queue i pins to CpuAffinity[i % len(CpuAffinity)] —
|
||||
// shorter lists than `routines` cycle. Empty list keeps the default
|
||||
// pin-to-(i % NumCPU) behavior. Only consulted when PinThreads is true.
|
||||
CpuAffinity []int
|
||||
// PinThreads controls whether each TUN reader OS thread is pinned to a
|
||||
// single CPU (via tun.pin_threads, default true). Pinning keeps each
|
||||
// goroutine's UDP sends on one XPS-selected NIC TX ring so per-flow
|
||||
// packets stay ordered on the wire.
|
||||
PinThreads bool
|
||||
|
||||
l *slog.Logger
|
||||
}
|
||||
|
||||
type Interface struct {
|
||||
@@ -73,7 +87,16 @@ type Interface struct {
|
||||
routines int
|
||||
disconnectInvalid atomic.Bool
|
||||
closed atomic.Bool
|
||||
relayManager *relayManager
|
||||
// cpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
|
||||
// should pin to. Queue i pins to cpuAffinity[i % len(cpuAffinity)].
|
||||
// Empty falls back to the default pin-to-(allowed CPU) behavior.
|
||||
// Only consulted when pinThreads is true.
|
||||
cpuAffinity []int
|
||||
// pinThreads controls whether listenIn pins each TUN reader OS thread to
|
||||
// a CPU at all (tun.pin_threads, default true). When false, threads are
|
||||
// left free to migrate as on stock nebula.
|
||||
pinThreads bool
|
||||
relayManager *relayManager
|
||||
|
||||
tryPromoteEvery atomic.Uint32
|
||||
reQueryEvery atomic.Uint32
|
||||
@@ -90,7 +113,7 @@ type Interface struct {
|
||||
|
||||
ctx context.Context
|
||||
writers []udp.Conn
|
||||
readers []io.ReadWriteCloser
|
||||
queues []tio.Queue
|
||||
wg sync.WaitGroup
|
||||
|
||||
// fatalErr holds the first unexpected reader error that caused shutdown.
|
||||
@@ -189,7 +212,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
|
||||
routines: c.routines,
|
||||
version: c.version,
|
||||
writers: make([]udp.Conn, c.routines),
|
||||
readers: make([]io.ReadWriteCloser, c.routines),
|
||||
myVpnNetworks: cs.myVpnNetworks,
|
||||
myVpnNetworksTable: cs.myVpnNetworksTable,
|
||||
myVpnAddrs: cs.myVpnAddrs,
|
||||
@@ -198,6 +220,8 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
|
||||
relayManager: c.relayManager,
|
||||
connectionManager: c.connectionManager,
|
||||
conntrackCacheTimeout: c.ConntrackCacheTimeout,
|
||||
cpuAffinity: c.CpuAffinity,
|
||||
pinThreads: c.PinThreads,
|
||||
|
||||
metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
|
||||
messageMetrics: c.MessageMetrics,
|
||||
@@ -240,27 +264,27 @@ func (f *Interface) activate() error {
|
||||
"boringcrypto", boringEnabled(),
|
||||
)
|
||||
|
||||
if f.routines > 1 {
|
||||
if !f.inside.SupportsMultiqueue() || !f.outside.SupportsMultipleReaders() {
|
||||
f.routines = 1
|
||||
f.l.Warn("routines is not supported on this platform, falling back to a single routine")
|
||||
}
|
||||
if f.routines > 1 && !f.outside.SupportsMultipleReaders() {
|
||||
f.routines = 1
|
||||
f.l.Warn("multiple udp readers are not supported on this platform, falling back to a single routine")
|
||||
}
|
||||
|
||||
// Prepare the tun queues. A device that can't open that many hands back
|
||||
// fewer (a single queue on platforms without multiqueue support) and we
|
||||
// size the reader routines to what we actually got.
|
||||
queues, err := f.inside.Queues(f.routines)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(queues) < f.routines {
|
||||
f.l.Warn("tun multiqueue is not supported on this platform, falling back to fewer routines",
|
||||
"requested", f.routines, "opened", len(queues))
|
||||
f.routines = len(queues)
|
||||
}
|
||||
f.queues = queues
|
||||
|
||||
metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
|
||||
|
||||
// Prepare n tun queues
|
||||
var reader io.ReadWriteCloser = f.inside
|
||||
for i := 0; i < f.routines; i++ {
|
||||
if i > 0 {
|
||||
reader, err = f.inside.NewMultiQueueReader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
f.readers[i] = reader
|
||||
}
|
||||
|
||||
// On error the caller owns the cleanup, Control.Start cancels the service context
|
||||
// before releasing our resources so a waiter never observes a live context
|
||||
if err = f.inside.Activate(); err != nil {
|
||||
@@ -281,7 +305,7 @@ func (f *Interface) run() {
|
||||
// Launch n queues to read packets from tun dev
|
||||
for i := 0; i < f.routines; i++ {
|
||||
f.wg.Go(func() {
|
||||
f.listenIn(f.readers[i], i)
|
||||
f.listenIn(f.queues[i], i)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -336,8 +360,29 @@ func (f *Interface) listenOut(i int) {
|
||||
f.l.Debug("underlay reader is done", "reader", i)
|
||||
}
|
||||
|
||||
func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
|
||||
packet := make([]byte, mtu)
|
||||
func (f *Interface) listenIn(queue tio.Queue, i int) {
|
||||
// Pinning this thread (and goroutine) to a single CPU keeps every UDP send from this goroutine going through
|
||||
// the same TX ring on the nic (XPS selects the ring by CPU), so the wire sees per-flow order. Skip entirely
|
||||
// when tun.pin_threads is false.
|
||||
if f.pinThreads {
|
||||
var cpu int
|
||||
if n := len(f.cpuAffinity); n > 0 {
|
||||
// Explicit tun.cpu_affinity list wins; parseCpuAffinity already
|
||||
// validated the entries against the allowed CPU set.
|
||||
cpu = f.cpuAffinity[i%n]
|
||||
} else if allowed, err := util.AllowedCPUs(); err == nil && len(allowed) > 0 {
|
||||
// Default: spread queues across the CPUs we're actually allowed to
|
||||
// run on. Under a cpuset/taskset mask these aren't 0..NumCPU-1, so
|
||||
// i % NumCPU would pick unrunnable IDs and every pin would fail.
|
||||
cpu = allowed[i%len(allowed)]
|
||||
} else {
|
||||
cpu = i % runtime.NumCPU()
|
||||
}
|
||||
if err := util.PinThreadToCPU(cpu); err != nil {
|
||||
f.l.Warn("failed to pin tun reader to CPU", "queue", i, "cpu", cpu, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]byte, mtu)
|
||||
fwPacket := &firewall.Packet{}
|
||||
nb := make([]byte, 12, 12)
|
||||
@@ -345,7 +390,7 @@ func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
|
||||
conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
|
||||
|
||||
for {
|
||||
n, err := reader.Read(packet)
|
||||
pkts, err := queue.Read()
|
||||
if err != nil {
|
||||
// Same shutdown noise handling as listenOut
|
||||
if !f.closed.Load() && f.ctx.Err() == nil {
|
||||
@@ -355,7 +400,11 @@ func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
|
||||
break
|
||||
}
|
||||
|
||||
f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get())
|
||||
for _, pkt := range pkts {
|
||||
// borrowed: pkt.Bytes is owned by the queue and only valid until
|
||||
// the next Read; consumeInsidePacket reads it synchronously.
|
||||
f.consumeInsidePacket(pkt.Bytes, fwPacket, nb, out, i, conntrackCache.Get())
|
||||
}
|
||||
}
|
||||
|
||||
f.l.Debug("overlay reader is done", "reader", i)
|
||||
|
||||
+8
-30
@@ -2,16 +2,11 @@ package iputil
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
|
||||
"golang.org/x/net/ipv4"
|
||||
"golang.org/x/net/ipv6"
|
||||
)
|
||||
|
||||
// ErrIPv6CouldNotFindPayload is returned when the ipv6 extension header chain is truncated before a terminal
|
||||
// upper layer protocol is reached.
|
||||
var ErrIPv6CouldNotFindPayload = errors.New("could not find payload in ipv6 packet")
|
||||
|
||||
const (
|
||||
// MaxIPv4RejectPacketSize is the largest IPv4 reject packet:
|
||||
// - 20 byte ipv4 header
|
||||
@@ -204,8 +199,8 @@ func ipv4CreateRejectTCPPacket(packet []byte, out []byte) []byte {
|
||||
}
|
||||
|
||||
func ipv6CreateRejectPacket(packet []byte, out []byte) []byte {
|
||||
proto, offset, isFragment, err := IPv6FindUpperProtocol(packet)
|
||||
if err != nil || isFragment {
|
||||
proto, offset, isFragment := ipv6FindUpperProtocol(packet)
|
||||
if isFragment {
|
||||
return nil
|
||||
}
|
||||
switch proto {
|
||||
@@ -338,18 +333,7 @@ func ipv6CreateRejectTCPPacket(packet []byte, out []byte, offset int) []byte {
|
||||
return out
|
||||
}
|
||||
|
||||
// IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the
|
||||
// offset it begins at, and whether the packet is a non-first fragment. Only the RFC 8200 and IANA extension
|
||||
// headers below are walked. Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental
|
||||
// 253/254, and real upper layer protocols like SCTP or GRE, is terminal. Walking those as extension headers
|
||||
// is a firewall bypass, so they fail closed. For a non-first fragment the returned protocol is the fragmented
|
||||
// protocol and offset points at the fragment header, there is no transport header to locate. Returns
|
||||
// ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header or the chain is truncated before a
|
||||
// terminal protocol is reached.
|
||||
func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool, err error) {
|
||||
if len(packet) < ipv6.HeaderLen {
|
||||
return 0, 0, false, ErrIPv6CouldNotFindPayload
|
||||
}
|
||||
func ipv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool) {
|
||||
nextHeader = packet[6]
|
||||
offset = ipv6.HeaderLen
|
||||
|
||||
@@ -357,36 +341,30 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm
|
||||
switch nextHeader {
|
||||
case 0, 43, 60: // Hop-by-Hop, Routing, Destination
|
||||
if len(packet) < offset+2 {
|
||||
return nextHeader, offset, isFragment, ErrIPv6CouldNotFindPayload
|
||||
return nextHeader, offset, isFragment
|
||||
}
|
||||
nextHeader = packet[offset]
|
||||
offset += (int(packet[offset+1]) + 1) << 3
|
||||
|
||||
case 44: // Fragment
|
||||
if len(packet) < offset+8 {
|
||||
return nextHeader, offset, isFragment, ErrIPv6CouldNotFindPayload
|
||||
return nextHeader, offset, isFragment
|
||||
}
|
||||
// Non-first fragments carry no transport header, report the fragmented protocol and stop
|
||||
if packet[offset+2] != 0 || packet[offset+3]&0xf8 != 0 {
|
||||
return packet[offset], offset, true, nil
|
||||
isFragment = true
|
||||
}
|
||||
nextHeader = packet[offset]
|
||||
offset += 8
|
||||
|
||||
case 51: // AH
|
||||
if len(packet) < offset+2 {
|
||||
return nextHeader, offset, isFragment, ErrIPv6CouldNotFindPayload
|
||||
return nextHeader, offset, isFragment
|
||||
}
|
||||
nextHeader = packet[offset]
|
||||
offset += (int(packet[offset+1]) + 2) << 2
|
||||
|
||||
default:
|
||||
// A prior extension header can declare a length that advances offset past the packet. The terminal
|
||||
// protocol's header isn't actually here, so treat the chain as truncated rather than classifying it.
|
||||
if offset > len(packet) {
|
||||
return nextHeader, offset, isFragment, ErrIPv6CouldNotFindPayload
|
||||
}
|
||||
return nextHeader, offset, isFragment, nil
|
||||
return nextHeader, offset, isFragment
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/net/ipv4"
|
||||
"golang.org/x/net/ipv6"
|
||||
)
|
||||
@@ -475,61 +474,3 @@ func TestCreateICMPEchoResponse_IPv6_NotICMPv6(t *testing.T) {
|
||||
result := CreateICMPEchoResponse(packet, out)
|
||||
assert.Nil(t, result)
|
||||
}
|
||||
|
||||
func Test_IPv6FindUpperProtocol(t *testing.T) {
|
||||
src := net.ParseIP("fd00::1")
|
||||
dst := net.ParseIP("fd00::2")
|
||||
|
||||
// 8 byte extension/transport stand-ins, first byte is the next header, second is the length field
|
||||
extToTCP := []byte{6, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = TCP
|
||||
extToUDP := []byte{17, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = UDP
|
||||
extToRouting := []byte{43, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = Routing
|
||||
ahToUDP := []byte{17, 0, 0, 0, 0, 0, 0, 0} // AH len 0 -> (0+2)<<2 = 8 bytes, next = UDP
|
||||
firstFragToUDP := []byte{17, 0, 0, 1, 0, 0, 0, 1} // frag offset 0, M=1, next = UDP
|
||||
nonFirstFrag := []byte{17, 0, 0, 9, 0, 0, 0, 1} // frag offset non-zero, next = UDP
|
||||
transport := []byte{0, 80, 1, 187, 0, 0, 0, 0} // stand-in bytes, IPv6FindUpperProtocol never reads ports
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
nextHeader uint8
|
||||
payload []byte
|
||||
wantProto uint8
|
||||
wantOffset int
|
||||
wantFragment bool
|
||||
wantErr error
|
||||
}{
|
||||
{"plain udp", 17, transport, 17, ipv6.HeaderLen, false, nil},
|
||||
{"hop-by-hop then tcp", 0, append(extToTCP, transport...), 6, ipv6.HeaderLen + 8, false, nil},
|
||||
{"routing then tcp", 43, append(extToTCP, transport...), 6, ipv6.HeaderLen + 8, false, nil},
|
||||
{"destination then udp", 60, append(extToUDP, transport...), 17, ipv6.HeaderLen + 8, false, nil},
|
||||
{"hop-by-hop, routing, then tcp", 0, append(append(extToRouting, extToTCP...), transport...), 6, ipv6.HeaderLen + 16, false, nil},
|
||||
{"ah then udp", 51, append(ahToUDP, transport...), 17, ipv6.HeaderLen + 8, false, nil},
|
||||
{"first fragment walks to transport", 44, append(firstFragToUDP, transport...), 17, ipv6.HeaderLen + 8, false, nil},
|
||||
{"non-first fragment stops", 44, append(nonFirstFrag, transport...), 17, ipv6.HeaderLen, true, nil},
|
||||
{"unknown protocol is terminal", 132, transport, 132, ipv6.HeaderLen, false, nil}, // SCTP
|
||||
{"truncated extension header", 0, nil, 0, ipv6.HeaderLen, false, ErrIPv6CouldNotFindPayload},
|
||||
// Destination Options with a declared length (255+1)*8 = 2048 that runs past the 48 byte buffer, next = SCTP
|
||||
{"extension length past buffer", 60, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 132, ipv6.HeaderLen + 2048, false, ErrIPv6CouldNotFindPayload},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
packet := makeIPv6Packet(src, dst, tt.nextHeader, tt.payload)
|
||||
proto, offset, isFragment, err := IPv6FindUpperProtocol(packet)
|
||||
if tt.wantErr != nil {
|
||||
assert.ErrorIs(t, err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.wantProto, proto)
|
||||
assert.Equal(t, tt.wantOffset, offset)
|
||||
assert.Equal(t, tt.wantFragment, isFragment)
|
||||
})
|
||||
}
|
||||
|
||||
// A packet smaller than an ipv6 header must error rather than panic reading byte 6
|
||||
t.Run("shorter than ipv6 header", func(t *testing.T) {
|
||||
_, _, _, err := IPv6FindUpperProtocol(make([]byte, 6))
|
||||
assert.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
})
|
||||
}
|
||||
|
||||
+1
-9
@@ -36,10 +36,6 @@ type LightHouse struct {
|
||||
myVpnNetworksTable *bart.Lite
|
||||
punchy *Punchy
|
||||
|
||||
// localAddrsFn enumerates the underlay addresses we advertise. It is a field so tests can supply simulated
|
||||
// addresses rather than whatever this machine's NICs happen to be. Set it before Start.
|
||||
localAddrsFn func(*LocalAllowList) []netip.Addr
|
||||
|
||||
// Local cache of answers from light houses
|
||||
// map of vpn addr to answers
|
||||
addrMap map[netip.Addr]*RemoteList
|
||||
@@ -111,10 +107,6 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
|
||||
queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
|
||||
l: l,
|
||||
}
|
||||
h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
|
||||
return localAddrs(h.l, al)
|
||||
}
|
||||
|
||||
lighthouses := make([]netip.Addr, 0)
|
||||
h.lighthouses.Store(&lighthouses)
|
||||
staticList := make(map[netip.Addr]struct{})
|
||||
@@ -926,7 +918,7 @@ func (lh *LightHouse) SendUpdate() {
|
||||
}
|
||||
|
||||
lal := lh.GetLocalAllowList()
|
||||
for _, e := range lh.localAddrsFn(lal) {
|
||||
for _, e := range localAddrs(lh.l, lal) {
|
||||
if lh.myVpnNetworksTable.Contains(e) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -7,11 +7,11 @@ import (
|
||||
"net"
|
||||
"net/netip"
|
||||
"runtime/debug"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/noiseutil"
|
||||
"github.com/slackhq/nebula/overlay"
|
||||
"github.com/slackhq/nebula/sshd"
|
||||
"github.com/slackhq/nebula/udp"
|
||||
@@ -21,12 +21,6 @@ import (
|
||||
|
||||
type m = map[string]any
|
||||
|
||||
// maxRoutines caps routines below the RejectHeadroom nonce gap so concurrent senders can't race the counter past wrap.
|
||||
const maxRoutines = 1 << 16
|
||||
|
||||
// The reject headroom must exceed every sender that can be mid-reservation at once, about two per routine.
|
||||
const _ = noiseutil.RejectHeadroom - 4*maxRoutines
|
||||
|
||||
func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
// Automatically cancel the context if Main returns an error, to signal all created goroutines to quit.
|
||||
@@ -88,6 +82,9 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
|
||||
if routines < 1 {
|
||||
routines = 1
|
||||
}
|
||||
if routines > 1 {
|
||||
l.Info("Using multiple routines", "routines", routines)
|
||||
}
|
||||
} else {
|
||||
// deprecated and undocumented
|
||||
tunQueues := c.GetInt("tun.routines", 1)
|
||||
@@ -97,12 +94,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
|
||||
l.Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead", "routines", routines)
|
||||
}
|
||||
}
|
||||
if routines > maxRoutines {
|
||||
l.Warn("Using multiple routines", "routines", maxRoutines, "clamped", true, "requestedRoutines", routines)
|
||||
routines = maxRoutines
|
||||
} else if routines > 1 {
|
||||
l.Info("Using multiple routines", "routines", routines)
|
||||
}
|
||||
|
||||
// EXPERIMENTAL
|
||||
// Intentionally not documented yet while we do more testing and determine
|
||||
@@ -241,6 +232,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
|
||||
relayManager: NewRelayManager(ctx, l, hostMap, c),
|
||||
punchy: punchy,
|
||||
ConntrackCacheTimeout: conntrackCacheTimeout,
|
||||
CpuAffinity: parseCpuAffinity(c, l, routines),
|
||||
PinThreads: c.GetBool("tun.pin_threads", true),
|
||||
l: l,
|
||||
}
|
||||
|
||||
@@ -278,8 +271,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
|
||||
|
||||
attachCommands(l, c, ssh, ifce)
|
||||
|
||||
networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
|
||||
|
||||
return &Control{
|
||||
state: StateReady,
|
||||
f: ifce,
|
||||
@@ -290,11 +281,74 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
|
||||
statsStart: stats.Start,
|
||||
dnsStart: ds.Start,
|
||||
lighthouseStart: lightHouse.StartUpdateWorker,
|
||||
networkChangeStart: networkChanges.Start,
|
||||
connectionManagerStart: connManager.Start,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// parseCpuAffinity reads `tun.cpu_affinity` from the config — a list of
|
||||
// integer CPU IDs, one per TUN reader goroutine. Empty / unset returns nil
|
||||
// (listenIn falls back to spreading queues across the allowed CPU set).
|
||||
// Length mismatch with `routines` is a warning, not an error: shorter lists
|
||||
// are modulo-cycled across queues, longer lists' tail is ignored. Invalid
|
||||
// entries (non-integer, or a CPU ID we're not allowed to run on) are also a
|
||||
// warning and disable the override entirely so we don't silently pin to the
|
||||
// wrong CPU. Entries are validated against the process's current affinity
|
||||
// mask (util.AllowedCPUs) rather than 0..NumCPU-1: under a cgroup cpuset or
|
||||
// taskset the runnable IDs are frequently not that contiguous range, and
|
||||
// pinning to an unrunnable ID always fails. If the allowed set can't be
|
||||
// determined we fall back to a plain non-negative check.
|
||||
func parseCpuAffinity(c *config.C, l *slog.Logger, routines int) []int {
|
||||
raw := c.Get("tun.cpu_affinity")
|
||||
if raw == nil {
|
||||
return nil
|
||||
}
|
||||
rv, ok := raw.([]any)
|
||||
if !ok {
|
||||
l.Warn("tun.cpu_affinity must be a list of integers; ignoring", "value", raw)
|
||||
return nil
|
||||
}
|
||||
// allowed is the set of CPU IDs we're actually permitted to run on. A nil
|
||||
// slice (unsupported platform or lookup error) means "can't tell", so we
|
||||
// only apply the weaker non-negative check in that case.
|
||||
allowed, err := util.AllowedCPUs()
|
||||
if err != nil {
|
||||
l.Warn("could not determine allowed CPUs; validating tun.cpu_affinity against non-negative only", "error", err)
|
||||
allowed = nil
|
||||
}
|
||||
cpus := make([]int, 0, len(rv))
|
||||
for i, e := range rv {
|
||||
var cpu int
|
||||
switch v := e.(type) {
|
||||
case int:
|
||||
cpu = v
|
||||
case int64:
|
||||
cpu = int(v)
|
||||
case float64:
|
||||
cpu = int(v)
|
||||
default:
|
||||
l.Warn("tun.cpu_affinity entry not an integer; ignoring affinity",
|
||||
"index", i, "value", e)
|
||||
return nil
|
||||
}
|
||||
if cpu < 0 {
|
||||
l.Warn("tun.cpu_affinity entry out of range; ignoring affinity",
|
||||
"index", i, "cpu", cpu)
|
||||
return nil
|
||||
}
|
||||
if len(allowed) > 0 && !slices.Contains(allowed, cpu) {
|
||||
l.Warn("tun.cpu_affinity entry not in allowed CPU set; ignoring affinity",
|
||||
"index", i, "cpu", cpu, "allowed", allowed)
|
||||
return nil
|
||||
}
|
||||
cpus = append(cpus, cpu)
|
||||
}
|
||||
if len(cpus) != routines {
|
||||
l.Warn("tun.cpu_affinity length doesn't match routines; queues will modulo-cycle through the list",
|
||||
"affinity_len", len(cpus), "routines", routines)
|
||||
}
|
||||
return cpus
|
||||
}
|
||||
|
||||
func moduleVersion() string {
|
||||
info, ok := debug.ReadBuildInfo()
|
||||
if !ok {
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package nebula
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/slackhq/nebula/util"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestParseCpuAffinity(t *testing.T) {
|
||||
l := test.NewLogger()
|
||||
|
||||
// newConfig returns a config.C with tun.cpu_affinity set to v. A nil v
|
||||
// leaves the key unset.
|
||||
newConfig := func(v any) *config.C {
|
||||
c := config.NewC(l)
|
||||
if v != nil {
|
||||
c.Settings["tun"] = map[string]any{"cpu_affinity": v}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// unset -> nil (listenIn falls back to spreading across the allowed set)
|
||||
assert.Nil(t, parseCpuAffinity(newConfig(nil), l, 1))
|
||||
|
||||
// Pick a CPU we're actually allowed to run on so a valid list survives
|
||||
// validation regardless of the host's affinity mask.
|
||||
allowed, _ := util.AllowedCPUs()
|
||||
validCPU := 0
|
||||
if len(allowed) > 0 {
|
||||
validCPU = allowed[0]
|
||||
}
|
||||
|
||||
// valid list -> parsed through unchanged
|
||||
assert.Equal(t, []int{validCPU, validCPU}, parseCpuAffinity(newConfig([]any{validCPU, validCPU}), l, 2))
|
||||
|
||||
// a negative entry is out of range on every platform -> disables the override
|
||||
assert.Nil(t, parseCpuAffinity(newConfig([]any{validCPU, -1}), l, 2))
|
||||
|
||||
// a non-integer entry -> disables the override
|
||||
assert.Nil(t, parseCpuAffinity(newConfig([]any{validCPU, "not-a-cpu"}), l, 2))
|
||||
|
||||
// a CPU id outside the allowed set -> disables the override. Only assertable
|
||||
// where we can enumerate the allowed set (e.g. linux); 1<<20 is far beyond
|
||||
// any representable CPU id so it can never be in the mask.
|
||||
if len(allowed) > 0 {
|
||||
assert.Nil(t, parseCpuAffinity(newConfig([]any{1 << 20}), l, 1))
|
||||
}
|
||||
}
|
||||
+4
-13
@@ -14,8 +14,7 @@ type MessageMetrics struct {
|
||||
rxUnknown metrics.Counter
|
||||
txUnknown metrics.Counter
|
||||
|
||||
rxInvalid metrics.Counter
|
||||
txExhausted metrics.Counter
|
||||
rxInvalid metrics.Counter
|
||||
}
|
||||
|
||||
func (m *MessageMetrics) Rx(t header.MessageType, s header.MessageSubType, i int64) {
|
||||
@@ -42,13 +41,6 @@ func (m *MessageMetrics) RxInvalid(i int64) {
|
||||
}
|
||||
}
|
||||
|
||||
// TxExhausted counts outbound packets dropped because the tunnel's message counter is spent.
|
||||
func (m *MessageMetrics) TxExhausted(i int64) {
|
||||
if m != nil && m.txExhausted != nil {
|
||||
m.txExhausted.Inc(i)
|
||||
}
|
||||
}
|
||||
|
||||
func newMessageMetrics() *MessageMetrics {
|
||||
gen := func(t string) [][]metrics.Counter {
|
||||
return [][]metrics.Counter{
|
||||
@@ -69,10 +61,9 @@ func newMessageMetrics() *MessageMetrics {
|
||||
rx: gen("rx"),
|
||||
tx: gen("tx"),
|
||||
|
||||
rxUnknown: metrics.GetOrRegisterCounter("messages.rx.other", nil),
|
||||
txUnknown: metrics.GetOrRegisterCounter("messages.tx.other", nil),
|
||||
rxInvalid: metrics.GetOrRegisterCounter("messages.rx.invalid", nil),
|
||||
txExhausted: metrics.GetOrRegisterCounter("messages.tx.exhausted", nil),
|
||||
rxUnknown: metrics.GetOrRegisterCounter("messages.rx.other", nil),
|
||||
txUnknown: metrics.GetOrRegisterCounter("messages.tx.other", nil),
|
||||
rxInvalid: metrics.GetOrRegisterCounter("messages.rx.invalid", nil),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,9 +25,6 @@ func (s *CipherStateAESGCM) EncryptDanger(out, ad, plaintext []byte, n uint64, n
|
||||
if s == nil {
|
||||
return nil, errors.New("no cipher state available to encrypt")
|
||||
}
|
||||
if n >= RejectAfterMessages {
|
||||
return nil, ErrMessageCounterExhausted
|
||||
}
|
||||
nb[0] = 0
|
||||
nb[1] = 0
|
||||
nb[2] = 0
|
||||
|
||||
@@ -24,9 +24,6 @@ func (s *CipherStateChaChaPoly) EncryptDanger(out, ad, plaintext []byte, n uint6
|
||||
if s == nil {
|
||||
return nil, errors.New("no cipher state available to encrypt")
|
||||
}
|
||||
if n >= RejectAfterMessages {
|
||||
return nil, ErrMessageCounterExhausted
|
||||
}
|
||||
nb[0] = 0
|
||||
nb[1] = 0
|
||||
nb[2] = 0
|
||||
|
||||
@@ -1,22 +1,11 @@
|
||||
package noiseutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/flynn/noise"
|
||||
)
|
||||
|
||||
// RejectHeadroom is the wrap gap for senders racing the counter, sized large enough for any routine count.
|
||||
const RejectHeadroom = uint64(1) << 40
|
||||
|
||||
// RejectAfterMessages is the nonce ceiling: encrypting stops RejectHeadroom short of the wrap.
|
||||
const RejectAfterMessages = math.MaxUint64 - RejectHeadroom
|
||||
|
||||
// ErrMessageCounterExhausted is returned by EncryptDanger once the nonce reaches RejectAfterMessages.
|
||||
var ErrMessageCounterExhausted = errors.New("message counter exhausted")
|
||||
|
||||
// CipherState is the post-handshake AEAD cipher used for the data plane.
|
||||
// Each supported cipher has its own concrete implementation in this package with the nonce endianness hardcoded,
|
||||
// so the encrypt/decrypt fast path avoids interface dispatch on the byte order.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package noiseutil
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/flynn/noise"
|
||||
@@ -90,24 +89,6 @@ func roundtrip(t *testing.T, enc, dec CipherState) {
|
||||
assert.Equal(t, 16, enc.Overhead())
|
||||
}
|
||||
|
||||
func TestEncryptRejectsExhaustedCounter(t *testing.T) {
|
||||
// Pin the headroom below the uint64 wrap so a typo can't silently move the ceiling.
|
||||
require.Equal(t, uint64(1)<<40, RejectHeadroom)
|
||||
require.Equal(t, math.MaxUint64-RejectHeadroom, RejectAfterMessages)
|
||||
|
||||
encA, _ := buildCipherStates(t, CipherAESGCM)
|
||||
encC, _ := buildCipherStates(t, noise.CipherChaChaPoly)
|
||||
nb := make([]byte, 12)
|
||||
|
||||
for _, cs := range []CipherState{NewCipherStateAESGCM(encA), NewCipherStateChaChaPoly(encC)} {
|
||||
_, err := cs.EncryptDanger(nil, nil, []byte("x"), RejectAfterMessages-1, nb)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = cs.EncryptDanger(nil, nil, []byte("x"), RejectAfterMessages, nb)
|
||||
require.ErrorIs(t, err, ErrMessageCounterExhausted)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkCipherStateEncryptAESGCM(b *testing.B) {
|
||||
enc, _ := buildCipherStatesB(b, CipherAESGCM)
|
||||
benchEncryptCipherState(b, NewCipherState(enc, CipherAESGCM))
|
||||
|
||||
+141
-70
@@ -13,7 +13,6 @@ import (
|
||||
|
||||
"github.com/slackhq/nebula/firewall"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/iputil"
|
||||
"golang.org/x/net/ipv4"
|
||||
)
|
||||
|
||||
@@ -103,31 +102,27 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
|
||||
return
|
||||
}
|
||||
|
||||
if len(packet) < header.Len+hostinfo.ConnectionState.dKey.Overhead() {
|
||||
f.messageMetrics.RxInvalid(1)
|
||||
if f.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||
f.l.Debug("packet too small", "from", via, "length", len(packet))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// All remaining packets are encrypted
|
||||
if isMessageRelay {
|
||||
// Relay packets are special, this branch should always early-return
|
||||
if err = hostinfo.ConnectionState.VerifyRelay(f.l, h.MessageCounter, packet, nb); err != nil {
|
||||
if f.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||
hostinfo.logger(f.l).Debug("Failed to verify relay packet", "error", err, "from", via, "header", h)
|
||||
}
|
||||
return
|
||||
}
|
||||
f.handleOutsideRelayPacket(hostinfo, via, out, packet, h, fwPacket, lhf, nb, q, localCache)
|
||||
ci := hostinfo.ConnectionState
|
||||
if !ci.window.Check(f.l, h.MessageCounter) {
|
||||
return
|
||||
}
|
||||
|
||||
out, err = hostinfo.ConnectionState.Decrypt(f.l, h.MessageCounter, out, packet, nb)
|
||||
// Relay packets are special
|
||||
if isMessageRelay {
|
||||
f.handleOutsideRelayPacket(hostinfo, via, out, packet, h, fwPacket, lhf, nb, q, localCache)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
out, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
|
||||
if err != nil {
|
||||
if f.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||
hostinfo.logger(f.l).Debug("Failed to decrypt packet", "error", err, "from", via, "header", h)
|
||||
hostinfo.logger(f.l).Debug("Failed to decrypt packet",
|
||||
"error", err,
|
||||
"from", via,
|
||||
"header", h,
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -156,7 +151,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
|
||||
// No-op, useful for the Roaming and connectionManager side-effects above
|
||||
case header.TestRequest:
|
||||
//recycle the input packet ciphertext as our output buffer
|
||||
f.send(header.Test, header.TestReply, hostinfo.ConnectionState, hostinfo, out, nb, packet)
|
||||
f.send(header.Test, header.TestReply, ci, hostinfo, out, nb, packet)
|
||||
default:
|
||||
hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected test subtype seen", "from", via, "header", h)
|
||||
return
|
||||
@@ -175,8 +170,27 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
|
||||
}
|
||||
|
||||
func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
|
||||
// Successfully validated the thing. Get rid of the Relay header and the AEAD tag
|
||||
signedPayload := packet[header.Len : len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
|
||||
// The entire body is sent as AD, not encrypted.
|
||||
// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
|
||||
// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
|
||||
// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
|
||||
// which will gracefully fail in the DecryptDanger call.
|
||||
signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
|
||||
signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
|
||||
var err error
|
||||
out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// Advance the replay window now that the frame is authenticated
|
||||
if !hostinfo.ConnectionState.window.Update(f.l, h.MessageCounter) {
|
||||
if f.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||
hostinfo.logger(f.l).Debug("dropping out of window relay packet", "header", h)
|
||||
}
|
||||
return
|
||||
}
|
||||
// Successfully validated the thing. Get rid of the Relay header.
|
||||
signedPayload = signedPayload[header.Len:]
|
||||
// Pull the Roaming parts up here, and return in all call paths.
|
||||
f.handleHostRoaming(hostinfo, via)
|
||||
// Track usage of both the HostInfo and the Relay for the received & authenticated packet
|
||||
@@ -200,6 +214,7 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
|
||||
via = ViaSender{
|
||||
UdpAddr: via.UdpAddr,
|
||||
relayHI: hostinfo,
|
||||
remoteIdx: relay.RemoteIndex,
|
||||
relay: relay,
|
||||
IsRelayed: true,
|
||||
}
|
||||
@@ -220,10 +235,9 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
|
||||
if targetRelay.State == Established {
|
||||
switch targetRelay.Type {
|
||||
case ForwardingType:
|
||||
// Forward this packet through the relay tunnel, rebuilding it in place.
|
||||
// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
|
||||
fwdBuf := packet[:0:len(packet)] // Cap to len(packet) to protect memory from a larger parent buffer
|
||||
f.SendVia(targetHI, targetRelay, signedPayload, nb, fwdBuf, true)
|
||||
// Forward this packet through the relay tunnel
|
||||
// Find the target HostInfo
|
||||
f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
|
||||
case TerminalType:
|
||||
hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
|
||||
return
|
||||
@@ -300,6 +314,7 @@ var (
|
||||
ErrIPv4InvalidHeaderLength = errors.New("invalid ipv4 header length")
|
||||
ErrIPv4PacketTooShort = errors.New("ipv4 packet is too short")
|
||||
ErrIPv6PacketTooShort = errors.New("ipv6 packet is too short")
|
||||
ErrIPv6CouldNotFindPayload = errors.New("could not find payload in ipv6 packet")
|
||||
)
|
||||
|
||||
// newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers
|
||||
@@ -332,59 +347,101 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
|
||||
fp.RemoteAddr, _ = netip.AddrFromSlice(data[24:40])
|
||||
}
|
||||
|
||||
// Walk the extension header chain to the upper layer protocol. iputil.IPv6FindUpperProtocol is the single
|
||||
// source of truth for which headers are extension headers, so this stays in lockstep with the reject path
|
||||
// and cannot drift into misreading an unknown protocol (SCTP, GRE, etc.) as a forged transport.
|
||||
proto, offset, isFragment, err := iputil.IPv6FindUpperProtocol(data)
|
||||
if err != nil {
|
||||
return ErrIPv6PacketTooShort
|
||||
}
|
||||
|
||||
fp.Protocol = proto
|
||||
fp.Fragment = isFragment
|
||||
if isFragment {
|
||||
// Non-first fragments carry no transport header, so we have no ports to read
|
||||
fp.RemotePort = 0
|
||||
fp.LocalPort = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
switch layers.IPProtocol(proto) {
|
||||
case layers.IPProtocolICMPv6:
|
||||
// An ICMPv6 message is at least type, code and checksum, 4 bytes. Only echo carries more than we read.
|
||||
if dataLen < offset+4 {
|
||||
return ErrIPv6PacketTooShort
|
||||
protoAt := 6 // NextHeader is at 6 bytes into the ipv6 header
|
||||
offset := ipv6.HeaderLen // Start at the end of the ipv6 header
|
||||
next := 0
|
||||
for {
|
||||
if protoAt >= dataLen {
|
||||
break
|
||||
}
|
||||
fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6
|
||||
switch data[offset] { //icmp type
|
||||
case layers.ICMPv6TypeEchoRequest, layers.ICMPv6TypeEchoReply:
|
||||
proto := layers.IPProtocol(data[protoAt])
|
||||
|
||||
switch proto {
|
||||
case layers.IPProtocolESP, layers.IPProtocolNoNextHeader:
|
||||
fp.Protocol = uint8(proto)
|
||||
fp.RemotePort = 0
|
||||
fp.LocalPort = 0
|
||||
fp.Fragment = false
|
||||
return nil
|
||||
|
||||
case layers.IPProtocolICMPv6:
|
||||
if dataLen < offset+6 {
|
||||
return ErrIPv6PacketTooShort
|
||||
}
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier
|
||||
fp.Protocol = uint8(proto)
|
||||
fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6
|
||||
icmptype := data[offset+1]
|
||||
switch icmptype {
|
||||
case layers.ICMPv6TypeEchoRequest, layers.ICMPv6TypeEchoReply:
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier
|
||||
default:
|
||||
fp.RemotePort = 0
|
||||
}
|
||||
fp.Fragment = false
|
||||
return nil
|
||||
|
||||
case layers.IPProtocolTCP, layers.IPProtocolUDP:
|
||||
if dataLen < offset+4 {
|
||||
return ErrIPv6PacketTooShort
|
||||
}
|
||||
|
||||
fp.Protocol = uint8(proto)
|
||||
if incoming {
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset : offset+2])
|
||||
fp.LocalPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
|
||||
} else {
|
||||
fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
|
||||
}
|
||||
|
||||
fp.Fragment = false
|
||||
return nil
|
||||
|
||||
case layers.IPProtocolIPv6Fragment:
|
||||
// Fragment header is 8 bytes, need at least offset+4 to read the offset field
|
||||
if dataLen < offset+8 {
|
||||
return ErrIPv6PacketTooShort
|
||||
}
|
||||
|
||||
// Check if this is the first fragment
|
||||
fragmentOffset := binary.BigEndian.Uint16(data[offset+2:offset+4]) &^ uint16(0x7) // Remove the reserved and M flag bits
|
||||
if fragmentOffset != 0 {
|
||||
// Non-first fragment, use what we have now and stop processing
|
||||
fp.Protocol = data[offset]
|
||||
fp.Fragment = true
|
||||
fp.RemotePort = 0
|
||||
fp.LocalPort = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
// The next loop should be the transport layer since we are the first fragment
|
||||
next = 8 // Fragment headers are always 8 bytes
|
||||
|
||||
case layers.IPProtocolAH:
|
||||
// Auth headers, used by IPSec, have a different meaning for header length
|
||||
if dataLen <= offset+1 {
|
||||
break
|
||||
}
|
||||
next = (int(data[offset+1]) + 2) << 2
|
||||
|
||||
default:
|
||||
fp.RemotePort = 0
|
||||
// Normal ipv6 header length processing
|
||||
if dataLen <= offset+1 {
|
||||
break
|
||||
}
|
||||
next = (int(data[offset+1]) + 1) << 3
|
||||
}
|
||||
|
||||
case layers.IPProtocolTCP, layers.IPProtocolUDP:
|
||||
if dataLen < offset+4 {
|
||||
return ErrIPv6PacketTooShort
|
||||
}
|
||||
if incoming {
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset : offset+2])
|
||||
fp.LocalPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
|
||||
} else {
|
||||
fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
|
||||
if next <= 0 {
|
||||
// Safety check, each ipv6 header has to be at least 8 bytes
|
||||
next = 8
|
||||
}
|
||||
|
||||
default:
|
||||
// don't set ports for protocols Nebula doesn't inspect
|
||||
fp.RemotePort = 0
|
||||
fp.LocalPort = 0
|
||||
protoAt = offset
|
||||
offset = offset + next
|
||||
}
|
||||
|
||||
return nil
|
||||
return ErrIPv6CouldNotFindPayload
|
||||
}
|
||||
|
||||
func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
|
||||
@@ -447,6 +504,20 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []byte, h *header.H, nb []byte) ([]byte, error) {
|
||||
var err error
|
||||
out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], mc, nb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !hostinfo.ConnectionState.window.Update(f.l, mc) {
|
||||
return nil, ErrOutOfWindow
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) {
|
||||
err := newPacket(out, true, fwPacket)
|
||||
if err != nil {
|
||||
@@ -471,7 +542,7 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, p
|
||||
return
|
||||
}
|
||||
|
||||
_, err = f.readers[q].Write(out)
|
||||
_, err = f.queues[q].Write(out)
|
||||
if err != nil {
|
||||
f.l.Error("Failed to write to tun", "error", err)
|
||||
}
|
||||
|
||||
+9
-89
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/net/ipv4"
|
||||
"golang.org/x/net/ipv6"
|
||||
)
|
||||
|
||||
func Test_newPacket(t *testing.T) {
|
||||
@@ -116,12 +115,12 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
err = newPacket(buffer.Bytes(), true, p)
|
||||
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
|
||||
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
|
||||
// A v6 packet with a hop-by-hop extension
|
||||
// ICMPv6 Payload (Echo Request)
|
||||
icmpLayer := layers.ICMPv6{
|
||||
TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeEchoRequest, 0),
|
||||
TypeCode: layers.ICMPv6TypeEchoRequest,
|
||||
}
|
||||
// Hop-by-Hop Extension Header
|
||||
hopOption := layers.IPv6HopByHopOption{}
|
||||
@@ -150,12 +149,12 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
// A full IPv6 header and 1 byte in the first extension, but missing
|
||||
// the length byte.
|
||||
err = newPacket(buffer.Bytes()[:41], true, p)
|
||||
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
|
||||
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
|
||||
// A full IPv6 header plus 1 full extension, but only 1 byte of the
|
||||
// next layer, missing length byte
|
||||
err = newPacket(buffer.Bytes()[:49], true, p)
|
||||
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
|
||||
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
err = nil
|
||||
|
||||
// A good ICMP packet
|
||||
@@ -168,7 +167,7 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
}
|
||||
|
||||
icmp := layers.ICMPv6{
|
||||
TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeEchoRequest, 0),
|
||||
TypeCode: layers.ICMPv6TypeEchoRequest,
|
||||
Checksum: 0x1234,
|
||||
}
|
||||
|
||||
@@ -190,18 +189,6 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
|
||||
// A minimal 4 byte non-echo ICMPv6 message (type, code, checksum), no identifier to read
|
||||
icmpMin := make([]byte, ipv6.HeaderLen+4)
|
||||
copy(icmpMin, buffer.Bytes()[:ipv6.HeaderLen])
|
||||
icmpMin[6] = byte(layers.IPProtocolICMPv6)
|
||||
icmpMin[ipv6.HeaderLen] = 1 // type 1, destination unreachable, not echo
|
||||
err = newPacket(icmpMin, true, p)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint8(layers.IPProtocolICMPv6), p.Protocol)
|
||||
assert.Equal(t, uint16(0), p.RemotePort)
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
|
||||
// A good ESP packet
|
||||
b := buffer.Bytes()
|
||||
b[6] = byte(layers.IPProtocolESP)
|
||||
@@ -226,15 +213,11 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
|
||||
// An unknown protocol packet, we don't dissect it so we fail closed on its true protocol with no ports
|
||||
// An unknown protocol packet
|
||||
b = buffer.Bytes()
|
||||
b[6] = 255 // 255 is a reserved protocol number
|
||||
err = newPacket(b, true, p)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint8(255), p.Protocol)
|
||||
assert.Equal(t, uint16(0), p.RemotePort)
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
|
||||
// A good UDP packet
|
||||
ip = layers.IPv6{
|
||||
@@ -351,14 +334,14 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
assert.Equal(t, uint16(22), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
|
||||
// Ensure buffer bounds checking during processing, a truncated AH header can't reach the payload
|
||||
// Ensure buffer bounds checking during processing
|
||||
err = newPacket(b[:41], true, p)
|
||||
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
|
||||
|
||||
// Invalid AH header
|
||||
b = buffer.Bytes()
|
||||
err = newPacket(b, true, p)
|
||||
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
|
||||
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
}
|
||||
|
||||
func Test_newPacket_ipv6Fragment(t *testing.T) {
|
||||
@@ -692,66 +675,3 @@ func Test_newPacket_v6ExtHeaderOverflow(t *testing.T) {
|
||||
// the host delivers to, not the forged 443 at the overflowed offset.
|
||||
assert.Equal(t, uint16(22), p.LocalPort, "firewall must parse the real transport header, not the overflowed offset")
|
||||
}
|
||||
|
||||
// Test_newPacket_v6ExtHeaderPastBuffer is a regression test for an extension header whose declared length
|
||||
// advances the walk past the end of the packet. The upper layer protocol's header isn't actually present,
|
||||
// so parseV6 must drop the packet rather than classify it as the terminal protocol with no ports.
|
||||
func Test_newPacket_v6ExtHeaderPastBuffer(t *testing.T) {
|
||||
p := &firewall.Packet{}
|
||||
|
||||
pkt := make([]byte, 48)
|
||||
pkt[0] = 0x60
|
||||
pkt[6] = byte(layers.IPProtocolIPv6Destination) // Destination Options
|
||||
pkt[7] = 64 // hop limit
|
||||
pkt[40] = byte(layers.IPProtocolSCTP) // Dest Options next header = SCTP
|
||||
pkt[41] = 255 // declared length (255+1)*8 = 2048, past the 48 byte buffer
|
||||
|
||||
require.ErrorIs(t, newPacket(pkt, true, p), ErrIPv6PacketTooShort)
|
||||
}
|
||||
|
||||
// Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized
|
||||
// Next Header as if it were an ipv6 extension header. A real upper layer protocol Nebula doesn't
|
||||
// dissect (SCTP here) is not walkable, so applying the (len+1)*8 formula marched into the SCTP
|
||||
// payload and landed on a byte that looked like UDP, forging a protocol/port pair the firewall
|
||||
// would trust while the host delivered the real SCTP datagram. The fix fails closed: the packet
|
||||
// is classified as its true protocol with no ports, so it only matches an `any` rule.
|
||||
func Test_newPacket_v6ExtHeaderConfusion(t *testing.T) {
|
||||
p := &firewall.Packet{}
|
||||
|
||||
pkt := make([]byte, 52)
|
||||
pkt[0] = 0x60 // version 6
|
||||
pkt[6] = byte(layers.IPProtocolSCTP) // NextHeader = SCTP, a real protocol, not an extension header
|
||||
pkt[7] = 64 // hop limit
|
||||
|
||||
// Real SCTP header at offset 40. Pre-fix parseV6 walked SCTP as an extension header: byte 41 (0x00, the
|
||||
// low byte of the src port below) was read as the header length, giving next=(0+1)*8=8, which landed the
|
||||
// walk on byte 40 (0x11), misread as NextHeader=UDP, then bytes 48-51 as ports.
|
||||
binary.BigEndian.PutUint16(pkt[40:42], 0x1100) // SCTP src port; byte 40=0x11, byte 41=0x00
|
||||
binary.BigEndian.PutUint16(pkt[42:44], 445) // SCTP dst port, never read by parseV6
|
||||
binary.BigEndian.PutUint16(pkt[48:50], 53) // SCTP checksum bytes, pre-fix forged RemotePort
|
||||
binary.BigEndian.PutUint16(pkt[50:52], 53) // pre-fix forged LocalPort
|
||||
|
||||
require.NoError(t, newPacket(pkt, true, p))
|
||||
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must classify as the true protocol, not the forged UDP")
|
||||
assert.Equal(t, uint16(0), p.RemotePort)
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
|
||||
// Same confusion, but the unknown protocol sits after a real extension header. The HopByHop is walked
|
||||
// correctly, then SCTP must still fail closed instead of being walked into its own payload. Protocol is
|
||||
// the only assertion that discriminates the fix here, a regression that walked SCTP would misclassify it.
|
||||
chained := make([]byte, 60)
|
||||
chained[0] = 0x60 // version 6
|
||||
chained[6] = byte(layers.IPProtocolIPv6HopByHop) // NextHeader = HopByHop extension
|
||||
chained[7] = 64 // hop limit
|
||||
chained[40] = byte(layers.IPProtocolSCTP) // HopByHop NextHeader = SCTP
|
||||
chained[41] = 0 // HopByHop length 0 -> 8 bytes, SCTP begins at offset 48
|
||||
binary.BigEndian.PutUint16(chained[48:50], 0x1100) // SCTP src port, pre-fix forged NextHeader/length bait
|
||||
binary.BigEndian.PutUint16(chained[50:52], 445) // SCTP dst port, never read by parseV6
|
||||
|
||||
require.NoError(t, newPacket(chained, true, p))
|
||||
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must fail closed on the unknown protocol after the extension header")
|
||||
assert.Equal(t, uint16(0), p.RemotePort)
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
}
|
||||
|
||||
+13
-3
@@ -4,15 +4,25 @@ import (
|
||||
"io"
|
||||
"net/netip"
|
||||
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
)
|
||||
|
||||
// defaultBatchBufSize is the per-Queue scratch size for Read. 65535 covers
|
||||
// any single IP packet.
|
||||
const defaultBatchBufSize = 65535
|
||||
|
||||
type Device interface {
|
||||
io.ReadWriteCloser
|
||||
io.Closer
|
||||
Activate() error
|
||||
Networks() []netip.Prefix
|
||||
Name() string
|
||||
RoutesFor(netip.Addr) routing.Gateways
|
||||
SupportsMultiqueue() bool
|
||||
NewMultiQueueReader() (io.ReadWriteCloser, error)
|
||||
// Queues returns the device's packet queues, opening additional ones as
|
||||
// needed until there are n. Platforms without multiqueue support return
|
||||
// their single queue regardless of n, so callers must size reader loops
|
||||
// to len(result), not n; implementations never return more than n. An
|
||||
// error means a queue that should have opened could not; the caller owns
|
||||
// cleanup via Close. Called once, during interface activation.
|
||||
Queues(n int) ([]tio.Queue, error)
|
||||
}
|
||||
|
||||
@@ -3,10 +3,9 @@
|
||||
package overlaytest
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/netip"
|
||||
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
)
|
||||
|
||||
@@ -31,20 +30,16 @@ func (NoopTun) Name() string {
|
||||
return "noop"
|
||||
}
|
||||
|
||||
func (NoopTun) Read([]byte) (int, error) {
|
||||
return 0, nil
|
||||
func (NoopTun) Read() ([]tio.Packet, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (NoopTun) Write([]byte) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (NoopTun) SupportsMultiqueue() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (NoopTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return nil, errors.New("unsupported")
|
||||
func (NoopTun) Queues(int) ([]tio.Queue, error) {
|
||||
return []tio.Queue{NoopTun{}}, nil
|
||||
}
|
||||
|
||||
func (NoopTun) Close() error {
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
//go:build linux && !android
|
||||
// +build linux,!android
|
||||
|
||||
package tio
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// blockOn parks the calling goroutine until fd is ready (events is POLLIN for
|
||||
// reads, POLLOUT for writes) or shutdownFd signals teardown. It builds the
|
||||
// pollfd array on the stack every call, so concurrent callers on the same
|
||||
// Queue never share Revents storage.
|
||||
//
|
||||
// Returns os.ErrClosed when shutdown was signaled (POLLIN on shutdownFd)
|
||||
// or either fd reported a problem condition (POLLHUP|POLLNVAL|POLLERR).
|
||||
func blockOn(fd, shutdownFd int32, events int16) error {
|
||||
const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
|
||||
pfds := [2]unix.PollFd{
|
||||
{Fd: fd, Events: events},
|
||||
{Fd: shutdownFd, Events: unix.POLLIN},
|
||||
}
|
||||
var err error
|
||||
for {
|
||||
_, err = unix.Poll(pfds[:], -1)
|
||||
if err != unix.EINTR {
|
||||
break
|
||||
}
|
||||
}
|
||||
tunEvents := pfds[0].Revents
|
||||
shutdownEvents := pfds[1].Revents
|
||||
// Check err before trusting the potentially bogus bits we just got.
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
|
||||
return os.ErrClosed
|
||||
}
|
||||
if tunEvents&problemFlags != 0 {
|
||||
return os.ErrClosed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
//go:build linux && !android
|
||||
// +build linux,!android
|
||||
|
||||
package tio
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
type pollQueueSet struct {
|
||||
pq []*Poll
|
||||
// pqi is exactly the same as pq, but stored as the interface type
|
||||
pqi []Queue
|
||||
shutdownFd int
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func NewPollQueueSet() (QueueSet, error) {
|
||||
shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create eventfd: %w", err)
|
||||
}
|
||||
|
||||
out := &pollQueueSet{
|
||||
pq: []*Poll{},
|
||||
pqi: []Queue{},
|
||||
shutdownFd: shutdownFd,
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *pollQueueSet) Queues() []Queue {
|
||||
return c.pqi
|
||||
}
|
||||
|
||||
func (c *pollQueueSet) Add(fd int) error {
|
||||
x, err := newPoll(fd, c.shutdownFd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.pq = append(c.pq, x)
|
||||
c.pqi = append(c.pqi, x)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *pollQueueSet) wakeForShutdown() error {
|
||||
var buf [8]byte
|
||||
binary.NativeEndian.PutUint64(buf[:], 1)
|
||||
_, err := unix.Write(int(c.shutdownFd), buf[:])
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *pollQueueSet) Close() error {
|
||||
if c.closed.Swap(true) {
|
||||
return nil
|
||||
}
|
||||
|
||||
errs := []error{}
|
||||
|
||||
// Wake any reader blocked in poll so it observes POLLIN on the shutdown
|
||||
// eventfd and returns os.ErrClosed.
|
||||
if err := c.wakeForShutdown(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
// Close the per-queue tun fds; this also unblocks any in-flight reads.
|
||||
// The per-queue Close deliberately leaves shutdownFd alone - it belongs
|
||||
// to this container.
|
||||
for _, x := range c.pq {
|
||||
if err := x.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Close the shutdown eventfd last: every reader's pollfd set references
|
||||
// it, so it must outlive the wake + per-queue teardown above.
|
||||
if err := unix.Close(c.shutdownFd); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
c.shutdownFd = -1
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package tio
|
||||
|
||||
import "io"
|
||||
|
||||
// singleQueue adapts a legacy one-datagram-per-Read source into a Queue.
|
||||
// Read fills a private scratch buffer and returns exactly one Packet whose
|
||||
// Bytes borrow from that buffer, valid only until the next Read, per the
|
||||
// Queue contract. Single-reader like every Queue; Write is exactly as safe
|
||||
// for concurrent use as the underlying source's Write.
|
||||
type singleQueue struct {
|
||||
rw io.ReadWriter
|
||||
closer io.Closer // nil: Close is a no-op (the source is shared and owned elsewhere)
|
||||
buf []byte
|
||||
ret [1]Packet
|
||||
}
|
||||
|
||||
// NewSingleQueue wraps a one-datagram-per-Read ReadWriteCloser (a legacy tun
|
||||
// device) into a Queue. bufSize is the per-queue read scratch size and must
|
||||
// be at least the largest datagram the source can return. Close closes rwc.
|
||||
func NewSingleQueue(rwc io.ReadWriteCloser, bufSize int) Queue {
|
||||
return &singleQueue{rw: rwc, closer: rwc, buf: make([]byte, bufSize)}
|
||||
}
|
||||
|
||||
// NewSingleQueueNoClose is NewSingleQueue for a source owned by someone else,
|
||||
// e.g. several queues sharing one device. Close on the returned Queue is a
|
||||
// no-op so one queue can't tear the shared source out from under its
|
||||
// siblings; the owner remains responsible for closing the source itself.
|
||||
func NewSingleQueueNoClose(rw io.ReadWriter, bufSize int) Queue {
|
||||
return &singleQueue{rw: rw, buf: make([]byte, bufSize)}
|
||||
}
|
||||
|
||||
func (q *singleQueue) Read() ([]Packet, error) {
|
||||
n, err := q.rw.Read(q.buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q.ret[0] = Packet{Bytes: q.buf[:n]}
|
||||
return q.ret[:], nil
|
||||
}
|
||||
|
||||
func (q *singleQueue) Write(p []byte) (int, error) {
|
||||
return q.rw.Write(p)
|
||||
}
|
||||
|
||||
func (q *singleQueue) Close() error {
|
||||
if q.closer == nil {
|
||||
return nil
|
||||
}
|
||||
return q.closer.Close()
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package tio
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// QueueSet holds one or many Queue objects and helps close them in an orderly way.
|
||||
type QueueSet interface {
|
||||
io.Closer
|
||||
Queues() []Queue
|
||||
|
||||
// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
|
||||
Add(fd int) error
|
||||
}
|
||||
|
||||
// Queue is a readable/writable packet queue. Concurrency contract: a single
|
||||
// read goroutine drives Read; plain Write is safe for concurrent callers.
|
||||
type Queue interface {
|
||||
io.Closer
|
||||
|
||||
// Read returns one or more packets. The returned Packet.Bytes slices
|
||||
// are borrowed from the Queue's internal buffer and are only valid
|
||||
// until the next Read or Close on this Queue - callers must encrypt
|
||||
// or copy each slice before the next call. Single-reader only: not
|
||||
// safe for concurrent Reads (it reuses per-queue rx scratch each call).
|
||||
Read() ([]Packet, error)
|
||||
|
||||
// Write emits a single packet on the plaintext (outside→inside)
|
||||
// delivery path. Safe for concurrent use.
|
||||
Write(p []byte) (int, error)
|
||||
}
|
||||
|
||||
// Packet is the unit Queue.Read returns. Bytes points into the queue's
|
||||
// internal buffer and is only valid until the next Read or Close on the
|
||||
// queue that produced it.
|
||||
type Packet struct {
|
||||
Bytes []byte
|
||||
}
|
||||
|
||||
// Clone returns a Packet whose Bytes is a freshly allocated copy of p.Bytes,
|
||||
// safe to retain past the next Read or Close on the originating Queue.
|
||||
// Use this only when a caller genuinely needs to outlive the borrowed-slice
|
||||
// contract — the hot path reads should continue to consume the borrow
|
||||
// synchronously to avoid the allocation.
|
||||
func (p Packet) Clone() Packet {
|
||||
if p.Bytes == nil {
|
||||
return p
|
||||
}
|
||||
cp := make([]byte, len(p.Bytes))
|
||||
copy(cp, p.Bytes)
|
||||
return Packet{Bytes: cp}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
//go:build linux && !android
|
||||
// +build linux,!android
|
||||
|
||||
package tio
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// Maximum size we accept for a single read from a TUN. 65535 covers any
|
||||
// single IP packet.
|
||||
const tunReadBufSize = 65535
|
||||
|
||||
type Poll struct {
|
||||
fd int
|
||||
shutdownFd int
|
||||
closed atomic.Bool
|
||||
|
||||
readBuf []byte
|
||||
batchRet [1]Packet
|
||||
}
|
||||
|
||||
// newPoll wraps an existing tun fd. On failure it does NOT close fd: the
|
||||
// caller owns fd and is the sole closer (see pollQueueSet.Add callers in
|
||||
// overlay/tun_linux.go, which unix.Close on Add error). This keeps closes
|
||||
// at exactly one on every path.
|
||||
func newPoll(fd int, shutdownFd int) (*Poll, error) {
|
||||
if err := unix.SetNonblock(fd, true); err != nil {
|
||||
return nil, fmt.Errorf("failed to set Poll device as nonblocking: %w", err)
|
||||
}
|
||||
|
||||
out := &Poll{
|
||||
fd: fd,
|
||||
shutdownFd: shutdownFd,
|
||||
readBuf: make([]byte, tunReadBufSize),
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// blockOnRead waits until the Poll fd is readable or shutdown has been signaled.
|
||||
// Returns os.ErrClosed if Close was called.
|
||||
func (t *Poll) blockOnRead() error {
|
||||
return blockOn(int32(t.fd), int32(t.shutdownFd), unix.POLLIN)
|
||||
}
|
||||
|
||||
func (t *Poll) blockOnWrite() error {
|
||||
return blockOn(int32(t.fd), int32(t.shutdownFd), unix.POLLOUT)
|
||||
}
|
||||
|
||||
func (t *Poll) Read() ([]Packet, error) {
|
||||
n, err := t.readOne(t.readBuf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.batchRet[0] = Packet{Bytes: t.readBuf[:n]}
|
||||
return t.batchRet[:], nil
|
||||
}
|
||||
|
||||
func (t *Poll) readOne(to []byte) (int, error) {
|
||||
for {
|
||||
n, errno := unix.Read(t.fd, to)
|
||||
if errno == nil {
|
||||
return n, nil
|
||||
}
|
||||
switch errno {
|
||||
case unix.EAGAIN:
|
||||
if err := t.blockOnRead(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
case unix.EINTR:
|
||||
// retry
|
||||
case unix.EBADF:
|
||||
return 0, os.ErrClosed
|
||||
default:
|
||||
return 0, errno
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Write is safe for concurrent use
|
||||
func (t *Poll) Write(from []byte) (int, error) {
|
||||
for {
|
||||
n, errno := unix.Write(t.fd, from)
|
||||
if errno == nil {
|
||||
return n, nil
|
||||
}
|
||||
switch errno {
|
||||
case unix.EAGAIN:
|
||||
if err := t.blockOnWrite(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
case unix.EINTR:
|
||||
// retry
|
||||
case unix.EBADF:
|
||||
return 0, os.ErrClosed
|
||||
default:
|
||||
return 0, errno
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Poll) Close() error {
|
||||
if t.closed.Swap(true) {
|
||||
return nil
|
||||
}
|
||||
//shutdownFd is owned by the container, so we should not close it
|
||||
// Close the underlying fd but do NOT null t.fd: a reader may still be
|
||||
// loading it in readOne, and mutating the field would race that load.
|
||||
// It gets EBADF -> os.ErrClosed (or wakes via the shutdown eventfd's
|
||||
// ppoll first). closed.Swap already guarantees we only close once.
|
||||
return unix.Close(t.fd)
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
//go:build linux && !android && !e2e_testing
|
||||
// +build linux,!android,!e2e_testing
|
||||
|
||||
package tio
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
|
||||
// The caller takes ownership of the read fd (pass it into a QueueSet).
|
||||
func newReadPipe(t *testing.T) int {
|
||||
t.Helper()
|
||||
var fds [2]int
|
||||
if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
|
||||
t.Fatalf("pipe2: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = unix.Close(fds[1]) })
|
||||
return fds[0]
|
||||
}
|
||||
|
||||
func TestPoll_WakeForShutdown_WakesFriends(t *testing.T) {
|
||||
pipe1 := newReadPipe(t)
|
||||
pipe2 := newReadPipe(t)
|
||||
parent, err := NewPollQueueSet()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, parent.Add(pipe1))
|
||||
require.NoError(t, parent.Add(pipe2))
|
||||
t.Cleanup(func() {
|
||||
_ = unix.Close(pipe1)
|
||||
_ = unix.Close(pipe2)
|
||||
})
|
||||
|
||||
readers := parent.Queues()
|
||||
errs := make([]error, len(readers))
|
||||
var wg sync.WaitGroup
|
||||
for i, r := range readers {
|
||||
wg.Add(1)
|
||||
go func(i int, r Queue) {
|
||||
defer wg.Done()
|
||||
_, errs[i] = r.Read()
|
||||
}(i, r)
|
||||
}
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
if err := parent.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() { wg.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("readers did not wake")
|
||||
}
|
||||
|
||||
for i, err := range errs {
|
||||
if !errors.Is(err, os.ErrClosed) {
|
||||
t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPoll_ConcurrentWrite_NoRace hammers a single Poll queue from two writer
|
||||
// goroutines while a reader drains the other end of the pipe. The writers
|
||||
// overflow the pipe buffer, so both repeatedly park in blockOnWrite at the same
|
||||
// time — the exact scenario that raced on the old shared writePoll member
|
||||
// array. Run under -race; a shared-array regression trips the detector here.
|
||||
func TestPoll_ConcurrentWrite_NoRace(t *testing.T) {
|
||||
var fds [2]int
|
||||
require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
|
||||
readFd, writeFd := fds[0], fds[1]
|
||||
|
||||
shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = unix.Close(shutdownFd) })
|
||||
|
||||
p, err := newPoll(writeFd, shutdownFd)
|
||||
require.NoError(t, err)
|
||||
|
||||
const writers = 2
|
||||
const perWriter = 4000
|
||||
payload := make([]byte, 100)
|
||||
total := writers * perWriter * len(payload)
|
||||
|
||||
// Reader: drain the read end (blocking) until every writer's bytes are
|
||||
// consumed, so the writers keep making progress rather than wedging on a
|
||||
// permanently full pipe.
|
||||
readDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(readDone)
|
||||
buf := make([]byte, 4096)
|
||||
got := 0
|
||||
for got < total {
|
||||
n, rerr := unix.Read(readFd, buf)
|
||||
got += n
|
||||
if rerr != nil {
|
||||
if rerr == unix.EINTR {
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
if n == 0 { // EOF
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < writers; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < perWriter; i++ {
|
||||
if _, werr := p.Write(payload); werr != nil {
|
||||
t.Errorf("write: %v", werr)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
select {
|
||||
case <-readDone:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("reader did not drain")
|
||||
}
|
||||
|
||||
require.NoError(t, p.Close())
|
||||
_ = unix.Close(readFd)
|
||||
}
|
||||
|
||||
// TestPoll_NewPoll_DoesNotCloseFdOnFailure pins the ownership rule: when
|
||||
// newPoll fails, it must leave fd open so the caller (pollQueueSet.Add's
|
||||
// callers in tun_linux.go) is the sole closer. If newPoll also closed fd,
|
||||
// the poll path would double-close on Add error. We force the failure with
|
||||
// an O_PATH descriptor: fcntl(F_SETFL) — which SetNonblock performs — is not
|
||||
// permitted on O_PATH fds and fails with EBADF, while the fd itself stays
|
||||
// open so we can observe that newPoll left it alone.
|
||||
func TestPoll_NewPoll_DoesNotCloseFdOnFailure(t *testing.T) {
|
||||
fd, err := unix.Open("/", unix.O_PATH|unix.O_CLOEXEC, 0)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = unix.Close(fd) })
|
||||
|
||||
p, err := newPoll(fd, 1)
|
||||
require.Error(t, err, "SetNonblock on an O_PATH fd should fail")
|
||||
require.Nil(t, p)
|
||||
|
||||
// If newPoll had closed fd, F_GETFD would report it closed. It staying
|
||||
// open proves newPoll left the fd for the caller to close exactly once.
|
||||
require.True(t, fdOpen(t, fd), "newPoll must not close fd on failure; caller is the sole closer")
|
||||
}
|
||||
|
||||
func TestPoll_Close_Idempotent(t *testing.T) {
|
||||
tf, err := newPoll(newReadPipe(t), 1)
|
||||
require.NoError(t, err)
|
||||
if err := tf.Close(); err != nil {
|
||||
t.Fatalf("first Close: %v", err)
|
||||
}
|
||||
if err := tf.Close(); err != nil {
|
||||
t.Fatalf("second Close should be a no-op, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// fdOpen reports whether fd currently refers to an open file description.
|
||||
// A closed (or never-allocated) fd makes F_GETFD fail with EBADF.
|
||||
func fdOpen(t *testing.T, fd int) bool {
|
||||
t.Helper()
|
||||
_, err := unix.FcntlInt(uintptr(fd), unix.F_GETFD, 0)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
if errors.Is(err, unix.EBADF) {
|
||||
return false
|
||||
}
|
||||
t.Fatalf("unexpected fcntl(F_GETFD) error on fd %d: %v", fd, err)
|
||||
return false
|
||||
}
|
||||
|
||||
// TestPollQueueSet_Close_ClosesShutdownFd is the regression test for the
|
||||
// leaked shutdown eventfd: the container that owns shutdownFd must close it in
|
||||
// Close, and a second Close must be a safe no-op.
|
||||
func TestPollQueueSet_Close_ClosesShutdownFd(t *testing.T) {
|
||||
qs, err := NewPollQueueSet()
|
||||
require.NoError(t, err)
|
||||
c, ok := qs.(*pollQueueSet)
|
||||
require.True(t, ok)
|
||||
require.NoError(t, qs.Add(newReadPipe(t)))
|
||||
|
||||
shutdownFd := c.shutdownFd
|
||||
require.True(t, fdOpen(t, shutdownFd), "shutdown eventfd should be open before Close")
|
||||
|
||||
require.NoError(t, qs.Close())
|
||||
require.False(t, fdOpen(t, shutdownFd), "shutdown eventfd should be closed after Close")
|
||||
|
||||
// Second Close must not touch fds (shutdownFd is now -1) and must return nil.
|
||||
require.NoError(t, qs.Close())
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
"github.com/slackhq/nebula/util"
|
||||
)
|
||||
@@ -63,7 +64,7 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
|
||||
return r
|
||||
}
|
||||
|
||||
func (t tun) Activate() error {
|
||||
func (t *tun) Activate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -96,10 +97,6 @@ func (t *tun) Name() string {
|
||||
return "android"
|
||||
}
|
||||
|
||||
func (t *tun) SupportsMultiqueue() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
|
||||
func (t *tun) Queues(int) ([]tio.Queue, error) {
|
||||
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ package overlay
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
@@ -16,6 +15,7 @@ import (
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
"github.com/slackhq/nebula/util"
|
||||
netroute "golang.org/x/net/route"
|
||||
@@ -606,10 +606,6 @@ func (t *tun) Name() string {
|
||||
return t.Device
|
||||
}
|
||||
|
||||
func (t *tun) SupportsMultiqueue() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
|
||||
func (t *tun) Queues(int) ([]tio.Queue, error) {
|
||||
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
|
||||
}
|
||||
|
||||
+26
-24
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/rcrowley/go-metrics"
|
||||
"github.com/slackhq/nebula/iputil"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
)
|
||||
|
||||
@@ -23,6 +24,23 @@ type disabledTun struct {
|
||||
l *slog.Logger
|
||||
}
|
||||
|
||||
// Read hands the next queued packet to a reader, copying it into b. Reads
|
||||
// from concurrent queues are safe: the channel receive serializes them and
|
||||
// each queue copies into its own private scratch buffer.
|
||||
func (t *disabledTun) Read(b []byte) (int, error) {
|
||||
r, ok := <-t.read
|
||||
if !ok {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
t.tx.Inc(1)
|
||||
if t.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||
t.l.Debug("Write payload", "raw", prettyPacket(r))
|
||||
}
|
||||
|
||||
return copy(b, r), nil
|
||||
}
|
||||
|
||||
func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled bool, l *slog.Logger) *disabledTun {
|
||||
tun := &disabledTun{
|
||||
vpnNetworks: vpnNetworks,
|
||||
@@ -57,24 +75,6 @@ func (*disabledTun) Name() string {
|
||||
return "disabled"
|
||||
}
|
||||
|
||||
func (t *disabledTun) Read(b []byte) (int, error) {
|
||||
r, ok := <-t.read
|
||||
if !ok {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
if len(r) > len(b) {
|
||||
return 0, fmt.Errorf("packet larger than mtu: %d > %d bytes", len(r), len(b))
|
||||
}
|
||||
|
||||
t.tx.Inc(1)
|
||||
if t.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||
t.l.Debug("Write payload", "raw", prettyPacket(r))
|
||||
}
|
||||
|
||||
return copy(b, r), nil
|
||||
}
|
||||
|
||||
func (t *disabledTun) handleICMPEchoRequest(b []byte) bool {
|
||||
out := make([]byte, len(b))
|
||||
out = iputil.CreateICMPEchoResponse(b, out)
|
||||
@@ -106,12 +106,14 @@ func (t *disabledTun) Write(b []byte) (int, error) {
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
func (t *disabledTun) SupportsMultiqueue() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return t, nil
|
||||
func (t *disabledTun) Queues(n int) ([]tio.Queue, error) {
|
||||
out := make([]tio.Queue, n)
|
||||
for i := range out {
|
||||
// NoClose: the shared channel and metrics are owned by the
|
||||
// disabledTun; Close on the device tears them down once for everybody.
|
||||
out[i] = tio.NewSingleQueueNoClose(t, defaultBatchBufSize)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (t *disabledTun) Close() error {
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
//go:build linux && !android && !e2e_testing
|
||||
// +build linux,!android,!e2e_testing
|
||||
|
||||
package overlay
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
|
||||
// The caller takes ownership of the read fd (pass it to newTunFd / newFriend).
|
||||
func newReadPipe(t *testing.T) int {
|
||||
t.Helper()
|
||||
var fds [2]int
|
||||
if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
|
||||
t.Fatalf("pipe2: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = unix.Close(fds[1]) })
|
||||
return fds[0]
|
||||
}
|
||||
|
||||
func TestTunFile_WakeForShutdown_UnblocksRead(t *testing.T) {
|
||||
tf, err := newTunFd(newReadPipe(t))
|
||||
if err != nil {
|
||||
t.Fatalf("newTunFd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = tf.Close() })
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := tf.Read(make([]byte, 64))
|
||||
done <- err
|
||||
}()
|
||||
|
||||
// Verify Read is actually blocked in poll.
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Read returned before shutdown signal: %v", err)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
if err := tf.wakeForShutdown(); err != nil {
|
||||
t.Fatalf("wakeForShutdown: %v", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, os.ErrClosed) {
|
||||
t.Fatalf("expected os.ErrClosed, got %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Read did not wake on shutdown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTunFile_WakeForShutdown_WakesFriends(t *testing.T) {
|
||||
parent, err := newTunFd(newReadPipe(t))
|
||||
if err != nil {
|
||||
t.Fatalf("newTunFd: %v", err)
|
||||
}
|
||||
friend, err := parent.newFriend(newReadPipe(t))
|
||||
if err != nil {
|
||||
_ = parent.Close()
|
||||
t.Fatalf("newFriend: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = friend.Close()
|
||||
_ = parent.Close()
|
||||
})
|
||||
|
||||
readers := []*tunFile{parent, friend}
|
||||
errs := make([]error, len(readers))
|
||||
var wg sync.WaitGroup
|
||||
for i, r := range readers {
|
||||
wg.Add(1)
|
||||
go func(i int, r *tunFile) {
|
||||
defer wg.Done()
|
||||
_, errs[i] = r.Read(make([]byte, 64))
|
||||
}(i, r)
|
||||
}
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
if err := parent.wakeForShutdown(); err != nil {
|
||||
t.Fatalf("wakeForShutdown: %v", err)
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() { wg.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("readers did not wake")
|
||||
}
|
||||
|
||||
for i, err := range errs {
|
||||
if !errors.Is(err, os.ErrClosed) {
|
||||
t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTunFile_Close_Idempotent(t *testing.T) {
|
||||
tf, err := newTunFd(newReadPipe(t))
|
||||
if err != nil {
|
||||
t.Fatalf("newTunFd: %v", err)
|
||||
}
|
||||
if err := tf.Close(); err != nil {
|
||||
t.Fatalf("first Close: %v", err)
|
||||
}
|
||||
if err := tf.Close(); err != nil {
|
||||
t.Fatalf("second Close should be a no-op, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
@@ -20,7 +19,7 @@ import (
|
||||
"github.com/gaissmai/bart"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
"github.com/slackhq/nebula/util"
|
||||
netroute "golang.org/x/net/route"
|
||||
@@ -561,12 +560,8 @@ func (t *tun) Name() string {
|
||||
return t.Device
|
||||
}
|
||||
|
||||
func (t *tun) SupportsMultiqueue() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
|
||||
func (t *tun) Queues(int) ([]tio.Queue, error) {
|
||||
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
|
||||
}
|
||||
|
||||
func (t *tun) addRoutes(logErrors bool) error {
|
||||
|
||||
+3
-6
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
"github.com/slackhq/nebula/util"
|
||||
"golang.org/x/sys/unix"
|
||||
@@ -159,10 +160,6 @@ func (t *tun) Name() string {
|
||||
return "iOS"
|
||||
}
|
||||
|
||||
func (t *tun) SupportsMultiqueue() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
|
||||
func (t *tun) Queues(int) ([]tio.Queue, error) {
|
||||
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
|
||||
}
|
||||
|
||||
+75
-250
@@ -4,10 +4,7 @@
|
||||
package overlay
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
@@ -20,180 +17,15 @@ import (
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
"github.com/slackhq/nebula/util"
|
||||
"github.com/vishvananda/netlink"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// tunFile wraps a TUN file descriptor with poll-based reads. The FD provided will be changed to non-blocking.
|
||||
// A shared eventfd allows Close to wake all readers blocked in poll.
|
||||
type tunFile struct {
|
||||
fd int
|
||||
shutdownFd int
|
||||
lastOne bool
|
||||
readPoll [2]unix.PollFd
|
||||
writePoll [2]unix.PollFd
|
||||
closed bool
|
||||
}
|
||||
|
||||
// newFriend makes a tunFile for a MultiQueueReader that copies the shutdown eventfd from the parent tun
|
||||
func (r *tunFile) newFriend(fd int) (*tunFile, error) {
|
||||
if err := unix.SetNonblock(fd, true); err != nil {
|
||||
return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
|
||||
}
|
||||
return &tunFile{
|
||||
fd: fd,
|
||||
shutdownFd: r.shutdownFd,
|
||||
readPoll: [2]unix.PollFd{
|
||||
{Fd: int32(fd), Events: unix.POLLIN},
|
||||
{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
|
||||
},
|
||||
writePoll: [2]unix.PollFd{
|
||||
{Fd: int32(fd), Events: unix.POLLOUT},
|
||||
{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newTunFd(fd int) (*tunFile, error) {
|
||||
if err := unix.SetNonblock(fd, true); err != nil {
|
||||
return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
|
||||
}
|
||||
|
||||
shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create eventfd: %w", err)
|
||||
}
|
||||
|
||||
out := &tunFile{
|
||||
fd: fd,
|
||||
shutdownFd: shutdownFd,
|
||||
lastOne: true,
|
||||
readPoll: [2]unix.PollFd{
|
||||
{Fd: int32(fd), Events: unix.POLLIN},
|
||||
{Fd: int32(shutdownFd), Events: unix.POLLIN},
|
||||
},
|
||||
writePoll: [2]unix.PollFd{
|
||||
{Fd: int32(fd), Events: unix.POLLOUT},
|
||||
{Fd: int32(shutdownFd), Events: unix.POLLIN},
|
||||
},
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *tunFile) blockOnRead() error {
|
||||
const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
|
||||
var err error
|
||||
for {
|
||||
_, err = unix.Poll(r.readPoll[:], -1)
|
||||
if err != unix.EINTR {
|
||||
break
|
||||
}
|
||||
}
|
||||
//always reset these!
|
||||
tunEvents := r.readPoll[0].Revents
|
||||
shutdownEvents := r.readPoll[1].Revents
|
||||
r.readPoll[0].Revents = 0
|
||||
r.readPoll[1].Revents = 0
|
||||
//do the err check before trusting the potentially bogus bits we just got
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
|
||||
return os.ErrClosed
|
||||
} else if tunEvents&problemFlags != 0 {
|
||||
return os.ErrClosed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *tunFile) blockOnWrite() error {
|
||||
const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
|
||||
var err error
|
||||
for {
|
||||
_, err = unix.Poll(r.writePoll[:], -1)
|
||||
if err != unix.EINTR {
|
||||
break
|
||||
}
|
||||
}
|
||||
//always reset these!
|
||||
tunEvents := r.writePoll[0].Revents
|
||||
shutdownEvents := r.writePoll[1].Revents
|
||||
r.writePoll[0].Revents = 0
|
||||
r.writePoll[1].Revents = 0
|
||||
//do the err check before trusting the potentially bogus bits we just got
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
|
||||
return os.ErrClosed
|
||||
} else if tunEvents&problemFlags != 0 {
|
||||
return os.ErrClosed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *tunFile) Read(buf []byte) (int, error) {
|
||||
for {
|
||||
if n, err := unix.Read(r.fd, buf); err == nil {
|
||||
return n, nil
|
||||
} else if err == unix.EAGAIN {
|
||||
if err = r.blockOnRead(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
continue
|
||||
} else if err == unix.EINTR {
|
||||
continue
|
||||
} else if err == unix.EBADF {
|
||||
return 0, os.ErrClosed
|
||||
} else {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *tunFile) Write(buf []byte) (int, error) {
|
||||
for {
|
||||
if n, err := unix.Write(r.fd, buf); err == nil {
|
||||
return n, nil
|
||||
} else if err == unix.EAGAIN {
|
||||
if err = r.blockOnWrite(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
continue
|
||||
} else if err == unix.EINTR {
|
||||
continue
|
||||
} else if err == unix.EBADF {
|
||||
return 0, os.ErrClosed
|
||||
} else {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *tunFile) wakeForShutdown() error {
|
||||
var buf [8]byte
|
||||
binary.NativeEndian.PutUint64(buf[:], 1)
|
||||
_, err := unix.Write(int(r.readPoll[1].Fd), buf[:])
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *tunFile) Close() error {
|
||||
if r.closed { // avoid closing more than once. Technically a fd could get re-used, which would be a problem
|
||||
return nil
|
||||
}
|
||||
r.closed = true
|
||||
if r.lastOne {
|
||||
_ = unix.Close(r.shutdownFd)
|
||||
}
|
||||
return unix.Close(r.fd)
|
||||
}
|
||||
|
||||
type tun struct {
|
||||
*tunFile
|
||||
readers []*tunFile
|
||||
readers tio.QueueSet
|
||||
closeLock sync.Mutex
|
||||
Device string
|
||||
vpnNetworks []netip.Prefix
|
||||
@@ -250,44 +82,57 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
|
||||
// openTunDev opens /dev/net/tun, creating the device node first if it's
|
||||
// missing (docker containers occasionally omit it).
|
||||
func openTunDev() (int, error) {
|
||||
fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
|
||||
if err != nil {
|
||||
// If /dev/net/tun doesn't exist, try to create it (will happen in docker)
|
||||
if os.IsNotExist(err) {
|
||||
err = os.MkdirAll("/dev/net", 0755)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
|
||||
}
|
||||
err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err)
|
||||
}
|
||||
|
||||
fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
|
||||
}
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
if err == nil {
|
||||
return fd, nil
|
||||
}
|
||||
if !os.IsNotExist(err) {
|
||||
return -1, err
|
||||
}
|
||||
if err = os.MkdirAll("/dev/net", 0755); err != nil {
|
||||
return -1, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
|
||||
}
|
||||
if err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200))); err != nil {
|
||||
return -1, fmt.Errorf("failed to create /dev/net/tun: %w", err)
|
||||
}
|
||||
fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0)
|
||||
if err != nil {
|
||||
return -1, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
|
||||
}
|
||||
return fd, nil
|
||||
}
|
||||
|
||||
// tunSetIff runs TUNSETIFF with the given flags and returns the kernel-chosen
|
||||
// device name on success.
|
||||
func tunSetIff(fd int, name string, flags uint16) (string, error) {
|
||||
var req ifReq
|
||||
req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
|
||||
req.Flags = flags
|
||||
copy(req.Name[:], name)
|
||||
if err := ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.Trim(string(req.Name[:]), "\x00"), nil
|
||||
}
|
||||
|
||||
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
|
||||
baseFlags := uint16(unix.IFF_TUN | unix.IFF_NO_PI)
|
||||
if multiqueue {
|
||||
req.Flags |= unix.IFF_MULTI_QUEUE
|
||||
baseFlags |= unix.IFF_MULTI_QUEUE
|
||||
}
|
||||
nameStr := c.GetString("tun.dev", "")
|
||||
copy(req.Name[:], nameStr)
|
||||
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
|
||||
|
||||
fd, err := openTunDev()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name, err := tunSetIff(fd, nameStr, baseFlags)
|
||||
if err != nil {
|
||||
_ = unix.Close(fd)
|
||||
return nil, &NameError{
|
||||
Name: nameStr,
|
||||
Underlying: err,
|
||||
}
|
||||
return nil, &NameError{Name: nameStr, Underlying: err}
|
||||
}
|
||||
name := strings.Trim(string(req.Name[:]), "\x00")
|
||||
|
||||
t, err := newTunGeneric(c, l, fd, vpnNetworks)
|
||||
if err != nil {
|
||||
@@ -299,16 +144,22 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error.
|
||||
// newTunGeneric does all the stuff common to different tun initialization
|
||||
// paths. It will close your files on error.
|
||||
func newTunGeneric(c *config.C, l *slog.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) {
|
||||
tfd, err := newTunFd(fd)
|
||||
qs, err := tio.NewPollQueueSet()
|
||||
if err != nil {
|
||||
_ = unix.Close(fd)
|
||||
return nil, err
|
||||
}
|
||||
err = qs.Add(fd)
|
||||
if err != nil {
|
||||
_ = unix.Close(fd)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
t := &tun{
|
||||
tunFile: tfd,
|
||||
readers: []*tunFile{tfd},
|
||||
readers: qs,
|
||||
closeLock: sync.Mutex{},
|
||||
vpnNetworks: vpnNetworks,
|
||||
TXQueueLen: c.GetInt("tun.tx_queue", 500),
|
||||
@@ -407,36 +258,41 @@ func (t *tun) reload(c *config.C, initial bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *tun) SupportsMultiqueue() bool {
|
||||
return true
|
||||
// Queues opens additional kernel multiqueue fds until the device has n
|
||||
// queues, then returns them all. The first queue was opened by newTun.
|
||||
func (t *tun) Queues(n int) ([]tio.Queue, error) {
|
||||
for len(t.readers.Queues()) < n {
|
||||
if err := t.addQueue(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return t.readers.Queues(), nil
|
||||
}
|
||||
|
||||
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
// addQueue opens one more IFF_MULTI_QUEUE fd on the device and adds it to
|
||||
// the queue set.
|
||||
func (t *tun) addQueue() error {
|
||||
t.closeLock.Lock()
|
||||
defer t.closeLock.Unlock()
|
||||
|
||||
fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
var req ifReq
|
||||
req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
|
||||
copy(req.Name[:], t.Device)
|
||||
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
|
||||
flags := uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
|
||||
if _, err = tunSetIff(fd, t.Device, flags); err != nil {
|
||||
_ = unix.Close(fd)
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
out, err := t.tunFile.newFriend(fd)
|
||||
err = t.readers.Add(fd)
|
||||
if err != nil {
|
||||
_ = unix.Close(fd)
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
t.readers = append(t.readers, out)
|
||||
|
||||
return out, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
|
||||
@@ -484,16 +340,7 @@ func (t *tun) addIPs(link netlink.Link) error {
|
||||
//iterate over remainder, remove whoever shouldn't be there
|
||||
al, err := netlink.AddrList(link, netlink.FAMILY_ALL)
|
||||
if err != nil {
|
||||
//RTM_GETADDR dumps the whole system, so any concurrent address change
|
||||
//interrupts it - including the kernel's async tentative->preferred
|
||||
//flip of an IPv6 address the AddrReplace calls above just added,
|
||||
//which makes this a race against our own setup. Partial results are
|
||||
//still returned; the worst case is a stale address surviving until
|
||||
//the next config reload, which beats failing startup over it.
|
||||
if !errors.Is(err, netlink.ErrDumpInterrupted) {
|
||||
return fmt.Errorf("failed to get tun address list: %s", err)
|
||||
}
|
||||
t.l.Warn("tun address list dump was interrupted, stale addresses may remain")
|
||||
return fmt.Errorf("failed to get tun address list: %s", err)
|
||||
}
|
||||
|
||||
for i := range al {
|
||||
@@ -888,32 +735,10 @@ func (t *tun) Close() error {
|
||||
t.routeChan = nil
|
||||
}
|
||||
|
||||
// Signal all readers blocked in poll to wake up and exit
|
||||
_ = t.tunFile.wakeForShutdown()
|
||||
|
||||
if t.ioctlFd > 0 {
|
||||
_ = unix.Close(int(t.ioctlFd))
|
||||
t.ioctlFd = 0
|
||||
}
|
||||
|
||||
for i := range t.readers {
|
||||
if i == 0 {
|
||||
continue //we want to close the zeroth reader last
|
||||
}
|
||||
err := t.readers[i].Close()
|
||||
if err != nil {
|
||||
t.l.Error("error closing tun reader", "reader", i, "error", err)
|
||||
} else {
|
||||
t.l.Info("closed tun reader", "reader", i)
|
||||
}
|
||||
}
|
||||
|
||||
//this is t.readers[0] too
|
||||
err := t.tunFile.Close()
|
||||
if err != nil {
|
||||
t.l.Error("error closing tun reader", "reader", 0, "error", err)
|
||||
} else {
|
||||
t.l.Info("closed tun reader", "reader", 0)
|
||||
}
|
||||
return err
|
||||
return t.readers.Close()
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ package overlay
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
@@ -17,6 +16,7 @@ import (
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
"github.com/slackhq/nebula/util"
|
||||
netroute "golang.org/x/net/route"
|
||||
@@ -390,12 +390,8 @@ func (t *tun) Name() string {
|
||||
return t.Device
|
||||
}
|
||||
|
||||
func (t *tun) SupportsMultiqueue() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd")
|
||||
func (t *tun) Queues(int) ([]tio.Queue, error) {
|
||||
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
|
||||
}
|
||||
|
||||
func (t *tun) addRoutes(logErrors bool) error {
|
||||
|
||||
@@ -6,7 +6,6 @@ package overlay
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
@@ -17,6 +16,7 @@ import (
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
"github.com/slackhq/nebula/util"
|
||||
netroute "golang.org/x/net/route"
|
||||
@@ -138,8 +138,8 @@ func tunWritev(fd int, iovecs []unix.Iovec) (n int, err error)
|
||||
//go:noescape
|
||||
func tunReadv(fd int, iovecs []unix.Iovec) (n int, err error)
|
||||
|
||||
// Read pulls one IP packet off the tun device, scattering the 4 byte protocol header away from the
|
||||
// packet so the payload lands directly in to.
|
||||
// Read pulls one IP packet off the tun device, scattering the 4 byte protocol header away from
|
||||
// the packet so the payload lands directly in to.
|
||||
func (t *tun) Read(to []byte) (int, error) {
|
||||
var head [4]byte
|
||||
|
||||
@@ -369,12 +369,8 @@ func (t *tun) Name() string {
|
||||
return t.Device
|
||||
}
|
||||
|
||||
func (t *tun) SupportsMultiqueue() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd")
|
||||
func (t *tun) Queues(int) ([]tio.Queue, error) {
|
||||
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
|
||||
}
|
||||
|
||||
func (t *tun) addRoutes(logErrors bool) error {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
"github.com/slackhq/nebula/udp"
|
||||
)
|
||||
@@ -177,10 +178,6 @@ func (t *TestTun) Read(b []byte) (int, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (t *TestTun) SupportsMultiqueue() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return nil, fmt.Errorf("TODO: multiqueue not implemented")
|
||||
func (t *TestTun) Queues(int) ([]tio.Queue, error) {
|
||||
return []tio.Queue{tio.NewSingleQueue(t, udp.MTU)}, nil
|
||||
}
|
||||
|
||||
+7
-11
@@ -6,7 +6,6 @@ package overlay
|
||||
import (
|
||||
"crypto"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
@@ -18,6 +17,7 @@ import (
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
"github.com/slackhq/nebula/util"
|
||||
"github.com/slackhq/nebula/wintun"
|
||||
@@ -47,6 +47,10 @@ type winTun struct {
|
||||
tun *wintun.NativeTun
|
||||
}
|
||||
|
||||
func (t *winTun) Read(b []byte) (int, error) {
|
||||
return t.tun.Read(b, 0)
|
||||
}
|
||||
|
||||
func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (Device, error) {
|
||||
return nil, fmt.Errorf("newTunFromFd not supported in Windows")
|
||||
}
|
||||
@@ -255,20 +259,12 @@ func (t *winTun) Name() string {
|
||||
return t.Device
|
||||
}
|
||||
|
||||
func (t *winTun) Read(b []byte) (int, error) {
|
||||
return t.tun.Read(b, 0)
|
||||
}
|
||||
|
||||
func (t *winTun) Write(b []byte) (int, error) {
|
||||
return t.tun.Write(b, 0)
|
||||
}
|
||||
|
||||
func (t *winTun) SupportsMultiqueue() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
|
||||
func (t *winTun) Queues(int) ([]tio.Queue, error) {
|
||||
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
|
||||
}
|
||||
|
||||
func (t *winTun) Close() error {
|
||||
|
||||
+13
-6
@@ -6,6 +6,7 @@ import (
|
||||
"net/netip"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
)
|
||||
|
||||
@@ -46,12 +47,16 @@ func (d *UserDevice) RoutesFor(ip netip.Addr) routing.Gateways {
|
||||
return routing.Gateways{routing.NewGateway(ip, 1)}
|
||||
}
|
||||
|
||||
func (d *UserDevice) SupportsMultiqueue() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (d *UserDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
|
||||
return d, nil
|
||||
func (d *UserDevice) Queues(n int) ([]tio.Queue, error) {
|
||||
out := make([]tio.Queue, n)
|
||||
for i := range out {
|
||||
// All queues share the underlying pipes (the io.Pipe serializes
|
||||
// concurrent callers) but each owns a private scratch buffer so
|
||||
// concurrent Reads across queues never alias. NoClose: the pipes are
|
||||
// owned by the UserDevice and torn down once by UserDevice.Close.
|
||||
out[i] = tio.NewSingleQueueNoClose(d, defaultBatchBufSize)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *UserDevice) Pipe() (*io.PipeReader, *io.PipeWriter) {
|
||||
@@ -61,9 +66,11 @@ func (d *UserDevice) Pipe() (*io.PipeReader, *io.PipeWriter) {
|
||||
func (d *UserDevice) Read(p []byte) (n int, err error) {
|
||||
return d.outboundReader.Read(p)
|
||||
}
|
||||
|
||||
func (d *UserDevice) Write(p []byte) (n int, err error) {
|
||||
return d.inboundWriter.Write(p)
|
||||
}
|
||||
|
||||
func (d *UserDevice) Close() error {
|
||||
d.inboundWriter.Close()
|
||||
d.outboundWriter.Close()
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package overlay
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/slackhq/nebula/overlay/tio"
|
||||
)
|
||||
|
||||
// newTestUserDevice returns the concrete *UserDevice so tests can reach Pipe()
|
||||
// and the internal queue plumbing.
|
||||
func newTestUserDevice(t *testing.T) *UserDevice {
|
||||
t.Helper()
|
||||
dev, err := NewUserDevice([]netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
|
||||
if err != nil {
|
||||
t.Fatalf("NewUserDevice: %v", err)
|
||||
}
|
||||
ud, ok := dev.(*UserDevice)
|
||||
if !ok {
|
||||
t.Fatalf("NewUserDevice returned %T, want *UserDevice", dev)
|
||||
}
|
||||
return ud
|
||||
}
|
||||
|
||||
// TestUserDeviceReadersDistinctBuffers ensures each Queue is actually different
|
||||
func TestUserDeviceReadersDistinctBuffers(t *testing.T) {
|
||||
d := newTestUserDevice(t)
|
||||
|
||||
readers, err := d.Queues(2)
|
||||
if err != nil {
|
||||
t.Fatalf("Queues: %v", err)
|
||||
}
|
||||
if len(readers) != 2 {
|
||||
t.Fatalf("Queues(2) returned %d queues, want 2", len(readers))
|
||||
}
|
||||
|
||||
// Distinct queue objects.
|
||||
if readers[0] == readers[1] {
|
||||
t.Fatal("Queues(2) returned the same queue object twice")
|
||||
}
|
||||
|
||||
// Drive one packet through each queue and confirm the borrowed bytes from
|
||||
// the first read are NOT clobbered by the second read. With a shared
|
||||
// buffer, reading pkt1 into q1 would corrupt q0's still-borrowed slice.
|
||||
_, ow := d.Pipe()
|
||||
|
||||
pkt0 := []byte("packet-zero-aaaaaaaa")
|
||||
pkt1 := []byte("packet-one-bbbbbbbbb")
|
||||
|
||||
// The pipe is unbuffered, so writes block until a reader consumes them.
|
||||
// Serialize: write pkt0 (read on q0), then write pkt1 (read on q1).
|
||||
go func() {
|
||||
if _, err := ow.Write(pkt0); err != nil {
|
||||
t.Errorf("write pkt0: %v", err)
|
||||
}
|
||||
if _, err := ow.Write(pkt1); err != nil {
|
||||
t.Errorf("write pkt1: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
got0, err := readers[0].Read()
|
||||
if err != nil {
|
||||
t.Fatalf("q0.Read: %v", err)
|
||||
}
|
||||
if len(got0) != 1 || string(got0[0].Bytes) != string(pkt0) {
|
||||
t.Fatalf("q0 first read = %q, want %q", firstBytes(got0), pkt0)
|
||||
}
|
||||
// Hold onto q0's borrowed slice across q1's read.
|
||||
borrowed := got0[0].Bytes
|
||||
|
||||
got1, err := readers[1].Read()
|
||||
if err != nil {
|
||||
t.Fatalf("q1.Read: %v", err)
|
||||
}
|
||||
if len(got1) != 1 || string(got1[0].Bytes) != string(pkt1) {
|
||||
t.Fatalf("q1 read = %q, want %q", firstBytes(got1), pkt1)
|
||||
}
|
||||
|
||||
// q0's borrowed bytes must still hold pkt0 - a shared buffer would now
|
||||
// show pkt1's contents.
|
||||
if string(borrowed) != string(pkt0) {
|
||||
t.Fatalf("q0 borrowed bytes were clobbered by q1's read: got %q, want %q", borrowed, pkt0)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUserDeviceReadersConcurrentRace exercises two queues reading distinct
|
||||
// packets concurrently. Run it under `go test -race`: with the old
|
||||
// shared-buffer implementation the concurrent Reads raced on readBuf/batchRet
|
||||
// and corrupted each other's returned slices.
|
||||
func TestUserDeviceReadersConcurrentRace(t *testing.T) {
|
||||
d := newTestUserDevice(t)
|
||||
readers, err := d.Queues(2)
|
||||
if err != nil {
|
||||
t.Fatalf("Queues: %v", err)
|
||||
}
|
||||
_, ow := d.Pipe()
|
||||
|
||||
const iterations = 200
|
||||
|
||||
errs := make(chan error, 3)
|
||||
|
||||
// Each reader parks in Read on the shared outboundReader; io.Pipe hands
|
||||
// each write to whichever reader is currently waiting. We only care that
|
||||
// concurrent Reads into distinct buffers are race-free, so any parked
|
||||
// reader may serve any write.
|
||||
var wg sync.WaitGroup
|
||||
run := func(idx int) {
|
||||
defer wg.Done()
|
||||
for i := 0; i < iterations; i++ {
|
||||
pkts, err := readers[idx].Read()
|
||||
if err != nil {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
if len(pkts) != 1 {
|
||||
errs <- fmt.Errorf("reader %d: got %d packets, want 1", idx, len(pkts))
|
||||
return
|
||||
}
|
||||
// Touch every byte of the borrowed slice while the other reader
|
||||
// may be mid-Read; a shared buffer would race here.
|
||||
total := 0
|
||||
for _, c := range pkts[0].Bytes {
|
||||
total += int(c)
|
||||
}
|
||||
_ = total
|
||||
}
|
||||
}
|
||||
|
||||
wg.Add(2)
|
||||
go run(0)
|
||||
go run(1)
|
||||
|
||||
// Feed 2*iterations packets. io.Pipe copies each write straight into the
|
||||
// waiting reader's private buffer, so reusing buf between writes is safe.
|
||||
go func() {
|
||||
buf := make([]byte, 32)
|
||||
for i := 0; i < 2*iterations; i++ {
|
||||
for j := range buf {
|
||||
buf[j] = byte(i + j)
|
||||
}
|
||||
if _, err := ow.Write(buf); err != nil {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
select {
|
||||
case err := <-errs:
|
||||
t.Fatalf("concurrent reader failed: %v", err)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func firstBytes(p []tio.Packet) []byte {
|
||||
if len(p) == 0 {
|
||||
return nil
|
||||
}
|
||||
return p[0].Bytes
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
graphite "github.com/cyberdelia/go-metrics-graphite"
|
||||
mp "github.com/nbrownus/go-metrics-prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
@@ -252,7 +253,7 @@ func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) {
|
||||
// loadStatsConfig already resolved and validated the address; re-parse
|
||||
// the resolved form (no DNS lookup) to get a *net.TCPAddr.
|
||||
addr, _ := net.ResolveTCPAddr(cfg.graphite.protocol, cfg.graphite.resolvedAddr)
|
||||
gcfg := graphiteConfigExport{
|
||||
gcfg := graphite.Config{
|
||||
Addr: addr,
|
||||
Registry: metrics.DefaultRegistry,
|
||||
FlushInterval: cfg.interval,
|
||||
@@ -261,7 +262,7 @@ func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) {
|
||||
Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999},
|
||||
}
|
||||
captureFns = append(captureFns, func() {
|
||||
if err := graphiteOnce(gcfg); err != nil {
|
||||
if err := graphite.Once(gcfg); err != nil {
|
||||
s.l.Error("Graphite export failed", "error", err)
|
||||
}
|
||||
})
|
||||
|
||||
+1
-1
@@ -371,7 +371,7 @@ func waitForListening(t *testing.T, addr string) {
|
||||
})
|
||||
}
|
||||
|
||||
// graphiteSink is a minimal TCP accept-and-discard server so graphiteOnce
|
||||
// graphiteSink is a minimal TCP accept-and-discard server so graphite.Once
|
||||
// calls in tests don't spam error logs or wedge on connection refused.
|
||||
type graphiteSink struct {
|
||||
ln net.Listener
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
package udp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
)
|
||||
|
||||
// NetworkChangeMonitor rebinds the udp listener when the local network moves out from under it.
|
||||
//
|
||||
// Detection lives here in the udp package, next to the socket it concerns and the platform matrix that already knows
|
||||
// which sockets go stale. What to do about a change — updating the lighthouse, requerying tunnels — is not the udp
|
||||
// package's business, so Start takes the reaction as a plain function. Passing it at Start rather than holding it
|
||||
// keeps this package from referencing whatever owns the rebind.
|
||||
//
|
||||
// On platforms whose sockets do not go stale, watchNetworkChanges hands back a nil channel and Start returns.
|
||||
type NetworkChangeMonitor struct {
|
||||
l *slog.Logger
|
||||
ctx context.Context
|
||||
enabled bool
|
||||
}
|
||||
|
||||
// NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start
|
||||
// is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it.
|
||||
func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor {
|
||||
return &NetworkChangeMonitor{
|
||||
l: l,
|
||||
ctx: ctx,
|
||||
enabled: c.GetBool("listen.rebind_on_network_change", true),
|
||||
}
|
||||
}
|
||||
|
||||
// Start watches for network changes until the context is cancelled, calling rebind once per settled change. It
|
||||
// blocks, so callers run it in a goroutine, and it no-ops when disabled, unsupported, or with nothing to rebind.
|
||||
func (m *NetworkChangeMonitor) Start(rebind func()) {
|
||||
if !m.enabled || rebind == nil || m.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
changes, err := watchNetworkChanges(m.ctx, m.l)
|
||||
if err != nil {
|
||||
// Not fatal. Everything else still works, we just won't notice a network change on our own.
|
||||
m.l.Error("Failed to watch for network changes, will not rebind the udp listener when the network moves",
|
||||
"error", err,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if changes == nil {
|
||||
// This platform's sockets don't go stale, so there is nothing to watch for.
|
||||
return
|
||||
}
|
||||
|
||||
m.l.Info("Watching for network changes to rebind the udp listener")
|
||||
|
||||
for range changes {
|
||||
m.l.Info("Local network changed, rebinding the udp listener")
|
||||
rebind()
|
||||
}
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
//go:build darwin && !ios && !e2e_testing
|
||||
// +build darwin,!ios,!e2e_testing
|
||||
|
||||
package udp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const (
|
||||
// netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A
|
||||
// single network change is never a single message, it is a burst: the link drops, addresses go away, new ones
|
||||
// arrive, routes get rewritten. Reporting part way through that just means reporting again.
|
||||
netChangeSettleWindow = time.Second
|
||||
|
||||
// netChangeReadBuffer is sized well past any rt_msghdr plus its addresses. A short read would be discarded by
|
||||
// the kernel, so being generous here is how we avoid missing a message.
|
||||
netChangeReadBuffer = 4096
|
||||
)
|
||||
|
||||
// watchNetworkChanges reports when the local network moves out from under us, so the listener can be rebound.
|
||||
//
|
||||
// Darwin scopes a udp socket to whatever interface it came up on. Move between networks and we keep sending out an
|
||||
// interface that no longer has a route, which surfaces as an instant "no route to host" with no packet ever leaving
|
||||
// the box. Rebind clears that, but only if something notices the change and calls it. iOS has always been told by
|
||||
// the host app off NWPathMonitor. This is the equivalent for everything else that runs on darwin.
|
||||
//
|
||||
// The returned channel is buffered and coalescing: a send is dropped if one is already pending, since both mean the
|
||||
// same thing to a reader. It is closed when ctx is cancelled or the routing socket fails, so a caller can simply
|
||||
// range over it. Platforms whose sockets do not need rebinding return a nil channel and no error.
|
||||
func watchNetworkChanges(ctx context.Context, l *slog.Logger) (<-chan struct{}, error) {
|
||||
sock, err := openRouteSocket()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
changes := make(chan struct{}, 1)
|
||||
|
||||
go func() {
|
||||
defer close(changes)
|
||||
defer func() { _ = sock.Close() }()
|
||||
|
||||
// Closing the socket is what unblocks the read in watchRouteSocket, so this turns cancellation into a
|
||||
// close. It is scoped to this call so it cannot outlive the watch it belongs to.
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = sock.Close()
|
||||
case <-done:
|
||||
}
|
||||
}()
|
||||
|
||||
watchRouteSocket(l, sock, changes)
|
||||
}()
|
||||
|
||||
return changes, nil
|
||||
}
|
||||
|
||||
// watchRouteSocket blocks reading the routing socket, reporting once per settled burst of changes. It returns when
|
||||
// the socket is closed, which is how cancellation gets us out of here.
|
||||
func watchRouteSocket(l *slog.Logger, sock *os.File, changes chan<- struct{}) {
|
||||
buf := make([]byte, netChangeReadBuffer)
|
||||
|
||||
for {
|
||||
n, err := sock.Read(buf)
|
||||
if err != nil {
|
||||
logRouteSocketError(l, err)
|
||||
return
|
||||
}
|
||||
|
||||
if !isNetworkChange(buf[:n]) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Swallow the rest of the burst. The deadline is absolute and not extended by what arrives, so this always
|
||||
// ends after the settle window no matter how chatty the socket is. Changes that land after the window
|
||||
// simply produce another report, which is the correct outcome anyway.
|
||||
deadline := time.Now().Add(netChangeSettleWindow)
|
||||
for {
|
||||
if err = sock.SetReadDeadline(deadline); err != nil {
|
||||
logRouteSocketError(l, err)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err = sock.Read(buf); err != nil {
|
||||
if os.IsTimeout(err) {
|
||||
break
|
||||
}
|
||||
logRouteSocketError(l, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err = sock.SetReadDeadline(time.Time{}); err != nil {
|
||||
logRouteSocketError(l, err)
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case changes <- struct{}{}:
|
||||
default:
|
||||
// One already pending, and a second "the network moved" tells the reader nothing new.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// logRouteSocketError reports a routing socket failure unless it is just us shutting the socket down.
|
||||
func logRouteSocketError(l *slog.Logger, err error) {
|
||||
if errors.Is(err, os.ErrClosed) {
|
||||
return
|
||||
}
|
||||
|
||||
l.Error("Error reading the routing socket, will no longer notice local network changes", "error", err)
|
||||
}
|
||||
|
||||
// openRouteSocket returns the routing socket as a non blocking os.File. Going through os.File puts reads on the go
|
||||
// poller, which buys us both a working read deadline and a Close that unblocks a read in progress.
|
||||
func openRouteSocket() (*os.File, error) {
|
||||
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = unix.SetNonblock(fd, true); err != nil {
|
||||
_ = unix.Close(fd)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return os.NewFile(uintptr(fd), "route"), nil
|
||||
}
|
||||
|
||||
// isNetworkChange reports whether a routing message means our local addressing may have moved out from under us.
|
||||
//
|
||||
// We read the header instead of parsing the message because the type is the only part we need, and a full parse can
|
||||
// fail on shapes we don't care about, which would turn "a message I can't parse" into "a change I missed".
|
||||
// rt_msghdr, if_msghdr and ifa_msghdr all begin with the same three fields, so this is the same for every type.
|
||||
func isNetworkChange(msg []byte) bool {
|
||||
if len(msg) < 4 {
|
||||
return false
|
||||
}
|
||||
|
||||
// u_short msglen, u_char version, u_char type
|
||||
if int(binary.NativeEndian.Uint16(msg[0:2])) > len(msg) || msg[2] != unix.RTM_VERSION {
|
||||
return false
|
||||
}
|
||||
|
||||
switch msg[3] {
|
||||
case unix.RTM_NEWADDR, unix.RTM_DELADDR, unix.RTM_IFINFO:
|
||||
// An address arrived or left, or a link changed state. Anything else on this socket is either a route
|
||||
// churning underneath us, which a rebind doesn't help with, or unrelated traffic.
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -1,244 +0,0 @@
|
||||
//go:build darwin && !ios && !e2e_testing
|
||||
// +build darwin,!ios,!e2e_testing
|
||||
|
||||
package udp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/goleak"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// routeMsg builds the first four bytes of a routing message, which is all isNetworkChange reads.
|
||||
func routeMsg(msgType uint8, extra int) []byte {
|
||||
msg := make([]byte, 4+extra)
|
||||
binary.NativeEndian.PutUint16(msg[0:2], uint16(len(msg)))
|
||||
msg[2] = unix.RTM_VERSION
|
||||
msg[3] = msgType
|
||||
return msg
|
||||
}
|
||||
|
||||
func TestIsNetworkChange(t *testing.T) {
|
||||
// The three that mean our addressing may have moved
|
||||
assert.True(t, isNetworkChange(routeMsg(unix.RTM_NEWADDR, 0)))
|
||||
assert.True(t, isNetworkChange(routeMsg(unix.RTM_DELADDR, 0)))
|
||||
assert.True(t, isNetworkChange(routeMsg(unix.RTM_IFINFO, 0)))
|
||||
|
||||
// Route churn is not something a rebind helps with
|
||||
assert.False(t, isNetworkChange(routeMsg(unix.RTM_ADD, 0)))
|
||||
assert.False(t, isNetworkChange(routeMsg(unix.RTM_DELETE, 0)))
|
||||
assert.False(t, isNetworkChange(routeMsg(unix.RTM_GET, 0)))
|
||||
|
||||
// Garbage must not be mistaken for a change
|
||||
assert.False(t, isNetworkChange(nil), "empty")
|
||||
assert.False(t, isNetworkChange([]byte{0, 0, 0}), "short header")
|
||||
|
||||
wrongVersion := routeMsg(unix.RTM_NEWADDR, 0)
|
||||
wrongVersion[2] = unix.RTM_VERSION + 1
|
||||
assert.False(t, isNetworkChange(wrongVersion), "wrong rtm_version")
|
||||
|
||||
lying := routeMsg(unix.RTM_NEWADDR, 0)
|
||||
binary.NativeEndian.PutUint16(lying[0:2], 512)
|
||||
assert.False(t, isNetworkChange(lying), "msglen longer than what we read")
|
||||
}
|
||||
|
||||
// socketPair returns a connected pair of datagram sockets, the first wrapped the same way the routing socket is. It
|
||||
// stands in for the kernel so the watch loop can be driven with synthetic messages.
|
||||
func socketPair(t *testing.T) (*os.File, int) {
|
||||
t.Helper()
|
||||
|
||||
fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, unix.SetNonblock(fds[0], true))
|
||||
|
||||
f := os.NewFile(uintptr(fds[0]), "route")
|
||||
t.Cleanup(func() {
|
||||
_ = f.Close()
|
||||
_ = unix.Close(fds[1])
|
||||
})
|
||||
|
||||
return f, fds[1]
|
||||
}
|
||||
|
||||
func TestWatchRouteSocketCoalescesABurst(t *testing.T) {
|
||||
sock, kernel := socketPair(t)
|
||||
changes := make(chan struct{}, 1)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
watchRouteSocket(test.NewLogger(), sock, changes)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
// One network change is a burst of messages. All of these land inside the settle window, so they must produce
|
||||
// exactly one report rather than one apiece.
|
||||
for range 5 {
|
||||
_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
// Uninteresting messages in the middle of a burst must not add a report of their own either.
|
||||
_, err := unix.Write(kernel, routeMsg(unix.RTM_ADD, 8))
|
||||
require.NoError(t, err)
|
||||
|
||||
select {
|
||||
case <-changes:
|
||||
case <-time.After(netChangeSettleWindow * 4):
|
||||
t.Fatal("a burst should have reported a change")
|
||||
}
|
||||
|
||||
// Nothing more from that burst
|
||||
select {
|
||||
case <-changes:
|
||||
t.Fatal("a burst should report exactly once")
|
||||
case <-time.After(netChangeSettleWindow):
|
||||
}
|
||||
|
||||
// A change after the window has closed is a separate event and gets its own report.
|
||||
_, err = unix.Write(kernel, routeMsg(unix.RTM_IFINFO, 8))
|
||||
require.NoError(t, err)
|
||||
select {
|
||||
case <-changes:
|
||||
case <-time.After(netChangeSettleWindow * 4):
|
||||
t.Fatal("a later change should report again")
|
||||
}
|
||||
|
||||
// Closing the socket is how the real thing shuts down
|
||||
require.NoError(t, sock.Close())
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("watchRouteSocket did not return after the socket was closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchRouteSocketIgnoresUninterestingMessages(t *testing.T) {
|
||||
sock, kernel := socketPair(t)
|
||||
changes := make(chan struct{}, 1)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
watchRouteSocket(test.NewLogger(), sock, changes)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
for _, msgType := range []uint8{unix.RTM_ADD, unix.RTM_DELETE, unix.RTM_GET, unix.RTM_MISS} {
|
||||
_, err := unix.Write(kernel, routeMsg(msgType, 8))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-changes:
|
||||
t.Fatal("route churn alone must not report a change")
|
||||
case <-time.After(netChangeSettleWindow * 2):
|
||||
}
|
||||
|
||||
require.NoError(t, sock.Close())
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("watchRouteSocket did not return after the socket was closed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWatchRouteSocketDropsRatherThanBlocks covers the coalescing send. A reader that is busy rebinding must not
|
||||
// wedge the watcher, and a second pending "the network moved" tells it nothing new anyway.
|
||||
func TestWatchRouteSocketDropsRatherThanBlocks(t *testing.T) {
|
||||
sock, kernel := socketPair(t)
|
||||
changes := make(chan struct{}, 1)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
watchRouteSocket(test.NewLogger(), sock, changes)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
// Nobody is reading changes, so after the first report the buffer is full for the rest of this test
|
||||
for range 3 {
|
||||
_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
|
||||
require.NoError(t, err)
|
||||
time.Sleep(netChangeSettleWindow + time.Millisecond*250)
|
||||
}
|
||||
|
||||
// The watcher must still be alive and responsive to a close
|
||||
require.NoError(t, sock.Close())
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("watchRouteSocket wedged on a full channel")
|
||||
}
|
||||
|
||||
assert.Len(t, changes, 1, "the pending report should have coalesced, not queued")
|
||||
}
|
||||
|
||||
// TestWatchNetworkChangesStopsWithContext covers the detection path against a real routing socket, including that
|
||||
// cancelling the context closes the channel so a ranging caller falls out of its loop.
|
||||
func TestWatchNetworkChangesStopsWithContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
changes, err := watchNetworkChanges(ctx, test.NewLogger())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, changes, "darwin should support watching")
|
||||
|
||||
drained := make(chan struct{})
|
||||
go func() {
|
||||
for range changes {
|
||||
}
|
||||
close(drained)
|
||||
}()
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case <-drained:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("cancelling the context should close the changes channel")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetworkChangeMonitorStopsWithContext drives the whole monitor against a real routing socket: Start must block
|
||||
// watching, and cancelling the context (which is all Control does on shutdown, it never stops the monitor directly)
|
||||
// must return it and clean up the watch goroutines.
|
||||
func TestNetworkChangeMonitorStopsWithContext(t *testing.T) {
|
||||
// IgnoreCurrent because other tests in this package leave readers running; we only care about what this test
|
||||
// leaks itself.
|
||||
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
l := test.NewLogger()
|
||||
c := config.NewC(l)
|
||||
require.NoError(t, c.LoadString("listen:\n rebind_on_network_change: true\n"))
|
||||
m := NewNetworkChangeMonitor(ctx, l, c)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
m.Start(func() {})
|
||||
close(done)
|
||||
}()
|
||||
|
||||
// Start should be sitting on the routing socket, not have fallen out. If it returned early it either failed to
|
||||
// watch or no-op'd, both of which we want to catch.
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("Start returned instead of watching")
|
||||
case <-time.After(time.Millisecond * 250):
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("Start did not return after the context was cancelled")
|
||||
}
|
||||
|
||||
// Starting again after the context is dead must not open anything.
|
||||
m.Start(func() {})
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
//go:build !darwin || ios || e2e_testing
|
||||
// +build !darwin ios e2e_testing
|
||||
|
||||
package udp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// watchNetworkChanges is a no-op outside of darwin.
|
||||
//
|
||||
// Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket
|
||||
// goes stale when the local network changes. Everywhere else Rebind has nothing to do, so there is nothing to watch
|
||||
// for. iOS is excluded on purpose even though it is darwin: the host app already drives the rebind off NWPathMonitor,
|
||||
// and two things racing to rebind the same socket is worse than one.
|
||||
//
|
||||
// A nil channel means "not supported here", which callers must treat as "do not start a watcher" rather than
|
||||
// selecting on it, since a receive from a nil channel blocks forever.
|
||||
func watchNetworkChanges(_ context.Context, _ *slog.Logger) (<-chan struct{}, error) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package udp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newMonitor(t *testing.T, ctx context.Context, cfg string) *NetworkChangeMonitor {
|
||||
t.Helper()
|
||||
l := test.NewLogger()
|
||||
c := config.NewC(l)
|
||||
require.NoError(t, c.LoadString(cfg))
|
||||
return NewNetworkChangeMonitor(ctx, l, c)
|
||||
}
|
||||
|
||||
func TestNetworkChangeMonitorDefaultsOn(t *testing.T) {
|
||||
// Says nothing about rebinding, so this covers the default.
|
||||
m := newMonitor(t, context.Background(), "listen:\n host: 0.0.0.0\n")
|
||||
assert.True(t, m.enabled, "should default to on")
|
||||
}
|
||||
|
||||
func TestNetworkChangeMonitorDisabledIsANoOp(t *testing.T) {
|
||||
m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: false\n")
|
||||
require.False(t, m.enabled)
|
||||
|
||||
// Must return without opening a socket. If it watched anything this would block.
|
||||
m.Start(func() {})
|
||||
}
|
||||
|
||||
func TestNetworkChangeMonitorNilRebindIsANoOp(t *testing.T) {
|
||||
// Nothing to rebind, so there is no point watching, on any platform.
|
||||
m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: true\n")
|
||||
m.Start(nil)
|
||||
}
|
||||
+5
-4
@@ -187,9 +187,6 @@ func (u *StdConn) SupportsMultipleReaders() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Rebind clears the interface the kernel scoped this socket to, so that sends are routed against the current
|
||||
// routing table instead of the interface we happened to be on when the socket was created. Darwin pins sockets
|
||||
// this way on its own, which is what strands us after the underlying network changes.
|
||||
func (u *StdConn) Rebind() error {
|
||||
var err error
|
||||
if u.isV4 {
|
||||
@@ -198,5 +195,9 @@ func (u *StdConn) Rebind() error {
|
||||
err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0)
|
||||
}
|
||||
|
||||
return err
|
||||
if err != nil {
|
||||
u.l.Error("Failed to rebind udp socket", "error", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
+166
-167
@@ -4,13 +4,12 @@
|
||||
package udp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
|
||||
@@ -20,51 +19,58 @@ import (
|
||||
)
|
||||
|
||||
type StdConn struct {
|
||||
sysFd int
|
||||
closed atomic.Bool
|
||||
isV4 bool
|
||||
l *slog.Logger
|
||||
batch int
|
||||
udpConn *net.UDPConn
|
||||
rawConn syscall.RawConn
|
||||
isV4 bool
|
||||
l *slog.Logger
|
||||
batch int
|
||||
}
|
||||
|
||||
func setReusePort(network, address string, c syscall.RawConn) error {
|
||||
var opErr error
|
||||
err := c.Control(func(fd uintptr) {
|
||||
opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
|
||||
//CloseOnExec already set by the runtime
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return opErr
|
||||
}
|
||||
|
||||
func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
|
||||
af := unix.AF_INET6
|
||||
if ip.Is4() {
|
||||
af = unix.AF_INET
|
||||
}
|
||||
syscall.ForkLock.RLock()
|
||||
fd, err := unix.Socket(af, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
|
||||
if err == nil {
|
||||
unix.CloseOnExec(fd)
|
||||
}
|
||||
syscall.ForkLock.RUnlock()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to open socket: %w", err)
|
||||
}
|
||||
|
||||
listen := netip.AddrPortFrom(ip, uint16(port))
|
||||
lc := net.ListenConfig{}
|
||||
if multi {
|
||||
if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
|
||||
_ = unix.Close(fd)
|
||||
return nil, fmt.Errorf("unable to set SO_REUSEPORT: %w", err)
|
||||
}
|
||||
lc.Control = setReusePort
|
||||
}
|
||||
//this context is only used during the bind operation, you can't cancel it to kill the socket
|
||||
pc, err := lc.ListenPacket(context.Background(), "udp", listen.String())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to open socket: %s", err)
|
||||
}
|
||||
udpConn := pc.(*net.UDPConn)
|
||||
rawConn, err := udpConn.SyscallConn()
|
||||
if err != nil {
|
||||
_ = udpConn.Close()
|
||||
return nil, err
|
||||
}
|
||||
//gotta find out if we got an AF_INET6 socket or not:
|
||||
out := &StdConn{
|
||||
udpConn: udpConn,
|
||||
rawConn: rawConn,
|
||||
l: l,
|
||||
batch: batch,
|
||||
}
|
||||
|
||||
var sa unix.Sockaddr
|
||||
if ip.Is4() {
|
||||
sa4 := &unix.SockaddrInet4{Port: port}
|
||||
sa4.Addr = ip.As4()
|
||||
sa = sa4
|
||||
} else {
|
||||
sa6 := &unix.SockaddrInet6{Port: port}
|
||||
sa6.Addr = ip.As16()
|
||||
sa = sa6
|
||||
}
|
||||
if err = unix.Bind(fd, sa); err != nil {
|
||||
_ = unix.Close(fd)
|
||||
return nil, fmt.Errorf("unable to bind to socket: %w", err)
|
||||
af, err := out.getSockOptInt(unix.SO_DOMAIN)
|
||||
if err != nil {
|
||||
_ = out.Close()
|
||||
return nil, err
|
||||
}
|
||||
out.isV4 = af == unix.AF_INET
|
||||
|
||||
return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (u *StdConn) SupportsMultipleReaders() bool {
|
||||
@@ -75,111 +81,134 @@ func (u *StdConn) Rebind() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *StdConn) getSockOptInt(opt int) (int, error) {
|
||||
if u.rawConn == nil {
|
||||
return 0, fmt.Errorf("no UDP connection")
|
||||
}
|
||||
var out int
|
||||
var opErr error
|
||||
err := u.rawConn.Control(func(fd uintptr) {
|
||||
out, opErr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, opt)
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return out, opErr
|
||||
}
|
||||
|
||||
func (u *StdConn) setSockOptInt(opt int, n int) error {
|
||||
if u.rawConn == nil {
|
||||
return fmt.Errorf("no UDP connection")
|
||||
}
|
||||
var opErr error
|
||||
err := u.rawConn.Control(func(fd uintptr) {
|
||||
opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, opt, n)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return opErr
|
||||
}
|
||||
|
||||
func (u *StdConn) SetRecvBuffer(n int) error {
|
||||
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n)
|
||||
return u.setSockOptInt(unix.SO_RCVBUFFORCE, n)
|
||||
}
|
||||
|
||||
func (u *StdConn) SetSendBuffer(n int) error {
|
||||
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
|
||||
return u.setSockOptInt(unix.SO_SNDBUFFORCE, n)
|
||||
}
|
||||
|
||||
func (u *StdConn) SetSoMark(mark int) error {
|
||||
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK, mark)
|
||||
return u.setSockOptInt(unix.SO_MARK, mark)
|
||||
}
|
||||
|
||||
func (u *StdConn) GetRecvBuffer() (int, error) {
|
||||
return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUF)
|
||||
return u.getSockOptInt(unix.SO_RCVBUF)
|
||||
}
|
||||
|
||||
func (u *StdConn) GetSendBuffer() (int, error) {
|
||||
return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF)
|
||||
return u.getSockOptInt(unix.SO_SNDBUF)
|
||||
}
|
||||
|
||||
func (u *StdConn) GetSoMark() (int, error) {
|
||||
return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK)
|
||||
return u.getSockOptInt(unix.SO_MARK)
|
||||
}
|
||||
|
||||
func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
|
||||
sa, err := unix.Getsockname(u.sysFd)
|
||||
if err != nil {
|
||||
return netip.AddrPort{}, err
|
||||
}
|
||||
switch sa := sa.(type) {
|
||||
case *unix.SockaddrInet4:
|
||||
return netip.AddrPortFrom(netip.AddrFrom4(sa.Addr), uint16(sa.Port)), nil
|
||||
case *unix.SockaddrInet6:
|
||||
return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr), uint16(sa.Port)), nil
|
||||
a := u.udpConn.LocalAddr()
|
||||
|
||||
switch v := a.(type) {
|
||||
case *net.UDPAddr:
|
||||
addr, ok := netip.AddrFromSlice(v.IP)
|
||||
if !ok {
|
||||
return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP)
|
||||
}
|
||||
return netip.AddrPortFrom(addr, uint16(v.Port)), nil
|
||||
|
||||
default:
|
||||
return netip.AddrPort{}, fmt.Errorf("unsupported sock type: %T", sa)
|
||||
return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// recvmmsg does one blocking recvmmsg (MSG_WAITFORONE), reading up to len(msgs) datagrams
|
||||
func (u *StdConn) recvmmsg(msgs []rawMessage) (int, error) {
|
||||
r, _, errno := unix.Syscall6(
|
||||
func recvmmsg(fd uintptr, msgs []rawMessage) (int, bool, error) {
|
||||
var errno syscall.Errno
|
||||
n, _, errno := unix.Syscall6(
|
||||
unix.SYS_RECVMMSG,
|
||||
uintptr(u.sysFd),
|
||||
fd,
|
||||
uintptr(unsafe.Pointer(&msgs[0])),
|
||||
uintptr(len(msgs)),
|
||||
unix.MSG_WAITFORONE,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
if errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK {
|
||||
// No data available, block for I/O and try again.
|
||||
return int(n), false, nil
|
||||
}
|
||||
if errno != 0 {
|
||||
if u.closed.Load() {
|
||||
return 0, net.ErrClosed
|
||||
}
|
||||
return 0, &net.OpError{Op: "recvmmsg", Err: errno}
|
||||
return int(n), true, &net.OpError{Op: "recvmmsg", Err: errno}
|
||||
}
|
||||
n := int(r)
|
||||
if (n == 0 || msgs[0].Len == 0) && u.closed.Load() {
|
||||
return 0, net.ErrClosed
|
||||
}
|
||||
return n, nil
|
||||
return int(n), true, nil
|
||||
}
|
||||
|
||||
// recvmsg does one blocking recvmsg into msgs[0]
|
||||
func (u *StdConn) recvmsg(msgs []rawMessage) (int, error) {
|
||||
r, _, errno := unix.Syscall6(
|
||||
unix.SYS_RECVMSG,
|
||||
uintptr(u.sysFd),
|
||||
uintptr(unsafe.Pointer(&msgs[0].Hdr)),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
if errno != 0 {
|
||||
if u.closed.Load() {
|
||||
return 0, net.ErrClosed
|
||||
func (u *StdConn) listenOutSingle(r EncReader) error {
|
||||
var err error
|
||||
var n int
|
||||
var from netip.AddrPort
|
||||
buffer := make([]byte, MTU)
|
||||
|
||||
for {
|
||||
n, from, err = u.udpConn.ReadFromUDPAddrPort(buffer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return 0, &net.OpError{Op: "recvmsg", Err: errno}
|
||||
from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port())
|
||||
r(from, buffer[:n])
|
||||
}
|
||||
if r == 0 && u.closed.Load() {
|
||||
return 0, net.ErrClosed
|
||||
}
|
||||
msgs[0].Len = uint32(r)
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (u *StdConn) ListenOut(r EncReader) error {
|
||||
func (u *StdConn) listenOutBatch(r EncReader) error {
|
||||
var ip netip.Addr
|
||||
var n int
|
||||
var operr error
|
||||
|
||||
msgs, buffers, names := u.PrepareRawMessages(u.batch)
|
||||
read := u.recvmmsg
|
||||
if u.batch == 1 {
|
||||
read = u.recvmsg
|
||||
|
||||
//reader needs to capture variables from this function, since it's used as a lambda with rawConn.Read
|
||||
//defining it outside the loop so it gets re-used
|
||||
reader := func(fd uintptr) (done bool) {
|
||||
n, done, operr = recvmmsg(fd, msgs)
|
||||
return done
|
||||
}
|
||||
|
||||
for {
|
||||
n, err := read(msgs)
|
||||
err := u.rawConn.Read(reader)
|
||||
if err != nil {
|
||||
if errors.Is(err, unix.EINTR) {
|
||||
continue // interrupted by a signal, retry the read
|
||||
}
|
||||
// net.ErrClosed after Close() is teardown, absorbed by the caller's
|
||||
// closed flag like the other platforms; anything else is a real error.
|
||||
return err
|
||||
}
|
||||
if operr != nil {
|
||||
return operr
|
||||
}
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
|
||||
@@ -193,68 +222,26 @@ func (u *StdConn) ListenOut(r EncReader) error {
|
||||
}
|
||||
}
|
||||
|
||||
func (u *StdConn) ListenOut(r EncReader) error {
|
||||
if u.batch == 1 {
|
||||
return u.listenOutSingle(r)
|
||||
} else {
|
||||
return u.listenOutBatch(r)
|
||||
}
|
||||
}
|
||||
|
||||
func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
|
||||
if u.isV4 {
|
||||
return u.writeTo4(b, ip)
|
||||
}
|
||||
return u.writeTo6(b, ip)
|
||||
}
|
||||
|
||||
func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
|
||||
var rsa unix.RawSockaddrInet6
|
||||
rsa.Family = unix.AF_INET6
|
||||
rsa.Addr = ip.Addr().As16()
|
||||
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
|
||||
|
||||
for {
|
||||
_, _, err := unix.Syscall6(
|
||||
unix.SYS_SENDTO,
|
||||
uintptr(u.sysFd),
|
||||
uintptr(unsafe.Pointer(&b[0])),
|
||||
uintptr(len(b)),
|
||||
uintptr(0),
|
||||
uintptr(unsafe.Pointer(&rsa)),
|
||||
uintptr(unix.SizeofSockaddrInet6),
|
||||
)
|
||||
if err != 0 {
|
||||
return &net.OpError{Op: "sendto", Err: err}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
|
||||
if !ip.Addr().Is4() {
|
||||
return ErrInvalidIPv6RemoteForSocket
|
||||
}
|
||||
|
||||
var rsa unix.RawSockaddrInet4
|
||||
rsa.Family = unix.AF_INET
|
||||
rsa.Addr = ip.Addr().As4()
|
||||
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
|
||||
|
||||
for {
|
||||
_, _, err := unix.Syscall6(
|
||||
unix.SYS_SENDTO,
|
||||
uintptr(u.sysFd),
|
||||
uintptr(unsafe.Pointer(&b[0])),
|
||||
uintptr(len(b)),
|
||||
uintptr(0),
|
||||
uintptr(unsafe.Pointer(&rsa)),
|
||||
uintptr(unix.SizeofSockaddrInet4),
|
||||
)
|
||||
if err != 0 {
|
||||
return &net.OpError{Op: "sendto", Err: err}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
_, err := u.udpConn.WriteToUDPAddrPort(b, ip)
|
||||
return err
|
||||
}
|
||||
|
||||
func (u *StdConn) ReloadConfig(c *config.C) {
|
||||
b := c.GetInt("listen.read_buffer", 0)
|
||||
if b > 0 {
|
||||
if err := u.SetRecvBuffer(b); err == nil {
|
||||
if s, err := u.GetRecvBuffer(); err == nil {
|
||||
err := u.SetRecvBuffer(b)
|
||||
if err == nil {
|
||||
s, err := u.GetRecvBuffer()
|
||||
if err == nil {
|
||||
u.l.Info("listen.read_buffer was set", "size", s)
|
||||
} else {
|
||||
u.l.Warn("Failed to get listen.read_buffer", "error", err)
|
||||
@@ -266,8 +253,10 @@ func (u *StdConn) ReloadConfig(c *config.C) {
|
||||
|
||||
b = c.GetInt("listen.write_buffer", 0)
|
||||
if b > 0 {
|
||||
if err := u.SetSendBuffer(b); err == nil {
|
||||
if s, err := u.GetSendBuffer(); err == nil {
|
||||
err := u.SetSendBuffer(b)
|
||||
if err == nil {
|
||||
s, err := u.GetSendBuffer()
|
||||
if err == nil {
|
||||
u.l.Info("listen.write_buffer was set", "size", s)
|
||||
} else {
|
||||
u.l.Warn("Failed to get listen.write_buffer", "error", err)
|
||||
@@ -280,8 +269,10 @@ func (u *StdConn) ReloadConfig(c *config.C) {
|
||||
b = c.GetInt("listen.so_mark", 0)
|
||||
s, err := u.GetSoMark()
|
||||
if b > 0 || (err == nil && s != 0) {
|
||||
if err := u.SetSoMark(b); err == nil {
|
||||
if s, err := u.GetSoMark(); err == nil {
|
||||
err := u.SetSoMark(b)
|
||||
if err == nil {
|
||||
s, err := u.GetSoMark()
|
||||
if err == nil {
|
||||
u.l.Info("listen.so_mark was set", "mark", s)
|
||||
} else {
|
||||
u.l.Warn("Failed to get listen.so_mark", "error", err)
|
||||
@@ -294,20 +285,28 @@ func (u *StdConn) ReloadConfig(c *config.C) {
|
||||
|
||||
func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
|
||||
var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
|
||||
_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
|
||||
if err != 0 {
|
||||
|
||||
if u.rawConn == nil {
|
||||
return fmt.Errorf("no UDP connection")
|
||||
}
|
||||
var opErr error
|
||||
err := u.rawConn.Control(func(fd uintptr) {
|
||||
_, _, syserr := unix.Syscall6(unix.SYS_GETSOCKOPT, fd, uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
|
||||
if syserr != 0 {
|
||||
opErr = syserr
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
return opErr
|
||||
}
|
||||
|
||||
func (u *StdConn) Close() error {
|
||||
u.closed.Store(true)
|
||||
// Wake the reader parked in recvmmsg/recvmsg. shutdown(2) on an unconnected socket
|
||||
// returns ENOTCONN but still wakes it, so ignore the error.
|
||||
// The reader then sees closed and stops touching the fd, making the Close below safe.
|
||||
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
|
||||
return unix.Close(u.sysFd)
|
||||
if u.udpConn != nil {
|
||||
return u.udpConn.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewUDPStatsEmitter(udpConns []Conn) func() {
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
//go:build linux && !android && !e2e_testing
|
||||
|
||||
package udp
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func testLogger() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
|
||||
}
|
||||
|
||||
// TestShutdownWakesAfterRx_Mechanism exercises the kernel quirk our teardown
|
||||
// relies on: once a socket has received a packet, shutdown(2) wakes a blocked
|
||||
// recvmmsg with n>=1/Len==0 (not n==0). recvmmsg must turn that into net.ErrClosed
|
||||
// once Close set closed, so a parked reader exits instead of spinning.
|
||||
func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
|
||||
c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, 64)
|
||||
if err != nil {
|
||||
t.Fatalf("NewListener: %v", err)
|
||||
}
|
||||
sc := c.(*StdConn)
|
||||
addr, err := sc.LocalAddr()
|
||||
if err != nil {
|
||||
t.Fatalf("LocalAddr: %v", err)
|
||||
}
|
||||
msgs, _, _ := sc.PrepareRawMessages(sc.batch)
|
||||
|
||||
// Receive a real packet so the socket has carried data.
|
||||
send, err := net.Dial("udp", addr.String())
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
if _, err := send.Write([]byte("hello")); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
n, err := sc.recvmmsg(msgs)
|
||||
t.Logf("drain of real packet: n=%d err=%v msgs[0].Len=%d", n, err, msgs[0].Len)
|
||||
_ = send.Close()
|
||||
|
||||
// Block a reader on the now-empty queue, then tear down as Close() does.
|
||||
// recvmmsg must return net.ErrClosed (not hang, not spin) even post-rx.
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := sc.recvmmsg(msgs)
|
||||
done <- err
|
||||
}()
|
||||
time.Sleep(150 * time.Millisecond) // let it park in recvmmsg
|
||||
|
||||
sc.closed.Store(true)
|
||||
if serr := unix.Shutdown(sc.sysFd, unix.SHUT_RDWR); serr != nil {
|
||||
t.Logf("shutdown returned %v (expected ENOTCONN on unconnected UDP)", serr)
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, net.ErrClosed) {
|
||||
t.Errorf("recvmmsg after post-rx shutdown returned %v, want net.ErrClosed", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatalf("HANG: recvmmsg did not return after shutdown following a received packet")
|
||||
}
|
||||
_ = unix.Close(sc.sysFd)
|
||||
}
|
||||
|
||||
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
|
||||
// reader must tear down cleanly on Close() regardless of what the socket has
|
||||
// carried. The three cases the report called out:
|
||||
//
|
||||
// no traffic ever -> works (shutdown wakes recvmmsg with n==0)
|
||||
// ping once, then idle -> historically HUNG: once the socket has received a
|
||||
// packet, shutdown(2) wakes recvmmsg with n>=1/Len==0,
|
||||
// which an n==0-only teardown check misses
|
||||
// continuous traffic -> works (a real packet is always arriving)
|
||||
//
|
||||
// All three must return within the deadline; a hang dumps goroutines so the
|
||||
// stuck reader is visible.
|
||||
func TestListenOutTeardown_TrafficPatterns(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
traffic func(send net.Conn, stop <-chan struct{})
|
||||
}{
|
||||
{"no_traffic_ever", func(net.Conn, <-chan struct{}) {}},
|
||||
{"ping_once_then_idle", func(send net.Conn, _ <-chan struct{}) {
|
||||
_, _ = send.Write([]byte("hello"))
|
||||
}},
|
||||
{"continuous", func(send net.Conn, stop <-chan struct{}) {
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
default:
|
||||
_, _ = send.Write([]byte("hello"))
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}},
|
||||
}
|
||||
|
||||
// batch 1 exercises the recvmsg path, batch 64 the recvmmsg path; both must
|
||||
// tear down cleanly.
|
||||
for _, batch := range []int{1, 64} {
|
||||
for _, tc := range cases {
|
||||
t.Run(fmt.Sprintf("batch%d/%s", batch, tc.name), func(t *testing.T) {
|
||||
runTeardownCase(t, batch, tc.name, tc.traffic)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net.Conn, stop <-chan struct{})) {
|
||||
c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, batch)
|
||||
if err != nil {
|
||||
t.Fatalf("NewListener: %v", err)
|
||||
}
|
||||
sc := c.(*StdConn)
|
||||
addr, err := sc.LocalAddr()
|
||||
if err != nil {
|
||||
t.Fatalf("LocalAddr: %v", err)
|
||||
}
|
||||
|
||||
var received atomic.Int64
|
||||
loopDone := make(chan error, 1)
|
||||
go func() {
|
||||
loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
|
||||
received.Add(1)
|
||||
})
|
||||
}()
|
||||
|
||||
send, err := net.Dial("udp", addr.String())
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
defer send.Close()
|
||||
|
||||
stop := make(chan struct{})
|
||||
trafficDone := make(chan struct{})
|
||||
go func() {
|
||||
traffic(send, stop)
|
||||
close(trafficDone)
|
||||
}()
|
||||
|
||||
// Let the pattern run and, for the idle case, the reader park again on an
|
||||
// empty queue with the socket already having received a packet.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
start := time.Now()
|
||||
if err := sc.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
close(stop)
|
||||
|
||||
select {
|
||||
case err := <-loopDone:
|
||||
// Clean teardown surfaces as net.ErrClosed (propagated like the other
|
||||
// platforms); the caller absorbs it via its closed flag.
|
||||
if err != nil && !errors.Is(err, net.ErrClosed) {
|
||||
t.Fatalf("%s: ListenOut returned unexpected error on teardown: %v", name, err)
|
||||
}
|
||||
t.Logf("%s: closed in %v (received %d packets)", name, time.Since(start), received.Load())
|
||||
case <-time.After(3 * time.Second):
|
||||
buf := make([]byte, 1<<20)
|
||||
n := runtime.Stack(buf, true)
|
||||
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
|
||||
}
|
||||
<-trafficDone
|
||||
}
|
||||
+6
-20
@@ -10,7 +10,6 @@ import (
|
||||
"net/netip"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/header"
|
||||
@@ -65,9 +64,7 @@ func acquirePacket() *Packet {
|
||||
}
|
||||
|
||||
type TesterConn struct {
|
||||
// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
|
||||
// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
|
||||
addr atomic.Pointer[netip.AddrPort]
|
||||
Addr netip.AddrPort
|
||||
|
||||
RxPackets chan *Packet // Packets to receive into nebula
|
||||
TxPackets chan *Packet // Packets transmitted outside by nebula
|
||||
@@ -85,24 +82,13 @@ type TesterConn struct {
|
||||
}
|
||||
|
||||
func NewListener(l *slog.Logger, ip netip.Addr, port int, _ bool, _ int) (Conn, error) {
|
||||
c := &TesterConn{
|
||||
return &TesterConn{
|
||||
Addr: netip.AddrPortFrom(ip, uint16(port)),
|
||||
RxPackets: make(chan *Packet, 10),
|
||||
TxPackets: make(chan *Packet, 10),
|
||||
done: make(chan struct{}),
|
||||
l: l,
|
||||
}
|
||||
c.SetAddr(netip.AddrPortFrom(ip, uint16(port)))
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// GetAddr returns the underlay address this conn currently sends from.
|
||||
func (u *TesterConn) GetAddr() netip.AddrPort {
|
||||
return *u.addr.Load()
|
||||
}
|
||||
|
||||
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
|
||||
func (u *TesterConn) SetAddr(addr netip.AddrPort) {
|
||||
u.addr.Store(&addr)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Send will place a UdpPacket onto the receive queue for nebula to consume
|
||||
@@ -161,7 +147,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
||||
p.Data = p.Data[:len(b)]
|
||||
}
|
||||
copy(p.Data, b)
|
||||
p.From = u.GetAddr()
|
||||
p.From = u.Addr
|
||||
p.To = addr
|
||||
select {
|
||||
case <-u.done:
|
||||
@@ -192,7 +178,7 @@ func NewUDPStatsEmitter(_ []Conn) func() {
|
||||
}
|
||||
|
||||
func (u *TesterConn) LocalAddr() (netip.AddrPort, error) {
|
||||
return u.GetAddr(), nil
|
||||
return u.Addr, nil
|
||||
}
|
||||
|
||||
func (u *TesterConn) SupportsMultipleReaders() bool {
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
//go:build linux && !android && !e2e_testing
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// PinThreadToCPU restricts the calling OS thread to the given CPU via
|
||||
// sched_setaffinity(2). Combined with runtime.LockOSThread on the
|
||||
// goroutine, this prevents the kernel from migrating us across CPUs and
|
||||
// in turn keeps every UDP send from this goroutine going through the
|
||||
// same XPS-selected TX ring, eliminating the wire-side reorder that
|
||||
// otherwise fragments one nebula flow across multiple rings.
|
||||
func PinThreadToCPU(cpu int) error {
|
||||
runtime.LockOSThread()
|
||||
var set unix.CPUSet
|
||||
set.Zero()
|
||||
set.Set(cpu)
|
||||
return unix.SchedSetaffinity(0, &set)
|
||||
}
|
||||
|
||||
// AllowedCPUs returns the CPU IDs the calling process is currently allowed to
|
||||
// run on, as reported by sched_getaffinity(2). Under a cgroup cpuset or a
|
||||
// `taskset` mask the allowed IDs are frequently not the contiguous range
|
||||
// 0..NumCPU-1 (e.g. pinned to CPUs 4-7: NumCPU reports 4 while the valid IDs
|
||||
// are 4,5,6,7). Callers that need a real CPU to pin to must choose from this
|
||||
// set rather than assuming i % NumCPU is runnable, or every pin fails.
|
||||
func AllowedCPUs() ([]int, error) {
|
||||
var set unix.CPUSet
|
||||
if err := unix.SchedGetaffinity(0, &set); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cpus := make([]int, 0, set.Count())
|
||||
for cpu := 0; cpu < len(set)*64; cpu++ {
|
||||
if set.IsSet(cpu) {
|
||||
cpus = append(cpus, cpu)
|
||||
}
|
||||
}
|
||||
return cpus, nil
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//go:build !linux || android || e2e_testing
|
||||
|
||||
package util
|
||||
|
||||
// PinThreadToCPU is a no-op outside Linux: only Linux exposes a stable
|
||||
// per-thread CPU affinity API and only Linux has XPS-driven TX ring
|
||||
// selection in the first place. On every other platform there's nothing
|
||||
// to fix here.
|
||||
func PinThreadToCPU(_ int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// AllowedCPUs has no meaningful answer off Linux (no sched_getaffinity), so it
|
||||
// reports "unknown" by returning a nil slice and nil error. Callers treat an
|
||||
// empty result as "fall back to the default CPU choice".
|
||||
func AllowedCPUs() ([]int, error) {
|
||||
return nil, nil
|
||||
}
|
||||
Reference in New Issue
Block a user