From 4b108e94958d42edefab2613ce0c88c8075725d6 Mon Sep 17 00:00:00 2001 From: curtis lee fulton Date: Wed, 2 Sep 2026 18:45:58 -0700 Subject: [PATCH] feat: otel metrics publishing to otlp #89 --- .github/workflows/ci-publish.yml | 223 ++- .github/workflows/ci-tests.yml | 3 +- .github/workflows/release-branch.yml | 104 ++ .github/workflows/rust-tests-workflow.yml | 22 +- Cargo.lock | 71 +- Cargo.toml | 54 +- Dockerfile | 6 +- README.md | 428 ++++-- ci/Dockerfile | 4 + components/Cargo.toml | 2 + components/src/discovery/db.rs | 118 +- components/src/discovery/http.rs | 115 +- components/src/discovery/memory.rs | 6 + components/src/lib.rs | 1 + components/src/metrics/db.rs | 379 +++++ components/src/metrics/grpc.rs | 73 + components/src/metrics/http.rs | 151 ++ components/src/metrics/mod.rs | 21 + components/src/offer/db.rs | 211 ++- components/src/offer/http.rs | 195 ++- components/src/offer/memory.rs | 7 + components/src/pool/client_pool.rs | 4 - components/src/pool/cln/grpc/client.rs | 57 +- components/src/pool/error.rs | 11 +- components/src/pool/lnd/grpc/client.rs | 52 +- components/src/pool/mod.rs | 5 +- components/src/secrets/tls.rs | 16 +- metrics/Cargo.toml | 21 + metrics/src/lib.rs | 137 ++ metrics/src/macros.rs | 149 ++ metrics/src/value.rs | 340 +++++ .../tests/compile_fail/duration_counter.rs | 7 + .../compile_fail/duration_counter.stderr | 50 + .../tests/compile_fail/non_const_label_key.rs | 11 + .../compile_fail/non_const_label_key.stderr | 12 + metrics/tests/compile_fail/non_numeric.rs | 5 + metrics/tests/compile_fail/non_numeric.stderr | 46 + .../compile_fail/signed_where_unsigned.rs | 7 + .../compile_fail/signed_where_unsigned.stderr | 89 ++ .../tests/compile_fail/unsigned_counter.rs | 5 + .../compile_fail/unsigned_counter.stderr | 50 + metrics/tests/metrics.rs | 610 ++++++++ metrics/tests/ui.rs | 14 + pingora/src/balance.rs | 2 - server/Cargo.toml | 6 +- server/config/discovery-standalone.yaml | 21 +- server/config/lnurl-standalone.yaml | 21 +- server/config/memory-basic-no-tls.yaml | 69 +- server/config/memory-basic.yaml | 69 +- server/config/offers-standalone.yaml | 21 +- server/config/persistence-with-secrets.yaml | 63 +- server/config/persistence.yaml | 69 +- server/src/commands/discovery/backend.rs | 2 +- server/src/commands/error.rs | 12 +- server/src/commands/offer/metadata.rs | 2 +- server/src/commands/offer/record.rs | 2 +- server/src/commands/services.rs | 23 +- server/src/config.rs | 45 +- server/src/di/error.rs | 57 + .../di/inject/injectors/service/balance.rs | 28 +- .../di/inject/injectors/service/discovery.rs | 28 +- server/src/di/inject/injectors/service/mod.rs | 2 + .../src/di/inject/injectors/service/offer.rs | 28 +- .../src/di/inject/injectors/service/otlp.rs | 208 +++ .../di/inject/injectors/service/subscriber.rs | 305 ++++ .../di/inject/injectors/service/tracing.rs | 204 +-- server/src/error.rs | 50 +- server/src/level.rs | 160 +++ server/src/lib.rs | 1 + server/src/main.rs | 35 +- server/tests/features/cli_offer_manage.rs | 31 +- server/tests/features/common/context/cli.rs | 4 +- .../tests/features/common/context/global.rs | 22 +- .../tests/features/common/context/server.rs | 48 +- .../tests/features/common/otlp_collector.rs | 36 +- .../tests/features/common/step_functions.rs | 362 ++++- server/tests/features/main.rs | 3 + .../otlp-discovery-store-metrics.feature | 118 ++ server/tests/features/otlp-metrics.feature | 223 +++ .../features/otlp-offer-store-metrics.feature | 152 ++ .../features/otlp_discovery_store_metrics.rs | 199 +++ server/tests/features/otlp_metrics.rs | 560 ++++++++ .../features/otlp_offer_store_metrics.rs | 235 ++++ server/tests/features/otlp_spans.rs | 4 +- server/tests/features/service_logs.rs | 32 +- ...ans__otlp_spans__dead_offer_store_502.snap | 30 +- ...pans__otlp_spans__error_get_offer_404.snap | 6 +- ...lp_spans__otlp_spans__success_invoice.snap | 302 +--- service/src/axum/extract/host.rs | 2 +- service/src/discovery/service.rs | 1 + service/src/lnurl/pay/handler.rs | 3 - service/src/lnurl/service.rs | 1 + service/src/offer/service.rs | 1 + {switchgear/src => src}/lib.rs | 0 switchgear/Cargo.toml | 17 - switchgear/README.md | 1237 ----------------- testing/Cargo.toml | 2 + testing/README.md | 15 +- testing/docker-compose.yml | 33 + testing/otel-collector/entrypoint.sh | 1 + .../otel-collector/otel-collector-config.yml | 21 +- testing/setup/Dockerfile | 2 +- testing/setup/credentials.sh | 8 + testing/src/credentials/influx.rs | 53 + testing/src/credentials/mod.rs | 1 + testing/src/influx.rs | 103 ++ testing/src/lib.rs | 1 + testing/src/services.rs | 24 + testing/testing.env | 5 +- 109 files changed, 6915 insertions(+), 2408 deletions(-) create mode 100644 .github/workflows/release-branch.yml create mode 100644 components/src/metrics/db.rs create mode 100644 components/src/metrics/grpc.rs create mode 100644 components/src/metrics/http.rs create mode 100644 components/src/metrics/mod.rs create mode 100644 metrics/Cargo.toml create mode 100644 metrics/src/lib.rs create mode 100644 metrics/src/macros.rs create mode 100644 metrics/src/value.rs create mode 100644 metrics/tests/compile_fail/duration_counter.rs create mode 100644 metrics/tests/compile_fail/duration_counter.stderr create mode 100644 metrics/tests/compile_fail/non_const_label_key.rs create mode 100644 metrics/tests/compile_fail/non_const_label_key.stderr create mode 100644 metrics/tests/compile_fail/non_numeric.rs create mode 100644 metrics/tests/compile_fail/non_numeric.stderr create mode 100644 metrics/tests/compile_fail/signed_where_unsigned.rs create mode 100644 metrics/tests/compile_fail/signed_where_unsigned.stderr create mode 100644 metrics/tests/compile_fail/unsigned_counter.rs create mode 100644 metrics/tests/compile_fail/unsigned_counter.stderr create mode 100644 metrics/tests/metrics.rs create mode 100644 metrics/tests/ui.rs create mode 100644 server/src/di/inject/injectors/service/otlp.rs create mode 100644 server/src/di/inject/injectors/service/subscriber.rs create mode 100644 server/src/level.rs create mode 100644 server/tests/features/otlp-discovery-store-metrics.feature create mode 100644 server/tests/features/otlp-metrics.feature create mode 100644 server/tests/features/otlp-offer-store-metrics.feature create mode 100644 server/tests/features/otlp_discovery_store_metrics.rs create mode 100644 server/tests/features/otlp_metrics.rs create mode 100644 server/tests/features/otlp_offer_store_metrics.rs rename {switchgear/src => src}/lib.rs (100%) delete mode 100644 switchgear/Cargo.toml delete mode 100644 switchgear/README.md create mode 100644 testing/src/credentials/influx.rs create mode 100644 testing/src/influx.rs diff --git a/.github/workflows/ci-publish.yml b/.github/workflows/ci-publish.yml index 1ff8e0c..8afd43c 100644 --- a/.github/workflows/ci-publish.yml +++ b/.github/workflows/ci-publish.yml @@ -1,38 +1,141 @@ name: CI Publish on: - push: - tags: [ 'v*.*.*' ] + workflow_dispatch: + inputs: + sha: + description: 'Commit sha to release (must be an ancestor of main for a live run)' + required: true + type: string + dry_run: + description: 'Dry run: build and verify without pushing or publishing' + required: false + default: true + type: boolean + +concurrency: + group: publish + cancel-in-progress: false + +permissions: + contents: read + packages: read env: IMAGE_NAME: docker.io/bitshock/switchgear jobs: + preflight: + name: Preflight + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}/ci-rust:1.97.1 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + outputs: + version: ${{ steps.version.outputs.version }} + dry_run: ${{ steps.version.outputs.dry_run }} + have_docker: ${{ steps.creds.outputs.have_docker }} + have_crates: ${{ steps.creds.outputs.have_crates }} + + steps: + - name: Checkout release commit + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + fetch-depth: 0 + + - name: Configure git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Resolve workspace version + id: version + env: + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + VERSION=$(cargo metadata --format-version 1 --no-deps | jq -r '.packages[0].version') + if [ -z "$VERSION" ] || [ "$VERSION" = "null" ]; then + echo "::error::could not resolve workspace version from cargo metadata" + exit 1 + fi + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + echo "dry_run=${DRY_RUN}" >> "$GITHUB_OUTPUT" + + - name: Detect credentials + id: creds + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: | + set -euo pipefail + if [ -n "${DOCKERHUB_TOKEN:-}" ] && [ -n "${DOCKERHUB_USERNAME:-}" ]; then + echo "have_docker=true" >> "$GITHUB_OUTPUT" + else + echo "have_docker=false" >> "$GITHUB_OUTPUT" + fi + if [ -n "${CARGO_REGISTRY_TOKEN:-}" ]; then + echo "have_crates=true" >> "$GITHUB_OUTPUT" + else + echo "have_crates=false" >> "$GITHUB_OUTPUT" + fi + + - name: Assert sha is an ancestor of main + if: ${{ !inputs.dry_run }} + env: + SHA: ${{ inputs.sha }} + run: | + set -euo pipefail + git fetch --no-tags origin main + if ! git merge-base --is-ancestor "$SHA" FETCH_HEAD; then + echo "::error::$SHA is not an ancestor of main" + exit 1 + fi + echo "$SHA is an ancestor of main" + + - name: Summary + env: + VERSION: ${{ steps.version.outputs.version }} + DRY_RUN: ${{ steps.version.outputs.dry_run }} + HAVE_DOCKER: ${{ steps.creds.outputs.have_docker }} + HAVE_CRATES: ${{ steps.creds.outputs.have_crates }} + SHA: ${{ inputs.sha }} + run: | + set -euo pipefail + { + echo "### Publish \`$VERSION\`" + echo + echo "| | |" + echo "|---|---|" + echo "| sha | \`$SHA\` |" + echo "| dry run | \`$DRY_RUN\` |" + echo "| docker credentials | \`$HAVE_DOCKER\` |" + echo "| crates.io credentials | \`$HAVE_CRATES\` |" + } >> "$GITHUB_STEP_SUMMARY" + tests: name: Rust Tests For Publish uses: ./.github/workflows/rust-tests-workflow.yml -# tests: -# name: Call Rust Tests -# runs-on: ubuntu-latest -# steps: -# - name: Skip tests (temporary) -# run: echo "Tests skipped for debugging" - docker: name: Docker Publish - needs: tests + needs: [ preflight, tests ] runs-on: ubuntu-latest - if: vars.PUBLISH_DRY_RUN == 'true' || github.repository == 'bitshock-src/switchgear' + steps: - - name: Checkout repository + - name: Checkout release commit uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Log in to Docker Hub - if: vars.PUBLISH_DRY_RUN != 'true' + if: needs.preflight.outputs.have_docker == 'true' && needs.preflight.outputs.dry_run == 'false' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -44,14 +147,25 @@ jobs: with: images: ${{ env.IMAGE_NAME }} tags: | - type=semver,pattern={{version}} + type=raw,value=${{ needs.preflight.outputs.version }} + type=raw,value=latest - - name: Log dry run - if: vars.PUBLISH_DRY_RUN == 'true' + - name: Log skipped push + if: ${{ !(needs.preflight.outputs.have_docker == 'true' && needs.preflight.outputs.dry_run == 'false') }} + env: + TAGS: ${{ steps.meta.outputs.tags }} + HAVE_DOCKER: ${{ needs.preflight.outputs.have_docker }} + DRY_RUN: ${{ needs.preflight.outputs.dry_run }} run: | - echo "== DOCKER PUSH DRY RUN ==" - echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n' | while read -r tag; do - echo " - $tag" + set -euo pipefail + if [ "$DRY_RUN" != "true" ] && [ "$HAVE_DOCKER" != "true" ]; then + echo "::warning::DOCKERHUB_USERNAME/DOCKERHUB_TOKEN are not set, building without pushing" + fi + echo "== DOCKER PUSH SKIPPED ==" + printf '%s\n' "$TAGS" | while read -r tag; do + if [ -n "$tag" ]; then + echo " - $tag" + fi done - name: Build and push multi-arch image @@ -59,30 +173,73 @@ jobs: with: context: . platforms: linux/amd64,linux/arm64 - push: ${{ vars.PUBLISH_DRY_RUN != 'true' }} + push: ${{ needs.preflight.outputs.have_docker == 'true' && needs.preflight.outputs.dry_run == 'false' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - cargo: name: Cargo Publish - needs: tests + needs: [ preflight, docker ] runs-on: ubuntu-latest + environment: publish container: - image: ghcr.io/bitshock-src/switchgear/ci-rust:1.0.0 + image: ghcr.io/${{ github.repository }}/ci-rust:1.97.1 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - if: vars.PUBLISH_DRY_RUN == 'true' || github.repository == 'bitshock-src/switchgear' steps: - - name: Checkout repository + - name: Checkout release commit uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} - - name: Publish dry run - if: vars.PUBLISH_DRY_RUN == 'true' - run: cargo publish --dry-run - - - name: Publish crates.io - if: vars.PUBLISH_DRY_RUN != 'true' - run: cargo publish + - name: Publish to crates.io env: + HAVE_CRATES: ${{ needs.preflight.outputs.have_crates }} + DRY_RUN: ${{ needs.preflight.outputs.dry_run }} CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: | + set -euo pipefail + if [ "$HAVE_CRATES" = "true" ] && [ "$DRY_RUN" = "false" ]; then + cargo publish --workspace + else + if [ "$DRY_RUN" != "true" ] && [ "$HAVE_CRATES" != "true" ]; then + echo "::warning::CARGO_REGISTRY_TOKEN is not set, running a dry run instead" + fi + cargo publish --workspace --dry-run + fi + + git-tag: + name: Git Tag + needs: [ preflight, cargo ] + runs-on: ubuntu-latest + if: needs.preflight.outputs.dry_run == 'false' + permissions: + contents: write + + steps: + - name: Checkout release commit + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + fetch-depth: 0 + + - name: Tag release + env: + VERSION: ${{ needs.preflight.outputs.version }} + SHA: ${{ inputs.sha }} + run: | + set -euo pipefail + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + + if git ls-remote --exit-code --tags origin "v$VERSION" > /dev/null 2>&1; then + echo "::error::tag v$VERSION already exists on origin" + exit 1 + fi + + git tag "v$VERSION" "$SHA" + git push origin "v$VERSION" + + echo "### Tagged \`v$VERSION\` at \`$SHA\`" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 4828790..0fd2ea9 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -16,9 +16,10 @@ on: permissions: contents: read + packages: read statuses: write jobs: tests: name: Rust Tests For Branch Push - uses: ./.github/workflows/rust-tests-workflow.yml \ No newline at end of file + uses: ./.github/workflows/rust-tests-workflow.yml diff --git a/.github/workflows/release-branch.yml b/.github/workflows/release-branch.yml new file mode 100644 index 0000000..132edea --- /dev/null +++ b/.github/workflows/release-branch.yml @@ -0,0 +1,104 @@ +name: Release Branch + +on: + workflow_dispatch: + inputs: + version: + description: 'Explicit semver (n.n.n). Leave blank to bump by level.' + required: false + type: string + level: + description: 'Bump level, used when version is blank' + required: false + default: patch + type: choice + options: + - patch + - minor + - major + +permissions: + contents: write + packages: read + +jobs: + release-branch: + name: Create Release Branch + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}/ci-rust:1.97.1 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + + - name: Configure git + run: | + set -euo pipefail + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Bump workspace version + id: bump + env: + VERSION: ${{ inputs.version }} + LEVEL: ${{ inputs.level }} + run: | + set -euo pipefail + if [ -n "$VERSION" ]; then + cargo set-version --workspace "$VERSION" + else + cargo set-version --workspace --bump "$LEVEL" + fi + cargo build + NEW=$(cargo metadata --format-version 1 --no-deps | jq -r '.packages[0].version') + echo "version=${NEW}" >> "$GITHUB_OUTPUT" + echo "branch=release-${NEW}" >> "$GITHUB_OUTPUT" + + - name: Create release branch + env: + NEW: ${{ steps.bump.outputs.version }} + BRANCH: ${{ steps.bump.outputs.branch }} + run: | + set -euo pipefail + + if git ls-remote --exit-code --heads origin "$BRANCH" > /dev/null 2>&1; then + echo "::error::branch $BRANCH already exists on origin" + exit 1 + fi + + if git diff --quiet -- Cargo.lock '*Cargo.toml'; then + echo "::error::version is already $NEW, nothing to commit" + exit 1 + fi + + git checkout -b "$BRANCH" + git add -- Cargo.lock '*Cargo.toml' + git commit -m "release $NEW" + git push origin "$BRANCH" + + - name: Summary + env: + NEW: ${{ steps.bump.outputs.version }} + BRANCH: ${{ steps.bump.outputs.branch }} + run: | + set -euo pipefail + { + echo "### Release branch \`$BRANCH\`" + echo + echo "Version \`$NEW\` off \`main\`." + echo + echo "[Open the pull request]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/compare/main...$BRANCH?expand=1) to trigger CI on it." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/rust-tests-workflow.yml b/.github/workflows/rust-tests-workflow.yml index f899485..37204b2 100644 --- a/.github/workflows/rust-tests-workflow.yml +++ b/.github/workflows/rust-tests-workflow.yml @@ -6,13 +6,19 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + CARGO_INCREMENTAL: 0 + CARGO_PROFILE_DEV_DEBUG: line-tables-only + CARGO_PROFILE_TEST_DEBUG: line-tables-only jobs: tests: name: Format, Clippy, Tests runs-on: ubuntu-latest container: - image: ghcr.io/bitshock-src/switchgear/ci-rust:1.0.0 + image: ghcr.io/${{ github.repository }}/ci-rust:1.97.1 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: -v /var/run/docker.sock:/var/run/docker.sock volumes: - /var/run/docker.sock:/var/run/docker.sock @@ -26,14 +32,20 @@ jobs: - name: Start Docker Compose services run: | cd testing - nohup /bin/sh -c 'docker compose --env-file ./testing.env up -d --build --wait; touch ./compose-ready' > compose.log 2>&1 & + date +%s > ./compose-start + nohup /bin/sh -c 'docker compose --env-file ./testing.env up -d --build --wait; date +%s > ./compose-ready' > compose.log 2>&1 & cd .. - name: Rust cache uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true - - name: Run clippy - run: cargo clippy -- -D warnings + - name: Run clippy for panics + run: cargo clippy -- -D warnings -D clippy::unwrap_used -D clippy::expect_used -D clippy::panic + + - name: Run clippy on tests + run: cargo clippy --tests -- -D warnings - name: Compile tests run: | @@ -57,6 +69,8 @@ jobs: exit 1 fi + echo "compose services ready after $(( $(cat ./compose-ready) - $(cat ./compose-start) ))s" + . ./testing.env && docker network connect $SERVICES_NETWORK_NAME $(hostname) cd .. diff --git a/Cargo.lock b/Cargo.lock index 6568ca7..7c775bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5127,11 +5127,11 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "switchgear" -version = "0.1.27" +version = "0.1.28" [[package]] name = "switchgear-components" -version = "0.1.27" +version = "0.1.28" dependencies = [ "anyhow", "async-trait", @@ -5148,6 +5148,7 @@ dependencies = [ "lightning-invoice", "moka", "opentelemetry", + "opentelemetry-semantic-conventions", "p256", "pkcs8", "prost", @@ -5164,6 +5165,7 @@ dependencies = [ "sqlx", "sqlx-core", "switchgear-error", + "switchgear-metrics", "switchgear-migration", "switchgear-service-api", "switchgear-testing", @@ -5181,14 +5183,23 @@ dependencies = [ [[package]] name = "switchgear-error" -version = "0.1.27" +version = "0.1.28" +dependencies = [ + "tracing", +] + +[[package]] +name = "switchgear-metrics" +version = "0.1.28" dependencies = [ "tracing", + "tracing-subscriber", + "trybuild", ] [[package]] name = "switchgear-migration" -version = "0.1.27" +version = "0.1.28" dependencies = [ "sea-orm-migration", "tokio", @@ -5196,7 +5207,7 @@ dependencies = [ [[package]] name = "switchgear-pingora" -version = "0.1.27" +version = "0.1.28" dependencies = [ "arc-swap", "async-trait", @@ -5218,7 +5229,7 @@ dependencies = [ [[package]] name = "switchgear-server" -version = "0.1.27" +version = "0.1.28" dependencies = [ "anyhow", "async-trait", @@ -5234,7 +5245,6 @@ dependencies = [ "jemallocator", "jsonwebtoken", "lightning-invoice", - "log", "opentelemetry", "opentelemetry-otlp", "opentelemetry-proto", @@ -5259,6 +5269,7 @@ dependencies = [ "signal-hook-tokio", "switchgear-components", "switchgear-error", + "switchgear-metrics", "switchgear-pingora", "switchgear-service", "switchgear-service-api", @@ -5267,7 +5278,7 @@ dependencies = [ "tempfile", "tokio", "tokio-stream", - "toml", + "toml 0.9.12+spec-1.1.0", "tonic", "tracing", "tracing-ecs-formatter", @@ -5279,7 +5290,7 @@ dependencies = [ [[package]] name = "switchgear-service" -version = "0.1.27" +version = "0.1.28" dependencies = [ "async-trait", "axum", @@ -5314,7 +5325,7 @@ dependencies = [ [[package]] name = "switchgear-service-api" -version = "0.1.27" +version = "0.1.28" dependencies = [ "async-trait", "axum", @@ -5336,7 +5347,7 @@ dependencies = [ [[package]] name = "switchgear-testing" -version = "0.1.27" +version = "0.1.28" dependencies = [ "anyhow", "dotenvy", @@ -5345,7 +5356,9 @@ dependencies = [ "hex", "indexmap 2.14.0", "rand 0.9.5", + "reqwest", "secp256k1 0.31.1", + "serde_json", "sqlx", "tar", "tempfile", @@ -5443,6 +5456,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-triple" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a6bfce3d99adfa72d24750a61f782f3036a81e7f86d8841ee1326deaebd171" + [[package]] name = "tempfile" version = "3.27.0" @@ -5673,6 +5692,21 @@ dependencies = [ "winnow 0.7.15", ] +[[package]] +name = "toml" +version = "1.1.4+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +dependencies = [ + "indexmap 2.14.0", + "serde_core", + "serde_spanned", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.4", +] + [[package]] name = "toml_datetime" version = "0.7.5+spec-1.1.0" @@ -5960,6 +5994,21 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "trybuild" +version = "1.0.120" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e605bf6b39357663d8ba4e984f8be8da8df6bb32e81031d6889024ea8fd68e4" +dependencies = [ + "glob", + "serde", + "serde_derive", + "serde_json", + "target-triple", + "termcolor", + "toml 1.1.4+spec-1.1.0", +] + [[package]] name = "typeid" version = "1.0.3" diff --git a/Cargo.toml b/Cargo.toml index 1f18c49..bc83620 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,31 +1,67 @@ [workspace] members = [ + ".", "components", "error", + "metrics", + "migration", + "pingora", + "server", + "service", + "service-api", + "testing" +] + +# Required: the workspace root is now a package, so bare `cargo build`/`test`/`clippy` +# would otherwise select only it. This restores full-workspace selection. +default-members = [ + ".", + "components", + "error", + "metrics", "migration", "pingora", "server", "service", "service-api", - "switchgear", "testing" ] resolver = "2" +[package] +name = "switchgear" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors = ["Bitshock "] +description = "High availability LNURL load balancer server for enterprise Bitcoin Lightning payment providers" +documentation = "https://github.com/bitshock-src/switchgear" +homepage = "https://bitshock.com" +repository = "https://github.com/bitshock-src/switchgear" +license = "Apache-2.0" +keywords = ["bitcoin", "lightning", "lnurl", "load-balancer", "payments"] +categories = ["command-line-utilities", "network-programming", "web-programming"] +readme = "README.md" +publish = true +include = ["src/**/*", "README.md", "LICENSE"] + +[dependencies] + [workspace.package] -version = "0.1.27" +version = "0.1.28" edition = "2024" rust-version = "1.97" [workspace.dependencies] -switchgear-components = { version = "0.1.26", path = "components" } -switchgear-error = { version = "0.1.26", path = "error" } -switchgear-migration = { version = "0.1.26", path = "migration" } -switchgear-pingora = { version = "0.1.26", path = "pingora" } -switchgear-service = { version = "0.1.26", path = "service" } -switchgear-service-api = { version = "0.1.26", path = "service-api" } -switchgear-testing = { version = "0.1.26", path = "testing" } +switchgear-components = { version = "0.1.28", path = "components" } +switchgear-error = { version = "0.1.28", path = "error" } +switchgear-metrics = { version = "0.1.28", path = "metrics" } +switchgear-migration = { version = "0.1.28", path = "migration" } +switchgear-pingora = { version = "0.1.28", path = "pingora" } +switchgear-service = { version = "0.1.28", path = "service" } +switchgear-service-api = { version = "0.1.28", path = "service-api" } +switchgear-testing = { version = "0.1.28", path = "testing" } secrecy = { version = "0.10", features = ["serde"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "registry", "tracing-log"] } diff --git a/Dockerfile b/Dockerfile index ccad3cb..32f8f6a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,14 +24,16 @@ COPY components/build.rs ./components/build.rs COPY error/src ./error/src COPY error/Cargo.toml ./error/Cargo.toml +COPY metrics/src ./metrics/src +COPY metrics/Cargo.toml ./metrics/Cargo.toml + COPY pingora/src ./pingora/src COPY pingora/Cargo.toml ./pingora/Cargo.toml COPY migration/src ./migration/src COPY migration/Cargo.toml ./migration/Cargo.toml -COPY switchgear/src ./switchgear/src -COPY switchgear/Cargo.toml ./switchgear/Cargo.toml +COPY src ./src COPY testing/Cargo.toml ./testing/Cargo-src.toml diff --git a/README.md b/README.md index 5638cb9..cdc18e9 100644 --- a/README.md +++ b/README.md @@ -16,104 +16,112 @@ scale massive multi-region Lightning Node fleets at five nines uptime. ## Table of Contents - [User Guide](#user-guide) - - [Learn](#learn) - - [Features](#features) - - [Why Bitcoin Lightning Payments Fail](#why-bitcoin-lightning-payments-fail) - - [Single Points Of Failure](#single-points-of-failure) - - [No Liquidity Bias](#no-liquidity-bias) - - [External Load Balancer Resource Exhaustion](#external-load-balancer-resource-exhaustion) - - [Feature Reference](#feature-reference) - - [Liquidity Bias](#liquidity-bias) - - [Negative Capacity Bias example (restrictive):](#negative-capacity-bias-example-restrictive) - - [Positive Capacity Bias example (permissive):](#positive-capacity-bias-example-permissive) - - [Inbound Capacity Measurement](#inbound-capacity-measurement) - - [Partitioning](#partitioning) - - [Balancing Switchgear](#balancing-switchgear) - - [LNURL Service](#lnurl-service) - - [Consistent Backend-Selection](#consistent-backend-selection) - - [Discovery Service](#discovery-service) - - [Offer Service](#offer-service) + - [Learn](#learn) + - [Features](#features) + - [Why Bitcoin Lightning Payments Fail](#why-bitcoin-lightning-payments-fail) + - [Single Points Of Failure](#single-points-of-failure) + - [No Liquidity Bias](#no-liquidity-bias) + - [External Load Balancer Resource Exhaustion](#external-load-balancer-resource-exhaustion) + - [Feature Reference](#feature-reference) + - [Liquidity Bias](#liquidity-bias) + - [Negative Capacity Bias example (restrictive):](#negative-capacity-bias-example-restrictive) + - [Positive Capacity Bias example (permissive):](#positive-capacity-bias-example-permissive) + - [Inbound Capacity Measurement](#inbound-capacity-measurement) + - [Partitioning](#partitioning) + - [Balancing Switchgear](#balancing-switchgear) + - [LNURL Service](#lnurl-service) + - [Consistent Backend-Selection](#consistent-backend-selection) + - [Discovery Service](#discovery-service) + - [Offer Service](#offer-service) - [Deployment](#deployment) - - [Getting Started](#getting-started) - - [Install](#install) - - [Host](#host) - - [Docker](#docker) - - [Starting Switchgear Services](#starting-switchgear-services) - - [Docker](#docker-1) - - [Administration](#administration) - - [REST Administration](#rest-administration) - - [CLI Administration](#cli-administration) - - [Docker](#docker-2) - - [Feature Reference](#feature-reference-1) - - [Configuring Switchgear Services](#configuring-switchgear-services) - - [Env Var Shell Expansion](#env-var-shell-expansion) - - [Secrets Expansion](#secrets-expansion) - - [LNURL Service Configuration](#lnurl-service-configuration) - - [Discovery Service Configuration](#discovery-service-configuration) - - [Authentication Setup](#authentication-setup) - - [Offer Service Configuration](#offer-service-configuration) - - [Authentication Setup](#authentication-setup-1) - - [Persistence](#persistence) - - [Common Storage Types](#common-storage-types) - - [Database Storage (SQLite/MySQL/PostgreSQL)](#database-storage-sqlitemysqlpostgresql) - - [HTTP Storage (Remote Service)](#http-storage-remote-service) - - [In-memory Storage](#in-memory-storage) - - [Configuration Examples](#configuration-examples) - - [Using Same Database for Both Stores](#using-same-database-for-both-stores) - - [Mixed Storage Types](#mixed-storage-types) - - [Remote Service Configuration](#remote-service-configuration) - - [Database Connection URLs](#database-connection-urls) - - [Sqlite](#sqlite) - - [MySQL](#mysql) - - [Postgres](#postgres) - - [Observability](#observability) - - [Logging](#logging) - - [Service Identification](#service-identification) - - [Access Logs](#access-logs) - - [Error Logs](#error-logs) - - [Log-to-Trace Correlation](#log-to-trace-correlation) - - [Level Filtering](#level-filtering) - - [CLI Output](#cli-output) - - [OTLP Tracing](#otlp-tracing) + - [Getting Started](#getting-started) + - [Install](#install) + - [Host](#host) + - [Docker](#docker) + - [Starting Switchgear Services](#starting-switchgear-services) + - [Docker](#docker-1) + - [Administration](#administration) + - [REST Administration](#rest-administration) + - [CLI Administration](#cli-administration) + - [Docker](#docker-2) + - [Feature Reference](#feature-reference-1) + - [Configuring Switchgear Services](#configuring-switchgear-services) + - [Env Var Shell Expansion](#env-var-shell-expansion) + - [Secrets Expansion](#secrets-expansion) + - [LNURL Service Configuration](#lnurl-service-configuration) + - [Discovery Service Configuration](#discovery-service-configuration) + - [Authentication Setup](#authentication-setup) + - [Offer Service Configuration](#offer-service-configuration) + - [Authentication Setup](#authentication-setup-1) + - [Persistence](#persistence) + - [Common Storage Types](#common-storage-types) + - [Database Storage (SQLite/MySQL/PostgreSQL)](#database-storage-sqlitemysqlpostgresql) + - [HTTP Storage (Remote Service)](#http-storage-remote-service) + - [In-memory Storage](#in-memory-storage) + - [Configuration Examples](#configuration-examples) + - [Using Same Database for Both Stores](#using-same-database-for-both-stores) + - [Mixed Storage Types](#mixed-storage-types) + - [Remote Service Configuration](#remote-service-configuration) + - [Database Connection URLs](#database-connection-urls) + - [Sqlite](#sqlite) + - [MySQL](#mysql) + - [Postgres](#postgres) + - [Observability](#observability) + - [Logging](#logging) + - [Service Identification](#service-identification) + - [Access Logs](#access-logs) + - [Error Logs](#error-logs) + - [Log-to-Trace Correlation](#log-to-trace-correlation) + - [Level Filtering](#level-filtering) + - [CLI Output](#cli-output) + - [OTLP Tracing](#otlp-tracing) + - [OTLP Metrics](#otlp-metrics) + - [Enabling And Disabling](#enabling-and-disabling) + - [Metric Level](#metric-level) + - [Temporality](#temporality) + - [Emitted Metrics](#emitted-metrics) + - [`db.client.operation.duration`](#dbclientoperationduration) + - [`http.client.request.duration`](#httpclientrequestduration) + - [`rpc.client.call.duration`](#rpcclientcallduration) - [Developer](#developer) - - [API Reference](#api-reference) - - [Manage Lightning Node Backends With Discovery Service](#manage-lightning-node-backends-with-discovery-service) - - [REST API](#rest-api) - - [Authentication](#authentication) - - [Register A New Backend](#register-a-new-backend) - - [List All Backends](#list-all-backends) - - [Get A Specific Backend](#get-a-specific-backend) - - [Update A Backend](#update-a-backend) - - [Delete A Backend](#delete-a-backend) - - [Health Check](#health-check) - - [CLI](#cli) - - [Token Management](#token-management) - - [Backend Management](#backend-management) - - [Discovery Data Model](#discovery-data-model) - - [Manage LNURLs With Offer Service](#manage-lnurls-with-offer-service) - - [LNURL](#lnurl) - - [REST API](#rest-api-1) - - [Authentication](#authentication-1) - - [Create A New Offer](#create-a-new-offer) - - [List All Offers In A Partition](#list-all-offers-in-a-partition) - - [Get A Specific Offer](#get-a-specific-offer) - - [Update An Offer](#update-an-offer) - - [Delete An Offer](#delete-an-offer) - - [Create Metadata](#create-metadata) - - [List All Metadata In A Partition](#list-all-metadata-in-a-partition) - - [Get Specific Metadata](#get-specific-metadata) - - [Update Metadata](#update-metadata) - - [Delete Metadata](#delete-metadata) - - [Health Check](#health-check-1) - - [CLI](#cli-1) - - [Token Management](#token-management-1) - - [Offer Management](#offer-management) - - [Metadata Management](#metadata-management) - - [Offer Data Model](#offer-data-model) - - [Image Support](#image-support) - - [Identifier Types](#identifier-types) + - [API Reference](#api-reference) + - [Manage Lightning Node Backends With Discovery Service](#manage-lightning-node-backends-with-discovery-service) + - [REST API](#rest-api) + - [Authentication](#authentication) + - [Register A New Backend](#register-a-new-backend) + - [List All Backends](#list-all-backends) + - [Get A Specific Backend](#get-a-specific-backend) + - [Update A Backend](#update-a-backend) + - [Delete A Backend](#delete-a-backend) + - [Health Check](#health-check) + - [CLI](#cli) + - [Token Management](#token-management) + - [Backend Management](#backend-management) + - [Discovery Data Model](#discovery-data-model) + - [Manage LNURLs With Offer Service](#manage-lnurls-with-offer-service) + - [LNURL](#lnurl) + - [REST API](#rest-api-1) + - [Authentication](#authentication-1) + - [Create A New Offer](#create-a-new-offer) + - [List All Offers In A Partition](#list-all-offers-in-a-partition) + - [Get A Specific Offer](#get-a-specific-offer) + - [Update An Offer](#update-an-offer) + - [Delete An Offer](#delete-an-offer) + - [Create Metadata](#create-metadata) + - [List All Metadata In A Partition](#list-all-metadata-in-a-partition) + - [Get Specific Metadata](#get-specific-metadata) + - [Update Metadata](#update-metadata) + - [Delete Metadata](#delete-metadata) + - [Health Check](#health-check-1) + - [CLI](#cli-1) + - [Token Management](#token-management-1) + - [Offer Management](#offer-management) + - [Metadata Management](#metadata-management) + - [Offer Data Model](#offer-data-model) + - [Image Support](#image-support) + - [Identifier Types](#identifier-types) - [Meta](#meta) - - [Status](#status) + - [Status](#status) @@ -615,8 +623,8 @@ lnurl-service: backend-selection: "round-robin" # For consistent hashing, specify max iterations (only used with "consistent") # backend-selection: - # type: "consistent" - # max-iterations: 10000 + # type: "consistent" + # max-iterations: 10000 # Optional: Bias factor for capacity-influenced selection # Negative values are restrictive: @@ -645,7 +653,10 @@ lnurl-service: # Optional: OTLP telemetry export otlp: - tracing: + # Required. Shared transport for every signal below: a signal that declares + # its own `export` block replaces this one in full (no field-level merge), + # and signals that do not share one gRPC connection. + export: # OTLP collector endpoint (gRPC) endpoint: "http://127.0.0.1:4317" # Name of the secret whose contents are sent as the bearer token @@ -654,6 +665,8 @@ lnurl-service: trusted-roots: "/etc/ssl/certs/otlp-ca.pem" # Optional: pinned collector address for the trusted-roots bundle trusted-root-address: "127.0.0.1:4317" + # Optional: per-request timeout on the gRPC channel + export-timeout-secs: 5.0 # Optional: mTLS client identity, referencing secret names client-identity: cert-secret: "OTEL_CLIENT_CERT" @@ -668,6 +681,19 @@ lnurl-service: path: "/etc/ssl/certs/otlp-client.pem" OTEL_CLIENT_KEY: path: "/etc/ssl/certs/otlp-client-key.pem" + # Present = span export is on. Omit the block to turn tracing off. + tracing: + # always-on | always-off | trace-id-ratio | parent-based-trace-id-ratio + # `parent-based-trace-id-ratio` with `ratio: 1.0` is the OTel default. + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + # Present = metric export is on. Omit the block to turn metrics off. + metrics: + # cumulative | delta | low-memory + temporality: cumulative + # trace | debug | info | warn | error | off. Optional; defaults to info. + level: info ``` #### Discovery Service Configuration @@ -692,7 +718,10 @@ discovery-service: # Optional: OTLP telemetry export otlp: - tracing: + # Required. Shared transport for every signal below: a signal that declares + # its own `export` block replaces this one in full (no field-level merge), + # and signals that do not share one gRPC connection. + export: # OTLP collector endpoint (gRPC) endpoint: "http://127.0.0.1:4317" # Name of the secret whose contents are sent as the bearer token @@ -701,6 +730,8 @@ discovery-service: trusted-roots: "/etc/ssl/certs/otlp-ca.pem" # Optional: pinned collector address for the trusted-roots bundle trusted-root-address: "127.0.0.1:4317" + # Optional: per-request timeout on the gRPC channel + export-timeout-secs: 5.0 # Optional: mTLS client identity, referencing secret names client-identity: cert-secret: "OTEL_CLIENT_CERT" @@ -715,6 +746,19 @@ discovery-service: path: "/etc/ssl/certs/otlp-client.pem" OTEL_CLIENT_KEY: path: "/etc/ssl/certs/otlp-client-key.pem" + # Present = span export is on. Omit the block to turn tracing off. + tracing: + # always-on | always-off | trace-id-ratio | parent-based-trace-id-ratio + # `parent-based-trace-id-ratio` with `ratio: 1.0` is the OTel default. + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + # Present = metric export is on. Omit the block to turn metrics off. + metrics: + # cumulative | delta | low-memory + temporality: cumulative + # trace | debug | info | warn | error | off. Optional; defaults to info. + level: info ``` ##### Authentication Setup @@ -757,7 +801,10 @@ offer-service: # Optional: OTLP telemetry export otlp: - tracing: + # Required. Shared transport for every signal below: a signal that declares + # its own `export` block replaces this one in full (no field-level merge), + # and signals that do not share one gRPC connection. + export: # OTLP collector endpoint (gRPC) endpoint: "http://127.0.0.1:4317" # Name of the secret whose contents are sent as the bearer token @@ -766,6 +813,8 @@ offer-service: trusted-roots: "/etc/ssl/certs/otlp-ca.pem" # Optional: pinned collector address for the trusted-roots bundle trusted-root-address: "127.0.0.1:4317" + # Optional: per-request timeout on the gRPC channel + export-timeout-secs: 5.0 # Optional: mTLS client identity, referencing secret names client-identity: cert-secret: "OTEL_CLIENT_CERT" @@ -780,6 +829,19 @@ offer-service: path: "/etc/ssl/certs/otlp-client.pem" OTEL_CLIENT_KEY: path: "/etc/ssl/certs/otlp-client-key.pem" + # Present = span export is on. Omit the block to turn tracing off. + tracing: + # always-on | always-off | trace-id-ratio | parent-based-trace-id-ratio + # `parent-based-trace-id-ratio` with `ratio: 1.0` is the OTel default. + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + # Present = metric export is on. Omit the block to turn metrics off. + metrics: + # cumulative | delta | low-memory + temporality: cumulative + # trace | debug | info | warn | error | off. Optional; defaults to info. + level: info ``` ##### Authentication Setup @@ -1127,20 +1189,164 @@ swgr --log-level debug discovery ls ##### OTLP Tracing Each service can export OpenTelemetry spans to an OTLP-compatible collector (Jaeger, Tempo, Elastic APM, Grafana Cloud, -etc.). Configure the exporter in the service block's `otlp.tracing` field. See -the [LNURL](#lnurl-service-configuration), [Discovery](#discovery-service-configuration), -and [Offer](#offer-service-configuration) service configuration samples above. +etc.). Presence of the `otlp.tracing` block turns span export on; the transport comes from `otlp.export` unless +`otlp.tracing.export` overrides it. See the [LNURL](#lnurl-service-configuration), +[Discovery](#discovery-service-configuration), and [Offer](#offer-service-configuration) service configuration samples +above. + +##### OTLP Metrics + +Each service can also export OpenTelemetry metrics over the same OTLP pipeline. Presence of the `otlp.metrics` block +turns metric export on; the transport comes from `otlp.export` unless `otlp.metrics.export` overrides it. + +```yaml + otlp: + export: + endpoint: "http://127.0.0.1:4317" + auth-token: "OTEL_AUTH_TOKEN" + # ... shared transport, as in the samples above + metrics: + temporality: cumulative +``` + +###### Enabling And Disabling + +Metric export is on when the service's `otlp.metrics` block is present. Setting the OTel-standard +`OTEL_METRICS_EXPORTER` environment variable to `none` forces it off regardless of configuration: + +```shell +OTEL_METRICS_EXPORTER=none swgr service --config ./config.yaml +``` + +`none` is the only value that changes anything: any other setting, and an unset variable, leave metrics on. Log records +and spans are unaffected either way. + +###### Metric Level + +`otlp.metrics.level` sets how much metric detail is recorded: + +```yaml + otlp: + metrics: + temporality: cumulative + level: debug +``` + +| Value | Recorded | +|------------------------|------------------------------------------------------------------------| +| `off`, `error`, `warn` | Nothing | +| `info` | The standard metric set. This is the default when the field is omitted | +| `debug` | The standard set, plus finer-grained per-operation detail | +| `trace` | Everything | + +Values are case-insensitive. + +Leave the level at `info` for normal running. Raise it to `debug` when you are investigating something and need +per-operation detail; expect more datapoints and higher tag cardinality at your metrics sink, so lower it again +afterward. + +Setting `off` stops metrics being recorded but leaves the exporter connected. To shut the pipeline down entirely, omit +the `otlp.metrics` block or set `OTEL_METRICS_EXPORTER=none` (see [Enabling And Disabling](#enabling-and-disabling) +above). + +[`RUST_LOG`](#level-filtering) has no effect on metrics — it governs log records only. `otlp.metrics.level` is the +only setting that changes metric verbosity. + +###### Temporality + +`otlp.metrics.temporality` selects how cumulative state is reported: + +| Value | Behaviour | +|--------------|-------------------------------------------------------------------------------| +| `cumulative` | Each export carries the running total since process start | +| `delta` | Each export carries only what changed since the previous one | +| `low-memory` | Delta for counters and histograms, cumulative for up-down counters and gauges | + +##### Emitted Metrics + +| Metric | Emitted when | +|--------------------------------------------------------------|-------------------------------------------------------------------------------------------| +| [`db.client.operation.duration`](#dbclientoperationduration) | A discovery or offer store is backed by a database | +| [`http.client.request.duration`](#httpclientrequestduration) | A discovery or offer store is backed by a remote service, see [Persistence](#persistence) | +| [`rpc.client.call.duration`](#rpcclientcallduration) | The LNURL service calls a CLN or LND node | + +Metric and attribute names follow the [OpenTelemetry semantic +conventions](https://opentelemetry.io/docs/specs/semconv/), so a collector, dashboard or alert built against those +conventions understands them without mapping. Three things hold across all three: + +- **Values are milliseconds, not seconds.** Switchgear records milliseconds. Divide by 1000 when comparing against a + dashboard built to the convention's unit. +- **Attributes that do not apply are absent, not empty.** Write queries to tolerate a missing attribute rather than an + empty one. +- **All three are recorded at `info`**, so they are exported at the default [metric level](#metric-level). + +###### `db.client.operation.duration` + +How long a database call took. + +Source: [Database client metrics](https://opentelemetry.io/docs/specs/semconv/database/database-metrics/) — histogram, +**stable**. + +| Attribute | Present | Value | +|----------------------------------------|-----------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `db.system.name` | Always | `sqlite`, `mysql` or `postgresql` | +| `db.namespace` \| `db.collection.name` | Always | The database and the table. For SQLite the database is the file name without its extension, or `:memory:` | +| `server.address` \| `server.port` | Except SQLite | Host and port of the database server. Never any credential from the connection URL | +| `error.type` | On failure only | `unique_constraint`, `foreign_key_constraint`, `constraint`, `connection_acquire`, `connection`, `statement`, `conversion`, or `_OTHER`. The same set whichever database you run | +| `db.response.status_code` | On failure only | The database's own error code, which differs between SQLite, MySQL and Postgres | +| `swgr.operation` | Always | The store operation, see below | + +`swgr.operation` is finer-grained than the table alone. The pairings are fixed: + +| `db.collection.name` | `swgr.operation` | +|--------------------------|---------------------------------------------------------------------------------------------------------------------| +| `offer_record` | `get_offer`, `get_offers`, `post_offer`, `put_offer_upsert`, `put_offer_fetch`, `delete_offer` | +| `offer_metadata` | `get_metadata`, `get_all_metadata`, `post_metadata`, `put_metadata_upsert`, `put_metadata_fetch`, `delete_metadata` | +| `discovery_backend` | `get`, `get_all_backends`, `post`, `put`, `patch`, `delete` | +| `discovery_backend_etag` | `get_all_etag` | + +###### `http.client.request.duration` + +How long a request to a remote discovery or offer service took. + +Source: [HTTP client metrics](https://opentelemetry.io/docs/specs/semconv/http/http-metrics/) — histogram, **stable**. + +| Attribute | Present | Value | +|-----------------------------------------|-------------------------|--------------------------------------------------------------------------------------------------------------| +| `http.request.method` \| `url.template` | Always | The method, and the route pattern rather than the requested path, so no identifier reaches your metrics sink | +| `server.address` \| `server.port` | Always | Host and port of the remote store | +| `http.response.status_code` | When a response arrived | The status, including a 4xx or 5xx | +| `error.type` | On failure only | The status, when one was returned, otherwise `timeout`, `connect`, `decode` or `request` | + +The route patterns are fixed: + +| Store | `url.template` | `http.request.method` | +|-----------|--------------------------------------------------------------------|-----------------------------------------| +| Offer | `/offers`, `/offers/{partition}`, `/offers/{partition}/{id}` | `GET`, `POST`, `PUT`, `DELETE` | +| Offer | `/metadata`, `/metadata/{partition}`, `/metadata/{partition}/{id}` | `GET`, `POST`, `PUT`, `DELETE` | +| Discovery | `/discovery`, `/discovery/{public_key}` | `GET`, `POST`, `PUT`, `PATCH`, `DELETE` | + +###### `rpc.client.call.duration` + +How long a gRPC call to a Lightning node took. + +Source: [RPC client metrics](https://opentelemetry.io/docs/specs/semconv/rpc/rpc-metrics/) — histogram, **release +candidate**. Older tooling may know this metric by its previous name, `rpc.client.duration`. -The exporter uses: +| Attribute | Present | Value | +|-----------------------------------|-----------------|-------------------------------------------------------------| +| `rpc.system.name` | Always | `grpc` | +| `rpc.method` | Always | The node call, see below | +| `server.address` \| `server.port` | Always | Host and port of the node | +| `rpc.response.status_code` | Always | The gRPC status, including `OK` on success | +| `error.type` | On failure only | The gRPC status, set only when the call did not return `OK` | -* gRPC transport (`opentelemetry-otlp` with tonic) -* Optional TLS with a custom trusted-root PEM bundle -* Optional pinned server name via `trusted-root-address` (for connecting to a collector by IP) -* Optional mTLS client identity via the `client-identity.cert-secret` / `key-secret` secret references -* Bearer authentication via the `auth-token` secret reference +The methods are fixed: -`service.name` on exported spans matches the service's log `service.name` (`swgr.lnurl`, `swgr.discovery`,`swgr.offer`), -and `service.version` matches `service.version` in logs, providing a single identity across logs and traces. +| Node | `rpc.method` | +|------|----------------------------------------------------------------| +| CLN | `cln.Node/Invoice`, `cln.Node/ListPeerChannels` | +| LND | `lnrpc.Lightning/AddInvoice`, `lnrpc.Lightning/ChannelBalance` | ## Developer diff --git a/ci/Dockerfile b/ci/Dockerfile index e3b52e6..a262b49 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update && \ curl \ gnupg \ iputils-ping \ + jq \ libclang-dev \ protobuf-compiler && \ install -m 0755 -d /etc/apt/keyrings && \ @@ -24,3 +25,6 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* RUN rustup component add rustfmt clippy + +RUN cargo install cargo-edit --locked && \ + rm -rf "$CARGO_HOME/registry" diff --git a/components/Cargo.toml b/components/Cargo.toml index 111a9a0..db475bd 100644 --- a/components/Cargo.toml +++ b/components/Cargo.toml @@ -18,6 +18,7 @@ async-trait = "0.1" axum = { version = "0.8", features = ["macros"] } axum-tracing-opentelemetry = { version = "0.38", features = ["tracing_level_info"] } opentelemetry = { version = "0.32", default-features = false, features = ["trace"] } +opentelemetry-semantic-conventions = { version = "0.32", features = ["semconv_experimental"] } tracing-opentelemetry = { version = "0.33", default-features = false } base64 = "0.22" chrono = { version = "0.4", features = ["serde"] } @@ -40,6 +41,7 @@ sha2 = "0.10" sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "tls-rustls-aws-lc-rs", "sqlite", "postgres", "mysql"] } sqlx-core = { version = "0.9", default-features = false, features = ["rustls-native-certs"] } switchgear-error.workspace = true +switchgear-metrics.workspace = true switchgear-migration.workspace = true switchgear-service-api.workspace = true tempfile = "3" diff --git a/components/src/discovery/db.rs b/components/src/discovery/db.rs index d2adcd8..4b2b059 100644 --- a/components/src/discovery/db.rs +++ b/components/src/discovery/db.rs @@ -1,4 +1,5 @@ use crate::discovery::error::DefaultDiscoveryBackendStoreError; +use crate::metrics::db::{DbTarget, record_db_operation}; use crate::secrets::SecretStore; use async_trait::async_trait; use chrono::Utc; @@ -15,7 +16,7 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; use std::str::FromStr; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant}; use switchgear_error::{ChainedContext, ErrorOrigin, ForeignContext}; use switchgear_migration::{DISCOVERY_BACKEND_GET_ALL_ETAG_ID, MigratorTrait}; use switchgear_service_api::discovery::{ @@ -207,11 +208,11 @@ impl CredentialRotation { #[derive(Clone, Debug)] pub struct DbDiscoveryBackendStore { db: DatabaseConnection, + db_target: DbTarget, rotation: Option, } impl DbDiscoveryBackendStore { - #[tracing::instrument(skip_all)] pub async fn connect( uri: &str, secrets: Option<&SecretStore>, @@ -234,14 +235,19 @@ impl DbDiscoveryBackendStore { ErrorOrigin::Internal, )?; + let db_target = crate::metrics::db::db_target(&db); + let rotation = secrets .map(|s| CredentialRotation::spawn(db.clone(), uri.to_owned(), s.clone(), resolved)) .transpose()?; - Ok(Self { db, rotation }) + Ok(Self { + db, + db_target, + rotation, + }) } - #[tracing::instrument(skip_all)] pub async fn migrate_up(&self) -> Result<(), DefaultDiscoveryBackendStoreError> { switchgear_migration::DiscoveryBackendMigrator::up(&self.db, None) .await @@ -249,7 +255,6 @@ impl DbDiscoveryBackendStore { Ok(()) } - #[tracing::instrument(skip_all)] pub async fn migrate_down(&self) -> Result<(), DefaultDiscoveryBackendStoreError> { switchgear_migration::DiscoveryBackendMigrator::down(&self.db, None) .await @@ -257,7 +262,6 @@ impl DbDiscoveryBackendStore { Ok(()) } - #[tracing::instrument(skip_all)] fn model_to_domain( model: Model, ) -> Result { @@ -283,13 +287,21 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { #[tracing::instrument(skip_all)] async fn get(&self, public_key: &PublicKey) -> Result, Self::Error> { + let started = Instant::now(); let result = Entity::find_by_id(public_key.serialize()) .one(&self.db) - .await - .with_foreign_context( - || format!("fetching backend for public key {public_key}"), - ErrorOrigin::Internal, - )?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "discovery_backend", + "get", + &result, + ); + let result = result.with_foreign_context( + || format!("fetching backend for public key {public_key}"), + ErrorOrigin::Internal, + )?; match result { Some(model) => Ok(Some(Self::model_to_domain(model)?)), @@ -299,9 +311,18 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { #[tracing::instrument(skip_all)] async fn get_all(&self, request_etag: Option) -> Result { + let started = Instant::now(); let response_etag = etag::Entity::find_by_id(DISCOVERY_BACKEND_GET_ALL_ETAG_ID) .one(&self.db) - .await + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "discovery_backend_etag", + "get_all_etag", + &response_etag, + ); + let response_etag = response_etag .foreign_context("fetching etag value", ErrorOrigin::Internal)? .map(|e| e.value as u64) .unwrap_or(0); @@ -312,12 +333,20 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { backends: None, }) } else { + let started = Instant::now(); let models = Entity::find() .order_by_asc(Column::CreatedAt) .order_by_asc(Column::Id) .all(&self.db) - .await - .foreign_context("fetching all backends", ErrorOrigin::Internal)?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "discovery_backend", + "get_all_backends", + &models, + ); + let models = models.foreign_context("fetching all backends", ErrorOrigin::Internal)?; let backends = models .into_iter() @@ -344,7 +373,8 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { updated_at: Set(now.into()), }; - let (insert_result, etag_result) = self + let started = Instant::now(); + let transaction = self .db .transaction::<_, (Result<_, _>, Option>), sea_orm::DbErr>(|txn| { Box::pin(async move { @@ -366,8 +396,16 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { Ok((insert, etag)) }) }) - .await - .foreign_context("post transaction", ErrorOrigin::Internal)?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "discovery_backend", + "post", + &transaction, + ); + let (insert_result, etag_result) = + transaction.foreign_context("post transaction", ErrorOrigin::Internal)?; etag_result .transpose() @@ -406,7 +444,8 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { updated_at: Set(now.into()), }; - let (upsert_result, fetch_result, etag_result) = self + let started = Instant::now(); + let transaction = self .db .transaction::<_, (Result<_, _>, Result<_, _>, Option>), sea_orm::DbErr>( |txn| { @@ -458,8 +497,16 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { }) }, ) - .await - .foreign_context("put transaction", ErrorOrigin::Internal)?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "discovery_backend", + "put", + &transaction, + ); + let (upsert_result, fetch_result, etag_result) = + transaction.foreign_context("put transaction", ErrorOrigin::Internal)?; upsert_result.with_foreign_context( || format!("upserting backend for public key {}", backend.public_key), @@ -514,7 +561,8 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { update = update.col_expr(Column::UpdatedAt, Expr::value(Utc::now())); - let (patch_result, etag_result) = self + let started = Instant::now(); + let transaction = self .db .transaction::<_, _, _>(|txn| { Box::pin(async move { @@ -543,8 +591,16 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { Ok((patch, etag)) }) }) - .await - .foreign_context("patch transaction", ErrorOrigin::Internal)?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "discovery_backend", + "patch", + &transaction, + ); + let (patch_result, etag_result) = + transaction.foreign_context("patch transaction", ErrorOrigin::Internal)?; etag_result .transpose() @@ -562,7 +618,8 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { async fn delete(&self, public_key: &PublicKey) -> Result { let id = public_key.serialize(); - let (delete_result, etag_result) = self + let started = Instant::now(); + let transaction = self .db .transaction::<_, _, _>(|txn| { Box::pin(async move { @@ -591,8 +648,16 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { Ok((delete, etag)) }) }) - .await - .foreign_context("delete transaction", ErrorOrigin::Internal)?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "discovery_backend", + "delete", + &transaction, + ); + let (delete_result, etag_result) = + transaction.foreign_context("delete transaction", ErrorOrigin::Internal)?; etag_result .transpose() @@ -606,6 +671,7 @@ impl DiscoveryBackendStore for DbDiscoveryBackendStore { Ok(result.rows_affected > 0) } + #[tracing::instrument(skip_all)] async fn disconnect(&self) -> Result<(), Self::Error> { if let Some(r) = &self.rotation { r.shutdown().await?; diff --git a/components/src/discovery/http.rs b/components/src/discovery/http.rs index 6ce0285..34d73ab 100644 --- a/components/src/discovery/http.rs +++ b/components/src/discovery/http.rs @@ -1,11 +1,12 @@ use crate::discovery::error::DefaultDiscoveryBackendStoreError; +use crate::metrics::http::record_http_request; use crate::secrets::SecretStore; use async_trait::async_trait; use reqwest::{Certificate, Client, ClientBuilder, IntoUrl, StatusCode}; use rustls::pki_types::CertificateDer; use secp256k1::PublicKey; use secrecy::{ExposeSecret, SecretString}; -use std::time::Duration; +use std::time::{Duration, Instant}; use switchgear_error::{ChainedContext, ErrorOrigin, ForeignContext}; use switchgear_service_api::discovery::{ DiscoveryBackend, DiscoveryBackendPatch, DiscoveryBackendStore, DiscoveryBackends, @@ -20,10 +21,11 @@ pub struct HttpDiscoveryBackendStore { bearer_secret: String, discovery_url: String, health_check_url: String, + server_address: String, + server_port: u16, } impl HttpDiscoveryBackendStore { - #[tracing::instrument(skip_all)] pub fn create( base_url: U, total_timeout: Duration, @@ -54,7 +56,6 @@ impl HttpDiscoveryBackendStore { Self::with_client(client, base_url, secret_store, bearer_secret.into()) } - #[tracing::instrument(skip_all)] pub fn with_client( client: Client, base_url: U, @@ -63,10 +64,15 @@ impl HttpDiscoveryBackendStore { ) -> Result { let base_url = base_url.as_str().trim_end_matches('/').to_string(); let discovery_url = format!("{base_url}/discovery"); - Url::parse(&discovery_url).with_foreign_context( + let parsed_discovery_url = Url::parse(&discovery_url).with_foreign_context( || format!("parsing service url {discovery_url}"), ErrorOrigin::Upstream, )?; + let server_address = parsed_discovery_url + .host_str() + .unwrap_or_default() + .to_owned(); + let server_port = parsed_discovery_url.port_or_known_default().unwrap_or(0); let health_check_url = format!("{base_url}/health"); Url::parse(&health_check_url).with_foreign_context( @@ -80,6 +86,8 @@ impl HttpDiscoveryBackendStore { bearer_secret, discovery_url, health_check_url, + server_address, + server_port, }) } @@ -101,7 +109,6 @@ impl HttpDiscoveryBackendStore { ) } - #[tracing::instrument(skip_all)] fn general_error(status: StatusCode, context: &str) -> DefaultDiscoveryBackendStoreError { if status.is_success() { return DefaultDiscoveryBackendStoreError::message( @@ -131,12 +138,23 @@ impl DiscoveryBackendStore for HttpDiscoveryBackendStore { let url = self.discovery_public_key_url(public_key); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .get(&url) .bearer_auth(token.expose_secret()) .send() - .await + .await; + record_http_request( + started.elapsed(), + "GET", + "/discovery/{public_key}", + &self.server_address, + self.server_port, + &response, + ); + let response = response .with_foreign_context(|| format!("get backend {url}"), ErrorOrigin::Upstream)?; match response.status() { @@ -157,6 +175,7 @@ impl DiscoveryBackendStore for HttpDiscoveryBackendStore { let url = &self.discovery_url; let token = self.bearer()?; let client = self.client.get(url).bearer_auth(token.expose_secret()); + let client = if let Some(requested_etag) = requested_etag { client.header( reqwest::header::IF_NONE_MATCH, @@ -165,9 +184,18 @@ impl DiscoveryBackendStore for HttpDiscoveryBackendStore { } else { client }; - let response = client - .send() - .await + + let started = Instant::now(); + let response = client.send().await; + record_http_request( + started.elapsed(), + "GET", + "/discovery", + &self.server_address, + self.server_port, + &response, + ); + let response = response .with_foreign_context(|| format!("get all backends {url}"), ErrorOrigin::Upstream)?; let response_etag = response @@ -222,22 +250,32 @@ impl DiscoveryBackendStore for HttpDiscoveryBackendStore { #[tracing::instrument(skip_all)] async fn post(&self, backend: DiscoveryBackend) -> Result, Self::Error> { let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .post(&self.discovery_url) .bearer_auth(token.expose_secret()) .json(&backend) .send() - .await - .with_foreign_context( - || { - format!( - "post backend: {}, url: {}", - backend.public_key, self.discovery_url - ) - }, - ErrorOrigin::Upstream, - )?; + .await; + record_http_request( + started.elapsed(), + "POST", + "/discovery", + &self.server_address, + self.server_port, + &response, + ); + let response = response.with_foreign_context( + || { + format!( + "post backend: {}, url: {}", + backend.public_key, self.discovery_url + ) + }, + ErrorOrigin::Upstream, + )?; match response.status() { StatusCode::CREATED => Ok(Some(backend.public_key)), @@ -257,13 +295,24 @@ impl DiscoveryBackendStore for HttpDiscoveryBackendStore { let url = self.discovery_public_key_url(&backend.public_key); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .put(&url) .bearer_auth(token.expose_secret()) .json(&backend.backend) .send() - .await + .await; + record_http_request( + started.elapsed(), + "PUT", + "/discovery/{public_key}", + &self.server_address, + self.server_port, + &response, + ); + let response = response .with_foreign_context(|| format!("put backend {url}"), ErrorOrigin::Upstream)?; match response.status() { @@ -278,13 +327,24 @@ impl DiscoveryBackendStore for HttpDiscoveryBackendStore { let url = self.discovery_public_key_url(&backend.public_key); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .patch(&url) .bearer_auth(token.expose_secret()) .json(&backend.backend) .send() - .await + .await; + record_http_request( + started.elapsed(), + "PATCH", + "/discovery/{public_key}", + &self.server_address, + self.server_port, + &response, + ); + let response = response .with_foreign_context(|| format!("patch backend {url}"), ErrorOrigin::Upstream)?; match response.status() { @@ -299,12 +359,23 @@ impl DiscoveryBackendStore for HttpDiscoveryBackendStore { let url = self.discovery_public_key_url(public_key); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .delete(&url) .bearer_auth(token.expose_secret()) .send() - .await + .await; + record_http_request( + started.elapsed(), + "DELETE", + "/discovery/{public_key}", + &self.server_address, + self.server_port, + &response, + ); + let response = response .with_foreign_context(|| format!("delete backend {url}"), ErrorOrigin::Upstream)?; match response.status() { diff --git a/components/src/discovery/memory.rs b/components/src/discovery/memory.rs index 6b359c7..e9ab911 100644 --- a/components/src/discovery/memory.rs +++ b/components/src/discovery/memory.rs @@ -40,11 +40,13 @@ impl MemoryDiscoveryBackendStore { impl DiscoveryBackendStore for MemoryDiscoveryBackendStore { type Error = DefaultDiscoveryBackendStoreError; + #[tracing::instrument(skip_all)] async fn get(&self, public_key: &PublicKey) -> Result, Self::Error> { let store = self.store.lock().await; Ok(store.get(public_key).map(|b| b.backend.clone())) } + #[tracing::instrument(skip_all)] async fn get_all(&self, request_etag: Option) -> Result { let store = self.store.lock().await; let mut backends: Vec = store.values().cloned().collect(); @@ -71,6 +73,7 @@ impl DiscoveryBackendStore for MemoryDiscoveryBackendStore { } } + #[tracing::instrument(skip_all)] async fn post(&self, backend: DiscoveryBackend) -> Result, Self::Error> { let mut store = self.store.lock().await; if store.contains_key(&backend.public_key) { @@ -88,6 +91,7 @@ impl DiscoveryBackendStore for MemoryDiscoveryBackendStore { Ok(Some(key)) } + #[tracing::instrument(skip_all)] async fn put(&self, backend: DiscoveryBackend) -> Result { let mut store = self.store.lock().await; let key = backend.public_key; @@ -100,6 +104,7 @@ impl DiscoveryBackendStore for MemoryDiscoveryBackendStore { Ok(was_new) } + #[tracing::instrument(skip_all)] async fn patch(&self, backend: DiscoveryBackendPatch) -> Result { let mut store = self.store.lock().await; let entry = match store.get_mut(&backend.public_key) { @@ -122,6 +127,7 @@ impl DiscoveryBackendStore for MemoryDiscoveryBackendStore { Ok(true) } + #[tracing::instrument(skip_all)] async fn delete(&self, public_key: &PublicKey) -> Result { let mut store = self.store.lock().await; let was_found = store.remove(public_key).is_some(); diff --git a/components/src/lib.rs b/components/src/lib.rs index 226a001..8f652bf 100644 --- a/components/src/lib.rs +++ b/components/src/lib.rs @@ -1,5 +1,6 @@ pub mod axum; pub mod discovery; +pub(crate) mod metrics; pub mod offer; pub mod pool; pub mod secrets; diff --git a/components/src/metrics/db.rs b/components/src/metrics/db.rs new file mode 100644 index 0000000..ed3fa28 --- /dev/null +++ b/components/src/metrics/db.rs @@ -0,0 +1,379 @@ +use opentelemetry_semantic_conventions::attribute::{ + DB_COLLECTION_NAME, DB_NAMESPACE, DB_RESPONSE_STATUS_CODE, DB_SYSTEM_NAME, ERROR_TYPE, + SERVER_ADDRESS, SERVER_PORT, +}; +use sea_orm::{ + DatabaseConnection, DatabaseConnectionType, DbBackend, DbErr, RuntimeErr, SqlErr, + TransactionError, +}; +use sqlx::mysql::MySqlConnectOptions; +use sqlx::postgres::PgConnectOptions; +use sqlx::sqlite::SqliteConnectOptions; +use std::path::Path; +use std::time::Duration; + +const ERROR_TYPE_OTHER: &str = "_OTHER"; +const SWGR_OPERATION: &str = "swgr.operation"; + +const SQLITE: &str = "sqlite"; +const MYSQL: &str = "mysql"; +const POSTGRESQL: &str = "postgresql"; +const SQLX_IN_MEMORY_PREFIX: &str = "file:sqlx-in-memory-"; + +#[derive(Clone, Debug)] +pub(crate) struct DbTarget { + pub system: &'static str, + pub namespace: Option, + pub address: Option, + pub port: Option, +} + +pub(crate) fn db_target(db: &DatabaseConnection) -> DbTarget { + match &db.inner { + DatabaseConnectionType::SqlxPostgresPoolConnection(_) => { + postgres_target(&db.get_postgres_connection_pool().connect_options()) + } + DatabaseConnectionType::SqlxMySqlPoolConnection(_) => { + mysql_target(&db.get_mysql_connection_pool().connect_options()) + } + DatabaseConnectionType::SqlxSqlitePoolConnection(_) => { + sqlite_target(&db.get_sqlite_connection_pool().connect_options()) + } + _ => DbTarget { + system: unsupported_system(db.get_database_backend()), + namespace: None, + address: None, + port: None, + }, + } +} + +fn unsupported_system(backend: DbBackend) -> &'static str { + match backend { + DbBackend::Sqlite => SQLITE, + DbBackend::MySql => MYSQL, + DbBackend::Postgres => POSTGRESQL, + _ => ERROR_TYPE_OTHER, + } +} + +fn postgres_target(options: &PgConnectOptions) -> DbTarget { + DbTarget { + system: POSTGRESQL, + namespace: options.get_database().map(str::to_owned), + address: non_empty(options.get_host()), + port: Some(options.get_port()), + } +} + +fn mysql_target(options: &MySqlConnectOptions) -> DbTarget { + DbTarget { + system: MYSQL, + namespace: options.get_database().map(str::to_owned), + address: non_empty(options.get_host()), + port: Some(options.get_port()), + } +} + +fn sqlite_target(options: &SqliteConnectOptions) -> DbTarget { + DbTarget { + system: SQLITE, + namespace: sqlite_namespace(options.get_filename()), + address: None, + port: None, + } +} + +fn sqlite_namespace(filename: &Path) -> Option { + let filename = filename.to_string_lossy(); + if filename == ":memory:" || filename.starts_with(SQLX_IN_MEMORY_PREFIX) { + return Some(":memory:".to_owned()); + } + Path::new(filename.as_ref()) + .file_stem() + .map(|stem| stem.to_string_lossy().into_owned()) +} + +fn non_empty(value: &str) -> Option { + (!value.is_empty()).then(|| value.to_owned()) +} + +pub(crate) trait DbOutcome { + fn error_type(&self) -> &'static str; + fn status_code(&self) -> Option; +} + +impl DbOutcome for DbErr { + fn error_type(&self) -> &'static str { + if let Some(sql_err) = self.sql_err() { + return match sql_err { + SqlErr::UniqueConstraintViolation(_) => "unique_constraint", + SqlErr::ForeignKeyConstraintViolation(_) => "foreign_key_constraint", + _ => "constraint", + }; + } + match self { + DbErr::ConnectionAcquire(_) => "connection_acquire", + DbErr::Conn(_) => "connection", + DbErr::Query(_) | DbErr::Exec(_) => "statement", + DbErr::Type(_) | DbErr::Json(_) | DbErr::TryIntoErr { .. } => "conversion", + _ => ERROR_TYPE_OTHER, + } + } + + fn status_code(&self) -> Option { + let runtime = match self { + DbErr::Conn(e) | DbErr::Exec(e) | DbErr::Query(e) => e, + _ => return None, + }; + let RuntimeErr::SqlxError(e) = runtime else { + return None; + }; + match e.as_ref() { + sqlx::Error::Database(db) => db.code().map(|code| code.into_owned()), + _ => None, + } + } +} + +impl DbOutcome for TransactionError { + fn error_type(&self) -> &'static str { + match self { + TransactionError::Connection(e) | TransactionError::Transaction(e) => e.error_type(), + } + } + + fn status_code(&self) -> Option { + match self { + TransactionError::Connection(e) | TransactionError::Transaction(e) => e.status_code(), + } + } +} + +pub(crate) fn record_db_operation( + elapsed: Duration, + target: &DbTarget, + collection: &'static str, + operation: &'static str, + result: &Result, +) { + let error = result.as_ref().err(); + let status_code = error.and_then(|e| e.status_code()); + switchgear_metrics::histogram!( + "db.client.operation.duration", + elapsed, + DB_SYSTEM_NAME => target.system, + DB_NAMESPACE => target.namespace.as_deref(), + SERVER_ADDRESS => target.address.as_deref(), + SERVER_PORT => target.port, + DB_COLLECTION_NAME => collection, + SWGR_OPERATION => operation, + ERROR_TYPE => error.map(|e| e.error_type()), + DB_RESPONSE_STATUS_CODE => status_code.as_deref(), + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pg(url: &str) -> PgConnectOptions { + url.parse().expect("postgres options") + } + + fn my(url: &str) -> MySqlConnectOptions { + url.parse().expect("mysql options") + } + + fn lite(url: &str) -> SqliteConnectOptions { + url.parse().expect("sqlite options") + } + + #[test] + fn postgres_target_reads_the_driver_options() { + let target = postgres_target(&pg("postgres://user:pw@db.example:6543/offers")); + + assert_eq!(target.system, POSTGRESQL); + assert_eq!(target.address.as_deref(), Some("db.example")); + assert_eq!(target.port, Some(6543)); + assert_eq!(target.namespace.as_deref(), Some("offers")); + } + + #[test] + fn mysql_target_reads_the_driver_options() { + let target = mysql_target(&my("mysql://user:pw@db.example:3307/offers")); + + assert_eq!(target.system, MYSQL); + assert_eq!(target.address.as_deref(), Some("db.example")); + assert_eq!(target.port, Some(3307)); + assert_eq!(target.namespace.as_deref(), Some("offers")); + } + + #[test] + fn a_target_carries_no_credentials() { + let rendered = format!( + "{:?} {:?}", + postgres_target(&pg("postgres://hunter2user:hunter2pass@h/db")), + mysql_target(&my("mysql://hunter2user:hunter2pass@h/db")), + ); + + assert!(!rendered.contains("hunter2"), "{rendered}"); + } + + #[test] + fn sqlite_target_reads_the_file_stem() { + let target = sqlite_target(&lite("sqlite:///data/offer_store.db?mode=rwc")); + + assert_eq!(target.system, SQLITE); + assert_eq!(target.namespace.as_deref(), Some("offer_store")); + assert_eq!(target.address, None); + assert_eq!(target.port, None); + } + + #[test] + fn sqlite_target_names_an_in_memory_database() { + let target = sqlite_target(&lite("sqlite::memory:")); + + assert_eq!(target.namespace.as_deref(), Some(":memory:")); + assert_eq!(target.address, None); + assert_eq!(target.port, None); + } + + #[tokio::test] + async fn db_target_reads_a_live_connection() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let path = dir.path().join("offer_store.db"); + let db = + sea_orm::Database::connect(format!("sqlite://{}?mode=rwc", path.to_string_lossy())) + .await + .expect("connect"); + + let target = db_target(&db); + + assert_eq!(target.system, SQLITE); + assert_eq!(target.namespace.as_deref(), Some("offer_store")); + assert_eq!(target.address, None); + assert_eq!(target.port, None); + } + + #[test] + fn db_outcome_classifies_each_db_err_variant() { + let cases: Vec<(DbErr, &str)> = vec![ + ( + DbErr::ConnectionAcquire(sea_orm::ConnAcquireErr::Timeout), + "connection_acquire", + ), + ( + DbErr::ConnectionAcquire(sea_orm::ConnAcquireErr::ConnectionClosed), + "connection_acquire", + ), + (DbErr::Conn(RuntimeErr::Internal("x".into())), "connection"), + (DbErr::Query(RuntimeErr::Internal("x".into())), "statement"), + (DbErr::Exec(RuntimeErr::Internal("x".into())), "statement"), + (DbErr::Type("x".into()), "conversion"), + (DbErr::Json("x".into()), "conversion"), + ( + DbErr::TryIntoErr { + from: "a", + into: "b", + source: std::sync::Arc::new(std::fmt::Error), + }, + "conversion", + ), + ]; + + for (err, expected) in cases { + assert_eq!(err.error_type(), expected, "{err:?}"); + assert_eq!( + TransactionError::Transaction(err.clone()).error_type(), + expected, + "{err:?}" + ); + assert_eq!( + TransactionError::Connection(err.clone()).error_type(), + expected, + "{err:?}" + ); + } + } + + #[test] + fn db_outcome_falls_back_to_the_registry_value() { + for err in [ + DbErr::RecordNotFound("x".into()), + DbErr::RecordNotInserted, + DbErr::RecordNotUpdated, + DbErr::Custom("x".into()), + DbErr::Migration("x".into()), + DbErr::UnpackInsertId, + DbErr::MutexPoisonError, + DbErr::AccessDenied { + permission: "x".into(), + resource: "y".into(), + }, + ] { + assert_eq!(err.error_type(), "_OTHER", "{err:?}"); + } + } + + #[test] + fn db_outcome_has_no_status_code_without_a_driver_error() { + assert_eq!( + DbErr::Query(RuntimeErr::Internal("x".into())).status_code(), + None + ); + assert_eq!(DbErr::RecordNotInserted.status_code(), None); + } + + #[tokio::test] + async fn db_outcome_extracts_a_driver_status_code() { + use sqlx::Connection; + + let mut conn = sqlx::SqliteConnection::connect("sqlite::memory:") + .await + .expect("connect"); + sqlx::query("CREATE TABLE t (id INTEGER PRIMARY KEY)") + .execute(&mut conn) + .await + .expect("create"); + sqlx::query("INSERT INTO t (id) VALUES (1)") + .execute(&mut conn) + .await + .expect("first insert"); + let sqlx_err = sqlx::query("INSERT INTO t (id) VALUES (1)") + .execute(&mut conn) + .await + .expect_err("unique violation"); + + let err = DbErr::Exec(RuntimeErr::SqlxError(std::sync::Arc::new(sqlx_err))); + + assert_eq!(err.error_type(), "unique_constraint"); + assert_eq!( + foreign_key_error(&mut conn).await.error_type(), + "foreign_key_constraint" + ); + let code = err.status_code().expect("driver code"); + assert!(code.parse::().is_ok(), "{code}"); + assert_eq!( + TransactionError::Transaction(err).status_code().as_deref(), + Some(code.as_str()) + ); + } + + async fn foreign_key_error(conn: &mut sqlx::SqliteConnection) -> DbErr { + sqlx::query("PRAGMA foreign_keys = ON") + .execute(&mut *conn) + .await + .expect("pragma"); + sqlx::query("CREATE TABLE child (id INTEGER PRIMARY KEY, parent INTEGER REFERENCES t(id))") + .execute(&mut *conn) + .await + .expect("create child"); + let err = sqlx::query("INSERT INTO child (id, parent) VALUES (1, 999)") + .execute(&mut *conn) + .await + .expect_err("foreign key violation"); + + DbErr::Exec(RuntimeErr::SqlxError(std::sync::Arc::new(err))) + } +} diff --git a/components/src/metrics/grpc.rs b/components/src/metrics/grpc.rs new file mode 100644 index 0000000..25ec175 --- /dev/null +++ b/components/src/metrics/grpc.rs @@ -0,0 +1,73 @@ +use opentelemetry_semantic_conventions::attribute::{ + ERROR_TYPE, RPC_METHOD, RPC_RESPONSE_STATUS_CODE, RPC_SYSTEM_NAME, SERVER_ADDRESS, SERVER_PORT, +}; +use std::time::Duration; + +pub(crate) fn record_rpc_call( + elapsed: Duration, + method: &'static str, + address: &str, + port: u16, + result: &Result, tonic::Status>, +) { + let code = match result { + Ok(_) => tonic::Code::Ok, + Err(status) => status.code(), + }; + let status_code = grpc_status_name(code); + switchgear_metrics::histogram!( + "rpc.client.call.duration", + elapsed, + RPC_SYSTEM_NAME => "grpc", + RPC_METHOD => method, + SERVER_ADDRESS => address, + SERVER_PORT => port, + RPC_RESPONSE_STATUS_CODE => status_code, + ERROR_TYPE => (code != tonic::Code::Ok).then_some(status_code), + ); +} + +fn grpc_status_name(code: tonic::Code) -> &'static str { + match code { + tonic::Code::Ok => "OK", + tonic::Code::Cancelled => "CANCELLED", + tonic::Code::Unknown => "UNKNOWN", + tonic::Code::InvalidArgument => "INVALID_ARGUMENT", + tonic::Code::DeadlineExceeded => "DEADLINE_EXCEEDED", + tonic::Code::NotFound => "NOT_FOUND", + tonic::Code::AlreadyExists => "ALREADY_EXISTS", + tonic::Code::PermissionDenied => "PERMISSION_DENIED", + tonic::Code::ResourceExhausted => "RESOURCE_EXHAUSTED", + tonic::Code::FailedPrecondition => "FAILED_PRECONDITION", + tonic::Code::Aborted => "ABORTED", + tonic::Code::OutOfRange => "OUT_OF_RANGE", + tonic::Code::Unimplemented => "UNIMPLEMENTED", + tonic::Code::Internal => "INTERNAL", + tonic::Code::Unavailable => "UNAVAILABLE", + tonic::Code::DataLoss => "DATA_LOSS", + tonic::Code::Unauthenticated => "UNAUTHENTICATED", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn grpc_status_names_are_canonical() { + assert_eq!(grpc_status_name(tonic::Code::Ok), "OK"); + assert_eq!(grpc_status_name(tonic::Code::Unavailable), "UNAVAILABLE"); + assert_eq!( + grpc_status_name(tonic::Code::DeadlineExceeded), + "DEADLINE_EXCEEDED" + ); + assert_eq!( + grpc_status_name(tonic::Code::InvalidArgument), + "INVALID_ARGUMENT" + ); + assert_eq!( + grpc_status_name(tonic::Code::Unauthenticated), + "UNAUTHENTICATED" + ); + } +} diff --git a/components/src/metrics/http.rs b/components/src/metrics/http.rs new file mode 100644 index 0000000..0f18b2c --- /dev/null +++ b/components/src/metrics/http.rs @@ -0,0 +1,151 @@ +use opentelemetry_semantic_conventions::attribute::{ + ERROR_TYPE, HTTP_REQUEST_METHOD, HTTP_RESPONSE_STATUS_CODE, SERVER_ADDRESS, SERVER_PORT, + URL_TEMPLATE, +}; +use std::time::Duration; + +pub(crate) fn record_http_request( + elapsed: Duration, + method: &'static str, + template: &'static str, + address: &str, + port: u16, + result: &Result, +) { + let status = match result { + Ok(response) => Some(response.status()), + Err(e) => e.status(), + }; + let failed = status.is_some_and(|s| s.is_client_error() || s.is_server_error()); + let error_type = if failed { + status.as_ref().map(reqwest::StatusCode::as_str) + } else { + result.as_ref().err().map(classify_request_error) + }; + switchgear_metrics::histogram!( + "http.client.request.duration", + elapsed, + HTTP_REQUEST_METHOD => method, + SERVER_ADDRESS => address, + SERVER_PORT => port, + URL_TEMPLATE => template, + HTTP_RESPONSE_STATUS_CODE => status.map(|s| s.as_u16()), + ERROR_TYPE => error_type, + ); +} + +fn classify_request_error(e: &reqwest::Error) -> &'static str { + if e.is_timeout() { + "timeout" + } else if e.is_connect() { + "connect" + } else if e.is_decode() { + "decode" + } else { + "request" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::SocketAddr; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + fn install_crypto_provider() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + } + + #[tokio::test] + async fn request_errors_are_classified_by_predicate() { + install_crypto_provider(); + + let connect = reqwest::Client::new() + .get("http://127.0.0.1:1/") + .send() + .await + .expect_err("connection refused"); + assert!(connect.is_connect(), "{connect:?}"); + assert_eq!(classify_request_error(&connect), "connect"); + + let (silent, _silent_task) = spawn_tcp(None).await; + let timeout = reqwest::Client::builder() + .timeout(Duration::from_millis(250)) + .build() + .expect("client") + .get(format!("http://{silent}/")) + .send() + .await + .expect_err("timeout"); + assert!(timeout.is_timeout(), "{timeout:?}"); + assert_eq!(classify_request_error(&timeout), "timeout"); + + let (server, _server_task) = spawn_tcp(Some( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 9\r\n\r\nnot json!", + )) + .await; + let decode = reqwest::Client::new() + .get(format!("http://{server}/")) + .send() + .await + .expect("response") + .json::() + .await + .expect_err("decode"); + assert!(decode.is_decode(), "{decode:?}"); + assert_eq!(classify_request_error(&decode), "decode"); + + let other = reqwest::Client::new() + .get("ftp://127.0.0.1/") + .send() + .await + .expect_err("unsupported scheme"); + assert!( + !other.is_timeout() && !other.is_connect() && !other.is_decode(), + "{other:?}" + ); + assert_eq!(classify_request_error(&other), "request"); + } + + #[tokio::test] + async fn a_failing_response_carries_its_status_as_the_error_type() { + install_crypto_provider(); + + let (server, _task) = spawn_tcp(Some( + "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\n\r\n", + )) + .await; + let response = reqwest::Client::new() + .get(format!("http://{server}/")) + .send() + .await + .expect("response"); + + let status = response.status(); + assert!(status.is_server_error()); + assert_eq!(status.as_str(), "503"); + } + + async fn spawn_tcp( + response: Option<&'static str>, + ) -> (SocketAddr, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("local addr"); + let task = tokio::spawn(async move { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let mut buffer = [0u8; 1024]; + let _ = socket.read(&mut buffer).await; + match response { + Some(response) => { + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.flush().await; + } + None => std::future::pending::<()>().await, + } + }); + (address, task) + } +} diff --git a/components/src/metrics/mod.rs b/components/src/metrics/mod.rs new file mode 100644 index 0000000..7a39910 --- /dev/null +++ b/components/src/metrics/mod.rs @@ -0,0 +1,21 @@ +pub(crate) mod db; +pub(crate) mod grpc; +pub(crate) mod http; + +#[cfg(test)] +mod tests { + use opentelemetry_semantic_conventions::metric; + + #[test] + fn metric_names_match_the_semantic_conventions() { + assert_eq!( + "db.client.operation.duration", + metric::DB_CLIENT_OPERATION_DURATION + ); + assert_eq!( + "http.client.request.duration", + metric::HTTP_CLIENT_REQUEST_DURATION + ); + assert_eq!("rpc.client.call.duration", metric::RPC_CLIENT_CALL_DURATION); + } +} diff --git a/components/src/offer/db.rs b/components/src/offer/db.rs index 45a2b73..3d97090 100644 --- a/components/src/offer/db.rs +++ b/components/src/offer/db.rs @@ -1,4 +1,5 @@ use crate::discovery::db::Column; +use crate::metrics::db::{DbTarget, record_db_operation}; use crate::offer::db_orm::prelude::*; use crate::offer::db_orm::{offer_metadata_table, offer_record_table}; use crate::offer::error::DefaultOfferStoreError; @@ -12,7 +13,7 @@ use sea_orm::{ use secrecy::{ExposeSecret, SecretString}; use std::str::FromStr; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant}; use switchgear_error::{ChainedContext, ErrorOrigin, ForeignContext}; use switchgear_migration::OnConflict; use switchgear_migration::{Expr, MigratorTrait}; @@ -165,11 +166,11 @@ impl CredentialRotation { #[derive(Clone, Debug)] pub struct DbOfferStore { db: DatabaseConnection, + db_target: DbTarget, rotation: Option, } impl DbOfferStore { - #[tracing::instrument(skip_all)] pub async fn connect( uri: &str, secrets: Option<&SecretStore>, @@ -191,14 +192,19 @@ impl DbOfferStore { .await .foreign_context("connecting to offer database", ErrorOrigin::Internal)?; + let db_target = crate::metrics::db::db_target(&db); + let rotation = secrets .map(|s| CredentialRotation::spawn(db.clone(), uri.to_owned(), s.clone(), resolved)) .transpose()?; - Ok(Self { db, rotation }) + Ok(Self { + db, + db_target, + rotation, + }) } - #[tracing::instrument(skip_all)] pub async fn migrate_up(&self) -> Result<(), DefaultOfferStoreError> { switchgear_migration::OfferMigrator::up(&self.db, None) .await @@ -206,7 +212,6 @@ impl DbOfferStore { Ok(()) } - #[tracing::instrument(skip_all)] pub async fn migrate_down(&self) -> Result<(), DefaultOfferStoreError> { switchgear_migration::OfferMigrator::down(&self.db, None) .await @@ -228,14 +233,22 @@ impl OfferStore for DbOfferStore { ) -> Result, Self::Error> { let sparse = sparse.unwrap_or(true); + let started = Instant::now(); let result = OfferRecordTable::find_by_id((partition.to_string(), *id)) .find_also_related(OfferMetadataTable) .one(&self.db) - .await - .with_foreign_context( - || format!("getting offer with metadata for partition {partition} id {id}"), - ErrorOrigin::Internal, - )?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_record", + "get_offer", + &result, + ); + let result = result.with_foreign_context( + || format!("getting offer with metadata for partition {partition} id {id}"), + ErrorOrigin::Internal, + )?; let (offer_model, metadata_model) = match (result, sparse) { (Some((offer, Some(metadata))), false) => { @@ -270,6 +283,7 @@ impl OfferStore for DbOfferStore { start: usize, count: usize, ) -> Result, Self::Error> { + let started = Instant::now(); let models = OfferRecordTable::find() .filter(offer_record_table::Column::Partition.eq(partition)) .order_by_asc(offer_record_table::Column::CreatedAt) @@ -277,11 +291,18 @@ impl OfferStore for DbOfferStore { .offset(start as u64) .limit(count as u64) .all(&self.db) - .await - .with_foreign_context( - || format!("getting offers for partition {partition}"), - ErrorOrigin::Internal, - )?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_record", + "get_offers", + &models, + ); + let models = models.with_foreign_context( + || format!("getting offers for partition {partition}"), + ErrorOrigin::Internal, + )?; let mut offers = Vec::new(); for model in models { @@ -317,7 +338,16 @@ impl OfferStore for DbOfferStore { updated_at: Set(now.into()), }; - match OfferRecordTable::insert(active_model).exec(&self.db).await { + let started = Instant::now(); + let result = OfferRecordTable::insert(active_model).exec(&self.db).await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_record", + "post_offer", + &result, + ); + match result { Ok(_) => Ok(Some(offer.id)), // Unique constraint violation (Postgres via Query, SQLite via Exec) Err(sea_orm::DbErr::Query(sea_orm::RuntimeErr::SqlxError(sqlx_err))) @@ -366,7 +396,8 @@ impl OfferStore for DbOfferStore { updated_at: Set(now.into()), }; - let _result = match OfferRecordTable::insert(active_model) + let started = Instant::now(); + let result = OfferRecordTable::insert(active_model) .on_conflict( OnConflict::columns([ offer_record_table::Column::Partition, @@ -383,8 +414,15 @@ impl OfferStore for DbOfferStore { .to_owned(), ) .exec(&self.db) - .await - { + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_record", + "put_offer_upsert", + &result, + ); + let _result = match result { Ok(result) => result, // Foreign key constraint violation (metadata_id doesn't exist) Err(sea_orm::DbErr::Query(sea_orm::RuntimeErr::SqlxError(sqlx_err))) @@ -411,6 +449,7 @@ impl OfferStore for DbOfferStore { }; // Fetch only the timestamps to compare + let started = Instant::now(); let result = OfferRecordTable::find() .filter(offer_record_table::Column::Partition.eq(offer.partition.clone())) .filter(offer_record_table::Column::Id.eq(offer.id)) @@ -422,7 +461,15 @@ impl OfferStore for DbOfferStore { chrono::DateTime, )>() .one(&self.db) - .await + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_record", + "put_offer_fetch", + &result, + ); + let result = result .with_foreign_context( || { format!( @@ -445,17 +492,26 @@ impl OfferStore for DbOfferStore { #[tracing::instrument(skip_all)] async fn delete_offer(&self, partition: &str, id: &Uuid) -> Result { + let started = Instant::now(); let result = OfferRecordTable::delete_by_id((partition.to_string(), *id)) .exec(&self.db) - .await - .with_foreign_context( - || format!("deleting offer for partition {partition} id {id}"), - ErrorOrigin::Internal, - )?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_record", + "delete_offer", + &result, + ); + let result = result.with_foreign_context( + || format!("deleting offer for partition {partition} id {id}"), + ErrorOrigin::Internal, + )?; Ok(result.rows_affected > 0) } + #[tracing::instrument(skip_all)] async fn disconnect(&self) -> Result<(), Self::Error> { if let Some(r) = &self.rotation { r.shutdown().await?; @@ -474,13 +530,21 @@ impl OfferMetadataStore for DbOfferStore { partition: &str, id: &Uuid, ) -> Result, Self::Error> { + let started = Instant::now(); let model = OfferMetadataTable::find_by_id((partition.to_string(), *id)) .one(&self.db) - .await - .with_foreign_context( - || format!("getting metadata for partition {partition} id {id}"), - ErrorOrigin::Internal, - )?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_metadata", + "get_metadata", + &model, + ); + let model = model.with_foreign_context( + || format!("getting metadata for partition {partition} id {id}"), + ErrorOrigin::Internal, + )?; match model { Some(model) => { @@ -506,6 +570,7 @@ impl OfferMetadataStore for DbOfferStore { start: usize, count: usize, ) -> Result, Self::Error> { + let started = Instant::now(); let models = OfferMetadataTable::find() .filter(offer_metadata_table::Column::Partition.eq(partition)) .order_by_asc(offer_metadata_table::Column::CreatedAt) @@ -513,11 +578,18 @@ impl OfferMetadataStore for DbOfferStore { .offset(start as u64) .limit(count as u64) .all(&self.db) - .await - .with_foreign_context( - || format!("getting all metadata for partition {partition}"), - ErrorOrigin::Internal, - )?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_metadata", + "get_all_metadata", + &models, + ); + let models = models.with_foreign_context( + || format!("getting all metadata for partition {partition}"), + ErrorOrigin::Internal, + )?; let mut metadata_list = Vec::new(); for model in models { @@ -562,10 +634,18 @@ impl OfferMetadataStore for DbOfferStore { updated_at: Set(now.into()), }; - match OfferMetadataTable::insert(active_model) + let started = Instant::now(); + let result = OfferMetadataTable::insert(active_model) .exec(&self.db) - .await - { + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_metadata", + "post_metadata", + &result, + ); + match result { Ok(_) => Ok(Some(offer.id)), // Unique constraint violation (Postgres via Query, SQLite via Exec) Err(sea_orm::DbErr::Query(sea_orm::RuntimeErr::SqlxError(sqlx_err))) @@ -607,7 +687,8 @@ impl OfferMetadataStore for DbOfferStore { updated_at: Set(now.into()), }; - let _result = OfferMetadataTable::insert(active_model) + let started = Instant::now(); + let result = OfferMetadataTable::insert(active_model) .on_conflict( OnConflict::columns([ offer_metadata_table::Column::Partition, @@ -618,18 +699,26 @@ impl OfferMetadataStore for DbOfferStore { .to_owned(), ) .exec(&self.db) - .await - .with_foreign_context( - || { - format!( - "upserting metadata for partition {} id {}", - offer.partition, offer.id - ) - }, - ErrorOrigin::Internal, - )?; + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_metadata", + "put_metadata_upsert", + &result, + ); + result.with_foreign_context( + || { + format!( + "upserting metadata for partition {} id {}", + offer.partition, offer.id + ) + }, + ErrorOrigin::Internal, + )?; // Fetch only the timestamps to compare + let started = Instant::now(); let result = OfferMetadataTable::find() .filter(offer_metadata_table::Column::Partition.eq(offer.partition.clone())) .filter(offer_metadata_table::Column::Id.eq(offer.id)) @@ -641,7 +730,15 @@ impl OfferMetadataStore for DbOfferStore { chrono::DateTime, )>() .one(&self.db) - .await + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_metadata", + "put_metadata_fetch", + &result, + ); + let result = result .with_foreign_context( || { format!( @@ -664,10 +761,18 @@ impl OfferMetadataStore for DbOfferStore { #[tracing::instrument(skip_all)] async fn delete_metadata(&self, partition: &str, id: &Uuid) -> Result { - let result = match OfferMetadataTable::delete_by_id((partition.to_string(), *id)) + let started = Instant::now(); + let result = OfferMetadataTable::delete_by_id((partition.to_string(), *id)) .exec(&self.db) - .await - { + .await; + record_db_operation( + started.elapsed(), + &self.db_target, + "offer_metadata", + "delete_metadata", + &result, + ); + let result = match result { Ok(result) => result, Err(sea_orm::DbErr::Exec(sea_orm::RuntimeErr::SqlxError(sqlx_err))) if matches!( diff --git a/components/src/offer/http.rs b/components/src/offer/http.rs index f60441e..c3ea858 100644 --- a/components/src/offer/http.rs +++ b/components/src/offer/http.rs @@ -1,10 +1,11 @@ +use crate::metrics::http::record_http_request; use crate::offer::error::DefaultOfferStoreError; use crate::secrets::SecretStore; use async_trait::async_trait; use reqwest::{Certificate, Client, ClientBuilder, IntoUrl, StatusCode}; use rustls::pki_types::CertificateDer; use secrecy::{ExposeSecret, SecretString}; -use std::time::Duration; +use std::time::{Duration, Instant}; use switchgear_error::{ChainedContext, ErrorOrigin, ForeignContext}; use switchgear_service_api::offer::{ HttpOfferClient, OfferMetadata, OfferMetadataStore, OfferRecord, OfferStore, @@ -20,10 +21,11 @@ pub struct HttpOfferStore { offer_url: String, metadata_url: String, health_check_url: String, + server_address: String, + server_port: u16, } impl HttpOfferStore { - #[tracing::instrument(skip_all)] pub fn create( base_url: U, total_timeout: Duration, @@ -52,7 +54,6 @@ impl HttpOfferStore { Self::with_client(client, base_url, secret_store, bearer_secret.into()) } - #[tracing::instrument(skip_all)] fn with_client( client: Client, base_url: U, @@ -62,10 +63,12 @@ impl HttpOfferStore { let base_url = base_url.as_str().trim_end_matches('/').to_string(); let offer_url = format!("{base_url}/offers"); - Url::parse(&offer_url).with_foreign_context( + let parsed_offer_url = Url::parse(&offer_url).with_foreign_context( || format!("parsing service url {offer_url}"), ErrorOrigin::Upstream, )?; + let server_address = parsed_offer_url.host_str().unwrap_or_default().to_owned(); + let server_port = parsed_offer_url.port_or_known_default().unwrap_or(0); let metadata_url = format!("{base_url}/metadata"); Url::parse(&metadata_url).with_foreign_context( @@ -86,6 +89,8 @@ impl HttpOfferStore { offer_url, metadata_url, health_check_url, + server_address, + server_port, }) } @@ -114,7 +119,6 @@ impl HttpOfferStore { ) } - #[tracing::instrument(skip_all)] fn general_error(status: StatusCode, context: &str) -> DefaultOfferStoreError { if status.is_success() { return DefaultOfferStoreError::message( @@ -150,13 +154,24 @@ impl OfferStore for HttpOfferStore { let url = self.offers_partition_id_url(partition, id); let url = format!("{url}?sparse={sparse}"); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .get(&url) .bearer_auth(token.expose_secret()) .send() - .await - .with_foreign_context(|| format!("get offer {url}"), ErrorOrigin::Upstream)?; + .await; + record_http_request( + started.elapsed(), + "GET", + "/offers/{partition}/{id}", + &self.server_address, + self.server_port, + &response, + ); + let response = + response.with_foreign_context(|| format!("get offer {url}"), ErrorOrigin::Upstream)?; match response.status() { StatusCode::OK => { @@ -181,12 +196,23 @@ impl OfferStore for HttpOfferStore { let url = self.offers_partition_url(partition); let url = format!("{url}?start={start}&count={count}"); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .get(&url) .bearer_auth(token.expose_secret()) .send() - .await + .await; + record_http_request( + started.elapsed(), + "GET", + "/offers/{partition}", + &self.server_address, + self.server_port, + &response, + ); + let response = response .with_foreign_context(|| format!("get all offers {url}"), ErrorOrigin::Upstream)?; match response.status() { @@ -210,17 +236,27 @@ impl OfferStore for HttpOfferStore { #[tracing::instrument(skip_all)] async fn post_offer(&self, offer: OfferRecord) -> Result, Self::Error> { let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .post(&self.offer_url) .bearer_auth(token.expose_secret()) .json(&offer) .send() - .await - .with_foreign_context( - || format!("post offer: {}, url: {}", offer.id, self.offer_url), - ErrorOrigin::Upstream, - )?; + .await; + record_http_request( + started.elapsed(), + "POST", + "/offers", + &self.server_address, + self.server_port, + &response, + ); + let response = response.with_foreign_context( + || format!("post offer: {}, url: {}", offer.id, self.offer_url), + ErrorOrigin::Upstream, + )?; match response.status() { StatusCode::CREATED => Ok(Some(offer.id)), @@ -236,14 +272,25 @@ impl OfferStore for HttpOfferStore { async fn put_offer(&self, offer: OfferRecord) -> Result { let url = self.offers_partition_id_url(&offer.partition, &offer.id); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .put(&url) .bearer_auth(token.expose_secret()) .json(&offer) .send() - .await - .with_foreign_context(|| format!("put offer {url}"), ErrorOrigin::Upstream)?; + .await; + record_http_request( + started.elapsed(), + "PUT", + "/offers/{partition}/{id}", + &self.server_address, + self.server_port, + &response, + ); + let response = + response.with_foreign_context(|| format!("put offer {url}"), ErrorOrigin::Upstream)?; match response.status() { StatusCode::CREATED => Ok(true), @@ -256,12 +303,23 @@ impl OfferStore for HttpOfferStore { async fn delete_offer(&self, partition: &str, id: &Uuid) -> Result { let url = self.offers_partition_id_url(partition, id); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .delete(&url) .bearer_auth(token.expose_secret()) .send() - .await + .await; + record_http_request( + started.elapsed(), + "DELETE", + "/offers/{partition}/{id}", + &self.server_address, + self.server_port, + &response, + ); + let response = response .with_foreign_context(|| format!("delete offer {url}"), ErrorOrigin::Upstream)?; match response.status() { @@ -284,16 +342,26 @@ impl OfferMetadataStore for HttpOfferStore { ) -> Result, Self::Error> { let url = self.metadata_partition_id_url(partition, id); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .get(&url) .bearer_auth(token.expose_secret()) .send() - .await - .with_foreign_context( - || format!("get offer metadata {url}"), - ErrorOrigin::Upstream, - )?; + .await; + record_http_request( + started.elapsed(), + "GET", + "/metadata/{partition}/{id}", + &self.server_address, + self.server_port, + &response, + ); + let response = response.with_foreign_context( + || format!("get offer metadata {url}"), + ErrorOrigin::Upstream, + )?; match response.status() { StatusCode::OK => { @@ -324,12 +392,23 @@ impl OfferMetadataStore for HttpOfferStore { let url = self.metadata_partition_url(partition); let url = format!("{url}?start={start}&count={count}"); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .get(&url) .bearer_auth(token.expose_secret()) .send() - .await + .await; + record_http_request( + started.elapsed(), + "GET", + "/metadata/{partition}", + &self.server_address, + self.server_port, + &response, + ); + let response = response .with_foreign_context(|| format!("get all metadata {url}"), ErrorOrigin::Upstream)?; match response.status() { @@ -353,22 +432,32 @@ impl OfferMetadataStore for HttpOfferStore { #[tracing::instrument(skip_all)] async fn post_metadata(&self, metadata: OfferMetadata) -> Result, Self::Error> { let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .post(&self.metadata_url) .bearer_auth(token.expose_secret()) .json(&metadata) .send() - .await - .with_foreign_context( - || { - format!( - "post offer metadata {}, url: {}", - metadata.id, self.metadata_url - ) - }, - ErrorOrigin::Upstream, - )?; + .await; + record_http_request( + started.elapsed(), + "POST", + "/metadata", + &self.server_address, + self.server_port, + &response, + ); + let response = response.with_foreign_context( + || { + format!( + "post offer metadata {}, url: {}", + metadata.id, self.metadata_url + ) + }, + ErrorOrigin::Upstream, + )?; match response.status() { StatusCode::CREATED => Ok(Some(metadata.id)), @@ -387,17 +476,27 @@ impl OfferMetadataStore for HttpOfferStore { async fn put_metadata(&self, metadata: OfferMetadata) -> Result { let url = self.metadata_partition_id_url(&metadata.partition, &metadata.id); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .put(&url) .bearer_auth(token.expose_secret()) .json(&metadata) .send() - .await - .with_foreign_context( - || format!("put offer metadata {url}"), - ErrorOrigin::Upstream, - )?; + .await; + record_http_request( + started.elapsed(), + "PUT", + "/metadata/{partition}/{id}", + &self.server_address, + self.server_port, + &response, + ); + let response = response.with_foreign_context( + || format!("put offer metadata {url}"), + ErrorOrigin::Upstream, + )?; match response.status() { StatusCode::CREATED => Ok(true), @@ -413,16 +512,26 @@ impl OfferMetadataStore for HttpOfferStore { async fn delete_metadata(&self, partition: &str, id: &Uuid) -> Result { let url = self.metadata_partition_id_url(partition, id); let token = self.bearer()?; + + let started = Instant::now(); let response = self .client .delete(&url) .bearer_auth(token.expose_secret()) .send() - .await - .with_foreign_context( - || format!("delete offer metadata {url}"), - ErrorOrigin::Upstream, - )?; + .await; + record_http_request( + started.elapsed(), + "DELETE", + "/metadata/{partition}/{id}", + &self.server_address, + self.server_port, + &response, + ); + let response = response.with_foreign_context( + || format!("delete offer metadata {url}"), + ErrorOrigin::Upstream, + )?; match response.status() { StatusCode::NO_CONTENT => Ok(true), @@ -440,6 +549,7 @@ impl HttpOfferClient for HttpOfferStore { #[tracing::instrument(skip_all)] async fn health(&self) -> Result<(), ::Error> { let token = self.bearer()?; + let response = self .client .get(&self.health_check_url) @@ -447,6 +557,7 @@ impl HttpOfferClient for HttpOfferStore { .send() .await .foreign_context("health check", ErrorOrigin::Upstream)?; + if !response.status().is_success() { return Err(DefaultOfferStoreError::message( ErrorOrigin::Upstream, diff --git a/components/src/offer/memory.rs b/components/src/offer/memory.rs index 9e41cd8..db170e4 100644 --- a/components/src/offer/memory.rs +++ b/components/src/offer/memory.rs @@ -43,6 +43,7 @@ impl Default for MemoryOfferStore { impl OfferStore for MemoryOfferStore { type Error = DefaultOfferStoreError; + #[tracing::instrument(skip_all)] async fn get_offer( &self, partition: &str, @@ -68,6 +69,7 @@ impl OfferStore for MemoryOfferStore { })) } + #[tracing::instrument(skip_all)] async fn get_offers( &self, partition: &str, @@ -152,6 +154,7 @@ impl OfferStore for MemoryOfferStore { Ok(was_new) } + #[tracing::instrument(skip_all)] async fn delete_offer(&self, partition: &str, id: &Uuid) -> Result { let mut store = self.offer.lock().await; Ok(store.remove(&(partition.to_string(), *id)).is_some()) @@ -162,6 +165,7 @@ impl OfferStore for MemoryOfferStore { impl OfferMetadataStore for MemoryOfferStore { type Error = DefaultOfferStoreError; + #[tracing::instrument(skip_all)] async fn get_metadata( &self, partition: &str, @@ -173,6 +177,7 @@ impl OfferMetadataStore for MemoryOfferStore { .map(|o| o.metadata.clone())) } + #[tracing::instrument(skip_all)] async fn get_all_metadata( &self, partition: &str, @@ -202,6 +207,7 @@ impl OfferMetadataStore for MemoryOfferStore { Ok(metadata) } + #[tracing::instrument(skip_all)] async fn post_metadata(&self, metadata: OfferMetadata) -> Result, Self::Error> { let mut store = self.metadata.lock().await; if let std::collections::hash_map::Entry::Vacant(e) = @@ -218,6 +224,7 @@ impl OfferMetadataStore for MemoryOfferStore { } } + #[tracing::instrument(skip_all)] async fn put_metadata(&self, metadata: OfferMetadata) -> Result { let mut store = self.metadata.lock().await; let was_new = store diff --git a/components/src/pool/client_pool.rs b/components/src/pool/client_pool.rs index de3b6e5..c5cf40a 100644 --- a/components/src/pool/client_pool.rs +++ b/components/src/pool/client_pool.rs @@ -40,7 +40,6 @@ where } } - #[tracing::instrument(skip_all)] async fn get_client( &self, key: &K, @@ -63,7 +62,6 @@ where Ok(client.clone()) } - #[tracing::instrument(skip_all)] pub async fn get_invoice( &self, offer: &Offer, @@ -89,7 +87,6 @@ where .await } - #[tracing::instrument(skip_all)] pub async fn get_metrics(&self, key: &K) -> Result { let client = self.get_client(key).await?; @@ -106,7 +103,6 @@ where Ok(metrics) } - #[tracing::instrument(skip_all)] pub fn connect(&self, key: K, backend: &DiscoveryBackend) -> Result<(), LnPoolError> { let implementation: DiscoveryBackendImplementation = serde_json::from_slice(backend.backend.implementation.as_slice()) diff --git a/components/src/pool/cln/grpc/client.rs b/components/src/pool/cln/grpc/client.rs index 6737103..a92f4a8 100644 --- a/components/src/pool/cln/grpc/client.rs +++ b/components/src/pool/cln/grpc/client.rs @@ -1,3 +1,4 @@ +use crate::metrics::grpc::record_rpc_call; use crate::pool::cln::grpc::config::{ClnGrpcClientAuth, ClnGrpcDiscoveryBackendImplementation}; use crate::pool::error::LnPoolError; use crate::pool::{Bolt11InvoiceDescription, LnFeatures, LnMetrics, LnRpcClient}; @@ -11,10 +12,11 @@ use rustls::pki_types::{CertificateDer, ServerName}; use rustls::{ClientConfig, RootCertStore}; use sha2::Digest; use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use switchgear_error::{ErrorOrigin, ForeignContext}; use tokio::sync::Mutex; use tonic::transport::{Channel, Endpoint}; +use url::Url; #[allow(clippy::all)] pub mod cln { @@ -33,7 +35,6 @@ pub struct TonicClnGrpcClient { } impl TonicClnGrpcClient { - #[tracing::instrument(skip_all)] pub fn create( timeout: Duration, config: ClnGrpcDiscoveryBackendImplementation, @@ -55,7 +56,6 @@ impl TonicClnGrpcClient { }) } - #[tracing::instrument(skip_all)] async fn inner_connect(&self) -> Result, LnPoolError> { let mut inner = self.inner.lock().await; match inner.as_ref() { @@ -66,7 +66,7 @@ impl TonicClnGrpcClient { &self.trusted_roots, &self.config.auth, self.secrets.clone(), - self.config.url.to_string(), + self.config.url.clone(), self.config.domain.as_deref(), ) .await?, @@ -78,7 +78,6 @@ impl TonicClnGrpcClient { } } - #[tracing::instrument(skip_all)] async fn inner_disconnect(&self) { let mut inner = self.inner.lock().await; *inner = None; @@ -120,6 +119,7 @@ impl LnRpcClient for TonicClnGrpcClient { r } + #[tracing::instrument(skip_all)] fn get_features(&self) -> Option<&LnFeatures> { self.features.as_ref() } @@ -127,17 +127,18 @@ impl LnRpcClient for TonicClnGrpcClient { struct InnerTonicClnGrpcClient { client: NodeClient, - url: String, + url: Url, + server_address: String, + server_port: u16, } impl InnerTonicClnGrpcClient { - #[tracing::instrument(skip_all)] async fn connect( timeout: Duration, trusted_roots: &[CertificateDer<'static>], auth: &ClnGrpcClientAuth, secrets: SecretStore, - url: String, + url: Url, domain: Option<&str>, ) -> Result { let tls_config = build_client_tls_config(trusted_roots, auth, &secrets)?; @@ -158,20 +159,24 @@ impl InnerTonicClnGrpcClient { } let https = connector_builder.enable_http2().build(); - let endpoint = Endpoint::from_shared(url.clone()) + let endpoint = Endpoint::from_shared(url.to_string()) .with_foreign_context(|| format!("parsing CLN endpoint address {url}"), None)? .connect_timeout(timeout) .timeout(timeout); let channel = Channel::new(https, endpoint); + let server_address = url.host_str().unwrap_or_default().to_owned(); + let server_port = url.port_or_known_default().unwrap_or(0); + Ok(Self { client: NodeClient::new(channel), url, + server_address, + server_port, }) } - #[tracing::instrument(skip_all)] async fn get_invoice<'a>( &self, amount_msat: Option, @@ -220,25 +225,38 @@ impl InnerTonicClnGrpcClient { ..Default::default() }; - let response = client - .invoice(request) - .await + let started = Instant::now(); + let response = client.invoice(request).await; + record_rpc_call( + started.elapsed(), + "cln.Node/Invoice", + &self.server_address, + self.server_port, + &response, + ); + let response = response .with_foreign_context(|| format!("requesting CLN invoice from {}", self.url), None)? .into_inner(); Ok(response.bolt11) } - #[tracing::instrument(skip_all)] async fn get_metrics(&self) -> Result { let channels_request = cln::ListpeerchannelsRequest { id: None, short_channel_id: None, }; let mut client = self.client.clone(); - let channels_response = client - .list_peer_channels(channels_request) - .await + let started = Instant::now(); + let channels_response = client.list_peer_channels(channels_request).await; + record_rpc_call( + started.elapsed(), + "cln.Node/ListPeerChannels", + &self.server_address, + self.server_port, + &channels_response, + ); + let channels_response = channels_response .with_foreign_context( || format!("listing CLN peer channels for {}", self.url), None, @@ -305,7 +323,10 @@ fn build_client_tls_config( Ok(ClientConfig::builder_with_provider(crypto.clone()) .with_safe_default_protocol_versions() - .expect("rustls safe default protocol versions") + .foreign_context( + "configuring CLN gRPC client TLS protocol versions", + ErrorOrigin::Internal, + )? .with_root_certificates(roots) .with_client_cert_resolver(Arc::new(ClientCertResolver::new( secrets.clone(), diff --git a/components/src/pool/error.rs b/components/src/pool/error.rs index 4e374fc..dc6706f 100644 --- a/components/src/pool/error.rs +++ b/components/src/pool/error.rs @@ -1,8 +1,9 @@ +use crate::pool::LnRpcClientError; use std::borrow::Cow; use std::error::Error; use std::fmt::{Debug, Display, Formatter}; use std::panic::Location; -use switchgear_error::{ContextError, ErrorOrigin, IntoContextError}; +use switchgear_error::{ContextError, ErrorOrigin, IntoBoxedTrait, IntoContextError}; use tonic::{Code, Status, transport}; #[derive(Debug)] @@ -119,6 +120,14 @@ impl ContextError for LnPoolError { } } +impl LnRpcClientError for LnPoolError {} + +impl IntoBoxedTrait for LnPoolError { + fn into_boxed(self) -> Box { + Box::new(self) + } +} + fn origin_from_tonic_status(status: &Status) -> ErrorOrigin { match status.code() { Code::InvalidArgument | Code::OutOfRange | Code::AlreadyExists => ErrorOrigin::Downstream, diff --git a/components/src/pool/lnd/grpc/client.rs b/components/src/pool/lnd/grpc/client.rs index aa0566a..d1bbdca 100644 --- a/components/src/pool/lnd/grpc/client.rs +++ b/components/src/pool/lnd/grpc/client.rs @@ -1,3 +1,4 @@ +use crate::metrics::grpc::record_rpc_call; use crate::pool::error::LnPoolError; use crate::pool::lnd::grpc::config::LndGrpcDiscoveryBackendImplementation; use crate::pool::{Bolt11InvoiceDescription, LnFeatures, LnMetrics, LnRpcClient}; @@ -7,10 +8,11 @@ use rustls::pki_types::CertificateDer; use sha2::Digest; use std::fs; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use switchgear_error::ForeignContext; use tokio::sync::Mutex; use tonic::transport::{Certificate, Channel, ClientTlsConfig}; +use url::Url; #[allow(clippy::all)] pub mod lnrpc { @@ -29,7 +31,6 @@ pub struct TonicLndGrpcClient { } impl TonicLndGrpcClient { - #[tracing::instrument(skip_all)] pub fn create( timeout: Duration, config: LndGrpcDiscoveryBackendImplementation, @@ -72,7 +73,6 @@ impl TonicLndGrpcClient { }) } - #[tracing::instrument(skip_all)] async fn inner_connect(&self) -> Result, LnPoolError> { let mut inner = self.inner.lock().await; match inner.as_ref() { @@ -83,7 +83,7 @@ impl TonicLndGrpcClient { self.ca_certificates.clone(), self.secrets.clone(), self.config.auth.macaroon_secret.clone(), - self.config.url.to_string(), + self.config.url.clone(), self.config.domain.as_deref(), self.config.amp_invoice, ) @@ -96,7 +96,6 @@ impl TonicLndGrpcClient { } } - #[tracing::instrument(skip_all)] async fn inner_disconnect(&self) { let mut inner = self.inner.lock().await; *inner = None; @@ -144,6 +143,7 @@ impl LnRpcClient for TonicLndGrpcClient { r } + #[tracing::instrument(skip_all)] fn get_features(&self) -> Option<&LnFeatures> { self.features.as_ref() } @@ -153,22 +153,23 @@ struct InnerTonicLndGrpcClient { client: LightningClient< tonic::service::interceptor::InterceptedService, >, - url: String, + url: Url, amp_invoice: bool, + server_address: String, + server_port: u16, } impl InnerTonicLndGrpcClient { - #[tracing::instrument(skip_all)] async fn connect( timeout: Duration, ca_certificates: Vec, secrets: SecretStore, macaroon_secret: String, - url: String, + url: Url, domain: Option<&str>, amp_invoice: bool, ) -> Result { - let endpoint = Channel::from_shared(url.clone()) + let endpoint = Channel::from_shared(url.to_string()) .with_foreign_context(|| format!("parsing LND endpoint address {url}"), None)?; let mut tls_config = ClientTlsConfig::new() @@ -193,14 +194,18 @@ impl InnerTonicLndGrpcClient { let interceptor = SecretHeaderInterceptor::macaroon_hex(secrets, macaroon_secret); let client = LightningClient::with_interceptor(channel, interceptor); + let server_address = url.host_str().unwrap_or_default().to_owned(); + let server_port = url.port_or_known_default().unwrap_or(0); + Ok(Self { client, url, amp_invoice, + server_address, + server_port, }) } - #[tracing::instrument(skip_all)] async fn get_invoice<'a>( &self, amount_msat: Option, @@ -226,23 +231,36 @@ impl InnerTonicLndGrpcClient { ..Default::default() }; - let response = client - .add_invoice(invoice_request) - .await + let started = Instant::now(); + let response = client.add_invoice(invoice_request).await; + record_rpc_call( + started.elapsed(), + "lnrpc.Lightning/AddInvoice", + &self.server_address, + self.server_port, + &response, + ); + let response = response .with_foreign_context(|| format!("requesting LND invoice from {}", self.url), None)? .into_inner(); Ok(response.payment_request) } - #[tracing::instrument(skip_all)] async fn get_metrics(&self) -> Result { let mut client = self.client.clone(); let channel_balance_request = lnrpc::ChannelBalanceRequest {}; - let channels_balance_response = client - .channel_balance(channel_balance_request) - .await + let started = Instant::now(); + let channels_balance_response = client.channel_balance(channel_balance_request).await; + record_rpc_call( + started.elapsed(), + "lnrpc.Lightning/ChannelBalance", + &self.server_address, + self.server_port, + &channels_balance_response, + ); + let channels_balance_response = channels_balance_response .with_foreign_context( || format!("querying LND channel balance for {}", self.url), None, diff --git a/components/src/pool/mod.rs b/components/src/pool/mod.rs index 7b7328a..db69c5a 100644 --- a/components/src/pool/mod.rs +++ b/components/src/pool/mod.rs @@ -7,6 +7,7 @@ use crate::pool::cln::grpc::config::ClnGrpcDiscoveryBackendImplementation; use crate::pool::lnd::grpc::config::LndGrpcDiscoveryBackendImplementation; use async_trait::async_trait; use serde::{Deserialize, Serialize}; +use switchgear_error::{ContextError, IntoBoxedTrait}; pub use client_pool::LnClientPool; @@ -18,9 +19,11 @@ pub enum DiscoveryBackendImplementation { LndGrpc(LndGrpcDiscoveryBackendImplementation), } +pub trait LnRpcClientError: ContextError {} + #[async_trait] pub trait LnRpcClient { - type Error: std::error::Error + Send + Sync + 'static; + type Error: LnRpcClientError + IntoBoxedTrait; async fn get_invoice<'a>( &self, diff --git a/components/src/secrets/tls.rs b/components/src/secrets/tls.rs index 78a8d0f..a1c985b 100644 --- a/components/src/secrets/tls.rs +++ b/components/src/secrets/tls.rs @@ -79,7 +79,13 @@ impl ClientCertResolver { let key_hash = hash_bytes(key_bytes); { - let cache = self.cache.lock().unwrap(); + let Ok(cache) = self.cache.lock() else { + tracing::warn!( + "client cert cache lock poisoned for secret {}", + self.cert_secret_name + ); + return None; + }; if let Some(cached) = cache.as_ref() && cached.cert_hash == cert_hash && cached.key_hash == key_hash @@ -124,7 +130,13 @@ impl ClientCertResolver { } }; - let mut cache = self.cache.lock().unwrap(); + let Ok(mut cache) = self.cache.lock() else { + tracing::warn!( + "client cert cache lock poisoned for secret {}", + self.cert_secret_name + ); + return None; + }; *cache = Some(CachedCertKey { cert_hash, key_hash, diff --git a/metrics/Cargo.toml b/metrics/Cargo.toml new file mode 100644 index 0000000..736cfe6 --- /dev/null +++ b/metrics/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "switchgear-metrics" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors = ["Bitshock "] +description = "Metric emission macros (monotonic_counter, counter, histogram, gauge) over tracing for the Switchgear workspace" +documentation = "https://github.com/bitshock-src/switchgear" +homepage = "https://bitshock.com" +repository = "https://github.com/bitshock-src/switchgear" +license = "Apache-2.0" +keywords = ["metrics", "tracing", "opentelemetry", "otlp"] +categories = ["development-tools::debugging", "development-tools::profiling"] +publish = true + +[dependencies] +tracing.workspace = true + +[dev-dependencies] +tracing-subscriber.workspace = true +trybuild = "1" diff --git a/metrics/src/lib.rs b/metrics/src/lib.rs new file mode 100644 index 0000000..81d8572 --- /dev/null +++ b/metrics/src/lib.rs @@ -0,0 +1,137 @@ +//! Metric emission for the Switchgear workspace. +//! +//! Metrics are `tracing` events on a dedicated target, bridged to +//! OpenTelemetry instruments by `tracing-opentelemetry`'s `MetricsLayer`. +//! +//! ``` +//! use std::time::Instant; +//! +//! let started = Instant::now(); +//! switchgear_metrics::histogram!( +//! "swgr_ln_grpc_invoice_request_ms", +//! started.elapsed(), +//! "outcome" => "success", +//! "ln.backend" => "cln", +//! ); +//! ``` +//! +//! # Macros +//! +//! | Macro | Field prefix | Instrument | +//! |---|---|---| +//! | [`monotonic_counter!`] | `monotonic_counter.` | counter | +//! | [`counter!`] | `counter.` | up-down counter | +//! | [`histogram!`] | `histogram.` | histogram | +//! | [`gauge!`] | `gauge.` | gauge | +//! +//! All four share one shape: +//! +//! ```text +//! !( [level: ,] , [, => [%|?] ]* [,] ) +//! ``` +//! +//! * `level:` — optional named parameter, before the name. Defaults to +//! `tracing::Level::INFO`. See [Levels](#levels). +//! * `` — a string literal. The macro prepends the prefix, so +//! `histogram!("swgr_x_ms", …)` emits the field `histogram.swgr_x_ms` and +//! the instrument is named `swgr_x_ms`. +//! * `` — constrained per kind. See [Value types](#value-types). +//! * Label keys — constant expressions: a string literal +//! (`"ln.backend" => …`) or a `const` (`BACKEND => …`). +//! * Label values — anything implementing `tracing::Value`, or `%expr` for +//! `Display` and `?expr` for `Debug`. The three forms mix freely in one +//! call. +//! * Field order — labels first, metric field last. +//! +//! ``` +//! const BACKEND: &str = "ln.backend"; +//! # #[derive(Debug)] struct BackendId(u32); +//! # let backend_id = BackendId(7); +//! # let url = "https://node.example:9736"; +//! switchgear_metrics::counter!( +//! "swgr_x_inflight", +//! 1i64, +//! BACKEND => "cln", // const key +//! "url" => %url, // Display +//! "backend.id" => ?backend_id, // Debug +//! "attempt" => 2u64, // non-string value +//! ); +//! ``` +//! +//! # Levels +//! +//! A metric's level is compared against the `EnvFilter` directive for the +//! metrics target. The threshold comes from service config, not `RUST_LOG`: +//! +//! ```yaml +//! otlp: +//! metrics: +//! temporality: cumulative +//! level: debug # trace|debug|info|warn|error|off; optional, defaults to info +//! ``` +//! +//! ``` +//! use tracing::Level; +//! # let started = std::time::Instant::now(); +//! switchgear_metrics::histogram!( +//! "swgr_ln_grpc_invoice_request_ms", +//! started.elapsed(), +//! "ln.backend" => "cln", +//! ); +//! +//! switchgear_metrics::histogram!( +//! level: Level::DEBUG, +//! "swgr_ln_grpc_invoice_detail_ms", +//! started.elapsed(), +//! "ln.backend" => "cln", +//! ); +//! ``` +//! +//! | `otlp.metrics` | `level` | directive | INFO metric | DEBUG metric | +//! |---|---|---|---|---| +//! | present | unset | `swgr::metrics=info` | recorded | dropped | +//! | present | `debug` | `swgr::metrics=debug` | recorded | recorded | +//! | present | `off` | `swgr::metrics=off` | dropped | dropped | +//! | absent, or `OTEL_METRICS_EXPORTER=none` | — | `swgr::metrics=off` | dropped | dropped | +//! +//! `otlp.metrics.level` is the sole control over metric verbosity at any +//! `RUST_LOG`. A `RUST_LOG` directive naming the metrics target is overridden. +//! +//! # Value types +//! +//! | Macro | Accepts | Recorded as | +//! |---|---|---| +//! | [`monotonic_counter!`] | `u8`–`u64`, `usize`, `f32`, `f64`, `Duration` | `u64` / `f64` | +//! | [`counter!`] | `i8`–`i64`, `isize`, `u8`–`u32`, `f32`, `f64` | `i64` / `f64` | +//! | [`histogram!`] | `u8`–`u64`, `usize`, `f32`, `f64`, `Duration` | `u64` / `f64` | +//! | [`gauge!`] | all integers, `f32`, `f64`, `Duration` | `u64` / `i64` / `f64` | +//! +//! A `Duration` records as milliseconds in `f64` — on [`histogram!`], +//! [`gauge!`] and [`monotonic_counter!`], but not [`counter!`]. +//! +//! # Metric names +//! +//! Record a given metric name as one numeric type everywhere. Instrument maps +//! are keyed by name, so the same name recorded as `u64` in one place and +//! `f64` in another creates two instruments and splits the series. + +mod macros; + +#[doc(hidden)] +pub mod value; + +#[doc(hidden)] +pub mod __private { + pub use tracing; + + pub const TARGET: &str = "swgr::metrics"; +} + +/// The tracing target metric events are emitted on, for building subscriber +/// filters. +/// +/// Constant: it does not vary by service, by whether metrics are enabled, or +/// at runtime. +pub fn metrics_target() -> &'static str { + __private::TARGET +} diff --git a/metrics/src/macros.rs b/metrics/src/macros.rs new file mode 100644 index 0000000..82bf6e0 --- /dev/null +++ b/metrics/src/macros.rs @@ -0,0 +1,149 @@ +#[doc(hidden)] +#[macro_export] +macro_rules! __metric { + (@munch [$lvl:expr] [$field:expr] [$value:expr] [$($out:tt)*]) => { + $crate::__private::tracing::event!( + target: $crate::__private::TARGET, + $lvl, + { $($out)* { $field } = $value } + ) + }; + + (@munch [$lvl:expr] [$field:expr] [$value:expr] [$($out:tt)*] + $key:expr => %$val:expr $(, $($rest:tt)*)?) => { + $crate::__metric!(@munch [$lvl] [$field] [$value] [$($out)* { $key } = %$val,] $($($rest)*)?) + }; + + (@munch [$lvl:expr] [$field:expr] [$value:expr] [$($out:tt)*] + $key:expr => ?$val:expr $(, $($rest:tt)*)?) => { + $crate::__metric!(@munch [$lvl] [$field] [$value] [$($out)* { $key } = ?$val,] $($($rest)*)?) + }; + + (@munch [$lvl:expr] [$field:expr] [$value:expr] [$($out:tt)*] + $key:expr => $val:expr $(, $($rest:tt)*)?) => { + $crate::__metric!(@munch [$lvl] [$field] [$value] [$($out)* { $key } = $val,] $($($rest)*)?) + }; + + ($lvl:expr, $field:expr, $value:expr $(, $($labels:tt)*)?) => { + $crate::__metric!(@munch [$lvl] [$field] [$value] [] $($($labels)*)?) + }; +} + +/// Records a monotonically increasing count, as `monotonic_counter.`. +/// +/// ``` +/// switchgear_metrics::monotonic_counter!("swgr_x_total", 1u64, "outcome" => "success"); +/// ``` +/// +/// Takes an unsigned integer, `usize`, a float, or a `Duration` (recorded as +/// milliseconds). See [Value types](crate#value-types). +#[macro_export] +macro_rules! monotonic_counter { + (level: $lvl:expr, $name:literal, $value:expr $(, $($labels:tt)*)?) => { + $crate::__metric!( + $lvl, + ::core::concat!("monotonic_counter.", $name), + $crate::value::monotonic_counter_value($value) + $(, $($labels)*)? + ) + }; + ($name:literal, $value:expr $(, $($labels:tt)*)?) => { + $crate::monotonic_counter!( + level: $crate::__private::tracing::Level::INFO, + $name, + $value + $(, $($labels)*)? + ) + }; +} + +/// Records a count that can go up or down, as `counter.`. +/// +/// ``` +/// switchgear_metrics::counter!("swgr_x_inflight", -1i64, "outcome" => "success"); +/// ``` +/// +/// Takes a signed integer, `isize`, a `u8`/`u16`/`u32`, or a float. See +/// [Value types](crate#value-types). +#[macro_export] +macro_rules! counter { + (level: $lvl:expr, $name:literal, $value:expr $(, $($labels:tt)*)?) => { + $crate::__metric!( + $lvl, + ::core::concat!("counter.", $name), + $crate::value::counter_value($value) + $(, $($labels)*)? + ) + }; + ($name:literal, $value:expr $(, $($labels:tt)*)?) => { + $crate::counter!( + level: $crate::__private::tracing::Level::INFO, + $name, + $value + $(, $($labels)*)? + ) + }; +} + +/// Records an observation in a distribution, as `histogram.`. +/// +/// ``` +/// use std::time::Instant; +/// let started = Instant::now(); +/// switchgear_metrics::histogram!( +/// "swgr_ln_grpc_invoice_request_ms", +/// started.elapsed(), +/// "ln.backend" => "cln", +/// ); +/// ``` +/// +/// Takes an unsigned integer, `usize`, a float, or a `Duration` (recorded as +/// milliseconds). See [Value types](crate#value-types). +#[macro_export] +macro_rules! histogram { + (level: $lvl:expr, $name:literal, $value:expr $(, $($labels:tt)*)?) => { + $crate::__metric!( + $lvl, + ::core::concat!("histogram.", $name), + $crate::value::histogram_value($value) + $(, $($labels)*)? + ) + }; + ($name:literal, $value:expr $(, $($labels:tt)*)?) => { + $crate::histogram!( + level: $crate::__private::tracing::Level::INFO, + $name, + $value + $(, $($labels)*)? + ) + }; +} + +/// Records the current value of something that goes up and down, as +/// `gauge.`. +/// +/// ``` +/// switchgear_metrics::gauge!("swgr_x_open", 3i32, "ln.backend" => "lnd"); +/// ``` +/// +/// Takes any integer, a float, or a `Duration` (recorded as milliseconds). +/// See [Value types](crate#value-types). +#[macro_export] +macro_rules! gauge { + (level: $lvl:expr, $name:literal, $value:expr $(, $($labels:tt)*)?) => { + $crate::__metric!( + $lvl, + ::core::concat!("gauge.", $name), + $crate::value::gauge_value($value) + $(, $($labels)*)? + ) + }; + ($name:literal, $value:expr $(, $($labels:tt)*)?) => { + $crate::gauge!( + level: $crate::__private::tracing::Level::INFO, + $name, + $value + $(, $($labels)*)? + ) + }; +} diff --git a/metrics/src/value.rs b/metrics/src/value.rs new file mode 100644 index 0000000..9003b2f --- /dev/null +++ b/metrics/src/value.rs @@ -0,0 +1,340 @@ +use std::time::Duration; + +#[inline] +fn millis(d: Duration) -> f64 { + d.as_secs_f64() * 1_000.0 +} + +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be recorded as a monotonic counter value", + label = "monotonic counters take an unsigned integer, a float, or a Duration", + note = "tracing-opentelemetry casts a signed value to u64 for `monotonic_counter.`, so a \ + negative one wraps to an enormous positive count", + note = "cast explicitly if you meant it, e.g. `value as u64` or `value as f64`" +)] +pub trait MonotonicCounterValue { + type Out: tracing::Value; + fn coerce(self) -> Self::Out; +} + +impl MonotonicCounterValue for u8 { + type Out = u64; + fn coerce(self) -> u64 { + u64::from(self) + } +} + +impl MonotonicCounterValue for u16 { + type Out = u64; + fn coerce(self) -> u64 { + u64::from(self) + } +} + +impl MonotonicCounterValue for u32 { + type Out = u64; + fn coerce(self) -> u64 { + u64::from(self) + } +} + +impl MonotonicCounterValue for u64 { + type Out = u64; + fn coerce(self) -> u64 { + self + } +} + +impl MonotonicCounterValue for usize { + type Out = u64; + fn coerce(self) -> u64 { + self as u64 + } +} + +impl MonotonicCounterValue for f32 { + type Out = f64; + fn coerce(self) -> f64 { + f64::from(self) + } +} + +impl MonotonicCounterValue for f64 { + type Out = f64; + fn coerce(self) -> f64 { + self + } +} + +impl MonotonicCounterValue for Duration { + type Out = f64; + fn coerce(self) -> f64 { + millis(self) + } +} +#[inline] +pub fn monotonic_counter_value(v: V) -> V::Out { + v.coerce() +} +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be recorded as a counter value", + label = "counters take a signed integer, a `u8`/`u16`/`u32`, or a float", + note = "tracing-opentelemetry drops a `counter.` u64 above i64::MAX, so `u64` and `usize` \ + are not accepted", + note = "a Duration is not a counter value: time an operation with `histogram!`, or report \ + a level with `gauge!`", + note = "cast explicitly if you meant it, e.g. `value as i64` or `value as f64`" +)] +pub trait CounterValue { + type Out: tracing::Value; + fn coerce(self) -> Self::Out; +} + +impl CounterValue for i8 { + type Out = i64; + fn coerce(self) -> i64 { + i64::from(self) + } +} + +impl CounterValue for i16 { + type Out = i64; + fn coerce(self) -> i64 { + i64::from(self) + } +} + +impl CounterValue for i32 { + type Out = i64; + fn coerce(self) -> i64 { + i64::from(self) + } +} + +impl CounterValue for i64 { + type Out = i64; + fn coerce(self) -> i64 { + self + } +} + +impl CounterValue for isize { + type Out = i64; + fn coerce(self) -> i64 { + self as i64 + } +} + +impl CounterValue for u8 { + type Out = i64; + fn coerce(self) -> i64 { + i64::from(self) + } +} + +impl CounterValue for u16 { + type Out = i64; + fn coerce(self) -> i64 { + i64::from(self) + } +} + +impl CounterValue for u32 { + type Out = i64; + fn coerce(self) -> i64 { + i64::from(self) + } +} + +impl CounterValue for f32 { + type Out = f64; + fn coerce(self) -> f64 { + f64::from(self) + } +} + +impl CounterValue for f64 { + type Out = f64; + fn coerce(self) -> f64 { + self + } +} +#[inline] +pub fn counter_value(v: V) -> V::Out { + v.coerce() +} +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be recorded as a histogram value", + label = "histograms take an unsigned integer, a float, or a Duration", + note = "tracing-opentelemetry has no i64 histogram: a signed value is silently recorded as \ + an attribute, not a metric", + note = "cast explicitly if you meant it, e.g. `value as u64` or `value as f64`" +)] +pub trait HistogramValue { + type Out: tracing::Value; + fn coerce(self) -> Self::Out; +} + +impl HistogramValue for u8 { + type Out = u64; + fn coerce(self) -> u64 { + u64::from(self) + } +} + +impl HistogramValue for u16 { + type Out = u64; + fn coerce(self) -> u64 { + u64::from(self) + } +} + +impl HistogramValue for u32 { + type Out = u64; + fn coerce(self) -> u64 { + u64::from(self) + } +} + +impl HistogramValue for u64 { + type Out = u64; + fn coerce(self) -> u64 { + self + } +} + +impl HistogramValue for usize { + type Out = u64; + fn coerce(self) -> u64 { + self as u64 + } +} + +impl HistogramValue for f32 { + type Out = f64; + fn coerce(self) -> f64 { + f64::from(self) + } +} + +impl HistogramValue for f64 { + type Out = f64; + fn coerce(self) -> f64 { + self + } +} + +impl HistogramValue for Duration { + type Out = f64; + fn coerce(self) -> f64 { + millis(self) + } +} +#[inline] +pub fn histogram_value(v: V) -> V::Out { + v.coerce() +} +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be recorded as a gauge value", + label = "gauges take any integer, a float, or a Duration", + note = "cast explicitly if you meant it, e.g. `value as i64` or `value as f64`" +)] +pub trait GaugeValue { + type Out: tracing::Value; + fn coerce(self) -> Self::Out; +} + +impl GaugeValue for u8 { + type Out = u64; + fn coerce(self) -> u64 { + u64::from(self) + } +} + +impl GaugeValue for u16 { + type Out = u64; + fn coerce(self) -> u64 { + u64::from(self) + } +} + +impl GaugeValue for u32 { + type Out = u64; + fn coerce(self) -> u64 { + u64::from(self) + } +} + +impl GaugeValue for u64 { + type Out = u64; + fn coerce(self) -> u64 { + self + } +} + +impl GaugeValue for usize { + type Out = u64; + fn coerce(self) -> u64 { + self as u64 + } +} + +impl GaugeValue for i8 { + type Out = i64; + fn coerce(self) -> i64 { + i64::from(self) + } +} + +impl GaugeValue for i16 { + type Out = i64; + fn coerce(self) -> i64 { + i64::from(self) + } +} + +impl GaugeValue for i32 { + type Out = i64; + fn coerce(self) -> i64 { + i64::from(self) + } +} + +impl GaugeValue for i64 { + type Out = i64; + fn coerce(self) -> i64 { + self + } +} + +impl GaugeValue for isize { + type Out = i64; + fn coerce(self) -> i64 { + self as i64 + } +} + +impl GaugeValue for f32 { + type Out = f64; + fn coerce(self) -> f64 { + f64::from(self) + } +} + +impl GaugeValue for f64 { + type Out = f64; + fn coerce(self) -> f64 { + self + } +} + +impl GaugeValue for Duration { + type Out = f64; + fn coerce(self) -> f64 { + millis(self) + } +} +#[inline] +pub fn gauge_value(v: V) -> V::Out { + v.coerce() +} diff --git a/metrics/tests/compile_fail/duration_counter.rs b/metrics/tests/compile_fail/duration_counter.rs new file mode 100644 index 0000000..8121f4a --- /dev/null +++ b/metrics/tests/compile_fail/duration_counter.rs @@ -0,0 +1,7 @@ +// A Duration is not an up-down counter value. + +use std::time::Duration; + +fn main() { + switchgear_metrics::counter!("m", Duration::from_secs(1)); +} diff --git a/metrics/tests/compile_fail/duration_counter.stderr b/metrics/tests/compile_fail/duration_counter.stderr new file mode 100644 index 0000000..95d4a4a --- /dev/null +++ b/metrics/tests/compile_fail/duration_counter.stderr @@ -0,0 +1,50 @@ +error[E0277]: `Duration` cannot be recorded as a counter value + --> tests/compile_fail/duration_counter.rs:6:39 + | +6 | switchgear_metrics::counter!("m", Duration::from_secs(1)); + | ----------------------------------^^^^^^^^^^^^^^^^^^^^^^- + | | | + | | counters take a signed integer, a `u8`/`u16`/`u32`, or a float + | required by a bound introduced by this call + | + = help: the trait `switchgear_metrics::value::CounterValue` is not implemented for `Duration` + = note: tracing-opentelemetry drops a `counter.` u64 above i64::MAX, so `u64` and `usize` are not accepted + = note: a Duration is not a counter value: time an operation with `histogram!`, or report a level with `gauge!` + = note: cast explicitly if you meant it, e.g. `value as i64` or `value as f64` + = help: the following other types implement trait `switchgear_metrics::value::CounterValue`: + f32 + f64 + i16 + i32 + i64 + i8 + isize + u16 + and $N others +note: required by a bound in `switchgear_metrics::value::counter_value` + --> src/value.rs + | + | pub fn counter_value(v: V) -> V::Out { + | ^^^^^^^^^^^^ required by this bound in `counter_value` + +error[E0277]: `Duration` cannot be recorded as a counter value + --> tests/compile_fail/duration_counter.rs:6:5 + | +6 | switchgear_metrics::counter!("m", Duration::from_secs(1)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ counters take a signed integer, a `u8`/`u16`/`u32`, or a float + | + = help: the trait `switchgear_metrics::value::CounterValue` is not implemented for `Duration` + = note: tracing-opentelemetry drops a `counter.` u64 above i64::MAX, so `u64` and `usize` are not accepted + = note: a Duration is not a counter value: time an operation with `histogram!`, or report a level with `gauge!` + = note: cast explicitly if you meant it, e.g. `value as i64` or `value as f64` + = help: the following other types implement trait `switchgear_metrics::value::CounterValue`: + f32 + f64 + i16 + i32 + i64 + i8 + isize + u16 + and $N others + = note: this error originates in the macro `$crate::counter` which comes from the expansion of the macro `switchgear_metrics::counter` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/metrics/tests/compile_fail/non_const_label_key.rs b/metrics/tests/compile_fail/non_const_label_key.rs new file mode 100644 index 0000000..2c49f9a --- /dev/null +++ b/metrics/tests/compile_fail/non_const_label_key.rs @@ -0,0 +1,11 @@ +// A label key becomes part of the callsite's 'static metadata, so it has to +// be a constant expression: E0435. + +// `key` reads as used here, but the expansion only reaches it from a `static` +// initialiser, so rustc reports it unused as well. That warning is noise on +// top of the error under test. +#[allow(unused_variables)] +fn main() { + let key = "ln.backend"; + switchgear_metrics::histogram!("m", 1u64, key => "cln"); +} diff --git a/metrics/tests/compile_fail/non_const_label_key.stderr b/metrics/tests/compile_fail/non_const_label_key.stderr new file mode 100644 index 0000000..fec3f3c --- /dev/null +++ b/metrics/tests/compile_fail/non_const_label_key.stderr @@ -0,0 +1,12 @@ +error[E0435]: attempt to use a non-constant value in a constant + --> tests/compile_fail/non_const_label_key.rs:10:47 + | +10 | switchgear_metrics::histogram!("m", 1u64, key => "cln"); + | ^^^ non-constant value + | +help: consider using `let` instead of `static` + --> $CARGO/tracing-$VERSION/src/macros.rs + | + - static __CALLSITE: $crate::callsite::DefaultCallsite = $crate::callsite2! { + + let __CALLSITE: $crate::callsite::DefaultCallsite = $crate::callsite2! { + | diff --git a/metrics/tests/compile_fail/non_numeric.rs b/metrics/tests/compile_fail/non_numeric.rs new file mode 100644 index 0000000..4863980 --- /dev/null +++ b/metrics/tests/compile_fail/non_numeric.rs @@ -0,0 +1,5 @@ +// A metric value has to be a number. + +fn main() { + switchgear_metrics::histogram!("m", "nope"); +} diff --git a/metrics/tests/compile_fail/non_numeric.stderr b/metrics/tests/compile_fail/non_numeric.stderr new file mode 100644 index 0000000..e9ebd62 --- /dev/null +++ b/metrics/tests/compile_fail/non_numeric.stderr @@ -0,0 +1,46 @@ +error[E0277]: `&str` cannot be recorded as a histogram value + --> tests/compile_fail/non_numeric.rs:4:41 + | +4 | switchgear_metrics::histogram!("m", "nope"); + | ------------------------------------^^^^^^- + | | | + | | histograms take an unsigned integer, a float, or a Duration + | required by a bound introduced by this call + | + = help: the trait `switchgear_metrics::value::HistogramValue` is not implemented for `&str` + = note: tracing-opentelemetry has no i64 histogram: a signed value is silently recorded as an attribute, not a metric + = note: cast explicitly if you meant it, e.g. `value as u64` or `value as f64` + = help: the following other types implement trait `switchgear_metrics::value::HistogramValue`: + Duration + f32 + f64 + u16 + u32 + u64 + u8 + usize +note: required by a bound in `switchgear_metrics::value::histogram_value` + --> src/value.rs + | + | pub fn histogram_value(v: V) -> V::Out { + | ^^^^^^^^^^^^^^ required by this bound in `histogram_value` + +error[E0277]: `&str` cannot be recorded as a histogram value + --> tests/compile_fail/non_numeric.rs:4:5 + | +4 | switchgear_metrics::histogram!("m", "nope"); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ histograms take an unsigned integer, a float, or a Duration + | + = help: the trait `switchgear_metrics::value::HistogramValue` is not implemented for `&str` + = note: tracing-opentelemetry has no i64 histogram: a signed value is silently recorded as an attribute, not a metric + = note: cast explicitly if you meant it, e.g. `value as u64` or `value as f64` + = help: the following other types implement trait `switchgear_metrics::value::HistogramValue`: + Duration + f32 + f64 + u16 + u32 + u64 + u8 + usize + = note: this error originates in the macro `$crate::histogram` which comes from the expansion of the macro `switchgear_metrics::histogram` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/metrics/tests/compile_fail/signed_where_unsigned.rs b/metrics/tests/compile_fail/signed_where_unsigned.rs new file mode 100644 index 0000000..65aa77f --- /dev/null +++ b/metrics/tests/compile_fail/signed_where_unsigned.rs @@ -0,0 +1,7 @@ +// `histogram.` has no i64 arm in tracing-opentelemetry, and +// `monotonic_counter.` casts a signed value to u64. + +fn main() { + switchgear_metrics::histogram!("m", -3i64); + switchgear_metrics::monotonic_counter!("m", -1i32); +} diff --git a/metrics/tests/compile_fail/signed_where_unsigned.stderr b/metrics/tests/compile_fail/signed_where_unsigned.stderr new file mode 100644 index 0000000..8e55cea --- /dev/null +++ b/metrics/tests/compile_fail/signed_where_unsigned.stderr @@ -0,0 +1,89 @@ +error[E0277]: `i64` cannot be recorded as a histogram value + --> tests/compile_fail/signed_where_unsigned.rs:5:41 + | +5 | switchgear_metrics::histogram!("m", -3i64); + | ------------------------------------^^^^^- + | | | + | | histograms take an unsigned integer, a float, or a Duration + | required by a bound introduced by this call + | + = help: the trait `switchgear_metrics::value::HistogramValue` is not implemented for `i64` + = note: tracing-opentelemetry has no i64 histogram: a signed value is silently recorded as an attribute, not a metric + = note: cast explicitly if you meant it, e.g. `value as u64` or `value as f64` + = help: the following other types implement trait `switchgear_metrics::value::HistogramValue`: + f32 + f64 + u16 + u32 + u64 + u8 + usize +note: required by a bound in `switchgear_metrics::value::histogram_value` + --> src/value.rs + | + | pub fn histogram_value(v: V) -> V::Out { + | ^^^^^^^^^^^^^^ required by this bound in `histogram_value` + +error[E0277]: `i32` cannot be recorded as a monotonic counter value + --> tests/compile_fail/signed_where_unsigned.rs:6:49 + | +6 | switchgear_metrics::monotonic_counter!("m", -1i32); + | --------------------------------------------^^^^^- + | | | + | | monotonic counters take an unsigned integer, a float, or a Duration + | required by a bound introduced by this call + | + = help: the trait `switchgear_metrics::value::MonotonicCounterValue` is not implemented for `i32` + = note: tracing-opentelemetry casts a signed value to u64 for `monotonic_counter.`, so a negative one wraps to an enormous positive count + = note: cast explicitly if you meant it, e.g. `value as u64` or `value as f64` + = help: the following other types implement trait `switchgear_metrics::value::MonotonicCounterValue`: + f32 + f64 + u16 + u32 + u64 + u8 + usize +note: required by a bound in `switchgear_metrics::value::monotonic_counter_value` + --> src/value.rs + | + | pub fn monotonic_counter_value(v: V) -> V::Out { + | ^^^^^^^^^^^^^^^^^^^^^ required by this bound in `monotonic_counter_value` + +error[E0277]: `i64` cannot be recorded as a histogram value + --> tests/compile_fail/signed_where_unsigned.rs:5:5 + | +5 | switchgear_metrics::histogram!("m", -3i64); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ histograms take an unsigned integer, a float, or a Duration + | + = help: the trait `switchgear_metrics::value::HistogramValue` is not implemented for `i64` + = note: tracing-opentelemetry has no i64 histogram: a signed value is silently recorded as an attribute, not a metric + = note: cast explicitly if you meant it, e.g. `value as u64` or `value as f64` + = help: the following other types implement trait `switchgear_metrics::value::HistogramValue`: + f32 + f64 + u16 + u32 + u64 + u8 + usize + = note: this error originates in the macro `$crate::histogram` which comes from the expansion of the macro `switchgear_metrics::histogram` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: `i32` cannot be recorded as a monotonic counter value + --> tests/compile_fail/signed_where_unsigned.rs:6:5 + | +6 | switchgear_metrics::monotonic_counter!("m", -1i32); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ monotonic counters take an unsigned integer, a float, or a Duration + | + = help: the trait `switchgear_metrics::value::MonotonicCounterValue` is not implemented for `i32` + = note: tracing-opentelemetry casts a signed value to u64 for `monotonic_counter.`, so a negative one wraps to an enormous positive count + = note: cast explicitly if you meant it, e.g. `value as u64` or `value as f64` + = help: the following other types implement trait `switchgear_metrics::value::MonotonicCounterValue`: + f32 + f64 + u16 + u32 + u64 + u8 + usize + = note: this error originates in the macro `$crate::monotonic_counter` which comes from the expansion of the macro `switchgear_metrics::monotonic_counter` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/metrics/tests/compile_fail/unsigned_counter.rs b/metrics/tests/compile_fail/unsigned_counter.rs new file mode 100644 index 0000000..91eaeb0 --- /dev/null +++ b/metrics/tests/compile_fail/unsigned_counter.rs @@ -0,0 +1,5 @@ +// `counter.` drops a u64 above i64::MAX rather than recording it. + +fn main() { + switchgear_metrics::counter!("m", 1u64); +} diff --git a/metrics/tests/compile_fail/unsigned_counter.stderr b/metrics/tests/compile_fail/unsigned_counter.stderr new file mode 100644 index 0000000..5e72682 --- /dev/null +++ b/metrics/tests/compile_fail/unsigned_counter.stderr @@ -0,0 +1,50 @@ +error[E0277]: `u64` cannot be recorded as a counter value + --> tests/compile_fail/unsigned_counter.rs:4:39 + | +4 | switchgear_metrics::counter!("m", 1u64); + | ----------------------------------^^^^- + | | | + | | counters take a signed integer, a `u8`/`u16`/`u32`, or a float + | required by a bound introduced by this call + | + = help: the trait `switchgear_metrics::value::CounterValue` is not implemented for `u64` + = note: tracing-opentelemetry drops a `counter.` u64 above i64::MAX, so `u64` and `usize` are not accepted + = note: a Duration is not a counter value: time an operation with `histogram!`, or report a level with `gauge!` + = note: cast explicitly if you meant it, e.g. `value as i64` or `value as f64` + = help: the following other types implement trait `switchgear_metrics::value::CounterValue`: + f32 + f64 + i16 + i32 + i64 + i8 + isize + u16 + and $N others +note: required by a bound in `switchgear_metrics::value::counter_value` + --> src/value.rs + | + | pub fn counter_value(v: V) -> V::Out { + | ^^^^^^^^^^^^ required by this bound in `counter_value` + +error[E0277]: `u64` cannot be recorded as a counter value + --> tests/compile_fail/unsigned_counter.rs:4:5 + | +4 | switchgear_metrics::counter!("m", 1u64); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ counters take a signed integer, a `u8`/`u16`/`u32`, or a float + | + = help: the trait `switchgear_metrics::value::CounterValue` is not implemented for `u64` + = note: tracing-opentelemetry drops a `counter.` u64 above i64::MAX, so `u64` and `usize` are not accepted + = note: a Duration is not a counter value: time an operation with `histogram!`, or report a level with `gauge!` + = note: cast explicitly if you meant it, e.g. `value as i64` or `value as f64` + = help: the following other types implement trait `switchgear_metrics::value::CounterValue`: + f32 + f64 + i16 + i32 + i64 + i8 + isize + u16 + and $N others + = note: this error originates in the macro `$crate::counter` which comes from the expansion of the macro `switchgear_metrics::counter` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/metrics/tests/metrics.rs b/metrics/tests/metrics.rs new file mode 100644 index 0000000..90086a0 --- /dev/null +++ b/metrics/tests/metrics.rs @@ -0,0 +1,610 @@ +//! Behavioural tests for the emission macros. +//! +//! Everything is asserted through a subscriber rather than by inspecting +//! expansions: what matters is the event that reaches a layer — its target, +//! its level, its field names, their order, and the recorded value types. +//! +//! The subscribers here are shaped like the one `server` installs per +//! service: an optional metrics layer selected on the metrics target, a +//! global `EnvFilter` carrying `swgr::metrics=`, and a log layer that +//! excludes the metrics target. + +use std::fmt; +use std::io; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; + +use tracing::field::{Field, Visit}; +use tracing::{Event, Level, Metadata, Subscriber}; +use tracing_subscriber::filter::filter_fn; +use tracing_subscriber::layer::{Context, SubscriberExt}; +use tracing_subscriber::registry::LookupSpan; +use tracing_subscriber::{EnvFilter, Layer}; + +// --------------------------------------------------------------------------- +// capture +// --------------------------------------------------------------------------- + +/// A recorded field value, by the `Visit` method it arrived on. +#[derive(Clone, Debug, PartialEq)] +enum Val { + U64(u64), + I64(i64), + F64(f64), + Bool(bool), + Str(String), + /// `?expr`, and also `%expr` — `tracing` records a `DisplayValue` through + /// `record_debug` with a `Debug` impl that defers to `Display`. + Debug(String), +} + +#[derive(Clone, Debug)] +struct Captured { + target: String, + level: Level, + fields: Vec<(String, Val)>, +} + +impl Captured { + fn names(&self) -> Vec<&str> { + self.fields.iter().map(|(k, _)| k.as_str()).collect() + } + + fn get(&self, name: &str) -> Option<&Val> { + self.fields.iter().find(|(k, _)| k == name).map(|(_, v)| v) + } +} + +#[derive(Clone, Default)] +struct Capture(Arc>>); + +impl Capture { + fn events(&self) -> Vec { + lock(&self.0).clone() + } +} + +struct CaptureVisitor<'a>(&'a mut Vec<(String, Val)>); + +impl Visit for CaptureVisitor<'_> { + fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { + self.0 + .push((field.name().to_owned(), Val::Debug(format!("{value:?}")))); + } + + fn record_u64(&mut self, field: &Field, value: u64) { + self.0.push((field.name().to_owned(), Val::U64(value))); + } + + fn record_i64(&mut self, field: &Field, value: i64) { + self.0.push((field.name().to_owned(), Val::I64(value))); + } + + fn record_f64(&mut self, field: &Field, value: f64) { + self.0.push((field.name().to_owned(), Val::F64(value))); + } + + fn record_bool(&mut self, field: &Field, value: bool) { + self.0.push((field.name().to_owned(), Val::Bool(value))); + } + + fn record_str(&mut self, field: &Field, value: &str) { + self.0 + .push((field.name().to_owned(), Val::Str(value.to_owned()))); + } +} + +impl Layer for Capture { + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + let mut fields = Vec::new(); + event.record(&mut CaptureVisitor(&mut fields)); + let meta = event.metadata(); + lock(&self.0).push(Captured { + target: meta.target().to_owned(), + level: *meta.level(), + fields, + }); + } +} + +/// An in-memory `MakeWriter`, for asserting on what the log layer writes. +#[derive(Clone, Default)] +struct LogBuffer(Arc>>); + +impl LogBuffer { + fn contents(&self) -> String { + String::from_utf8_lossy(&lock(&self.0)).into_owned() + } +} + +impl io::Write for LogBuffer { + fn write(&mut self, buf: &[u8]) -> io::Result { + lock(&self.0).extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for LogBuffer { + type Writer = LogBuffer; + + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } +} + +/// A poisoned lock in a test is not interesting: the panic that poisoned it +/// is what the runner reports. +fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(|e| e.into_inner()) +} + +// --------------------------------------------------------------------------- +// subscribers +// --------------------------------------------------------------------------- + +/// Reproduces `tracing_opentelemetry::MetricsFilter`, the per-layer filter +/// `MetricsLayer` carries internally. Kept here so this crate need not depend +/// on `tracing-opentelemetry`; `server` exercises the real one. +fn is_metrics_event(meta: &Metadata<'_>) -> bool { + meta.is_event() + && meta.fields().iter().any(|field| { + let name = field.name(); + name.starts_with("monotonic_counter.") + || name.starts_with("counter.") + || name.starts_with("histogram.") + || name.starts_with("gauge.") + }) +} + +/// `server`'s `metrics_env_filter`: `RUST_LOG` plus a directive pinning the +/// metrics target to the configured level. +fn env_filter(rust_log: &str, metrics_level: &str) -> EnvFilter { + let target = switchgear_metrics::metrics_target(); + let directive = format!("{target}={metrics_level}"); + EnvFilter::new(rust_log).add_directive( + directive + .parse() + .unwrap_or_else(|e| panic!("parsing {directive}: {e}")), + ) +} + +fn metrics_layer(capture: Option) -> impl Layer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + let target = switchgear_metrics::metrics_target(); + capture.with_filter(filter_fn(move |m| { + m.target() == target && is_metrics_event(m) + })) +} + +/// Metrics layer present, everything recorded: the shape most of these tests +/// want. +fn recording(f: impl FnOnce() -> T) -> (T, Vec) { + with_metrics("trace", "info", f) +} + +fn with_metrics( + metrics_level: &str, + rust_log: &str, + f: impl FnOnce() -> T, +) -> (T, Vec) { + let capture = Capture::default(); + let subscriber = tracing_subscriber::registry() + .with(metrics_layer(Some(capture.clone()))) + .with(env_filter(rust_log, metrics_level)); + let out = tracing::subscriber::with_default(subscriber, f); + (out, capture.events()) +} + +fn only(events: Vec) -> Captured { + assert_eq!(events.len(), 1, "expected exactly one event: {events:?}"); + events + .into_iter() + .next() + .unwrap_or_else(|| unreachable!("length asserted")) +} + +// --------------------------------------------------------------------------- +// the four macros +// --------------------------------------------------------------------------- + +#[test] +fn monotonic_counter_emits_a_prefixed_field_on_the_metrics_target() { + let (_, events) = recording(|| { + switchgear_metrics::monotonic_counter!("swgr_x_total", 7u64, "outcome" => "success"); + }); + + let event = only(events); + assert_eq!(event.target, "swgr::metrics"); + assert_eq!(event.target, switchgear_metrics::metrics_target()); + assert_eq!(event.level, Level::INFO); + assert_eq!(event.names(), ["outcome", "monotonic_counter.swgr_x_total"]); + assert_eq!( + event.get("monotonic_counter.swgr_x_total"), + Some(&Val::U64(7)) + ); +} + +#[test] +fn counter_emits_a_prefixed_field_on_the_metrics_target() { + let (_, events) = recording(|| { + switchgear_metrics::counter!("swgr_x_inflight", 7i64, "outcome" => "success"); + }); + + let event = only(events); + assert_eq!(event.target, "swgr::metrics"); + assert_eq!(event.level, Level::INFO); + assert_eq!(event.names(), ["outcome", "counter.swgr_x_inflight"]); + assert_eq!(event.get("counter.swgr_x_inflight"), Some(&Val::I64(7))); +} + +#[test] +fn histogram_emits_a_prefixed_field_on_the_metrics_target() { + let (_, events) = recording(|| { + switchgear_metrics::histogram!("swgr_x_ms", 7u64, "outcome" => "success"); + }); + + let event = only(events); + assert_eq!(event.target, "swgr::metrics"); + assert_eq!(event.level, Level::INFO); + assert_eq!(event.names(), ["outcome", "histogram.swgr_x_ms"]); + assert_eq!(event.get("histogram.swgr_x_ms"), Some(&Val::U64(7))); +} + +#[test] +fn gauge_emits_a_prefixed_field_on_the_metrics_target() { + let (_, events) = recording(|| { + switchgear_metrics::gauge!("swgr_x_open", 7u64, "outcome" => "success"); + }); + + let event = only(events); + assert_eq!(event.target, "swgr::metrics"); + assert_eq!(event.level, Level::INFO); + assert_eq!(event.names(), ["outcome", "gauge.swgr_x_open"]); + assert_eq!(event.get("gauge.swgr_x_open"), Some(&Val::U64(7))); +} + +#[test] +fn the_metric_field_comes_after_every_label() { + let (_, events) = recording(|| { + switchgear_metrics::histogram!( + "swgr_x_ms", + 1u64, + "a" => "1", + "b" => "2", + "c" => "3", + ); + }); + + assert_eq!(only(events).names(), ["a", "b", "c", "histogram.swgr_x_ms"]); +} + +// --------------------------------------------------------------------------- +// label forms +// --------------------------------------------------------------------------- + +#[test] +fn no_label_form_emits_only_the_metric_field() { + let (_, events) = recording(|| { + switchgear_metrics::counter!("swgr_x_inflight", 1i64); + }); + + assert_eq!(only(events).names(), ["counter.swgr_x_inflight"]); +} + +#[test] +fn a_trailing_comma_is_accepted_with_and_without_labels() { + let (_, events) = recording(|| { + switchgear_metrics::counter!("swgr_a", 1i64,); + switchgear_metrics::counter!("swgr_b", 1i64, "k" => "v",); + }); + + assert_eq!(events.len(), 2); + assert_eq!(events[0].names(), ["counter.swgr_a"]); + assert_eq!(events[1].names(), ["k", "counter.swgr_b"]); +} + +#[test] +fn label_forms_mix_in_one_call() { + const BACKEND: &str = "ln.backend"; + + let url = "https://node.example:9736"; + + let (_, events) = recording(|| { + switchgear_metrics::gauge!( + "swgr_x_open", + 1u64, + "outcome" => "success", // string-literal key, string value + BACKEND => "cln", // const key + "attempt" => 2u64, // non-string value + "url" => %url, // Display + "backend.id" => ?Some(9u32) // Debug + ); + }); + + let event = only(events); + assert_eq!( + event.names(), + [ + "outcome", + "ln.backend", + "attempt", + "url", + "backend.id", + "gauge.swgr_x_open" + ] + ); + assert_eq!(event.get("outcome"), Some(&Val::Str("success".to_owned()))); + assert_eq!(event.get("ln.backend"), Some(&Val::Str("cln".to_owned()))); + assert_eq!(event.get("attempt"), Some(&Val::U64(2))); + assert_eq!(event.get("url"), Some(&Val::Debug(url.to_owned()))); + assert_eq!( + event.get("backend.id"), + Some(&Val::Debug("Some(9)".to_owned())) + ); +} + +#[test] +fn a_label_value_can_be_an_arbitrary_expression() { + let response: Result<(), ()> = Ok(()); + + let (_, events) = recording(|| { + switchgear_metrics::histogram!( + "swgr_x_ms", + 1u64, + "outcome" => if response.is_ok() { "success" } else { "error" }, + ); + }); + + assert_eq!( + only(events).get("outcome"), + Some(&Val::Str("success".to_owned())) + ); +} + +// --------------------------------------------------------------------------- +// coercion +// --------------------------------------------------------------------------- + +#[test] +fn narrow_unsigned_widens_to_u64() { + let (_, events) = recording(|| { + switchgear_metrics::histogram!("swgr_a", 5u32); + switchgear_metrics::monotonic_counter!("swgr_b", 5usize); + switchgear_metrics::gauge!("swgr_c", 5u8); + }); + + assert_eq!(events[0].get("histogram.swgr_a"), Some(&Val::U64(5))); + assert_eq!( + events[1].get("monotonic_counter.swgr_b"), + Some(&Val::U64(5)) + ); + assert_eq!(events[2].get("gauge.swgr_c"), Some(&Val::U64(5))); +} + +#[test] +fn narrow_signed_widens_to_i64_and_keeps_its_sign() { + let (_, events) = recording(|| { + switchgear_metrics::counter!("swgr_a", -3i32); + switchgear_metrics::counter!("swgr_b", 3u32); + switchgear_metrics::gauge!("swgr_c", -3i16); + }); + + assert_eq!(events[0].get("counter.swgr_a"), Some(&Val::I64(-3))); + assert_eq!(events[1].get("counter.swgr_b"), Some(&Val::I64(3))); + assert_eq!(events[2].get("gauge.swgr_c"), Some(&Val::I64(-3))); +} + +#[test] +fn f32_widens_to_f64() { + let (_, events) = recording(|| { + switchgear_metrics::histogram!("swgr_a", 1.5f32); + switchgear_metrics::counter!("swgr_b", -1.5f32); + }); + + assert_eq!(events[0].get("histogram.swgr_a"), Some(&Val::F64(1.5))); + assert_eq!(events[1].get("counter.swgr_b"), Some(&Val::F64(-1.5))); +} + +#[test] +fn duration_records_as_milliseconds_in_f64() { + let elapsed = Duration::from_micros(1_234_567); + let hand_written = elapsed.as_secs_f64() * 1_000.0; + + let (_, events) = recording(|| { + switchgear_metrics::histogram!("swgr_a_ms", elapsed); + switchgear_metrics::gauge!("swgr_b_ms", elapsed); + switchgear_metrics::monotonic_counter!("swgr_c_ms", elapsed); + }); + + assert_eq!( + events[0].get("histogram.swgr_a_ms"), + Some(&Val::F64(hand_written)) + ); + assert_eq!( + events[1].get("gauge.swgr_b_ms"), + Some(&Val::F64(hand_written)) + ); + assert_eq!( + events[2].get("monotonic_counter.swgr_c_ms"), + Some(&Val::F64(hand_written)) + ); +} + +// --------------------------------------------------------------------------- +// levels +// --------------------------------------------------------------------------- + +#[test] +fn level_defaults_to_info() { + let (_, events) = with_metrics("info", "info", || { + switchgear_metrics::histogram!("swgr_default_level_ms", 1u64); + }); + + assert_eq!(only(events).level, Level::INFO); +} + +#[test] +fn an_explicit_level_is_carried_onto_the_event() { + let (_, events) = with_metrics("trace", "info", || { + switchgear_metrics::histogram!(level: Level::WARN, "swgr_warn_ms", 1u64); + }); + + assert_eq!(only(events).level, Level::WARN); +} + +#[test] +fn a_debug_metric_is_dropped_under_an_info_directive() { + let (_, events) = with_metrics("info", "info", || { + switchgear_metrics::histogram!(level: Level::DEBUG, "swgr_detail_a_ms", 1u64); + }); + + assert!(events.is_empty(), "recorded under =info: {events:?}"); +} + +#[test] +fn a_debug_metric_is_recorded_under_a_debug_directive() { + let (_, events) = with_metrics("debug", "info", || { + switchgear_metrics::histogram!(level: Level::DEBUG, "swgr_detail_b_ms", 1u64); + }); + + let event = only(events); + assert_eq!(event.level, Level::DEBUG); + assert_eq!(event.names(), ["histogram.swgr_detail_b_ms"]); +} + +#[test] +fn an_info_metric_survives_a_debug_directive() { + let (_, events) = with_metrics("debug", "info", || { + switchgear_metrics::histogram!("swgr_detail_c_ms", 1u64); + }); + + assert_eq!(only(events).level, Level::INFO); +} + +/// A dropped metric costs the filter check and nothing else: the value and the +/// label expressions sit inside `event!`'s enabled branch. +/// +/// This holds while a dispatcher has been set — which `swgr service` always +/// does. `tracing`'s `log` feature is on in this workspace (sea-orm, sqlx, +/// tower pull it in), and its fallback path *does* reach the value set, but +/// only when `dispatcher::has_been_set()` is false. +#[test] +fn a_dropped_metric_evaluates_neither_its_value_nor_its_labels() { + static VALUES: AtomicUsize = AtomicUsize::new(0); + static LABELS: AtomicUsize = AtomicUsize::new(0); + + fn elapsed_ms() -> u64 { + VALUES.fetch_add(1, Ordering::Relaxed); + 7 + } + + fn backend() -> &'static str { + LABELS.fetch_add(1, Ordering::Relaxed); + "cln" + } + + fn emit() { + switchgear_metrics::histogram!("swgr_lazy_ms", elapsed_ms(), "ln.backend" => backend()); + } + + let (_, events) = with_metrics("off", "info", emit); + assert!(events.is_empty(), "recorded under =off: {events:?}"); + assert_eq!(VALUES.load(Ordering::Relaxed), 0, "value expression ran"); + assert_eq!(LABELS.load(Ordering::Relaxed), 0, "label expression ran"); + + let (_, events) = with_metrics("info", "info", emit); + assert_eq!( + only(events).get("histogram.swgr_lazy_ms"), + Some(&Val::U64(7)) + ); + assert_eq!( + VALUES.load(Ordering::Relaxed), + 1, + "value expression skipped" + ); + assert_eq!( + LABELS.load(Ordering::Relaxed), + 1, + "label expression skipped" + ); +} + +// --------------------------------------------------------------------------- +// the log layer +// --------------------------------------------------------------------------- + +#[test] +fn the_log_layer_writes_no_metric_record() { + for metrics_layer_present in [true, false] { + let capture = Capture::default(); + let logs = LogBuffer::default(); + let target = switchgear_metrics::metrics_target(); + + let subscriber = tracing_subscriber::registry() + .with(metrics_layer( + metrics_layer_present.then(|| capture.clone()), + )) + .with(env_filter( + "info", + if metrics_layer_present { "info" } else { "off" }, + )) + .with( + tracing_subscriber::fmt::layer() + .with_writer(logs.clone()) + .with_ansi(false) + .with_filter(filter_fn(move |m| m.target() != target)), + ); + + tracing::subscriber::with_default(subscriber, || { + tracing::info!(answer = 42, "an ordinary log line"); + switchgear_metrics::histogram!( + "swgr_log_layer_ms", + 1u64, + "ln.backend" => "cln", + ); + }); + + let written = logs.contents(); + let case = format!("metrics={metrics_layer_present}"); + assert!( + written.contains("an ordinary log line"), + "log line missing, {case}: {written}" + ); + assert!( + !written.contains("swgr::metrics"), + "metrics target in log output, {case}: {written}" + ); + assert!( + !written.contains("histogram."), + "metric field in log output, {case}: {written}" + ); + assert!( + !written.contains("swgr_log_layer_ms"), + "metric name in log output, {case}: {written}" + ); + assert!( + !written.contains("off"), + "directive level in log output, {case}: {written}" + ); + } +} + +// --------------------------------------------------------------------------- +// target +// --------------------------------------------------------------------------- + +#[test] +fn metrics_target_is_constant() { + assert_eq!(switchgear_metrics::metrics_target(), "swgr::metrics"); + assert_eq!( + switchgear_metrics::metrics_target(), + switchgear_metrics::metrics_target() + ); +} diff --git a/metrics/tests/ui.rs b/metrics/tests/ui.rs new file mode 100644 index 0000000..476bc39 --- /dev/null +++ b/metrics/tests/ui.rs @@ -0,0 +1,14 @@ +//! Compile-fail coverage for the value traits and the const-key requirement. +//! +//! These assert on rustc's rendered diagnostics, so the `.stderr` files are +//! tied to the toolchain pinned in `rust-toolchain.toml` — and, for the E0435 +//! case, to `tracing`'s own macro expansion. On a toolchain or `tracing` bump, +//! re-record them with `TRYBUILD=overwrite cargo test -p switchgear-metrics +//! --test ui` and read the diff: the point of the fixtures is that a call site +//! passing the wrong numeric type is told *why*, not merely that it failed. + +#[test] +fn compile_fail() { + let cases = trybuild::TestCases::new(); + cases.compile_fail("tests/compile_fail/*.rs"); +} diff --git a/pingora/src/balance.rs b/pingora/src/balance.rs index ed4d294..c164c1c 100644 --- a/pingora/src/balance.rs +++ b/pingora/src/balance.rs @@ -125,7 +125,6 @@ where } } - #[tracing::instrument(skip_all)] fn select_backend( &self, offer: &Offer, @@ -160,7 +159,6 @@ where }) } - #[tracing::instrument(skip_all)] async fn get_invoice_from_backend( &self, offer: &Offer, diff --git a/server/Cargo.toml b/server/Cargo.toml index 4b4e2b0..49f6c3d 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -25,9 +25,8 @@ hyper-rustls = { version = "0.27", default-features = false, features = ["http2" hyper-util = { version = "0.1", features = ["client-legacy", "http2", "tokio"] } jemallocator = "0.5" jsonwebtoken = "10" -log = "0.4" opentelemetry = "0.32" -opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace", "tls-aws-lc", "tls-roots", "internal-logs"] } +opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace", "metrics", "tls-aws-lc", "tls-roots", "internal-logs"] } opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] } p256 = { version = "0.13", features = ["ecdsa"] } pingora-load-balancing = { version = "0.6", default-features = false } @@ -47,6 +46,7 @@ signal-hook = "0.3" signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] } switchgear-components.workspace = true switchgear-error.workspace = true +switchgear-metrics.workspace = true switchgear-pingora.workspace = true switchgear-service-api.workspace = true switchgear-service.workspace = true @@ -65,7 +65,7 @@ chrono = { version = "0.4", features = ["serde"] } insta = { version = "1", features = ["json", "redactions"] } jemallocator = "0.5" lightning-invoice = { version = "0.34", features = ["serde", "std"] } -opentelemetry-proto = { version = "0.32", features = ["gen-tonic", "trace"] } +opentelemetry-proto = { version = "0.32", features = ["gen-tonic", "trace", "metrics"] } rand = "0.8" rcgen = { version = "0.14", default-features = false, features = ["crypto", "aws_lc_rs", "pem"] } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots-no-provider"] } diff --git a/server/config/discovery-standalone.yaml b/server/config/discovery-standalone.yaml index 84b4672..5042d3b 100644 --- a/server/config/discovery-standalone.yaml +++ b/server/config/discovery-standalone.yaml @@ -8,11 +8,12 @@ discovery-service: cert-path: "${DISCOVERY_SERVICE_TLS_CERT_PATH:-/etc/ssl/certs/discovery-cert.pem}" key-path: "${DISCOVERY_SERVICE_TLS_KEY_PATH:-/etc/ssl/certs/discovery-key.pem}" otlp: - tracing: - endpoint: "${DISCOVERY_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + export: + endpoint: "${DISCOVERY_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${DISCOVERY_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${DISCOVERY_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${DISCOVERY_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${DISCOVERY_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -20,11 +21,17 @@ discovery-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${DISCOVERY_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${DISCOVERY_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${DISCOVERY_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative # Storage configuration store: discover: diff --git a/server/config/lnurl-standalone.yaml b/server/config/lnurl-standalone.yaml index de908a2..f8d1185 100644 --- a/server/config/lnurl-standalone.yaml +++ b/server/config/lnurl-standalone.yaml @@ -25,11 +25,12 @@ lnurl-service: bech32-qr-light: 255 bech32-qr-dark: 0 otlp: - tracing: - endpoint: "${LNURL_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + export: + endpoint: "${LNURL_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${LNURL_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${LNURL_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${LNURL_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${LNURL_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -37,11 +38,17 @@ lnurl-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${LNURL_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${LNURL_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${LNURL_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${LNURL_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${LNURL_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${LNURL_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative # Remote storage configuration # Connect to external discovery and offers services store: diff --git a/server/config/memory-basic-no-tls.yaml b/server/config/memory-basic-no-tls.yaml index 3de9b5a..d9c24be 100644 --- a/server/config/memory-basic-no-tls.yaml +++ b/server/config/memory-basic-no-tls.yaml @@ -23,11 +23,12 @@ lnurl-service: bech32-qr-light: 255 bech32-qr-dark: 0 otlp: - tracing: - endpoint: "${LNURL_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + export: + endpoint: "${LNURL_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${LNURL_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${LNURL_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${LNURL_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${LNURL_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -35,20 +36,29 @@ lnurl-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${LNURL_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${LNURL_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${LNURL_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${LNURL_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${LNURL_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${LNURL_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative discovery-service: address: "${DISCOVERY_SERVICE_ADDRESS:-127.0.0.1:8081}" auth-authority: "${DISCOVERY_SERVICE_AUTH_AUTHORITY_PATH:-/etc/ssl/certs/discovery-auth-authority.pem}" otlp: - tracing: - endpoint: "${DISCOVERY_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + # Required. Both signals below export over this one connection unless they + # carry an `export` block of their own, which replaces it in full. + export: + endpoint: "${DISCOVERY_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${DISCOVERY_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${DISCOVERY_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${DISCOVERY_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${DISCOVERY_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -56,21 +66,31 @@ discovery-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${DISCOVERY_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${DISCOVERY_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${DISCOVERY_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + # `ratio: 1.0` under parent-based-trace-id-ratio reproduces the SDK default. + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative offer-service: auth-authority: "${OFFER_SERVICE_AUTH_AUTHORITY_PATH:-/etc/ssl/certs/offer-auth-authority.pem}" address: "${OFFER_SERVICE_ADDRESS:-127.0.0.1:8082}" max-page-size: 100 otlp: - tracing: - endpoint: "${OFFER_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + # Required. Both signals below export over this one connection unless they + # carry an `export` block of their own, which replaces it in full. + export: + endpoint: "${OFFER_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${OFFER_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${OFFER_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${OFFER_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${OFFER_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -78,11 +98,18 @@ offer-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${OFFER_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${OFFER_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${OFFER_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${OFFER_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${OFFER_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${OFFER_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + # `ratio: 1.0` under parent-based-trace-id-ratio reproduces the SDK default. + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative store: offer: type: "memory" diff --git a/server/config/memory-basic.yaml b/server/config/memory-basic.yaml index b85fc81..2bfe7b4 100644 --- a/server/config/memory-basic.yaml +++ b/server/config/memory-basic.yaml @@ -25,11 +25,12 @@ lnurl-service: bech32-qr-light: 255 bech32-qr-dark: 0 otlp: - tracing: - endpoint: "${LNURL_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + export: + endpoint: "${LNURL_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${LNURL_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${LNURL_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${LNURL_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${LNURL_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -37,11 +38,17 @@ lnurl-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${LNURL_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${LNURL_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${LNURL_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${LNURL_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${LNURL_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${LNURL_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative discovery-service: address: "${DISCOVERY_SERVICE_ADDRESS:-127.0.0.1:8081}" auth-authority: "${DISCOVERY_SERVICE_AUTH_AUTHORITY_PATH:-/etc/ssl/certs/discovery-auth-authority.pem}" @@ -49,11 +56,14 @@ discovery-service: cert-path: "${DISCOVERY_SERVICE_TLS_CERT_PATH:-/etc/ssl/certs/discovery-cert.pem}" key-path: "${DISCOVERY_SERVICE_TLS_KEY_PATH:-/etc/ssl/certs/discovery-key.pem}" otlp: - tracing: - endpoint: "${DISCOVERY_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + # Required. Both signals below export over this one connection unless they + # carry an `export` block of their own, which replaces it in full. + export: + endpoint: "${DISCOVERY_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${DISCOVERY_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${DISCOVERY_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${DISCOVERY_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${DISCOVERY_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -61,11 +71,18 @@ discovery-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${DISCOVERY_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${DISCOVERY_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${DISCOVERY_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + # `ratio: 1.0` under parent-based-trace-id-ratio reproduces the SDK default. + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative offer-service: address: "${OFFER_SERVICE_ADDRESS:-127.0.0.1:8082}" auth-authority: "${OFFER_SERVICE_AUTH_AUTHORITY_PATH:-/etc/ssl/certs/offer-auth-authority.pem}" @@ -74,11 +91,14 @@ offer-service: key-path: "${OFFER_SERVICE_TLS_KEY_PATH:-/etc/ssl/certs/offer-key.pem}" max-page-size: 100 otlp: - tracing: - endpoint: "${OFFER_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + # Required. Both signals below export over this one connection unless they + # carry an `export` block of their own, which replaces it in full. + export: + endpoint: "${OFFER_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${OFFER_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${OFFER_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${OFFER_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${OFFER_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -86,11 +106,18 @@ offer-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${OFFER_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${OFFER_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${OFFER_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${OFFER_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${OFFER_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${OFFER_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + # `ratio: 1.0` under parent-based-trace-id-ratio reproduces the SDK default. + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative store: offer: type: "memory" diff --git a/server/config/offers-standalone.yaml b/server/config/offers-standalone.yaml index 26263b1..00922ab 100644 --- a/server/config/offers-standalone.yaml +++ b/server/config/offers-standalone.yaml @@ -9,11 +9,12 @@ offer-service: key-path: "${OFFER_SERVICE_TLS_KEY_PATH:-/etc/ssl/certs/offer-key.pem}" max-page-size: 100 otlp: - tracing: - endpoint: "${OFFER_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + export: + endpoint: "${OFFER_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${OFFER_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${OFFER_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${OFFER_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${OFFER_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -21,11 +22,17 @@ offer-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${OFFER_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${OFFER_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${OFFER_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${OFFER_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${OFFER_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${OFFER_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative # Storage configuration store: offer: diff --git a/server/config/persistence-with-secrets.yaml b/server/config/persistence-with-secrets.yaml index 846ef9c..0480e86 100644 --- a/server/config/persistence-with-secrets.yaml +++ b/server/config/persistence-with-secrets.yaml @@ -25,11 +25,12 @@ lnurl-service: bech32-qr-light: 255 bech32-qr-dark: 0 otlp: - tracing: - endpoint: "${LNURL_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + export: + endpoint: "${LNURL_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${LNURL_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${LNURL_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${LNURL_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${LNURL_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -37,11 +38,17 @@ lnurl-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${LNURL_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${LNURL_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${LNURL_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${LNURL_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${LNURL_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${LNURL_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative discovery-service: address: "${DISCOVERY_SERVICE_ADDRESS:-127.0.0.1:8081}" auth-authority: "${DISCOVERY_SERVICE_AUTH_AUTHORITY_PATH:-/etc/ssl/certs/discovery-auth-authority.pem}" @@ -49,11 +56,12 @@ discovery-service: cert-path: "${DISCOVERY_SERVICE_TLS_CERT_PATH:-/etc/ssl/certs/discovery-cert.pem}" key-path: "${DISCOVERY_SERVICE_TLS_KEY_PATH:-/etc/ssl/certs/discovery-key.pem}" otlp: - tracing: - endpoint: "${DISCOVERY_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + export: + endpoint: "${DISCOVERY_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${DISCOVERY_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${DISCOVERY_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${DISCOVERY_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${DISCOVERY_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -61,11 +69,17 @@ discovery-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${DISCOVERY_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${DISCOVERY_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${DISCOVERY_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative offer-service: address: "${OFFER_SERVICE_ADDRESS:-127.0.0.1:8082}" auth-authority: "${OFFER_SERVICE_AUTH_AUTHORITY_PATH:-/etc/ssl/certs/offer-auth-authority.pem}" @@ -74,11 +88,12 @@ offer-service: key-path: "${OFFER_SERVICE_TLS_KEY_PATH:-/etc/ssl/certs/offer-key.pem}" max-page-size: 100 otlp: - tracing: - endpoint: "${OFFER_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + export: + endpoint: "${OFFER_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${OFFER_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${OFFER_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${OFFER_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${OFFER_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -86,11 +101,17 @@ offer-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${OFFER_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${OFFER_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${OFFER_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${OFFER_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${OFFER_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${OFFER_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative store: offer: type: "database" diff --git a/server/config/persistence.yaml b/server/config/persistence.yaml index fdc67cd..a079920 100644 --- a/server/config/persistence.yaml +++ b/server/config/persistence.yaml @@ -25,11 +25,14 @@ lnurl-service: bech32-qr-light: 255 bech32-qr-dark: 0 otlp: - tracing: - endpoint: "${LNURL_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + # Required. Both signals below export over this one connection unless they + # carry an `export` block of their own, which replaces it in full. + export: + endpoint: "${LNURL_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${LNURL_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${LNURL_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${LNURL_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${LNURL_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -37,11 +40,18 @@ lnurl-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${LNURL_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${LNURL_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${LNURL_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${LNURL_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${LNURL_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${LNURL_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + # `ratio: 1.0` under parent-based-trace-id-ratio reproduces the SDK default. + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative discovery-service: address: "${DISCOVERY_SERVICE_ADDRESS:-127.0.0.1:8081}" auth-authority: "${DISCOVERY_SERVICE_AUTH_AUTHORITY_PATH:-/etc/ssl/certs/discovery-auth-authority.pem}" @@ -49,11 +59,12 @@ discovery-service: cert-path: "${DISCOVERY_SERVICE_TLS_CERT_PATH:-/etc/ssl/certs/discovery-cert.pem}" key-path: "${DISCOVERY_SERVICE_TLS_KEY_PATH:-/etc/ssl/certs/discovery-key.pem}" otlp: - tracing: - endpoint: "${DISCOVERY_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + export: + endpoint: "${DISCOVERY_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${DISCOVERY_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${DISCOVERY_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${DISCOVERY_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${DISCOVERY_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -61,11 +72,17 @@ discovery-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${DISCOVERY_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${DISCOVERY_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${DISCOVERY_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative offer-service: address: "${OFFER_SERVICE_ADDRESS:-127.0.0.1:8082}" auth-authority: "${OFFER_SERVICE_AUTH_AUTHORITY_PATH:-/etc/ssl/certs/offer-auth-authority.pem}" @@ -74,11 +91,14 @@ offer-service: key-path: "${OFFER_SERVICE_TLS_KEY_PATH:-/etc/ssl/certs/offer-key.pem}" max-page-size: 100 otlp: - tracing: - endpoint: "${OFFER_SERVICE_OTLP_TRACING_ENDPOINT:-http://127.0.0.1:4317}" + # Required. Both signals below export over this one connection unless they + # carry an `export` block of their own, which replaces it in full. + export: + endpoint: "${OFFER_SERVICE_OTLP_ENDPOINT:-http://127.0.0.1:4317}" auth-token: "OTEL_AUTH_TOKEN" - trusted-roots: ${OFFER_SERVICE_OTLP_TRACING_TRUSTED_ROOTS:-} - trusted-root-address: ${OFFER_SERVICE_OTLP_TRACING_TRUSTED_ROOT_ADDRESS:-} + trusted-roots: ${OFFER_SERVICE_OTLP_TRUSTED_ROOTS:-} + trusted-root-address: ${OFFER_SERVICE_OTLP_TRUSTED_ROOT_ADDRESS:-} + export-timeout-secs: 5.0 client-identity: cert-secret: "OTEL_CLIENT_CERT" key-secret: "OTEL_CLIENT_KEY" @@ -86,11 +106,18 @@ offer-service: ttl: 300.0 secrets: OTEL_AUTH_TOKEN: - path: "${OFFER_SERVICE_OTLP_TRACING_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" + path: "${OFFER_SERVICE_OTLP_AUTH_TOKEN:-/etc/ssl/certs/otlp-auth.token}" OTEL_CLIENT_CERT: - path: "${OFFER_SERVICE_OTLP_TRACING_CLIENT_CERT:-}" + path: "${OFFER_SERVICE_OTLP_CLIENT_CERT:-}" OTEL_CLIENT_KEY: - path: "${OFFER_SERVICE_OTLP_TRACING_CLIENT_KEY:-}" + path: "${OFFER_SERVICE_OTLP_CLIENT_KEY:-}" + tracing: + # `ratio: 1.0` under parent-based-trace-id-ratio reproduces the SDK default. + sampler: + type: parent-based-trace-id-ratio + ratio: 1.0 + metrics: + temporality: cumulative store: offer: type: "database" diff --git a/server/src/commands/discovery/backend.rs b/server/src/commands/discovery/backend.rs index 908f54d..7424bcc 100644 --- a/server/src/commands/discovery/backend.rs +++ b/server/src/commands/discovery/backend.rs @@ -1,7 +1,6 @@ use crate::commands::error::CliError; use crate::commands::{cli_read_to_string, cli_write_all}; use clap::{Parser, ValueEnum}; -use log::info; use rustls::pki_types::CertificateDer; use rustls::pki_types::pem::PemObject; use std::collections::BTreeMap; @@ -23,6 +22,7 @@ use switchgear_service_api::discovery::{ DiscoveryBackend, DiscoveryBackendPatch, DiscoveryBackendPatchSparse, DiscoveryBackendSparse, DiscoveryBackendStore, }; +use tracing::info; use url::Url; #[derive(Parser, Debug)] diff --git a/server/src/commands/error.rs b/server/src/commands/error.rs index fad5cf1..2914b8f 100644 --- a/server/src/commands/error.rs +++ b/server/src/commands/error.rs @@ -430,11 +430,13 @@ impl CliErrorAccumulator { } } - pub fn finish(self) -> Result<(), CliError> { - match self.errors.len() { - 0 => Ok(()), - 1 => Err(self.errors.into_iter().next().expect("len == 1")), - _ => Err(CliError::multi(self.errors)), + pub fn finish(mut self) -> Result<(), CliError> { + if self.errors.len() > 1 { + return Err(CliError::multi(self.errors)); + } + match self.errors.pop() { + Some(err) => Err(err), + None => Ok(()), } } } diff --git a/server/src/commands/offer/metadata.rs b/server/src/commands/offer/metadata.rs index c05869c..9347c7d 100644 --- a/server/src/commands/offer/metadata.rs +++ b/server/src/commands/offer/metadata.rs @@ -2,10 +2,10 @@ use crate::commands::error::CliError; use crate::commands::offer::{OfferManagementClientConfig, create_offer_client}; use crate::commands::{cli_read_to_string, cli_write_all}; use clap::Parser; -use log::info; use std::path::{Path, PathBuf}; use switchgear_error::{ChainedContext, ErrorOrigin, ForeignContext}; use switchgear_service_api::offer::{OfferMetadata, OfferMetadataSparse, OfferMetadataStore}; +use tracing::info; use uuid::Uuid; #[derive(Parser, Debug)] diff --git a/server/src/commands/offer/record.rs b/server/src/commands/offer/record.rs index 1719d02..2c9cf27 100644 --- a/server/src/commands/offer/record.rs +++ b/server/src/commands/offer/record.rs @@ -3,10 +3,10 @@ use crate::commands::offer::{OfferManagementClientConfig, create_offer_client}; use crate::commands::{cli_read_to_string, cli_write_all}; use chrono::{DateTime, Utc}; use clap::Parser; -use log::info; use std::path::{Path, PathBuf}; use switchgear_error::{ChainedContext, ErrorOrigin, ForeignContext}; use switchgear_service_api::offer::{OfferRecord, OfferRecordSparse, OfferStore}; +use tracing::info; use uuid::Uuid; #[derive(Parser, Debug)] diff --git a/server/src/commands/services.rs b/server/src/commands/services.rs index 1d1dc1b..5782b1d 100644 --- a/server/src/commands/services.rs +++ b/server/src/commands/services.rs @@ -9,7 +9,6 @@ use crate::di::inject::injectors::store::discovery::DiscoveryStoreInjector; use crate::di::inject::injectors::store::offer::OfferStoreInjector; use crate::signals::get_signals_fut; use clap::ValueEnum; -use opentelemetry_sdk::trace::SdkTracerProvider; use signal_hook::low_level::signal_name; use switchgear_error::{ChainedContext, ErrorOrigin, ForeignContext}; use switchgear_service_api::discovery::DiscoveryBackendStore; @@ -30,7 +29,6 @@ pub enum ServiceEnablement { pub async fn execute( config_injector: ServerConfigInjector, enablement: Vec, - otel_providers: &mut Vec<(&'static str, SdkTracerProvider)>, ) -> Result<(), CliError> { info!("starting services"); @@ -74,7 +72,7 @@ pub async fn execute( ); let discovery_service_fut = discovery_service_injector - .connect(otel_providers) + .connect() .await .chained_context("connecting discovery service", None)?; let discovery_service_fut = async move { @@ -85,7 +83,7 @@ pub async fn execute( }; let offer_service_fut = offer_service_injector - .connect(otel_providers) + .connect() .await .chained_context("connecting offer service", None)?; let offer_service_fut = async move { @@ -96,7 +94,7 @@ pub async fn execute( }; let balancer_service_fut = balancer_service_injector - .connect(otel_providers) + .connect() .await .chained_context("connecting lnurl service", None)?; let balancer_service_fut = async move { @@ -191,6 +189,21 @@ pub async fn execute( info!("stores disconnected"); + info!("shutting down service tracing"); + + let (lnurl_res, discovery_res, offer_res) = tokio::join!( + balancer_service_injector.shutdown_tracing(), + discovery_service_injector.shutdown_tracing(), + offer_service_injector.shutdown_tracing(), + ); + errors.push_result(lnurl_res.chained_context("shutting down lnurl service tracing", None)); + errors.push_result( + discovery_res.chained_context("shutting down discovery service tracing", None), + ); + errors.push_result(offer_res.chained_context("shutting down offer service tracing", None)); + + info!("service tracing shut down"); + signals_handle.close(); info!("signal stream closed"); diff --git a/server/src/config.rs b/server/src/config.rs index 4129be8..d57aa0f 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -1,3 +1,4 @@ +use crate::level::Level; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use std::net::SocketAddr; @@ -143,18 +144,60 @@ pub struct TlsConfig { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct OtlpConfig { + pub export: OtlpExportConfig, pub tracing: Option, + pub metrics: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] -pub struct OtlpTracingConfig { +pub struct OtlpExportConfig { pub endpoint: Url, pub auth_token: String, pub trusted_roots: Option, pub trusted_root_address: Option, pub client_identity: Option, pub secrets: SecretStoreConfig, + pub export_timeout_secs: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct OtlpTracingConfig { + pub sampler: SamplerConfig, + pub export: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct OtlpMetricsConfig { + pub temporality: TemporalityConfig, + pub export: Option, + #[serde(default)] + pub level: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "kebab-case")] +pub enum SamplerConfig { + AlwaysOn, + AlwaysOff, + #[serde(rename_all = "kebab-case")] + TraceIdRatio { + ratio: f64, + }, + #[serde(rename_all = "kebab-case")] + ParentBasedTraceIdRatio { + ratio: f64, + }, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum TemporalityConfig { + Cumulative, + Delta, + LowMemory, } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/server/src/di/error.rs b/server/src/di/error.rs index 1685dab..38d15c2 100644 --- a/server/src/di/error.rs +++ b/server/src/di/error.rs @@ -7,10 +7,13 @@ use std::panic::Location; use std::str::Utf8Error; use opentelemetry_otlp::ExporterBuildError; +use opentelemetry_sdk::error::OTelSdkError; use switchgear_components::secrets::SecretContextError; use switchgear_error::{ContextError, ErrorOrigin, IntoBoxedTrait, IntoContextError}; use switchgear_service_api::discovery::DiscoveryBackendStoreError; use switchgear_service_api::offer::OfferStoreError; +use tokio::task::JoinError; +use tracing_subscriber::filter::ParseError; use tracing_subscriber::util::TryInitError; pub trait DiContextError: ContextError {} @@ -26,7 +29,10 @@ pub enum DiErrorSourceKind { Rustls(Box), Jwt(Box), TracingInit(Box), + FilterDirective(Box), OtlpExporterBuild(Box), + OtelSdk(Box), + Join(Box), OfferStore(Box), DiscoveryBackendStore(Box), Di(Box), @@ -45,7 +51,10 @@ impl Display for DiErrorSourceKind { Self::Rustls(e) => write!(f, "rustls error: {e}"), Self::Jwt(e) => write!(f, "jwt error: {e}"), Self::TracingInit(e) => write!(f, "tracing init error: {e}"), + Self::FilterDirective(e) => write!(f, "filter directive error: {e}"), Self::OtlpExporterBuild(e) => write!(f, "otlp exporter build error: {e}"), + Self::OtelSdk(e) => write!(f, "otel sdk error: {e}"), + Self::Join(e) => write!(f, "task join error: {e}"), Self::OfferStore(e) => Display::fmt(e, f), Self::DiscoveryBackendStore(e) => Display::fmt(e, f), Self::Di(e) => Display::fmt(e, f), @@ -66,7 +75,10 @@ impl Error for DiErrorSourceKind { Self::Rustls(e) => Some(&**e), Self::Jwt(e) => Some(&**e), Self::TracingInit(e) => Some(&**e), + Self::FilterDirective(e) => Some(&**e), Self::OtlpExporterBuild(e) => Some(&**e), + Self::OtelSdk(e) => Some(&**e), + Self::Join(e) => Some(&**e), Self::OfferStore(e) => Some(&**e), Self::DiscoveryBackendStore(e) => Some(&**e), Self::Di(e) => Some(&**e), @@ -298,6 +310,21 @@ impl IntoContextError for DiError { } } +impl IntoContextError for DiError { + #[track_caller] + fn error>>( + source: Box, + message: M, + origin: Option, + ) -> Self { + Self::new( + DiErrorSourceKind::FilterDirective(source), + origin.unwrap_or(ErrorOrigin::Internal), + message, + ) + } +} + impl IntoContextError for DiError { #[track_caller] fn error>>( @@ -313,6 +340,36 @@ impl IntoContextError for DiError { } } +impl IntoContextError for DiError { + #[track_caller] + fn error>>( + source: Box, + message: M, + origin: Option, + ) -> Self { + Self::new( + DiErrorSourceKind::OtelSdk(source), + origin.unwrap_or(ErrorOrigin::Internal), + message, + ) + } +} + +impl IntoContextError for DiError { + #[track_caller] + fn error>>( + source: Box, + message: M, + origin: Option, + ) -> Self { + Self::new( + DiErrorSourceKind::Join(source), + origin.unwrap_or(ErrorOrigin::Internal), + message, + ) + } +} + impl IntoContextError for DiError { #[track_caller] fn error>>( diff --git a/server/src/di/inject/injectors/service/balance.rs b/server/src/di/inject/injectors/service/balance.rs index ddfb731..a83fe69 100644 --- a/server/src/di/inject/injectors/service/balance.rs +++ b/server/src/di/inject/injectors/service/balance.rs @@ -1,10 +1,10 @@ use crate::di::error::DiError; use crate::di::inject::injectors::balance::BalancerInjector; use crate::di::inject::injectors::config::{ServerConfigInjector, ServiceEnablementInjector}; +use crate::di::inject::injectors::service::subscriber::ServiceTracing; use crate::di::inject::injectors::service::tls::load_server_x509_credentials; -use crate::di::inject::injectors::service::tracing::{ServiceSubscriber, build_service_subscriber}; use crate::di::inject::injectors::store::offer::OfferStoreInjector; -use opentelemetry_sdk::trace::SdkTracerProvider; +use std::cell::RefCell; use std::future::Future; use std::net::{SocketAddr, TcpListener}; use std::pin::Pin; @@ -26,6 +26,7 @@ pub struct BalancerServiceInjector { enablement: ServiceEnablementInjector, balancer_injector: BalancerInjector, offer_store: OfferStoreInjector, + tracing: RefCell>, } impl BalancerServiceInjector { @@ -40,12 +41,12 @@ impl BalancerServiceInjector { enablement, balancer_injector, offer_store, + tracing: RefCell::default(), } } pub async fn connect( &self, - otel_providers: &mut Vec<(&'static str, SdkTracerProvider)>, ) -> Result>>>>, DiError> { if !self.enablement.lnurl_enabled() { return Ok(None); @@ -58,12 +59,10 @@ impl BalancerServiceInjector { .as_ref() .ok_or_else(|| DiError::internal("lnurl service enabled but has no config"))?; - let ServiceSubscriber { dispatch, provider } = - build_service_subscriber(SERVICE_NAME, service_config.otlp.as_ref()) - .chained_context("building lnurl service tracing subscriber", None)?; - if let Some(p) = provider { - otel_providers.push((SERVICE_NAME, p)); - } + let tracing = ServiceTracing::build(SERVICE_NAME, service_config.otlp.as_ref()) + .chained_context("building lnurl service tracing subscriber", None)?; + let dispatch = tracing.dispatch().clone(); + *self.tracing.borrow_mut() = Some(tracing); let balancer = self .balancer_injector @@ -145,4 +144,15 @@ impl BalancerServiceInjector { Ok(Some(Box::pin(f))) } + + pub async fn shutdown_tracing(&self) -> Result<(), DiError> { + let tracing = self.tracing.borrow_mut().take(); + match tracing { + Some(t) => t + .shutdown() + .await + .chained_context("shutting down lnurl service tracing", None), + None => Ok(()), + } + } } diff --git a/server/src/di/inject/injectors/service/discovery.rs b/server/src/di/inject/injectors/service/discovery.rs index 0b04ccd..bf05145 100644 --- a/server/src/di/inject/injectors/service/discovery.rs +++ b/server/src/di/inject/injectors/service/discovery.rs @@ -1,10 +1,10 @@ use crate::di::error::DiError; use crate::di::inject::injectors::config::{ServerConfigInjector, ServiceEnablementInjector}; +use crate::di::inject::injectors::service::subscriber::ServiceTracing; use crate::di::inject::injectors::service::tls::load_server_x509_credentials; -use crate::di::inject::injectors::service::tracing::{ServiceSubscriber, build_service_subscriber}; use crate::di::inject::injectors::store::discovery::DiscoveryStoreInjector; use jsonwebtoken::DecodingKey; -use opentelemetry_sdk::trace::SdkTracerProvider; +use std::cell::RefCell; use std::future::Future; use std::net::{SocketAddr, TcpListener}; use std::pin::Pin; @@ -22,6 +22,7 @@ pub struct DiscoveryServiceInjector { config: ServerConfigInjector, enablement: ServiceEnablementInjector, store_injector: DiscoveryStoreInjector, + tracing: RefCell>, } impl DiscoveryServiceInjector { @@ -34,12 +35,12 @@ impl DiscoveryServiceInjector { config, enablement, store_injector, + tracing: RefCell::default(), } } pub async fn connect( &self, - otel_providers: &mut Vec<(&'static str, SdkTracerProvider)>, ) -> Result>>>>, DiError> { if !self.enablement.discovery_enabled() { return Ok(None); @@ -52,12 +53,10 @@ impl DiscoveryServiceInjector { .as_ref() .ok_or_else(|| DiError::internal("discover service enabled but has no config"))?; - let ServiceSubscriber { dispatch, provider } = - build_service_subscriber(SERVICE_NAME, service_config.otlp.as_ref()) - .chained_context("building discovery service tracing subscriber", None)?; - if let Some(p) = provider { - otel_providers.push((SERVICE_NAME, p)); - } + let tracing = ServiceTracing::build(SERVICE_NAME, service_config.otlp.as_ref()) + .chained_context("building discovery service tracing subscriber", None)?; + let dispatch = tracing.dispatch().clone(); + *self.tracing.borrow_mut() = Some(tracing); let store = self .store_injector @@ -135,4 +134,15 @@ impl DiscoveryServiceInjector { Ok(Some(Box::pin(f))) } + + pub async fn shutdown_tracing(&self) -> Result<(), DiError> { + let tracing = self.tracing.borrow_mut().take(); + match tracing { + Some(t) => t + .shutdown() + .await + .chained_context("shutting down discovery service tracing", None), + None => Ok(()), + } + } } diff --git a/server/src/di/inject/injectors/service/mod.rs b/server/src/di/inject/injectors/service/mod.rs index bee3072..4abc16a 100644 --- a/server/src/di/inject/injectors/service/mod.rs +++ b/server/src/di/inject/injectors/service/mod.rs @@ -2,5 +2,7 @@ pub mod balance; pub mod balance_background; pub mod discovery; pub mod offer; +mod otlp; +mod subscriber; mod tls; pub mod tracing; diff --git a/server/src/di/inject/injectors/service/offer.rs b/server/src/di/inject/injectors/service/offer.rs index bf09512..938c0fd 100644 --- a/server/src/di/inject/injectors/service/offer.rs +++ b/server/src/di/inject/injectors/service/offer.rs @@ -1,11 +1,11 @@ use crate::di::inject::injectors::config::{ServerConfigInjector, ServiceEnablementInjector}; use crate::di::error::DiError; +use crate::di::inject::injectors::service::subscriber::ServiceTracing; use crate::di::inject::injectors::service::tls::load_server_x509_credentials; -use crate::di::inject::injectors::service::tracing::{ServiceSubscriber, build_service_subscriber}; use crate::di::inject::injectors::store::offer::OfferStoreInjector; use jsonwebtoken::DecodingKey; -use opentelemetry_sdk::trace::SdkTracerProvider; +use std::cell::RefCell; use std::future::Future; use std::net::{SocketAddr, TcpListener}; use std::pin::Pin; @@ -23,6 +23,7 @@ pub struct OfferServiceInjector { config: ServerConfigInjector, enablement: ServiceEnablementInjector, store_injector: OfferStoreInjector, + tracing: RefCell>, } impl OfferServiceInjector { @@ -35,12 +36,12 @@ impl OfferServiceInjector { config, enablement, store_injector, + tracing: RefCell::default(), } } pub async fn connect( &self, - otel_providers: &mut Vec<(&'static str, SdkTracerProvider)>, ) -> Result>>>>, DiError> { if !self.enablement.offer_enabled() { return Ok(None); @@ -53,12 +54,10 @@ impl OfferServiceInjector { .as_ref() .ok_or_else(|| DiError::internal("offer service enabled but has no config"))?; - let ServiceSubscriber { dispatch, provider } = - build_service_subscriber(SERVICE_NAME, service_config.otlp.as_ref()) - .chained_context("building offer service tracing subscriber", None)?; - if let Some(p) = provider { - otel_providers.push((SERVICE_NAME, p)); - } + let tracing = ServiceTracing::build(SERVICE_NAME, service_config.otlp.as_ref()) + .chained_context("building offer service tracing subscriber", None)?; + let dispatch = tracing.dispatch().clone(); + *self.tracing.borrow_mut() = Some(tracing); let store = self .store_injector @@ -141,4 +140,15 @@ impl OfferServiceInjector { Ok(Some(Box::pin(f))) } + + pub async fn shutdown_tracing(&self) -> Result<(), DiError> { + let tracing = self.tracing.borrow_mut().take(); + match tracing { + Some(t) => t + .shutdown() + .await + .chained_context("shutting down offer service tracing", None), + None => Ok(()), + } + } } diff --git a/server/src/di/inject/injectors/service/otlp.rs b/server/src/di/inject/injectors/service/otlp.rs new file mode 100644 index 0000000..bbd8bc8 --- /dev/null +++ b/server/src/di/inject/injectors/service/otlp.rs @@ -0,0 +1,208 @@ +use crate::config::{OtlpExportConfig, SamplerConfig, TemporalityConfig}; +use crate::di::error::DiError; +use hyper_rustls::{FixedServerNameResolver, HttpsConnectorBuilder}; +use opentelemetry::KeyValue; +use opentelemetry_otlp::{MetricExporter, SpanExporter, WithTonicConfig}; +use opentelemetry_sdk::Resource; +use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider, Temporality}; +use opentelemetry_sdk::trace::{Sampler, SdkTracerProvider}; +use rustls::crypto::CryptoProvider; +use rustls::pki_types::pem::PemObject; +use rustls::pki_types::{CertificateDer, ServerName}; +use rustls::{ClientConfig, RootCertStore}; +use std::sync::Arc; +use std::time::Duration; +use switchgear_components::secrets::{ClientCertResolver, SecretHeaderInterceptor, SecretStore}; +use switchgear_error::{ErrorOrigin, ForeignContext}; +use tonic::transport::{Channel, Endpoint}; + +impl From<&SamplerConfig> for Sampler { + fn from(config: &SamplerConfig) -> Self { + match config { + SamplerConfig::AlwaysOn => Sampler::AlwaysOn, + SamplerConfig::AlwaysOff => Sampler::AlwaysOff, + SamplerConfig::TraceIdRatio { ratio } => Sampler::TraceIdRatioBased(*ratio), + SamplerConfig::ParentBasedTraceIdRatio { ratio } => { + Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(*ratio))) + } + } + } +} + +impl From<&TemporalityConfig> for Temporality { + fn from(config: &TemporalityConfig) -> Self { + match config { + TemporalityConfig::Cumulative => Temporality::Cumulative, + TemporalityConfig::Delta => Temporality::Delta, + TemporalityConfig::LowMemory => Temporality::LowMemory, + } + } +} + +pub fn resource(service_name: &'static str) -> Resource { + Resource::builder() + .with_service_name(service_name) + .with_attribute(KeyValue::new("service.version", env!("CARGO_PKG_VERSION"))) + .build() +} + +pub fn client_tls_config( + export: &OtlpExportConfig, + secrets: &SecretStore, +) -> Result { + let crypto = CryptoProvider::get_default() + .ok_or_else(|| DiError::internal("no rustls crypto provider installed for otel exporter"))? + .clone(); + + let mut roots = RootCertStore::empty(); + roots.add_parsable_certificates(rustls_native_certs::load_native_certs().certs); + if let Some(path) = export.trusted_roots.as_ref() { + let pem = std::fs::read(path).with_foreign_context( + || format!("reading otel trusted roots from {}", path.display()), + None, + )?; + let extra: Vec> = CertificateDer::pem_slice_iter(&pem) + .collect::>() + .with_foreign_context( + || format!("parsing otel trusted roots from {}", path.display()), + None, + )?; + roots.add_parsable_certificates(extra); + } + + let builder = ClientConfig::builder_with_provider(crypto.clone()) + .with_safe_default_protocol_versions() + .foreign_context( + "configuring otel exporter TLS protocol versions", + ErrorOrigin::Internal, + )? + .with_root_certificates(roots); + + Ok(match export.client_identity.as_ref() { + Some(id) => builder.with_client_cert_resolver(Arc::new(ClientCertResolver::new( + secrets.clone(), + id.cert_secret.clone(), + id.key_secret.clone(), + crypto, + ))), + None => builder.with_no_client_auth(), + }) +} + +pub fn channel(export: &OtlpExportConfig, secrets: &SecretStore) -> Result { + let tls_config = client_tls_config(export, secrets)?; + + let mut connector_builder = HttpsConnectorBuilder::new() + .with_tls_config(tls_config) + .https_or_http(); + if let Some(addr) = export.trusted_root_address.as_ref() { + let name: ServerName<'static> = + ServerName::try_from(addr.ip().to_string()).map_err(|e| { + DiError::internal(format!( + "parsing otel trusted-root-address {} as TLS server name: {e}", + addr.ip() + )) + })?; + connector_builder = + connector_builder.with_server_name_resolver(FixedServerNameResolver::new(name)); + } + let https = connector_builder.enable_http2().build(); + + let mut endpoint = Endpoint::from_shared(export.endpoint.to_string()).map_err(|e| { + DiError::internal(format!( + "parsing otel endpoint {} for tonic channel: {e}", + export.endpoint + )) + })?; + if let Some(secs) = export.export_timeout_secs { + endpoint = endpoint.timeout(Duration::from_secs_f64(secs)); + } + + Ok(Channel::new(https, endpoint)) +} + +pub fn transport(export: &OtlpExportConfig) -> Result<(Channel, SecretStore), DiError> { + let secrets = SecretStore::create(&export.secrets); + let channel = channel(export, &secrets)?; + Ok((channel, secrets)) +} + +pub fn span_exporter( + export: &OtlpExportConfig, + channel: Channel, + secrets: SecretStore, +) -> Result { + let interceptor = SecretHeaderInterceptor::bearer(secrets, export.auth_token.clone()); + + SpanExporter::builder() + .with_tonic() + .with_channel(channel) + .with_interceptor(interceptor) + .build() + .with_foreign_context( + || { + format!( + "building otel span exporter for endpoint {}", + export.endpoint + ) + }, + ErrorOrigin::Internal, + ) +} + +pub fn tracer_provider( + service_name: &'static str, + export: &OtlpExportConfig, + channel: Channel, + secrets: SecretStore, + sampler: &SamplerConfig, +) -> Result { + let exporter = span_exporter(export, channel, secrets)?; + + Ok(SdkTracerProvider::builder() + .with_resource(resource(service_name)) + .with_sampler(Sampler::from(sampler)) + .with_batch_exporter(exporter) + .build()) +} + +pub fn metric_exporter( + export: &OtlpExportConfig, + channel: Channel, + secrets: SecretStore, + temporality: &TemporalityConfig, +) -> Result { + let interceptor = SecretHeaderInterceptor::bearer(secrets, export.auth_token.clone()); + + MetricExporter::builder() + .with_tonic() + .with_channel(channel) + .with_interceptor(interceptor) + .with_temporality(Temporality::from(temporality)) + .build() + .with_foreign_context( + || { + format!( + "building otel metric exporter for endpoint {}", + export.endpoint + ) + }, + ErrorOrigin::Internal, + ) +} + +pub fn meter_provider( + service_name: &'static str, + export: &OtlpExportConfig, + channel: Channel, + secrets: SecretStore, + temporality: &TemporalityConfig, +) -> Result { + let exporter = metric_exporter(export, channel, secrets, temporality)?; + let reader = PeriodicReader::builder(exporter).build(); + + Ok(SdkMeterProvider::builder() + .with_resource(resource(service_name)) + .with_reader(reader) + .build()) +} diff --git a/server/src/di/inject/injectors/service/subscriber.rs b/server/src/di/inject/injectors/service/subscriber.rs new file mode 100644 index 0000000..4d42587 --- /dev/null +++ b/server/src/di/inject/injectors/service/subscriber.rs @@ -0,0 +1,305 @@ +use crate::config::{OtlpConfig, OtlpExportConfig, OtlpMetricsConfig, OtlpTracingConfig}; +use crate::di::error::DiError; +use crate::di::inject::injectors::service::otlp; +use crate::level::Level; +use opentelemetry::trace::TracerProvider; +use opentelemetry_sdk::metrics::SdkMeterProvider; +use opentelemetry_sdk::trace::SdkTracerProvider; +use switchgear_error::{ErrorOrigin, ForeignContext}; +use tracing::{Dispatch, Subscriber}; +use tracing_ecs_formatter::EcsFormatter; +use tracing_opentelemetry::MetricsLayer; +use tracing_subscriber::filter::{Directive, LevelFilter, filter_fn}; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::registry::LookupSpan; +use tracing_subscriber::{EnvFilter, Layer}; + +fn resolve_tracing(otlp: &OtlpConfig) -> Option<(&OtlpExportConfig, &OtlpTracingConfig)> { + let tracing = otlp.tracing.as_ref()?; + Some((tracing.export.as_ref().unwrap_or(&otlp.export), tracing)) +} + +fn resolve_metrics(otlp: &OtlpConfig) -> Option<(&OtlpExportConfig, &OtlpMetricsConfig)> { + let metrics = otlp.metrics.as_ref()?; + Some((metrics.export.as_ref().unwrap_or(&otlp.export), metrics)) +} + +pub fn env_filter() -> EnvFilter { + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")) +} + +pub fn ecs_layer(service_name: &'static str) -> impl Layer + Send + Sync +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + tracing_subscriber::fmt::layer() + .with_writer(std::io::stderr) + .event_format(EcsFormatter::new(service_name, env!("CARGO_PKG_VERSION"))) +} + +pub fn metrics_env_filter(level: LevelFilter) -> Result { + let target = switchgear_metrics::metrics_target(); + let directive = format!("{target}={level}"); + let parsed: Directive = directive.parse().with_foreign_context( + || format!("parsing metrics filter directive {directive}"), + ErrorOrigin::Internal, + )?; + Ok(env_filter().add_directive(parsed)) +} + +pub struct ServiceTracing { + dispatch: Dispatch, + provider: Option, + meter_provider: Option, +} + +impl ServiceTracing { + pub fn build(service_name: &'static str, otlp: Option<&OtlpConfig>) -> Result { + let metrics_off = std::env::var("OTEL_METRICS_EXPORTER").as_deref() == Ok("none"); + + let tracing_signal = otlp.and_then(resolve_tracing); + let metrics_signal = match otlp { + Some(otlp) if !metrics_off => resolve_metrics(otlp), + _ => None, + }; + + let shared = matches!( + (tracing_signal, metrics_signal), + (Some((t, _)), Some((m, _))) if std::ptr::eq(t, m) + ); + + let (provider, otel_layer, tracing_transport) = match tracing_signal { + None => (None, None, None), + Some((export, tracing)) => { + let (channel, secrets) = otlp::transport(export)?; + let provider = otlp::tracer_provider( + service_name, + export, + channel.clone(), + secrets.clone(), + &tracing.sampler, + )?; + let layer = tracing_opentelemetry::layer() + .with_tracer(provider.tracer(service_name)) + .with_location(false) + .with_tracked_inactivity(false) + .with_threads(false); + (Some(provider), Some(layer), Some((channel, secrets))) + } + }; + + let (meter_provider, metrics_layer, metrics_level) = match metrics_signal { + None => (None, None, LevelFilter::OFF), + Some((export, metrics)) => { + let (channel, secrets) = match (shared, &tracing_transport) { + (true, Some(transport)) => transport.clone(), + _ => otlp::transport(export)?, + }; + let provider = otlp::meter_provider( + service_name, + export, + channel, + secrets, + &metrics.temporality, + )?; + let layer = MetricsLayer::new(provider.clone()); + let level = LevelFilter::from(metrics.level.unwrap_or(Level::Info)); + (Some(provider), Some(layer), level) + } + }; + + let metrics_target = switchgear_metrics::metrics_target(); + + let subscriber = tracing_subscriber::registry() + .with(otel_layer.with_filter(filter_fn(|m| !m.is_event()))) + .with(metrics_layer.with_filter(filter_fn(move |m| m.target() == metrics_target))) + .with(metrics_env_filter(metrics_level)?) + .with( + ecs_layer(service_name) + .with_filter(filter_fn(move |m| m.target() != metrics_target)), + ); + + Ok(Self { + dispatch: Dispatch::new(subscriber), + provider, + meter_provider, + }) + } + + pub fn dispatch(&self) -> &Dispatch { + &self.dispatch + } + + pub async fn shutdown(self) -> Result<(), DiError> { + drop(self.dispatch); + let (tracer, meter) = (self.provider, self.meter_provider); + let (t, m) = tokio::join!( + async { + match tracer { + None => Ok(()), + Some(p) => tokio::task::spawn_blocking(move || p.shutdown()) + .await + .foreign_context( + "joining otel tracer provider shutdown task", + ErrorOrigin::Internal, + )? + .foreign_context( + "shutting down otel tracer provider", + ErrorOrigin::Internal, + ), + } + }, + async { + match meter { + None => Ok(()), + Some(p) => tokio::task::spawn_blocking(move || p.shutdown()) + .await + .foreign_context( + "joining otel meter provider shutdown task", + ErrorOrigin::Internal, + )? + .foreign_context( + "shutting down otel meter provider", + ErrorOrigin::Internal, + ), + } + }, + ); + t.and(m) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Arc, Mutex, MutexGuard}; + use tracing::{Event, Level, Metadata}; + use tracing_subscriber::layer::Context; + + /// Every value `otlp.metrics.level` can hold. + const ALL: [LevelFilter; 6] = [ + LevelFilter::OFF, + LevelFilter::ERROR, + LevelFilter::WARN, + LevelFilter::INFO, + LevelFilter::DEBUG, + LevelFilter::TRACE, + ]; + + /// Records nothing. What matters is that its filter votes on every + /// callsite, because a callsite's `Interest` is the sum of all of them. + struct Silent; + + impl Layer for Silent {} + + /// Stands where the metrics layer goes, and keeps what reaches it. + #[derive(Clone, Default)] + struct Capture(Arc>>); + + impl Capture { + fn levels(&self) -> Vec { + lock(&self.0).clone() + } + } + + impl Layer for Capture { + fn on_event(&self, event: &Event<'_>, _cx: Context<'_, S>) { + lock(&self.0).push(*event.metadata().level()); + } + } + + fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// Reproduces `tracing_opentelemetry::MetricsFilter`, the per-layer filter + /// `MetricsLayer` carries internally. + /// + /// The real `MetricsLayer` cannot be used here. It owns an + /// `SdkMeterProvider` whose `Drop` logs through `tracing`; when a stale + /// dispatcher is reaped inside `rebuild_callsite_interest`, that log + /// registers a callsite and re-enters `tracing-core`'s dispatcher lock, + /// which the rebuild already holds. The deadlock is intermittent, because + /// a callsite only registers once. `tests/features/otlp_metrics.rs` covers + /// the real layer end to end instead. + fn is_metrics_event(meta: &Metadata<'_>) -> bool { + meta.is_event() + && meta.fields().iter().any(|field| { + let name = field.name(); + name.starts_with("monotonic_counter.") + || name.starts_with("counter.") + || name.starts_with("histogram.") + || name.starts_with("gauge.") + }) + } + + /// Emits one `INFO` and one `DEBUG` metric under `filter`, in the layer + /// stack `build` assembles, and reports which of them reached the metrics + /// layer. + /// + /// The whole stack matters, not just the metrics layer: a callsite's + /// `Interest` is the sum of every per-layer filter's answer, and three of + /// these four exclude metric events. + fn record(filter: EnvFilter) -> Vec { + let metrics_target = switchgear_metrics::metrics_target(); + let capture = Capture::default(); + + let subscriber = tracing_subscriber::registry() + .with(Silent.with_filter(filter_fn(|m| !m.is_event()))) + .with(capture.clone().with_filter(filter_fn(move |m| { + m.target() == metrics_target && is_metrics_event(m) + }))) + .with(filter) + .with( + ecs_layer("swgr.test") + .with_filter(filter_fn(move |m| m.target() != metrics_target)), + ); + + tracing::subscriber::with_default(subscriber, || { + switchgear_metrics::histogram!("swgr_gate_info_ms", 1u64); + switchgear_metrics::histogram!(level: Level::DEBUG, "swgr_gate_debug_ms", 1u64); + }); + + capture.levels() + } + + #[test] + fn metrics_env_filter_builds_for_every_level_filter() { + for level in ALL { + metrics_env_filter(level).unwrap_or_else(|e| panic!("{level}: {e}")); + } + } + + #[test] + fn the_directive_gates_the_metrics_target_at_the_configured_level() { + for (level, expected) in [ + (LevelFilter::INFO, vec![Level::INFO]), + (LevelFilter::DEBUG, vec![Level::INFO, Level::DEBUG]), + (LevelFilter::OFF, vec![]), + ] { + let filter = metrics_env_filter(level).unwrap_or_else(|e| panic!("{level}: {e}")); + assert_eq!(record(filter), expected, "level: {level}"); + } + } + + #[test] + fn the_directive_replaces_one_from_rust_log_for_the_same_target() { + let target = switchgear_metrics::metrics_target(); + let rust_log = format!("info,{target}=trace"); + + assert_eq!( + record(EnvFilter::new(&rust_log)), + vec![Level::INFO, Level::DEBUG], + "RUST_LOG alone should have opened the metrics target" + ); + + let directive = format!("{target}=off"); + let filter = EnvFilter::new(&rust_log).add_directive( + directive + .parse() + .unwrap_or_else(|e| panic!("parsing {directive}: {e}")), + ); + + assert!(record(filter).is_empty()); + } +} diff --git a/server/src/di/inject/injectors/service/tracing.rs b/server/src/di/inject/injectors/service/tracing.rs index 1b5c462..de2ae91 100644 --- a/server/src/di/inject/injectors/service/tracing.rs +++ b/server/src/di/inject/injectors/service/tracing.rs @@ -1,26 +1,12 @@ -use crate::config::{OtlpConfig, OtlpTracingConfig}; use crate::di::error::DiError; +use crate::di::inject::injectors::service::subscriber::{ecs_layer, metrics_env_filter}; +use crate::level::Level; use crate::{CliArgs, RootCommands}; -use hyper_rustls::{FixedServerNameResolver, HttpsConnectorBuilder}; -use log::LevelFilter as LogLevelFilter; -use opentelemetry::KeyValue; -use opentelemetry::trace::TracerProvider; -use opentelemetry_otlp::{SpanExporter, WithTonicConfig}; -use opentelemetry_sdk::Resource; -use opentelemetry_sdk::trace::SdkTracerProvider; -use rustls::crypto::CryptoProvider; -use rustls::pki_types::pem::PemObject; -use rustls::pki_types::{CertificateDer, ServerName}; -use rustls::{ClientConfig, RootCertStore}; -use std::sync::Arc; -use switchgear_components::secrets::{ClientCertResolver, SecretHeaderInterceptor, SecretStore}; -use switchgear_error::{ErrorOrigin, ForeignContext}; -use tonic::transport::{Channel, Endpoint}; -use tracing_ecs_formatter::EcsFormatter; -use tracing_subscriber::filter::{Directive, filter_fn}; +use switchgear_error::ForeignContext; +use tracing_subscriber::Layer; +use tracing_subscriber::filter::{Directive, LevelFilter, filter_fn}; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; -use tracing_subscriber::{EnvFilter, Layer}; const GLOBAL_SERVICE_NAME: &str = "swgr"; @@ -31,83 +17,27 @@ impl TracingSystemInjector { match &args.command { RootCommands::Service { .. } => init_subscriber(), _ => { - let level = args.log_level.unwrap_or(LogLevelFilter::Info); + let level = args.log_level.unwrap_or(Level::Info); init_cli_subscriber(level) } } } } -pub struct ServiceSubscriber { - pub dispatch: tracing::Dispatch, - pub provider: Option, -} - -pub fn build_service_subscriber( - service_name: &'static str, - otel: Option<&OtlpConfig>, -) -> Result { - let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - - let fmt_layer = tracing_subscriber::fmt::layer() - .with_writer(std::io::stderr) - .event_format(EcsFormatter::new(service_name, env!("CARGO_PKG_VERSION"))); - - let (provider, otel_layer) = match otel { - None => (None, None), - Some(otel) => match &otel.tracing { - None => (None, None), - Some(tracing) => { - let provider = build_tracing_provider(service_name, tracing)?; - let layer = tracing_opentelemetry::layer() - .with_tracer(provider.tracer(service_name)) - .with_location(false) - .with_tracked_inactivity(false) - .with_threads(false); - (Some(provider), Some(layer)) - } - }, - }; - - let subscriber = tracing_subscriber::registry() - .with(otel_layer.with_filter(filter_fn(|m| !m.is_event()))) - .with(env_filter) - .with(fmt_layer); - - Ok(ServiceSubscriber { - dispatch: tracing::Dispatch::new(subscriber), - provider, - }) -} - fn init_subscriber() -> Result<(), DiError> { - let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - - let fmt_layer = tracing_subscriber::fmt::layer() - .with_writer(std::io::stderr) - .event_format(EcsFormatter::new( - GLOBAL_SERVICE_NAME, - env!("CARGO_PKG_VERSION"), - )); - + let metrics_target = switchgear_metrics::metrics_target(); tracing_subscriber::registry() - .with(env_filter) - .with(fmt_layer) + .with(metrics_env_filter(LevelFilter::OFF)?) + .with( + ecs_layer(GLOBAL_SERVICE_NAME) + .with_filter(filter_fn(move |m| m.target() != metrics_target)), + ) .try_init() .foreign_context("initializing tracing subscriber", None) } -fn init_cli_subscriber(level: LogLevelFilter) -> Result<(), DiError> { - let directive: Directive = match level { - LogLevelFilter::Off => "off", - LogLevelFilter::Error => "error", - LogLevelFilter::Warn => "warn", - LogLevelFilter::Info => "info", - LogLevelFilter::Debug => "debug", - LogLevelFilter::Trace => "trace", - } - .parse() - .expect("static log level parses as directive"); +fn init_cli_subscriber(level: Level) -> Result<(), DiError> { + let directive: Directive = LevelFilter::from(level).into(); serif::Config::new() .with_output(serif::Output::Stderr) @@ -118,109 +48,3 @@ fn init_cli_subscriber(level: LogLevelFilter) -> Result<(), DiError> { Ok(()) } - -fn build_tracing_provider( - service_name: &'static str, - tracing: &OtlpTracingConfig, -) -> Result { - let secrets = SecretStore::create(&tracing.secrets); - let exporter = build_tracing_exporter(tracing, secrets)?; - - let resource = Resource::builder() - .with_service_name(service_name) - .with_attribute(KeyValue::new("service.version", env!("CARGO_PKG_VERSION"))) - .build(); - - Ok(SdkTracerProvider::builder() - .with_resource(resource) - .with_batch_exporter(exporter) - .build()) -} - -fn build_tracing_exporter( - tracing: &OtlpTracingConfig, - secrets: SecretStore, -) -> Result { - let tls_config = build_client_tls_config(tracing, &secrets)?; - - let mut connector_builder = HttpsConnectorBuilder::new() - .with_tls_config(tls_config) - .https_or_http(); - if let Some(addr) = tracing.trusted_root_address.as_ref() { - let name: ServerName<'static> = - ServerName::try_from(addr.ip().to_string()).map_err(|e| { - DiError::internal(format!( - "parsing otel trusted-root-address {} as TLS server name: {e}", - addr.ip() - )) - })?; - connector_builder = - connector_builder.with_server_name_resolver(FixedServerNameResolver::new(name)); - } - let https = connector_builder.enable_http2().build(); - - let endpoint = Endpoint::from_shared(tracing.endpoint.to_string()).map_err(|e| { - DiError::internal(format!( - "parsing otel endpoint {} for tonic channel: {e}", - tracing.endpoint - )) - })?; - let channel = Channel::new(https, endpoint); - - let interceptor = SecretHeaderInterceptor::bearer(secrets, tracing.auth_token.clone()); - - SpanExporter::builder() - .with_tonic() - .with_channel(channel) - .with_interceptor(interceptor) - .build() - .with_foreign_context( - || { - format!( - "building otel span exporter for endpoint {}", - tracing.endpoint - ) - }, - ErrorOrigin::Internal, - ) -} - -fn build_client_tls_config( - tracing: &OtlpTracingConfig, - secrets: &SecretStore, -) -> Result { - let crypto = CryptoProvider::get_default() - .ok_or_else(|| DiError::internal("no rustls crypto provider installed for otel exporter"))? - .clone(); - - let mut roots = RootCertStore::empty(); - roots.add_parsable_certificates(rustls_native_certs::load_native_certs().certs); - if let Some(path) = tracing.trusted_roots.as_ref() { - let pem = std::fs::read(path).with_foreign_context( - || format!("reading otel trusted roots from {}", path.display()), - None, - )?; - let extra: Vec> = CertificateDer::pem_slice_iter(&pem) - .collect::>() - .with_foreign_context( - || format!("parsing otel trusted roots from {}", path.display()), - None, - )?; - roots.add_parsable_certificates(extra); - } - - let builder = ClientConfig::builder_with_provider(crypto.clone()) - .with_safe_default_protocol_versions() - .expect("rustls safe default protocol versions") - .with_root_certificates(roots); - - Ok(match tracing.client_identity.as_ref() { - Some(id) => builder.with_client_cert_resolver(Arc::new(ClientCertResolver::new( - secrets.clone(), - id.cert_secret.clone(), - id.key_secret.clone(), - crypto, - ))), - None => builder.with_no_client_auth(), - }) -} diff --git a/server/src/error.rs b/server/src/error.rs index 2a70ebc..6cec83c 100644 --- a/server/src/error.rs +++ b/server/src/error.rs @@ -3,9 +3,7 @@ use std::error::Error; use std::fmt::{Debug, Display, Formatter}; use std::panic::Location; -use opentelemetry_sdk::error::OTelSdkError; use switchgear_error::{ContextError, ContextErrorExt, ErrorOrigin, IntoContextError}; -use tokio::task::JoinError; use crate::commands::error::CliContextError; use crate::di::error::DiContextError; @@ -17,8 +15,6 @@ const OUTCOME_FAILURE: &str = "failure"; #[derive(Debug)] pub enum ServerErrorSourceKind { - OtelSdk(Box), - Join(Box), Cli(Box), Di(Box), } @@ -26,8 +22,6 @@ pub enum ServerErrorSourceKind { impl Display for ServerErrorSourceKind { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - Self::OtelSdk(e) => write!(f, "otel sdk error: {e}"), - Self::Join(e) => write!(f, "task join error: {e}"), Self::Cli(e) => Display::fmt(e, f), Self::Di(e) => Display::fmt(e, f), } @@ -37,8 +31,6 @@ impl Display for ServerErrorSourceKind { impl Error for ServerErrorSourceKind { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::OtelSdk(e) => Some(&**e), - Self::Join(e) => Some(&**e), Self::Cli(e) => Some(&**e), Self::Di(e) => Some(&**e), } @@ -185,36 +177,6 @@ impl ServerError { } } -impl IntoContextError for ServerError { - #[track_caller] - fn error>>( - source: Box, - message: M, - origin: Option, - ) -> Self { - Self::new( - ServerErrorSourceKind::OtelSdk(source), - origin.unwrap_or(ErrorOrigin::Internal), - message, - ) - } -} - -impl IntoContextError for ServerError { - #[track_caller] - fn error>>( - source: Box, - message: M, - origin: Option, - ) -> Self { - Self::new( - ServerErrorSourceKind::Join(source), - origin.unwrap_or(ErrorOrigin::Internal), - message, - ) - } -} - impl IntoContextError for ServerError { #[track_caller] fn error>>( @@ -262,11 +224,13 @@ impl ServerErrorAccumulator { } } - pub fn finish(self) -> Result<(), ServerError> { - match self.errors.len() { - 0 => Ok(()), - 1 => Err(self.errors.into_iter().next().expect("len == 1")), - _ => Err(ServerError::multi(self.errors)), + pub fn finish(mut self) -> Result<(), ServerError> { + if self.errors.len() > 1 { + return Err(ServerError::multi(self.errors)); + } + match self.errors.pop() { + Some(err) => Err(err), + None => Ok(()), } } } diff --git a/server/src/level.rs b/server/src/level.rs new file mode 100644 index 0000000..5c9df3b --- /dev/null +++ b/server/src/level.rs @@ -0,0 +1,160 @@ +use clap::ValueEnum; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::error::Error; +use std::fmt::{Display, Formatter}; +use std::str::FromStr; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, ValueEnum)] +#[clap(rename_all = "lowercase")] +pub enum Level { + Off, + Error, + Warn, + Info, + Debug, + Trace, +} + +impl Level { + pub const ALL: [Level; 6] = [ + Level::Off, + Level::Error, + Level::Warn, + Level::Info, + Level::Debug, + Level::Trace, + ]; + + pub fn as_str(self) -> &'static str { + match self { + Level::Off => "off", + Level::Error => "error", + Level::Warn => "warn", + Level::Info => "info", + Level::Debug => "debug", + Level::Trace => "trace", + } + } +} + +impl Display for Level { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug)] +pub struct ParseLevelError(String); + +impl Display for ParseLevelError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "unknown level {:?}, expected one of off, error, warn, info, debug, trace", + self.0 + ) + } +} + +impl Error for ParseLevelError {} + +impl FromStr for Level { + type Err = ParseLevelError; + + fn from_str(s: &str) -> Result { + match Level::ALL + .into_iter() + .find(|l| s.eq_ignore_ascii_case(l.as_str())) + { + Some(level) => Ok(level), + None => Err(ParseLevelError(s.to_owned())), + } + } +} + +impl Serialize for Level { + fn serialize(&self, s: S) -> Result { + s.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for Level { + fn deserialize>(d: D) -> Result { + let text = String::deserialize(d)?; + match text.parse() { + Ok(level) => Ok(level), + Err(e) => Err(serde::de::Error::custom(e)), + } + } +} + +impl From for tracing_subscriber::filter::LevelFilter { + fn from(level: Level) -> Self { + match level { + Level::Off => Self::OFF, + Level::Error => Self::ERROR, + Level::Warn => Self::WARN, + Level::Info => Self::INFO, + Level::Debug => Self::DEBUG, + Level::Trace => Self::TRACE, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tracing_subscriber::filter::LevelFilter; + + #[test] + fn a_level_parses_from_its_own_name_in_any_case() { + for level in Level::ALL { + let name = level.as_str(); + for spelling in [name.to_owned(), name.to_uppercase(), first_upper(name)] { + let parsed = spelling + .parse::() + .unwrap_or_else(|e| panic!("{spelling}: {e}")); + assert_eq!(parsed, level, "{spelling}"); + } + } + } + + #[test] + fn a_name_that_is_not_a_level_is_rejected() { + assert!("verbose".parse::().is_err()); + assert!("".parse::().is_err()); + } + + #[test] + fn every_level_converts_to_its_filter() { + for (level, expected) in [ + (Level::Off, LevelFilter::OFF), + (Level::Error, LevelFilter::ERROR), + (Level::Warn, LevelFilter::WARN), + (Level::Info, LevelFilter::INFO), + (Level::Debug, LevelFilter::DEBUG), + (Level::Trace, LevelFilter::TRACE), + ] { + assert_eq!(LevelFilter::from(level), expected, "{level}"); + } + } + + #[test] + fn clap_offers_every_level_under_the_name_it_is_written_with() { + for level in Level::value_variants() { + let value = level + .to_possible_value() + .unwrap_or_else(|| panic!("{level} has no clap value")); + assert_eq!(value.get_name(), level.as_str()); + } + assert_eq!(Level::value_variants().len(), Level::ALL.len()); + } + + fn first_upper(s: &str) -> String { + let mut chars = s.chars(); + match chars.next() { + Some(c) => c.to_uppercase().collect::() + chars.as_str(), + None => String::new(), + } + } +} diff --git a/server/src/lib.rs b/server/src/lib.rs index ef68c36..f8d8497 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1 +1,2 @@ pub mod config; +pub mod level; diff --git a/server/src/main.rs b/server/src/main.rs index 2cbc00b..a7751ef 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -2,6 +2,7 @@ mod commands; mod config; mod di; mod error; +mod level; mod signals; use crate::commands::offer::metadata::OfferMetadataManagementCommands; @@ -9,25 +10,24 @@ use crate::commands::offer::record::OfferRecordManagementCommands; use crate::di::inject::injectors::config::ServerConfigInjector; use crate::di::inject::injectors::service::tracing::TracingSystemInjector; use crate::error::{ServerError, ServerErrorAccumulator}; +use crate::level::Level; use clap::{Parser, Subcommand}; use commands::discovery::DiscoveryCommands; use commands::discovery::backend::DiscoveryBackendManagementCommands; use commands::offer::OfferCommands; use commands::services::ServiceEnablement; use commands::token::TokenCommands; -use log::LevelFilter as LogLevelFilter; -use opentelemetry_sdk::trace::SdkTracerProvider; use std::path::PathBuf; use std::process::ExitCode; -use switchgear_error::{ChainedContext, ErrorOrigin, ForeignContext}; +use switchgear_error::ChainedContext; /// lnurl load balance server #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] pub(crate) struct CliArgs { /// log level (CLI subcommands only; `swgr service` reads RUST_LOG) - #[clap(short, long, value_parser)] - pub(crate) log_level: Option, + #[clap(short, long, value_enum, ignore_case = true)] + pub(crate) log_level: Option, #[clap(subcommand)] pub(crate) command: RootCommands, @@ -81,14 +81,13 @@ async fn _main(args: CliArgs) -> Result<(), ServerError> { .install_default() .map_err(|_| ServerError::internal("failed to stand up rustls encryption platform"))?; - let mut otel_providers: Vec<(&'static str, SdkTracerProvider)> = Vec::new(); let mut errors = ServerErrorAccumulator::new(); let cli_result: Result<(), crate::commands::error::CliError> = match args.command { RootCommands::Service { config, enablement } => match ServerConfigInjector::new(config) .chained_context("loading server configuration", None) { - Ok(ci) => commands::services::execute(ci, enablement, &mut otel_providers).await, + Ok(ci) => commands::services::execute(ci, enablement).await, Err(e) => Err(e), }, RootCommands::Offer(offer) => match offer { @@ -308,28 +307,6 @@ async fn _main(args: CliArgs) -> Result<(), ServerError> { cli_result.with_chained_context(|| format!("cli command '{}'", summarize_command()), None); errors.push_result(result); - let mut shutdowns = tokio::task::JoinSet::new(); - for (name, provider) in otel_providers { - shutdowns.spawn_blocking(move || (name, provider.shutdown())); - } - while let Some(res) = shutdowns.join_next().await { - match res { - Ok((_, Ok(()))) => {} - Ok((name, Err(e))) => { - let r: Result<(), ServerError> = Err(e).with_foreign_context( - || format!("shutting down otel tracer provider for {name}"), - ErrorOrigin::Internal, - ); - errors.push_result(r); - } - Err(e) => { - let r: Result<(), ServerError> = - Err(e).foreign_context("otel shutdown task join failed", ErrorOrigin::Internal); - errors.push_result(r); - } - } - } - errors.finish() } diff --git a/server/tests/features/cli_offer_manage.rs b/server/tests/features/cli_offer_manage.rs index 0dfc9c8..620853c 100644 --- a/server/tests/features/cli_offer_manage.rs +++ b/server/tests/features/cli_offer_manage.rs @@ -1046,16 +1046,7 @@ async fn test_offer_post_invalid_metadata() { .expect("assert"); // Scenario steps - step_given_an_offer_json_with_non_existent_metadata_id_exists(&mut ctx, &mut cli_ctx) - .await - .expect("assert"); - step_when_i_run_swgr_offer_post(&mut ctx, &mut cli_ctx, CertificateLocation::Arg) - .await - .expect("assert"); - step_then_the_command_should_fail(&mut cli_ctx) - .await - .expect("assert"); - step_then_a_user_error_message_should_be_shown(&mut cli_ctx) + step_when_i_run_swgr_offer_post_with_unknown_metadata_expecting_failure(&mut ctx, &mut cli_ctx) .await .expect("assert"); @@ -1104,25 +1095,7 @@ async fn test_offer_metadata_delete_referenced() { .expect("assert"); // Scenario steps - step_given_a_valid_offer_json_exists(&mut ctx, &mut cli_ctx) - .await - .expect("assert"); - let metadata_id = extract_metadata_id_from_offer(&cli_ctx) - .await - .expect("assert"); - step_when_i_run_swgr_offer_post(&mut ctx, &mut cli_ctx, CertificateLocation::Arg) - .await - .expect("assert"); - step_then_the_command_should_succeed(&mut cli_ctx) - .await - .expect("assert"); - step_when_i_run_swgr_offer_metadata_delete(&mut ctx, &mut cli_ctx, &metadata_id) - .await - .expect("assert"); - step_then_the_command_should_fail(&mut cli_ctx) - .await - .expect("assert"); - step_then_a_user_error_message_should_be_shown(&mut cli_ctx) + step_when_i_run_swgr_offer_metadata_delete_referenced_expecting_failure(&mut ctx, &mut cli_ctx) .await .expect("assert"); diff --git a/server/tests/features/common/context/cli.rs b/server/tests/features/common/context/cli.rs index 2841c99..de89196 100644 --- a/server/tests/features/common/context/cli.rs +++ b/server/tests/features/common/context/cli.rs @@ -27,7 +27,7 @@ impl CliContext { let metadata_json_path = temp_dir.path().join("metadata.json"); Ok(Self { - cli: SwgrCli::create(log::Level::Info)?, + cli: SwgrCli::create(switchgear_server::level::Level::Info)?, temp_dir, public_key_path, private_key_path, @@ -76,7 +76,7 @@ struct SwgrCli { } impl SwgrCli { - pub fn create(log_level: log::Level) -> anyhow::Result { + pub fn create(log_level: switchgear_server::level::Level) -> anyhow::Result { let rust_log = std::env::var("RUST_LOG") .unwrap_or_else(|_| "".to_string()) .to_lowercase(); diff --git a/server/tests/features/common/context/global.rs b/server/tests/features/common/context/global.rs index 113a9f5..aaf7236 100644 --- a/server/tests/features/common/context/global.rs +++ b/server/tests/features/common/context/global.rs @@ -207,7 +207,7 @@ impl GlobalContext { pub async fn start_active_server( &mut self, start_services: &[Service], - log_level: log::Level, + log_level: switchgear_server::level::Level, ) -> anyhow::Result { self.get_active_server_mut()? .start_server(start_services, log_level) @@ -280,6 +280,18 @@ impl GlobalContext { JaegerClient::new(&self.otel_collector.jaeger_query_endpoint) } + /// Set the active server's OTLP resource attributes (`k=v,k=v`). Used by + /// metrics tests to tag the child's telemetry with a value they can select + /// on. Call before starting the server. + pub fn set_active_otlp_resource_attributes( + &mut self, + otlp_resource_attributes: Option, + ) -> anyhow::Result<()> { + self.get_active_server_mut()? + .set_otlp_resource_attributes(otlp_resource_attributes); + Ok(()) + } + /// Build a raw HTTP client aimed at the active server's given service. pub fn active_raw_client_for(&self, service: Service) -> anyhow::Result { self.get_active_server()?.raw_client_for(service) @@ -352,6 +364,14 @@ impl GlobalContext { self.get_active_server_mut()?.wait_exit_code() } + /// Every server's exit code, keyed by the name it was added under. + pub fn wait_all_exit_codes(&mut self) -> anyhow::Result> { + self.servers + .iter_mut() + .map(|(name, server)| Ok((name.clone(), server.wait_exit_code()?))) + .collect() + } + pub fn has_active_server_process(&self) -> anyhow::Result { Ok(self.get_active_server()?.has_server_process()) } diff --git a/server/tests/features/common/context/server.rs b/server/tests/features/common/context/server.rs index d82164f..f362c8b 100644 --- a/server/tests/features/common/context/server.rs +++ b/server/tests/features/common/context/server.rs @@ -78,6 +78,7 @@ pub struct ServerContext { ln_trusted_roots_path: Option, otel_collector: OtelCollector, + otlp_resource_attributes: Option, } impl ServerContext { @@ -260,6 +261,7 @@ impl ServerContext { }, ln_trusted_roots_path: None, otel_collector, + otlp_resource_attributes: None, }) } @@ -437,7 +439,7 @@ impl ServerContext { pub async fn start_server( &mut self, start_services: &[Service], - log_level: log::Level, + log_level: switchgear_server::level::Level, ) -> anyhow::Result { let rust_log = std::env::var("RUST_LOG") .unwrap_or_else(|_| "".to_string()) @@ -569,54 +571,58 @@ impl ServerContext { command .env( - "LNURL_SERVICE_OTLP_TRACING_ENDPOINT", + "LNURL_SERVICE_OTLP_ENDPOINT", &self.otel_collector.grpc_endpoint, ) .env( - "LNURL_SERVICE_OTLP_TRACING_AUTH_TOKEN", + "LNURL_SERVICE_OTLP_AUTH_TOKEN", &self.otel_collector.bearer_token_path, ) .env( - "LNURL_SERVICE_OTLP_TRACING_TRUSTED_ROOTS", + "LNURL_SERVICE_OTLP_TRUSTED_ROOTS", &self.otel_collector.ca_cert_path, ) .env( - "DISCOVERY_SERVICE_OTLP_TRACING_ENDPOINT", + "DISCOVERY_SERVICE_OTLP_ENDPOINT", &self.otel_collector.grpc_endpoint, ) .env( - "DISCOVERY_SERVICE_OTLP_TRACING_AUTH_TOKEN", + "DISCOVERY_SERVICE_OTLP_AUTH_TOKEN", &self.otel_collector.bearer_token_path, ) .env( - "DISCOVERY_SERVICE_OTLP_TRACING_TRUSTED_ROOTS", + "DISCOVERY_SERVICE_OTLP_TRUSTED_ROOTS", &self.otel_collector.ca_cert_path, ) .env( - "OFFER_SERVICE_OTLP_TRACING_ENDPOINT", + "OFFER_SERVICE_OTLP_ENDPOINT", &self.otel_collector.grpc_endpoint, ) .env( - "OFFER_SERVICE_OTLP_TRACING_AUTH_TOKEN", + "OFFER_SERVICE_OTLP_AUTH_TOKEN", &self.otel_collector.bearer_token_path, ) .env( - "OFFER_SERVICE_OTLP_TRACING_TRUSTED_ROOTS", + "OFFER_SERVICE_OTLP_TRUSTED_ROOTS", &self.otel_collector.ca_cert_path, ); if let Some(cert) = &self.otel_collector.client_cert_path { command - .env("LNURL_SERVICE_OTLP_TRACING_CLIENT_CERT", cert) - .env("DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_CERT", cert) - .env("OFFER_SERVICE_OTLP_TRACING_CLIENT_CERT", cert); + .env("LNURL_SERVICE_OTLP_CLIENT_CERT", cert) + .env("DISCOVERY_SERVICE_OTLP_CLIENT_CERT", cert) + .env("OFFER_SERVICE_OTLP_CLIENT_CERT", cert); } if let Some(key) = &self.otel_collector.client_key_path { command - .env("LNURL_SERVICE_OTLP_TRACING_CLIENT_KEY", key) - .env("DISCOVERY_SERVICE_OTLP_TRACING_CLIENT_KEY", key) - .env("OFFER_SERVICE_OTLP_TRACING_CLIENT_KEY", key); + .env("LNURL_SERVICE_OTLP_CLIENT_KEY", key) + .env("DISCOVERY_SERVICE_OTLP_CLIENT_KEY", key) + .env("OFFER_SERVICE_OTLP_CLIENT_KEY", key); } + if let Some(resource_attributes) = &self.otlp_resource_attributes { + command.env("OTEL_RESOURCE_ATTRIBUTES", resource_attributes); + } + if has_rust_log { println!("[STDOUT] Executing command: {command:?}"); let lnurl_profile = self.get_service_profile(Service::LnUrl)?; @@ -804,6 +810,14 @@ impl ServerContext { self.discovery_store_database_uri = discovery_store_database_uri; } + /// Resource attributes for the child's OTLP exporters, as the SDK's + /// `OTEL_RESOURCE_ATTRIBUTES` wire format (`k=v,k=v`). Unset by default: + /// they become resource keys on every exported span, which the OTLP + /// snapshots assert on. Call before start. + pub fn set_otlp_resource_attributes(&mut self, otlp_resource_attributes: Option) { + self.otlp_resource_attributes = otlp_resource_attributes; + } + pub fn set_ln_trusted_roots_path(&mut self, ln_trusted_roots_path: Option) { self.ln_trusted_roots_path = ln_trusted_roots_path; } @@ -826,7 +840,7 @@ impl ServerContext { /// Point the swgr child at a different OTLP collector than the shared /// container-backed one. `start_server` reads `self.otel_collector` when - /// wiring the `OTLP_TRACING_*` env vars, so this must be called before start. + /// wiring the `OTLP_*` env vars, so this must be called before start. pub fn set_otel_collector_override(&mut self, otel_collector: OtelCollector) { self.otel_collector = otel_collector; } diff --git a/server/tests/features/common/otlp_collector.rs b/server/tests/features/common/otlp_collector.rs index 05e4bb0..28f9946 100644 --- a/server/tests/features/common/otlp_collector.rs +++ b/server/tests/features/common/otlp_collector.rs @@ -1,4 +1,4 @@ -//! In-process OTLP trace gRPC collector for OTLP-compliance snapshot tests. +//! In-process OTLP gRPC collector for OTLP-compliance snapshot tests. //! //! Runs on a per-test tokio task, receives spans via the OTLP protobuf //! ExportTraceServiceRequest, buffers them lossless in memory, and on @@ -6,11 +6,21 @@ //! deterministically-sorted JSON-lines file (one line per root span) that //! `insta` snapshots as the OTLP compliance assertion. //! +//! It also serves the OTLP metrics service, which accepts and discards +//! everything: the child exports both signals over one connection, and an +//! UNIMPLEMENTED reply would only add exporter errors to its log. +//! //! Runs over plain HTTP/2 (h2c) — no TLS. TLS adds no assertion value //! (we're snapshotting on-wire OTLP shape, not transport crypto) and only //! introduces IP-SAN / SNI surface that flakes across rustls versions. use anyhow::{Context, Result, anyhow}; +use opentelemetry_proto::tonic::collector::metrics::v1::metrics_service_server::{ + MetricsService, MetricsServiceServer, +}; +use opentelemetry_proto::tonic::collector::metrics::v1::{ + ExportMetricsServiceRequest, ExportMetricsServiceResponse, +}; use opentelemetry_proto::tonic::collector::trace::v1::trace_service_server::{ TraceService, TraceServiceServer, }; @@ -95,6 +105,26 @@ impl TraceService for CollectorService { } } +/// Accepts and discards metrics. +/// +/// The trace pipeline is what these snapshots assert; metrics only need to be +/// received, because the SDK's `PeriodicReader` exports on its own schedule and +/// on shutdown, and an UNIMPLEMENTED reply would surface as exporter errors in +/// the child's log — noise that other assertions would then have to tolerate. +struct MetricsCollectorService; + +#[tonic::async_trait] +impl MetricsService for MetricsCollectorService { + async fn export( + &self, + _request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + Ok(tonic::Response::new(ExportMetricsServiceResponse { + partial_success: None, + })) + } +} + fn ingest_resource_spans(state: &mut CollectorState, rs: ResourceSpans) { let resource_attrs = rs .resource @@ -198,7 +228,9 @@ impl TestOtlpCollector { state: Arc::clone(&state), }; let (tx, rx) = oneshot::channel::<()>(); - let server = Server::builder().add_service(TraceServiceServer::new(service)); + let server = Server::builder() + .add_service(TraceServiceServer::new(service)) + .add_service(MetricsServiceServer::new(MetricsCollectorService)); let serve_handle = tokio::spawn(async move { server diff --git a/server/tests/features/common/step_functions.rs b/server/tests/features/common/step_functions.rs index 7601a68..d68e865 100644 --- a/server/tests/features/common/step_functions.rs +++ b/server/tests/features/common/step_functions.rs @@ -18,6 +18,7 @@ use chrono::{Duration as ChronoDuration, Utc}; use rand::{Rng, distributions::Alphanumeric}; use reqwest::{StatusCode, Url}; use secp256k1::PublicKey; +use serde_json::Value; use std::collections::BTreeMap; use std::time::{Duration, SystemTime}; use std::vec; @@ -37,6 +38,8 @@ use switchgear_service_api::offer::{ OfferMetadata, OfferMetadataSparse, OfferMetadataStore, OfferRecord, OfferRecordSparse, OfferStore, }; +use switchgear_testing::credentials::influx::InfluxCredentials; +use switchgear_testing::influx::InfluxClient; use tokio::time::sleep as tokio_sleep; use uuid::Uuid; // ============================================================================= @@ -119,7 +122,7 @@ pub async fn step_when_i_start_the_lnurl_server_with_the_configuration( let _pid = ctx .start_active_server( &[Service::LnUrl, Service::Discovery, Service::Offer], - log::Level::Info, + switchgear_server::level::Level::Info, ) .await?; Ok(()) @@ -136,7 +139,9 @@ pub async fn step_when_i_send_a_signal_to_the_server_process( } /// Step: "When I send a SIGTERM signal to the server process" -/// Sends a SIGTERM signal to the running server process for graceful shutdown +/// Sends SIGTERM to every server in the scenario, not only the active one, +/// and waits for each to exit. For metrics scenarios this is also the flush: +/// SIGTERM drains each child's meter provider. pub async fn step_when_i_send_a_sigterm_signal_to_the_server_process( ctx: &mut GlobalContext, ) -> Result<()> { @@ -162,7 +167,13 @@ pub async fn step_and_all_services_should_be_listening_on_their_configured_ports ) -> Result<()> { let services = vec![Service::LnUrl, Service::Discovery, Service::Offer]; - check_services_listening_status(ctx, services, true, 100, 5).await?; + // 1000ms/10s matches every other positive check (the 100ms here was the + // negative-check value: a short cap is right when asserting nothing + // responds, wrong when waiting for a cold-started child to bind three TLS + // ports). The loop breaks on the first unready service, so a failing pass + // costs probe+200ms — a probe that would answer in ~150ms was scored as a + // failure on every iteration, and this step checks the most services. + check_services_listening_status(ctx, services, true, 1000, 10).await?; Ok(()) } @@ -272,6 +283,18 @@ pub async fn step_then_the_server_should_exit_with_code_0(ctx: &mut GlobalContex Ok(()) } +/// Step: "Then all servers should exit with code 0" +/// Verifies every server in the scenario shut down cleanly +pub async fn step_then_all_servers_should_exit_with_code_0(ctx: &mut GlobalContext) -> Result<()> { + for (server, exit_code) in ctx.wait_all_exit_codes()? { + if exit_code != 0 { + bail_log!("server {server} exited with code {exit_code}, expected 0"); + } + } + + Ok(()) +} + /// Step: "Then the server should exit with a non-zero code" /// Verifies that the server process exits with an error exit code (non-zero) pub async fn step_then_the_server_should_exit_with_a_non_zero_code( @@ -1027,18 +1050,33 @@ pub async fn step_given_the_payer_can_generate_invoices_successfully( Ok(()) } +/// Step: "When the admin disables the {payee_id} backend" +/// Disables the discovery backend registered for that payee's LN node +pub async fn step_when_the_admin_disables_the_backend( + ctx: &mut GlobalContext, + payee_id: &str, +) -> Result<()> { + let public_key = payee_backend_public_key(ctx, payee_id)?; + + enable_disable_backend(ctx, &public_key, false).await +} + +/// The public key of the LN node a payee targets — the identity its +/// discovery backend is registered under. +fn payee_backend_public_key(ctx: &GlobalContext, payee_id: &str) -> Result { + let payee = get_payee_from_context(ctx, payee_id)?; + + match payee.target_ln_node.as_str() { + "cln" => Ok(payee.ln_nodes.cln.public_key), + "lnd" => Ok(payee.ln_nodes.lnd.public_key), + node => bail_log!("payee {payee_id} targets unknown ln node {node}"), + } +} + /// Step: "When the admin disables the first backend" /// Disables the first backend (LND) pub async fn step_when_the_admin_disables_the_first_backend(ctx: &mut GlobalContext) -> Result<()> { - // Get LND backend location - let lnd_payee = ctx - .get_payee("lnd") - .ok_or_else(|| anyhow_log!("LND payee not found in context"))? - .clone(); - - enable_disable_backend(ctx, &lnd_payee.ln_nodes.lnd.public_key, false).await?; - - Ok(()) + step_when_the_admin_disables_the_backend(ctx, "lnd").await } /// Step: "Then the payer can still get invoices" @@ -1057,15 +1095,7 @@ pub async fn step_then_the_payer_can_still_generate_invoices( pub async fn step_when_the_admin_disables_the_second_backend( ctx: &mut GlobalContext, ) -> Result<()> { - // Get CLN backend location - let cln_payee = ctx - .get_payee("cln") - .ok_or_else(|| anyhow_log!("CLN payee not found in context"))? - .clone(); - - enable_disable_backend(ctx, &cln_payee.ln_nodes.cln.public_key, false).await?; - - Ok(()) + step_when_the_admin_disables_the_backend(ctx, "cln").await } /// Step: "Then the payer cannot get invoices" @@ -1163,32 +1193,27 @@ async fn enable_disable_backend( Ok(()) } +/// Step: "When the admin deletes the {payee_id} backend" +/// Deletes that payee's backend from the discovery service +pub async fn step_when_the_admin_deletes_the_backend( + ctx: &mut GlobalContext, + payee_id: &str, +) -> Result<()> { + let public_key = payee_backend_public_key(ctx, payee_id)?; + + delete_backend(ctx, &public_key).await +} + /// Step: "When the admin deletes the first backend" /// Deletes the first backend (LND) from the discovery service pub async fn step_when_the_admin_deletes_the_first_backend(ctx: &mut GlobalContext) -> Result<()> { - // Get LND backend location - let lnd_payee = ctx - .get_payee("lnd") - .ok_or_else(|| anyhow_log!("LND payee not found in context"))? - .clone(); - - delete_backend(ctx, &lnd_payee.ln_nodes.lnd.public_key).await?; - - Ok(()) + step_when_the_admin_deletes_the_backend(ctx, "lnd").await } /// Step: "When the admin deletes the second backend" /// Deletes the second backend (CLN) from the discovery service pub async fn step_when_the_admin_deletes_the_second_backend(ctx: &mut GlobalContext) -> Result<()> { - // Get CLN backend location - let cln_payee = ctx - .get_payee("cln") - .ok_or_else(|| anyhow_log!("CLN payee not found in context"))? - .clone(); - - delete_backend(ctx, &cln_payee.ln_nodes.cln.public_key).await?; - - Ok(()) + step_when_the_admin_deletes_the_backend(ctx, "cln").await } /// Step: "And both nodes are ready to be registered as backends" @@ -1248,7 +1273,7 @@ pub async fn step_when_i_start_the_lnurl_server_with_enablement_flags( start_services: &[Service], ) -> Result<()> { let _pid = ctx - .start_active_server(start_services, log::Level::Info) + .start_active_server(start_services, switchgear_server::level::Level::Info) .await?; Ok(()) } @@ -2047,7 +2072,10 @@ pub async fn step_when_i_start_server_1_with_offers_and_discovery_services( ctx: &mut GlobalContext, ) -> Result<()> { let _pid = ctx - .start_active_server(&[Service::Discovery, Service::Offer], log::Level::Info) + .start_active_server( + &[Service::Discovery, Service::Offer], + switchgear_server::level::Level::Info, + ) .await?; Ok(()) @@ -2059,7 +2087,7 @@ pub async fn step_when_i_start_server_2_with_only_lnurl_service( ctx: &mut GlobalContext, ) -> Result<()> { let _pid = ctx - .start_active_server(&[Service::LnUrl], log::Level::Info) + .start_active_server(&[Service::LnUrl], switchgear_server::level::Level::Info) .await?; Ok(()) } @@ -5117,3 +5145,253 @@ pub async fn step_when_i_run_swgr_offer_metadata_delete_for_non_existent_metadat cli_ctx.command(empty_env, args)?; Ok(()) } + +// ============================================================================= +// OTLP METRICS STEP FUNCTIONS +// ============================================================================= + +/// The three instruments the workspace emits. +pub const DB_OPERATION_DURATION: &str = "db.client.operation.duration"; +pub const HTTP_REQUEST_DURATION: &str = "http.client.request.duration"; +pub const RPC_CALL_DURATION: &str = "rpc.client.call.duration"; + +/// The `service.name` each service's dispatch stamps on its telemetry. +pub const LNURL_SERVICE: &str = "swgr.lnurl"; +pub const OFFER_SERVICE: &str = "swgr.offer"; +pub const DISCOVERY_SERVICE: &str = "swgr.discovery"; + +/// Every metric name, for asserting that none of them reaches an ECS log +/// record. +const METRIC_NAMES: &[&str] = &[ + DB_OPERATION_DURATION, + HTTP_REQUEST_DURATION, + RPC_CALL_DURATION, +]; + +/// The `error.type` half of a histogram row selector. +#[derive(Clone, Copy, Debug)] +pub enum MetricOutcome { + /// The call succeeded, so the row carries no `error.type` at all. + Success, + /// The call failed with this `error.type`. + Error(&'static str), +} + +/// A random per-scenario `test_identity`. Hex, because it is interpolated +/// into a SQL string literal. +pub fn new_test_identity() -> String { + format!("{:032x}", rand::random::()) +} + +/// Step: "Given the child server is stamped with a unique test_identity" +/// InfluxDB is shared by every test and every previous run, so each scenario +/// tags its child's telemetry with a value it can select on. Call before +/// starting the server. +pub async fn step_given_the_active_server_is_stamped_with_test_identity( + ctx: &mut GlobalContext, + identity: &str, +) -> Result<()> { + ctx.set_active_otlp_resource_attributes(Some(format!("test_identity={identity}")))?; + + Ok(()) +} + +/// An InfluxDB client aimed at the shared test database. +/// +/// The credentials' `TempDir` is dropped on return, which is safe: +/// `InfluxClient` owns a copy of the token, so the extracted file is never +/// read again. +pub fn influx_client() -> Result { + let credentials = InfluxCredentials::create()?; + let influx = credentials.get_influx()?; + + Ok(InfluxClient::new(&influx.query_endpoint, &influx.token) + .with_timeout(Duration::from_secs(30))) +} + +/// Step: "Then InfluxDB should have a {metric} row for this test_identity" +/// +/// `selectors` are the datapoint attributes that pick out one call site, so +/// one step serves all three metrics. Every attribute column carries a dot +/// and must therefore be a quoted SQL identifier — unquoted, `error.type` +/// parses as column `type` of table `error`. +pub async fn step_then_influx_should_have_a_histogram_row( + influx: &InfluxClient, + identity: &str, + metric: &str, + service: &str, + selectors: &[(&str, &str)], + outcome: MetricOutcome, +) -> Result { + let mut base = format!( + "SELECT * FROM \"{metric}\" WHERE test_identity = '{identity}' \ + AND \"service.name\" = '{service}'" + ); + for (column, value) in selectors { + base.push_str(&format!(" AND \"{column}\" = '{value}'")); + } + + // Wait for this run's datapoints to land before naming `error.type`: + // InfluxDB creates a tag column on the first write that carries it, and + // naming a column the measurement has never seen is a planning error + // rather than an empty result. + influx.wait_for_rows(&format!("{base} LIMIT 1")).await?; + + let sql = match outcome { + MetricOutcome::Error(error_type) => { + format!("{base} AND \"error.type\" = '{error_type}' LIMIT 1") + } + MetricOutcome::Success if has_error_type_column(influx, metric).await? => { + format!("{base} AND \"error.type\" IS NULL LIMIT 1") + } + // Nothing in this measurement has ever carried `error.type`, so + // every row that matches is a success and the clause is redundant. + MetricOutcome::Success => format!("{base} LIMIT 1"), + }; + + influx + .wait_for_rows(&sql) + .await? + .into_iter() + .next() + .ok_or_else(|| anyhow_log!("no row returned for {sql}")) +} + +/// Whether any row of `metric` has ever carried an `error.type` tag. +async fn has_error_type_column(influx: &InfluxClient, metric: &str) -> Result { + let rows = influx + .query(&format!( + "SELECT column_name FROM information_schema.columns \ + WHERE table_name = '{metric}' AND column_name = 'error.type'" + )) + .await?; + + Ok(!rows.is_empty()) +} + +/// Step: "Then the histogram row should be well formed" +/// Checks the resource attributes shared with logs and traces, and the +/// histogram body the collector wrote. +pub fn step_then_the_histogram_row_should_be_well_formed(row: &Value) -> Result<()> { + if row_string(row, "service.version")?.is_empty() { + bail_log!("service.version is empty, row = {row}"); + } + + // `count >= 1`, not `== 1`: `get_offer` fires twice per LNURL flow and + // `get_all` once per balancer retry. + let count = row_number(row, "count")?; + if count < 1.0 { + bail_log!("count is {count}, expected at least 1, row = {row}"); + } + + let sum = row_number(row, "sum")?; + if sum <= 0.0 { + bail_log!("sum is {sum}, expected greater than 0, row = {row}"); + } + + let infinity = row_number(row, "+Inf")?; + if infinity != count { + bail_log!("+Inf bucket is {infinity}, expected count {count}, row = {row}"); + } + + let (max, min) = (row_number(row, "max")?, row_number(row, "min")?); + if max < min { + bail_log!("max {max} is below min {min}, row = {row}"); + } + + Ok(()) +} + +/// Step: "Then no ECS log record should carry the metrics target or a metric +/// field" +/// +/// The metrics layer and the log layer partition events by target, so a +/// metric event must never reach an ECS record. `event.module` carries the +/// emitting event's own target, so it is never the metrics target either — +/// and `swgr::metrics=off`, being a *level* in a directive string, never +/// becomes event metadata and cannot surface as a field. +/// +/// This runs on scenarios that have just proved a metric reached InfluxDB, +/// so the metric events demonstrably happened: an empty stderr assertion +/// here is an assertion about real events, not a vacuous one. +pub fn step_then_no_ecs_log_record_should_carry_a_metric_field( + ctx: &mut GlobalContext, +) -> Result<()> { + let buffer = ctx.get_active_stderr_buffer()?; + let lines = buffer.lock().unwrap_or_else(|e| e.into_inner()).clone(); + + for line in &lines { + let Ok(Value::Object(record)) = serde_json::from_str::(line) else { + continue; + }; + if !record.contains_key("ecs.version") { + continue; + } + if record.get("event.module").and_then(Value::as_str) == Some("swgr::metrics") { + bail_log!("metrics target as event.module: {line}"); + } + for key in record.keys() { + if key.starts_with("histogram.") + || key.starts_with("counter.") + || key.starts_with("monotonic_counter.") + || key.starts_with("gauge.") + { + bail_log!("metric field {key:?} in an ECS record: {line}"); + } + } + for metric in METRIC_NAMES { + if record.contains_key(*metric) { + bail_log!("metric name {metric:?} as an ECS field: {line}"); + } + } + } + + Ok(()) +} + +/// A string column of an InfluxDB row. +fn row_string(row: &Value, key: &str) -> Result { + row.get(key) + .and_then(Value::as_str) + .map(str::to_owned) + .with_context(|| format!("row has no string column {key:?}: {row}")) +} + +/// A numeric column of an InfluxDB row. +fn row_number(row: &Value, key: &str) -> Result { + row.get(key) + .and_then(Value::as_f64) + .with_context(|| format!("row has no numeric column {key:?}: {row}")) +} + +/// The `cli_offer_manage.rs` "post an offer whose metadata does not exist" +/// sequence: an offer POST the offer store rejects on its foreign key. +pub async fn step_when_i_run_swgr_offer_post_with_unknown_metadata_expecting_failure( + ctx: &mut GlobalContext, + cli_ctx: &mut CliContext, +) -> Result<()> { + step_given_an_offer_json_with_non_existent_metadata_id_exists(ctx, cli_ctx).await?; + step_when_i_run_swgr_offer_post(ctx, cli_ctx, CertificateLocation::Arg).await?; + step_then_the_command_should_fail(cli_ctx).await?; + step_then_a_user_error_message_should_be_shown(cli_ctx).await?; + + Ok(()) +} + +/// The `cli_offer_manage.rs` "delete metadata an offer still references" +/// sequence: posts an offer, then a metadata DELETE the offer store rejects +/// on its foreign key. +pub async fn step_when_i_run_swgr_offer_metadata_delete_referenced_expecting_failure( + ctx: &mut GlobalContext, + cli_ctx: &mut CliContext, +) -> Result<()> { + step_given_a_valid_offer_json_exists(ctx, cli_ctx).await?; + let metadata_id = extract_metadata_id_from_offer(cli_ctx).await?; + step_when_i_run_swgr_offer_post(ctx, cli_ctx, CertificateLocation::Arg).await?; + step_then_the_command_should_succeed(cli_ctx).await?; + step_when_i_run_swgr_offer_metadata_delete(ctx, cli_ctx, &metadata_id).await?; + step_then_the_command_should_fail(cli_ctx).await?; + step_then_a_user_error_message_should_be_shown(cli_ctx).await?; + + Ok(()) +} diff --git a/server/tests/features/main.rs b/server/tests/features/main.rs index eda737e..e3c71e6 100644 --- a/server/tests/features/main.rs +++ b/server/tests/features/main.rs @@ -13,6 +13,9 @@ mod http_remote_stores; mod invalid_configuration_rejection; mod lnurl_pay_invoice_generation; mod lnurl_pay_multi_backend_invoice_generation; +mod otlp_discovery_store_metrics; +mod otlp_metrics; +mod otlp_offer_store_metrics; mod otlp_spans; mod secrets; mod server_lifecycle_with_graceful_shutdown; diff --git a/server/tests/features/otlp-discovery-store-metrics.feature b/server/tests/features/otlp-discovery-store-metrics.feature new file mode 100644 index 0000000..51b41a9 --- /dev/null +++ b/server/tests/features/otlp-discovery-store-metrics.feature @@ -0,0 +1,118 @@ +@integration @otlp-metrics @otlp-compliance @discovery-store +Feature: OTLP metrics for the discovery database store + As a platform maintainer + I want every database call the discovery service makes to be measured + So that a slow backend lookup is visible on a dashboard, and so that + an uninstrumented call site fails a test rather than leaving a silent + gap + + # Scope: this feature covers `db.client.operation.duration` for the + # seven call sites in `components/src/discovery/db.rs`. It is the + # mirror of `otlp-offer-store-metrics.feature` — same pipeline, same + # SQLite-backed server, same `swgr` CLI driving it — over the + # discovery service's own store. + # + # Attributes on each row: + # + # | attribute | scope | value | + # | db.system.name | store | sqlite | + # | db.namespace | store | the SQLite file stem | + # | db.collection.name | site | the table the call is about | + # | swgr.operation | site | the call site | + # + # `swgr.operation` is not from the semantic-convention registry — it + # names the call site inside this workspace, under the `swgr.` prefix + # so it can never collide with a registry attribute. `server.address` + # and `server.port` are absent throughout: SQLite has no server. + # + # Two call sites are worth naming. `get_all` makes two round trips — + # an etag lookup and, when the caller's etag is stale, the backend + # fetch — so it contributes both `get_all_etag` and + # `get_all_backends`. And the four writes are each a single + # `DatabaseConnection::transaction` spanning `discovery_backend` and + # `discovery_backend_etag`: the transaction is the unit measured, its + # own Result decides the outcome, and `db.collection.name` names the + # table the call is about while the etag bump is incidental. + # + # `patch` covers three CLI commands: `discovery patch`, `discovery + # disable` and `discovery enable` all reach the same call site. + # + # Isolation and flush work as in `otlp-metrics.feature`: a random + # `test_identity` resource attribute scopes the run, and SIGTERM + # drains the meter provider, so the query happens after the child has + # exited with code 0. `count` is asserted as at least 1, never + # exactly 1 — `patch` alone runs three times here. + # + # No `error.type` row: the discovery schema has no foreign keys, so + # no call this API surface can make will fail the transaction. The + # `error.type` mapping itself is unit-tested against every `DbErr` + # variant in `components/src/metrics.rs`, and its end-to-end path is + # covered by `otlp-offer-store-metrics.feature`, whose foreign keys + # do reject. + # + # Out of scope: the MySQL and PostgreSQL variants of this scenario, + # which are a one-line change via + # `ctx.set_discovery_store_database_uri`; and the discovery store's + # HTTP implementation, whose write surface no service calls. + + Background: + Given a valid configuration file exists + And the server is not already running + And the OTel collector and InfluxDB containers are running + And the swgr CLI is available + And the server is configured with SQLite persistence + And the child server is stamped with a unique test_identity resource attribute + + # ----------------------------------------------------------------- + # One scenario for all seven call sites: they share a server, a store + # and a test_identity, and the reads need the rows the writes create. + # ----------------------------------------------------------------- + + @otlp-metrics-discovery-store-operations + Scenario: Every discovery store operation reaches InfluxDB + When I start the LNURL server with the configuration + Then the server should start successfully + And all services should be listening on their configured ports + Given a valid backend JSON exists + When I run "swgr discovery post" with backend JSON + Then the command should succeed + When I run "swgr discovery get" for the backend public key + Then the command should succeed + When I run "swgr discovery ls" + Then the command should succeed + When I run "swgr discovery get-all" + Then the command should succeed + Given updated backend JSON exists + When I run "swgr discovery put" for the backend public key + Then the command should succeed + Given backend patch JSON exists + When I run "swgr discovery patch" for the backend public key + Then the command should succeed + When I run "swgr discovery disable" for the backend public key + Then the command should succeed + When I run "swgr discovery enable" for the backend public key + Then the command should succeed + When I run "swgr discovery get" for a non-existent backend public key + Then the command should fail + # The store still succeeded: the empty result becomes `Ok(None)` and + # only the CLI treats it as an error. + When I run "swgr discovery delete" for the backend public key + Then the command should succeed + When I send a SIGTERM signal to the server process + # SIGTERM drains the meter provider: the PeriodicReader's own + # interval is 60s, so shutdown is what puts the run's datapoints on + # the wire. + Then the server should exit with code 0 + And InfluxDB should have a "db.client.operation.duration" row for this test_identity under service.name "swgr.discovery" for each operation: + | swgr.operation | db.collection.name | driven by | + | get | discovery_backend | discovery get | + | get_all_etag | discovery_backend_etag | discovery ls, discovery get-all | + | get_all_backends | discovery_backend | discovery ls, discovery get-all | + | post | discovery_backend | discovery post | + | put | discovery_backend | discovery put | + | patch | discovery_backend | patch, disable, enable | + | delete | discovery_backend | discovery delete | + And each row's "db.system.name" should be "sqlite" + And each row should carry no "error.type" + And each row should be a well-formed histogram + And no ECS log record should carry the metrics target or a metric field diff --git a/server/tests/features/otlp-metrics.feature b/server/tests/features/otlp-metrics.feature new file mode 100644 index 0000000..2799395 --- /dev/null +++ b/server/tests/features/otlp-metrics.feature @@ -0,0 +1,223 @@ +@integration @otlp-metrics @otlp-compliance +Feature: OTLP metrics export + As a platform maintainer + I want swgr's metric events to reach a metrics sink through the OTLP + pipeline with the resource and datapoint attributes intact + So that dashboards and alerts can be built on them, and so that a + regression in the metrics bridge fails a test rather than silently + emptying a dashboard + + # Scope: this feature covers the whole metrics path end to end — + # `switchgear_metrics::histogram!("", …)` at the call site, which + # expands to a `tracing` event on the `swgr::metrics` target carrying a + # `histogram.` field; the `MetricsLayer` bridge in the service + # dispatch; the `SdkMeterProvider` + `PeriodicReader`; OTLP gRPC export to + # the containerised OTel collector; and the collector's InfluxDB exporter. + # The assertion is a SQL query against InfluxDB, so it proves the metric + # is queryable, not merely that bytes left the process. + # + # Three instruments are emitted, all histograms of milliseconds: + # + # | metric | call sites | + # | db.client.operation.duration | offer/db.rs, discovery/db.rs | + # | http.client.request.duration | offer/http.rs, discovery/http.rs | + # | rpc.client.call.duration | pool/{cln,lnd}/grpc/client.rs | + # + # Their attributes follow the OpenTelemetry semantic conventions, with one + # addition: `swgr.operation` names the call site inside this workspace and + # is not from the registry. Conditional attributes are absent rather than + # empty — `error.type` appears only on a failure, and a SQLite store has + # no server address or port at all. + # + # This feature covers the LNURL service's read path: its gRPC calls to a + # Lightning node, and the requests its stores make when those stores are + # HTTP or database-backed. The database stores' own call sites — every + # `swgr.operation` in `offer/db.rs` and `discovery/db.rs`, driven by the + # CLI — live in `otlp-offer-store-metrics.feature` and + # `otlp-discovery-store-metrics.feature`. + # + # Each assertion has exactly one home. `rpc.client.call.duration` belongs + # to the two gRPC scenarios below, which exist to cover both node types' + # call sites; the HTTP- and database-store scenarios run the same invoice + # flow but do not re-assert it, because the step that requests the invoice + # already fails the scenario if no invoice comes back. + # + # The same scenarios assert the other side of the partition: the log layer + # excludes the metrics target, so the child's ECS stderr carries no metric + # record — no `swgr::metrics` as `event.module`, and no metric-prefixed + # field. + # + # The collector writes with the `telegraf-prometheus-v1` schema: one + # measurement per metric name, with resource, scope and datapoint + # attributes as tags and the histogram body (`count`, `sum`, `min`, + # `max`, and one column per bucket boundary) as fields. The bucket + # boundaries are the SDK's defaults, which is why the call sites record + # milliseconds — seconds-valued data would collapse into the first two + # buckets. + # + # `count` is asserted as at least 1, never exactly 1: a single LNURL flow + # fetches its offer twice, and the balancer's retry refreshes discovery + # once per attempt. + # + # Isolation: InfluxDB is shared by every test and every previous run, so + # each scenario stamps its children with a random + # `OTEL_RESOURCE_ATTRIBUTES=test_identity=` and selects on it. The + # tag is set on the children only — the test process exports nothing. + # + # Flush: the `PeriodicReader`'s export interval is 60s, far longer than a + # scenario. SIGTERM drains the meter provider, so shutdown is what puts + # the run's datapoints on the wire — the query happens after the children + # have exited with code 0. + # + # Out of scope: OTLP metric wire shape (there is no metrics equivalent of + # the `otlp_spans.rs` snapshots), exporter retry behaviour, and the + # `otlp.metrics` disabled/`OTEL_METRICS_EXPORTER=none` paths. Also out of + # scope: metrics from the CLI, which runs without a metrics layer; the + # HTTP stores' write surface, which no service calls; and the balancer's + # background refresh loop, which runs outside any service dispatch and so + # reaches no subscriber. + + Background: + Given a valid configuration file exists + And the server is not already running + And the OTel collector and InfluxDB containers are running + And the child servers are stamped with a unique test_identity resource attribute + + # ----------------------------------------------------------------- + # rpc.client.call.duration — one scenario per Lightning backend, + # because `rpc.method` is the wire path of that node's own service. + # ----------------------------------------------------------------- + + @otlp-metrics-grpc-invoice-request-cln + Scenario: CLN gRPC invoice request timing reaches InfluxDB + Given the single payee has a CLN lightning node available + When I start the LNURL server with the configuration + Then the server should start successfully + And the lnurl service should be listening on the configured port + And the discovery service should be listening on the configured port + And the offers service should be listening on the configured port + When the single payee creates an offer for their lightning node + And the single payee registers their lightning node as a backend + And the payer requests the LNURL offer from the single payee + And the payer requests an invoice for 100 sats using the single payee's callback URL + And I send a SIGTERM signal to the server process + Then the server should exit with code 0 + And InfluxDB should have a "rpc.client.call.duration" row for this test_identity + And the row's "service.name" should be "swgr.lnurl" + And the row's "rpc.method" should be "cln.Node/Invoice" + And the row should have no "error.type" + And the row should be a well-formed histogram + And no ECS log record should carry the metrics target or a metric field + + @otlp-metrics-grpc-invoice-request-lnd + Scenario: LND gRPC invoice request timing reaches InfluxDB + Given the single payee has an LND lightning node available + When I start the LNURL server with the configuration + Then the server should start successfully + And the lnurl service should be listening on the configured port + And the discovery service should be listening on the configured port + And the offers service should be listening on the configured port + When the single payee creates an offer for their lightning node + And the single payee registers their lightning node as a backend + And the payer requests the LNURL offer from the single payee + And the payer requests an invoice for 100 sats using the single payee's callback URL + And I send a SIGTERM signal to the server process + Then the server should exit with code 0 + And InfluxDB should have a "rpc.client.call.duration" row for this test_identity + And the row's "service.name" should be "swgr.lnurl" + And the row's "rpc.method" should be "lnrpc.Lightning/AddInvoice" + And the row should have no "error.type" + And the row should be a well-formed histogram + And no ECS log record should carry the metrics target or a metric field + + # ----------------------------------------------------------------- + # http.client.request.duration — only the LNURL service has HTTP + # stores, and only its read path calls them. + # ----------------------------------------------------------------- + + @otlp-metrics-http-store-reads + Scenario: HTTP store reads reach InfluxDB + Given the single payee has a CLN lightning node available + And server 1 serves the offers and discovery services over memory stores + And server 2 runs only the lnurl service against server 1 over HTTP + When I start server 1 with offers and discovery services + And I start server 2 with only lnurl service + Then server 1 should have offers and discovery services listening + And server 2 should have only lnurl service listening + When the single payee creates an offer for their lightning node on server 1 + And the single payee registers their lightning node as a backend on server 1 + And the payer requests the LNURL offer from the single payee on server 2 + And the payer requests an invoice for 100 sats using the single payee's callback URL + And I send a SIGTERM signal to the server process + Then all servers should exit with code 0 + And InfluxDB should have a "http.client.request.duration" row for this test_identity with method "GET" and url.template "/offers/{partition}/{id}" under service.name "swgr.lnurl" + And InfluxDB should have a "http.client.request.duration" row for this test_identity with method "GET" and url.template "/discovery" under service.name "swgr.lnurl" + And both rows should carry no "error.type" + And each row should be a well-formed histogram + And no ECS log record should carry the metrics target or a metric field + + @otlp-metrics-dead-offer-store + Scenario: A dead offer store records a connect error + Given the lnurl server's offer and discovery store URLs point at a dead port + When I start server 2 with only lnurl service + Then the server should start successfully + And the lnurl service should be listening on the configured port + When I request an offer expecting failure + And I send a SIGTERM signal to the server process + Then the server should exit with code 0 + And InfluxDB should have a "http.client.request.duration" row for this test_identity with method "GET", url.template "/offers/{partition}/{id}" and "error.type" "connect" under service.name "swgr.lnurl" + And the row should carry no "http.response.status_code" + And the row should be a well-formed histogram + And no ECS log record should carry the metrics target or a metric field + # The discovery store's connect failure is not reachable from this + # topology: every request that would refresh the balancer's backends + # goes through the invoice handler, which fetches the offer first and so + # never gets past the dead offer store. The next scenario covers it. + + @otlp-metrics-dead-discovery-store + Scenario: A dead discovery store records a connect error + Given the single payee has a CLN lightning node available + And server 1 serves the offers and discovery services over memory stores + And server 2 runs only the lnurl service, with its offer store on server 1 and its discovery store URL on a dead port + When I start server 1 with offers and discovery services + And I start server 2 with only lnurl service + Then server 1 should have offers and discovery services listening + And server 2 should have only lnurl service listening + When the single payee creates an offer for their lightning node on server 1 + And the payer requests the LNURL offer from the single payee on server 2 + And the payer requests an invoice for 100 sats using the single payee's callback URL, expecting failure + And I send a SIGTERM signal to the server process + Then all servers should exit with code 0 + And InfluxDB should have a "http.client.request.duration" row for this test_identity with method "GET", url.template "/discovery" and "error.type" "connect" under service.name "swgr.lnurl" + And the row should be a well-formed histogram + And no ECS log record should carry the metrics target or a metric field + + # ----------------------------------------------------------------- + # The LNURL read path over database stores. This configuration runs + # all three services in one process against one pair of SQLite files, + # which is the only place `service.name` attribution is under real + # pressure: the setup steps and the LNURL request path share a store, + # and each row must still be attributed to the dispatch that made the + # call. One write per other service is enough to show that — the + # per-service call-site inventories live in their own features. + # ----------------------------------------------------------------- + + @otlp-metrics-db-store-reads + Scenario: Database store reads reach InfluxDB + Given the server is configured with SQLite persistence + And the single payee has a CLN lightning node available + When I start the LNURL server with the configuration + Then the server should start successfully + And all services should be listening on their configured ports + When the single payee creates an offer for their lightning node + And the single payee registers their lightning node as a backend + And the payer requests the LNURL offer from the single payee + And the payer requests an invoice for 100 sats using the single payee's callback URL + And I send a SIGTERM signal to the server process + Then the server should exit with code 0 + And InfluxDB should have a "db.client.operation.duration" row for this test_identity for each of "get_offer", "get_all_etag" and "get_all_backends" under service.name "swgr.lnurl" + And InfluxDB should have a "db.client.operation.duration" row for "post_offer" under service.name "swgr.offer" + And InfluxDB should have a "db.client.operation.duration" row for "post" under service.name "swgr.discovery" + And each row's "db.system.name" should be "sqlite" + And each row should be a well-formed histogram + And no ECS log record should carry the metrics target or a metric field diff --git a/server/tests/features/otlp-offer-store-metrics.feature b/server/tests/features/otlp-offer-store-metrics.feature new file mode 100644 index 0000000..d0ad6de --- /dev/null +++ b/server/tests/features/otlp-offer-store-metrics.feature @@ -0,0 +1,152 @@ +@integration @otlp-metrics @otlp-compliance @offer-store +Feature: OTLP metrics for the offer database store + As a platform maintainer + I want every database call the offer service makes to be measured + So that a slow query or a rising constraint-failure rate is visible + on a dashboard, and so that an uninstrumented call site fails a test + rather than leaving a silent gap + + # Scope: this feature covers `db.client.operation.duration` for the + # twelve call sites in `components/src/offer/db.rs`. The pipeline is + # the one `otlp-metrics.feature` describes end to end — a `tracing` + # event on the `swgr::metrics` target, the `MetricsLayer` bridge in + # the offer service's dispatch, the `SdkMeterProvider`, OTLP gRPC to + # the containerised collector, and the collector's InfluxDB exporter. + # What differs is the driver: the `swgr` CLI against a server whose + # stores are SQLite, so one scenario walks every call site including + # the two the foreign keys reject. + # + # Attributes on each row: + # + # | attribute | scope | value | + # | db.system.name | store | sqlite | + # | db.namespace | store | the SQLite file stem | + # | db.collection.name | site | the table the call is for | + # | swgr.operation | site | the call site | + # | error.type | failure | the class of failure | + # | db.response.status_code | failure | the driver's own code | + # + # `error.type` is a class, not a detail: the convention asks for a value + # that is predictable and low-cardinality. Constraint violations are + # classified by sea-orm's own `DbErr::sql_err`, which is portable across + # SQLite, MySQL and Postgres, giving `unique_constraint` and + # `foreign_key_constraint`; anything else falls back to + # `connection_acquire`, `connection`, `statement`, `conversion`, or the + # registry's `_OTHER`. The precise identifier goes in + # `db.response.status_code`, which stays backend-specific. + # + # `swgr.operation` is not from the semantic-convention registry — it + # names the call site inside this workspace, under the `swgr.` prefix + # so it can never collide with a registry attribute. `server.address` + # and `server.port` are absent throughout: SQLite has no server. + # Conditional attributes are absent rather than empty, so a success + # row carries no `error.type` column value at all. + # + # The recording rule: the metric records the round trip, not the + # store's reading of it. `get_offer` mapping an empty result to + # `Ok(None)`, `post_offer` mapping a unique violation to `Ok(None)`, + # and `delete_offer` mapping a miss to `Ok(false)` are store + # semantics the metric does not see — all three are successes here. + # `put_offer` and `put_metadata` each make two round trips, so each + # contributes two `swgr.operation` values. + # + # Isolation and flush work as in `otlp-metrics.feature`: a random + # `test_identity` resource attribute scopes the run, and SIGTERM + # drains the meter provider, so the query happens after the child has + # exited with code 0. `count` is asserted as at least 1, never + # exactly 1. + # + # Out of scope: the MySQL and PostgreSQL variants of this scenario, + # which are a one-line change via `ctx.set_offer_store_database_uri`; + # and the offer store's HTTP implementation, whose write surface no + # service calls. + + Background: + Given a valid configuration file exists + And the server is not already running + And the OTel collector and InfluxDB containers are running + And the swgr CLI is available + And the server is configured with SQLite persistence + And the child server is stamped with a unique test_identity resource attribute + + # ----------------------------------------------------------------- + # One scenario for all twelve call sites: they share a server, a + # store and a test_identity, and the two failures need the rows the + # successes create. + # ----------------------------------------------------------------- + + @otlp-metrics-offer-store-operations + Scenario: Every offer store operation reaches InfluxDB + When I start the LNURL server with the configuration + Then the server should start successfully + And all services should be listening on their configured ports + Given a valid offer JSON exists + When I run "swgr offer post" with offer JSON + Then the command should succeed + When I run "swgr offer get" for the offer ID + Then the command should succeed + When I run "swgr offer get" with no parameters + Then the command should succeed + When I run "swgr offer metadata get" for the metadata ID + Then the command should succeed + When I run "swgr offer metadata get" with no parameters + Then the command should succeed + Given updated offer JSON exists + When I run "swgr offer put" for the offer ID + Then the command should succeed + Given updated offer metadata JSON exists + When I run "swgr offer metadata put" for the metadata ID + Then the command should succeed + When I run "swgr offer get" for a non-existent offer ID + Then the command should fail + # The store still succeeded: the empty result becomes `Ok(None)` and + # only the CLI treats it as an error. + Given an offer JSON with a non-existent metadata ID exists + When I run "swgr offer post" with offer JSON + Then the command should fail + And a user error message should be shown + Given a valid offer JSON exists + When I run "swgr offer post" with offer JSON + Then the command should succeed + When I run "swgr offer metadata delete" for metadata an offer references + Then the command should fail + And a user error message should be shown + When I run "swgr offer delete" for the offer ID + Then the command should succeed + When I run "swgr offer metadata delete" for the metadata ID + Then the command should succeed + When I send a SIGTERM signal to the server process + # SIGTERM drains the meter provider: the PeriodicReader's own + # interval is 60s, so shutdown is what puts the run's datapoints on + # the wire. + Then the server should exit with code 0 + And InfluxDB should have a "db.client.operation.duration" row for this test_identity under service.name "swgr.offer" for each operation: + | swgr.operation | db.collection.name | + | get_offer | offer_record | + | get_offers | offer_record | + | post_offer | offer_record | + | put_offer_upsert | offer_record | + | put_offer_fetch | offer_record | + | delete_offer | offer_record | + | get_metadata | offer_metadata | + | get_all_metadata | offer_metadata | + | post_metadata | offer_metadata | + | put_metadata_upsert | offer_metadata | + | put_metadata_fetch | offer_metadata | + | delete_metadata | offer_metadata | + And each row's "db.system.name" should be "sqlite" + And each row should carry no "error.type" + And each row should be a well-formed histogram + And InfluxDB should have a failed "db.client.operation.duration" row for each rejection: + | swgr.operation | error.type | db.response.status_code | + | post_offer | foreign_key_constraint | 787 (CONSTRAINT_FOREIGNKEY) | + | delete_metadata | statement | 1811 (CONSTRAINT_TRIGGER) | + # One error.type for both: it is the class of failure, which the + # convention asks to keep predictable and low-cardinality. The precise + # identifier is db.response.status_code, the domain-specific attribute + # the convention pairs with error.type. + And no ECS log record should carry the metrics target or a metric field + + # `get_offer` appears twice in this scenario — once for the offer + # that exists and once for the one that does not — which is why the + # histogram assertion is `count >= 1` rather than `count == 1`. diff --git a/server/tests/features/otlp_discovery_store_metrics.rs b/server/tests/features/otlp_discovery_store_metrics.rs new file mode 100644 index 0000000..97ebf48 --- /dev/null +++ b/server/tests/features/otlp_discovery_store_metrics.rs @@ -0,0 +1,199 @@ +//! `db.client.operation.duration` for the discovery service's database +//! store. +//! +//! The mirror of `otlp_offer_store_metrics.rs`: the `swgr` CLI drives a +//! server whose stores are SQLite, and one scenario walks every call site in +//! `components/src/discovery/db.rs`. +//! +//! No `error.type` row here. The discovery schema has no foreign keys, so no +//! call the API surface can make will fail the transaction; the `error.type` +//! mapping itself is unit-tested in `components/src/metrics.rs`. + +use crate::FEATURE_TEST_CONFIG_PATH; +use crate::common::context::Protocol; +use crate::common::context::cli::CliContext; +use crate::common::context::global::GlobalContext; +use crate::common::context::server::CertificateLocation; +use crate::common::step_functions::*; +use std::path::PathBuf; + +/// Every `swgr.operation` in `discovery/db.rs`, with the table each names. +/// The four writes are transactions spanning both tables; each names the one +/// the call is about, and the etag bump is incidental. +const OPERATIONS: &[(&str, &str)] = &[ + ("get", "discovery_backend"), + ("get_all_etag", "discovery_backend_etag"), + ("get_all_backends", "discovery_backend"), + ("post", "discovery_backend"), + ("put", "discovery_backend"), + ("patch", "discovery_backend"), + ("delete", "discovery_backend"), +]; + +#[tokio::test] +async fn test_discovery_store_operations_reach_influx() { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let feature_test_config_path = manifest_dir.join(FEATURE_TEST_CONFIG_PATH); + let mut ctx = GlobalContext::create(&feature_test_config_path).expect("assert"); + + let server1 = "server1"; + ctx.add_server( + server1, + manifest_dir.join("config/persistence.yaml"), + Protocol::Https, + Protocol::Https, + Protocol::Https, + ) + .expect("assert"); + ctx.activate_server(server1); + + let identity = new_test_identity(); + step_given_the_active_server_is_stamped_with_test_identity(&mut ctx, &identity) + .await + .expect("assert"); + + let mut cli_ctx = CliContext::create().expect("assert"); + step_given_the_swgr_cli_is_available(&mut cli_ctx) + .await + .expect("assert"); + + step_given_the_server_is_not_already_running(&mut ctx) + .await + .expect("assert"); + step_given_the_lnurl_server_is_ready_to_start(&mut ctx) + .await + .expect("assert"); + step_when_i_start_the_lnurl_server_with_the_configuration(&mut ctx) + .await + .expect("assert"); + step_then_the_server_should_start_successfully(&mut ctx) + .await + .expect("assert"); + step_and_all_services_should_be_listening_on_their_configured_ports(&mut ctx) + .await + .expect("assert"); + + step_given_a_valid_backend_json_exists(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + let public_key = extract_backend_public_key(&cli_ctx) + .await + .expect("assert") + .to_string(); + + step_when_i_run_swgr_discovery_post(&mut ctx, &mut cli_ctx, CertificateLocation::Arg) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_discovery_get( + &mut ctx, + &mut cli_ctx, + &public_key, + CertificateLocation::Arg, + ) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_discovery_ls(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_discovery_get_all(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_and_updated_backend_json_exists(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + step_when_i_run_swgr_discovery_put(&mut ctx, &mut cli_ctx, &public_key) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_and_backend_patch_json_exists(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + step_when_i_run_swgr_discovery_patch(&mut ctx, &mut cli_ctx, &public_key) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_discovery_disable(&mut ctx, &mut cli_ctx, &public_key) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_discovery_enable(&mut ctx, &mut cli_ctx, &public_key) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + // A miss is still a successful round trip: `get` maps the empty result to + // `Ok(None)` and the metric records no `error.type`. + step_when_i_run_swgr_discovery_get_for_non_existent_backend(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + step_then_the_command_should_fail(&mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_discovery_delete(&mut ctx, &mut cli_ctx, &public_key) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + // SIGTERM drains the meter provider: the `PeriodicReader`'s own interval + // is 60s, so shutdown is what puts this run's datapoints on the wire. + step_when_i_send_a_sigterm_signal_to_the_server_process(&mut ctx) + .await + .expect("assert"); + step_then_the_server_should_exit_with_code_0(&mut ctx) + .await + .expect("assert"); + + let influx = influx_client().expect("assert"); + + for (operation, collection) in OPERATIONS { + let row = step_then_influx_should_have_a_histogram_row( + &influx, + &identity, + DB_OPERATION_DURATION, + DISCOVERY_SERVICE, + &[ + ("swgr.operation", operation), + ("db.collection.name", collection), + ("db.system.name", "sqlite"), + ], + MetricOutcome::Success, + ) + .await + .expect("assert"); + step_then_the_histogram_row_should_be_well_formed(&row).expect("assert"); + } + + step_then_no_ecs_log_record_should_carry_a_metric_field(&mut ctx).expect("assert"); + + ctx.stop_all_servers().expect("assert"); +} diff --git a/server/tests/features/otlp_metrics.rs b/server/tests/features/otlp_metrics.rs new file mode 100644 index 0000000..5f36b40 --- /dev/null +++ b/server/tests/features/otlp_metrics.rs @@ -0,0 +1,560 @@ +//! OTLP metrics end-to-end test. +//! +//! Unlike `otlp_spans.rs`, which snapshots the wire shape against an +//! in-process collector, this drives the real pipeline: the child server +//! exports OTLP metrics to the containerised OTel collector, which writes +//! them to InfluxDB under the `telegraf-prometheus-v1` schema (one +//! measurement per metric name; resource, scope and datapoint attributes +//! become tags). The assertion is a SQL query against InfluxDB. +//! +//! Runs are isolated by a random `test_identity` resource attribute set on +//! the children through `OTEL_RESOURCE_ATTRIBUTES`, so concurrent and +//! previous runs sharing the container never collide. +//! +//! The scenarios here cover the LNURL service's read path, one topology per +//! store backing: gRPC to a Lightning node, HTTP stores on both the success +//! and the connect-failure path, and database stores. Each assertion is made +//! in exactly one place — `rpc.client.call.duration` belongs to the two gRPC +//! scenarios, which exist to cover both node types' call sites, so the +//! HTTP- and database-store scenarios do not re-assert it. The database +//! stores' own call sites live in `otlp_offer_store_metrics.rs` and +//! `otlp_discovery_store_metrics.rs`. + +use crate::FEATURE_TEST_CONFIG_PATH; +use crate::common::context::Protocol; +use crate::common::context::global::GlobalContext; +use crate::common::context::server::CertificateLocation; +use crate::common::step_functions::*; +use std::path::PathBuf; + +/// A port with nothing listening, so a request to it fails at connect. +const DEAD_STORE_URL: &str = "https://127.0.0.1:1"; + +/// How server 2's discovery store is wired in the two-server topology. +#[derive(Clone, Copy)] +enum DiscoveryStore { + /// Reachable, on server 1. + OnServer1, + /// Unreachable, so every backend refresh fails at connect. + DeadPort, +} + +/// The two-server topology from `http_remote_stores.rs`: server 1 serves the +/// offers and discovery services over memory stores, and server 2 runs only +/// the LNURL service, reaching them over HTTP. Both children export, so both +/// carry the scenario's `identity`. +/// +/// Returns with both started, their services listening, a CLN payee +/// available, and server 1 active. +async fn start_remote_store_topology( + identity: &str, + discovery_store: DiscoveryStore, +) -> GlobalContext { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let feature_test_config_path = manifest_dir.join(FEATURE_TEST_CONFIG_PATH); + let mut ctx = GlobalContext::create(&feature_test_config_path).expect("assert"); + + let server1 = "server1"; + ctx.add_server( + server1, + manifest_dir.join("config/memory-basic.yaml"), + Protocol::Https, + Protocol::Https, + Protocol::Https, + ) + .expect("assert"); + + let server2 = "server2"; + ctx.add_server( + server2, + manifest_dir.join("config/lnurl-standalone.yaml"), + Protocol::Https, + Protocol::Https, + Protocol::Https, + ) + .expect("assert"); + + ctx.set_offer_store_url(server1, server2).expect("assert"); + ctx.set_offer_store_authorization(server1, server2) + .expect("assert"); + ctx.set_discovery_store_authorization(server1, server2) + .expect("assert"); + ctx.set_certificate_location(server2, CertificateLocation::Env) + .expect("assert"); + + for server in [server1, server2] { + ctx.activate_server(server); + step_given_the_active_server_is_stamped_with_test_identity(&mut ctx, identity) + .await + .expect("assert"); + } + + match discovery_store { + DiscoveryStore::OnServer1 => { + ctx.set_discovery_store_url(server1, server2) + .expect("assert"); + } + DiscoveryStore::DeadPort => { + ctx.activate_server(server2); + ctx.set_active_discovery_store_url_raw(Some(DEAD_STORE_URL.to_string())) + .expect("assert"); + } + } + + ctx.activate_server(server1); + step_given_the_payee_has_a_lightning_node_available(&mut ctx, "cln") + .await + .expect("assert"); + step_given_the_server_is_not_already_running(&mut ctx) + .await + .expect("assert"); + step_given_the_lnurl_server_is_ready_to_start(&mut ctx) + .await + .expect("assert"); + step_when_i_start_server_1_with_offers_and_discovery_services(&mut ctx) + .await + .expect("assert"); + step_then_server_1_should_have_offers_and_discovery_services_listening(&mut ctx) + .await + .expect("assert"); + + ctx.activate_server(server2); + step_given_the_lnurl_server_is_ready_to_start(&mut ctx) + .await + .expect("assert"); + step_when_i_start_server_2_with_only_lnurl_service(&mut ctx) + .await + .expect("assert"); + step_then_server_2_should_have_only_lnurl_service_listening(&mut ctx) + .await + .expect("assert"); + + ctx.activate_server(server1); + ctx +} + +/// The wire path of the invoice RPC each node type serves. +fn invoice_rpc_method(node_type: &str) -> &'static str { + match node_type { + "cln" => "cln.Node/Invoice", + _ => "lnrpc.Lightning/AddInvoice", + } +} + +#[tokio::test] +async fn test_grpc_invoice_request_reaches_influx_cln() { + test_grpc_invoice_request_reaches_influx_inner("cln").await; +} + +#[tokio::test] +async fn test_grpc_invoice_request_reaches_influx_lnd() { + test_grpc_invoice_request_reaches_influx_inner("lnd").await; +} + +/// The sole home of the `rpc.client.call.duration` assertion: one run per +/// node type, because `rpc.method` is the wire path of that node's own +/// service and the two call sites are separate code. +async fn test_grpc_invoice_request_reaches_influx_inner(node_type: &str) { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let feature_test_config_path = manifest_dir.join(FEATURE_TEST_CONFIG_PATH); + let mut ctx = GlobalContext::create(&feature_test_config_path).expect("assert"); + + let server1 = "server1"; + ctx.add_server( + server1, + manifest_dir.join("config/memory-basic.yaml"), + Protocol::Https, + Protocol::Https, + Protocol::Https, + ) + .expect("assert"); + ctx.activate_server(server1); + + let identity = new_test_identity(); + step_given_the_active_server_is_stamped_with_test_identity(&mut ctx, &identity) + .await + .expect("assert"); + + step_given_the_server_is_not_already_running(&mut ctx) + .await + .expect("assert"); + step_given_the_lnurl_server_is_ready_to_start(&mut ctx) + .await + .expect("assert"); + step_given_the_payee_has_a_lightning_node_available(&mut ctx, node_type) + .await + .expect("assert"); + step_when_i_start_the_lnurl_server_with_the_configuration(&mut ctx) + .await + .expect("assert"); + step_then_the_server_should_start_successfully(&mut ctx) + .await + .expect("assert"); + step_and_all_services_should_be_listening_on_their_configured_ports(&mut ctx) + .await + .expect("assert"); + + step_when_the_payee_creates_an_offer_for_their_lightning_node(&mut ctx, "single") + .await + .expect("assert"); + step_when_the_payee_registers_their_lightning_node_as_a_backend(&mut ctx, "single", true) + .await + .expect("assert"); + step_when_the_payer_requests_the_lnurl_offer_from_the_payee(&mut ctx, "single") + .await + .expect("assert"); + step_when_the_payer_requests_an_invoice_for_100_sats_using_the_payee_callback_url( + &mut ctx, + "single", + &Protocol::Https, + ) + .await + .expect("assert"); + + // SIGTERM drains the meter provider: the `PeriodicReader`'s own interval is + // 60s, so shutdown is what puts this run's datapoint on the wire. + step_when_i_send_a_sigterm_signal_to_the_server_process(&mut ctx) + .await + .expect("assert"); + step_then_the_server_should_exit_with_code_0(&mut ctx) + .await + .expect("assert"); + + let influx = influx_client().expect("assert"); + let row = step_then_influx_should_have_a_histogram_row( + &influx, + &identity, + RPC_CALL_DURATION, + LNURL_SERVICE, + &[("rpc.method", invoice_rpc_method(node_type))], + MetricOutcome::Success, + ) + .await + .expect("assert"); + step_then_the_histogram_row_should_be_well_formed(&row).expect("assert"); + + step_then_no_ecs_log_record_should_carry_a_metric_field(&mut ctx).expect("assert"); + + ctx.stop_all_servers().expect("assert"); +} + +/// The LNURL service reading through HTTP stores: both reads its request +/// path makes, on the success path. +/// +/// The invoice itself is asserted by the step that requests it, so the gRPC +/// metric is left to the two scenarios above. +#[tokio::test] +async fn test_http_store_reads_reach_influx() { + let identity = new_test_identity(); + let mut ctx = start_remote_store_topology(&identity, DiscoveryStore::OnServer1).await; + + step_when_the_payee_creates_an_offer_for_their_lightning_node(&mut ctx, "single") + .await + .expect("assert"); + step_when_the_payee_registers_their_lightning_node_as_a_backend(&mut ctx, "single", true) + .await + .expect("assert"); + + ctx.activate_server("server2"); + step_when_the_payer_requests_the_lnurl_offer_from_the_payee(&mut ctx, "single") + .await + .expect("assert"); + step_when_the_payer_requests_an_invoice_for_100_sats_using_the_payee_callback_url( + &mut ctx, + "single", + &Protocol::Https, + ) + .await + .expect("assert"); + step_then_the_payer_should_receive_a_valid_lightning_invoice(&mut ctx, "single") + .await + .expect("assert"); + + step_when_i_send_a_sigterm_signal_to_the_server_process(&mut ctx) + .await + .expect("assert"); + step_then_all_servers_should_exit_with_code_0(&mut ctx) + .await + .expect("assert"); + + let influx = influx_client().expect("assert"); + // `/discovery` comes from the balancer's in-request retry, which + // refreshes its backends before each attempt. + for template in ["/offers/{partition}/{id}", "/discovery"] { + let row = step_then_influx_should_have_a_histogram_row( + &influx, + &identity, + HTTP_REQUEST_DURATION, + LNURL_SERVICE, + &[("http.request.method", "GET"), ("url.template", template)], + MetricOutcome::Success, + ) + .await + .expect("assert"); + step_then_the_histogram_row_should_be_well_formed(&row).expect("assert"); + } + + step_then_no_ecs_log_record_should_carry_a_metric_field(&mut ctx).expect("assert"); + + ctx.stop_all_servers().expect("assert"); +} + +/// The LNURL service's offer store pointed at a dead port. +/// +/// Topology from `otlp_spans.rs::test_dead_offer_store_502_otlp_shape`. The +/// offer read fails at reqwest's connect, so the call site records +/// `error.type = connect` and no `http.response.status_code` at all. +/// +/// The discovery store's connect failure is not reachable from this +/// topology: every request that would refresh the balancer's backends goes +/// through the invoice handler, which fetches the offer first and so never +/// gets past the dead offer store. It is covered by +/// `test_dead_discovery_store_records_connect_errors` below. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_dead_offer_store_records_connect_errors() { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let feature_test_config_path = manifest_dir.join(FEATURE_TEST_CONFIG_PATH); + let mut ctx = GlobalContext::create(&feature_test_config_path).expect("assert"); + + let server1 = "server1"; + ctx.add_server( + server1, + manifest_dir.join("config/lnurl-standalone.yaml"), + Protocol::Https, + Protocol::Https, + Protocol::Https, + ) + .expect("assert"); + ctx.activate_server(server1); + + // The auth pointers and certificate location still have to resolve for + // the config to load. + ctx.set_active_offer_store_url_raw(Some(DEAD_STORE_URL.to_string())) + .expect("assert"); + ctx.set_active_discovery_store_url_raw(Some(DEAD_STORE_URL.to_string())) + .expect("assert"); + ctx.set_offer_store_authorization(server1, server1) + .expect("assert"); + ctx.set_discovery_store_authorization(server1, server1) + .expect("assert"); + ctx.set_certificate_location(server1, CertificateLocation::Env) + .expect("assert"); + + let identity = new_test_identity(); + step_given_the_active_server_is_stamped_with_test_identity(&mut ctx, &identity) + .await + .expect("assert"); + + step_given_the_server_is_not_already_running(&mut ctx) + .await + .expect("assert"); + step_given_the_lnurl_server_is_ready_to_start(&mut ctx) + .await + .expect("assert"); + step_when_i_start_server_2_with_only_lnurl_service(&mut ctx) + .await + .expect("assert"); + step_then_the_server_should_start_successfully(&mut ctx) + .await + .expect("assert"); + step_and_the_lnurl_service_should_be_listening_on_the_configured_port(&mut ctx) + .await + .expect("assert"); + + // Any partition/id — the store's request never reaches a real server. + step_when_i_request_offer_expecting_failure(&mut ctx, "default") + .await + .expect("assert"); + + step_when_i_send_a_sigterm_signal_to_the_server_process(&mut ctx) + .await + .expect("assert"); + step_then_the_server_should_exit_with_code_0(&mut ctx) + .await + .expect("assert"); + + let influx = influx_client().expect("assert"); + let row = step_then_influx_should_have_a_histogram_row( + &influx, + &identity, + HTTP_REQUEST_DURATION, + LNURL_SERVICE, + &[ + ("http.request.method", "GET"), + ("url.template", "/offers/{partition}/{id}"), + ], + MetricOutcome::Error("connect"), + ) + .await + .expect("assert"); + step_then_the_histogram_row_should_be_well_formed(&row).expect("assert"); + + step_then_no_ecs_log_record_should_carry_a_metric_field(&mut ctx).expect("assert"); + + ctx.stop_all_servers().expect("assert"); +} + +/// A live offer store and a dead discovery store. +/// +/// The LNURL offer request succeeds against server 1, so the invoice request +/// gets past the offer fetch and reaches the balancer. With no backends +/// readable it retries, and each retry refreshes the discovery store +/// in-request — which is where `GET /discovery` records its connect failure. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_dead_discovery_store_records_connect_errors() { + let identity = new_test_identity(); + let mut ctx = start_remote_store_topology(&identity, DiscoveryStore::DeadPort).await; + + step_when_the_payee_creates_an_offer_for_their_lightning_node(&mut ctx, "single") + .await + .expect("assert"); + + ctx.activate_server("server2"); + step_when_the_payer_requests_the_lnurl_offer_from_the_payee(&mut ctx, "single") + .await + .expect("assert"); + step_but_when_the_payer_requests_an_invoice_for_100_sats_using_the_payee_callback_url_expecting_failure( + &mut ctx, + "single", + &Protocol::Https, + ) + .await + .expect("assert"); + + step_when_i_send_a_sigterm_signal_to_the_server_process(&mut ctx) + .await + .expect("assert"); + step_then_all_servers_should_exit_with_code_0(&mut ctx) + .await + .expect("assert"); + + let influx = influx_client().expect("assert"); + let row = step_then_influx_should_have_a_histogram_row( + &influx, + &identity, + HTTP_REQUEST_DURATION, + LNURL_SERVICE, + &[ + ("http.request.method", "GET"), + ("url.template", "/discovery"), + ], + MetricOutcome::Error("connect"), + ) + .await + .expect("assert"); + step_then_the_histogram_row_should_be_well_formed(&row).expect("assert"); + + step_then_no_ecs_log_record_should_carry_a_metric_field(&mut ctx).expect("assert"); + + ctx.stop_all_servers().expect("assert"); +} + +/// The same LNURL flow over database stores. +/// +/// `config/persistence.yaml` runs all three services in one process against +/// one pair of SQLite files, which is the only place `service.name` +/// attribution is put under real pressure: the setup steps and the LNURL +/// request path share a store, and each row must still be attributed to the +/// dispatch that made the call. The per-service call-site inventories belong +/// to `otlp_offer_store_metrics.rs` and `otlp_discovery_store_metrics.rs`, +/// so only one write per service is checked here. +#[tokio::test] +async fn test_db_store_reads_reach_influx() { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let feature_test_config_path = manifest_dir.join(FEATURE_TEST_CONFIG_PATH); + let mut ctx = GlobalContext::create(&feature_test_config_path).expect("assert"); + + let server1 = "server1"; + ctx.add_server( + server1, + manifest_dir.join("config/persistence.yaml"), + Protocol::Https, + Protocol::Https, + Protocol::Https, + ) + .expect("assert"); + ctx.activate_server(server1); + + let identity = new_test_identity(); + step_given_the_active_server_is_stamped_with_test_identity(&mut ctx, &identity) + .await + .expect("assert"); + + step_given_the_server_is_not_already_running(&mut ctx) + .await + .expect("assert"); + step_given_the_lnurl_server_is_ready_to_start(&mut ctx) + .await + .expect("assert"); + step_given_the_payee_has_a_lightning_node_available(&mut ctx, "cln") + .await + .expect("assert"); + step_when_i_start_the_lnurl_server_with_the_configuration(&mut ctx) + .await + .expect("assert"); + step_then_the_server_should_start_successfully(&mut ctx) + .await + .expect("assert"); + step_and_all_services_should_be_listening_on_their_configured_ports(&mut ctx) + .await + .expect("assert"); + + step_when_the_payee_creates_an_offer_for_their_lightning_node(&mut ctx, "single") + .await + .expect("assert"); + step_when_the_payee_registers_their_lightning_node_as_a_backend(&mut ctx, "single", true) + .await + .expect("assert"); + step_when_the_payer_requests_the_lnurl_offer_from_the_payee(&mut ctx, "single") + .await + .expect("assert"); + step_when_the_payer_requests_an_invoice_for_100_sats_using_the_payee_callback_url( + &mut ctx, + "single", + &Protocol::Https, + ) + .await + .expect("assert"); + + step_when_i_send_a_sigterm_signal_to_the_server_process(&mut ctx) + .await + .expect("assert"); + step_then_the_server_should_exit_with_code_0(&mut ctx) + .await + .expect("assert"); + + let influx = influx_client().expect("assert"); + + // The LNURL service's own reads. `get_all_*` come from the balancer's + // in-request retry, which refreshes its backends before each attempt. + for (service, operation, collection) in [ + (LNURL_SERVICE, "get_offer", "offer_record"), + (LNURL_SERVICE, "get_all_etag", "discovery_backend_etag"), + (LNURL_SERVICE, "get_all_backends", "discovery_backend"), + // One write per other service, on the same store and the same + // identity: the row is proof the dispatch, not the store, decides + // `service.name`. + (OFFER_SERVICE, "post_offer", "offer_record"), + (DISCOVERY_SERVICE, "post", "discovery_backend"), + ] { + let row = step_then_influx_should_have_a_histogram_row( + &influx, + &identity, + DB_OPERATION_DURATION, + service, + &[ + ("swgr.operation", operation), + ("db.collection.name", collection), + ("db.system.name", "sqlite"), + ], + MetricOutcome::Success, + ) + .await + .expect("assert"); + step_then_the_histogram_row_should_be_well_formed(&row).expect("assert"); + } + + step_then_no_ecs_log_record_should_carry_a_metric_field(&mut ctx).expect("assert"); + + ctx.stop_all_servers().expect("assert"); +} diff --git a/server/tests/features/otlp_offer_store_metrics.rs b/server/tests/features/otlp_offer_store_metrics.rs new file mode 100644 index 0000000..652ebff --- /dev/null +++ b/server/tests/features/otlp_offer_store_metrics.rs @@ -0,0 +1,235 @@ +//! `db.client.operation.duration` for the offer service's database store. +//! +//! Same pipeline as `otlp_metrics.rs`: the child exports OTLP metrics to the +//! containerised collector, which writes them to InfluxDB, and the assertion +//! is a SQL query. What differs is the driver — the `swgr` CLI against a +//! server whose stores are SQLite, so one scenario walks every call site in +//! `components/src/offer/db.rs`, including the two the foreign key rejects. + +use crate::FEATURE_TEST_CONFIG_PATH; +use crate::common::context::Protocol; +use crate::common::context::cli::CliContext; +use crate::common::context::global::GlobalContext; +use crate::common::context::server::CertificateLocation; +use crate::common::step_functions::*; +use std::path::PathBuf; + +/// Every `swgr.operation` in `offer/db.rs`, with the table each names. +const OPERATIONS: &[(&str, &str)] = &[ + ("get_offer", "offer_record"), + ("get_offers", "offer_record"), + ("post_offer", "offer_record"), + ("put_offer_upsert", "offer_record"), + ("put_offer_fetch", "offer_record"), + ("delete_offer", "offer_record"), + ("get_metadata", "offer_metadata"), + ("get_all_metadata", "offer_metadata"), + ("post_metadata", "offer_metadata"), + ("put_metadata_upsert", "offer_metadata"), + ("put_metadata_fetch", "offer_metadata"), + ("delete_metadata", "offer_metadata"), +]; + +#[tokio::test] +async fn test_offer_store_operations_reach_influx() { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let feature_test_config_path = manifest_dir.join(FEATURE_TEST_CONFIG_PATH); + let mut ctx = GlobalContext::create(&feature_test_config_path).expect("assert"); + + let server1 = "server1"; + ctx.add_server( + server1, + manifest_dir.join("config/persistence.yaml"), + Protocol::Https, + Protocol::Https, + Protocol::Https, + ) + .expect("assert"); + ctx.activate_server(server1); + + let identity = new_test_identity(); + step_given_the_active_server_is_stamped_with_test_identity(&mut ctx, &identity) + .await + .expect("assert"); + + let mut cli_ctx = CliContext::create().expect("assert"); + step_given_the_swgr_cli_is_available(&mut cli_ctx) + .await + .expect("assert"); + + step_given_the_server_is_not_already_running(&mut ctx) + .await + .expect("assert"); + step_given_the_lnurl_server_is_ready_to_start(&mut ctx) + .await + .expect("assert"); + step_when_i_start_the_lnurl_server_with_the_configuration(&mut ctx) + .await + .expect("assert"); + step_then_the_server_should_start_successfully(&mut ctx) + .await + .expect("assert"); + step_and_all_services_should_be_listening_on_their_configured_ports(&mut ctx) + .await + .expect("assert"); + + // Posts the first metadata and leaves an offer JSON referencing it. + step_given_a_valid_offer_json_exists(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + let offer_id = extract_offer_id(&cli_ctx).await.expect("assert"); + let metadata_id = extract_metadata_id_from_offer(&cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_offer_post(&mut ctx, &mut cli_ctx, CertificateLocation::Arg) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_offer_get(&mut ctx, &mut cli_ctx, &offer_id, CertificateLocation::Arg) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_offer_get_all(&mut ctx, &mut cli_ctx, None, None) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_offer_metadata_get( + &mut ctx, + &mut cli_ctx, + &metadata_id, + CertificateLocation::Arg, + ) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_offer_metadata_get_all(&mut ctx, &mut cli_ctx, None, None) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_and_updated_offer_json_exists(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + step_when_i_run_swgr_offer_put(&mut ctx, &mut cli_ctx, &offer_id) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_and_updated_offer_metadata_json_exists(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + step_when_i_run_swgr_offer_metadata_put(&mut ctx, &mut cli_ctx, &metadata_id) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + // A miss is still a successful round trip: `get_offer` maps the empty + // result to `Ok(None)` and the metric records no `error.type`. + step_when_i_run_swgr_offer_get_for_non_existent_offer(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + step_then_the_command_should_fail(&mut cli_ctx) + .await + .expect("assert"); + + // The two foreign-key rejections, which are the only failures the offer + // store's API surface can provoke. The second posts the run's second + // metadata and a second offer along the way. + step_when_i_run_swgr_offer_post_with_unknown_metadata_expecting_failure(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + step_when_i_run_swgr_offer_metadata_delete_referenced_expecting_failure(&mut ctx, &mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_offer_delete(&mut ctx, &mut cli_ctx, &offer_id) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + step_when_i_run_swgr_offer_metadata_delete(&mut ctx, &mut cli_ctx, &metadata_id) + .await + .expect("assert"); + step_then_the_command_should_succeed(&mut cli_ctx) + .await + .expect("assert"); + + // SIGTERM drains the meter provider: the `PeriodicReader`'s own interval + // is 60s, so shutdown is what puts this run's datapoints on the wire. + step_when_i_send_a_sigterm_signal_to_the_server_process(&mut ctx) + .await + .expect("assert"); + step_then_the_server_should_exit_with_code_0(&mut ctx) + .await + .expect("assert"); + + let influx = influx_client().expect("assert"); + + for (operation, collection) in OPERATIONS { + let row = step_then_influx_should_have_a_histogram_row( + &influx, + &identity, + DB_OPERATION_DURATION, + OFFER_SERVICE, + &[ + ("swgr.operation", operation), + ("db.collection.name", collection), + ("db.system.name", "sqlite"), + ], + MetricOutcome::Success, + ) + .await + .expect("assert"); + step_then_the_histogram_row_should_be_well_formed(&row).expect("assert"); + } + + // Both foreign-key rejections carry the same `error.type`: it is the + // class of failure, kept predictable and low-cardinality. What separates + // them is `db.response.status_code` — SQLITE_CONSTRAINT_FOREIGNKEY on + // the insert, SQLITE_CONSTRAINT_TRIGGER on the delete — which is the + // domain-specific attribute the convention pairs with `error.type`. + for (operation, status_code, error_type) in [ + ("post_offer", "787", "foreign_key_constraint"), + ("delete_metadata", "1811", "statement"), + ] { + let row = step_then_influx_should_have_a_histogram_row( + &influx, + &identity, + DB_OPERATION_DURATION, + OFFER_SERVICE, + &[ + ("swgr.operation", operation), + ("db.response.status_code", status_code), + ], + MetricOutcome::Error(error_type), + ) + .await + .expect("assert"); + step_then_the_histogram_row_should_be_well_formed(&row).expect("assert"); + } + + step_then_no_ecs_log_record_should_carry_a_metric_field(&mut ctx).expect("assert"); + + ctx.stop_all_servers().expect("assert"); +} diff --git a/server/tests/features/otlp_spans.rs b/server/tests/features/otlp_spans.rs index 99950d3..3474787 100644 --- a/server/tests/features/otlp_spans.rs +++ b/server/tests/features/otlp_spans.rs @@ -51,7 +51,7 @@ async fn test_success_invoice_otlp_shape() { // ca_cert_path stays pointed at the existing PKI root: swgr's OTLP // client only consults it for https:// endpoints, and our collector is - // http:// (h2c). Path just needs to exist so `OTLP_TRACING_TRUSTED_ROOTS` is a + // http:// (h2c). Path just needs to exist so `OTLP_TRUSTED_ROOTS` is a // readable file when the child reads it. let ca_cert_path = ctx.get_pki_root_certificate_path().to_path_buf(); ctx.set_active_otel_collector(OtelCollector { @@ -156,7 +156,7 @@ async fn test_error_get_offer_404_otlp_shape() { // ca_cert_path stays pointed at the existing PKI root: swgr's OTLP // client only consults it for https:// endpoints, and our collector is - // http:// (h2c). Path just needs to exist so `OTLP_TRACING_TRUSTED_ROOTS` is a + // http:// (h2c). Path just needs to exist so `OTLP_TRUSTED_ROOTS` is a // readable file when the child reads it. let ca_cert_path = ctx.get_pki_root_certificate_path().to_path_buf(); ctx.set_active_otel_collector(OtelCollector { diff --git a/server/tests/features/service_logs.rs b/server/tests/features/service_logs.rs index 5c30989..b9aea30 100644 --- a/server/tests/features/service_logs.rs +++ b/server/tests/features/service_logs.rs @@ -388,8 +388,8 @@ async fn test_error_conditions_are_properly_logged() { /// Provokes PingoraLnError::no_available_nodes: invoice endpoint runs, offer /// is fetched, then the balancer has no backend to route to and terminates at -/// pingora/src/balance.rs:222. Captured span is PingoraLnBalancer::get_invoice; -/// status 502. +/// pingora/src/balance.rs:217-222. Captured span is +/// PingoraLnBalancer::get_invoice; status 502. #[tokio::test] async fn test_invoice_no_available_nodes_error_logged_and_traced() { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -457,9 +457,8 @@ async fn test_invoice_no_available_nodes_error_logged_and_traced() { /// Provokes LnUrlPayServiceError::bad_request: invoice handler validates /// `amount` against the offer's [min_sendable, max_sendable] bounds and returns -/// bad_request at service/src/lnurl/pay/handler.rs:97-100. Captured span is -/// the invoice -/// handler; status 400. +/// bad_request at service/src/lnurl/pay/handler.rs:95-99. Captured span is the +/// invoice handler; status 400. #[tokio::test] async fn test_invoice_out_of_range_amount_error_logged_and_traced() { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -528,7 +527,7 @@ async fn test_invoice_out_of_range_amount_error_logged_and_traced() { /// Provokes CrudError::conflict via DiscoveryCrudError: registering the same /// backend twice makes the second POST return None from the store, which the /// handler at service/src/discovery/handler.rs:101 converts to a `conflict` -/// capture at service/src/axum/crud/error.rs:111. Captured span is the +/// capture at service/src/axum/crud/error.rs:260. Captured span is the /// post_backend handler; status 409. #[tokio::test] async fn test_duplicate_backend_post_conflict_logged_and_traced() { @@ -600,11 +599,11 @@ async fn test_duplicate_backend_post_conflict_logged_and_traced() { ctx.stop_all_servers().expect("assert"); } -/// Provokes CrudError::unauthorized (report §1 line 138) via the newly -/// instrumented `BearerTokenAuthService` reject path. Raw HTTP POST to -/// /discovery without a bearer header short-circuits at the auth middleware; -/// the `auth` span opened by `unauthorized_in_span` becomes the captured -/// span, so the rejection is now visible in Jaeger. +/// Provokes CrudError::unauthorized (report §1 line 138) via the +/// `BearerTokenAuthService` reject path. Raw HTTP POST to /discovery without a +/// bearer header short-circuits at the auth middleware, before any instrumented +/// handler runs, so the rejection has no span of its own — this test asserts on +/// the ECS log line only (POST /discovery, 401, WARN on service=discovery). #[tokio::test] async fn test_unauthorized_backend_post_logged_and_traced() { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -723,10 +722,10 @@ async fn test_get_offers_over_limit_logged_and_traced_bad_request() { ctx.stop_all_servers().expect("assert"); } -/// Provokes LnUrlPayServiceError::not_found from PartitionsService's newly -/// instrumented reject path (report §2 line 76). The `partitions` span opened -/// by the middleware is the captured span, so the reject is now attributable -/// in Jaeger — previously it produced only a log event with no OTLP span. +/// Provokes LnUrlPayServiceError::not_found from the PartitionsService reject +/// path (report §2 line 76). The middleware rejects before any instrumented +/// handler runs, so the rejection has no span of its own — this test asserts on +/// the ECS log line only (GET /offers/non-existent-partition/..., 404, WARN). #[tokio::test] async fn test_non_existent_partition_reject_logged_and_traced() { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -793,7 +792,8 @@ async fn test_non_existent_partition_reject_logged_and_traced() { /// Server1 runs lnurl-standalone (HTTP offer + discovery stores). The offer /// store base URL is redirected to a dead port so the HttpOfferStore's request /// fails at connect. Captured span is the innermost HTTP-store span in the -/// offer chain (`get_offer`); the request logger tags it with swgr.lnurl. +/// offer chain — `HttpOfferStore::get_offer`, now a direct child of the +/// `OfferProvider::offer` span; the request logger tags it with swgr.lnurl. #[tokio::test] async fn test_lnurl_with_dead_offer_store_url_logged_and_traced() { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); diff --git a/server/tests/features/snapshots/features__otlp_spans__otlp_spans__dead_offer_store_502.snap b/server/tests/features/snapshots/features__otlp_spans__otlp_spans__dead_offer_store_502.snap index 4104e1c..d3742df 100644 --- a/server/tests/features/snapshots/features__otlp_spans__otlp_spans__dead_offer_store_502.snap +++ b/server/tests/features/snapshots/features__otlp_spans__otlp_spans__dead_offer_store_502.snap @@ -1,6 +1,6 @@ --- source: server/tests/features/otlp_spans.rs -assertion_line: 312 +assertion_line: 318 expression: lines --- [ @@ -32,37 +32,13 @@ expression: lines "attributes": { "target": "[string]" }, - "children": [ - { - "attributes": { - "target": "[string]" - }, - "dropped_attributes_count": "[int]", - "dropped_events_count": "[int]", - "dropped_links_count": "[int]", - "end_time_unix_nano": "[int]", - "flags": "[int]", - "kind": "[int]", - "name": "get_offer", - "parent_span_id": "[bytes]", - "span_id": "[bytes]", - "start_time_unix_nano": "[int]", - "status": { - "code": "STATUS_CODE_UNSET", - "message": "[string]" - }, - "trace_id": "[bytes]", - "trace_state": "[string]", - "type": "span" - } - ], "dropped_attributes_count": "[int]", "dropped_events_count": "[int]", "dropped_links_count": "[int]", "end_time_unix_nano": "[int]", "flags": "[int]", "kind": "[int]", - "name": "offer", + "name": "get_offer", "parent_span_id": "[bytes]", "span_id": "[bytes]", "start_time_unix_nano": "[int]", @@ -81,7 +57,7 @@ expression: lines "end_time_unix_nano": "[int]", "flags": "[int]", "kind": "[int]", - "name": "get_offer", + "name": "offer", "parent_span_id": "[bytes]", "span_id": "[bytes]", "start_time_unix_nano": "[int]", diff --git a/server/tests/features/snapshots/features__otlp_spans__otlp_spans__error_get_offer_404.snap b/server/tests/features/snapshots/features__otlp_spans__otlp_spans__error_get_offer_404.snap index c80b497..69244fe 100644 --- a/server/tests/features/snapshots/features__otlp_spans__otlp_spans__error_get_offer_404.snap +++ b/server/tests/features/snapshots/features__otlp_spans__otlp_spans__error_get_offer_404.snap @@ -1,6 +1,6 @@ --- source: server/tests/features/otlp_spans.rs -assertion_line: 203 +assertion_line: 207 expression: lines --- [ @@ -75,7 +75,7 @@ expression: lines "end_time_unix_nano": "[int]", "flags": "[int]", "kind": "[int]", - "name": "offer", + "name": "get_offer", "parent_span_id": "[bytes]", "span_id": "[bytes]", "start_time_unix_nano": "[int]", @@ -94,7 +94,7 @@ expression: lines "end_time_unix_nano": "[int]", "flags": "[int]", "kind": "[int]", - "name": "get_offer", + "name": "offer", "parent_span_id": "[bytes]", "span_id": "[bytes]", "start_time_unix_nano": "[int]", diff --git a/server/tests/features/snapshots/features__otlp_spans__otlp_spans__success_invoice.snap b/server/tests/features/snapshots/features__otlp_spans__otlp_spans__success_invoice.snap index e6bdb49..cbbbb37 100644 --- a/server/tests/features/snapshots/features__otlp_spans__otlp_spans__success_invoice.snap +++ b/server/tests/features/snapshots/features__otlp_spans__otlp_spans__success_invoice.snap @@ -1,6 +1,6 @@ --- source: server/tests/features/otlp_spans.rs -assertion_line: 121 +assertion_line: 123 expression: lines --- [ @@ -33,54 +33,6 @@ expression: lines "attributes": { "target": "[string]" }, - "children": [ - { - "attributes": { - "target": "[string]" - }, - "children": [ - { - "attributes": { - "target": "[string]" - }, - "dropped_attributes_count": "[int]", - "dropped_events_count": "[int]", - "dropped_links_count": "[int]", - "end_time_unix_nano": "[int]", - "flags": "[int]", - "kind": "[int]", - "name": "create", - "parent_span_id": "[bytes]", - "span_id": "[bytes]", - "start_time_unix_nano": "[int]", - "status": { - "code": "STATUS_CODE_UNSET", - "message": "[string]" - }, - "trace_id": "[bytes]", - "trace_state": "[string]", - "type": "span" - } - ], - "dropped_attributes_count": "[int]", - "dropped_events_count": "[int]", - "dropped_links_count": "[int]", - "end_time_unix_nano": "[int]", - "flags": "[int]", - "kind": "[int]", - "name": "connect", - "parent_span_id": "[bytes]", - "span_id": "[bytes]", - "start_time_unix_nano": "[int]", - "status": { - "code": "STATUS_CODE_UNSET", - "message": "[string]" - }, - "trace_id": "[bytes]", - "trace_state": "[string]", - "type": "span" - } - ], "dropped_attributes_count": "[int]", "dropped_events_count": "[int]", "dropped_links_count": "[int]", @@ -99,6 +51,28 @@ expression: lines "trace_state": "[string]", "type": "span" }, + { + "attributes": { + "target": "[string]" + }, + "dropped_attributes_count": "[int]", + "dropped_events_count": "[int]", + "dropped_links_count": "[int]", + "end_time_unix_nano": "[int]", + "flags": "[int]", + "kind": "[int]", + "name": "get_all", + "parent_span_id": "[bytes]", + "span_id": "[bytes]", + "start_time_unix_nano": "[int]", + "status": { + "code": "STATUS_CODE_UNSET", + "message": "[string]" + }, + "trace_id": "[bytes]", + "trace_state": "[string]", + "type": "span" + }, { "attributes": { "target": "[string]" @@ -108,122 +82,28 @@ expression: lines "attributes": { "target": "[string]" }, - "children": [ - { - "attributes": { - "target": "[string]" - }, - "children": [ - { - "attributes": { - "target": "[string]" - }, - "dropped_attributes_count": "[int]", - "dropped_events_count": "[int]", - "dropped_links_count": "[int]", - "end_time_unix_nano": "[int]", - "flags": "[int]", - "kind": "[int]", - "name": "get_client", - "parent_span_id": "[bytes]", - "span_id": "[bytes]", - "start_time_unix_nano": "[int]", - "status": { - "code": "STATUS_CODE_UNSET", - "message": "[string]" - }, - "trace_id": "[bytes]", - "trace_state": "[string]", - "type": "span" - }, - { - "attributes": { - "target": "[string]" - }, - "children": [ - { - "attributes": { - "target": "[string]" - }, - "dropped_attributes_count": "[int]", - "dropped_events_count": "[int]", - "dropped_links_count": "[int]", - "end_time_unix_nano": "[int]", - "flags": "[int]", - "kind": "[int]", - "name": "get_invoice", - "parent_span_id": "[bytes]", - "span_id": "[bytes]", - "start_time_unix_nano": "[int]", - "status": { - "code": "STATUS_CODE_UNSET", - "message": "[string]" - }, - "trace_id": "[bytes]", - "trace_state": "[string]", - "type": "span" - }, - { - "attributes": { - "target": "[string]" - }, - "dropped_attributes_count": "[int]", - "dropped_events_count": "[int]", - "dropped_links_count": "[int]", - "end_time_unix_nano": "[int]", - "flags": "[int]", - "kind": "[int]", - "name": "inner_connect", - "parent_span_id": "[bytes]", - "span_id": "[bytes]", - "start_time_unix_nano": "[int]", - "status": { - "code": "STATUS_CODE_UNSET", - "message": "[string]" - }, - "trace_id": "[bytes]", - "trace_state": "[string]", - "type": "span" - } - ], - "dropped_attributes_count": "[int]", - "dropped_events_count": "[int]", - "dropped_links_count": "[int]", - "end_time_unix_nano": "[int]", - "flags": "[int]", - "kind": "[int]", - "name": "get_invoice", - "parent_span_id": "[bytes]", - "span_id": "[bytes]", - "start_time_unix_nano": "[int]", - "status": { - "code": "STATUS_CODE_UNSET", - "message": "[string]" - }, - "trace_id": "[bytes]", - "trace_state": "[string]", - "type": "span" - } - ], - "dropped_attributes_count": "[int]", - "dropped_events_count": "[int]", - "dropped_links_count": "[int]", - "end_time_unix_nano": "[int]", - "flags": "[int]", - "kind": "[int]", - "name": "get_invoice", - "parent_span_id": "[bytes]", - "span_id": "[bytes]", - "start_time_unix_nano": "[int]", - "status": { - "code": "STATUS_CODE_UNSET", - "message": "[string]" - }, - "trace_id": "[bytes]", - "trace_state": "[string]", - "type": "span" - } - ], + "dropped_attributes_count": "[int]", + "dropped_events_count": "[int]", + "dropped_links_count": "[int]", + "end_time_unix_nano": "[int]", + "flags": "[int]", + "kind": "[int]", + "name": "get_features", + "parent_span_id": "[bytes]", + "span_id": "[bytes]", + "start_time_unix_nano": "[int]", + "status": { + "code": "STATUS_CODE_UNSET", + "message": "[string]" + }, + "trace_id": "[bytes]", + "trace_state": "[string]", + "type": "span" + }, + { + "attributes": { + "target": "[string]" + }, "dropped_attributes_count": "[int]", "dropped_events_count": "[int]", "dropped_links_count": "[int]", @@ -249,73 +129,7 @@ expression: lines "end_time_unix_nano": "[int]", "flags": "[int]", "kind": "[int]", - "name": "get_invoice_from_backend", - "parent_span_id": "[bytes]", - "span_id": "[bytes]", - "start_time_unix_nano": "[int]", - "status": { - "code": "STATUS_CODE_UNSET", - "message": "[string]" - }, - "trace_id": "[bytes]", - "trace_state": "[string]", - "type": "span" - }, - { - "attributes": { - "target": "[string]" - }, - "dropped_attributes_count": "[int]", - "dropped_events_count": "[int]", - "dropped_links_count": "[int]", - "end_time_unix_nano": "[int]", - "flags": "[int]", - "kind": "[int]", - "name": "select_backend", - "parent_span_id": "[bytes]", - "span_id": "[bytes]", - "start_time_unix_nano": "[int]", - "status": { - "code": "STATUS_CODE_UNSET", - "message": "[string]" - }, - "trace_id": "[bytes]", - "trace_state": "[string]", - "type": "span" - }, - { - "attributes": { - "target": "[string]" - }, - "dropped_attributes_count": "[int]", - "dropped_events_count": "[int]", - "dropped_links_count": "[int]", - "end_time_unix_nano": "[int]", - "flags": "[int]", - "kind": "[int]", - "name": "select_backend", - "parent_span_id": "[bytes]", - "span_id": "[bytes]", - "start_time_unix_nano": "[int]", - "status": { - "code": "STATUS_CODE_UNSET", - "message": "[string]" - }, - "trace_id": "[bytes]", - "trace_state": "[string]", - "type": "span" - }, - { - "attributes": { - "target": "[string]" - }, - "dropped_attributes_count": "[int]", - "dropped_events_count": "[int]", - "dropped_links_count": "[int]", - "end_time_unix_nano": "[int]", - "flags": "[int]", - "kind": "[int]", - "name": "select_backend", + "name": "get_invoice", "parent_span_id": "[bytes]", "span_id": "[bytes]", "start_time_unix_nano": "[int]", @@ -350,6 +164,30 @@ expression: lines "attributes": { "target": "[string]" }, + "children": [ + { + "attributes": { + "target": "[string]" + }, + "dropped_attributes_count": "[int]", + "dropped_events_count": "[int]", + "dropped_links_count": "[int]", + "end_time_unix_nano": "[int]", + "flags": "[int]", + "kind": "[int]", + "name": "get_offer", + "parent_span_id": "[bytes]", + "span_id": "[bytes]", + "start_time_unix_nano": "[int]", + "status": { + "code": "STATUS_CODE_UNSET", + "message": "[string]" + }, + "trace_id": "[bytes]", + "trace_state": "[string]", + "type": "span" + } + ], "dropped_attributes_count": "[int]", "dropped_events_count": "[int]", "dropped_links_count": "[int]", diff --git a/service/src/axum/extract/host.rs b/service/src/axum/extract/host.rs index 0c65f13..9cb78de 100644 --- a/service/src/axum/extract/host.rs +++ b/service/src/axum/extract/host.rs @@ -76,5 +76,5 @@ fn parse_authority(auth: &Authority) -> &str { auth.as_str() .rsplit('@') .next() - .expect("split always has at least 1 item") + .unwrap_or_else(|| auth.as_str()) } diff --git a/service/src/discovery/service.rs b/service/src/discovery/service.rs index c85c6e0..fd333f9 100644 --- a/service/src/discovery/service.rs +++ b/service/src/discovery/service.rs @@ -42,6 +42,7 @@ impl DiscoveryService { .with_state(state) } + #[tracing::instrument(skip_all)] async fn health_check_handler() -> StatusCode { StatusCode::OK } diff --git a/service/src/lnurl/pay/handler.rs b/service/src/lnurl/pay/handler.rs index 063103b..0f97743 100644 --- a/service/src/lnurl/pay/handler.rs +++ b/service/src/lnurl/pay/handler.rs @@ -195,7 +195,6 @@ impl LnUrlPayHandlers { )) } - #[tracing::instrument(skip_all)] fn gen_bech32(callback: &str) -> io::Result { let callback = Url::parse(callback).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; @@ -205,7 +204,6 @@ impl LnUrlPayHandlers { Ok(callback) } - #[tracing::instrument(skip_all)] async fn get_offer( hostname: &str, partition: &str, @@ -234,7 +232,6 @@ impl LnUrlPayHandlers { Ok(offer) } - #[tracing::instrument(skip_all)] fn expires_headers( expires: Option>, ) -> Result { diff --git a/service/src/lnurl/service.rs b/service/src/lnurl/service.rs index d0006ad..84a8394 100644 --- a/service/src/lnurl/service.rs +++ b/service/src/lnurl/service.rs @@ -37,6 +37,7 @@ impl LnUrlBalancerService { .with_state(state) } + #[tracing::instrument(skip_all)] async fn health_check_handler() -> StatusCode { StatusCode::OK } diff --git a/service/src/offer/service.rs b/service/src/offer/service.rs index 2710161..b77bc09 100644 --- a/service/src/offer/service.rs +++ b/service/src/offer/service.rs @@ -50,6 +50,7 @@ impl OfferService { .with_state(state) } + #[tracing::instrument(skip_all)] async fn health_check_handler() -> StatusCode { StatusCode::OK } diff --git a/switchgear/src/lib.rs b/src/lib.rs similarity index 100% rename from switchgear/src/lib.rs rename to src/lib.rs diff --git a/switchgear/Cargo.toml b/switchgear/Cargo.toml deleted file mode 100644 index 6710511..0000000 --- a/switchgear/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "switchgear" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -authors = ["Bitshock "] -description = "High availability LNURL load balancer server for enterprise Bitcoin Lightning payment providers" -documentation = "https://github.com/bitshock-src/switchgear" -homepage = "https://bitshock.com" -repository = "https://github.com/bitshock-src/switchgear" -license = "Apache-2.0" -keywords = ["bitcoin", "lightning", "lnurl", "load-balancer", "payments"] -categories = ["command-line-utilities", "network-programming", "web-programming"] -readme = "./README.md" -publish = true - -[dependencies] diff --git a/switchgear/README.md b/switchgear/README.md deleted file mode 100644 index c76eb20..0000000 --- a/switchgear/README.md +++ /dev/null @@ -1,1237 +0,0 @@ -# Switchgear - -**By [Bitshock](https://bitshock.com) | [info@bitshock.com](mailto:info@bitshock.com)** - -> "The impediment to action advances action. What stands in the way becomes the way." – Marcus Aurelius - -Switchgear is a high availability LNURL balancer for enterprise Bitcoin Lightning payment providers. It is designed to scale massive multi-region Lightning Node fleets at five nines uptime. - -![image](https://raw.githubusercontent.com/bitshock-src/switchgear/main/doc/switchgear.png) - -## Features - -Professional LNURL load balancing for your enterprise: - -* CLN Lightning Node support with gRPC -* LND Lightning Node support with gRPC -* built on CloudFlare's [Pingora](https://github.com/cloudflare/pingora) Load Balancer -* Three balancing algorithms: Round Robin, Random and [Consistent](https://en.wikipedia.org/wiki/Consistent_hashing) (Ketama) -* Node health checks redirect invoice requests to healthy nodes -* Self-healing - if an unhealthy node transitions to healthy, it will automatically start taking invoice requests again -* Weighting can shift invoice requests to preferred nodes -* Liquidity biasing with inbound capacity checks will favor nodes most likely to accept payment -* Timed retries with exponential backoff for spurious node failures -* Partitions bind inbound invoice requests to subset of nodes -* Add nodes in seconds with REST Discovery API - new nodes will automatically start taking invoice requests -* Drop nodes in seconds with REST Discovery API - the balancer will safely direct invoices to remaining healthy nodes -* Publish new LNURLs in seconds with Offer API -* `bech32` and QR code generation endpoints -* Sqlite, MySql and Postgres database support for both Discovery and Offer data stores -* Remote REST stores for both Discovery and Offer - -![image](https://raw.githubusercontent.com/bitshock-src/switchgear/main/doc/sequence-LNURL_Pay_Multi_Backend_Invoice_Generation.png) - - -## Status - -Switchgear is in **ALPHA** status: - -* Critical features are implemented and work as designed -* Integration tests are complete -* APIs may change without warning - -See [ROADMAP.md](https://github.com/bitshock-src/switchgear/blob/HEAD/ROADMAP.md) for the Switchgear release roadmap. - -## Why Bitcoin Lightning Payments Fail - -### Single Points Of Failure - -Failure with a standard LNURL service configuration is highly-probable: - -![image](https://raw.githubusercontent.com/bitshock-src/switchgear/main/doc/sequence_invoice_failure-LNURL_Pay___Invoice_Generation_Failure.png) - - -### No Liquidity Bias - -Single LNURL + Lightning Node deployments make Liquidity Bias impossible. Liquidity bias deprioritizes nodes for invoice requests that have low inbound capacity, which make payment success unlikely. - -![image](https://raw.githubusercontent.com/bitshock-src/switchgear/main/doc/sequence_payment_failure-LNURL_Pay___Payment_Failure__Inadequate_Inbound_Liquidity_.png) - - -### External Load Balancer Resource Exhaustion - -Load balancing multiple LNURL instances with a single fixed Lightning Node will exhaust resources: - -* If a node fails, it makes the entire LNURL instance useless -* If a LNURL instance fails, it makes the Lightning Node unreachable -* Duplicating the entire LNURL instance just to attach it to a single Lightning Node raises cost, reducing feasibility of reaching scale -* High latency increases chances of catastrophic retry loop or flooding single node with failover requests - -![image](https://raw.githubusercontent.com/bitshock-src/switchgear/main/doc/lnurl-external-balancer.png) - -By contrast, every instance of Switchgear in a region can share the Lightning Node fleet, making scale feasible: - -* A single Switchgear instance failure does not make a Lightning Node unreachable -* Any Lightning Node failure does not affect the uptime of a Switchgear instance -* Switchgear instances and the Lightning Node fleet size can scale independently, reducing cost -* Health checks are low latency, keeping Lightning Node status accurate -* Internal Backoff runs through entire Backend selection before returning 502 to external balancer, reducing latency and wallet retry cascade - -![image](https://raw.githubusercontent.com/bitshock-src/switchgear/main/doc/switchgear-single-region.png) - -The internal backoff can be entered under two conditions: - -1. No healthy Lightning Nodes are available -2. A Lightning Node spurious failure or node failure between health checks - -![image](https://raw.githubusercontent.com/bitshock-src/switchgear/main/doc/get_invoice_loop-Get_Invoice_Loop.png) - -## Install - -The Switchgear binary runs all services as well as the CLI admin interface. - -### Host - -```shell -cargo install switchgear-server -``` - -### Docker - -The docker image is multi-platform for: - -* linux/amd64 -* linux/arm64 - -```shell -docker pull bitshock/switchgear -``` - -## Starting Switchgear Services - -``` shell -swgr service --config {path/to/config/file} {service enabledment list} -``` - -The configuration file is in YAML format, and controls settings for all services. - -The service enablement list can be any of: - -* `lnurl` - the public LNURL service -* `discovery` - the admin discovery service -* `offer` - the offer admin service -* `all` - all services - -If left empty, all services will be enabled (same as `all`). - -### Docker - -To run the Docker image: - -```shell -docker run bitshock/switchgear -``` - -The image is configured with a default configuration file path of `/etc/swgr/config.yaml` . Mount a volume on top of `/etc/swgr` to provide your own configuration file. - -To build the Docker image: - -```shell -docker buildx build --platform linux/arm64,linux/amd64 -t swgr . -``` - -Set the build arg `WEBPKI_ROOTS=true` if you need Mozilla's web PKI roots bundle installed on the image: - -```shell -docker buildx build --platform linux/arm64,linux/amd64 --build-arg WEBPKI_ROOTS=true -t swgr . -``` - -## Administration - -Switchgear can be configured by both the REST API and the CLI. - -### REST Administration - -Administration Service endpoints: - -``` -https://{host}/discovery -https://{host}/offers -``` - -See the [Manage Lightning Node Backends with Discovery Service](#manage-lightning-node-backends-with-discovery-service) and [Manage LNURLs with Offer Service](#manage-lnurls-with-offer-service) sections for complete REST API. - -### CLI Administration - -```shell -# Manage Lightning Node Backends -swgr discovery -# Manage LNURLs -swgr offer -``` -See the [Manage Lightning Node Backends with Discovery Service](#manage-lightning-node-backends-with-discovery-service) and [Manage LNURLs with Offer Service](#manage-lnurls-with-offer-service) sections for complete CLI manual. - -### Docker - -To run the CLI administration from Docker: - -```shell -docker run bitshock/switchgear {cli-options} -``` - -## Configuring Switchgear Services - -All service configuration is controlled by a yaml file passed to the server at startup. - -See [server/config](https://github.com/bitshock-src/switchgear/blob/HEAD/server/config) directory for more configuration examples. - -Each service has a root entry the configuration file: - -```yaml -# LNURL Service Configuration -lnurl-service: - -# Discovery Service Configuration -discovery-service: - -# Offer Service Configuration -offer-service: - -# Persistence Settings for Discovery and Offer data -store: - -# Optional secrets file to replace tokens in specific configuration fields -secrets: -``` - -See the service entries below for complete configuration manual. - -#### Env Var Shell Expansion - -Shell-style env var expansion is supported anywhere in the yaml configuration file. - -Example config.yaml: - -```yaml -lnurl-service: - address: "${MY_LNURL_SERVICE_ADDRESS:-127.0.0.1:8080}" -``` - -Run with: - -```shell -MY_LNURL_SERVICE_ADDRESS=192.168.1.100:8080 swgr service --config ./config.yaml -``` - -The configuration would be parsed as: - -```yaml -lnurl-service: - address: "192.168.1.100:8080" -``` - -If the env var is unset: - -```shell -swgr service --config ./config.yaml -``` - -The configuration would be parsed as: - -```yaml -lnurl-service: - address: "127.0.0.1:8080" -``` - -#### Secrets Expansion - -Secrets are read from a secrets file, and can be used to replace tokens in a configuration file. - -The secrets file format is a standard dot file: - -```shell -NAME="value" -``` - -Example: - -```shell -MYSQL_USERNAME=root -MYSQL_PASSWORD=mysql -POSTGRES_USERNAME=postgres -POSTGRES_PASSWORD=postgres -``` - -Secrets expansion works the same way as env var expansion, but with the different template syntax: - -``` -{secret.SECRET_NAME} -``` - -Example: - -```yaml - database-uri: "postgres://{secret.POSTGRES_USERNAME}:{secret.POSTGRES_PASSWORD}@localhost:5432/discovery" -``` - -Unlike env var expansion, secret expansion is selective. Only specific configuration fields have access to secrets. - -## Liquidity Bias - -Optional Liquidity Bias will prioritize nodes with inbound capacity at the moment of selection. If no nodes are in range of the capacity bias, selection will fall back to standard weight selection. - -#### Negative Capacity Bias example (restrictive): - -```yaml -lnurl-service: - selection-capacity-bias: -0.2 -``` - -Invoice request amount must be less than 20% of total inbound capacity of the node to be favored over other nodes that are not. - -#### Positive Capacity Bias example (permissive): - -```yaml -lnurl-service: - selection-capacity-bias: 0.1 -``` - -Invoice request amount may be over inbound capacity by up to 10% of the node and still be favored over other nodes that are not. - -#### Inbound Capacity Measurement - -Capacity is measured in the same cycle as the Lightning Node health check. It is the sum of inbound capacity for all active channels on the node. - -## Partitioning - -An organization may have a global Offer database. A Switchgear instance may be configured to serve a portion of that database, using partitions. Furthermore, every Lightning Node is configured in Discovery to be bound to one or more partitions, insuring payments only land on nodes they belong to. - -Each Switchgear instance is configured for the partitions it will serve. - -Example: - -```yaml -lnurl-service: - partitions: ["us", "it", "cr"] -``` - -Only Offers created in "us", "it" or "cr" partitions will be available on the Switchgear instance, even if the Offer exists the database: - -``` -https://example.com/offers/us/{id} - 200 success -https://example.com/offers/it/{id} - 200 success -https://example.com/offers/cr/{id} - 200 success -https://example.com/offers/ca/{id} - 404 not found -``` - -Each partition must have Discovery Backends configured for the invoice request to succeed. A single backend can serve multiple partitions. - -See the [Manage Lightning Node Backends with Discovery Service](#manage-lightning-node-backends-with-discovery-service) and [Manage LNURLs with Offer Service](#manage-lnurls-with-offer-service) sections for creating Discovery Backends and Offers with partitions. - -## Balancing Switchgear - -Switchgear itself can be balanced. Balancing multiple switchgear instances within a region: - -![image](https://raw.githubusercontent.com/bitshock-src/switchgear/main/doc/switchgear-single-region.png) - -For multi-region Global Load Balancer deployment, use the full health check to signal to the downstream balancer that a Switchgear instance has no healthy Lightning Nodes: - -``` -https://{host}/health/full -``` - -The full health check will return 500 if no Lightning Nodes are available. - -The regional balancer will forward the failing health status to the global balancer, which will send invoice requests to an alternate region: - -![image](https://raw.githubusercontent.com/bitshock-src/switchgear/main/doc/switchgear-multi-region.png) - -Switchgear partitions have predictable URLs. Use partitions and a global Application Load Balancer to map invoice requests to alternate regions that have Switchgear instances configured for the requested partition. - -## LNURL Service - -The OpenAPI LNURL Service specification: [doc/lnurl-service-openapi.yaml](https://github.com/bitshock-src/switchgear/blob/HEAD/doc/lnurl-service-openapi.yaml). - -The LNURL Service is public facing, and implements the [LNURL LUD-06 specification.](https://github.com/lnurl/luds/blob/luds/06.md) - -See the [Manage LNURLs with Offer Service](#manage-lnurls-with-offer-service) section for complete service manual. - -All Switchgear LNURLs are formatted as: - -``` -https://{host}/offers/{partition}/{id} -``` - -Where: - -* `partition` - the Offer partition -* `id` - the Offer id (Uuid) - -The returned callback is always the LNURL, with the postfix `/invoice` : - -``` -https://{host}/offers/{partition}/{id}/invoice -``` - -The bech32 and QR variants are available with: - -``` -https://{host}/offers/{partition}/{id}/bech32 -``` - -And: - -``` -https://{host}/offers/{partition}/{id}/bech32/qr -``` - -The QR image is in PNG format. - -### LNURL Service Configuration - -See [server/config](https://github.com/bitshock-src/switchgear/blob/HEAD/server/config) directory for more configuration examples. - -```yaml -lnurl-service: - # List of partitions this service will handle - # Partitions allow you to segment different Lightning node groups - partitions: ["default"] - - # Network address and port for the LNURL service to bind to - address: "127.0.0.1:8080" - - # Frequency in seconds for health checking Lightning node backends (float) - health-check-frequency-secs: 1.0 - - # Whether to perform health checks in parallel across all backends - parallel-health-check: true - - # Number of consecutive successful health checks needed to mark a backend as healthy - health-check-consecutive-success-to-healthy: 1 - - # Number of consecutive failed health checks needed to mark a backend as unhealthy - health-check-consecutive-failure-to-unhealthy: 1 - - # Frequency in seconds for updating backend node information (float) - backend-update-frequency-secs: 1.0 - - # Invoice expiry time in seconds (integer) - invoice-expiry-secs: 180 - - # Timeout in seconds for Lightning node client connections (float) - ln-client-timeout-secs: 2.0 - - # Optional trusted roots pem bundle for all LN clients - ln-trusted-roots: "/etc/ssl/certs/ln-ca.pem" - - # List of allowed host headers for incoming requests - # Used for safely generating callback/invoice URLs. - allowed-hosts: ["lnurl.example.com"] - - # Backoff configuration for retrying failed operations. - # Backoff is used when Lightning Node invoice request fails. - backoff: - # Type of backoff strategy: "stop" or "exponential" - type: "exponential" - # Optional: Initial interval in seconds for exponential backoff (float) - initial-interval-secs: 1.0 - # Optional: Randomization factor (0.0 to 1.0) - randomization-factor: 0.5 - # Optional: Multiplier for each retry attempt - multiplier: 2.0 - # Optional: Maximum interval between retries in seconds (float) - max-interval-secs: 60.0 - # Optional: Maximum total elapsed time before giving up in seconds (float) - max-elapsed-time-secs: 300.0 - - # Backend selection strategy for load balancing - # Options: "round-robin", "random", or "consistent" - backend-selection: "round-robin" - # For consistent hashing, specify max iterations (only used with "consistent") - # backend-selection: - # type: "consistent" - # max-iterations: 10000 - - # Optional: Bias factor for capacity-influenced selection - # Negative values are restrictive: - # prefer nodes with capacity higher than requested amount - # Positive values are lenient: - # refer nodes with capacity less than requested amount - selection-capacity-bias: -0.2 - - # Optional: Allow &comment query param in LNURL invoice request, sized in char len - # Used for Consistent backend selection - comment_allowed: 64, - - # Optional: TLS configuration for HTTPS support - tls: - # Path to TLS certificate file - cert-path: "/etc/ssl/certs/lnurl-cert.pem" - # Path to TLS private key file - key-path: "/etc/ssl/certs/lnurl-key.pem" - - # QR module width x height - bech32-qr-scale: 8 - # QR light gray level - bech32-qr-light: 255 - # QR dark gray level - bech32-qr-dark: 0 -``` - -### Consistent Backend-Selection - -Consistent uses the optional LNURL `comment` query parameter as a hash key, which guarantees the same node will always receive invoice requests for that key. The balancer will move on to the next closest key match if the node becomes unavailable. This is a specific use-case that provides optimized HTLC settlement between cooperating peers for high-frequency transactions. - -## Discovery Service - -The OpenAPI Discovery Service specification: [doc/discovery-service-openapi.yaml](https://github.com/bitshock-src/switchgear/blob/HEAD/doc/discovery-service-openapi.yaml). - -The Discovery Service is an administrative service used to manage connections to individual Lightning Nodes. - -The service is isolated from the LNURL Service and can be configured to run on any port. The service supports TLS. Access is protected by a bearer token. Do not run this service without TLS enabled if it is exposed to the public internet. - -See the [Manage Lightning Node Backends with Discovery Service](#manage-lightning-node-backends-with-discovery-service) section for complete service manual. - -### Discovery Service Configuration - -See [server/config](https://github.com/bitshock-src/switchgear/blob/HEAD/server/config) directory for more configuration examples. - -```yaml -discovery-service: - # Network address and port for the Discovery service to bind to - address: "127.0.0.1:8081" - - # Path to the authentication authority certificate/key file - # This file contains the public key used to verify API access - auth-authority: "/etc/ssl/certs/discovery-auth-authority.pem" - - # Optional: TLS configuration for HTTPS support - tls: - # Path to TLS certificate file - cert-path: "/etc/ssl/certs/discovery-cert.pem" - # Path to TLS private key file - key-path: "/etc/ssl/certs/discovery-key.pem" -``` - -#### Authentication Setup - -Generate key pairs and tokens for Discovery service authentication: - -```shell -# Generate a new key pair for token signing -swgr discovery token key --public discovery-public.pem --private discovery-private.pem - -# The public key (discovery-public.pem) should be used as the auth-authority in the configuration -# The private key (discovery-private.pem) is used to mint authentication tokens - -# Create a token (default 3600 seconds) -swgr discovery token mint --key discovery-private.pem --output discovery.token -``` - -## Offer Service - -The OpenAPI Offer Service specification: [doc/offer-service-openapi.yaml](https://github.com/bitshock-src/switchgear/blob/HEAD/doc/offer-service-openapi.yaml). - -The Offer Service is an administrative service used to manage Offers, which are used to generate LNURLs. - -The service is isolated from the LNURL Service and can be configured to run on any port. The service supports TLS. Access is protected by a bearer token. Do not run this service without TLS enabled if it is exposed to the public internet. - -See the [Manage LNURLs with Offer Service](#manage-lnurls-with-offer-service) section for complete service manual. - -### Offer Service Configuration - -See [server/config](https://github.com/bitshock-src/switchgear/blob/HEAD/server/config) directory for more configuration examples. - -```yaml -offer-service: - # Network address and port for the Offers service to bind to - address: "127.0.0.1:8082" - - # Path to the authentication authority certificate/key file - # This file contains the public key used to verify API access - auth-authority: "/etc/ssl/certs/offer-auth-authority.pem" - - # Optional: TLS configuration for HTTPS support - tls: - # Path to TLS certificate file - cert-path: "/etc/ssl/certs/offer-cert.pem" - # Path to TLS private key file - key-path: "/etc/ssl/certs/offer-key.pem" - - # max page size for get all queries - max-page-size: 100 -``` - -#### Authentication Setup - -Generate key pairs and tokens for Offer service authentication: - -```shell -# Generate a new key pair for token signing -swgr offer token key --public offer-public.pem --private offer-private.pem - -# The public key (offer-public.pem) should be used as the auth-authority in the configuration -# The private key (offer-private.pem) is used to mint authentication tokens - -# Create a token (default 3600 seconds) -swgr offer token mint --key offer-private.pem --output offer.token -``` - -## Persistence - -Both Discovery and Offer services support multiple storage backends. Configure persistence in the `store` section of your configuration file. - -### Common Storage Types - -Both Discovery and Offer stores support these storage backends: - -#### Database Storage (SQLite/MySQL/PostgreSQL) - -```yaml -store: - discover: # or 'offer' - type: "database" - # Database connection URI (SQLite/MySQL/PostgreSQL) - # Supports secrets expansion - database-uri: "connection-url" - # Maximum number of concurrent database connections - max-connections: 5 -``` - -For `database-uri` formats, see [Database Connection URLs](#database-connection-urls). - -#### HTTP Storage (Remote Service) - -Both Discovery and Offer can use a remote http store, making custom integrations straightforward. The store clients connect to the same REST API used for remote administration, making it possible for Switchgear to run headless as well, serving only as a database for other Switchgear instances. - -```yaml -store: - discover: # or 'offer' - type: "http" - # Base URL of the remote service - base-url: "https://service.example.com" - # Timeout in seconds for establishing connection - connect-timeout-secs: 2.0 - # Total timeout in seconds for complete request/response - total-timeout-secs: 5.0 - # Optional pem bundle of trusted CA certificate paths for TLS verification - trusted-roots: "/etc/ssl/certs/ca.pem" - # Path to bearer token file for authentication - authorization: "/etc/ssl/certs/auth.token" -``` - -#### In-memory Storage - -Volatile storage, data is lost on restart: - -```yaml -store: - discover: # or 'offer' - type: "memory" -``` - -### Configuration Examples - -#### Using Same Database for Both Stores - -```yaml -store: - # Discovery backend storage - discover: - type: "database" - database-uri: "postgres://{secret.POSTGRES_USERNAME}:{secret.POSTGRES_PASSWORD}@localhost:5432/switchgear" - max-connections: 5 - - # Offer storage (sharing same database) - offer: - type: "database" - database-uri: "postgres://{secret.POSTGRES_USERNAME}:{secret.POSTGRES_PASSWORD}@localhost:5432/switchgear" - max-connections: 10 -``` - -#### Mixed Storage Types - -```yaml -store: - # Memory storage for Discovery - discover: - type: "memory" - - # Database storage for Offers - offer: - type: "database" - database-uri: "sqlite:///var/lib/switchgear/offers.db?mode=rwc" - max-connections: 5 -``` - -#### Remote Service Configuration - -```yaml -store: - # Connect to remote Discovery service - discover: - type: "http" - base-url: "https://discovery.internal:8081" - connect-timeout-secs: 2.0 - total-timeout-secs: 5.0 - trusted-roots: "/etc/ssl/certs/internal-ca.pem" - authorization: "/etc/ssl/certs/discovery.token" - - # Local database for Offers - offer: - type: "database" - database-uri: "sqlite:///data/offers.db?mode=rwc" - max-connections: 10 -``` - -### Database Connection URLs - -Both Discovery and Offer data stores have a `database-uri` field to configure the database. - -### Sqlite - -``` -sqlite:///path/to/file.db?{options} -``` - -See [https://www.sqlite.org/uri.html](https://www.sqlite.org/uri.html) for all connection URL options. - -### MySQL - -``` -mysql://[host][/database][?properties] -``` - -Properties: - -| Parameter | Default | Description | -|----------------------------|-------------|-------------------------------------------------------------------------------------------------------------------| -| `ssl-mode` | `PREFERRED` | Determines whether or with what priority a secure SSL TCP/IP connection will be negotiated. See [`MySqlSslMode`]. | -| `ssl-ca` | `None` | Sets the name of a file containing a list of trusted SSL Certificate Authorities. | -| `statement-cache-capacity` | `100` | The maximum number of prepared statements stored in the cache. Set to `0` to disable. | -| `socket` | `None` | Path to the unix domain socket, which will be used instead of TCP if set. | - -### Postgres - -```text -postgresql://[user[:password]@][host][:port][/dbname][?param1=value1&...] -``` - -See [https://www.postgresql.org/docs/current/libpq-connect.html](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) for all connection URL options. - -## Manage Lightning Node Backends With Discovery Service - -The Discovery Service manages Lightning Node backends that connect to Switchgear for invoice requests. The service supports dynamic registration, updates, enablement and removal of CLN and LND nodes. - -The Discovery Service can be administered with both REST and the CLI. - -To get started quickly, use the CLI to write a new JSON data model template to a file: - -```shell -# Generate a template backend configuration -swgr discovery new cln-grpc --output cln-backend.json -swgr discovery new lnd-grpc --output lnd-backend.json -```` - -### REST API - -The Discovery Service provides a REST API for backend management. All endpoints except `/health` require bearer token authentication. - -#### Authentication - -First, generate a token: -```shell -# Create a token (expires in 3600 seconds) -# Note: Requires private key from Authentication Setup (see Discovery Service Configuration) -swgr discovery token mint --key discovery-private.pem --expires 3600 --output discovery.token - -# Set authorization header for curl commands -export AUTH_TOKEN=$(cat discovery.token) -``` - -#### Register A New Backend - -```shell -# Register a CLN node -curl -X POST http://localhost:3001/discovery \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "publicKey": "0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798", - "name": "CLN Node 1", - "partitions": ["default"], - "weight": 100, - "enabled": true, - "implementation": { - "type": "clnGrpc", - "url": "https://192.168.1.100:9736", - "domain": "cln-node.local", - "auth": { - "type": "path", - "caCertPath": "/path/to/ca.pem", - "clientCertPath": "/path/to/client.pem", - "clientKeyPath": "/path/to/client-key.pem" - } - } - }' - -# Register an LND node -curl -X POST http://localhost:3001/discovery \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "publicKey": "02c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5", - "name": "LND Node 1", - "partitions": ["default", "us", "eu"], - "weight": 50, - "enabled": true, - "implementation": { - "type": "lndGrpc", - "url": "https://192.168.1.101:10009", - "domain": "lnd-node.local", - "auth": { - "type": "path", - "tlsCertPath": "/path/to/tls.cert", - "macaroonPath": "/path/to/admin.macaroon" - }, - "ampInvoice": false - } - }' -``` - -#### List All Backends - -```shell -curl -X GET http://localhost:3001/discovery \ - -H "Authorization: Bearer $AUTH_TOKEN" -``` - -#### Get A Specific Backend - -```shell -# By public key -curl -X GET "http://localhost:3001/discovery/0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798" \ - -H "Authorization: Bearer $AUTH_TOKEN" -``` - -#### Update A Backend - -```shell -curl -X PUT "http://localhost:3001/discovery/0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798" \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "name": "CLN Node 1 Updated", - "partitions": ["default", "us"], - "weight": 200, - "enabled": false, - "implementation": { - "type": "clnGrpc", - "url": "https://192.168.1.100:9736", - "domain": "cln-node.local", - "auth": { - "type": "path", - "caCertPath": "/path/to/ca.pem", - "clientCertPath": "/path/to/client.pem", - "clientKeyPath": "/path/to/client-key.pem" - } - } - }' -``` - -#### Delete A Backend - -```shell -curl -X DELETE "http://localhost:3001/discovery/0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798" \ - -H "Authorization: Bearer $AUTH_TOKEN" -``` - -#### Health Check - -```shell -# No authentication required -curl http://localhost:3001/health -``` - -### CLI - -The `swgr discovery` command provides the same functionality as the REST interface for remote management of Lightning Node backends. - -#### Token Management - -```shell -# Create a token (default 3600 seconds) -# Note: Requires a private key generated during Authentication Setup (see Discovery Service Configuration) -swgr discovery token mint --key discovery-private.pem --output discovery.token - -# Create a token with custom expiration (86400 seconds = 24 hours) -swgr discovery token mint --key discovery-private.pem --expires 86400 --output discovery-24h.token - -# Verify a token -swgr discovery token verify --public discovery-public.pem --token discovery.token -``` - -#### Backend Management - -```shell -# Generate a template backend configuration -swgr discovery new cln-grpc --output cln-backend.json -swgr discovery new lnd-grpc --output lnd-backend.json - -# Set connection parameters (via environment or flags) -export DISCOVERY_STORE_HTTP_BASE_URL="https://discovery.example.com" -export DISCOVERY_STORE_HTTP_AUTHORIZATION="/path/to/discovery.token" -export DISCOVERY_STORE_HTTP_TRUSTED_ROOTS="/path/to/ca.pem" - -# List all backends (simple table format) -swgr discovery ls - - -# Get backend details (JSON output) -swgr discovery get 0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798 --output backend-details.json - -# Get all backends (JSON output) -swgr discovery get - -# Register a new backend from JSON file -swgr discovery post --input cln-backend.json - -# Update an existing backend -swgr discovery put 0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798 --input updated-backend.json - -# Patch an existing backend -swgr discovery patch 0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798 --input backend-patch.json - -# Enable an existing backend -swgr discovery enable 0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798 - -# Disable an existing backend -swgr discovery disable 0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798 - -# Delete a backend -swgr discovery delete 0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798 -``` - -### Discovery Data Model - -Discovery OpenAPI schema: [doc/discovery-service-openapi.yaml](https://github.com/bitshock-src/switchgear/blob/HEAD/doc/discovery-service-openapi.yaml). - -Example CLN backend configuration: -```json -{ - "publicKey": "0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798", - "name": "CLN Node 1", - "partitions": ["default"], - "weight": 1, - "enabled": true, - "implementation": { - "type": "clnGrpc", - "url": "https://127.0.0.1:9736", - "domain": "localhost", - "auth": { - "type": "path", - "caCertPath": "/path/to/ca.pem", - "clientCertPath": "/path/to/client.pem", - "clientKeyPath": "/path/to/client-key.pem" - } - } -} -``` - -Example LND backend configuration: -```json -{ - "publicKey": "02c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5", - "name": "LND Node 1", - "partitions": ["default", "us", "eu"], - "weight": 1, - "enabled": true, - "implementation": { - "type": "lndGrpc", - "url": "https://127.0.0.1:10009", - "domain": "localhost", - "auth": { - "type": "path", - "tlsCertPath": "/path/to/tls.cert", - "macaroonPath": "/path/to/admin.macaroon" - }, - "ampInvoice": false - } -} -``` - - - -## Manage LNURLs With Offer Service - -The Offer service manages Lightning payment offers and their metadata. It provides storage and retrieval of LNURL Pay offers with configurable payment limits and display information. - -The Offer Service can be administered with both REST and the CLI. - -To get started quickly, use the CLI to write a new JSON data model template to a file: - -```shell -# Generate a template offer configuration -swgr offer new --output offer-template.json - -# Generate a template metadata configuration -swgr offer metadata new --output metadata-template.json -```` - -### LNURL - -An Offer has two fields that combine to create a unique identifier: - -* `partition` -* `id` (Uuid) - -Both fields are used to make the final public LNURL: - -``` -https://{host}/offers/{partition}/{id} -``` - - -### REST API - -The Offer service provides a REST API for offer and metadata management. All endpoints except `/health` require bearer token authentication. - -#### Authentication - -First, generate a token: -```shell -# Create a bearer token (expires in 3600 seconds) -# Note: Requires private key from Authentication Setup (see Offer Service Configuration) -swgr offer token mint --key offer-private.pem --expires 3600 --output offer.token - -# Set authorization header for curl commands -export AUTH_TOKEN=$(cat offer.token) -``` - -#### Create A New Offer - -```shell -curl -X POST http://localhost:3002/offers \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "partition": "default", - "id": "6a38ebdd-83ef-4b94-b843-3b18cd90a833", - "maxSendable": 1000000, - "minSendable": 1000, - "metadataId": "88deff7e-ca45-4144-8fca-286a5a18fb1a", - "timestamp": "2024-01-01T00:00:00Z", - "expires": "2024-12-31T23:59:59Z" - }' -``` - -#### List All Offers In A Partition - -```shell -curl -X GET http://localhost:3002/offers/default \ - -H "Authorization: Bearer $AUTH_TOKEN" -``` - -#### Get A Specific Offer - -```shell -curl -X GET "http://localhost:3002/offers/default/6a38ebdd-83ef-4b94-b843-3b18cd90a833" \ - -H "Authorization: Bearer $AUTH_TOKEN" -``` - -#### Update An Offer - -```shell -curl -X PUT "http://localhost:3002/offers/default/6a38ebdd-83ef-4b94-b843-3b18cd90a833" \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "maxSendable": 2000000, - "minSendable": 1000, - "metadataId": "88deff7e-ca45-4144-8fca-286a5a18fb1a", - "timestamp": "2024-01-01T00:00:00Z", - "expires": "2024-12-31T23:59:59Z" - }' -``` - -#### Delete An Offer - -```shell -curl -X DELETE "http://localhost:3002/offers/default/6a38ebdd-83ef-4b94-b843-3b18cd90a833" \ - -H "Authorization: Bearer $AUTH_TOKEN" -``` - -#### Create Metadata - -```shell -curl -X POST http://localhost:3002/metadata \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "id": "88deff7e-ca45-4144-8fca-286a5a18fb1a", - "partition": "default", - "text": "Lightning Payment", - "longText": "Pay for premium services with Lightning Network", - "image": { - "png": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" - }, - "identifier": { - "email": "payments@example.com" - } - }' -``` - -#### List All Metadata In A Partition - -```shell -curl -X GET http://localhost:3002/metadata/default \ - -H "Authorization: Bearer $AUTH_TOKEN" -``` - -#### Get Specific Metadata - -```shell -curl -X GET "http://localhost:3002/metadata/default/88deff7e-ca45-4144-8fca-286a5a18fb1a" \ - -H "Authorization: Bearer $AUTH_TOKEN" -``` - -#### Update Metadata - -```shell -curl -X PUT "http://localhost:3002/metadata/default/88deff7e-ca45-4144-8fca-286a5a18fb1a" \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "text": "Updated Lightning Payment", - "longText": "Pay for premium services with Lightning Network - Updated", - "identifier": { - "email": "billing@example.com" - } - }' -``` - -#### Delete Metadata - -```shell -curl -X DELETE "http://localhost:3002/metadata/default/88deff7e-ca45-4144-8fca-286a5a18fb1a" \ - -H "Authorization: Bearer $AUTH_TOKEN" -``` - -#### Health Check - -```shell -# No authentication required -curl http://localhost:3002/health -``` - -### CLI - -The `swgr offer` command provides the same functionality as the REST interface for remote management of Lightning payment offers and metadata. - -#### Token Management - -```shell -# Create a token (default 3600 seconds) -# Note: Requires a private key generated during Authentication Setup (see Offer Service Configuration) -swgr offer token mint --key offer-private.pem --output offer.token - -# Create a token with custom expiration (86400 seconds = 24 hours) -swgr offer token mint --key offer-private.pem --expires 86400 --output offer-24h.token - -# Verify a token -swgr offer token verify --public offer-public.pem --token offer.token -``` - -#### Offer Management - -```shell -# Generate a template offer configuration -swgr offer new --output offer-template.json - -# Set connection parameters (via environment or flags) -export OFFER_STORE_HTTP_BASE_URL="https://offer.example.com" -export OFFER_STORE_HTTP_AUTHORIZATION="/path/to/offer.token" -export OFFER_STORE_HTTP_TRUSTED_ROOTS="/path/to/ca.pem" - -# Get offer details (JSON output) -swgr offer get default 6a38ebdd-83ef-4b94-b843-3b18cd90a833 --output offer-details.json - -# Get all offers in partition (JSON output) -swgr offer get default - -# Create a new offer from JSON file -swgr offer post --input offer.json - -# Update an existing offer -swgr offer put default 6a38ebdd-83ef-4b94-b843-3b18cd90a833 --input updated-offer.json - -# Delete an offer -swgr offer delete default 6a38ebdd-83ef-4b94-b843-3b18cd90a833 -``` - -#### Metadata Management - -```shell -# Generate a template metadata configuration -swgr offer metadata new --output metadata-template.json - -# Get metadata details (JSON output) -swgr offer metadata get default 88deff7e-ca45-4144-8fca-286a5a18fb1a --output metadata-details.json - -# Get all metadata in partition (JSON output) -swgr offer metadata get default - -# Create new metadata from JSON file -swgr offer metadata post --input metadata.json - -# Update existing metadata -swgr offer metadata put default 88deff7e-ca45-4144-8fca-286a5a18fb1a --input updated-metadata.json - -# Delete metadata -swgr offer metadata delete default 88deff7e-ca45-4144-8fca-286a5a18fb1a -``` - -### Offer Data Model - -Offer OpenAPI schema: [doc/offer-service-openapi.yaml](https://github.com/bitshock-src/switchgear/blob/HEAD/doc/offer-service-openapi.yaml). - -Example offer configuration: -```json -{ - "partition": "default", - "id": "6a38ebdd-83ef-4b94-b843-3b18cd90a833", - "maxSendable": 1000000, - "minSendable": 1000000, - "metadataId": "88deff7e-ca45-4144-8fca-286a5a18fb1a", - "timestamp": "1970-01-01T00:00:00Z", - "expires": null -} -``` - -Example metadata configuration: -```json -{ - "id": "88deff7e-ca45-4144-8fca-286a5a18fb1a", - "partition": "default", - "text": "mandatory offer text", - "longText": "optional long offer text", - "image": { - "png": "base64_encoded_png_data" - }, - "identifier": { - "email": "optional@email.com" - } -} -``` - -#### Image Support - -Metadata can include images in PNG or JPEG format, base64 encoded: -- PNG: `{"png": "base64_encoded_data"}` -- JPEG: `{"jpeg": "base64_encoded_data"}` - -#### Identifier Types - -Metadata identifiers can be: -- Email: `{"email": "contact@example.com"}` -- Text: `{"text": "contact@example.com"}` diff --git a/testing/Cargo.toml b/testing/Cargo.toml index 6e55e0d..486484c 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -19,7 +19,9 @@ fs4 = { version ="0.13" } hex = "0.4" indexmap = "2" rand = "0.9.2" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots-no-provider"] } secp256k1 = { version = "0.31", features = ["recovery", "serde"] } +serde_json = "1" sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "tls-rustls-aws-lc-rs", "sqlite", "postgres", "mysql"] } tar = "0.4" tempfile = "3" diff --git a/testing/README.md b/testing/README.md index bcc581a..d252045 100644 --- a/testing/README.md +++ b/testing/README.md @@ -1,6 +1,6 @@ # Testing -Docker-based regtest environment for testing with Lightning Network nodes (CLN, LND), Bitcoin Core, databases, and an OpenTelemetry tracing stack (OTEL Collector + Jaeger). +Docker-based regtest environment for testing with Lightning Network nodes (CLN, LND), Bitcoin Core, databases, and an OpenTelemetry telemetry stack (OTEL Collector + Jaeger for traces, InfluxDB for metrics). ## Local Testing @@ -25,6 +25,7 @@ MYSQL_HOSTNAME=localhost POSTGRES_HOSTNAME=localhost OTEL_GRPC_HOSTNAME=localhost JAEGER_HOSTNAME=localhost +INFLUXDB_HOSTNAME=localhost ``` 4. **Run tests:** @@ -58,12 +59,13 @@ For running tests inside a container with Docker socket access. cargo test ``` -## Tracing Stack +## Telemetry Stack -The compose bundle ships an OTLP-gRPC collector fronting a Jaeger v2 backend so tests can exercise real trace export and query spans back out. +The compose bundle ships an OTLP-gRPC collector fronting a Jaeger v2 backend for traces and an InfluxDB 3 backend for metrics, so tests can exercise real export and query telemetry back out. -- **OTEL Collector** (`otel-collector`, port `OTEL_GRPC_PORT`, default `4317`) — OTLP gRPC receiver that requires TLS, mTLS client-cert verification, **and** a bearer token on every request. Forwards traces to Jaeger. +- **OTEL Collector** (`otel-collector`, port `OTEL_GRPC_PORT`, default `4317`) — OTLP gRPC receiver that requires TLS, mTLS client-cert verification, **and** a bearer token on every request. Fans out: traces to Jaeger, metrics to InfluxDB. - **Jaeger** (`jaeger`, UI at `http://localhost:${JAEGER_QUERY_PORT}`, default `16686`) — receives spans over OTLP gRPC on `JAEGER_OTLP_GRPC_PORT`. +- **InfluxDB** (`influxdb`, port `INFLUXDB_PORT`, default `8181`) — InfluxDB 3 Core. The collector writes metrics into database `otel` using the `telegraf-prometheus-v1` schema: one measurement per metric name, with resource, scope, and datapoint attributes as tags. Query it with `POST /api/v3/query_sql`, passing `{"db": "otel", "q": ""}` and the admin token as a bearer. The container keeps its data in its own layer, so `down` discards the metrics and the admin token together. ### Credentials @@ -73,6 +75,7 @@ The compose bundle ships an OTLP-gRPC collector fronting a Jaeger v2 backend so - `credentials/otel-collector/cert.pem` + `key.pem` — server-side TLS material (fetched by the collector at startup, not needed by clients). - `credentials/otel-collector/client-cert.pem` + `client-key.pem` — client-side mTLS material every exporter must present. - `credentials/otel-collector/token` — bearer token required on every OTLP request. +- `credentials/influxdb/token` — InfluxDB 3 admin token, minted by the setup container against the running `influxdb` service. Required on every query. InfluxDB mints exactly one admin token per data directory and rejects a second create, so if `setup` is recreated while `influxdb` survives, minting fails with a 409 — recover with `docker compose --env-file ./testing.env down -v` then up. The server cert's SANs cover `localhost`, `docker`, and `otel-collector`, so the same cert verifies from host tests, DinD, and in-network containers. @@ -80,4 +83,8 @@ The server cert's SANs cover `localhost`, `docker`, and `otel-collector`, so the `testing::credentials::otel::OtelCredentials` downloads the tarball and hands back an `OtelCollector` with `grpc_endpoint`, `jaeger_query_endpoint`, `ca_cert_path`, `bearer_token_path`, `client_cert_path`, and `client_key_path` — wire these into an OTLP exporter to emit spans, and hit `jaeger_query_endpoint` to assert on them. +`switchgear_testing::credentials::influx::InfluxCredentials` is the metrics counterpart: it hands back an `Influx` with `query_endpoint`, `token`, and `token_path`. Keep the `InfluxCredentials` (and `OtelCredentials`) value bound for as long as you use the paths — each owns the `TempDir` those paths point into. +`switchgear_testing::influx::InfluxClient::new(query_endpoint, token)` queries the metrics database. `query` issues a single `POST /api/v3/query_sql` and returns the rows as `serde_json::Value`; `wait_for_rows` and `wait_for` poll until a row (optionally, a matching row) appears, retrying transport failures, non-2xx responses, and empty results alike — InfluxDB creates databases and tables lazily, so a missing table is a normal transient state. Defaults are database `otel`, a 30s timeout and a 500ms poll interval, each overridable with `with_db` / `with_timeout` / `with_poll_interval`. + +`server/tests/features/otlp_metrics.rs` is the live example: it stamps the child with a random `test_identity` resource attribute, exercises an invoice, and queries the histogram back out of InfluxDB. diff --git a/testing/docker-compose.yml b/testing/docker-compose.yml index 6a2d143..9de7cd8 100644 --- a/testing/docker-compose.yml +++ b/testing/docker-compose.yml @@ -32,6 +32,30 @@ services: retries: 5 start_period: 30s + influxdb: + image: influxdb:3.8-core + container_name: influxdb + hostname: "${INFLUXDB_HOSTNAME}" + command: + - influxdb3 + - serve + - --node-id=node0 + - --object-store=file + - --data-dir=/var/lib/influxdb3 + - --disable-authz=health + - --http-bind=0.0.0.0:${INFLUXDB_PORT} + ports: + - "${INFLUXDB_PORT}:${INFLUXDB_PORT}" + networks: + - services + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:${INFLUXDB_PORT}/health"] + interval: 5s + timeout: 5s + retries: 12 + start_period: 15s + setup: build: ./setup container_name: setup @@ -46,6 +70,11 @@ services: condition: service_healthy mysql: condition: service_healthy + influxdb: + condition: service_healthy + environment: + INFLUXDB_HOSTNAME: "${INFLUXDB_HOSTNAME}" + INFLUXDB_PORT: "${INFLUXDB_PORT}" volumes: - /var/run/docker.sock:/var/run/docker.sock - shared:/shared @@ -226,9 +255,13 @@ services: # Needed by entrypoint.sh to fetch the TLS cert + key + bearer token. - CREDENTIALS_SERVER_HOSTNAME=${CREDENTIALS_SERVER_HOSTNAME} - CREDENTIALS_SERVER_PORT=8888 + - INFLUXDB_HOSTNAME=${INFLUXDB_HOSTNAME} + - INFLUXDB_PORT=${INFLUXDB_PORT} depends_on: jaeger-health: condition: service_healthy + influxdb: + condition: service_healthy # The collector fetches its TLS material + bearer token from the # credentials-server, so it must be up before the collector starts. credentials-server: diff --git a/testing/otel-collector/entrypoint.sh b/testing/otel-collector/entrypoint.sh index bf55e54..9286238 100644 --- a/testing/otel-collector/entrypoint.sh +++ b/testing/otel-collector/entrypoint.sh @@ -9,5 +9,6 @@ wget -qO /tmp/otel_collector_cert "${CS}/credentials/otel-collector/cert.pem" wget -qO /tmp/otel_collector_key "${CS}/credentials/otel-collector/key.pem" wget -qO /tmp/otel_collector_client_ca "${CS}/credentials/otel-collector/ca.pem" wget -qO /tmp/otel_collector_token "${CS}/credentials/otel-collector/token" +wget -qO /tmp/influxdb_token "${CS}/credentials/influxdb/token" exec /otelcol-contrib --config /etc/otelcol-contrib/config.yaml diff --git a/testing/otel-collector/otel-collector-config.yml b/testing/otel-collector/otel-collector-config.yml index 9ba4a8f..e951ad7 100644 --- a/testing/otel-collector/otel-collector-config.yml +++ b/testing/otel-collector/otel-collector-config.yml @@ -7,6 +7,12 @@ extensions: bearertokenauth: scheme: Bearer filename: /tmp/otel_collector_token + # Bearer-token auth for the InfluxDB exporter. Distinct from the unnamed + # `bearertokenauth` above (the OTLP receiver's): this is the InfluxDB 3 admin + # token minted by the setup container, fetched at startup by entrypoint.sh. + bearertokenauth/influxdb: + scheme: Bearer + filename: /tmp/influxdb_token receivers: # TLS-only gRPC receiver — there is no cleartext transport. The cert + key @@ -32,9 +38,22 @@ exporters: tls: insecure: true + # `bucket` maps to an InfluxDB 3 database name, so queries pass + # `{"db": "otel"}`. The telegraf-prometheus-v1 schema writes one measurement + # per metric name, with resource + scope + datapoint attributes as tags. + influxdb: + endpoint: http://${env:INFLUXDB_HOSTNAME}:${env:INFLUXDB_PORT} + bucket: otel + metrics_schema: telegraf-prometheus-v1 + auth: + authenticator: bearertokenauth/influxdb + service: - extensions: [health_check, bearertokenauth] + extensions: [health_check, bearertokenauth, bearertokenauth/influxdb] pipelines: + metrics: + receivers: [otlp] + exporters: [influxdb] traces: receivers: [otlp] exporters: [otlp_grpc/jaeger] diff --git a/testing/setup/Dockerfile b/testing/setup/Dockerfile index 8814f0b..a3462bc 100644 --- a/testing/setup/Dockerfile +++ b/testing/setup/Dockerfile @@ -1,6 +1,6 @@ FROM docker:24-cli -RUN apk add --no-cache jq openssl +RUN apk add --no-cache jq openssl curl COPY init.sh /init.sh COPY ln.sh /ln.sh diff --git a/testing/setup/credentials.sh b/testing/setup/credentials.sh index 3a0ba8b..1a7d48d 100755 --- a/testing/setup/credentials.sh +++ b/testing/setup/credentials.sh @@ -8,6 +8,7 @@ mkdir -p "$CREDS_DIR/lnd" mkdir -p "$CREDS_DIR/postgres" mkdir -p "$CREDS_DIR/mysql" mkdir -p "$CREDS_DIR/otel-collector" +mkdir -p "$CREDS_DIR/influxdb" CLN_PUBKEY=$(docker exec cln-regtest lightning-cli --regtest getinfo | jq -r ".id") echo "$CLN_PUBKEY" > "$CREDS_DIR/cln/node_id" @@ -62,6 +63,13 @@ rm -f "$OTEL_DIR/ca.key" "$OTEL_DIR/server.csr" "$OTEL_DIR/server.ext" \ # the credentials tarball. openssl rand -hex 32 > "$OTEL_DIR/token" +# InfluxDB 3 mints one admin token per data directory and rejects a second +# create. A 409 here means the influxdb container outlived a `setup` recreate; +# recover with `docker compose --env-file ./testing.env down -v` then up. +curl -sS -f -X POST \ + "http://${INFLUXDB_HOSTNAME}:${INFLUXDB_PORT}/api/v3/configure/token/admin" \ + | jq -r '.token' > "$CREDS_DIR/influxdb/token" + chmod -R 644 "$CREDS_DIR" chmod -R +X "$CREDS_DIR" diff --git a/testing/src/credentials/influx.rs b/testing/src/credentials/influx.rs new file mode 100644 index 0000000..fa9f8e0 --- /dev/null +++ b/testing/src/credentials/influx.rs @@ -0,0 +1,53 @@ +use crate::credentials::download_credentials; +use crate::services::IntegrationTestServices; +use anyhow::{Context, anyhow}; +use std::path::PathBuf; +use tempfile::TempDir; + +#[derive(Clone, Debug)] +pub struct Influx { + pub query_endpoint: String, + pub token: String, + pub token_path: PathBuf, +} + +pub struct InfluxCredentials { + credentials_dir: TempDir, + query_endpoint: String, +} + +impl InfluxCredentials { + pub fn create() -> anyhow::Result { + let services = IntegrationTestServices::new(); + let credentials_dir = TempDir::new()?; + download_credentials(credentials_dir.path(), services.credentials())?; + Ok(Self { + credentials_dir, + query_endpoint: services.influx().query_endpoint.clone(), + }) + } + + pub fn get_influx(&self) -> anyhow::Result { + let token_path = self + .credentials_dir + .path() + .join("credentials") + .join("influxdb") + .join("token"); + let token_path = token_path.canonicalize().with_context(|| { + format!("canonicalizing influx token path {}", token_path.display()) + })?; + let token = std::fs::read_to_string(&token_path) + .with_context(|| format!("reading influx token {}", token_path.display()))? + .trim() + .to_owned(); + if token.is_empty() { + return Err(anyhow!("influx token at {} is empty", token_path.display())); + } + Ok(Influx { + query_endpoint: self.query_endpoint.clone(), + token, + token_path, + }) + } +} diff --git a/testing/src/credentials/mod.rs b/testing/src/credentials/mod.rs index 3f83968..548b7ad 100644 --- a/testing/src/credentials/mod.rs +++ b/testing/src/credentials/mod.rs @@ -5,6 +5,7 @@ use std::path::Path; use tar::Archive; pub mod db; +pub mod influx; pub mod lightning; pub mod otel; diff --git a/testing/src/influx.rs b/testing/src/influx.rs new file mode 100644 index 0000000..51e034d --- /dev/null +++ b/testing/src/influx.rs @@ -0,0 +1,103 @@ +use anyhow::{Context, anyhow}; +use serde_json::Value; +use std::time::{Duration, Instant}; + +const DEFAULT_DB: &str = "otel"; +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); +const DEFAULT_POLL_INTERVAL: Duration = Duration::from_millis(500); + +#[derive(Clone, Debug)] +pub struct InfluxClient { + client: reqwest::Client, + base_url: String, + token: String, + db: String, + timeout: Duration, + poll_interval: Duration, +} + +impl InfluxClient { + pub fn new(base_url: impl Into, token: impl Into) -> Self { + Self { + client: reqwest::Client::new(), + base_url: base_url.into().trim_end_matches('/').to_string(), + token: token.into(), + db: DEFAULT_DB.to_string(), + timeout: DEFAULT_TIMEOUT, + poll_interval: DEFAULT_POLL_INTERVAL, + } + } + + pub fn with_db(mut self, db: impl Into) -> Self { + self.db = db.into(); + self + } + + pub fn with_timeout(mut self, timeout: Duration) -> Self { + self.timeout = timeout; + self + } + + pub fn with_poll_interval(mut self, poll_interval: Duration) -> Self { + self.poll_interval = poll_interval; + self + } + + /// Single `POST /api/v3/query_sql`. Errors on transport failure or non-2xx. + pub async fn query(&self, sql: &str) -> anyhow::Result> { + let url = format!("{}/api/v3/query_sql", self.base_url); + let response = self + .client + .post(&url) + .bearer_auth(&self.token) + .json(&serde_json::json!({ "db": self.db, "q": sql })) + .send() + .await + .with_context(|| format!("POST {url}"))?; + + let status = response.status(); + let body = response + .text() + .await + .with_context(|| format!("reading body from {url}"))?; + if !status.is_success() { + return Err(anyhow!("InfluxDB status={status} body={body} sql={sql}")); + } + + serde_json::from_str::(&body) + .with_context(|| format!("parsing InfluxDB JSON: {body}"))? + .as_array() + .cloned() + .ok_or_else(|| anyhow!("InfluxDB returned a non-array body: {body}")) + } + + /// Poll until `sql` returns at least one row. + pub async fn wait_for_rows(&self, sql: &str) -> anyhow::Result> { + self.wait_for(sql, |_| true).await + } + + /// Poll until at least one returned row satisfies `predicate`. InfluxDB + /// creates databases and tables lazily, so every failure — transport, + /// non-2xx, or empty result — is retried until the deadline. + pub async fn wait_for(&self, sql: &str, predicate: F) -> anyhow::Result> + where + F: Fn(&Value) -> bool, + { + let deadline = Instant::now() + self.timeout; + let mut last_error; + loop { + match self.query(sql).await { + Ok(rows) if rows.iter().any(&predicate) => return Ok(rows), + Ok(_) => last_error = format!("no row matched, sql={sql}"), + Err(e) => last_error = format!("{e:#}"), + } + if Instant::now() >= deadline { + return Err(anyhow!( + "InfluxDB wait timed out after {:?}: {last_error}", + self.timeout + )); + } + tokio::time::sleep(self.poll_interval).await; + } + } +} diff --git a/testing/src/lib.rs b/testing/src/lib.rs index 2d1e647..04bf24a 100644 --- a/testing/src/lib.rs +++ b/testing/src/lib.rs @@ -1,4 +1,5 @@ pub mod credentials; pub mod db; +pub mod influx; pub mod ports; pub mod services; diff --git a/testing/src/services.rs b/testing/src/services.rs index ec74a49..c8c2309 100644 --- a/testing/src/services.rs +++ b/testing/src/services.rs @@ -9,6 +9,7 @@ pub struct IntegrationTestServices { mysql: String, lightning: LightningIntegrationTestServices, otel: OtelIntegrationTestServices, + influx: InfluxIntegrationTestServices, } #[derive(Debug, Clone)] @@ -17,6 +18,11 @@ pub struct OtelIntegrationTestServices { pub jaeger_query_endpoint: String, } +#[derive(Debug, Clone)] +pub struct InfluxIntegrationTestServices { + pub query_endpoint: String, +} + #[derive(Debug, Clone)] pub struct LightningIntegrationTestServices { pub cln: String, @@ -75,6 +81,14 @@ impl IntegrationTestServices { Self::env_or_panic("JAEGER_QUERY_PORT") ); + let influx = InfluxIntegrationTestServices { + query_endpoint: format!( + "http://{}:{}", + Self::env_or_panic("INFLUXDB_HOSTNAME"), + Self::env_or_panic("INFLUXDB_PORT") + ), + }; + Self { credentials, postgres, @@ -84,9 +98,15 @@ impl IntegrationTestServices { grpc_endpoint, jaeger_query_endpoint, }, + influx, } } + #[expect( + clippy::panic, + reason = "missing integration-test env var must fail \ + loudly with setup instructions" + )] fn env_or_panic(config_env: &str) -> String { env::var(config_env).unwrap_or_else(|_| { panic!( @@ -122,4 +142,8 @@ See testing/README.md to configure integration tests and services. pub fn otel(&self) -> &OtelIntegrationTestServices { &self.otel } + + pub fn influx(&self) -> &InfluxIntegrationTestServices { + &self.influx + } } diff --git a/testing/testing.env b/testing/testing.env index b229e1b..bcdbc1b 100644 --- a/testing/testing.env +++ b/testing/testing.env @@ -20,4 +20,7 @@ JAEGER_HOSTNAME=jaeger JAEGER_QUERY_PORT=16686 JAEGER_OTLP_GRPC_PORT=4317 -SERVICES_NETWORK_NAME=services_network \ No newline at end of file +SERVICES_NETWORK_NAME=services_network + +INFLUXDB_HOSTNAME=influxdb +INFLUXDB_PORT=8181