From 1ba1e0f9949cc5bbd394aee15564688666f524c6 Mon Sep 17 00:00:00 2001 From: chillin_since_26 Date: Thu, 5 Nov 2026 18:46:49 +0100 Subject: [PATCH 1/2] feat(node): add authenticated node health route --- CHANGELOG.md | 8 ++ api/components/schemas/access/NodeHealth.yaml | 27 +++++ .../schemas/access/RegistryOutboxHealth.yaml | 25 +++++ api/openapi.bundled.json | 91 +++++++++++++++++ api/openapi.bundled.yaml | 79 +++++++++++++++ api/openapi.yaml | 6 ++ api/paths/vault/vault_api_v1_node_health.yaml | 24 +++++ crates/dpp-common/src/metrics.rs | 55 +++++++++++ crates/dpp-node/src/boot/tasks.rs | 3 +- crates/dpp-node/src/infra/ruleset.rs | 2 +- crates/dpp-node/tests/openapi_contract.rs | 23 +++++ crates/dpp-node/tests/smoke.rs | 30 ++++++ crates/dpp-types/src/registry_sync.rs | 5 + .../dpp-vault/src/domain/service/publish.rs | 6 +- crates/dpp-vault/src/handlers/mod.rs | 1 + crates/dpp-vault/src/handlers/node_health.rs | 98 +++++++++++++++++++ crates/dpp-vault/src/router.rs | 2 + 17 files changed, 479 insertions(+), 6 deletions(-) create mode 100644 api/components/schemas/access/NodeHealth.yaml create mode 100644 api/components/schemas/access/RegistryOutboxHealth.yaml create mode 100644 api/paths/vault/vault_api_v1_node_health.yaml create mode 100644 crates/dpp-vault/src/handlers/node_health.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index be48fc70..57d3e09e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,14 @@ under the pre-1.0 conventions in [VERSIONING.md](docs/governance/VERSIONING.md): ### Added +- **An authenticated node health route reports the counters an operator pages + on.** `GET /vault/api/v1/node/health` returns the publishes refused for a failed + signature and the ruleset bundles refused on load, both since the node process + started, and the registry-sync outbox's pending, submitted, stalled and rejected + counts. A monitor that cannot scrape `/metrics` can read them with an API key. + The unauthenticated `/health` still answers liveness only. The two process + counters reset on restart; the outbox does not. (#138) + - **An authority or a person with a legitimate interest can read the passport as it stood at a past moment.** EN 18221:2026 clause 4.2 expects the version of a passport at a past moment to be retrievable by authenticated, authorised diff --git a/api/components/schemas/access/NodeHealth.yaml b/api/components/schemas/access/NodeHealth.yaml new file mode 100644 index 00000000..4f455f57 --- /dev/null +++ b/api/components/schemas/access/NodeHealth.yaml @@ -0,0 +1,27 @@ +type: object +required: + - signingFailuresTotal + - rulesetLoadFailuresTotal +properties: + signingFailuresTotal: + type: integer + format: int64 + minimum: 0 + description: | + Publishes refused because a passport could not be signed, since the node + process started. A restart resets it, so a restart hides failures from + before it. + example: 0 + rulesetLoadFailuresTotal: + type: integer + format: int64 + minimum: 0 + description: | + Compliance Current ruleset bundles refused on load, since the node process + started. A restart resets it. + example: 0 + registryOutbox: + $ref: ./RegistryOutboxHealth.yaml + description: | + The registry-sync outbox's counts. Absent on a deployment with no outbox + wired, such as a standalone vault. diff --git a/api/components/schemas/access/RegistryOutboxHealth.yaml b/api/components/schemas/access/RegistryOutboxHealth.yaml new file mode 100644 index 00000000..34e767b2 --- /dev/null +++ b/api/components/schemas/access/RegistryOutboxHealth.yaml @@ -0,0 +1,25 @@ +type: object +required: + - pending + - submitted + - stalled + - rejected +properties: + pending: + type: integer + format: int64 + description: Rows awaiting a drain attempt. + submitted: + type: integer + format: int64 + description: Rows accepted by the registry and awaiting its verdict. + stalled: + type: integer + format: int64 + description: | + Pending rows that have reached the stall threshold. A row here is not + going to succeed without someone looking at it. + rejected: + type: integer + format: int64 + description: Rows the registry terminally rejected. diff --git a/api/openapi.bundled.json b/api/openapi.bundled.json index 6576116f..295768a6 100644 --- a/api/openapi.bundled.json +++ b/api/openapi.bundled.json @@ -3576,6 +3576,39 @@ } } }, + "/vault/api/v1/node/health": { + "get": { + "operationId": "getNodeHealth", + "summary": "Node operational health counters", + "description": "Reports the counters an operator pages on: publishes refused for a failed\nsignature and ruleset bundles refused on load, both since the node process\nstarted, and the registry-sync outbox's pending, submitted, stalled and\nrejected counts. Authenticated, so a monitor that cannot scrape `/metrics`\ncan read them with an API key. The unauthenticated `/health` answers\nliveness only.\n", + "tags": [ + "Node" + ], + "security": [ + { + "BearerApiKey": [] + }, + { + "BasicAuth": [] + } + ], + "responses": { + "200": { + "description": "Node operational health counters.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NodeHealth" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + } + } + } + }, "/vault/api/v1/whoami": { "get": { "operationId": "whoami", @@ -9495,6 +9528,64 @@ } } }, + "NodeHealth": { + "type": "object", + "required": [ + "signingFailuresTotal", + "rulesetLoadFailuresTotal" + ], + "properties": { + "signingFailuresTotal": { + "type": "integer", + "format": "int64", + "minimum": 0, + "description": "Publishes refused because a passport could not be signed, since the node\nprocess started. A restart resets it, so a restart hides failures from\nbefore it.\n", + "example": 0 + }, + "rulesetLoadFailuresTotal": { + "type": "integer", + "format": "int64", + "minimum": 0, + "description": "Compliance Current ruleset bundles refused on load, since the node process\nstarted. A restart resets it.\n", + "example": 0 + }, + "registryOutbox": { + "$ref": "#/components/schemas/RegistryOutboxHealth", + "description": "The registry-sync outbox's counts. Absent on a deployment with no outbox\nwired, such as a standalone vault.\n" + } + } + }, + "RegistryOutboxHealth": { + "type": "object", + "required": [ + "pending", + "submitted", + "stalled", + "rejected" + ], + "properties": { + "pending": { + "type": "integer", + "format": "int64", + "description": "Rows awaiting a drain attempt." + }, + "submitted": { + "type": "integer", + "format": "int64", + "description": "Rows accepted by the registry and awaiting its verdict." + }, + "stalled": { + "type": "integer", + "format": "int64", + "description": "Pending rows that have reached the stall threshold. A row here is not\ngoing to succeed without someone looking at it.\n" + }, + "rejected": { + "type": "integer", + "format": "int64", + "description": "Rows the registry terminally rejected." + } + } + }, "NodeProfile": { "type": "string", "description": "The deployment profile a node was started under (`NODE_PROFILE`), which decides the trust tiers it will boot on. `development` boots on any tier, placeholders included, and is what an unset or unrecognised value means. `sandbox` is a real deployment against test authorities: it refuses a placeholder on a required trust port and accepts a sandbox tier. `production` accepts only live tiers on required ports.", diff --git a/api/openapi.bundled.yaml b/api/openapi.bundled.yaml index d5a56908..fd5f0c65 100644 --- a/api/openapi.bundled.yaml +++ b/api/openapi.bundled.yaml @@ -2978,6 +2978,31 @@ paths: $ref: '#/components/schemas/NodeState' '401': $ref: '#/components/responses/Unauthorized' + /vault/api/v1/node/health: + get: + operationId: getNodeHealth + summary: Node operational health counters + description: | + Reports the counters an operator pages on: publishes refused for a failed + signature and ruleset bundles refused on load, both since the node process + started, and the registry-sync outbox's pending, submitted, stalled and + rejected counts. Authenticated, so a monitor that cannot scrape `/metrics` + can read them with an API key. The unauthenticated `/health` answers + liveness only. + tags: + - Node + security: + - BearerApiKey: [] + - BasicAuth: [] + responses: + '200': + description: Node operational health counters. + content: + application/json: + schema: + $ref: '#/components/schemas/NodeHealth' + '401': + $ref: '#/components/responses/Unauthorized' /vault/api/v1/whoami: get: operationId: whoami @@ -8247,6 +8272,60 @@ components: the rules a passport was checked under are observable. Absent on a standalone vault. example: baseline + NodeHealth: + type: object + required: + - signingFailuresTotal + - rulesetLoadFailuresTotal + properties: + signingFailuresTotal: + type: integer + format: int64 + minimum: 0 + description: | + Publishes refused because a passport could not be signed, since the node + process started. A restart resets it, so a restart hides failures from + before it. + example: 0 + rulesetLoadFailuresTotal: + type: integer + format: int64 + minimum: 0 + description: | + Compliance Current ruleset bundles refused on load, since the node process + started. A restart resets it. + example: 0 + registryOutbox: + $ref: '#/components/schemas/RegistryOutboxHealth' + description: | + The registry-sync outbox's counts. Absent on a deployment with no outbox + wired, such as a standalone vault. + RegistryOutboxHealth: + type: object + required: + - pending + - submitted + - stalled + - rejected + properties: + pending: + type: integer + format: int64 + description: Rows awaiting a drain attempt. + submitted: + type: integer + format: int64 + description: Rows accepted by the registry and awaiting its verdict. + stalled: + type: integer + format: int64 + description: | + Pending rows that have reached the stall threshold. A row here is not + going to succeed without someone looking at it. + rejected: + type: integer + format: int64 + description: Rows the registry terminally rejected. NodeProfile: type: string description: 'The deployment profile a node was started under (`NODE_PROFILE`), which decides the trust tiers it will boot on. `development` boots on any tier, placeholders included, and is what an unset or unrecognised value means. `sandbox` is a real deployment against test authorities: it refuses a placeholder on a required trust port and accepts a sandbox tier. `production` accepts only live tiers on required ports.' diff --git a/api/openapi.yaml b/api/openapi.yaml index 32186de0..5fdae0f8 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -287,6 +287,10 @@ components: $ref: components/schemas/access/InstalledPlugin.yaml NodeState: $ref: components/schemas/access/NodeState.yaml + NodeHealth: + $ref: components/schemas/access/NodeHealth.yaml + RegistryOutboxHealth: + $ref: components/schemas/access/RegistryOutboxHealth.yaml NodeProfile: $ref: components/schemas/access/NodeProfile.yaml WhoamiResponse: @@ -446,6 +450,8 @@ paths: $ref: paths/vault/vault_api_v1_ruleset_reload.yaml /vault/api/v1/node/state: $ref: paths/vault/vault_api_v1_node_state.yaml + /vault/api/v1/node/health: + $ref: paths/vault/vault_api_v1_node_health.yaml /vault/api/v1/whoami: $ref: paths/vault/vault_api_v1_whoami.yaml /vault/public/dpp/{dppId}: diff --git a/api/paths/vault/vault_api_v1_node_health.yaml b/api/paths/vault/vault_api_v1_node_health.yaml new file mode 100644 index 00000000..e3ae4ca0 --- /dev/null +++ b/api/paths/vault/vault_api_v1_node_health.yaml @@ -0,0 +1,24 @@ +get: + operationId: getNodeHealth + summary: Node operational health counters + description: | + Reports the counters an operator pages on: publishes refused for a failed + signature and ruleset bundles refused on load, both since the node process + started, and the registry-sync outbox's pending, submitted, stalled and + rejected counts. Authenticated, so a monitor that cannot scrape `/metrics` + can read them with an API key. The unauthenticated `/health` answers + liveness only. + tags: + - Node + security: + - BearerApiKey: [] + - BasicAuth: [] + responses: + '200': + description: Node operational health counters. + content: + application/json: + schema: + $ref: ../../components/schemas/access/NodeHealth.yaml + '401': + $ref: ../../components/responses/Unauthorized.yaml diff --git a/crates/dpp-common/src/metrics.rs b/crates/dpp-common/src/metrics.rs index fb5c0f44..4f7e3ca9 100644 --- a/crates/dpp-common/src/metrics.rs +++ b/crates/dpp-common/src/metrics.rs @@ -17,6 +17,7 @@ //! breaking change for anyone already scraping it, so this is flagged, not //! fixed, here. +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Instant; use axum::{extract::Request, middleware::Next, response::Response}; @@ -62,3 +63,57 @@ pub async fn http_metrics_middleware(request: Request, next: Next) -> Response { response } + +/// Signing failures since this process started. The Prometheus recorder cannot be +/// read back, so the node-health route reads these instead. +static SIGNING_FAILURES: AtomicU64 = AtomicU64::new(0); +/// Ruleset bundles refused on load since this process started, as above. +static RULESET_LOAD_FAILURES: AtomicU64 = AtomicU64::new(0); + +/// Count one publish refused because a passport could not be signed. +/// +/// Every increment goes through here, not `metrics::counter!` directly, so the +/// count `/metrics` exports and the one the node-health route reports cannot drift. +pub fn record_signing_failure() { + SIGNING_FAILURES.fetch_add(1, Ordering::Relaxed); + metrics::counter!("signing_failures_total").increment(1); +} + +/// Count one ruleset bundle refused on load. See [`record_signing_failure`]. +pub fn record_ruleset_load_failure() { + RULESET_LOAD_FAILURES.fetch_add(1, Ordering::Relaxed); + metrics::counter!("ruleset_load_failures_total").increment(1); +} + +/// Signing failures since process start. A restart resets it. +#[must_use] +pub fn signing_failures() -> u64 { + SIGNING_FAILURES.load(Ordering::Relaxed) +} + +/// Ruleset bundles refused since process start. A restart resets it. +#[must_use] +pub fn ruleset_load_failures() -> u64 { + RULESET_LOAD_FAILURES.load(Ordering::Relaxed) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A refused signature shows up on the readable count, not only on the + /// exported counter the route cannot read. + #[test] + fn a_recorded_signing_failure_is_counted_where_the_route_reads_it() { + let before = signing_failures(); + record_signing_failure(); + assert_eq!(signing_failures(), before + 1); + } + + #[test] + fn a_recorded_ruleset_failure_is_counted_where_the_route_reads_it() { + let before = ruleset_load_failures(); + record_ruleset_load_failure(); + assert_eq!(ruleset_load_failures(), before + 1); + } +} diff --git a/crates/dpp-node/src/boot/tasks.rs b/crates/dpp-node/src/boot/tasks.rs index 8d59447a..897f6ef8 100644 --- a/crates/dpp-node/src/boot/tasks.rs +++ b/crates/dpp-node/src/boot/tasks.rs @@ -8,7 +8,7 @@ use dpp_domain::ports::passport_repo::PassportRepository; use dpp_domain::ports::registry_sync::RegistrySyncPort; use dpp_domain::ports::seal::SealPort; use dpp_integrator::infra::job_store::JobStore; -use dpp_types::registry_sync::{RegistrySyncCounts, RegistrySyncOutbox}; +use dpp_types::registry_sync::{RegistrySyncCounts, RegistrySyncOutbox, STALL_THRESHOLD}; use dpp_types::registry_transfer::{RegistryTransferCounts, RegistryTransferOutbox}; use dpp_types::scan::ScanTelemetryRepository; use dpp_types::seal::{SealOutbox, SealOutboxCounts}; @@ -88,7 +88,6 @@ const DRAIN_BATCH: i64 = 50; /// (one statement, no object-storage work); the drain still paces the actual /// reconciles at its own batch size. const SWEEP_BATCH: i64 = 500; -const STALL_THRESHOLD: i32 = 8; /// Reflect the registry-sync outbox's counts onto its gauges. Shared by the /// boot-time reconciliation log and every post-drain re-check so the two diff --git a/crates/dpp-node/src/infra/ruleset.rs b/crates/dpp-node/src/infra/ruleset.rs index 8710e3e0..524b3d38 100644 --- a/crates/dpp-node/src/infra/ruleset.rs +++ b/crates/dpp-node/src/infra/ruleset.rs @@ -514,7 +514,7 @@ pub async fn reload_and_report(active: &ActiveRuleset, at_boot: bool) { // `from_env` has already said so once at boot. Err(RulesetReloadError::NotConfigured) => {} Err(e) => { - metrics::counter!("ruleset_load_failures_total").increment(1); + dpp_common::metrics::record_ruleset_load_failure(); tracing::error!( code = event_codes::RULESET_LOAD_FAILED, error = %e, diff --git a/crates/dpp-node/tests/openapi_contract.rs b/crates/dpp-node/tests/openapi_contract.rs index 0076f738..ef6a4065 100644 --- a/crates/dpp-node/tests/openapi_contract.rs +++ b/crates/dpp-node/tests/openapi_contract.rs @@ -576,6 +576,8 @@ fn object_cases() -> Vec { case!("AmendRequest", fixtures::amend_request()); case!("SupersedeRequest", fixtures::supersede_request()); case!("NodeState", fixtures::node_state()); + case!("NodeHealth", fixtures::node_health()); + case!("RegistryOutboxHealth", fixtures::registry_outbox_health()); case!("VaultInfo", fixtures::vault_info()); case!( "TransferInitiateRequest", @@ -2506,6 +2508,7 @@ mod handler_sources { include_str!("../../dpp-vault/src/handlers/info.rs"), include_str!("../../dpp-vault/src/handlers/lint.rs"), include_str!("../../dpp-vault/src/handlers/list.rs"), + include_str!("../../dpp-vault/src/handlers/node_health.rs"), include_str!("../../dpp-vault/src/handlers/node_state.rs"), include_str!("../../dpp-vault/src/handlers/operator.rs"), include_str!("../../dpp-vault/src/handlers/personal_data.rs"), @@ -3014,6 +3017,7 @@ mod fixtures { eol::EolRequest, info::VaultInfo, list::PassportListResponse, + node_health::{NodeHealth, RegistryOutboxHealth}, node_state::NodeState, personal_data::PersonalDataErasureReceipt, registry_abandon::{AbandonRegistrationRequest, AbandonRegistrationResponse}, @@ -3858,6 +3862,25 @@ mod fixtures { } } + /// The node-health counters with an outbox wired, the shape a Postgres node + /// serves. The values are fixed and non-zero so each field is exercised. + pub fn node_health() -> NodeHealth { + NodeHealth { + signing_failures_total: 1, + ruleset_load_failures_total: 0, + registry_outbox: Some(registry_outbox_health()), + } + } + + pub fn registry_outbox_health() -> RegistryOutboxHealth { + RegistryOutboxHealth { + pending: 2, + submitted: 1, + stalled: 0, + rejected: 0, + } + } + /// Every deployment profile, as the node-state route serialises it. /// /// Read through `posture_json`, the one producer of `profile` on that diff --git a/crates/dpp-node/tests/smoke.rs b/crates/dpp-node/tests/smoke.rs index 308261de..7e91a4ca 100644 --- a/crates/dpp-node/tests/smoke.rs +++ b/crates/dpp-node/tests/smoke.rs @@ -186,6 +186,8 @@ async fn start_node_with_ruleset( .with_transfer_store(Arc::new(PgTransferRepo::new(dal.clone()))) .with_evidence_store(Arc::new(PgEvidenceDossierRepo::new(dal.clone()))) .with_registry_reader(operator_repo.clone()) + // Wired as `boot::db` wires it, so the node-health route reads a real outbox. + .with_registry_outbox(Arc::new(dpp_dal::pg::PgRegistrySyncRepo::new(dal.clone()))) .with_versions(version_store) // Wired as `boot::db` wires it, so the public-read branch that resolves // a superseded carrier on to its successor is actually exercised here. @@ -353,6 +355,33 @@ async fn authenticated_node_state_reports_trust_modes() { assert_eq!(body["rulesetVersion"], "baseline"); } +/// The operational counters are served to an entitled caller, with the outbox +/// they describe read from the database. Only the shape is asserted: the two +/// process counters are shared by every test in this binary, so their values +/// are not this test's to fix. +#[tokio::test(flavor = "multi_thread")] +async fn authenticated_node_health_reports_the_counters_and_the_outbox() { + let (base, _c) = start_db_and_node().await; + let token = make_jwt("00000000-0000-0000-0000-000000000092"); + let resp = reqwest::Client::new() + .get(format!("{base}/vault/api/v1/node/health")) + .bearer_auth(&token) + .send() + .await + .expect("GET /vault/api/v1/node/health failed"); + assert_eq!(resp.status(), 200); + + let body: serde_json::Value = resp.json().await.expect("node health is JSON"); + assert!(body["signingFailuresTotal"].is_u64(), "{body}"); + assert!(body["rulesetLoadFailuresTotal"].is_u64(), "{body}"); + for field in ["pending", "submitted", "stalled", "rejected"] { + assert!( + body["registryOutbox"][field].is_i64(), + "`registryOutbox.{field}` must be reported on a Postgres node: {body}" + ); + } +} + /// The issue this route exists to close, end to end: a node that is up and /// serving adopts a newly published ruleset, and the version it reports moves. /// @@ -733,6 +762,7 @@ async fn route_inventory_matches_assembled_router() { format!("/vault/api/v1/dpp/{FAKE_ID}/verify-tree"), ), (reqwest::Method::GET, "/vault/api/v1/node/state".into()), + (reqwest::Method::GET, "/vault/api/v1/node/health".into()), (reqwest::Method::GET, "/vault/api/v1/operator".into()), (reqwest::Method::PATCH, "/vault/api/v1/operator".into()), (reqwest::Method::GET, "/vault/api/v1/api-keys".into()), diff --git a/crates/dpp-types/src/registry_sync.rs b/crates/dpp-types/src/registry_sync.rs index c00bb263..5bef71a0 100644 --- a/crates/dpp-types/src/registry_sync.rs +++ b/crates/dpp-types/src/registry_sync.rs @@ -185,6 +185,11 @@ pub struct RegistrySyncRow { pub next_attempt_at: DateTime, } +/// Attempts after which a pending registry row counts as stalled. The registration +/// and transfer outboxes both report their stalled rows at this threshold, on the +/// node's gauges, its alarms and `GET /vault/api/v1/node/health`. +pub const STALL_THRESHOLD: i32 = 8; + /// Aggregate counts for boot reconciliation and metrics. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct RegistrySyncCounts { diff --git a/crates/dpp-vault/src/domain/service/publish.rs b/crates/dpp-vault/src/domain/service/publish.rs index b3425928..ae97ea23 100644 --- a/crates/dpp-vault/src/domain/service/publish.rs +++ b/crates/dpp-vault/src/domain/service/publish.rs @@ -396,7 +396,7 @@ impl PassportService { .await .map(|c| c.jws) .map_err(|e| { - metrics::counter!("signing_failures_total").increment(1); + dpp_common::metrics::record_signing_failure(); metrics::counter!("passport_publish_rejected_total", "reason" => REASON_SIGNING_FAILED).increment(1); tracing::error!( code = event_codes::JWS_UNSIGNED_PUBLISH_BLOCKED, @@ -424,7 +424,7 @@ impl PassportService { .await .map(|c| c.jws) .map_err(|e| { - metrics::counter!("signing_failures_total").increment(1); + dpp_common::metrics::record_signing_failure(); metrics::counter!("passport_publish_rejected_total", "reason" => REASON_SIGNING_FAILED).increment(1); tracing::error!( code = event_codes::JWS_UNSIGNED_PUBLISH_BLOCKED, @@ -449,7 +449,7 @@ impl PassportService { ) .await .map_err(|e| { - metrics::counter!("signing_failures_total").increment(1); + dpp_common::metrics::record_signing_failure(); metrics::counter!("passport_publish_rejected_total", "reason" => REASON_SIGNING_FAILED) .increment(1); tracing::error!( diff --git a/crates/dpp-vault/src/handlers/mod.rs b/crates/dpp-vault/src/handlers/mod.rs index ecc9f074..95ae8e51 100644 --- a/crates/dpp-vault/src/handlers/mod.rs +++ b/crates/dpp-vault/src/handlers/mod.rs @@ -14,6 +14,7 @@ pub mod history; pub mod info; pub mod lint; pub mod list; +pub mod node_health; pub mod node_state; pub mod operator; pub mod personal_data; diff --git a/crates/dpp-vault/src/handlers/node_health.rs b/crates/dpp-vault/src/handlers/node_health.rs new file mode 100644 index 00000000..998a8aa5 --- /dev/null +++ b/crates/dpp-vault/src/handlers/node_health.rs @@ -0,0 +1,98 @@ +use axum::{ + Json, + extract::{Extension, State}, + http::StatusCode, + response::IntoResponse, +}; +use serde::Serialize; + +use dpp_types::registry_sync::STALL_THRESHOLD; + +use crate::{middleware::auth::AuthContext, state::AppState}; + +use super::error::internal_error; + +/// The registry-sync outbox's counts, as the node's gauges report them. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RegistryOutboxHealth { + /// Rows awaiting a drain attempt. + pub pending: i64, + /// Rows accepted by the registry and awaiting its verdict. + pub submitted: i64, + /// Pending rows that have reached the stall threshold and need a person. + pub stalled: i64, + /// Rows the registry terminally rejected. + pub rejected: i64, +} + +/// The counters an operator pages on. Public so the OpenAPI contract test can +/// serialise it, as [`super::node_state::NodeState`] is. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NodeHealth { + /// Publishes refused because a passport could not be signed, since the + /// process started. + pub signing_failures_total: u64, + /// Ruleset bundles refused on load, since the process started. + pub ruleset_load_failures_total: u64, + /// Absent on a deployment with no registry-sync outbox wired, rather than + /// reported as zeros a monitor would read as an empty queue. + #[serde(skip_serializing_if = "Option::is_none")] + pub registry_outbox: Option, +} + +/// `GET /api/v1/node/health` — the operational counters a monitor needs, for a +/// deployment that cannot scrape `/metrics`. Authenticated, for the same reason +/// as `/node/state`: a failing signer or a stalled queue is a description of +/// where the node is weak. The unauthenticated `/health` answers liveness only. +/// +/// The two process counters reset when the node restarts, so a restart hides +/// failures from before it. The outbox is read from the database and does not. +pub async fn node_health_handler( + State(state): State, + Extension(_auth): Extension, +) -> impl IntoResponse { + let registry_outbox = match &state.service.registry_outbox { + Some(outbox) => match outbox.status_counts(STALL_THRESHOLD).await { + Ok(c) => Some(RegistryOutboxHealth { + pending: c.pending, + submitted: c.submitted, + stalled: c.stalled, + rejected: c.rejected, + }), + Err(e) => return internal_error(e), + }, + None => None, + }; + + ( + StatusCode::OK, + Json(NodeHealth { + signing_failures_total: dpp_common::metrics::signing_failures(), + ruleset_load_failures_total: dpp_common::metrics::ruleset_load_failures(), + registry_outbox, + }), + ) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// With no outbox wired the block is omitted entirely; the counters stay. + #[test] + fn an_unwired_outbox_is_omitted_rather_than_reported_as_empty() { + let json = serde_json::to_value(NodeHealth { + signing_failures_total: 0, + ruleset_load_failures_total: 0, + registry_outbox: None, + }) + .expect("serialises"); + + assert!(json.get("registryOutbox").is_none(), "{json}"); + assert_eq!(json["signingFailuresTotal"], 0); + assert_eq!(json["rulesetLoadFailuresTotal"], 0); + } +} diff --git a/crates/dpp-vault/src/router.rs b/crates/dpp-vault/src/router.rs index d2191e70..b0d27f0b 100644 --- a/crates/dpp-vault/src/router.rs +++ b/crates/dpp-vault/src/router.rs @@ -35,6 +35,7 @@ use crate::{ info::info_handler, lint::lint_handler, list::list_handler, + node_health::node_health_handler, node_state::node_state_handler, operator::{operator_get_handler, operator_patch_handler}, personal_data::personal_data_erase_handler, @@ -163,6 +164,7 @@ pub fn build(state: AppState) -> Router { .route("/evidence/{id}/verify", post(verify_evidence_handler)) // ── Node setup state ────────────────────────────────────────── .route("/node/state", get(node_state_handler)) + .route("/node/health", get(node_health_handler)) // ── Caller identity ─────────────────────────────────────────── // Any scope: it reports only what the caller already presented, and a // read-only key needs it precisely to learn that it is read-only. From c94f955956d01004459031d91483f33df9d7babf Mon Sep 17 00:00:00 2001 From: chillin_since_26 Date: Thu, 5 Nov 2026 19:44:54 +0100 Subject: [PATCH 2/2] fix(node): correct stall docs and count reloads --- .../schemas/access/RegistryOutboxHealth.yaml | 4 +- api/openapi.bundled.json | 2 +- api/openapi.bundled.yaml | 4 +- crates/dpp-common/src/metrics.rs | 93 ++++++++++-- crates/dpp-node/tests/smoke.rs | 142 +++++++++++++++--- crates/dpp-types/src/registry_sync.rs | 10 +- crates/dpp-vault/src/handlers/node_health.rs | 3 +- crates/dpp-vault/src/handlers/ruleset.rs | 7 +- 8 files changed, 226 insertions(+), 39 deletions(-) diff --git a/api/components/schemas/access/RegistryOutboxHealth.yaml b/api/components/schemas/access/RegistryOutboxHealth.yaml index 34e767b2..2dc12090 100644 --- a/api/components/schemas/access/RegistryOutboxHealth.yaml +++ b/api/components/schemas/access/RegistryOutboxHealth.yaml @@ -17,8 +17,8 @@ properties: type: integer format: int64 description: | - Pending rows that have reached the stall threshold. A row here is not - going to succeed without someone looking at it. + Pending and submitted rows that have reached the stall threshold. A row here + is not going to succeed without someone looking at it. rejected: type: integer format: int64 diff --git a/api/openapi.bundled.json b/api/openapi.bundled.json index 295768a6..0f5d4b8c 100644 --- a/api/openapi.bundled.json +++ b/api/openapi.bundled.json @@ -9577,7 +9577,7 @@ "stalled": { "type": "integer", "format": "int64", - "description": "Pending rows that have reached the stall threshold. A row here is not\ngoing to succeed without someone looking at it.\n" + "description": "Pending and submitted rows that have reached the stall threshold. A row here\nis not going to succeed without someone looking at it.\n" }, "rejected": { "type": "integer", diff --git a/api/openapi.bundled.yaml b/api/openapi.bundled.yaml index fd5f0c65..ec2f3814 100644 --- a/api/openapi.bundled.yaml +++ b/api/openapi.bundled.yaml @@ -8320,8 +8320,8 @@ components: type: integer format: int64 description: | - Pending rows that have reached the stall threshold. A row here is not - going to succeed without someone looking at it. + Pending and submitted rows that have reached the stall threshold. A row here + is not going to succeed without someone looking at it. rejected: type: integer format: int64 diff --git a/crates/dpp-common/src/metrics.rs b/crates/dpp-common/src/metrics.rs index 4f7e3ca9..21eed157 100644 --- a/crates/dpp-common/src/metrics.rs +++ b/crates/dpp-common/src/metrics.rs @@ -99,21 +99,94 @@ pub fn ruleset_load_failures() -> u64 { #[cfg(test)] mod tests { + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + use metrics::{ + Counter, CounterFn, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit, + }; + use super::*; - /// A refused signature shows up on the readable count, not only on the - /// exported counter the route cannot read. + /// A recorder that totals every counter by name: the exported side of a + /// refusal, for a test to read. Gauges and histograms are not kept. + #[derive(Clone, Default)] + struct ExportedCounters(Arc>>); + + struct Increments { + name: String, + totals: Arc>>, + } + + impl CounterFn for Increments { + fn increment(&self, value: u64) { + *self + .totals + .lock() + .unwrap() + .entry(self.name.clone()) + .or_default() += value; + } + + fn absolute(&self, value: u64) { + self.totals.lock().unwrap().insert(self.name.clone(), value); + } + } + + impl Recorder for ExportedCounters { + fn describe_counter(&self, _: KeyName, _: Option, _: SharedString) {} + + fn describe_gauge(&self, _: KeyName, _: Option, _: SharedString) {} + + fn describe_histogram(&self, _: KeyName, _: Option, _: SharedString) {} + + fn register_counter(&self, key: &Key, _: &Metadata<'_>) -> Counter { + Counter::from_arc(Arc::new(Increments { + name: key.name().to_owned(), + totals: Arc::clone(&self.0), + })) + } + + fn register_gauge(&self, _: &Key, _: &Metadata<'_>) -> Gauge { + Gauge::noop() + } + + fn register_histogram(&self, _: &Key, _: &Metadata<'_>) -> Histogram { + Histogram::noop() + } + } + + /// How far the exported counter called `name` moves while `record` runs. + fn exported_increase(name: &str, record: fn()) -> u64 { + let exported = ExportedCounters::default(); + metrics::with_local_recorder(&exported, record); + exported.0.lock().unwrap().get(name).copied().unwrap_or(0) + } + + /// Each refusal moves the exported counter `/metrics` serves and the readable + /// count the route reports by one each, so the two cannot drift apart. #[test] - fn a_recorded_signing_failure_is_counted_where_the_route_reads_it() { - let before = signing_failures(); - record_signing_failure(); - assert_eq!(signing_failures(), before + 1); + fn a_refused_signature_moves_the_exported_and_the_readable_count() { + let readable = signing_failures(); + let exported = exported_increase("signing_failures_total", record_signing_failure); + assert_eq!(exported, 1, "the exported counter moves once per refusal"); + assert_eq!( + signing_failures(), + readable + 1, + "the readable count moves once" + ); } #[test] - fn a_recorded_ruleset_failure_is_counted_where_the_route_reads_it() { - let before = ruleset_load_failures(); - record_ruleset_load_failure(); - assert_eq!(ruleset_load_failures(), before + 1); + fn a_refused_ruleset_bundle_moves_the_exported_and_the_readable_count() { + let readable = ruleset_load_failures(); + let exported = + exported_increase("ruleset_load_failures_total", record_ruleset_load_failure); + assert_eq!(exported, 1, "the exported counter moves once per refusal"); + assert_eq!( + ruleset_load_failures(), + readable + 1, + "the readable count moves once" + ); } } diff --git a/crates/dpp-node/tests/smoke.rs b/crates/dpp-node/tests/smoke.rs index 7e91a4ca..7e06c13e 100644 --- a/crates/dpp-node/tests/smoke.rs +++ b/crates/dpp-node/tests/smoke.rs @@ -30,6 +30,7 @@ use dpp_integrator::{infra::vault_client::VaultHttpClient, state::AppState as In use dpp_node::infra::credential_issuance::KeyStoreCredentialIssuer; use dpp_node::infra::pg_job_store::PgJobStore; use dpp_types::auth::{AuthContext, AuthError, AuthProvider}; +use dpp_types::registry_sync::{RegistrySyncOutbox, STALL_THRESHOLD}; use dpp_vault::domain::verify::{RefUnverifiable, RefVerification, verify_ref}; use dpp_vault::{ domain::{ @@ -355,31 +356,138 @@ async fn authenticated_node_state_reports_trust_modes() { assert_eq!(body["rulesetVersion"], "baseline"); } -/// The operational counters are served to an entitled caller, with the outbox -/// they describe read from the database. Only the shape is asserted: the two -/// process counters are shared by every test in this binary, so their values -/// are not this test's to fix. +/// The node-health counters follow what the node did. A passport's registration +/// row is stalled from the stall threshold and not one attempt before it, and a +/// refusal recorded in this process moves the counter the route reports by one. #[tokio::test(flavor = "multi_thread")] -async fn authenticated_node_health_reports_the_counters_and_the_outbox() { - let (base, _c) = start_db_and_node().await; +async fn node_health_counts_follow_the_outbox_and_the_refusals() { + let (base, pg) = start_db_and_node().await; let token = make_jwt("00000000-0000-0000-0000-000000000092"); + let client = reqwest::Client::new(); + + // Battery is in force, so publish also needs a default facility and a primary + // operator identifier (Annex III / Art. 13), as the route-inventory test seeds. + let facility = client + .post(format!("{base}/vault/api/v1/facilities")) + .bearer_auth(&token) + .json(&serde_json::json!({ + "name": "Default Plant", "identifierScheme": "gln", + "identifierValue": "4012345000009", "country": "DE", "isDefault": true + })) + .send() + .await + .expect("facility seed request failed"); + assert!( + facility.status().is_success(), + "facility seed failed: {}", + facility.status() + ); + let operator_id = client + .post(format!("{base}/vault/api/v1/operator-identifiers")) + .bearer_auth(&token) + .json(&serde_json::json!({ + "scheme": "lei", "value": "529900T8BM49AURSDO55", "isPrimary": true + })) + .send() + .await + .expect("operator-identifier seed request failed"); + assert!( + operator_id.status().is_success(), + "operator-identifier seed failed: {}", + operator_id.status() + ); + + let created: serde_json::Value = client + .post(format!("{base}/vault/api/v1/dpp")) + .bearer_auth(&token) + .json(&serde_json::json!({ + "productName": "Node Health Battery", + "manufacturer": {"name": "SmokeTestCorp", "address": "Berlin, DE"}, + "materials": [], + "productGroupData": { + "productGroup": "battery", + "productIdentifier": { "scheme": "gs1", "gtin": "09506000134352" }, + "batteryChemistry": "LFP", + "batteryType": "portable", + "nominalVoltageV": 48.0, + "nominalCapacityAh": 100.0, + "expectedLifetimeCycles": 3000, + "co2ePerUnitKg": 45.2 + } + })) + .send() + .await + .expect("create request failed") + .json() + .await + .expect("create response must be JSON"); + let id = created["id"] + .as_str() + .expect("id in create response") + .to_owned(); + let published = client + .post(format!("{base}/vault/api/v1/dpp/{id}/publish")) + .bearer_auth(&token) + .json(&serde_json::json!({})) + .send() + .await + .expect("publish request failed"); + let (publish_status, publish_body) = ( + published.status(), + published.text().await.unwrap_or_default(), + ); + assert!( + publish_status.is_success(), + "publish: {publish_status} {publish_body}" + ); + + // Publishing queued the registration. The drain is not running in this + // harness, so its attempts are driven here, one at a time. + let outbox = dpp_dal::pg::PgRegistrySyncRepo::new(pg.dal.clone()); + let passport = dpp_domain::passport::PassportId(id.parse().expect("a uuid")); + for _ in 1..STALL_THRESHOLD { + outbox + .mark_submitted_noting(passport, "waiting for a person".into()) + .await + .expect("attempt recorded"); + } + let short = node_health(&base, &token).await; + assert_eq!( + short["registryOutbox"]["stalled"], 0, + "one attempt short of the threshold: {short}" + ); + + outbox + .mark_submitted_noting(passport, "waiting for a person".into()) + .await + .expect("attempt recorded"); + let at_threshold = node_health(&base, &token).await; + assert_eq!( + at_threshold["registryOutbox"]["stalled"], 1, + "at the threshold: {at_threshold}" + ); + + let signing_before = at_threshold["signingFailuresTotal"] + .as_u64() + .expect("a count"); + dpp_common::metrics::record_signing_failure(); + let after = node_health(&base, &token).await; + assert_eq!( + after["signingFailuresTotal"].as_u64(), + Some(signing_before + 1), + "{after}" + ); +} + +async fn node_health(base: &str, token: &str) -> serde_json::Value { let resp = reqwest::Client::new() .get(format!("{base}/vault/api/v1/node/health")) - .bearer_auth(&token) + .bearer_auth(token) .send() .await .expect("GET /vault/api/v1/node/health failed"); assert_eq!(resp.status(), 200); - - let body: serde_json::Value = resp.json().await.expect("node health is JSON"); - assert!(body["signingFailuresTotal"].is_u64(), "{body}"); - assert!(body["rulesetLoadFailuresTotal"].is_u64(), "{body}"); - for field in ["pending", "submitted", "stalled", "rejected"] { - assert!( - body["registryOutbox"][field].is_i64(), - "`registryOutbox.{field}` must be reported on a Postgres node: {body}" - ); - } + resp.json().await.expect("node health is JSON") } /// The issue this route exists to close, end to end: a node that is up and diff --git a/crates/dpp-types/src/registry_sync.rs b/crates/dpp-types/src/registry_sync.rs index 5bef71a0..f7865651 100644 --- a/crates/dpp-types/src/registry_sync.rs +++ b/crates/dpp-types/src/registry_sync.rs @@ -185,9 +185,9 @@ pub struct RegistrySyncRow { pub next_attempt_at: DateTime, } -/// Attempts after which a pending registry row counts as stalled. The registration -/// and transfer outboxes both report their stalled rows at this threshold, on the -/// node's gauges, its alarms and `GET /vault/api/v1/node/health`. +/// Attempts after which a row counts as stalled: a registration row that is pending +/// or submitted, or a transfer row that is pending. The node's gauges, its alarms +/// and `GET /vault/api/v1/node/health` all count at this threshold. pub const STALL_THRESHOLD: i32 = 8; /// Aggregate counts for boot reconciliation and metrics. @@ -210,8 +210,8 @@ pub struct RegistrySyncCounts { /// Rows carrying an outstanding status intent. Independent of the counts /// above — a row can be `pending` *and* carry an intent. pub status_intents: i64, - /// Pending rows whose `attempts` have reached the stall threshold — these - /// are the ones a human must investigate (never silently dropped). + /// Pending and submitted rows whose `attempts` have reached the stall threshold + /// — these are the ones a human must investigate (never silently dropped). pub stalled: i64, } diff --git a/crates/dpp-vault/src/handlers/node_health.rs b/crates/dpp-vault/src/handlers/node_health.rs index 998a8aa5..20a8ea47 100644 --- a/crates/dpp-vault/src/handlers/node_health.rs +++ b/crates/dpp-vault/src/handlers/node_health.rs @@ -20,7 +20,8 @@ pub struct RegistryOutboxHealth { pub pending: i64, /// Rows accepted by the registry and awaiting its verdict. pub submitted: i64, - /// Pending rows that have reached the stall threshold and need a person. + /// Pending and submitted rows that have reached the stall threshold and need a + /// person. pub stalled: i64, /// Rows the registry terminally rejected. pub rejected: i64, diff --git a/crates/dpp-vault/src/handlers/ruleset.rs b/crates/dpp-vault/src/handlers/ruleset.rs index bc865c40..0fb214d1 100644 --- a/crates/dpp-vault/src/handlers/ruleset.rs +++ b/crates/dpp-vault/src/handlers/ruleset.rs @@ -45,7 +45,12 @@ pub async fn reload_ruleset_handler( match admin.reload().await { Ok(report) => (StatusCode::OK, Json(report)).into_response(), - Err(e) => reload_error(e), + Err(e) => { + if !matches!(e, RulesetReloadError::NotConfigured) { + dpp_common::metrics::record_ruleset_load_failure(); + } + reload_error(e) + } } }