feat(node): keep refusal counts across restarts

CHANGELOG.md

@@ -63,11 +63,12 @@ under the pre-1.0 conventions in [VERSIONING.md](docs/governance/VERSIONING.md):
 
 - **An authenticated node health route reports the counters an operator pages
   on.** `GET /vault/api/v1/node/health` returns the publishes refused for a failed
-  signature and the ruleset bundles refused on load, both since the node process
-  started, and the registry-sync outbox's pending, submitted, stalled and rejected
-  counts. A monitor that cannot scrape `/metrics` can read them with an API key.
-  The unauthenticated `/health` still answers liveness only. The two process
-  counters reset on restart; the outbox does not. (#138)
+  signature and the ruleset bundles refused on load, and the registry-sync
+  outbox's pending, submitted, stalled and rejected counts. A monitor that cannot
+  scrape `/metrics` can read them with an API key. The unauthenticated `/health`
+  still answers liveness only. The two refusal counts are kept in the database and
+  survive a restart, as the outbox counts always have. The Prometheus counters on
+  `/metrics` still reset on restart. (#138, #590)
 
 - **An authority or a person with a legitimate interest can read the passport
   as it stood at a past moment.** EN 18221:2026 clause 4.2 expects the version of

api/components/schemas/access/NodeHealth.yaml

@@ -1,24 +1,22 @@
 type: object
-required:
-  - signingFailuresTotal
-  - rulesetLoadFailuresTotal
 properties:
   signingFailuresTotal:
     type: integer
     format: int64
     minimum: 0
     description: |
-      Publishes refused because a passport could not be signed, since the node
-      process started. A restart resets it, so a restart hides failures from
-      before it.
+      Publishes refused because a passport could not be signed, since the node was
+      first started. Kept in the database, so a restart does not reset it. Absent
+      on a deployment with no durable counter wired.
     example: 0
   rulesetLoadFailuresTotal:
     type: integer
     format: int64
     minimum: 0
     description: |
-      Compliance Current ruleset bundles refused on load, since the node process
-      started. A restart resets it.
+      Compliance Current ruleset bundles refused on load, since the node was first
+      started. Kept in the database, so a restart does not reset it. Absent on a
+      deployment with no durable counter wired.
     example: 0
   registryOutbox:
     $ref: ./RegistryOutboxHealth.yaml

api/openapi.bundled.json

@@ -3580,7 +3580,7 @@
       "get": {
         "operationId": "getNodeHealth",
         "summary": "Node operational health counters",
-        "description": "Reports the counters an operator pages on: publishes refused for a failed\nsignature and ruleset bundles refused on load, both since the node process\nstarted, and the registry-sync outbox's pending, submitted, stalled and\nrejected counts. Authenticated, so a monitor that cannot scrape `/metrics`\ncan read them with an API key. The unauthenticated `/health` answers\nliveness only.\n",
+        "description": "Reports the counters an operator pages on: publishes refused for a failed\nsignature and ruleset bundles refused on load, counted since the node was\nfirst started and kept across restarts, and the registry-sync outbox's\npending, submitted, stalled and rejected counts. Authenticated, so a monitor\nthat cannot scrape `/metrics` can read them with an API key. The\nunauthenticated `/health` answers liveness only.\n",
         "tags": [
           "Node"
         ],
@@ -9530,23 +9530,19 @@
       },
       "NodeHealth": {
         "type": "object",
-        "required": [
-          "signingFailuresTotal",
-          "rulesetLoadFailuresTotal"
-        ],
         "properties": {
           "signingFailuresTotal": {
             "type": "integer",
             "format": "int64",
             "minimum": 0,
-            "description": "Publishes refused because a passport could not be signed, since the node\nprocess started. A restart resets it, so a restart hides failures from\nbefore it.\n",
+            "description": "Publishes refused because a passport could not be signed, since the node was\nfirst started. Kept in the database, so a restart does not reset it. Absent\non a deployment with no durable counter wired.\n",
             "example": 0
           },
           "rulesetLoadFailuresTotal": {
             "type": "integer",
             "format": "int64",
             "minimum": 0,
-            "description": "Compliance Current ruleset bundles refused on load, since the node process\nstarted. A restart resets it.\n",
+            "description": "Compliance Current ruleset bundles refused on load, since the node was first\nstarted. Kept in the database, so a restart does not reset it. Absent on a\ndeployment with no durable counter wired.\n",
             "example": 0
           },
           "registryOutbox": {

api/openapi.bundled.yaml

@@ -2984,11 +2984,11 @@ paths:
       summary: Node operational health counters
       description: |
         Reports the counters an operator pages on: publishes refused for a failed
-        signature and ruleset bundles refused on load, both since the node process
-        started, and the registry-sync outbox's pending, submitted, stalled and
-        rejected counts. Authenticated, so a monitor that cannot scrape `/metrics`
-        can read them with an API key. The unauthenticated `/health` answers
-        liveness only.
+        signature and ruleset bundles refused on load, counted since the node was
+        first started and kept across restarts, and the registry-sync outbox's
+        pending, submitted, stalled and rejected counts. Authenticated, so a monitor
+        that cannot scrape `/metrics` can read them with an API key. The
+        unauthenticated `/health` answers liveness only.
       tags:
         - Node
       security:
@@ -8274,26 +8274,24 @@ components:
           example: baseline
     NodeHealth:
       type: object
-      required:
-        - signingFailuresTotal
-        - rulesetLoadFailuresTotal
       properties:
         signingFailuresTotal:
           type: integer
           format: int64
           minimum: 0
           description: |
-            Publishes refused because a passport could not be signed, since the node
-            process started. A restart resets it, so a restart hides failures from
-            before it.
+            Publishes refused because a passport could not be signed, since the node was
+            first started. Kept in the database, so a restart does not reset it. Absent
+            on a deployment with no durable counter wired.
           example: 0
         rulesetLoadFailuresTotal:
           type: integer
           format: int64
           minimum: 0
           description: |
-            Compliance Current ruleset bundles refused on load, since the node process
-            started. A restart resets it.
+            Compliance Current ruleset bundles refused on load, since the node was first
+            started. Kept in the database, so a restart does not reset it. Absent on a
+            deployment with no durable counter wired.
           example: 0
         registryOutbox:
           $ref: '#/components/schemas/RegistryOutboxHealth'

api/paths/vault/vault_api_v1_node_health.yaml

@@ -3,11 +3,11 @@ get:
   summary: Node operational health counters
   description: |
     Reports the counters an operator pages on: publishes refused for a failed
-    signature and ruleset bundles refused on load, both since the node process
-    started, and the registry-sync outbox's pending, submitted, stalled and
-    rejected counts. Authenticated, so a monitor that cannot scrape `/metrics`
-    can read them with an API key. The unauthenticated `/health` answers
-    liveness only.
+    signature and ruleset bundles refused on load, counted since the node was
+    first started and kept across restarts, and the registry-sync outbox's
+    pending, submitted, stalled and rejected counts. Authenticated, so a monitor
+    that cannot scrape `/metrics` can read them with an API key. The
+    unauthenticated `/health` answers liveness only.
   tags:
     - Node
   security:

crates/dpp-common/src/metrics.rs

@@ -17,7 +17,6 @@
 //! breaking change for anyone already scraping it, so this is flagged, not
 //! fixed, here.
 
-use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::Instant;
 
 use axum::{extract::Request, middleware::Next, response::Response};
@@ -64,39 +63,18 @@ pub async fn http_metrics_middleware(request: Request, next: Next) -> Response {
     response
 }
 
-/// Signing failures since this process started. The Prometheus recorder cannot be
-/// read back, so the node-health route reads these instead.
-static SIGNING_FAILURES: AtomicU64 = AtomicU64::new(0);
-/// Ruleset bundles refused on load since this process started, as above.
-static RULESET_LOAD_FAILURES: AtomicU64 = AtomicU64::new(0);
-
-/// Count one publish refused because a passport could not be signed.
-///
-/// Every increment goes through here, not `metrics::counter!` directly, so the
-/// count `/metrics` exports and the one the node-health route reports cannot drift.
+/// Count one publish refused because a passport could not be signed, on the
+/// Prometheus counter `/metrics` exports. The durable count is kept beside it by
+/// the publish path, see `dpp_types::node_failure`.
 pub fn record_signing_failure() {
-    SIGNING_FAILURES.fetch_add(1, Ordering::Relaxed);
     metrics::counter!("signing_failures_total").increment(1);
 }
 
 /// Count one ruleset bundle refused on load. See [`record_signing_failure`].
 pub fn record_ruleset_load_failure() {
-    RULESET_LOAD_FAILURES.fetch_add(1, Ordering::Relaxed);
     metrics::counter!("ruleset_load_failures_total").increment(1);
 }
 
-/// Signing failures since process start. A restart resets it.
-#[must_use]
-pub fn signing_failures() -> u64 {
-    SIGNING_FAILURES.load(Ordering::Relaxed)
-}
-
-/// Ruleset bundles refused since process start. A restart resets it.
-#[must_use]
-pub fn ruleset_load_failures() -> u64 {
-    RULESET_LOAD_FAILURES.load(Ordering::Relaxed)
-}
-
 #[cfg(test)]
 mod tests {
     use std::collections::HashMap;
@@ -163,30 +141,21 @@ mod tests {
         exported.0.lock().unwrap().get(name).copied().unwrap_or(0)
     }
 
-    /// Each refusal moves the exported counter `/metrics` serves and the readable
-    /// count the route reports by one each, so the two cannot drift apart.
+    /// A refused signature moves the counter `/metrics` exports, under the name
+    /// monitors already scrape.
     #[test]
-    fn a_refused_signature_moves_the_exported_and_the_readable_count() {
-        let readable = signing_failures();
-        let exported = exported_increase("signing_failures_total", record_signing_failure);
-        assert_eq!(exported, 1, "the exported counter moves once per refusal");
+    fn a_refused_signature_moves_the_exported_counter() {
         assert_eq!(
-            signing_failures(),
-            readable + 1,
-            "the readable count moves once"
+            exported_increase("signing_failures_total", record_signing_failure),
+            1
         );
     }
 
     #[test]
-    fn a_refused_ruleset_bundle_moves_the_exported_and_the_readable_count() {
-        let readable = ruleset_load_failures();
-        let exported =
-            exported_increase("ruleset_load_failures_total", record_ruleset_load_failure);
-        assert_eq!(exported, 1, "the exported counter moves once per refusal");
+    fn a_refused_ruleset_bundle_moves_the_exported_counter() {
         assert_eq!(
-            ruleset_load_failures(),
-            readable + 1,
-            "the readable count moves once"
+            exported_increase("ruleset_load_failures_total", record_ruleset_load_failure),
+            1
         );
     }
 }

crates/dpp-dal/src/pg/mod.rs

@@ -14,6 +14,7 @@ pub mod repo_api_key;
 pub mod repo_audit;
 pub mod repo_evidence;
 pub mod repo_idempotency;
+pub mod repo_node_failure;
 pub mod repo_operator_config;
 pub mod repo_passport;
 pub mod repo_passport_version;
@@ -37,6 +38,7 @@ pub use repo_api_key::PgApiKeyRepo;
 pub use repo_audit::PgAuditRepo;
 pub use repo_evidence::PgEvidenceDossierRepo;
 pub use repo_idempotency::PgIdempotencyRepo;
+pub use repo_node_failure::PgNodeFailureRepo;
 pub use repo_operator_config::PgOperatorConfigRepo;
 pub use repo_passport::PgPassportRepo;
 pub use repo_passport_version::PgPassportVersionRepo;

crates/dpp-dal/src/pg/repo_node_failure.rs

@@ -0,0 +1,66 @@
+//! `NodeFailureCounter` on PostgreSQL: the refusals this node has counted, kept
+//! across restarts (`ops/pg/0048`).
+
+use async_trait::async_trait;
+use sqlx::Row;
+
+use dpp_domain::DppError;
+use dpp_types::node_failure::{FailureCounts, FailureKind, NodeFailureCounter};
+
+use super::{PgDal, db_err};
+
+/// PostgreSQL implementation of [`NodeFailureCounter`].
+pub struct PgNodeFailureRepo {
+    dal: PgDal,
+}
+
+impl PgNodeFailureRepo {
+    /// Construct a repo sharing the given pool handle.
+    pub fn new(dal: PgDal) -> Self {
+        Self { dal }
+    }
+}
+
+/// The name a kind is stored under. The CHECK in `ops/pg/0048` lists the same two.
+fn name_of(kind: FailureKind) -> &'static str {
+    match kind {
+        FailureKind::Signing => "signing",
+        FailureKind::RulesetLoad => "ruleset_load",
+    }
+}
+
+#[async_trait]
+impl NodeFailureCounter for PgNodeFailureRepo {
+    async fn record(&self, kind: FailureKind) -> Result<(), DppError> {
+        // One statement: the first refusal creates the row and later ones add to
+        // it, so two refusals of the same kind racing each other cannot lose one.
+        sqlx::query(
+            "INSERT INTO odal.node_failure_counter (name, refusals) VALUES ($1, 1) \
+             ON CONFLICT (name) DO UPDATE \
+             SET refusals = odal.node_failure_counter.refusals + 1",
+        )
+        .bind(name_of(kind))
+        .execute(self.dal.pool())
+        .await
+        .map_err(db_err)?;
+        Ok(())
+    }
+
+    async fn counts(&self) -> Result<FailureCounts, DppError> {
+        let rows = sqlx::query("SELECT name, refusals FROM odal.node_failure_counter")
+            .fetch_all(self.dal.pool())
+            .await
+            .map_err(db_err)?;
+        let mut counts = FailureCounts::default();
+        for row in rows {
+            let name: String = row.get("name");
+            let refusals: i64 = row.get("refusals");
+            match name.as_str() {
+                "signing" => counts.signing = refusals,
+                "ruleset_load" => counts.ruleset_load = refusals,
+                _ => {}
+            }
+        }
+        Ok(counts)
+    }
+}

crates/dpp-dal/tests/pg_node_failure.rs

@@ -0,0 +1,36 @@
+//! `PgNodeFailureRepo`: refusals are counted per kind, and the count outlives the
+//! repo that wrote it, as it does for a node that restarts.
+
+#![cfg(feature = "integration-tests")]
+
+use dpp_dal::pg::PgNodeFailureRepo;
+use dpp_dal::test_harness::start_pg;
+use dpp_types::node_failure::{FailureCounts, FailureKind, NodeFailureCounter};
+
+#[tokio::test(flavor = "multi_thread")]
+async fn refusals_are_counted_per_kind_and_survive_a_restart() {
+    let pg = start_pg().await;
+    let before = PgNodeFailureRepo::new(pg.dal.clone())
+        .counts()
+        .await
+        .unwrap();
+    assert_eq!(before, FailureCounts::default(), "nothing refused yet");
+
+    let repo = PgNodeFailureRepo::new(pg.dal.clone());
+    repo.record(FailureKind::Signing).await.unwrap();
+    repo.record(FailureKind::Signing).await.unwrap();
+    repo.record(FailureKind::RulesetLoad).await.unwrap();
+
+    // A new repo over the same database, as a node that has restarted holds.
+    let after_restart = PgNodeFailureRepo::new(pg.dal.clone())
+        .counts()
+        .await
+        .unwrap();
+    assert_eq!(
+        after_restart,
+        FailureCounts {
+            signing: 2,
+            ruleset_load: 1,
+        }
+    );
+}

crates/dpp-node/src/boot/db.rs

@@ -7,7 +7,7 @@ use anyhow::Context;
 use async_trait::async_trait;
 
 use dpp_dal::pg::{
-    PgApiKeyRepo, PgAuditRepo, PgDal, PgEvidenceDossierRepo, PgIdempotencyRepo,
+    PgApiKeyRepo, PgAuditRepo, PgDal, PgEvidenceDossierRepo, PgIdempotencyRepo, PgNodeFailureRepo,
     PgOperatorConfigRepo, PgPassportRepo, PgRegistryIdentityRepo, PgRegistrySyncRepo,
     PgRegistryTransferRepo, PgScanTelemetryRepo, PgSealOutboxRepo, PgSnapshotOutboxRepo,
     PgTransferRepo, PgWebhookRepo,
@@ -45,6 +45,8 @@ pub struct DbComponents {
     pub api_key_repo: Arc<dyn ApiKeyRepository>,
     pub registry_repo: Arc<dyn RegistryIdentityRepository>,
     pub registry_outbox: Arc<dyn RegistrySyncOutbox>,
+    /// Durable refusal counts the node-health route reports across restarts.
+    pub failure_counter: Arc<dyn dpp_types::node_failure::NodeFailureCounter>,
     pub transfer_store: Arc<dyn TransferStore>,
     pub transfer_outbox: Arc<dyn RegistryTransferOutbox>,
     pub evidence_store: Arc<dyn EvidenceDossierRepository>,
@@ -141,6 +143,7 @@ pub async fn init_db(cfg: &NodeConfig) -> anyhow::Result<DbComponents> {
         api_key_repo: Arc::new(PgApiKeyRepo::new(dal.clone())),
         registry_repo: Arc::new(PgRegistryIdentityRepo::new(dal.clone())),
         registry_outbox: Arc::new(PgRegistrySyncRepo::new(dal.clone())),
+        failure_counter: Arc::new(PgNodeFailureRepo::new(dal.clone())),
         transfer_store: Arc::new(PgTransferRepo::new(dal.clone())),
         transfer_outbox: Arc::new(PgRegistryTransferRepo::new(dal.clone())),
         evidence_store: Arc::new(PgEvidenceDossierRepo::new(dal.clone())),