Cap confirmation retries, persist reconciliation history, and alert on credit-usage spikes

backend/.env.example

@@ -151,8 +151,22 @@ PAYMENT_RECONCILE_BACKOFF_MAX_MINUTES=60
 PAYMENT_RECONCILE_MAX_BATCH=500
 # A payment unresolved this long escalates to MANUAL_REVIEW — but only on a
 # pass where the provider was actually reachable; a provider outage never
-# by itself causes escalation (see ReconciliationService).
+# by itself causes escalation (see ReconciliationService). The two hard caps
+# below are independent safety bounds: they stop further provider polling and
+# hand the payment to a human as soon as either cap is reached, even before
+# the age threshold. If a provider-error attempt hits the total-attempt cap
+# first, the next pass escalates it without making another provider call.
+# The provider-error-streak cap deliberately overrides the outage safeguard
+# only after a long, bounded per-payment streak; one bad run still does not
+# mass-flag payments.
 PAYMENT_MANUAL_REVIEW_AFTER_HOURS=24
+# Total scheduled confirmation polls for one payment before it is escalated
+# to MANUAL_REVIEW and excluded from future reconciliation selection.
+PAYMENT_CONFIRMATION_MAX_ATTEMPTS=20
+# Consecutive provider-unreachable polls for one payment before it is
+# escalated to MANUAL_REVIEW, even when the provider-error attempt is the one
+# that trips the cap.
+PAYMENT_CONFIRMATION_MAX_PROVIDER_ERROR_STREAK=10
 # When the manual-review queue depth exceeds this the reconciliation cron
 # logs a WARN, emails SUPPORT_EMAIL, and the depth stays visible on the
 # Prometheus /metrics endpoint (managehub_manual_review_queue_depth) for
@@ -204,6 +218,17 @@ CREDITS_SETTLEMENT_MAX_PAYOUT_ATTEMPTS=5
 # Upper bound on how many CONFIRMED payments one pass of the credit sweep
 # (top-ups and payment revenue splits) will apply.
 CREDITS_PAYMENT_SWEEP_MAX_BATCH=200
+# Usage-spike anomaly detection. After a new usage event is charged, the
+# service compares the tenant's current rolling window total with the
+# average of comparable prior windows. It alerts only when there are at
+# least MIN_SAMPLES prior events, the window is at least MULTIPLIER times
+# that baseline, and the window total is at least MIN_AMOUNT minor units.
+# The absolute floor prevents tiny tenants from alerting on rounding; SMTP
+# delivery is best-effort and never rolls back the charge.
+CREDITS_USAGE_SPIKE_WINDOW_MINUTES=60
+CREDITS_USAGE_SPIKE_MULTIPLIER=5.0
+CREDITS_USAGE_SPIKE_MIN_AMOUNT=10000
+CREDITS_USAGE_SPIKE_MIN_SAMPLES=3
 
 # Data retention (issue BE-144)
 # Set to false to stop the daily retention job entirely.

backend/src/common/metrics.service.ts

@@ -42,6 +42,11 @@ export class MetricsService {
     this.incrementCounter('managehub_settlement_payout_failures_total', {});
   }
 
+  /** Counts a detected tenant credit-usage anomaly by metered resource. */
+  recordCreditUsageSpike(resource: string): void {
+    this.incrementCounter('managehub_credit_usage_spikes_total', { resource });
+  }
+
   renderPrometheus(): string {
     const lines: string[] = [];
     this.appendCounters(lines);

backend/src/credits/metered-usage.service.spec.ts

@@ -4,27 +4,37 @@ import { InsufficientCreditException, LedgerService } from './ledger.service';
 import { MeteredUsageService } from './metered-usage.service';
 import { LedgerAccountKind } from './enums/ledger-account-kind.enum';
 import { MeteredResource } from './enums/metered-resource.enum';
+import { createTransport } from 'nodemailer';
 import {
   createLedgerHarness,
   fakeConfigService,
 } from './testing/in-memory-ledger';
 
-function build() {
+jest.mock('nodemailer', () => ({
+  createTransport: jest.fn(),
+}));
+
+function build(configValues: Record<string, unknown> = {}) {
   const harness = createLedgerHarness();
+  const config = fakeConfigService({
+    CREDITS_DEFAULT_CURRENCY: 'USD',
+    CREDITS_DEFAULT_OVERDRAFT_LIMIT: 0,
+    ...configValues,
+  });
   const ledger = new LedgerService(
     harness.accounts as any,
     harness.transactions as any,
     harness.entries as any,
   );
-  const credits = new CreditsService(
-    ledger,
-    fakeConfigService({
-      CREDITS_DEFAULT_CURRENCY: 'USD',
-      CREDITS_DEFAULT_OVERDRAFT_LIMIT: 0,
-    }),
+  const credits = new CreditsService(ledger, config);
+  const metrics = { recordCreditUsageSpike: jest.fn() };
+  const usage = new MeteredUsageService(
+    harness.usageEvents as any,
+    credits,
+    config as any,
+    metrics as any,
   );
-  const usage = new MeteredUsageService(harness.usageEvents as any, credits);
-  return { harness, ledger, credits, usage };
+  return { harness, ledger, credits, usage, config, metrics };
 }
 
 async function fund(credits: CreditsService, amount: number) {
@@ -45,6 +55,28 @@ const meterReading = {
 };
 
 describe('MeteredUsageService', () => {
+  beforeEach(() => {
+    (createTransport as jest.Mock).mockReset();
+  });
+
+  function seedPriorUsage(harness: any, amounts: number[]) {
+    const now = Date.now();
+    amounts.forEach((amount, index) => {
+      harness.usageEvents.rows.push({
+        id: `prior-${index}`,
+        userId: 'user-1',
+        resource: MeteredResource.RESOURCE_MINUTES,
+        units: 1,
+        unitPrice: amount,
+        amount,
+        currency: 'USD',
+        usageReference: `prior-${index}`,
+        ledgerTransactionId: `prior-transaction-${index}`,
+        createdAt: new Date(now - (70 + index * 10) * 60_000),
+      });
+    });
+  }
+
   it('prices the reading and charges it against the credit balance', async () => {
     const { usage, credits, harness } = build();
     await fund(credits, 1000);
@@ -163,4 +195,86 @@ describe('MeteredUsageService', () => {
     expect((await credits.getBalance('user-1')).balance).toBe(1000 - 250);
     expect(await usage.listForUser('user-1')).toHaveLength(3);
   });
+
+  it('does not alert when the current window is below the spike threshold', async () => {
+    const { usage, credits, harness, metrics } = build({
+      CREDITS_USAGE_SPIKE_WINDOW_MINUTES: 60,
+      CREDITS_USAGE_SPIKE_MULTIPLIER: 5,
+      CREDITS_USAGE_SPIKE_MIN_AMOUNT: 100,
+      CREDITS_USAGE_SPIKE_MIN_SAMPLES: 3,
+    });
+    seedPriorUsage(harness, [100, 100, 100]);
+    await fund(credits, 5000);
+
+    await usage.recordUsage({
+      ...meterReading,
+      units: 80,
+      usageReference: 'below-spike',
+    });
+
+    expect(metrics.recordCreditUsageSpike).not.toHaveBeenCalled();
+  });
+
+  it('does not alert without the minimum number of prior samples', async () => {
+    const { usage, credits, harness, metrics } = build({
+      CREDITS_USAGE_SPIKE_MIN_AMOUNT: 100,
+      CREDITS_USAGE_SPIKE_MIN_SAMPLES: 3,
+    });
+    seedPriorUsage(harness, [10, 10]);
+    await fund(credits, 5000);
+
+    await usage.recordUsage({
+      ...meterReading,
+      units: 200,
+      usageReference: 'insufficient-history',
+    });
+
+    expect(metrics.recordCreditUsageSpike).not.toHaveBeenCalled();
+  });
+
+  it('increments the usage-spike counter when a sufficiently large spike is detected', async () => {
+    const { usage, credits, harness, metrics } = build({
+      CREDITS_USAGE_SPIKE_MIN_AMOUNT: 100,
+      CREDITS_USAGE_SPIKE_MIN_SAMPLES: 3,
+    });
+    seedPriorUsage(harness, [10, 10, 10]);
+    await fund(credits, 5000);
+
+    await usage.recordUsage({
+      ...meterReading,
+      units: 200,
+      usageReference: 'large-spike',
+    });
+
+    expect(metrics.recordCreditUsageSpike).toHaveBeenCalledWith(
+      MeteredResource.RESOURCE_MINUTES,
+    );
+  });
+
+  it('does not let a failing SMTP alert break usage recording', async () => {
+    const sendMail = jest.fn().mockRejectedValue(new Error('smtp unavailable'));
+    (createTransport as jest.Mock).mockReturnValue({ sendMail });
+    const { usage, credits, harness, metrics } = build({
+      CREDITS_USAGE_SPIKE_MIN_AMOUNT: 100,
+      CREDITS_USAGE_SPIKE_MIN_SAMPLES: 3,
+      SMTP_HOST: 'smtp.example.test',
+      SMTP_USER: 'smtp-user',
+      SMTP_PASS: 'smtp-pass',
+      SMTP_FROM_EMAIL: 'alerts@example.test',
+      SUPPORT_EMAIL: 'support@example.test',
+    });
+    seedPriorUsage(harness, [10, 10, 10]);
+    await fund(credits, 5000);
+
+    await expect(
+      usage.recordUsage({
+        ...meterReading,
+        units: 200,
+        usageReference: 'smtp-failure-spike',
+      }),
+    ).resolves.toMatchObject({ charged: true });
+    expect(metrics.recordCreditUsageSpike).toHaveBeenCalled();
+    expect(sendMail).toHaveBeenCalledTimes(1);
+    expect((await credits.getBalance('user-1')).balance).toBe(4000);
+  });
 });