From 7adf3b18159b029be511ce7e4ba89e20bb95d682 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 13:34:39 +0300 Subject: [PATCH 01/27] fix(deps): pin patched System.Security.Cryptography.Pkcs in the SQL Server and Oracle adapters SqlClient 6.1.1 and Oracle.ManagedDataAccess.Core 23.9.1 bring in Pkcs 8.0.1 and 9.0.4. The advisories GHSA-846f-f764-76mx (high) and GHSA-2fmv-rv2c-59x7 (moderate) make restore fail with NU1903 and NU1902. Each target framework now pins the patched version of its own line. --- src/BackWave.Oracle/BackWave.Oracle.csproj | 8 ++++++++ src/BackWave.SqlServer/BackWave.SqlServer.csproj | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/src/BackWave.Oracle/BackWave.Oracle.csproj b/src/BackWave.Oracle/BackWave.Oracle.csproj index f511fce..5b0f149 100644 --- a/src/BackWave.Oracle/BackWave.Oracle.csproj +++ b/src/BackWave.Oracle/BackWave.Oracle.csproj @@ -23,6 +23,14 @@ + + + + + + + diff --git a/src/BackWave.SqlServer/BackWave.SqlServer.csproj b/src/BackWave.SqlServer/BackWave.SqlServer.csproj index ddd085a..25b1a70 100644 --- a/src/BackWave.SqlServer/BackWave.SqlServer.csproj +++ b/src/BackWave.SqlServer/BackWave.SqlServer.csproj @@ -19,6 +19,14 @@ + + + + + + + From 4ba2135a7c0280e768a2da75cdc1efeef2958e66 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Fri, 16 Oct 2026 22:56:36 +0300 Subject: [PATCH 02/27] test: turn off config reload in every test project that builds a host Dashboard.Tests hung for more than 25 minutes in FileSystemWatcher.StartRaisingEvents on macOS. This is the same hang that Hosting.Tests fixed. Dashboard, Pro.Dashboard, Pro.Mcp, Oracle, and the core tests now set the same switch. --- tests/BackWave.Dashboard.Tests/NoConfigReload.cs | 12 ++++++++++++ tests/BackWave.Oracle.Tests/NoConfigReload.cs | 12 ++++++++++++ tests/BackWave.Pro.Dashboard.Tests/NoConfigReload.cs | 12 ++++++++++++ tests/BackWave.Pro.Mcp.Tests/NoConfigReload.cs | 12 ++++++++++++ tests/BackWave.Tests/NoConfigReload.cs | 12 ++++++++++++ 5 files changed, 60 insertions(+) create mode 100644 tests/BackWave.Dashboard.Tests/NoConfigReload.cs create mode 100644 tests/BackWave.Oracle.Tests/NoConfigReload.cs create mode 100644 tests/BackWave.Pro.Dashboard.Tests/NoConfigReload.cs create mode 100644 tests/BackWave.Pro.Mcp.Tests/NoConfigReload.cs create mode 100644 tests/BackWave.Tests/NoConfigReload.cs diff --git a/tests/BackWave.Dashboard.Tests/NoConfigReload.cs b/tests/BackWave.Dashboard.Tests/NoConfigReload.cs new file mode 100644 index 0000000..ba33847 --- /dev/null +++ b/tests/BackWave.Dashboard.Tests/NoConfigReload.cs @@ -0,0 +1,12 @@ +using System.Runtime.CompilerServices; + +namespace BackWave.Dashboard.Tests; + +internal static class NoConfigReload +{ + // WebApplication.CreateBuilder watches appsettings.json for changes. On macOS, starting that + // file watcher sometimes blocks forever, and the whole run hangs. No test here reloads config. + [ModuleInitializer] + internal static void Initialize() => + Environment.SetEnvironmentVariable("DOTNET_hostBuilder__reloadConfigOnChange", "false"); +} diff --git a/tests/BackWave.Oracle.Tests/NoConfigReload.cs b/tests/BackWave.Oracle.Tests/NoConfigReload.cs new file mode 100644 index 0000000..7fcbd18 --- /dev/null +++ b/tests/BackWave.Oracle.Tests/NoConfigReload.cs @@ -0,0 +1,12 @@ +using System.Runtime.CompilerServices; + +namespace BackWave.Oracle.Tests; + +internal static class NoConfigReload +{ + // WebApplication.CreateBuilder watches appsettings.json for changes. On macOS, starting that + // file watcher sometimes blocks forever, and the whole run hangs. No test here reloads config. + [ModuleInitializer] + internal static void Initialize() => + Environment.SetEnvironmentVariable("DOTNET_hostBuilder__reloadConfigOnChange", "false"); +} diff --git a/tests/BackWave.Pro.Dashboard.Tests/NoConfigReload.cs b/tests/BackWave.Pro.Dashboard.Tests/NoConfigReload.cs new file mode 100644 index 0000000..19dd97e --- /dev/null +++ b/tests/BackWave.Pro.Dashboard.Tests/NoConfigReload.cs @@ -0,0 +1,12 @@ +using System.Runtime.CompilerServices; + +namespace BackWave.Pro.Dashboard.Tests; + +internal static class NoConfigReload +{ + // WebApplication.CreateBuilder watches appsettings.json for changes. On macOS, starting that + // file watcher sometimes blocks forever, and the whole run hangs. No test here reloads config. + [ModuleInitializer] + internal static void Initialize() => + Environment.SetEnvironmentVariable("DOTNET_hostBuilder__reloadConfigOnChange", "false"); +} diff --git a/tests/BackWave.Pro.Mcp.Tests/NoConfigReload.cs b/tests/BackWave.Pro.Mcp.Tests/NoConfigReload.cs new file mode 100644 index 0000000..bec24f9 --- /dev/null +++ b/tests/BackWave.Pro.Mcp.Tests/NoConfigReload.cs @@ -0,0 +1,12 @@ +using System.Runtime.CompilerServices; + +namespace BackWave.Pro.Mcp.Tests; + +internal static class NoConfigReload +{ + // WebApplication.CreateBuilder watches appsettings.json for changes. On macOS, starting that + // file watcher sometimes blocks forever, and the whole run hangs. No test here reloads config. + [ModuleInitializer] + internal static void Initialize() => + Environment.SetEnvironmentVariable("DOTNET_hostBuilder__reloadConfigOnChange", "false"); +} diff --git a/tests/BackWave.Tests/NoConfigReload.cs b/tests/BackWave.Tests/NoConfigReload.cs new file mode 100644 index 0000000..2d12e8d --- /dev/null +++ b/tests/BackWave.Tests/NoConfigReload.cs @@ -0,0 +1,12 @@ +using System.Runtime.CompilerServices; + +namespace BackWave.Tests; + +internal static class NoConfigReload +{ + // WebApplication.CreateBuilder watches appsettings.json for changes. On macOS, starting that + // file watcher sometimes blocks forever, and the whole run hangs. No test here reloads config. + [ModuleInitializer] + internal static void Initialize() => + Environment.SetEnvironmentVariable("DOTNET_hostBuilder__reloadConfigOnChange", "false"); +} From 3813c43a93ab845e380f03b44b1196e60c34113d Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 14:04:22 +0300 Subject: [PATCH 03/27] fix(oracle): classify network outage and restart faults as transient A worker group on Oracle fail-stopped on the first connect that failed during a store outage, and it did not claim again until the process restarted. ODP.NET raises codes for these faults that the transient list did not have: - ORA-12545 when the database host name stops resolving - ORA-50201 when the connection is refused (the database is down) - ORA-50000 when the connect times out - ORA-01089 and ORA-01109 while the database shuts down and opens again These codes now degrade the group, and it retries on the next poll. The pump runs the migration again on each poll, so a migration that fails on one of these codes also degrades the group. A wrong Data Source can give the same codes. A mistyped host name gives ORA-12545, and a wrong service name gives ORA-50201. A worker group with such a store now stays degraded and does not halt. The health check reports it as Degraded, not Unhealthy, so a readiness probe that accepts Degraded reports the deployment as ready. The release notes must state this tradeoff. The Postgres adapter already behaves this way. Fixes #12 --- src/BackWave.Oracle/OracleFaultCodes.cs | 8 +- .../OracleTransientFaultTests.cs | 90 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs diff --git a/src/BackWave.Oracle/OracleFaultCodes.cs b/src/BackWave.Oracle/OracleFaultCodes.cs index 7c52617..fd3f566 100644 --- a/src/BackWave.Oracle/OracleFaultCodes.cs +++ b/src/BackWave.Oracle/OracleFaultCodes.cs @@ -9,10 +9,16 @@ namespace BackWave.Oracle; // 12537 connection closed 12570 packet reader failure // 3113 end-of-file on channel 3114 not connected // 28 session killed 1033 database initializing/shutting down +// 1089 shutdown in progress 1109 database not open (a restart opens it after the listener) // 12154 could not resolve connect id (transient during cold DNS/TNS) +// 12545 target host does not exist (the host name stops resolving in a network outage) +// ODP.NET managed raises its own codes for a connect that fails before any listener answers: +// 50201 failed to connect to server (connection refused: the database is down) +// 50000 connection request timed out (the host does not answer, or the pool has no free connection) internal static class OracleFaultCodes { // True when the ORA code is one of the transient connectivity/timeout faults above. internal static bool IsConnectivityFault(int code) => code is - 12170 or 12541 or 12514 or 12518 or 12537 or 12570 or 3113 or 3114 or 28 or 1033 or 12154; + 12170 or 12541 or 12514 or 12518 or 12537 or 12570 or 3113 or 3114 or 28 or 1033 or 12154 + or 1089 or 1109 or 12545 or 50201 or 50000; } diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs new file mode 100644 index 0000000..4a0649a --- /dev/null +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -0,0 +1,90 @@ +using BackWave.Core; +using BackWave.Hosting; +using BackWave.Jobs; +using BackWave.Storage; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Oracle.ManagedDataAccess.Client; + +namespace BackWave.Oracle.Tests; + +/// +/// Issue #12: a network outage between a worker group and its Oracle database must degrade the group, not +/// fail-stop it. ODP.NET raises these faults while it opens a connection, so each case points a real store +/// at an endpoint that cannot answer and needs no database. +/// +public sealed class OracleTransientFaultTests +{ + // Each endpoint reproduces one outage the way ODP.NET reports it to the pump. + public static TheoryData Outages => new() + { + // The host name stops resolving (a DNS or network outage). + { Descriptor("bw-no-such-host", 1521), 12545 }, + // The host answers but nothing listens on the port (the database is down). + { Descriptor("127.0.0.1", 1), 50201 }, + // The host does not answer at all, so the connect times out. + { Descriptor("10.255.255.1", 1521), 50000 }, + }; + + [Theory] + [MemberData(nameof(Outages))] + public async Task A_network_outage_raises_a_fault_the_adapter_classifies_transient(string dataSource, int expected) + { + var store = Store(dataSource); + + var fault = await Assert.ThrowsAsync(async () => + await store.ClaimAsync(new ClaimRequest("w1", ["default"], 10, TimeSpan.FromMinutes(1), DateTimeOffset.UtcNow))); + + Assert.Equal(expected, fault.Number); + Assert.True(store.IsTransientFault(fault), $"ORA-{fault.Number} must classify transient"); + } + + [Theory] + [MemberData(nameof(Outages))] + public async Task A_network_outage_degrades_the_worker_group_instead_of_halting_it(string dataSource, int expected) + { + var builder = Host.CreateApplicationBuilder(); + builder.Services.AddBackWave(backwave => backwave + .UseStore(Store(dataSource)) + .UseRegistry(new JobRegistry([])) + .AddWorkerGroup(new WorkerGroupOptions + { + Name = "outage-workers", + Policy = new DispatchPolicy.Strict(["default"]), + PollInterval = TimeSpan.FromMilliseconds(200), + LeaseDuration = TimeSpan.FromSeconds(30), + })); + using var host = builder.Build(); + var health = host.Services.GetRequiredService(); + + await host.StartAsync(); + try + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(15); + while (DateTime.UtcNow < deadline + && !health.DegradedGroups.ContainsKey("outage-workers") + && !health.HaltedGroups.ContainsKey("outage-workers")) + { + await Task.Delay(50); + } + + Assert.False(health.HaltedGroups.TryGetValue("outage-workers", out var halt), + $"the outage fail-stopped the group: {halt?.ExceptionType}: {halt?.Message}"); + Assert.True(health.DegradedGroups.TryGetValue("outage-workers", out var cause), "the outage must mark the group degraded"); + Assert.Contains($"ORA-{expected}", cause); + } + finally + { + await host.StopAsync(); + } + } + + private static string Descriptor(string host, int port) => + $"(DESCRIPTION=(CONNECT_TIMEOUT=2)(ADDRESS=(PROTOCOL=TCP)(HOST={host})(PORT={port}))" + + "(CONNECT_DATA=(SERVICE_NAME=FREEPDB1)))"; + + private static OracleJobStore Store(string dataSource) => new(new OracleStoreOptions + { + ConnectionString = $"User Id=backwave;Password=backwave;Connection Timeout=3;Data Source={dataSource}", + }); +} From d74815430353f4dfa090d664107cb95627086667 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Fri, 16 Oct 2026 21:04:35 +0300 Subject: [PATCH 04/27] chore(verify): run the sample and bwv on a private Oracle store The sample accepts BackWave:Store=Oracle, and bwv launch --store oracle starts a private Oracle container on a fixed loopback port. With a fixed port, a run can stop and start the container to reproduce a store outage. --- .agents/skills/verify-backwave/SKILL.md | 14 ++++--- .../skills/verify-backwave/features/README.md | 2 +- .agents/skills/verify-backwave/scripts/bwv | 38 +++++++++++++++---- .../BackWave.Sample.Api.csproj | 1 + samples/BackWave.Sample.Api/Program.cs | 2 +- samples/BackWave.Sample.Api/StoreKind.cs | 28 ++++++++++++++ 6 files changed, 71 insertions(+), 14 deletions(-) diff --git a/.agents/skills/verify-backwave/SKILL.md b/.agents/skills/verify-backwave/SKILL.md index 0fd02ea..1ae865f 100644 --- a/.agents/skills/verify-backwave/SKILL.md +++ b/.agents/skills/verify-backwave/SKILL.md @@ -1,6 +1,6 @@ --- name: verify-backwave -description: Launch the real BackWave Sample.Api (the sample host with the BackWave dashboard at /backwave) on a private port with its own SQLite file or Postgres container, create real jobs through the sample's HTTP endpoints, drive the dashboard in a private headless Chrome, capture screenshot, accessibility, API, and database evidence, and clean up. Use to prove a BackWave change works end to end, to reproduce a bug that a user or operator would see, or whenever you need to see or screenshot the dashboard running. +description: Launch the real BackWave Sample.Api (the sample host with the BackWave dashboard at /backwave) on a private port with its own SQLite file, Postgres container, or Oracle container, create real jobs through the sample's HTTP endpoints, drive the dashboard in a private headless Chrome, capture screenshot, accessibility, API, and database evidence, and clean up. Use to prove a BackWave change works end to end, to reproduce a bug that a user or operator would see, or whenever you need to see or screenshot the dashboard running. --- # Verify BackWave @@ -32,6 +32,7 @@ Each run owns these resources. All of them use the same run id (`YYYYMMDD-HHMMSS | HTTP port | `http://127.0.0.1:` | A free port from the OS. `bwv` never uses the default `5283` from `appsettings.json`. | | Scratch directory | `${TMPDIR}/bw-verify-/` | Holds the app copy and the SQLite file `backwave.db`. | | Postgres (only with `--store postgres`) | container `backwave-verify-`, label `backwave.verify.run=` | `postgres:17-alpine` on a free loopback port. It never uses the compose containers on `5398` or `14331`. | +| Oracle (only with `--store oracle`) | container `backwave-verify-`, label `backwave.verify.run=` | `gvenzl/oracle-free:23-slim` on a fixed free loopback port. It never uses the compose container on `15210`. | | Browser | `CHROME_DEVTOOLS_AXI_SESSION=bw-verify-` | A private bridge and an isolated headless Chrome profile. `bwv` never uses the `default` session. | | Evidence | `.verify-evidence//` at the worktree root | Gitignored. Cleanup never deletes it. | @@ -44,6 +45,7 @@ Parallel runs in one worktree share `bin/` and `obj/`. For this reason, `bwv lau ```bash bwv launch # SQLite co-resident store (the default) bwv launch --store postgres # a private Postgres container +bwv launch --store oracle # a private Oracle container (first boot 30-90 s) bwv launch --store inmemory # no database, fastest start ``` @@ -51,7 +53,7 @@ This command does these steps: 1. Records the run in `.verify-evidence//run.env` and makes it the current run. It records each resource name before it creates the resource. 2. Runs `dotnet build` on `samples/BackWave.Sample.Api` under the build lock, and copies `bin/Debug/net10.0` to the scratch directory. -3. For `--store postgres`, starts the run's container and waits until Postgres accepts TCP connections. +3. For `--store postgres`, starts the run's container and waits until Postgres accepts TCP connections. For `--store oracle`, starts the run's container and waits until its `healthcheck.sh` passes. 4. Starts the app copy with `--urls http://127.0.0.1:` and the store flags. It sets `DOTNET_hostBuilder__reloadConfigOnChange=false` (see [Gotchas](#gotchas)). 5. Waits until `/backwave` returns 200, then makes sure that `app.log` names the expected store. @@ -75,9 +77,9 @@ This is a read-only check. Run it after launch, before every proof, and first wh - The recorded PID still runs this run's `/app/BackWave.Sample.Api.dll`. - That PID holds the listening socket on the run's port. - `/backwave` serves the page with the title `Overview · BackWave`. -- `app.log` names the store: `SqliteCoResident`, `Postgres`, or `InMemory`. +- `app.log` names the store: `SqliteCoResident`, `Postgres`, `Oracle`, or `InMemory`. - The build matches the worktree. The fingerprint covers the committed files, the uncommitted edits, and the untracked files. It ignores docs, `*.md`, `.agents`, and `.claude`. If you edit product code after the launch, this check fails. -- The store exists: the SQLite file, or the Postgres container with the run's label. +- The store exists: the SQLite file, or the Postgres or Oracle container with the run's label. - The state of the run's browser session. The exit code is not zero when a check fails. Do not drive a run that fails doctor. Correct the cause, or run `bwv cleanup` and then `bwv launch`. @@ -166,7 +168,7 @@ Run cleanup after every failed iteration too. `bwv runs` lists every run in this | Helper | Invocation | |---|---| -| `scripts/bwv` | `bwv launch [--store sqlite\|postgres\|inmemory] \| doctor \| url [path] \| api [name] [json] \| wait-job [timeout-s] \| browser \| shot \| sql \| cleanup \| runs`. Run `bwv` with no arguments to see the usage. | +| `scripts/bwv` | `bwv launch [--store sqlite\|postgres\|oracle\|inmemory] \| doctor \| url [path] \| api [name] [json] \| wait-job [timeout-s] \| browser \| shot \| sql \| cleanup \| runs`. Run `bwv` with no arguments to see the usage. | | `chrome-devtools-axi` | Through `bwv browser` only, so that the run's session is always set. Run `chrome-devtools-axi --help` for the full command list. | | Swagger UI | `bwv url /swagger` prints the URL. The page lists every sample endpoint and its parameters. | @@ -181,4 +183,6 @@ Run cleanup after every failed iteration too. `bwv runs` lists every run in this - **The Overview counts only queues that hold jobs.** The Queues page also lists `limited` (concurrency limit 1, set at startup by the actor `startup`). - **SQL Server is not wired into `bwv`.** The sample supports `--BackWave:Store=SqlServer`, but `bwv` does not start a private SQL Server container. Do not point a run at the compose container on port `14331`: other test runs share it. - **`sqlite3 -readonly` fails on this WAL database.** `bwv sql` uses `PRAGMA query_only=1` instead. +- **To reproduce a store outage,** stop and start the run's own container: `docker stop -t 2 `, then `docker start `. `CONTAINER` is in `run.env`. The Oracle port is fixed, so the app finds the database again. On Oracle, `SHUTDOWN IMMEDIATE` and `STARTUP` through `docker exec -i sqlplus -S / as sysdba` give the restart codes (`ORA-01089`, `ORA-01109`). A host-run app cannot lose DNS, so `ORA-12545` needs the app in a container on a private Docker network. +- **The Oracle store has no business table.** `POST /tx` answers 409, and `bwv sql` queries the `backwave` schema, for example `SELECT RAWTOHEX(job_id), state, attempt FROM jobs`. - **macOS has no `timeout` command.** Use `bwv wait-job` or a loop with `sleep` to wait. diff --git a/.agents/skills/verify-backwave/features/README.md b/.agents/skills/verify-backwave/features/README.md index 7de14ce..cbb6600 100644 --- a/.agents/skills/verify-backwave/features/README.md +++ b/.agents/skills/verify-backwave/features/README.md @@ -6,7 +6,7 @@ This directory is the maintained source for verifying the user-facing behavior o - `bwv launch` printed `READY`, and `bwv doctor` passes every check on the run that you drive. - The run's Sample.Api starts with no jobs. Each recipe creates its own jobs through the sample's HTTP endpoints with `bwv api`. -- The run's store is its own SQLite file (the default) or its own Postgres container (`--store postgres`). The queues are `critical`, `bulk`, `low`, and `limited` (concurrency limit 1). +- The run's store is its own SQLite file (the default) its own Postgres container (`--store postgres`), or its own Oracle container (`--store oracle`). The queues are `critical`, `bulk`, `low`, and `limited` (concurrency limit 1). - The operator actor on the dashboard and on `/ops/*` is `sample-operator`. The startup limit on `limited` has the actor `startup`. - Never drive an app, a container, or a browser session that this run did not start. diff --git a/.agents/skills/verify-backwave/scripts/bwv b/.agents/skills/verify-backwave/scripts/bwv index 3793ef6..ddc8613 100755 --- a/.agents/skills/verify-backwave/scripts/bwv +++ b/.agents/skills/verify-backwave/scripts/bwv @@ -6,12 +6,12 @@ # - a scratch dir ${TMPDIR}/bw-verify-/ (app copy + SQLite file) # - a free port http://127.0.0.1: # - a browser session CHROME_DEVTOOLS_AXI_SESSION=bw-verify- -# - (postgres only) container backwave-verify-, label backwave.verify.run= +# - (postgres/oracle only) container backwave-verify-, label backwave.verify.run= # - an evidence dir .verify-evidence// (gitignored, survives cleanup) # Nothing shared is reused or killed. See ../SKILL.md for the full contract. # # Usage: -# bwv launch [--store sqlite|postgres|inmemory] new run: build, copy, start, wait for READY +# bwv launch [--store sqlite|postgres|oracle|inmemory] new run: build, copy, start, wait for READY # bwv doctor read-only health check of the current run # bwv url [path] print the run's base URL (plus an optional path) # bwv api [name] [json-body] call the Sample API -> /api/.json @@ -33,6 +33,7 @@ PROJECT="$ROOT/samples/BackWave.Sample.Api/BackWave.Sample.Api.csproj" BUILD_OUT="$ROOT/samples/BackWave.Sample.Api/bin/Debug/net10.0" DLL="BackWave.Sample.Api.dll" PG_IMAGE="postgres:17-alpine" +ORA_IMAGE="gvenzl/oracle-free:23-slim" STATES=(Scheduled AwaitingParent Leased Succeeded Cancelled DeadLettered Quarantined) log() { printf "\033[1;34mbwv:\033[0m %s\n" "$*" >&2; } @@ -141,14 +142,14 @@ cmd_launch() { local store=sqlite while (( $# )); do case "$1" in - --store) store="${2:?--store needs sqlite|postgres|inmemory}"; shift 2 ;; + --store) store="${2:?--store needs sqlite|postgres|oracle|inmemory}"; shift 2 ;; *) die "unknown launch flag '$1'" ;; esac done - [[ "$store" =~ ^(sqlite|postgres|inmemory)$ ]] || die "store must be sqlite, postgres or inmemory" + [[ "$store" =~ ^(sqlite|postgres|oracle|inmemory)$ ]] || die "store must be sqlite, postgres, oracle or inmemory" command -v dotnet >/dev/null || die "dotnet is not on PATH" command -v chrome-devtools-axi >/dev/null || die "chrome-devtools-axi is not on PATH" - if [[ "$store" == postgres ]]; then docker info >/dev/null 2>&1 || die "Docker is not running"; fi + if [[ "$store" =~ ^(postgres|oracle)$ ]]; then docker info >/dev/null 2>&1 || die "Docker is not running"; fi RUN="$(date +%Y%m%d-%H%M%S)-$$" RUN_DIR="$EVIDENCE_ROOT/$RUN" @@ -170,6 +171,7 @@ cmd_launch() { set_var BROWSER_STARTED 0 set_var CONTAINER "" set_var PG_PORT "" + set_var ORA_PORT "" set_var SQLITE_DB "" log "run $RUN ($store) - evidence at $RUN_DIR" mkdir -p "$SCRATCH" @@ -203,6 +205,26 @@ cmd_launch() { store_args=(--BackWave:Store=Postgres "--BackWave:Postgres:ConnectionString=Host=127.0.0.1;Port=$PG_PORT;Username=backwave;Password=backwave;Database=backwave_sample") set_var STORE_NAME Postgres ;; + oracle) + set_var CONTAINER "backwave-verify-$RUN" + # A fixed host port, so that the run can stop and start the container (a store outage) + # and the app finds the database again on the same port. + set_var ORA_PORT "$(free_port)" + log "starting $ORA_IMAGE as $CONTAINER on 127.0.0.1:$ORA_PORT (first boot takes 30-90 s)" + # Oracle needs far more shared memory than the 64 MB Docker default. + docker run -d --name "$CONTAINER" --label "backwave.verify.run=$RUN" --shm-size 2g \ + -e ORACLE_PASSWORD=backwave -e APP_USER=backwave -e APP_USER_PASSWORD=backwave \ + -p "127.0.0.1:$ORA_PORT:1521" "$ORA_IMAGE" > /dev/null \ + || die "docker run failed - run 'bwv cleanup'" + local i + for i in $(seq 1 180); do + docker exec "$CONTAINER" healthcheck.sh >/dev/null 2>&1 && break + (( i == 180 )) && die "Oracle was not ready in 180 s - run 'bwv cleanup'" + sleep 1 + done + store_args=(--BackWave:Store=Oracle + "--BackWave:Oracle:ConnectionString=User Id=backwave;Password=backwave;Data Source=127.0.0.1:$ORA_PORT/FREEPDB1") + set_var STORE_NAME Oracle ;; esac log "starting Sample.Api on $(base) (log: $RUN_DIR/app.log)" @@ -264,9 +286,9 @@ cmd_doctor() { case "$STORE" in sqlite) [[ -f "$SQLITE_DB" ]] && ok "SQLite file $SQLITE_DB" || bad "SQLite file $SQLITE_DB missing" ;; - postgres) + postgres|oracle) if container_is_ours && [[ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER")" == true ]]; then - ok "container $CONTAINER running on 127.0.0.1:$PG_PORT (label run=$RUN)" + ok "container $CONTAINER running on 127.0.0.1:${PG_PORT:-$ORA_PORT} (label run=$RUN)" else bad "container $CONTAINER is not running with label backwave.verify.run=$RUN" fi ;; @@ -368,6 +390,8 @@ cmd_sql() { sqlite) sqlite3 -header -column -cmd '.timeout 5000' "$SQLITE_DB" "PRAGMA query_only=1; $query" ;; postgres) docker exec -e PGOPTIONS='-c default_transaction_read_only=on' "$CONTAINER" \ psql -U backwave -d backwave_sample -v ON_ERROR_STOP=1 -c "$query" ;; + oracle) printf 'SET MARKUP CSV ON\nWHENEVER SQLERROR EXIT FAILURE\nSET TRANSACTION READ ONLY;\n%s;\nEXIT\n' "${query%;}" \ + | docker exec -i "$CONTAINER" sqlplus -S -L backwave/backwave@localhost/FREEPDB1 ;; esac } > "$out" 2>&1 || { cat "$out"; die "query failed (saved db/$name.txt)"; } cat "$out" diff --git a/samples/BackWave.Sample.Api/BackWave.Sample.Api.csproj b/samples/BackWave.Sample.Api/BackWave.Sample.Api.csproj index 43d995c..9a957c4 100644 --- a/samples/BackWave.Sample.Api/BackWave.Sample.Api.csproj +++ b/samples/BackWave.Sample.Api/BackWave.Sample.Api.csproj @@ -31,6 +31,7 @@ + The SQLite Embedded Adapter, co-resident: BackWave's tables live in the app's OWN file. public static readonly StoreKind Sqlite = new SqliteCoResidentStore(); @@ -33,6 +35,7 @@ public static StoreKind FromConfiguration(IConfiguration configuration) var configured = configuration["BackWave:Store"] ?? nameof(InMemory); return configured.Equals(nameof(Postgres), StringComparison.OrdinalIgnoreCase) ? Postgres : configured.Equals(nameof(SqlServer), StringComparison.OrdinalIgnoreCase) ? SqlServer + : configured.Equals(nameof(Oracle), StringComparison.OrdinalIgnoreCase) ? Oracle : configured.Equals(nameof(SqliteDedicated), StringComparison.OrdinalIgnoreCase) ? SqliteDedicated : configured.Equals(nameof(Sqlite), StringComparison.OrdinalIgnoreCase) ? Sqlite : InMemory; @@ -155,6 +158,31 @@ private static string ConnectionString(IConfiguration configuration) => configuration["BackWave:SqlServer:ConnectionString"] ?? DefaultConnectionString; } + /// + /// The Oracle adapter. The sample registers no EF provider for Oracle, so it has no business table and + /// forgoes Transactional Enqueue: POST /tx answers 409. + /// + private sealed class OracleStore : StoreKind + { + private const string DefaultConnectionString = + "User Id=backwave;Password=backwave;Data Source=localhost:15210/FREEPDB1"; + + public override string Name => "Oracle"; + public override bool SupportsTransactionalEnqueue => false; + + public override IJobStore CreateStore(IConfiguration configuration, ILoggerFactory loggerFactory) + => new OracleJobStore(new OracleStoreOptions + { + ConnectionString = configuration["BackWave:Oracle:ConnectionString"] ?? DefaultConnectionString, + AutoMigrate = true, + LoggerFactory = loggerFactory, // opts in to the migration log; null (the default) is silent + }); + + public override void AddAdapterTracing(TracerProviderBuilder tracing) => tracing.AddBackWaveOracleInstrumentation(); + + public override void AddAdapterMetrics(MeterProviderBuilder metrics) => metrics.AddBackWaveOracleInstrumentation(); + } + /// /// The SQLite Embedded Adapter (no server, no Docker — just a local file). Two deployments: /// puts BackWave's tables in the application's OWN database file From fb13405a5312f4a63f5d7e4d70d28a286815e932 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 11:49:18 +0300 Subject: [PATCH 05/27] test(oracle): use outage endpoints that no network can resolve or route The DNS case used a single-label host name. A DNS search domain or a resolver that rewrites NXDOMAIN can make that name resolve, and then the test gets ORA-50000. The timeout case used 10.255.255.1. A VPN or a cloud network that routes 10/8 can reject it fast with a different code. The tests now use bw-no-such-host.invalid (RFC 6761) and 192.0.2.1 (TEST-NET-1, RFC 5737). --- tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs index 4a0649a..bfe6fec 100644 --- a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -18,12 +18,13 @@ public sealed class OracleTransientFaultTests // Each endpoint reproduces one outage the way ODP.NET reports it to the pump. public static TheoryData Outages => new() { - // The host name stops resolving (a DNS or network outage). - { Descriptor("bw-no-such-host", 1521), 12545 }, + // The host name stops resolving (a DNS or network outage). DNS never resolves the .invalid domain. + { Descriptor("bw-no-such-host.invalid", 1521), 12545 }, // The host answers but nothing listens on the port (the database is down). { Descriptor("127.0.0.1", 1), 50201 }, - // The host does not answer at all, so the connect times out. - { Descriptor("10.255.255.1", 1521), 50000 }, + // The host does not answer at all, so the connect times out. 192.0.2.1 is in TEST-NET-1, which no + // network routes. + { Descriptor("192.0.2.1", 1521), 50000 }, }; [Theory] From b2ee92cc0c1c64ae135db63f73a857901835f84b Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 12:05:48 +0300 Subject: [PATCH 06/27] chore(sample): give the sample its own Oracle service The Oracle store defaulted to localhost:15210, which is the shared test database of the repo-root compose file. The sample compose file now has an Oracle service on port 15211, and the default points at it. The sample README lists the Oracle store. --- samples/BackWave.Sample.Api/README.md | 10 ++++++++-- samples/BackWave.Sample.Api/StoreKind.cs | 2 +- samples/BackWave.Sample.Api/docker-compose.yml | 16 ++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/samples/BackWave.Sample.Api/README.md b/samples/BackWave.Sample.Api/README.md index d2189a3..938077f 100644 --- a/samples/BackWave.Sample.Api/README.md +++ b/samples/BackWave.Sample.Api/README.md @@ -27,7 +27,7 @@ Then open: ## Switching stores Storage is chosen by the `BackWave:Store` config key: `InMemory` (default), `Sqlite`, -`SqliteDedicated`, `Postgres`, or `SqlServer`. +`SqliteDedicated`, `Postgres`, `SqlServer`, or `Oracle`. | Store | Infra needed | Transactional Enqueue (`POST /tx`) | | ----------------- | ---------------------- | ------------------------------------------ | @@ -36,6 +36,7 @@ Storage is chosen by the `BackWave:Store` config key: `InMemory` (default), `Sql | `SqliteDedicated` | none — a local file | returns **409** (BackWave on its own file) | | `Postgres` | `docker compose up -d` | ✅ commits/rolls back atomically | | `SqlServer` | `docker compose up -d` | ✅ commits/rolls back atomically | +| `Oracle` | `docker compose up -d` | returns **409** (no business table) | The **SQLite Embedded Adapter** needs no server — just a local `.db` file (written under the content root). Two deployments demonstrate the spectrum: @@ -65,10 +66,15 @@ BackWave__Store=Postgres dotnet run # SQL Server (amd64 image; slow under Rosetta on Apple Silicon) docker compose up -d sqlserver BackWave__Store=SqlServer dotnet run + +# Oracle (no business table, so POST /tx returns 409) +docker compose up -d oracle +BackWave__Store=Oracle dotnet run ``` Connection strings default to the compose values and can be overridden via -`BackWave:Postgres:ConnectionString` / `BackWave:SqlServer:ConnectionString`; the SQLite file paths +`BackWave:Postgres:ConnectionString` / `BackWave:SqlServer:ConnectionString` / +`BackWave:Oracle:ConnectionString`; the SQLite file paths via `BackWave:Sqlite:DataSource` (and `BackWave:Sqlite:BusinessDataSource` for the dedicated business file) — see `appsettings.json`. diff --git a/samples/BackWave.Sample.Api/StoreKind.cs b/samples/BackWave.Sample.Api/StoreKind.cs index d88132d..9126260 100644 --- a/samples/BackWave.Sample.Api/StoreKind.cs +++ b/samples/BackWave.Sample.Api/StoreKind.cs @@ -165,7 +165,7 @@ private static string ConnectionString(IConfiguration configuration) private sealed class OracleStore : StoreKind { private const string DefaultConnectionString = - "User Id=backwave;Password=backwave;Data Source=localhost:15210/FREEPDB1"; + "User Id=backwave;Password=backwave;Data Source=localhost:15211/FREEPDB1"; public override string Name => "Oracle"; public override bool SupportsTransactionalEnqueue => false; diff --git a/samples/BackWave.Sample.Api/docker-compose.yml b/samples/BackWave.Sample.Api/docker-compose.yml index 3b80e02..4e36398 100644 --- a/samples/BackWave.Sample.Api/docker-compose.yml +++ b/samples/BackWave.Sample.Api/docker-compose.yml @@ -30,3 +30,19 @@ services: interval: 5s timeout: 5s retries: 60 + + oracle: + image: gvenzl/oracle-free:23-slim + # Oracle needs far more shared memory than Docker's 64 MB default. + shm_size: 2gb + environment: + ORACLE_PASSWORD: backwave + APP_USER: backwave + APP_USER_PASSWORD: backwave + ports: + - "15211:1521" + healthcheck: + test: ["CMD", "healthcheck.sh"] + interval: 5s + timeout: 5s + retries: 60 From 97fb43edffd043db2d3d4306b6a7fcdbfdedb675 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 11:49:26 +0300 Subject: [PATCH 07/27] fix(oracle): classify a pooled connection request timeout as transient A pool with no free connection raises ORA-50012, not ORA-50000. ODP.NET 23.9.1 sets IsTransient to false on it, so a busy node halted its worker group. Add 50012 to the transient connectivity codes and fix the comment on 50000. A live test holds the one connection of a pool of one and checks the store classifies the fault transient. --- src/BackWave.Oracle/OracleFaultCodes.cs | 5 ++- .../OraclePoolExhaustionFaultTests.cs | 38 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 tests/BackWave.Oracle.Tests/OraclePoolExhaustionFaultTests.cs diff --git a/src/BackWave.Oracle/OracleFaultCodes.cs b/src/BackWave.Oracle/OracleFaultCodes.cs index fd3f566..434e4d1 100644 --- a/src/BackWave.Oracle/OracleFaultCodes.cs +++ b/src/BackWave.Oracle/OracleFaultCodes.cs @@ -14,11 +14,12 @@ namespace BackWave.Oracle; // 12545 target host does not exist (the host name stops resolving in a network outage) // ODP.NET managed raises its own codes for a connect that fails before any listener answers: // 50201 failed to connect to server (connection refused: the database is down) -// 50000 connection request timed out (the host does not answer, or the pool has no free connection) +// 50000 connection request timed out (the host does not answer) +// 50012 pooled connection request timed out (the pool has no free connection) internal static class OracleFaultCodes { // True when the ORA code is one of the transient connectivity/timeout faults above. internal static bool IsConnectivityFault(int code) => code is 12170 or 12541 or 12514 or 12518 or 12537 or 12570 or 3113 or 3114 or 28 or 1033 or 12154 - or 1089 or 1109 or 12545 or 50201 or 50000; + or 1089 or 1109 or 12545 or 50201 or 50000 or 50012; } diff --git a/tests/BackWave.Oracle.Tests/OraclePoolExhaustionFaultTests.cs b/tests/BackWave.Oracle.Tests/OraclePoolExhaustionFaultTests.cs new file mode 100644 index 0000000..5f81e0e --- /dev/null +++ b/tests/BackWave.Oracle.Tests/OraclePoolExhaustionFaultTests.cs @@ -0,0 +1,38 @@ +using BackWave.Storage; +using Oracle.ManagedDataAccess.Client; + +namespace BackWave.Oracle.Tests; + +/// +/// Issue #12: a pump that waits for a pooled connection and does not get one in time must see a transient +/// fault, not a fail-stop. A busy node can drain its pool without any fault in the database. This is an +/// integration test: it needs "docker compose up -d oracle". +/// +[Collection("oracle")] +public sealed class OraclePoolExhaustionFaultTests(Xunit.Abstractions.ITestOutputHelper output) +{ + [Fact] + public async Task A_pool_with_no_free_connection_raises_a_fault_the_adapter_classifies_transient() + { + await OracleTestDatabase.CreateFreshStoreAsync(); + + // A pool of one connection, keyed by this exact string, so no other test shares it. + var connectionString = new OracleConnectionStringBuilder(OracleTestDatabase.ConnectionString) + { + MaxPoolSize = 1, + ConnectionTimeout = 2, + }.ConnectionString; + var store = new OracleJobStore(new OracleStoreOptions { ConnectionString = connectionString }); + + // Hold the one pooled connection, so the store's next open must wait for it. + await using var held = new OracleConnection(connectionString); + await held.OpenAsync(); + + var fault = await Assert.ThrowsAsync(async () => + await store.ClaimAsync(new ClaimRequest("w1", ["default"], 10, TimeSpan.FromMinutes(1), DateTimeOffset.UtcNow))); + output.WriteLine($"ORA-{fault.Number}: {fault.Message} (IsTransient={fault.IsTransient})"); + + Assert.Equal(50012, fault.Number); + Assert.True(store.IsTransientFault(fault), $"ORA-{fault.Number} must classify transient"); + } +} From 50bf57cb2232caa8791303c88a440951cb1550f8 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 12:05:48 +0300 Subject: [PATCH 08/27] docs(oracle): bring the transient fault code comments up to date The header now covers a store outage too: a pump that loses its database degrades and retries. The 50xxx codes come from ODP.NET, so the list is of ORA or ODP.NET codes. A wrong service name on a live listener also raises ORA-50201 (measured against the test database). A mistyped host name raises ORA-12545, so it leaves the group degraded and never halts it. --- src/BackWave.Oracle/OracleFaultCodes.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/BackWave.Oracle/OracleFaultCodes.cs b/src/BackWave.Oracle/OracleFaultCodes.cs index 434e4d1..1ac7790 100644 --- a/src/BackWave.Oracle/OracleFaultCodes.cs +++ b/src/BackWave.Oracle/OracleFaultCodes.cs @@ -2,8 +2,10 @@ namespace BackWave.Oracle; // The single source of truth for the transient Oracle connectivity/timeout error codes. A cold-booting // fleet can hit these listener/handshake-storm and connection-lost conditions without having caused them, -// so the migrator rides them out and the store tags them transient. Both reference this list so the two -// stay in lockstep. +// and a pump that loses its database in a store outage (network loss, a stop, a restart) sees the same +// codes until the database answers again. So the migrator rides them out and the store tags them +// transient: the worker group degrades and retries on the next poll instead of halting. Both reference +// this list so the two stay in lockstep. // 12170 TNS connect timeout 12541 no listener // 12514 listener does not know service 12518 listener could not hand off // 12537 connection closed 12570 packet reader failure @@ -12,13 +14,16 @@ namespace BackWave.Oracle; // 1089 shutdown in progress 1109 database not open (a restart opens it after the listener) // 12154 could not resolve connect id (transient during cold DNS/TNS) // 12545 target host does not exist (the host name stops resolving in a network outage) -// ODP.NET managed raises its own codes for a connect that fails before any listener answers: -// 50201 failed to connect to server (connection refused: the database is down) +// A mistyped host name also raises 12545 (like 12154), so it leaves the group degraded and never halts it. +// ODP.NET managed raises its own codes when a connect fails: +// 50201 failed to connect to server (connection refused while the database is down, or a service name +// the listener does not know during a restart; a wrong service name also raises it, so that too +// leaves the group degraded and never halts it) // 50000 connection request timed out (the host does not answer) // 50012 pooled connection request timed out (the pool has no free connection) internal static class OracleFaultCodes { - // True when the ORA code is one of the transient connectivity/timeout faults above. + // True when the ORA or ODP.NET code is one of the transient connectivity/timeout faults above. internal static bool IsConnectivityFault(int code) => code is 12170 or 12541 or 12514 or 12518 or 12537 or 12570 or 3113 or 3114 or 28 or 1033 or 12154 or 1089 or 1109 or 12545 or 50201 or 50000 or 50012; From b3b7c10d61bfc03a76a16719989f7916314cee3a Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 12:27:57 +0300 Subject: [PATCH 09/27] test(oracle): prove a worker group runs jobs again after its sessions are killed The outage tests only showed that a group degrades instead of halting. No test showed that it claims and runs jobs again, which is the symptom in issue #12. The new test runs a real host as a private bw_recover user. It runs one job, kills every session of that user as SYSTEM, and checks that the group degrades with ORA-00028 and never halts. Then a new job must reach Succeeded on the same group. The kill is not IMMEDIATE. ODP.NET drops a session with a closed socket from the pool before it hands it out, so only a plain kill makes the next claim run on a dead pooled session. At teardown the test locks the user, so the pool cannot open a new session, and disconnects every session of the user before DROP USER. --- .../OracleOutageRecoveryTests.cs | 253 ++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs diff --git a/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs b/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs new file mode 100644 index 0000000..714029b --- /dev/null +++ b/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs @@ -0,0 +1,253 @@ +using System.Text.Json.Serialization; +using BackWave.Core; +using BackWave.Hosting; +using BackWave.Jobs; +using BackWave.Storage; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Oracle.ManagedDataAccess.Client; + +namespace BackWave.Oracle.Tests; + +/// +/// Issue #12: after an outage a worker group on Oracle must claim and run jobs again without a process +/// restart. The outage is the database killing every session of the group while the pool still holds +/// them, so the next claim runs on a dead connection. The group runs as a private bw_recover user, so the +/// kill reaches this test's sessions and no other user's. +/// +[Collection("oracle")] +public sealed class OracleOutageRecoveryTests(Xunit.Abstractions.ITestOutputHelper output) : IAsyncLifetime +{ + private const string Schema = "bw_recover"; + private const string Group = "recovery-workers"; + + // Pooling stays on (the default): the dead pooled session is the outage this test models. A minimum + // pool size of zero stops the pool from opening a fresh session after ClearPool, which would block + // the DROP USER at teardown. + private static readonly string SchemaConnectionString = + new OracleConnectionStringBuilder(OracleTestDatabase.ConnectionString) + { + UserID = Schema, + Password = Schema, + MinPoolSize = 0, + }.ConnectionString; + + public Task InitializeAsync() => OracleTestDatabase.ResetUserAsync(Schema); + + public async Task DisposeAsync() + { + // Lock the user, so the pool cannot open a new session, then end every session it still has, so + // the DROP USER does not wait on them. + await LockUserAsync(); + OracleConnection.ClearPool(new OracleConnection(SchemaConnectionString)); + await DisconnectSessionsAsync(); + await OracleTestDatabase.DropUserAsync(Schema); + } + + [Fact] + public async Task A_worker_group_claims_and_runs_jobs_again_after_its_pooled_sessions_are_killed() + { + var store = new OracleJobStore(new OracleStoreOptions + { + ConnectionString = SchemaConnectionString, + SchemaName = Schema, + AutoMigrate = true, + }); + var builder = Host.CreateApplicationBuilder(); + builder.Services.AddTransient, RecoveryHandler>(); + builder.Services.AddBackWave(backwave => backwave + .UseStore(store) + .UseRegistry(new JobRegistry( + [ + JobRegistration.Create("recovery", OracleRecoveryJsonContext.Default.RecoveryJob), + ])) + .AddWorkerGroup(new WorkerGroupOptions + { + Name = Group, + Policy = new DispatchPolicy.Strict(["default"]), + PollInterval = TimeSpan.FromMilliseconds(200), + MaxPollInterval = TimeSpan.Zero, + LeaseDuration = TimeSpan.FromSeconds(30), + })); + using var host = builder.Build(); + var health = host.Services.GetRequiredService(); + var client = host.Services.GetRequiredService(); + + await host.StartAsync(); + try + { + // Baseline: the group runs a job before the outage. + var before = await client.EnqueueAsync(new RecoveryJob(1), dueTime: DateTimeOffset.UtcNow); + await WaitForSucceededAsync(store, health, before); + + var killed = await KillSessionsAsync(); + output.WriteLine($"killed {killed} {Schema} sessions"); + Assert.True(killed > 0, $"the group must hold at least one {Schema} session to kill"); + + // The next poll takes a killed session from the pool. The group must degrade, not halt. + var cause = await WaitForDegradedAsync(health); + output.WriteLine($"degraded: {cause}"); + Assert.Contains("ORA-00028", cause); + + // Each failed poll drops one dead session from the pool. Wait until none is left, so the + // enqueue below cannot take one either. + await WaitForAsync(async () => await CountKilledSessionsAsync() == 0, health, "the pool to drop every killed session"); + + // Recovery: a job enqueued after the outage is claimed and run by the same group. + var after = await client.EnqueueAsync(new RecoveryJob(2), dueTime: DateTimeOffset.UtcNow); + await WaitForSucceededAsync(store, health, after); + AssertNotHalted(health); + } + finally + { + await host.StopAsync(); + } + } + + // Kills every session of the private user as SYSTEM. The filter is the username alone, so no session + // of another user or test is touched. The outage kill is not IMMEDIATE on purpose: IMMEDIATE closes + // the socket, and ODP.NET then drops the session from the pool before it hands it out. A plain kill + // keeps the socket open, so the pool hands out the dead session and its next call raises the fault. + private static async Task KillSessionsAsync() + { + await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); + await admin.OpenAsync(); + + var sessions = new List(); + await using (var select = admin.CreateCommand()) + { + select.CommandText = "SELECT sid, serial# FROM v$session WHERE username = :username AND status <> 'KILLED'"; + select.Parameters.Add(new OracleParameter("username", Schema.ToUpperInvariant())); + await using var reader = await select.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + sessions.Add($"{reader.GetInt32(0)},{reader.GetInt32(1)}"); + } + } + + foreach (var session in sessions) + { + await using var kill = admin.CreateCommand(); + kill.CommandText = $"ALTER SYSTEM KILL SESSION '{session}'"; + try + { + await kill.ExecuteNonQueryAsync(); + } + // ORA-00031: the session is busy, so Oracle marks it and ends it on its next call. + // ORA-00030: the session ended after the select. + catch (OracleException exception) when (exception.Number is 31 or 30) + { + } + } + return sessions.Count; + } + + private static async Task LockUserAsync() + { + await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); + await admin.OpenAsync(); + await using var lockUser = admin.CreateCommand(); + lockUser.CommandText = $"ALTER USER {Schema} ACCOUNT LOCK"; + await lockUser.ExecuteNonQueryAsync(); + } + + // Ends every session of the private user, a killed one too, and waits until none is left. A killed + // session holds its process until its client calls again, and the client is gone at teardown. + private static async Task DisconnectSessionsAsync() + { + await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); + await admin.OpenAsync(); + + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (true) + { + var sessions = new List(); + await using (var select = admin.CreateCommand()) + { + select.CommandText = "SELECT sid, serial# FROM v$session WHERE username = :username"; + select.Parameters.Add(new OracleParameter("username", Schema.ToUpperInvariant())); + await using var reader = await select.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + sessions.Add($"{reader.GetInt32(0)},{reader.GetInt32(1)}"); + } + } + if (sessions.Count == 0 || DateTime.UtcNow > deadline) + { + return; + } + + foreach (var session in sessions) + { + await using var disconnect = admin.CreateCommand(); + disconnect.CommandText = $"ALTER SYSTEM DISCONNECT SESSION '{session}' IMMEDIATE"; + try + { + await disconnect.ExecuteNonQueryAsync(); + } + catch (OracleException exception) when (exception.Number is 31 or 30) + { + } + } + await Task.Delay(200); + } + } + + // A killed session stays in v$session until its client makes a call and is told it is dead. + private static async Task CountKilledSessionsAsync() + { + await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); + await admin.OpenAsync(); + await using var count = admin.CreateCommand(); + count.CommandText = "SELECT count(*) FROM v$session WHERE username = :username AND status = 'KILLED'"; + count.Parameters.Add(new OracleParameter("username", Schema.ToUpperInvariant())); + return Convert.ToInt32(await count.ExecuteScalarAsync()); + } + + // Samples the health state until the group is degraded. The next clean poll clears the mark, so the + // sample interval stays well under the poll interval. + private static async Task WaitForDegradedAsync(BackWaveHealth health) + { + var cause = ""; + await WaitForAsync(() => Task.FromResult(health.DegradedGroups.TryGetValue(Group, out cause!)), health, + "the killed sessions to mark the group degraded", TimeSpan.FromMilliseconds(5)); + return cause; + } + + private static Task WaitForSucceededAsync(OracleJobStore store, BackWaveHealth health, Guid jobId) => + WaitForAsync(async () => (await store.GetJobAsync(jobId))?.State == JobState.Succeeded, health, + $"job {jobId} to reach Succeeded"); + + // Polls the condition and fails the test the moment the group halts, so a fail-stop reports its cause + // instead of a timeout. + private static async Task WaitForAsync( + Func> condition, BackWaveHealth health, string description, TimeSpan? interval = null) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline) + { + AssertNotHalted(health); + if (await condition()) + { + return; + } + await Task.Delay(interval ?? TimeSpan.FromMilliseconds(50)); + } + AssertNotHalted(health); + Assert.Fail($"Timed out waiting for: {description}"); + } + + private static void AssertNotHalted(BackWaveHealth health) => + Assert.False(health.HaltedGroups.TryGetValue(Group, out var halt), + $"the outage fail-stopped the group: {halt?.ExceptionType}: {halt?.Message}"); +} + +public sealed record RecoveryJob(int Index); + +public sealed class RecoveryHandler : IJobHandler +{ + public Task HandleAsync(RecoveryJob job, JobContext context, CancellationToken cancellationToken) => Task.CompletedTask; +} + +[JsonSerializable(typeof(RecoveryJob))] +internal sealed partial class OracleRecoveryJsonContext : JsonSerializerContext; From 8f716e79bdcb65e0769aec06df2e58f0ee5b00ef Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 12:55:30 +0300 Subject: [PATCH 10/27] fix(oracle): bound the migrator's transient retry by elapsed time The coordinated migration retries a transient connect fault up to eight times. Since the connect-timeout code is transient, a host that does not answer costs a full Connection Timeout on each attempt. With the default 15 s timeout one migration call took about 2 min to fail, and a permanent misconfiguration was reported late. The retry now also stops once 5 s have passed, and the last fault propagates. The eight attempts still ride out a cold-boot handshake storm, because those faults come back fast and the backoff adds only 1.4 s. An auto-migrating store runs the migration again on its next poll, so an outage still retries at the pump. --- src/BackWave.Oracle/OracleMigrator.cs | 10 ++++++++-- .../OracleTransientFaultTests.cs | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/BackWave.Oracle/OracleMigrator.cs b/src/BackWave.Oracle/OracleMigrator.cs index 7aa206a..918ab20 100644 --- a/src/BackWave.Oracle/OracleMigrator.cs +++ b/src/BackWave.Oracle/OracleMigrator.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using BackWave.Diagnostics; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -78,8 +79,12 @@ public static async Task MigrateAsync( // listener/handshake faults it did nothing to cause. The schema script is idempotent and // concurrency-safe by construction (guarded DDL + DDL serialized on each object by Oracle), so // re-running it converges; ride out a bounded number of transient faults instead of crashing - // startup. On exhaustion the last fault propagates (fail-stop preserved). + // startup. The budget bounds the retry by time as well: a host that does not answer costs a full + // connect timeout on each attempt, and an auto-migrating store runs the migration again on its + // next poll. On exhaustion the last fault propagates (fail-stop preserved). const int maxAttempts = 8; + var budget = TimeSpan.FromSeconds(5); + var started = Stopwatch.GetTimestamp(); for (var attempt = 1; ; attempt++) { try @@ -87,7 +92,8 @@ public static async Task MigrateAsync( await ApplyScriptsAsync(connectionString, rewriter, cancellationToken).ConfigureAwait(false); return; } - catch (OracleException exception) when (attempt < maxAttempts && IsTransient(exception)) + catch (OracleException exception) when (attempt < maxAttempts && IsTransient(exception) + && Stopwatch.GetElapsedTime(started) < budget) { await Task.Delay(TimeSpan.FromMilliseconds(50 * attempt), cancellationToken).ConfigureAwait(false); } diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs index bfe6fec..17bed40 100644 --- a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using BackWave.Core; using BackWave.Hosting; using BackWave.Jobs; @@ -80,6 +81,21 @@ public async Task A_network_outage_degrades_the_worker_group_instead_of_halting_ } } + // A host that does not answer costs a full connect timeout on every attempt, so the migrator's retry + // stops on its time budget instead of waiting out all eight attempts (about 17 s here). + [Fact] + public async Task An_unreachable_host_fails_the_migration_once_the_retry_budget_is_spent() + { + var connectionString = + $"User Id=backwave;Password=backwave;Connection Timeout=2;Data Source={Descriptor("192.0.2.1", 1521)}"; + var elapsed = Stopwatch.StartNew(); + + var fault = await Assert.ThrowsAsync(() => OracleMigrator.MigrateAsync(connectionString)); + + Assert.Equal(50000, fault.Number); + Assert.True(elapsed.Elapsed < TimeSpan.FromSeconds(10), $"the migration took {elapsed.Elapsed.TotalSeconds:F1} s"); + } + private static string Descriptor(string host, int port) => $"(DESCRIPTION=(CONNECT_TIMEOUT=2)(ADDRESS=(PROTOCOL=TCP)(HOST={host})(PORT={port}))" + "(CONNECT_DATA=(SERVICE_NAME=FREEPDB1)))"; From 924f88dacf6363396452a3b35af24937db6d2b04 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 13:36:20 +0300 Subject: [PATCH 11/27] fix(oracle): always give the migrator three attempts before the time budget applies One slow transient fault, for example a TNS connect timeout with the default 15 s Connection Timeout, used the whole 5 s budget on the first attempt, so the migrator did not retry at all. An auto-migrating store tries again on its next poll, but a caller that runs MigrateAsync once at startup has no next poll, and that caller lost the cold-boot retry. The first three attempts now always retry. The 5 s budget applies after them, and the limit of eight attempts stays. The rule is one internal method, so a test covers it without a slow connect. --- src/BackWave.Oracle/OracleMigrator.cs | 17 +++++++++++++---- .../OracleTransientFaultTests.cs | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/BackWave.Oracle/OracleMigrator.cs b/src/BackWave.Oracle/OracleMigrator.cs index 918ab20..83b3c80 100644 --- a/src/BackWave.Oracle/OracleMigrator.cs +++ b/src/BackWave.Oracle/OracleMigrator.cs @@ -82,8 +82,6 @@ public static async Task MigrateAsync( // startup. The budget bounds the retry by time as well: a host that does not answer costs a full // connect timeout on each attempt, and an auto-migrating store runs the migration again on its // next poll. On exhaustion the last fault propagates (fail-stop preserved). - const int maxAttempts = 8; - var budget = TimeSpan.FromSeconds(5); var started = Stopwatch.GetTimestamp(); for (var attempt = 1; ; attempt++) { @@ -92,14 +90,25 @@ public static async Task MigrateAsync( await ApplyScriptsAsync(connectionString, rewriter, cancellationToken).ConfigureAwait(false); return; } - catch (OracleException exception) when (attempt < maxAttempts && IsTransient(exception) - && Stopwatch.GetElapsedTime(started) < budget) + catch (OracleException exception) when (IsTransient(exception) + && ShouldRetry(attempt, Stopwatch.GetElapsedTime(started))) { await Task.Delay(TimeSpan.FromMilliseconds(50 * attempt), cancellationToken).ConfigureAwait(false); } } } + // True when a transient fault on this attempt gets another attempt. The first attempts always retry, + // because one slow fault can use the whole budget and a caller that migrates once has no next poll. + internal static bool ShouldRetry(int attempt, TimeSpan elapsed) + { + const int maxAttempts = 8; + const int minAttempts = 3; + const int budgetSeconds = 5; + return attempt < maxAttempts + && (attempt < minAttempts || elapsed < TimeSpan.FromSeconds(budgetSeconds)); + } + // Runs every embedded schema script in version order. Each script is a single anonymous PL/SQL block // (guarded object-by-object), so one ExecuteNonQuery per script is correct and Oracle's implicit DDL // commit needs no explicit transaction. diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs index 17bed40..10d8203 100644 --- a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -96,6 +96,21 @@ public async Task An_unreachable_host_fails_the_migration_once_the_retry_budget_ Assert.True(elapsed.Elapsed < TimeSpan.FromSeconds(10), $"the migration took {elapsed.Elapsed.TotalSeconds:F1} s"); } + // A slow fault, for example a TNS connect timeout, can use the whole time budget on the first attempt. + // A caller that migrates once at startup has no next poll, so the first three attempts always run. + [Theory] + [InlineData(1, 0, true)] + [InlineData(1, 15, true)] + [InlineData(2, 30, true)] + [InlineData(3, 4, true)] + [InlineData(3, 45, false)] + [InlineData(7, 4, true)] + [InlineData(8, 0, false)] + public void The_migrator_retries_three_attempts_before_the_budget_applies(int attempt, int elapsedSeconds, bool retries) + { + Assert.Equal(retries, OracleMigrator.ShouldRetry(attempt, TimeSpan.FromSeconds(elapsedSeconds))); + } + private static string Descriptor(string host, int port) => $"(DESCRIPTION=(CONNECT_TIMEOUT=2)(ADDRESS=(PROTOCOL=TCP)(HOST={host})(PORT={port}))" + "(CONNECT_DATA=(SERVICE_NAME=FREEPDB1)))"; From 06d93ca20575bdbc969205e54698bf90ea226cbf Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 13:37:13 +0300 Subject: [PATCH 12/27] docs(oracle): name every misconfiguration that the transient codes leave degraded A probe with ODP.NET 23.9.1 showed what each kind of typo in a connection string raises: - A descriptor with a bad keyword raises ORA-50201 "failed to parse connect string". - A mistyped EZConnect string or TNS alias raises ORA-12154. - A wrong attribute name fails the open with an ArgumentException (ORA-50008). - A wrong password raises ORA-01017. The first two are transient, so they leave the group degraded. The last two halt it. The comment now lists all of them in one place. OracleConnectionStringBuilder catches only the attribute-name typos, and those already halt, so the store does not parse the string up front. --- src/BackWave.Oracle/OracleFaultCodes.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/BackWave.Oracle/OracleFaultCodes.cs b/src/BackWave.Oracle/OracleFaultCodes.cs index 1ac7790..afde701 100644 --- a/src/BackWave.Oracle/OracleFaultCodes.cs +++ b/src/BackWave.Oracle/OracleFaultCodes.cs @@ -14,13 +14,16 @@ namespace BackWave.Oracle; // 1089 shutdown in progress 1109 database not open (a restart opens it after the listener) // 12154 could not resolve connect id (transient during cold DNS/TNS) // 12545 target host does not exist (the host name stops resolving in a network outage) -// A mistyped host name also raises 12545 (like 12154), so it leaves the group degraded and never halts it. // ODP.NET managed raises its own codes when a connect fails: -// 50201 failed to connect to server (connection refused while the database is down, or a service name -// the listener does not know during a restart; a wrong service name also raises it, so that too -// leaves the group degraded and never halts it) +// 50201 failed to connect to server or failed to parse connect string (connection refused while the +// database is down, or a service name the listener does not know during a restart) // 50000 connection request timed out (the host does not answer) // 50012 pooled connection request timed out (the pool has no free connection) +// A misconfigured Data Source raises some of these codes too, so it leaves the group degraded and never +// halts it: a mistyped host name (12545), a mistyped EZConnect string or TNS alias (12154), and a wrong +// service name or a descriptor that ODP.NET cannot parse (50201). A wrong attribute name in the +// connection string fails the open with an ArgumentException, and a wrong password raises 1017. Neither +// is in this list, so both halt the group. internal static class OracleFaultCodes { // True when the ORA or ODP.NET code is one of the transient connectivity/timeout faults above. From 0743d4a6e8c47e86a2ebd3898852324adc250d40 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 13:38:39 +0300 Subject: [PATCH 13/27] test(oracle): assert that an outage code is transient, not that it is one exact code The outage tests expected one exact code per endpoint. A VPN or a CI runner can answer 192.0.2.1 with ICMP unreachable, and a kill on a busy session can surface as 3113 or 3135 instead of 28. The classification is still correct then, but the tests failed. The tests now write the code to the test output and assert that the adapter classifies it transient. A code that is not in the list still fails them. The comments keep the code measured for each endpoint. --- .../OracleOutageRecoveryTests.cs | 2 +- .../OracleTransientFaultTests.cs | 39 ++++++++++++------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs b/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs index 714029b..cc3513c 100644 --- a/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs @@ -87,7 +87,7 @@ public async Task A_worker_group_claims_and_runs_jobs_again_after_its_pooled_ses // The next poll takes a killed session from the pool. The group must degrade, not halt. var cause = await WaitForDegradedAsync(health); output.WriteLine($"degraded: {cause}"); - Assert.Contains("ORA-00028", cause); + Assert.True(OracleTransientFaultTests.NamesTransientCode(cause), $"the degraded cause must name a transient code: {cause}"); // Each failed poll drops one dead session from the pool. Wait until none is left, so the // enqueue below cannot take one either. diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs index 10d8203..5563c74 100644 --- a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Text.RegularExpressions; using BackWave.Core; using BackWave.Hosting; using BackWave.Jobs; @@ -14,36 +15,39 @@ namespace BackWave.Oracle.Tests; /// fail-stop it. ODP.NET raises these faults while it opens a connection, so each case points a real store /// at an endpoint that cannot answer and needs no database. /// -public sealed class OracleTransientFaultTests +public sealed class OracleTransientFaultTests(Xunit.Abstractions.ITestOutputHelper output) { - // Each endpoint reproduces one outage the way ODP.NET reports it to the pump. - public static TheoryData Outages => new() + // Each endpoint reproduces one outage the way ODP.NET reports it to the pump. The code in each comment + // is the one measured here. A network can report the same outage with another code, so the tests + // assert that the code is transient, not that it is this one. + public static TheoryData Outages => new() { // The host name stops resolving (a DNS or network outage). DNS never resolves the .invalid domain. - { Descriptor("bw-no-such-host.invalid", 1521), 12545 }, - // The host answers but nothing listens on the port (the database is down). - { Descriptor("127.0.0.1", 1), 50201 }, + // ORA-12545. + Descriptor("bw-no-such-host.invalid", 1521), + // The host answers but nothing listens on the port (the database is down). ORA-50201. + Descriptor("127.0.0.1", 1), // The host does not answer at all, so the connect times out. 192.0.2.1 is in TEST-NET-1, which no - // network routes. - { Descriptor("192.0.2.1", 1521), 50000 }, + // network routes. ORA-50000. + Descriptor("192.0.2.1", 1521), }; [Theory] [MemberData(nameof(Outages))] - public async Task A_network_outage_raises_a_fault_the_adapter_classifies_transient(string dataSource, int expected) + public async Task A_network_outage_raises_a_fault_the_adapter_classifies_transient(string dataSource) { var store = Store(dataSource); var fault = await Assert.ThrowsAsync(async () => await store.ClaimAsync(new ClaimRequest("w1", ["default"], 10, TimeSpan.FromMinutes(1), DateTimeOffset.UtcNow))); - Assert.Equal(expected, fault.Number); + output.WriteLine($"ORA-{fault.Number}: {fault.Message}"); Assert.True(store.IsTransientFault(fault), $"ORA-{fault.Number} must classify transient"); } [Theory] [MemberData(nameof(Outages))] - public async Task A_network_outage_degrades_the_worker_group_instead_of_halting_it(string dataSource, int expected) + public async Task A_network_outage_degrades_the_worker_group_instead_of_halting_it(string dataSource) { var builder = Host.CreateApplicationBuilder(); builder.Services.AddBackWave(backwave => backwave @@ -73,7 +77,8 @@ public async Task A_network_outage_degrades_the_worker_group_instead_of_halting_ Assert.False(health.HaltedGroups.TryGetValue("outage-workers", out var halt), $"the outage fail-stopped the group: {halt?.ExceptionType}: {halt?.Message}"); Assert.True(health.DegradedGroups.TryGetValue("outage-workers", out var cause), "the outage must mark the group degraded"); - Assert.Contains($"ORA-{expected}", cause); + output.WriteLine($"degraded: {cause}"); + Assert.True(NamesTransientCode(cause), $"the degraded cause must name a transient code: {cause}"); } finally { @@ -92,7 +97,8 @@ public async Task An_unreachable_host_fails_the_migration_once_the_retry_budget_ var fault = await Assert.ThrowsAsync(() => OracleMigrator.MigrateAsync(connectionString)); - Assert.Equal(50000, fault.Number); + output.WriteLine($"ORA-{fault.Number}: {fault.Message}"); + Assert.True(OracleFaultCodes.IsConnectivityFault(fault.Number), $"ORA-{fault.Number} must be transient"); Assert.True(elapsed.Elapsed < TimeSpan.FromSeconds(10), $"the migration took {elapsed.Elapsed.TotalSeconds:F1} s"); } @@ -111,6 +117,13 @@ public void The_migrator_retries_three_attempts_before_the_budget_applies(int at Assert.Equal(retries, OracleMigrator.ShouldRetry(attempt, TimeSpan.FromSeconds(elapsedSeconds))); } + // True when the cause names an ORA or ODP.NET code that the adapter classifies transient. + internal static bool NamesTransientCode(string cause) + { + var code = Regex.Match(cause, @"ORA-(\d+)"); + return code.Success && OracleFaultCodes.IsConnectivityFault(int.Parse(code.Groups[1].Value)); + } + private static string Descriptor(string host, int port) => $"(DESCRIPTION=(CONNECT_TIMEOUT=2)(ADDRESS=(PROTOCOL=TCP)(HOST={host})(PORT={port}))" + "(CONNECT_DATA=(SERVICE_NAME=FREEPDB1)))"; From 1c9bfcb7f6b886b5153c4b16cb1bb92ff137a4ed Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 13:41:11 +0300 Subject: [PATCH 14/27] test(oracle): watch the degraded group for three polls, and show a rejected login halts it The degrade test stopped at the first degraded mark and checked for a halt only once, so a later poll that halted the group went undetected. The test now watches three poll intervals after the mark, and it counts a halt of one pump too. A new negative test points the store at the live listener with a user that does not exist. The login fails with ORA-01017, which is not an outage, so the group must halt. The test fails if 1017 is added to the transient list. It does not use a wrong password for a real user, because repeated failed logins can lock that account. --- .../OracleTransientFaultTests.cs | 101 ++++++++++++++---- 1 file changed, 80 insertions(+), 21 deletions(-) diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs index 5563c74..714fca4 100644 --- a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -49,36 +49,57 @@ public async Task A_network_outage_raises_a_fault_the_adapter_classifies_transie [MemberData(nameof(Outages))] public async Task A_network_outage_degrades_the_worker_group_instead_of_halting_it(string dataSource) { - var builder = Host.CreateApplicationBuilder(); - builder.Services.AddBackWave(backwave => backwave - .UseStore(Store(dataSource)) - .UseRegistry(new JobRegistry([])) - .AddWorkerGroup(new WorkerGroupOptions - { - Name = "outage-workers", - Policy = new DispatchPolicy.Strict(["default"]), - PollInterval = TimeSpan.FromMilliseconds(200), - LeaseDuration = TimeSpan.FromSeconds(30), - })); - using var host = builder.Build(); + using var host = BuildHost(Store(dataSource)); var health = host.Services.GetRequiredService(); await host.StartAsync(); try { - var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(15); - while (DateTime.UtcNow < deadline - && !health.DegradedGroups.ContainsKey("outage-workers") - && !health.HaltedGroups.ContainsKey("outage-workers")) + await WaitForDegradedOrHaltedAsync(health); + + AssertNotHalted(health); + Assert.True(health.DegradedGroups.TryGetValue(Group, out var cause), "the outage must mark the group degraded"); + output.WriteLine($"degraded: {cause}"); + Assert.True(NamesTransientCode(cause), $"the degraded cause must name a transient code: {cause}"); + + // Later polls hit the same outage. Watch three poll intervals: none of them may halt the group. + var watchEnd = DateTime.UtcNow + 3 * PollInterval; + while (DateTime.UtcNow < watchEnd) { + AssertNotHalted(health); await Task.Delay(50); } + AssertNotHalted(health); + } + finally + { + await host.StopAsync(); + } + } - Assert.False(health.HaltedGroups.TryGetValue("outage-workers", out var halt), - $"the outage fail-stopped the group: {halt?.ExceptionType}: {halt?.Message}"); - Assert.True(health.DegradedGroups.TryGetValue("outage-workers", out var cause), "the outage must mark the group degraded"); - output.WriteLine($"degraded: {cause}"); - Assert.True(NamesTransientCode(cause), $"the degraded cause must name a transient code: {cause}"); + // The negative case: a fault that is not an outage must still halt the group. A user that does not + // exist raises ORA-01017 on the live listener. The test does not use a wrong password for a real user, + // because repeated failed logins can lock that account. This is an integration test: it needs + // "docker compose up -d oracle". + [Fact] + public async Task A_rejected_login_halts_the_worker_group() + { + var store = new OracleJobStore(new OracleStoreOptions + { + ConnectionString = "User Id=bw_no_such_user;Password=bw_no_such_user;Connection Timeout=3;" + + "Data Source=localhost:15210/FREEPDB1", + }); + using var host = BuildHost(store); + var health = host.Services.GetRequiredService(); + + await host.StartAsync(); + try + { + await WaitForDegradedOrHaltedAsync(health); + + Assert.True(health.HaltedGroups.TryGetValue(Group, out var halt), "a rejected login must halt the group"); + output.WriteLine($"halted: {halt.ExceptionType}: {halt.Message}"); + Assert.Contains("ORA-01017", halt.Message); } finally { @@ -117,6 +138,44 @@ public void The_migrator_retries_three_attempts_before_the_budget_applies(int at Assert.Equal(retries, OracleMigrator.ShouldRetry(attempt, TimeSpan.FromSeconds(elapsedSeconds))); } + private const string Group = "outage-workers"; + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(200); + + private static IHost BuildHost(OracleJobStore store) + { + var builder = Host.CreateApplicationBuilder(); + builder.Services.AddBackWave(backwave => backwave + .UseStore(store) + .UseRegistry(new JobRegistry([])) + .AddWorkerGroup(new WorkerGroupOptions + { + Name = Group, + Policy = new DispatchPolicy.Strict(["default"]), + PollInterval = PollInterval, + LeaseDuration = TimeSpan.FromSeconds(30), + })); + return builder.Build(); + } + + private static async Task WaitForDegradedOrHaltedAsync(BackWaveHealth health) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(15); + while (DateTime.UtcNow < deadline + && !health.DegradedGroups.ContainsKey(Group) + && !health.HaltedGroups.ContainsKey(Group)) + { + await Task.Delay(50); + } + } + + // A halt of one pump counts too, so the check reads the partly halted groups as well. + private static void AssertNotHalted(BackWaveHealth health) + { + var halted = health.HaltedGroups.TryGetValue(Group, out var halt) + || health.PartiallyHaltedGroups.TryGetValue(Group, out halt); + Assert.False(halted, $"the outage fail-stopped the group: {halt?.ExceptionType}: {halt?.Message}"); + } + // True when the cause names an ORA or ODP.NET code that the adapter classifies transient. internal static bool NamesTransientCode(string cause) { From 19e96360f83fba6cefebf485ae388de56078e4f3 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 13:43:05 +0300 Subject: [PATCH 15/27] test(oracle): drop the tracker number from the outage test summaries The summaries make sense without it, and the code does not cite tracker numbers. --- tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs | 2 +- tests/BackWave.Oracle.Tests/OraclePoolExhaustionFaultTests.cs | 2 +- tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs b/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs index cc3513c..72b3b31 100644 --- a/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs @@ -10,7 +10,7 @@ namespace BackWave.Oracle.Tests; /// -/// Issue #12: after an outage a worker group on Oracle must claim and run jobs again without a process +/// After an outage a worker group on Oracle must claim and run jobs again without a process /// restart. The outage is the database killing every session of the group while the pool still holds /// them, so the next claim runs on a dead connection. The group runs as a private bw_recover user, so the /// kill reaches this test's sessions and no other user's. diff --git a/tests/BackWave.Oracle.Tests/OraclePoolExhaustionFaultTests.cs b/tests/BackWave.Oracle.Tests/OraclePoolExhaustionFaultTests.cs index 5f81e0e..381063e 100644 --- a/tests/BackWave.Oracle.Tests/OraclePoolExhaustionFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OraclePoolExhaustionFaultTests.cs @@ -4,7 +4,7 @@ namespace BackWave.Oracle.Tests; /// -/// Issue #12: a pump that waits for a pooled connection and does not get one in time must see a transient +/// A pump that waits for a pooled connection and does not get one in time must see a transient /// fault, not a fail-stop. A busy node can drain its pool without any fault in the database. This is an /// integration test: it needs "docker compose up -d oracle". /// diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs index 714fca4..676028c 100644 --- a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -11,7 +11,7 @@ namespace BackWave.Oracle.Tests; /// -/// Issue #12: a network outage between a worker group and its Oracle database must degrade the group, not +/// A network outage between a worker group and its Oracle database must degrade the group, not /// fail-stop it. ODP.NET raises these faults while it opens a connection, so each case points a real store /// at an endpoint that cannot answer and needs no database. /// From 611e104994020126514605d15337b8f19c77cb78 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 13:47:11 +0300 Subject: [PATCH 16/27] fix(oracle): bound the migrator retry by time only on the store's ready path The store runs the migration while it holds its ready gate, and every store call that waits at the gate runs the migration again after it. The three-attempt minimum let one call hold the gate for three full connect timeouts, which is 45 s with the default 15 s Connection Timeout. Each waiting enqueue then added the same time again. The 5 s retry budget now applies only when the store migrates on first use. The pump migrates again on its next poll, so the store path does not need more attempts. The public MigrateAsync has no next poll, so it keeps all eight attempts with no time limit, as it did before this branch. A test points a store at a host that does not answer and runs a claim and an enqueue together. Both now fail in about 12 s. Without the budget the enqueue waited 35.8 s. --- src/BackWave.Oracle/OracleJobStore.cs | 4 +- src/BackWave.Oracle/OracleMigrator.cs | 32 +++++++----- .../OracleTransientFaultTests.cs | 49 +++++++++++-------- 3 files changed, 52 insertions(+), 33 deletions(-) diff --git a/src/BackWave.Oracle/OracleJobStore.cs b/src/BackWave.Oracle/OracleJobStore.cs index d0955c3..89f4916 100644 --- a/src/BackWave.Oracle/OracleJobStore.cs +++ b/src/BackWave.Oracle/OracleJobStore.cs @@ -187,7 +187,9 @@ private async ValueTask EnsureReadyAsync(CancellationToken cancellationToken) } if (options.AutoMigrate) { - await OracleMigrator.MigrateAsync(options.ConnectionString, options.SchemaName, options.CoordinateMigration, cancellationToken).ConfigureAwait(false); + await OracleMigrator.MigrateCoreAsync( + options.ConnectionString, options.SchemaName, options.CoordinateMigration, + OracleMigrator.StoreRetryBudget, cancellationToken).ConfigureAwait(false); BackWaveLog.MigrationApplied(_logger, "oracle"); } await OracleMigrator.VerifySchemaVersionAsync(options.ConnectionString, options.SchemaName, _logger, cancellationToken).ConfigureAwait(false); diff --git a/src/BackWave.Oracle/OracleMigrator.cs b/src/BackWave.Oracle/OracleMigrator.cs index 83b3c80..c6fe684 100644 --- a/src/BackWave.Oracle/OracleMigrator.cs +++ b/src/BackWave.Oracle/OracleMigrator.cs @@ -61,9 +61,21 @@ public static Task MigrateAsync(string connectionString, CancellationToken cance /// A task that completes once every script has run. /// is not a valid identifier. /// The token was cancelled. - public static async Task MigrateAsync( + public static Task MigrateAsync( string connectionString, string schemaName, bool coordinate = true, CancellationToken cancellationToken = default) + => MigrateCoreAsync(connectionString, schemaName, coordinate, retryBudget: null, cancellationToken); + + // The time limit on the transient retry when the store migrates on first use. The store holds its + // ready gate for the whole migration, so every other store call waits, and the pump migrates again on + // its next poll anyway. + internal static readonly TimeSpan StoreRetryBudget = TimeSpan.FromSeconds(5); + + // The migration itself. A null retry budget gives the full bounded attempts with no time limit, for a + // caller that migrates once and has no next poll. + internal static async Task MigrateCoreAsync( + string connectionString, string schemaName, bool coordinate, TimeSpan? retryBudget, + CancellationToken cancellationToken) { var rewriter = new SchemaRewriter(schemaName); @@ -79,9 +91,9 @@ public static async Task MigrateAsync( // listener/handshake faults it did nothing to cause. The schema script is idempotent and // concurrency-safe by construction (guarded DDL + DDL serialized on each object by Oracle), so // re-running it converges; ride out a bounded number of transient faults instead of crashing - // startup. The budget bounds the retry by time as well: a host that does not answer costs a full - // connect timeout on each attempt, and an auto-migrating store runs the migration again on its - // next poll. On exhaustion the last fault propagates (fail-stop preserved). + // startup. A retry budget bounds the retry by time as well, because a host that does not answer + // costs a full connect timeout on each attempt. On exhaustion the last fault propagates (fail-stop + // preserved). var started = Stopwatch.GetTimestamp(); for (var attempt = 1; ; attempt++) { @@ -91,22 +103,18 @@ public static async Task MigrateAsync( return; } catch (OracleException exception) when (IsTransient(exception) - && ShouldRetry(attempt, Stopwatch.GetElapsedTime(started))) + && ShouldRetry(attempt, Stopwatch.GetElapsedTime(started), retryBudget)) { await Task.Delay(TimeSpan.FromMilliseconds(50 * attempt), cancellationToken).ConfigureAwait(false); } } } - // True when a transient fault on this attempt gets another attempt. The first attempts always retry, - // because one slow fault can use the whole budget and a caller that migrates once has no next poll. - internal static bool ShouldRetry(int attempt, TimeSpan elapsed) + // True when a transient fault on this attempt gets another attempt. + internal static bool ShouldRetry(int attempt, TimeSpan elapsed, TimeSpan? retryBudget) { const int maxAttempts = 8; - const int minAttempts = 3; - const int budgetSeconds = 5; - return attempt < maxAttempts - && (attempt < minAttempts || elapsed < TimeSpan.FromSeconds(budgetSeconds)); + return attempt < maxAttempts && (retryBudget is not { } budget || elapsed < budget); } // Runs every embedded schema script in version order. Each script is a single anonymous PL/SQL block diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs index 676028c..0e09061 100644 --- a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -107,35 +107,44 @@ public async Task A_rejected_login_halts_the_worker_group() } } - // A host that does not answer costs a full connect timeout on every attempt, so the migrator's retry - // stops on its time budget instead of waiting out all eight attempts (about 17 s here). + // A host that does not answer costs a full connect timeout on every attempt. The store holds its ready + // gate while it migrates, so an enqueue waits for the claim's migration and then runs its own. The + // retry budget bounds both: without it each call waits out all eight attempts (about 17 s here). [Fact] - public async Task An_unreachable_host_fails_the_migration_once_the_retry_budget_is_spent() + public async Task An_unreachable_host_holds_the_store_ready_gate_only_for_the_retry_budget() { - var connectionString = - $"User Id=backwave;Password=backwave;Connection Timeout=2;Data Source={Descriptor("192.0.2.1", 1521)}"; + var store = new OracleJobStore(new OracleStoreOptions + { + ConnectionString = $"User Id=backwave;Password=backwave;Connection Timeout=2;Data Source={Descriptor("192.0.2.1", 1521)}", + AutoMigrate = true, + }); var elapsed = Stopwatch.StartNew(); - var fault = await Assert.ThrowsAsync(() => OracleMigrator.MigrateAsync(connectionString)); + var claim = Assert.ThrowsAsync(async () => + await store.ClaimAsync(new ClaimRequest("w1", ["default"], 10, TimeSpan.FromMinutes(1), DateTimeOffset.UtcNow))); + var enqueue = Assert.ThrowsAsync(async () => + await store.EnqueueAsync(new NewJob(Guid.NewGuid(), "gate-test", "{}"u8.ToArray(), "default", DateTimeOffset.UtcNow), DateTimeOffset.UtcNow)); + var faults = await Task.WhenAll(claim, enqueue); - output.WriteLine($"ORA-{fault.Number}: {fault.Message}"); - Assert.True(OracleFaultCodes.IsConnectivityFault(fault.Number), $"ORA-{fault.Number} must be transient"); - Assert.True(elapsed.Elapsed < TimeSpan.FromSeconds(10), $"the migration took {elapsed.Elapsed.TotalSeconds:F1} s"); + output.WriteLine($"both faulted after {elapsed.Elapsed.TotalSeconds:F1} s: ORA-{faults[0].Number}, ORA-{faults[1].Number}"); + Assert.All(faults, fault => Assert.True(store.IsTransientFault(fault), $"ORA-{fault.Number} must classify transient")); + Assert.True(elapsed.Elapsed < TimeSpan.FromSeconds(20), $"the enqueue waited {elapsed.Elapsed.TotalSeconds:F1} s"); } - // A slow fault, for example a TNS connect timeout, can use the whole time budget on the first attempt. - // A caller that migrates once at startup has no next poll, so the first three attempts always run. + // On the store path the pump migrates again on its next poll, so the budget stops the retry. A caller + // that migrates once has no next poll, so the public migrator keeps every attempt with no time limit. [Theory] - [InlineData(1, 0, true)] - [InlineData(1, 15, true)] - [InlineData(2, 30, true)] - [InlineData(3, 4, true)] - [InlineData(3, 45, false)] - [InlineData(7, 4, true)] - [InlineData(8, 0, false)] - public void The_migrator_retries_three_attempts_before_the_budget_applies(int attempt, int elapsedSeconds, bool retries) + [InlineData(1, 15, null, true)] + [InlineData(7, 100, null, true)] + [InlineData(8, 0, null, false)] + [InlineData(1, 0, 5, true)] + [InlineData(2, 6, 5, false)] + [InlineData(8, 0, 5, false)] + public void The_migrator_bounds_its_retry_by_time_only_when_a_budget_is_given( + int attempt, int elapsedSeconds, int? budgetSeconds, bool retries) { - Assert.Equal(retries, OracleMigrator.ShouldRetry(attempt, TimeSpan.FromSeconds(elapsedSeconds))); + var budget = budgetSeconds is { } seconds ? TimeSpan.FromSeconds(seconds) : (TimeSpan?)null; + Assert.Equal(retries, OracleMigrator.ShouldRetry(attempt, TimeSpan.FromSeconds(elapsedSeconds), budget)); } private const string Group = "outage-workers"; From aea210acd9f284b80609795778852ea514e3b565 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 14:03:47 +0300 Subject: [PATCH 17/27] test(oracle): prove a worker group recovers after its connections are refused The session-kill recovery test raises ORA-00028, which was transient before the outage fix, so it passes without that fix. This test raises a code the fix adds. The store connects through a free loopback port that nothing listens on, so ODP.NET gets a refused connect (ORA-50201). The group must degrade and not halt, and an enqueue in that window must fail with a transient fault. Then an in-process TCP forwarder opens the port in front of the real database. A job enqueued after that must succeed on the same group. Last, the forwarder closes every live connection, and one more job must succeed. Without the fix the group halts on ORA-50201 at the first poll. --- .../OracleRefusedConnectionRecoveryTests.cs | 373 ++++++++++++++++++ 1 file changed, 373 insertions(+) create mode 100644 tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs diff --git a/tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs b/tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs new file mode 100644 index 0000000..4de5fa5 --- /dev/null +++ b/tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs @@ -0,0 +1,373 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using System.Text.Json.Serialization; +using BackWave.Core; +using BackWave.Hosting; +using BackWave.Jobs; +using BackWave.Storage; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Oracle.ManagedDataAccess.Client; + +namespace BackWave.Oracle.Tests; + +/// +/// A worker group on Oracle that boots while its database refuses connections must degrade, not halt, and +/// run jobs once the database answers. The store connects through a loopback port that nothing listens on, +/// so ODP.NET gets a refused connect. A TCP forwarder then opens that port in front of the real database, +/// which models the database coming back. The group runs as a private bw_refused user, so the teardown +/// reaches this test's sessions and no other user's. +/// +[Collection("oracle")] +public sealed class OracleRefusedConnectionRecoveryTests(Xunit.Abstractions.ITestOutputHelper output) : IAsyncLifetime +{ + private const string Schema = "bw_refused"; + private const string Group = "refused-workers"; + + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(200); + + private static readonly int Port = ReserveFreePort(); + + // The store reaches the database only through the forwarder port. A minimum pool size of zero stops the + // pool from opening a fresh session after ClearPool, which would block the DROP USER at teardown. The + // short connect timeout keeps a connect that hangs from outlasting the test. + private static readonly string SchemaConnectionString = + new OracleConnectionStringBuilder(OracleTestDatabase.ConnectionString) + { + UserID = Schema, + Password = Schema, + DataSource = $"127.0.0.1:{Port}/{Upstream.Service}", + MinPoolSize = 0, + ConnectionTimeout = 3, + }.ConnectionString; + + private TcpForwarder? _forwarder; + + public Task InitializeAsync() => OracleTestDatabase.ResetUserAsync(Schema); + + public async Task DisposeAsync() + { + // Lock the user, so the pool cannot open a new session, then end every session it still has, so + // the DROP USER does not wait on them. + await LockUserAsync(); + OracleConnection.ClearPool(new OracleConnection(SchemaConnectionString)); + await DisconnectSessionsAsync(); + await OracleTestDatabase.DropUserAsync(Schema); + if (_forwarder is not null) + { + await _forwarder.DisposeAsync(); + } + } + + [Fact] + public async Task A_worker_group_that_boots_while_connections_are_refused_runs_jobs_once_the_database_answers() + { + var store = new OracleJobStore(new OracleStoreOptions + { + ConnectionString = SchemaConnectionString, + SchemaName = Schema, + AutoMigrate = true, + }); + var builder = Host.CreateApplicationBuilder(); + builder.Services.AddTransient, RefusedHandler>(); + builder.Services.AddBackWave(backwave => backwave + .UseStore(store) + .UseRegistry(new JobRegistry( + [ + JobRegistration.Create("refused", OracleRefusedJsonContext.Default.RefusedJob), + ])) + .AddWorkerGroup(new WorkerGroupOptions + { + Name = Group, + Policy = new DispatchPolicy.Strict(["default"]), + PollInterval = PollInterval, + MaxPollInterval = TimeSpan.Zero, + LeaseDuration = TimeSpan.FromSeconds(30), + })); + using var host = builder.Build(); + var health = host.Services.GetRequiredService(); + var client = host.Services.GetRequiredService(); + + await host.StartAsync(); + try + { + // The port is closed, so the first poll's migration gets a refused connect. The group must + // degrade, not halt: the host degrades a group only on a fault the store calls transient. + var cause = await WaitForDegradedAsync(health, "the refused connect to mark the group degraded"); + output.WriteLine($"degraded while refused: {cause}"); + Assert.Matches(@"ORA-\d+", cause); + + // An enqueue while the port is closed raises the same fault to the caller. The store must call it + // transient, so the caller can retry it as the group does. + var fault = await Assert.ThrowsAsync( + async () => await client.EnqueueAsync(new RefusedJob(0), dueTime: DateTimeOffset.UtcNow)); + output.WriteLine($"enqueue while refused: ORA-{fault.Number:D5}"); + Assert.True(store.IsTransientFault(fault), $"ORA-{fault.Number:D5} must be transient: {fault.Message}"); + + // Each poll runs the migration again and gets refused again. The enqueue above outlasted at + // least one more of those polls. None of them may halt the group. + await WatchNotHaltedAsync(health, 3 * PollInterval); + + // The database answers again. + _forwarder = TcpForwarder.Start(Port, Upstream.Host, Upstream.Port); + + // Recovery: a job enqueued after the database answers is claimed and run by the same group. + var first = await client.EnqueueAsync(new RefusedJob(1), dueTime: DateTimeOffset.UtcNow); + await WaitForSucceededAsync(store, health, first); + AssertNotHalted(health); + + // A drop mid-session: the forwarder closes every live connection, so no pooled session is + // usable. The job is due after the drop, so the group must claim it on a fresh session. The pool + // discards an idle connection with a closed socket when it hands it out, so the drop can pass + // without a fault; a call in flight at the drop gets ORA-12537. + var second = await client.EnqueueAsync(new RefusedJob(2), dueTime: DateTimeOffset.UtcNow.AddSeconds(1)); + var dropped = _forwarder.DropConnections(); + output.WriteLine($"dropped {dropped} live connections"); + Assert.True(dropped > 0, "the group must hold at least one live connection to drop"); + var dropCause = await WaitForSucceededAsync(store, health, second); + output.WriteLine($"degraded after drop: {dropCause ?? "never"}"); + AssertNotHalted(health); + } + finally + { + await host.StopAsync(); + } + } + + // Binds a listener on port 0 to get a free loopback port, then stops it, so nothing listens there until + // the forwarder starts. + private static int ReserveFreePort() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + // The real database endpoint, from a host:port/service data source. + private static class Upstream + { + private static readonly string DataSource = + new OracleConnectionStringBuilder(OracleTestDatabase.ConnectionString).DataSource; + + public static readonly string Host = DataSource[..DataSource.IndexOf(':')]; + public static readonly int Port = + int.Parse(DataSource[(DataSource.IndexOf(':') + 1)..DataSource.IndexOf('/')]); + public static readonly string Service = DataSource[(DataSource.IndexOf('/') + 1)..]; + } + + private static async Task LockUserAsync() + { + await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); + await admin.OpenAsync(); + await using var lockUser = admin.CreateCommand(); + lockUser.CommandText = $"ALTER USER {Schema} ACCOUNT LOCK"; + await lockUser.ExecuteNonQueryAsync(); + } + + // Ends every session of the private user, a killed one too, and waits until none is left. A killed + // session holds its process until its client calls again, and the client is gone at teardown. + private static async Task DisconnectSessionsAsync() + { + await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); + await admin.OpenAsync(); + + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (true) + { + var sessions = new List(); + await using (var select = admin.CreateCommand()) + { + select.CommandText = "SELECT sid, serial# FROM v$session WHERE username = :username"; + select.Parameters.Add(new OracleParameter("username", Schema.ToUpperInvariant())); + await using var reader = await select.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + sessions.Add($"{reader.GetInt32(0)},{reader.GetInt32(1)}"); + } + } + if (sessions.Count == 0 || DateTime.UtcNow > deadline) + { + return; + } + + foreach (var session in sessions) + { + await using var disconnect = admin.CreateCommand(); + disconnect.CommandText = $"ALTER SYSTEM DISCONNECT SESSION '{session}' IMMEDIATE"; + try + { + await disconnect.ExecuteNonQueryAsync(); + } + catch (OracleException exception) when (exception.Number is 31 or 30) + { + } + } + await Task.Delay(200); + } + } + + // Samples the health state until the group is degraded. The next clean poll clears the mark, so the + // sample interval stays well under the poll interval. + private static async Task WaitForDegradedAsync(BackWaveHealth health, string description) + { + var cause = ""; + await WaitForAsync(() => Task.FromResult(health.DegradedGroups.TryGetValue(Group, out cause!)), health, + description, TimeSpan.FromMilliseconds(5)); + return cause; + } + + private static async Task WatchNotHaltedAsync(BackWaveHealth health, TimeSpan duration) + { + var deadline = DateTime.UtcNow + duration; + while (DateTime.UtcNow < deadline) + { + AssertNotHalted(health); + await Task.Delay(TimeSpan.FromMilliseconds(20)); + } + AssertNotHalted(health); + } + + // Returns the first degraded cause seen while waiting, or null when the group never degraded. + private static async Task WaitForSucceededAsync(OracleJobStore store, BackWaveHealth health, Guid jobId) + { + string? cause = null; + await WaitForAsync(async () => + { + if (cause is null && health.DegradedGroups.TryGetValue(Group, out var degraded)) + { + cause = degraded; + } + return (await store.GetJobAsync(jobId))?.State == JobState.Succeeded; + }, health, $"job {jobId} to reach Succeeded", TimeSpan.FromMilliseconds(5)); + return cause; + } + + // Polls the condition and fails the test the moment the group halts, so a fail-stop reports its cause + // instead of a timeout. + private static async Task WaitForAsync( + Func> condition, BackWaveHealth health, string description, TimeSpan? interval = null) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline) + { + AssertNotHalted(health); + if (await condition()) + { + return; + } + await Task.Delay(interval ?? TimeSpan.FromMilliseconds(50)); + } + AssertNotHalted(health); + Assert.Fail($"Timed out waiting for: {description}"); + } + + private static void AssertNotHalted(BackWaveHealth health) => + Assert.False(health.HaltedGroups.TryGetValue(Group, out var halt), + $"the outage fail-stopped the group: {halt?.ExceptionType}: {halt?.Message}"); + + // Accepts each client on a loopback port and pumps its bytes both ways to the upstream endpoint. + private sealed class TcpForwarder : IAsyncDisposable + { + private readonly TcpListener _listener; + private readonly string _upstreamHost; + private readonly int _upstreamPort; + private readonly CancellationTokenSource _stopping = new(); + private readonly ConcurrentDictionary _connections = new(); + private readonly Task _accepting; + + private TcpForwarder(int port, string upstreamHost, int upstreamPort) + { + _upstreamHost = upstreamHost; + _upstreamPort = upstreamPort; + _listener = new TcpListener(IPAddress.Loopback, port); + _listener.Start(); + _accepting = AcceptAsync(); + } + + public static TcpForwarder Start(int port, string upstreamHost, int upstreamPort) => + new(port, upstreamHost, upstreamPort); + + // Closes every live connection and returns how many there were. The listener stays open, so a + // new connect still reaches the database. + public int DropConnections() + { + var live = _connections.Values.ToList(); + foreach (var (client, upstream) in live) + { + client.Dispose(); + upstream.Dispose(); + } + return live.Count; + } + + public async ValueTask DisposeAsync() + { + _stopping.Cancel(); + _listener.Stop(); + DropConnections(); + await Task.WhenAll(_connections.Keys.Append(_accepting)); + _stopping.Dispose(); + } + + private async Task AcceptAsync() + { + try + { + while (true) + { + var client = await _listener.AcceptTcpClientAsync(_stopping.Token); + var upstream = new TcpClient(); + var completed = new TaskCompletionSource(); + _connections[completed.Task] = (client, upstream); + _ = PumpAsync(client, upstream, completed); + } + } + catch (Exception exception) when (exception is OperationCanceledException or SocketException or ObjectDisposedException) + { + } + } + + // Pumps both ways until either side closes, then closes both, so the other side sees the drop too. + private async Task PumpAsync(TcpClient client, TcpClient upstream, TaskCompletionSource completed) + { + try + { + await upstream.ConnectAsync(_upstreamHost, _upstreamPort, _stopping.Token); + var clientStream = client.GetStream(); + var upstreamStream = upstream.GetStream(); + var copies = new[] + { + clientStream.CopyToAsync(upstreamStream, _stopping.Token), + upstreamStream.CopyToAsync(clientStream, _stopping.Token), + }; + await Task.WhenAny(copies); + client.Dispose(); + upstream.Dispose(); + await Task.WhenAll(copies); + } + catch (Exception exception) when (exception is OperationCanceledException or IOException or SocketException or ObjectDisposedException) + { + } + finally + { + client.Dispose(); + upstream.Dispose(); + _connections.TryRemove(completed.Task, out _); + completed.SetResult(); + } + } + } +} + +public sealed record RefusedJob(int Index); + +public sealed class RefusedHandler : IJobHandler +{ + public Task HandleAsync(RefusedJob job, JobContext context, CancellationToken cancellationToken) => Task.CompletedTask; +} + +[JsonSerializable(typeof(RefusedJob))] +internal sealed partial class OracleRefusedJsonContext : JsonSerializerContext; From 0c176bd32fd79426dc577927b3c82a8710b9d894 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 14:34:49 +0300 Subject: [PATCH 18/27] test(oracle): wait for the alert registration before the wake-up signal The hinted-wake test waited a fixed 750 ms for the consumer to register its DBMS_ALERT waiter. A signal before the registration is lost, and under load 750 ms was not always enough. The test now waits until a new session appears in sys.dbms_alert_info for the store's alert. The producer store also opens its connections and checks the schema before the stopwatch starts, so the latency measures the hint and not a cold start. The test passed 40 of 40 runs alone on a private Oracle container. The earlier timeouts occurred only while another worktree ran the Oracle suite on the shared database. --- .../OracleWakeUpHintTests.cs | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/tests/BackWave.Oracle.Tests/OracleWakeUpHintTests.cs b/tests/BackWave.Oracle.Tests/OracleWakeUpHintTests.cs index 84a84fd..d075c47 100644 --- a/tests/BackWave.Oracle.Tests/OracleWakeUpHintTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleWakeUpHintTests.cs @@ -33,16 +33,21 @@ public async Task Enqueue_wakes_an_idle_pump_far_faster_than_the_poll_interval() var recorder = new ConcurrencyRecorder(); // The consumer node: its pump waits on DBMS_ALERT while it polls every 5 s. + var registered = await AlertRegistrationsAsync(); var consumerStore = NewStore(enableWakeUpHints: true); using var consumer = BuildHost(consumerStore, recorder, WakeGroup()); await consumer.StartAsync(); - // Let the pump reach its idle state and the DBMS_ALERT waiter register before the signal fires. - await Task.Delay(TimeSpan.FromMilliseconds(750)); + // A signal before the waiter registers is lost, so wait for the consumer's registration. A fixed + // delay was not enough when the whole suite loaded the machine. + await WaitForNewRegistrationAsync(registered); - // The producer node: a separate store (separate connections) enqueues the probe and signals. - using var producer = BuildHost(NewStore(enableWakeUpHints: true), new ConcurrencyRecorder(), group: null); + // The producer node: a separate store (separate connections) enqueues the probe and signals. Its + // first call opens its connections and checks the schema, so that call runs before the stopwatch. + var producerStore = NewStore(enableWakeUpHints: true); + using var producer = BuildHost(producerStore, new ConcurrencyRecorder(), group: null); var producerClient = producer.Services.GetRequiredService(); + await producerStore.GetJobAsync(Guid.NewGuid()); recorder.ArmProbe(); var stopwatch = Stopwatch.StartNew(); @@ -255,6 +260,39 @@ private static async Task RevokeAlertAccessAsync() await revoke.ExecuteNonQueryAsync(); } + // The session ids that hold a registration on the store's alert. Oracle keeps a row for a session that + // has ended, so a new registration is a session id that an earlier read did not return. + private static async Task> AlertRegistrationsAsync() + { + await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); + await admin.OpenAsync(); + await using var select = admin.CreateCommand(); + select.CommandText = "SELECT sid FROM sys.dbms_alert_info WHERE name = :name"; + select.Parameters.Add(new OracleParameter("name", + new SchemaRewriter(SchemaRewriter.DefaultSchema).HintAlertName.ToUpperInvariant())); + var sessions = new HashSet(); + await using var reader = await select.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + sessions.Add(reader.GetString(0)); + } + return sessions; + } + + private static async Task WaitForNewRegistrationAsync(HashSet before) + { + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(10); + while (DateTimeOffset.UtcNow < deadline) + { + if ((await AlertRegistrationsAsync()).Except(before).Any()) + { + return; + } + await Task.Delay(50); + } + Assert.Fail("Timed out waiting for the consumer to register its DBMS_ALERT waiter"); + } + private static async Task WaitForAsync(Func condition, TimeSpan timeout, string description) { var deadline = DateTimeOffset.UtcNow + timeout; From a6405d8fb64eaf8d6526268ba547a69c8e99aa3a Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 14:47:57 +0300 Subject: [PATCH 19/27] fix(oracle): retry a library cache deadlock during a fleet migration Two nodes that create the same index at the same time can deadlock in the library cache. Oracle rolls back one statement with ORA-04020, and the migrator retried only connectivity faults, so that node failed its start. The 16-node first-boot storm test failed about 1 run in 3 with this error on a private container. The scripts are idempotent, so the coordinated migrator now retries ORA-04020 like a connectivity fault. The uncoordinated path is still caller-serialized and does not retry. Its storm test rides out the same codes. The storm test class passed 15 of 15 runs after the change. --- src/BackWave.Oracle/OracleMigrator.cs | 18 +++++++++++------- .../OracleCoordinatedMigrationTests.cs | 9 +++++---- .../OracleTransientFaultTests.cs | 11 +++++++++++ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/BackWave.Oracle/OracleMigrator.cs b/src/BackWave.Oracle/OracleMigrator.cs index c6fe684..92d8d07 100644 --- a/src/BackWave.Oracle/OracleMigrator.cs +++ b/src/BackWave.Oracle/OracleMigrator.cs @@ -20,16 +20,20 @@ public static class OracleMigrator /// The schema version this build of the adapter requires the database to be at. public const int ExpectedSchemaVersion = 2; - // Transient connection faults a cold-booting fleet can hit that the bounded retry should ride out - // rather than surface: the shared listener/handshake-storm and connection-lost connectivity set. - private static bool IsTransient(OracleException exception) - => OracleFaultCodes.IsConnectivityFault(exception.Number); + // Transient faults a cold-booting fleet can hit that the bounded retry should ride out rather than + // surface: the shared listener/handshake-storm and connection-lost connectivity set, and ORA-04020. + // Two nodes that create the same object at the same time can deadlock in the library cache. Oracle + // rolls back one statement, and the scripts are idempotent, so a new attempt converges. + internal static bool IsTransient(int number) + => OracleFaultCodes.IsConnectivityFault(number) || number == LibraryCacheDeadlock; + + private const int LibraryCacheDeadlock = 4020; /// /// Runs every schema script in version order, bringing the database up to the version this adapter /// requires. Idempotent - safe to run against an already-current database. Transient connection faults - /// from a fleet cold-boot are retried internally with a bounded backoff; a fault that persists past - /// those attempts propagates. + /// and library cache deadlocks (ORA-04020) from a fleet cold-boot are retried internally with a bounded + /// backoff; a fault that persists past those attempts propagates. /// /// The ODP.NET connection string for the target database. /// Token to cancel the migration. @@ -102,7 +106,7 @@ internal static async Task MigrateCoreAsync( await ApplyScriptsAsync(connectionString, rewriter, cancellationToken).ConfigureAwait(false); return; } - catch (OracleException exception) when (IsTransient(exception) + catch (OracleException exception) when (IsTransient(exception.Number) && ShouldRetry(attempt, Stopwatch.GetElapsedTime(started), retryBudget)) { await Task.Delay(TimeSpan.FromMilliseconds(50 * attempt), cancellationToken).ConfigureAwait(false); diff --git a/tests/BackWave.Oracle.Tests/OracleCoordinatedMigrationTests.cs b/tests/BackWave.Oracle.Tests/OracleCoordinatedMigrationTests.cs index c853b00..1264091 100644 --- a/tests/BackWave.Oracle.Tests/OracleCoordinatedMigrationTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleCoordinatedMigrationTests.cs @@ -63,9 +63,10 @@ public sealed class OracleCoordinatedMigrationTests : IAsyncLifetime // The coordinate:false path strips the migrator's own transient-connectivity retry, so a cold-boot // listener-handoff fault (ORA-12518, ORA-12570, and the rest of the connectivity set) can surface here - - // unrelated to the guarded-DDL convergence this test proves. Ride those out with a bounded retry. Any - // non-connectivity error (a real convergence fault such as ORA-00955 or ORA-00001) propagates unchanged - // and fails the storm as before. + // unrelated to the guarded-DDL convergence this test proves. The same path also surfaces ORA-04020, the + // library cache deadlock between two nodes that create the same object, which the coordinated path + // retries. Ride those out with a bounded retry. Any other error (a real convergence fault such as + // ORA-00955 or ORA-00001) propagates unchanged and fails the storm as before. private static async Task MigrateRidingOutConnectivityFaultsAsync(bool coordinate) { const int maxAttempts = 10; @@ -77,7 +78,7 @@ private static async Task MigrateRidingOutConnectivityFaultsAsync(bool coordinat return; } catch (OracleException exception) - when (OracleFaultCodes.IsConnectivityFault(exception.Number) && attempt < maxAttempts) + when (OracleMigrator.IsTransient(exception.Number) && attempt < maxAttempts) { await Task.Delay(250); } diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs index 0e09061..b942a0c 100644 --- a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -147,6 +147,17 @@ public void The_migrator_bounds_its_retry_by_time_only_when_a_budget_is_given( Assert.Equal(retries, OracleMigrator.ShouldRetry(attempt, TimeSpan.FromSeconds(elapsedSeconds), budget)); } + // Two migrators that create the same index at the same time can deadlock in the library cache. Oracle + // rolls back one statement with ORA-04020, and the measured 16-node storm hit it about 1 run in 3. A + // real schema fault such as ORA-00942 still stops the migration. + [Theory] + [InlineData(4020, true)] + [InlineData(12545, true)] + [InlineData(942, false)] + [InlineData(1031, false)] + public void The_migrator_retries_a_library_cache_deadlock_and_connectivity_faults_only(int number, bool retries) + => Assert.Equal(retries, OracleMigrator.IsTransient(number)); + private const string Group = "outage-workers"; private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(200); From 27537134c01ebf5b008d147107887baeed2869ee Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 17:20:34 +0300 Subject: [PATCH 20/27] fix(oracle): mark a library cache deadlock transient in the store too The store migrates on first use with a 5 s retry budget, but one DDL lock wait can last 30 s (ddl_lock_timeout). So an ORA-04020 can arrive after the budget and go out through EnsureReadyAsync. The store classifier accepted only ORA-00060 as a deadlock, so the worker group halted. OracleFaultCodes.IsDeadlock now names 60 and 4020, and the migrator and the store classifier both use it. A new test raises each code in the database and makes sure that the store marks it transient. It failed for 4020 before this change. --- src/BackWave.Oracle/OracleFaultCodes.cs | 5 ++++ src/BackWave.Oracle/OracleJobStore.cs | 6 ++--- src/BackWave.Oracle/OracleMigrator.cs | 11 +++----- .../OracleTransientFaultTests.cs | 25 +++++++++++++++++-- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/BackWave.Oracle/OracleFaultCodes.cs b/src/BackWave.Oracle/OracleFaultCodes.cs index afde701..8865f42 100644 --- a/src/BackWave.Oracle/OracleFaultCodes.cs +++ b/src/BackWave.Oracle/OracleFaultCodes.cs @@ -30,4 +30,9 @@ internal static class OracleFaultCodes internal static bool IsConnectivityFault(int code) => code is 12170 or 12541 or 12514 or 12518 or 12537 or 12570 or 3113 or 3114 or 28 or 1033 or 12154 or 1089 or 1109 or 12545 or 50201 or 50000 or 50012; + + // True when Oracle rolled back one statement to break a deadlock: 60 on rows, 4020 on the library cache + // (two nodes that create the same object at the same time). A new attempt can succeed, so the migrator + // retries both and the store tags both transient. + internal static bool IsDeadlock(int code) => code is 60 or 4020; } diff --git a/src/BackWave.Oracle/OracleJobStore.cs b/src/BackWave.Oracle/OracleJobStore.cs index 89f4916..49834b0 100644 --- a/src/BackWave.Oracle/OracleJobStore.cs +++ b/src/BackWave.Oracle/OracleJobStore.cs @@ -224,12 +224,12 @@ private Task FailpointAsync(string name, CancellationToken cancellationToken) // Classifies a store fault for the backwave.store.faults metric tag, mirroring the host's own // transient/terminal split: a bare TimeoutException plus the shared connectivity/timeout set and the - // ORA-00060 deadlock victim are the whole transient set. Emit-only - the host still makes the real - // retry/fail-stop decision from the rethrown exception. + // deadlock victims (ORA-00060, ORA-04020) are the whole transient set. Emit-only - the host still makes + // the real retry/fail-stop decision from the rethrown exception. private static bool IsTransientStoreFault(Exception exception) => exception is TimeoutException || (exception is OracleException o - && (OracleFaultCodes.IsConnectivityFault(o.Number) || o.Number == 60)); + && (OracleFaultCodes.IsConnectivityFault(o.Number) || OracleFaultCodes.IsDeadlock(o.Number))); /// public bool IsTransientFault(Exception exception) => IsTransientStoreFault(exception); diff --git a/src/BackWave.Oracle/OracleMigrator.cs b/src/BackWave.Oracle/OracleMigrator.cs index 92d8d07..6d6b980 100644 --- a/src/BackWave.Oracle/OracleMigrator.cs +++ b/src/BackWave.Oracle/OracleMigrator.cs @@ -21,18 +21,15 @@ public static class OracleMigrator public const int ExpectedSchemaVersion = 2; // Transient faults a cold-booting fleet can hit that the bounded retry should ride out rather than - // surface: the shared listener/handshake-storm and connection-lost connectivity set, and ORA-04020. - // Two nodes that create the same object at the same time can deadlock in the library cache. Oracle - // rolls back one statement, and the scripts are idempotent, so a new attempt converges. + // surface: the shared listener/handshake-storm and connection-lost connectivity set, and a deadlock. + // The scripts are idempotent, so a new attempt converges. internal static bool IsTransient(int number) - => OracleFaultCodes.IsConnectivityFault(number) || number == LibraryCacheDeadlock; - - private const int LibraryCacheDeadlock = 4020; + => OracleFaultCodes.IsConnectivityFault(number) || OracleFaultCodes.IsDeadlock(number); /// /// Runs every schema script in version order, bringing the database up to the version this adapter /// requires. Idempotent - safe to run against an already-current database. Transient connection faults - /// and library cache deadlocks (ORA-04020) from a fleet cold-boot are retried internally with a bounded + /// and deadlocks (ORA-00060, ORA-04020) from a fleet cold-boot are retried internally with a bounded /// backoff; a fault that persists past those attempts propagates. /// /// The ODP.NET connection string for the target database. diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs index b942a0c..40b83f8 100644 --- a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -149,15 +149,36 @@ public void The_migrator_bounds_its_retry_by_time_only_when_a_budget_is_given( // Two migrators that create the same index at the same time can deadlock in the library cache. Oracle // rolls back one statement with ORA-04020, and the measured 16-node storm hit it about 1 run in 3. A - // real schema fault such as ORA-00942 still stops the migration. + // row deadlock (ORA-00060) gets the same retry. A real schema fault such as ORA-00942 still stops the + // migration. [Theory] [InlineData(4020, true)] + [InlineData(60, true)] [InlineData(12545, true)] [InlineData(942, false)] [InlineData(1031, false)] - public void The_migrator_retries_a_library_cache_deadlock_and_connectivity_faults_only(int number, bool retries) + public void The_migrator_retries_deadlocks_and_connectivity_faults_only(int number, bool retries) => Assert.Equal(retries, OracleMigrator.IsTransient(number)); + // The store migrates on first use with a 5 s retry budget, and one DDL lock wait can last 30 s, so a + // deadlock can reach the pump after the budget. The store must then mark it transient, or the group + // halts. The database raises each code here, so the exception is the one ODP.NET gives. + [Theory] + [InlineData(4020)] + [InlineData(60)] + public async Task The_store_classifies_a_deadlock_transient(int number) + { + await using var connection = new OracleConnection(OracleTestDatabase.ConnectionString); + await connection.OpenAsync(); + await using var raise = connection.CreateCommand(); + raise.CommandText = $"DECLARE e EXCEPTION; PRAGMA EXCEPTION_INIT(e, -{number}); BEGIN RAISE e; END;"; + var fault = await Assert.ThrowsAsync(() => raise.ExecuteNonQueryAsync()); + + var store = new OracleJobStore(new OracleStoreOptions { ConnectionString = OracleTestDatabase.ConnectionString }); + Assert.Equal(number, fault.Number); + Assert.True(store.IsTransientFault(fault), $"ORA-{fault.Number} must classify transient"); + } + private const string Group = "outage-workers"; private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(200); From ffe78da7429668f98abea4e8bf13ef8314fae594 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 17:22:23 +0300 Subject: [PATCH 21/27] test(oracle): share the outage test helpers The two recovery test classes had their own copies of the halt check, the wait loops, and the private-user teardown. Their halt checks did not read PartiallyHaltedGroups, so a halt of one pump went unseen. The copy in OracleTransientFaultTests read it. OutageTestSupport now holds one copy of each helper, and its halt check reads both halted sets. --- .../OracleOutageRecoveryTests.cs | 99 ++--------------- .../OracleRefusedConnectionRecoveryTests.cs | 102 ++---------------- .../OracleTransientFaultTests.cs | 14 +-- .../OutageTestSupport.cs | 97 +++++++++++++++++ 4 files changed, 117 insertions(+), 195 deletions(-) create mode 100644 tests/BackWave.Oracle.Tests/OutageTestSupport.cs diff --git a/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs b/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs index 72b3b31..1a01437 100644 --- a/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleOutageRecoveryTests.cs @@ -38,9 +38,9 @@ public async Task DisposeAsync() { // Lock the user, so the pool cannot open a new session, then end every session it still has, so // the DROP USER does not wait on them. - await LockUserAsync(); + await OutageTestSupport.LockUserAsync(Schema); OracleConnection.ClearPool(new OracleConnection(SchemaConnectionString)); - await DisconnectSessionsAsync(); + await OutageTestSupport.DisconnectSessionsAsync(Schema); await OracleTestDatabase.DropUserAsync(Schema); } @@ -85,18 +85,19 @@ public async Task A_worker_group_claims_and_runs_jobs_again_after_its_pooled_ses Assert.True(killed > 0, $"the group must hold at least one {Schema} session to kill"); // The next poll takes a killed session from the pool. The group must degrade, not halt. - var cause = await WaitForDegradedAsync(health); + var cause = await OutageTestSupport.WaitForDegradedAsync(health, Group, "the killed sessions to mark the group degraded"); output.WriteLine($"degraded: {cause}"); Assert.True(OracleTransientFaultTests.NamesTransientCode(cause), $"the degraded cause must name a transient code: {cause}"); // Each failed poll drops one dead session from the pool. Wait until none is left, so the // enqueue below cannot take one either. - await WaitForAsync(async () => await CountKilledSessionsAsync() == 0, health, "the pool to drop every killed session"); + await OutageTestSupport.WaitForAsync(async () => await CountKilledSessionsAsync() == 0, health, Group, + "the pool to drop every killed session"); // Recovery: a job enqueued after the outage is claimed and run by the same group. var after = await client.EnqueueAsync(new RecoveryJob(2), dueTime: DateTimeOffset.UtcNow); await WaitForSucceededAsync(store, health, after); - AssertNotHalted(health); + OutageTestSupport.AssertNotHalted(health, Group); } finally { @@ -142,57 +143,6 @@ private static async Task KillSessionsAsync() return sessions.Count; } - private static async Task LockUserAsync() - { - await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); - await admin.OpenAsync(); - await using var lockUser = admin.CreateCommand(); - lockUser.CommandText = $"ALTER USER {Schema} ACCOUNT LOCK"; - await lockUser.ExecuteNonQueryAsync(); - } - - // Ends every session of the private user, a killed one too, and waits until none is left. A killed - // session holds its process until its client calls again, and the client is gone at teardown. - private static async Task DisconnectSessionsAsync() - { - await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); - await admin.OpenAsync(); - - var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); - while (true) - { - var sessions = new List(); - await using (var select = admin.CreateCommand()) - { - select.CommandText = "SELECT sid, serial# FROM v$session WHERE username = :username"; - select.Parameters.Add(new OracleParameter("username", Schema.ToUpperInvariant())); - await using var reader = await select.ExecuteReaderAsync(); - while (await reader.ReadAsync()) - { - sessions.Add($"{reader.GetInt32(0)},{reader.GetInt32(1)}"); - } - } - if (sessions.Count == 0 || DateTime.UtcNow > deadline) - { - return; - } - - foreach (var session in sessions) - { - await using var disconnect = admin.CreateCommand(); - disconnect.CommandText = $"ALTER SYSTEM DISCONNECT SESSION '{session}' IMMEDIATE"; - try - { - await disconnect.ExecuteNonQueryAsync(); - } - catch (OracleException exception) when (exception.Number is 31 or 30) - { - } - } - await Task.Delay(200); - } - } - // A killed session stays in v$session until its client makes a call and is told it is dead. private static async Task CountKilledSessionsAsync() { @@ -204,42 +154,9 @@ private static async Task CountKilledSessionsAsync() return Convert.ToInt32(await count.ExecuteScalarAsync()); } - // Samples the health state until the group is degraded. The next clean poll clears the mark, so the - // sample interval stays well under the poll interval. - private static async Task WaitForDegradedAsync(BackWaveHealth health) - { - var cause = ""; - await WaitForAsync(() => Task.FromResult(health.DegradedGroups.TryGetValue(Group, out cause!)), health, - "the killed sessions to mark the group degraded", TimeSpan.FromMilliseconds(5)); - return cause; - } - private static Task WaitForSucceededAsync(OracleJobStore store, BackWaveHealth health, Guid jobId) => - WaitForAsync(async () => (await store.GetJobAsync(jobId))?.State == JobState.Succeeded, health, - $"job {jobId} to reach Succeeded"); - - // Polls the condition and fails the test the moment the group halts, so a fail-stop reports its cause - // instead of a timeout. - private static async Task WaitForAsync( - Func> condition, BackWaveHealth health, string description, TimeSpan? interval = null) - { - var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); - while (DateTime.UtcNow < deadline) - { - AssertNotHalted(health); - if (await condition()) - { - return; - } - await Task.Delay(interval ?? TimeSpan.FromMilliseconds(50)); - } - AssertNotHalted(health); - Assert.Fail($"Timed out waiting for: {description}"); - } - - private static void AssertNotHalted(BackWaveHealth health) => - Assert.False(health.HaltedGroups.TryGetValue(Group, out var halt), - $"the outage fail-stopped the group: {halt?.ExceptionType}: {halt?.Message}"); + OutageTestSupport.WaitForAsync(async () => (await store.GetJobAsync(jobId))?.State == JobState.Succeeded, health, + Group, $"job {jobId} to reach Succeeded"); } public sealed record RecoveryJob(int Index); diff --git a/tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs b/tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs index 4de5fa5..de6016e 100644 --- a/tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs @@ -50,9 +50,9 @@ public async Task DisposeAsync() { // Lock the user, so the pool cannot open a new session, then end every session it still has, so // the DROP USER does not wait on them. - await LockUserAsync(); + await OutageTestSupport.LockUserAsync(Schema); OracleConnection.ClearPool(new OracleConnection(SchemaConnectionString)); - await DisconnectSessionsAsync(); + await OutageTestSupport.DisconnectSessionsAsync(Schema); await OracleTestDatabase.DropUserAsync(Schema); if (_forwarder is not null) { @@ -94,7 +94,7 @@ public async Task A_worker_group_that_boots_while_connections_are_refused_runs_j { // The port is closed, so the first poll's migration gets a refused connect. The group must // degrade, not halt: the host degrades a group only on a fault the store calls transient. - var cause = await WaitForDegradedAsync(health, "the refused connect to mark the group degraded"); + var cause = await OutageTestSupport.WaitForDegradedAsync(health, Group, "the refused connect to mark the group degraded"); output.WriteLine($"degraded while refused: {cause}"); Assert.Matches(@"ORA-\d+", cause); @@ -115,7 +115,7 @@ public async Task A_worker_group_that_boots_while_connections_are_refused_runs_j // Recovery: a job enqueued after the database answers is claimed and run by the same group. var first = await client.EnqueueAsync(new RefusedJob(1), dueTime: DateTimeOffset.UtcNow); await WaitForSucceededAsync(store, health, first); - AssertNotHalted(health); + OutageTestSupport.AssertNotHalted(health, Group); // A drop mid-session: the forwarder closes every live connection, so no pooled session is // usable. The job is due after the drop, so the group must claim it on a fresh session. The pool @@ -127,7 +127,7 @@ public async Task A_worker_group_that_boots_while_connections_are_refused_runs_j Assert.True(dropped > 0, "the group must hold at least one live connection to drop"); var dropCause = await WaitForSucceededAsync(store, health, second); output.WriteLine($"degraded after drop: {dropCause ?? "never"}"); - AssertNotHalted(health); + OutageTestSupport.AssertNotHalted(health, Group); } finally { @@ -158,116 +158,32 @@ private static class Upstream public static readonly string Service = DataSource[(DataSource.IndexOf('/') + 1)..]; } - private static async Task LockUserAsync() - { - await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); - await admin.OpenAsync(); - await using var lockUser = admin.CreateCommand(); - lockUser.CommandText = $"ALTER USER {Schema} ACCOUNT LOCK"; - await lockUser.ExecuteNonQueryAsync(); - } - - // Ends every session of the private user, a killed one too, and waits until none is left. A killed - // session holds its process until its client calls again, and the client is gone at teardown. - private static async Task DisconnectSessionsAsync() - { - await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); - await admin.OpenAsync(); - - var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); - while (true) - { - var sessions = new List(); - await using (var select = admin.CreateCommand()) - { - select.CommandText = "SELECT sid, serial# FROM v$session WHERE username = :username"; - select.Parameters.Add(new OracleParameter("username", Schema.ToUpperInvariant())); - await using var reader = await select.ExecuteReaderAsync(); - while (await reader.ReadAsync()) - { - sessions.Add($"{reader.GetInt32(0)},{reader.GetInt32(1)}"); - } - } - if (sessions.Count == 0 || DateTime.UtcNow > deadline) - { - return; - } - - foreach (var session in sessions) - { - await using var disconnect = admin.CreateCommand(); - disconnect.CommandText = $"ALTER SYSTEM DISCONNECT SESSION '{session}' IMMEDIATE"; - try - { - await disconnect.ExecuteNonQueryAsync(); - } - catch (OracleException exception) when (exception.Number is 31 or 30) - { - } - } - await Task.Delay(200); - } - } - - // Samples the health state until the group is degraded. The next clean poll clears the mark, so the - // sample interval stays well under the poll interval. - private static async Task WaitForDegradedAsync(BackWaveHealth health, string description) - { - var cause = ""; - await WaitForAsync(() => Task.FromResult(health.DegradedGroups.TryGetValue(Group, out cause!)), health, - description, TimeSpan.FromMilliseconds(5)); - return cause; - } - private static async Task WatchNotHaltedAsync(BackWaveHealth health, TimeSpan duration) { var deadline = DateTime.UtcNow + duration; while (DateTime.UtcNow < deadline) { - AssertNotHalted(health); + OutageTestSupport.AssertNotHalted(health, Group); await Task.Delay(TimeSpan.FromMilliseconds(20)); } - AssertNotHalted(health); + OutageTestSupport.AssertNotHalted(health, Group); } // Returns the first degraded cause seen while waiting, or null when the group never degraded. private static async Task WaitForSucceededAsync(OracleJobStore store, BackWaveHealth health, Guid jobId) { string? cause = null; - await WaitForAsync(async () => + await OutageTestSupport.WaitForAsync(async () => { if (cause is null && health.DegradedGroups.TryGetValue(Group, out var degraded)) { cause = degraded; } return (await store.GetJobAsync(jobId))?.State == JobState.Succeeded; - }, health, $"job {jobId} to reach Succeeded", TimeSpan.FromMilliseconds(5)); + }, health, Group, $"job {jobId} to reach Succeeded", TimeSpan.FromMilliseconds(5)); return cause; } - // Polls the condition and fails the test the moment the group halts, so a fail-stop reports its cause - // instead of a timeout. - private static async Task WaitForAsync( - Func> condition, BackWaveHealth health, string description, TimeSpan? interval = null) - { - var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); - while (DateTime.UtcNow < deadline) - { - AssertNotHalted(health); - if (await condition()) - { - return; - } - await Task.Delay(interval ?? TimeSpan.FromMilliseconds(50)); - } - AssertNotHalted(health); - Assert.Fail($"Timed out waiting for: {description}"); - } - - private static void AssertNotHalted(BackWaveHealth health) => - Assert.False(health.HaltedGroups.TryGetValue(Group, out var halt), - $"the outage fail-stopped the group: {halt?.ExceptionType}: {halt?.Message}"); - // Accepts each client on a loopback port and pumps its bytes both ways to the upstream endpoint. private sealed class TcpForwarder : IAsyncDisposable { diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs index 40b83f8..9312c51 100644 --- a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -57,7 +57,7 @@ public async Task A_network_outage_degrades_the_worker_group_instead_of_halting_ { await WaitForDegradedOrHaltedAsync(health); - AssertNotHalted(health); + OutageTestSupport.AssertNotHalted(health, Group); Assert.True(health.DegradedGroups.TryGetValue(Group, out var cause), "the outage must mark the group degraded"); output.WriteLine($"degraded: {cause}"); Assert.True(NamesTransientCode(cause), $"the degraded cause must name a transient code: {cause}"); @@ -66,10 +66,10 @@ public async Task A_network_outage_degrades_the_worker_group_instead_of_halting_ var watchEnd = DateTime.UtcNow + 3 * PollInterval; while (DateTime.UtcNow < watchEnd) { - AssertNotHalted(health); + OutageTestSupport.AssertNotHalted(health, Group); await Task.Delay(50); } - AssertNotHalted(health); + OutageTestSupport.AssertNotHalted(health, Group); } finally { @@ -209,14 +209,6 @@ private static async Task WaitForDegradedOrHaltedAsync(BackWaveHealth health) } } - // A halt of one pump counts too, so the check reads the partly halted groups as well. - private static void AssertNotHalted(BackWaveHealth health) - { - var halted = health.HaltedGroups.TryGetValue(Group, out var halt) - || health.PartiallyHaltedGroups.TryGetValue(Group, out halt); - Assert.False(halted, $"the outage fail-stopped the group: {halt?.ExceptionType}: {halt?.Message}"); - } - // True when the cause names an ORA or ODP.NET code that the adapter classifies transient. internal static bool NamesTransientCode(string cause) { diff --git a/tests/BackWave.Oracle.Tests/OutageTestSupport.cs b/tests/BackWave.Oracle.Tests/OutageTestSupport.cs new file mode 100644 index 0000000..0997c20 --- /dev/null +++ b/tests/BackWave.Oracle.Tests/OutageTestSupport.cs @@ -0,0 +1,97 @@ +using BackWave.Hosting; +using Oracle.ManagedDataAccess.Client; + +namespace BackWave.Oracle.Tests; + +/// The health waits and the private-user teardown that the outage tests share. +internal static class OutageTestSupport +{ + // A halt of one pump counts too, so the check reads the partly halted groups as well. + public static void AssertNotHalted(BackWaveHealth health, string group) + { + var halted = health.HaltedGroups.TryGetValue(group, out var halt) + || health.PartiallyHaltedGroups.TryGetValue(group, out halt); + Assert.False(halted, $"the outage fail-stopped the group: {halt?.ExceptionType}: {halt?.Message}"); + } + + // Polls the condition and fails the test the moment the group halts, so a fail-stop reports its cause + // instead of a timeout. + public static async Task WaitForAsync( + Func> condition, BackWaveHealth health, string group, string description, + TimeSpan? interval = null) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline) + { + AssertNotHalted(health, group); + if (await condition()) + { + return; + } + await Task.Delay(interval ?? TimeSpan.FromMilliseconds(50)); + } + AssertNotHalted(health, group); + Assert.Fail($"Timed out waiting for: {description}"); + } + + // Samples the health state until the group is degraded. The next clean poll clears the mark, so the + // sample interval stays well under the poll interval. + public static async Task WaitForDegradedAsync(BackWaveHealth health, string group, string description) + { + var cause = ""; + await WaitForAsync(() => Task.FromResult(health.DegradedGroups.TryGetValue(group, out cause!)), health, + group, description, TimeSpan.FromMilliseconds(5)); + return cause; + } + + public static async Task LockUserAsync(string user) + { + await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); + await admin.OpenAsync(); + await using var lockUser = admin.CreateCommand(); + lockUser.CommandText = $"ALTER USER {user} ACCOUNT LOCK"; + await lockUser.ExecuteNonQueryAsync(); + } + + // Ends every session of the private user, a killed one too, and waits until none is left. A killed + // session holds its process until its client calls again, and the client is gone at teardown. + public static async Task DisconnectSessionsAsync(string user) + { + await using var admin = new OracleConnection(OracleTestDatabase.SystemConnectionString); + await admin.OpenAsync(); + + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (true) + { + var sessions = new List(); + await using (var select = admin.CreateCommand()) + { + select.CommandText = "SELECT sid, serial# FROM v$session WHERE username = :username"; + select.Parameters.Add(new OracleParameter("username", user.ToUpperInvariant())); + await using var reader = await select.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + sessions.Add($"{reader.GetInt32(0)},{reader.GetInt32(1)}"); + } + } + if (sessions.Count == 0 || DateTime.UtcNow > deadline) + { + return; + } + + foreach (var session in sessions) + { + await using var disconnect = admin.CreateCommand(); + disconnect.CommandText = $"ALTER SYSTEM DISCONNECT SESSION '{session}' IMMEDIATE"; + try + { + await disconnect.ExecuteNonQueryAsync(); + } + catch (OracleException exception) when (exception.Number is 31 or 30) + { + } + } + await Task.Delay(200); + } + } +} From a3f3dcce8f7e3333857f16a45ce2667c34e8d72c Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 17:23:20 +0300 Subject: [PATCH 22/27] test(oracle): make the rejected-login test wait for the halt on the test database The test hard-coded localhost:15210 and ignored BACKWAVE_ORACLE_DSN, so on any other database it got ORA-50201 and failed. It stopped at the first degrade, so a listener fault on an early poll failed it too. Its class has no collection, so it ran in parallel with the 16-node migration storm. The test now takes its endpoint from OracleTestDatabase.ConnectionString with a user that does not exist, waits until the group halts, and lives in its own class in the oracle collection. The outage tests that need no database still run in parallel. --- .../OracleRejectedLoginTests.cs | 53 +++++++++++++++++++ .../OracleTransientFaultTests.cs | 34 +----------- 2 files changed, 55 insertions(+), 32 deletions(-) create mode 100644 tests/BackWave.Oracle.Tests/OracleRejectedLoginTests.cs diff --git a/tests/BackWave.Oracle.Tests/OracleRejectedLoginTests.cs b/tests/BackWave.Oracle.Tests/OracleRejectedLoginTests.cs new file mode 100644 index 0000000..8bb2e2d --- /dev/null +++ b/tests/BackWave.Oracle.Tests/OracleRejectedLoginTests.cs @@ -0,0 +1,53 @@ +using BackWave.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Oracle.ManagedDataAccess.Client; + +namespace BackWave.Oracle.Tests; + +/// +/// The negative case for the outage tests: a fault that is not an outage must still halt the group. A user +/// that does not exist raises ORA-01017 on the live listener. The test does not use a wrong password for a +/// real user, because repeated failed logins can lock that account. This is an integration test: it needs +/// "docker compose up -d oracle". +/// +[Collection("oracle")] +public sealed class OracleRejectedLoginTests(Xunit.Abstractions.ITestOutputHelper output) +{ + private const string Group = OracleTransientFaultTests.Group; + + [Fact] + public async Task A_rejected_login_halts_the_worker_group() + { + var store = new OracleJobStore(new OracleStoreOptions + { + ConnectionString = new OracleConnectionStringBuilder(OracleTestDatabase.ConnectionString) + { + UserID = "bw_no_such_user", + Password = "bw_no_such_user", + ConnectionTimeout = 3, + }.ConnectionString, + }); + using var host = OracleTransientFaultTests.BuildHost(store); + var health = host.Services.GetRequiredService(); + + await host.StartAsync(); + try + { + // A listener fault on an early poll only degrades the group, so the wait ignores degrades and + // ends at the halt. + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(15); + while (DateTime.UtcNow < deadline && !health.HaltedGroups.ContainsKey(Group)) + { + await Task.Delay(50); + } + + Assert.True(health.HaltedGroups.TryGetValue(Group, out var halt), "a rejected login must halt the group"); + output.WriteLine($"halted: {halt.ExceptionType}: {halt.Message}"); + Assert.Contains("ORA-01017", halt.Message); + } + finally + { + await host.StopAsync(); + } + } +} diff --git a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs index 9312c51..8e957f6 100644 --- a/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleTransientFaultTests.cs @@ -77,36 +77,6 @@ public async Task A_network_outage_degrades_the_worker_group_instead_of_halting_ } } - // The negative case: a fault that is not an outage must still halt the group. A user that does not - // exist raises ORA-01017 on the live listener. The test does not use a wrong password for a real user, - // because repeated failed logins can lock that account. This is an integration test: it needs - // "docker compose up -d oracle". - [Fact] - public async Task A_rejected_login_halts_the_worker_group() - { - var store = new OracleJobStore(new OracleStoreOptions - { - ConnectionString = "User Id=bw_no_such_user;Password=bw_no_such_user;Connection Timeout=3;" - + "Data Source=localhost:15210/FREEPDB1", - }); - using var host = BuildHost(store); - var health = host.Services.GetRequiredService(); - - await host.StartAsync(); - try - { - await WaitForDegradedOrHaltedAsync(health); - - Assert.True(health.HaltedGroups.TryGetValue(Group, out var halt), "a rejected login must halt the group"); - output.WriteLine($"halted: {halt.ExceptionType}: {halt.Message}"); - Assert.Contains("ORA-01017", halt.Message); - } - finally - { - await host.StopAsync(); - } - } - // A host that does not answer costs a full connect timeout on every attempt. The store holds its ready // gate while it migrates, so an enqueue waits for the claim's migration and then runs its own. The // retry budget bounds both: without it each call waits out all eight attempts (about 17 s here). @@ -179,10 +149,10 @@ public async Task The_store_classifies_a_deadlock_transient(int number) Assert.True(store.IsTransientFault(fault), $"ORA-{fault.Number} must classify transient"); } - private const string Group = "outage-workers"; + internal const string Group = "outage-workers"; private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(200); - private static IHost BuildHost(OracleJobStore store) + internal static IHost BuildHost(OracleJobStore store) { var builder = Host.CreateApplicationBuilder(); builder.Services.AddBackWave(backwave => backwave From 063ef249d721975f6a7a8361711ba6ecc1972d37 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 17:33:24 +0300 Subject: [PATCH 23/27] test(oracle): say why the refused-connect test releases its port A review asked to keep the port bound and not accept connects. On macOS a bound socket that does not listen makes the connect wait for its timeout, so the test got ORA-50000 instead of ORA-50201 and failed 20 of 20 runs. Only a port that nothing holds gets a refused connect. The comment now records this. If another process takes the port, the test fails and never passes by mistake. --- .../OracleRefusedConnectionRecoveryTests.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs b/tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs index de6016e..48083e0 100644 --- a/tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleRefusedConnectionRecoveryTests.cs @@ -136,7 +136,10 @@ public async Task A_worker_group_that_boots_while_connections_are_refused_runs_j } // Binds a listener on port 0 to get a free loopback port, then stops it, so nothing listens there until - // the forwarder starts. + // the forwarder starts. Only a port that nothing holds gets a refused connect (ORA-50201): on macOS a + // bound socket that does not listen makes the connect wait for its timeout (ORA-50000). Another + // process can take the port in the meantime, but then the connect is not refused or the forwarder + // cannot bind, so the test fails and never passes by mistake. private static int ReserveFreePort() { var listener = new TcpListener(IPAddress.Loopback, 0); From a4cbe392333f20b661e22cbb5e8ed70a6462f6ce Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 17:33:51 +0300 Subject: [PATCH 24/27] docs(oracle): name the deadlock retry in the coordinate parameter doc The coordinate doc named only the connection faults, and it called the script concurrency-safe on its own. The coordinated path also retries ORA-00060 and ORA-04020, and the uncoordinated path can surface them. --- src/BackWave.Oracle/OracleMigrator.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/BackWave.Oracle/OracleMigrator.cs b/src/BackWave.Oracle/OracleMigrator.cs index 6d6b980..ac00e99 100644 --- a/src/BackWave.Oracle/OracleMigrator.cs +++ b/src/BackWave.Oracle/OracleMigrator.cs @@ -53,10 +53,10 @@ public static Task MigrateAsync(string connectionString, CancellationToken cance /// /// /// When (the default), a cold-booting fleet rides out the transient connection - /// faults of a first-boot handshake storm with a bounded retry. The Oracle schema script is idempotent - /// and concurrency-safe on its own, so no database lock is taken either way. Setting it to - /// runs the script once with no transient retry; it is safe when the deployment - /// already serializes migration itself. + /// faults of a first-boot handshake storm and the deadlocks (ORA-00060, ORA-04020) of nodes that create + /// the same object at once, with a bounded retry. The Oracle schema script is idempotent, so no database + /// lock is taken either way. Setting it to runs the script once with no retry; + /// it is safe when the deployment already serializes migration itself. /// /// Token to cancel the migration. /// A task that completes once every script has run. From dc7351268315dee99c2c962874850a3fe8a6463d Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 17:34:05 +0300 Subject: [PATCH 25/27] fix(verify): turn off sqlplus substitution in bwv sql on Oracle An & in a query started a substitution prompt. sqlplus then took the next input line as the value, so SELECT 'a&b' FROM dual returned aEXIT. SET DEFINE OFF now runs first, and the same query returns a&b. --- .agents/skills/verify-backwave/scripts/bwv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/skills/verify-backwave/scripts/bwv b/.agents/skills/verify-backwave/scripts/bwv index ddc8613..53d42fb 100755 --- a/.agents/skills/verify-backwave/scripts/bwv +++ b/.agents/skills/verify-backwave/scripts/bwv @@ -390,7 +390,7 @@ cmd_sql() { sqlite) sqlite3 -header -column -cmd '.timeout 5000' "$SQLITE_DB" "PRAGMA query_only=1; $query" ;; postgres) docker exec -e PGOPTIONS='-c default_transaction_read_only=on' "$CONTAINER" \ psql -U backwave -d backwave_sample -v ON_ERROR_STOP=1 -c "$query" ;; - oracle) printf 'SET MARKUP CSV ON\nWHENEVER SQLERROR EXIT FAILURE\nSET TRANSACTION READ ONLY;\n%s;\nEXIT\n' "${query%;}" \ + oracle) printf 'SET DEFINE OFF\nSET MARKUP CSV ON\nWHENEVER SQLERROR EXIT FAILURE\nSET TRANSACTION READ ONLY;\n%s;\nEXIT\n' "${query%;}" \ | docker exec -i "$CONTAINER" sqlplus -S -L backwave/backwave@localhost/FREEPDB1 ;; esac } > "$out" 2>&1 || { cat "$out"; die "query failed (saved db/$name.txt)"; } From 15fc82924e29132431c5231d4b27822c63a1bbf6 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 17:34:05 +0300 Subject: [PATCH 26/27] docs(verify): name the sample Oracle port and add a missing comma A bwv run uses neither the shared test container on 15210 nor the sample compose container on 15211. The store sentence in features/README.md missed a comma. --- .agents/skills/verify-backwave/SKILL.md | 2 +- .agents/skills/verify-backwave/features/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.agents/skills/verify-backwave/SKILL.md b/.agents/skills/verify-backwave/SKILL.md index 1ae865f..67b07dc 100644 --- a/.agents/skills/verify-backwave/SKILL.md +++ b/.agents/skills/verify-backwave/SKILL.md @@ -32,7 +32,7 @@ Each run owns these resources. All of them use the same run id (`YYYYMMDD-HHMMSS | HTTP port | `http://127.0.0.1:` | A free port from the OS. `bwv` never uses the default `5283` from `appsettings.json`. | | Scratch directory | `${TMPDIR}/bw-verify-/` | Holds the app copy and the SQLite file `backwave.db`. | | Postgres (only with `--store postgres`) | container `backwave-verify-`, label `backwave.verify.run=` | `postgres:17-alpine` on a free loopback port. It never uses the compose containers on `5398` or `14331`. | -| Oracle (only with `--store oracle`) | container `backwave-verify-`, label `backwave.verify.run=` | `gvenzl/oracle-free:23-slim` on a fixed free loopback port. It never uses the compose container on `15210`. | +| Oracle (only with `--store oracle`) | container `backwave-verify-`, label `backwave.verify.run=` | `gvenzl/oracle-free:23-slim` on a fixed free loopback port. It never uses the compose container on `15210` or the sample compose container on `15211`. | | Browser | `CHROME_DEVTOOLS_AXI_SESSION=bw-verify-` | A private bridge and an isolated headless Chrome profile. `bwv` never uses the `default` session. | | Evidence | `.verify-evidence//` at the worktree root | Gitignored. Cleanup never deletes it. | diff --git a/.agents/skills/verify-backwave/features/README.md b/.agents/skills/verify-backwave/features/README.md index cbb6600..2da2f0a 100644 --- a/.agents/skills/verify-backwave/features/README.md +++ b/.agents/skills/verify-backwave/features/README.md @@ -6,7 +6,7 @@ This directory is the maintained source for verifying the user-facing behavior o - `bwv launch` printed `READY`, and `bwv doctor` passes every check on the run that you drive. - The run's Sample.Api starts with no jobs. Each recipe creates its own jobs through the sample's HTTP endpoints with `bwv api`. -- The run's store is its own SQLite file (the default) its own Postgres container (`--store postgres`), or its own Oracle container (`--store oracle`). The queues are `critical`, `bulk`, `low`, and `limited` (concurrency limit 1). +- The run's store is its own SQLite file (the default), its own Postgres container (`--store postgres`), or its own Oracle container (`--store oracle`). The queues are `critical`, `bulk`, `low`, and `limited` (concurrency limit 1). - The operator actor on the dashboard and on `/ops/*` is `sample-operator`. The startup limit on `limited` has the actor `startup`. - Never drive an app, a container, or a browser session that this run did not start. From fb4fa818430994bfceff38f6041ec6831cefc667 Mon Sep 17 00:00:00 2001 From: Paul DeVito Date: Sat, 17 Oct 2026 17:37:56 +0300 Subject: [PATCH 27/27] test(oracle): let the coordinated storm prove the migrator's own retry The storm helper rode out transient faults for both paths, so the coordinated storm passed even with no retry in the migrator. The coordinated path now calls MigrateAsync directly, and only the uncoordinated path keeps the helper retry. With the deadlock retry removed from the migrator, the storm class failed on its second run with ORA-04020. With the retry, it passed 12 of 12 runs. --- .../OracleCoordinatedMigrationTests.cs | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/BackWave.Oracle.Tests/OracleCoordinatedMigrationTests.cs b/tests/BackWave.Oracle.Tests/OracleCoordinatedMigrationTests.cs index 1264091..5543058 100644 --- a/tests/BackWave.Oracle.Tests/OracleCoordinatedMigrationTests.cs +++ b/tests/BackWave.Oracle.Tests/OracleCoordinatedMigrationTests.cs @@ -49,7 +49,15 @@ public sealed class OracleCoordinatedMigrationTests : IAsyncLifetime barrier.SignalAndWait(); try { - await MigrateRidingOutConnectivityFaultsAsync(coordinate); + // The coordinated path calls the migrator directly, so the storm proves its own retry. + if (coordinate) + { + await OracleMigrator.MigrateAsync(MigrationConnectionString, Schema); + } + else + { + await MigrateUncoordinatedRidingOutTransientFaultsAsync(); + } return (Exception?)null; } catch (Exception exception) @@ -61,20 +69,19 @@ public sealed class OracleCoordinatedMigrationTests : IAsyncLifetime return await Task.WhenAll(tasks); } - // The coordinate:false path strips the migrator's own transient-connectivity retry, so a cold-boot - // listener-handoff fault (ORA-12518, ORA-12570, and the rest of the connectivity set) can surface here - - // unrelated to the guarded-DDL convergence this test proves. The same path also surfaces ORA-04020, the - // library cache deadlock between two nodes that create the same object, which the coordinated path - // retries. Ride those out with a bounded retry. Any other error (a real convergence fault such as - // ORA-00955 or ORA-00001) propagates unchanged and fails the storm as before. - private static async Task MigrateRidingOutConnectivityFaultsAsync(bool coordinate) + // The coordinate:false path strips the migrator's own transient retry, so a cold-boot listener-handoff + // fault (ORA-12518, ORA-12570, and the rest of the connectivity set) or a deadlock between two nodes + // that create the same object (ORA-04020) can surface here - unrelated to the guarded-DDL convergence + // this test proves. Ride those out with a bounded retry, as the coordinated path does. Any other error + // (a real convergence fault such as ORA-00955 or ORA-00001) propagates unchanged and fails the storm. + private static async Task MigrateUncoordinatedRidingOutTransientFaultsAsync() { const int maxAttempts = 10; for (var attempt = 1; ; attempt++) { try { - await OracleMigrator.MigrateAsync(MigrationConnectionString, Schema, coordinate); + await OracleMigrator.MigrateAsync(MigrationConnectionString, Schema, coordinate: false); return; } catch (OracleException exception)